Plugin E2E: Describe how to start e2e test server in contributing docs (

#1097)

packages/plugin-e2e/.env.example

@@ -1 +1,6 @@
-GOOGLE_JWT_FILE=
+# google-sheets-datasource-jwt.yaml
+GOOGLE_JWT_FILE=
+
+# aws-redshift.yaml - data source with name 'AWS Redshift'
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=

packages/plugin-e2e/CONTRIBUTING.md

@@ -54,16 +54,38 @@ npm run build # used to build @grafana/plugin-e2e
 npm run dev # watches for changes to files and rebuilds @grafana/plugin-e2e automatically
 ```
 
+#### Running e2e tests locally
+
+There are two types of Playwright tests - those that require data source credentials (only available to members of the Grafana team) and those that don't.
+
+To run the Playwright tests that don't require credentials.
+
+1. Start the e2e test server:
+
 ```shell
-npm run playwright:test # runs all playwright tests headlessly
+npm run server
+
 ```
 
+2. Run the Playwright tests
+
 ```shell
-npm run playwright:test:ui # runs all playwright tests in Google chrome
+npm run playwright:test # runs all the playwright tests that don't require credentials
 ```
 
+To run the e2e tests that require data source credentials, you need to add a `/packages/plugin-e2e/.env` file and provide the necessary credentials (see `/packages/plugin-e2e/.env.example` to get an understanding of what variables you need to provide). You'll find all the necessary credentials in the [plugin-provisioning](https://github.com/grafana/plugin-provisioning) repo.
+
+1. Start the e2e test server:
+
+```shell
+npm run server
+
+```
+
+2. Run the Playwright integration tests
+
 ```shell
-npm run playwright:showreport # show Playwright report for the last test session
+npm run playwright:test:integration #runs all the playwright tests that integrates with third-party services
 ```
 
 ### VS Code Playwright extension

packages/plugin-e2e/docker-compose.yaml

@@ -1,5 +1,3 @@
-version: '3.7'
-
 services:
   grafana:
     image: grafana/${GRAFANA_IMAGE:-grafana-enterprise}:${GRAFANA_VERSION:-main}

packages/plugin-e2e/package.json

@@ -26,9 +26,9 @@
     "server": "docker compose up --build",
     "typecheck": "tsc --emitDeclarationOnly false --noEmit",
     "test": "vitest",
-    "playwright:test": "npx playwright test",
-    "playwright:test:ui": "npx playwright test --ui",
-    "playwright:showreport": "npx playwright show-report"
+    "playwright:test": "npx playwright test --grep-invert @integration",
+    "playwright:test:integration": "npx playwright test --grep @integration",
+    "playwright:all": "npx playwright test"
   },
   "engines": {
     "node": ">=18 <=20"
@@ -37,7 +37,7 @@
     "@playwright/test": "^1.41.2"
   },
   "devDependencies": {
-    "@playwright/test": "^1.41.2",
+    "@playwright/test": "^1.47.0",
     "@types/uuid": "^10.0.0",
     "dotenv": "^16.3.1"
   },

packages/plugin-e2e/tests/as-admin-user/datasource/alerting/alerting.spec.ts

@@ -2,16 +2,20 @@ import * as semver from 'semver';
 import { test, expect } from '../../../../src';
 
 const skipMsg = 'Alerting rule test API are only compatible with Grafana 9.5.0 and later';
-test('should evaluate to true if query is valid', async ({ grafanaVersion, page, alertRuleEditPage, selectors }) => {
-  test.skip(semver.lt(grafanaVersion, '9.5.0'), skipMsg);
-  const queryA = alertRuleEditPage.getAlertRuleQueryRow('A');
-  await queryA.datasource.set('AWS Redshift');
-  await alertRuleEditPage.alertRuleNameField.fill('Test Alert Rule');
-  await page.waitForFunction(() => window.monaco);
-  await queryA.getByGrafanaSelector(selectors.components.CodeEditor.container).click();
-  await page.keyboard.insertText('select * from long_format_example where $__timeFilter(time) ');
-  await expect(alertRuleEditPage.evaluate()).toBeOK();
-});
+test(
+  'should evaluate to true if query is valid',
+  { tag: '@integration' },
+  async ({ grafanaVersion, page, alertRuleEditPage, selectors }) => {
+    test.skip(semver.lt(grafanaVersion, '9.5.0'), skipMsg);
+    const queryA = alertRuleEditPage.getAlertRuleQueryRow('A');
+    await queryA.datasource.set('AWS Redshift');
+    await alertRuleEditPage.alertRuleNameField.fill('Test Alert Rule');
+    await page.waitForFunction(() => window.monaco);
+    await queryA.getByGrafanaSelector(selectors.components.CodeEditor.container).click();
+    await page.keyboard.insertText('select * from long_format_example where $__timeFilter(time) ');
+    await expect(alertRuleEditPage.evaluate()).toBeOK();
+  }
+);
 
 test('should evaluate to false if query is invalid', async ({ grafanaVersion, page, alertRuleEditPage, selectors }) => {
   test.skip(semver.lt(grafanaVersion, '9.5.0'), skipMsg);
@@ -36,16 +40,16 @@ test('should be possible to add multiple rows', async ({ grafanaVersion, alertRu
   await expect(alertRuleEditPage.getByGrafanaSelector(rows)).toHaveCount(rowCount + 2);
 });
 
-test('should evaluate to true when loading a provisioned query that is valid', async ({
-  grafanaVersion,
-  gotoAlertRuleEditPage,
-  readProvisionedAlertRule,
-}) => {
-  test.skip(semver.lt(grafanaVersion, '9.5.0'), skipMsg);
-  const alertRule = await readProvisionedAlertRule({ fileName: 'alerts.yml' });
-  const alertRuleEditPage = await gotoAlertRuleEditPage(alertRule);
-  await expect(alertRuleEditPage.evaluate()).toBeOK();
-});
+test(
+  'should evaluate to true when loading a provisioned query that is valid',
+  { tag: '@integration' },
+  async ({ grafanaVersion, gotoAlertRuleEditPage, readProvisionedAlertRule }) => {
+    test.skip(semver.lt(grafanaVersion, '9.5.0'), skipMsg);
+    const alertRule = await readProvisionedAlertRule({ fileName: 'alerts.yml' });
+    const alertRuleEditPage = await gotoAlertRuleEditPage(alertRule);
+    await expect(alertRuleEditPage.evaluate()).toBeOK();
+  }
+);
 
 test('should evaluate to false when loading a provisioned query that is invalid', async ({
   grafanaVersion,

packages/plugin-e2e/tests/as-admin-user/datasource/annotations/annotationQueryRunner.integration.spec.ts

@@ -2,45 +2,47 @@ import * as semver from 'semver';
 import semverLt from 'semver/functions/lt';
 import { test, expect } from '../../../../src';
 
-test('should run successfully if valid Redshift query was provided', async ({
-  annotationEditPage,
-  page,
-  selectors,
-  readProvisionedDataSource,
-  grafanaVersion,
-}, testInfo) => {
-  testInfo.skip(semverLt(grafanaVersion, '9.2.0'), 'Code editor seems to trigger one query per character typed');
-  const ds = await readProvisionedDataSource({ fileName: 'redshift.yaml' });
-  await annotationEditPage.datasource.set(ds.name);
-  await page.waitForFunction(() => (window as any).monaco);
-  await annotationEditPage.getByGrafanaSelector(selectors.components.CodeEditor.container).click();
-  await page.keyboard.insertText('SELECT starttime, eventname FROM event ORDER BY eventname ASC LIMIT 5 ');
-  await expect(annotationEditPage.runQuery()).toBeOK();
-  await expect(page.getByText('.38 Special')).toBeTruthy();
-});
+test.describe('annotation edit page', { tag: '@integration' }, () => {
+  test('should run successfully if valid Redshift query was provided', async ({
+    annotationEditPage,
+    page,
+    selectors,
+    readProvisionedDataSource,
+    grafanaVersion,
+  }, testInfo) => {
+    testInfo.skip(semverLt(grafanaVersion, '9.2.0'), 'Code editor seems to trigger one query per character typed');
+    const ds = await readProvisionedDataSource({ fileName: 'redshift.yaml' });
+    await annotationEditPage.datasource.set(ds.name);
+    await page.waitForFunction(() => (window as any).monaco);
+    await annotationEditPage.getByGrafanaSelector(selectors.components.CodeEditor.container).click();
+    await page.keyboard.insertText('SELECT starttime, eventname FROM event ORDER BY eventname ASC LIMIT 5 ');
+    await expect(annotationEditPage.runQuery()).toBeOK();
+    await expect(page.getByText('.38 Special')).toBeTruthy();
+  });
 
-test('should run successfully if valid Google Sheets query was provided', async ({
-  annotationEditPage,
-  page,
-  readProvisionedDataSource,
-}) => {
-  const ds = await readProvisionedDataSource({ fileName: 'google-sheets-datasource-jwt.yaml' });
-  await annotationEditPage.datasource.set(ds.name);
-  await page.getByText('Enter SpreadsheetID').click();
-  await page.keyboard.insertText('1TZlZX67Y0s4CvRro_3pCYqRCKuXer81oFp_xcsjPpe8');
-  await page.keyboard.press('Enter');
-  await expect(annotationEditPage.runQuery()).toBeOK();
-});
+  test('should run successfully if valid Google Sheets query was provided', async ({
+    annotationEditPage,
+    page,
+    readProvisionedDataSource,
+  }) => {
+    const ds = await readProvisionedDataSource({ fileName: 'google-sheets-datasource-jwt.yaml' });
+    await annotationEditPage.datasource.set(ds.name);
+    await page.getByText('Enter SpreadsheetID').click();
+    await page.keyboard.insertText('1TZlZX67Y0s4CvRro_3pCYqRCKuXer81oFp_xcsjPpe8');
+    await page.keyboard.press('Enter');
+    await expect(annotationEditPage.runQuery()).toBeOK();
+  });
 
-test('should run successfully if valid Redshift query was provided in provisioned dashboard', async ({
-  gotoAnnotationEditPage,
-  readProvisionedDashboard,
-  grafanaVersion,
-}) => {
-  const dashboard = await readProvisionedDashboard({ fileName: 'redshift.json' });
-  const annotationEditPage = await gotoAnnotationEditPage({ dashboard, id: '1' });
-  await expect(annotationEditPage.runQuery()).toBeOK();
-  if (semver.gte(grafanaVersion, '11.0.0')) {
-    await expect(annotationEditPage).toHaveAlert('warning', { hasText: 'No events found' });
-  }
+  test('should run successfully if valid Redshift query was provided in provisioned dashboard', async ({
+    gotoAnnotationEditPage,
+    readProvisionedDashboard,
+    grafanaVersion,
+  }) => {
+    const dashboard = await readProvisionedDashboard({ fileName: 'redshift.json' });
+    const annotationEditPage = await gotoAnnotationEditPage({ dashboard, id: '1' });
+    await expect(annotationEditPage.runQuery()).toBeOK();
+    if (semver.gte(grafanaVersion, '11.0.0')) {
+      await expect(annotationEditPage).toHaveAlert('warning', { hasText: 'No events found' });
+    }
+  });
 });

packages/plugin-e2e/tests/as-admin-user/datasource/config-editor/configEditor.spec.ts

@@ -8,12 +8,16 @@ test('invalid credentials should return an error', async ({ createDataSourceConf
   await expect(configPage.saveAndTest()).not.toBeOK();
 });
 
-test('valid credentials should return a 200 status code', async ({ createDataSourceConfigPage, page }) => {
-  const configPage = await createDataSourceConfigPage({ type: 'grafana-googlesheets-datasource' });
-  await page.getByTestId('Paste JWT button').click();
-  await page.getByTestId('Configuration text area').fill(process.env.GOOGLE_JWT_FILE!.replace(/'/g, ''));
-  await expect(configPage.saveAndTest()).toBeOK();
-});
+test(
+  'valid credentials should return a 200 status code',
+  { tag: '@integration' },
+  async ({ createDataSourceConfigPage, page }) => {
+    const configPage = await createDataSourceConfigPage({ type: 'grafana-googlesheets-datasource' });
+    await page.getByTestId('Paste JWT button').click();
+    await page.getByTestId('Configuration text area').fill(process.env.GOOGLE_JWT_FILE!.replace(/'/g, ''));
+    await expect(configPage.saveAndTest()).toBeOK();
+  }
+);
 
 test('should call a custom health endpoint when healthCheckPath is provided', async ({
   createDataSourceConfigPage,
@@ -33,11 +37,12 @@ test('should call a custom health endpoint when healthCheckPath is provided', as
   await expect(configPage).toHaveAlert('success', { hasNotText: 'Datasource updated' });
 });
 
-test('existing ds instance - valid credentials should return a 200 status code', async ({
-  readProvisionedDataSource,
-  gotoDataSourceConfigPage,
-}) => {
-  const datasource = await readProvisionedDataSource({ fileName: 'google-sheets-datasource-jwt.yaml' });
-  const configPage = await gotoDataSourceConfigPage(datasource.uid);
-  await expect(configPage.saveAndTest()).toBeOK();
-});
+test(
+  'existing ds instance - valid credentials should return a 200 status code',
+  { tag: '@integration' },
+  async ({ readProvisionedDataSource, gotoDataSourceConfigPage }) => {
+    const datasource = await readProvisionedDataSource({ fileName: 'google-sheets-datasource-jwt.yaml' });
+    const configPage = await gotoDataSourceConfigPage(datasource.uid);
+    await expect(configPage.saveAndTest()).toBeOK();
+  }
+);