fix: flakey tests reporting (#14851)

* fix: flakey tests reporting

* fix: remove unused param from constructor

.github/workflows/ci-core.yml

@@ -244,6 +244,93 @@ jobs:
  echo "path_output=${resultsFile}" >> $GITHUB_OUTPUT
  fi
 
+ detect-flakey-tests:
+ needs: [filter, core]
+ name: Flakey Test Detection
+ runs-on: ubuntu-latest
+ if: ${{ always() && github.actor != 'dependabot[bot]' }}
+ env:
+ CL_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable
+ permissions:
+ id-token: write
+ contents: read
+ steps:
+ - name: Checkout the repo
+ uses: actions/[email protected]
+ - name: Setup node
+ if: ${{ needs.filter.outputs.changes == 'true' }}
+ uses: actions/[email protected]
+ - name: Setup NodeJS
+ if: ${{ needs.filter.outputs.changes == 'true' }}
+ uses: ./.github/actions/setup-nodejs
+ with:
+ prod: "true"
+ - name: Setup Go
+ if: ${{ needs.filter.outputs.changes == 'true' }}
+ uses: ./.github/actions/setup-go
+ - name: Setup Postgres
+ if: ${{ needs.filter.outputs.changes == 'true' }}
+ uses: ./.github/actions/setup-postgres
+ - name: Touching core/web/assets/index.html
+ if: ${{ needs.filter.outputs.changes == 'true' }}
+ run: mkdir -p core/web/assets && touch core/web/assets/index.html
+ - name: Download Go vendor packages
+ if: ${{ needs.filter.outputs.changes == 'true' }}
+ run: go mod download
+ - name: Replace chainlink-evm deps
+ if: ${{ needs.filter.outputs.changes == 'true' && inputs.evm-ref != ''}}
+ shell: bash
+ run: go get github.com/smartcontractkit/chainlink-integrations/evm/relayer@${{ inputs.evm-ref }}
+ - name: Build binary
+ if: ${{ needs.filter.outputs.changes == 'true' }}
+ run: go build -o chainlink.test .
+ - name: Setup DB
+ if: ${{ needs.filter.outputs.changes == 'true' }}
+ run: ./chainlink.test local db preparetest
+ - name: Load test outputs
+ if: ${{ needs.filter.outputs.changes == 'true' }}
+ uses: actions/[email protected]
+ with:
+ name: go_core_tests_logs
+ path: ./artifacts
+ - name: Delete go_core_tests_logs/coverage.txt
+ if: ${{ needs.filter.outputs.changes == 'true' }}
+ shell: bash
+ run: |
+ # Need to delete coverage.txt so the disk doesn't fill up
+ rm -f ./artifacts/go_core_tests_logs/coverage.txt
+ - name: Build flakey test runner
+ if: ${{ needs.filter.outputs.changes == 'true' }}
+ run: go build ./tools/flakeytests/cmd/runner
+ - name: Re-run tests
+ if: ${{ needs.filter.outputs.changes == 'true' }}
+ env:
+ GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
+ GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
+ GITHUB_EVENT_PATH: ${{ github.event_path }}
+ GITHUB_EVENT_NAME: ${{ github.event_name }}
+ GITHUB_REPO: ${{ github.repository }}
+ GITHUB_RUN_ID: ${{ github.run_id }}
+ run: |
+ ./runner \
+ -grafana_auth=$GRAFANA_INTERNAL_BASIC_AUTH \
+ -grafana_host=$GRAFANA_INTERNAL_HOST \
+ -gh_sha=$GITHUB_SHA \
+ -gh_event_path=$GITHUB_EVENT_PATH \
+ -gh_event_name=$GITHUB_EVENT_NAME \
+ -gh_run_id=$GITHUB_RUN_ID \
+ -gh_repo=$GITHUB_REPO \
+ -command=./tools/bin/go_core_tests \
+ `ls -R ./artifacts/output.txt`
+ - name: Store logs artifacts
+ if: ${{ needs.filter.outputs.changes == 'true' && always() }}
+ uses: actions/[email protected]
+ with:
+ name: flakey_test_runner_logs
+ path: |
+ ./output.txt
+ retention-days: 7
+
  scan:
  name: SonarQube Scan
  needs: [core]

tools/flakeytests/cmd/runner/main.go

@@ -24,7 +24,6 @@ func main() {
 
  grafanaHost := flag.String("grafana_host", "", "grafana host URL")
  grafanaAuth := flag.String("grafana_auth", "", "grafana basic auth for Loki API")
- grafanaOrgID := flag.String("grafana_org_id", "", "grafana org ID")
  command := flag.String("command", "", "test command being rerun; used to tag metrics")
  ghSHA := flag.String("gh_sha", "", "commit sha for which we're rerunning tests")
  ghEventPath := flag.String("gh_event_path", "", "path to associated gh event")
@@ -46,10 +45,6 @@ func main() {
  log.Fatal("Error re-running flakey tests: `grafana_auth` is required")
  }
 
- if *grafanaOrgID == "" {
- log.Fatal("Error re-running flakey tests: `grafana_org_id` is required")
- }
-
  if *command == "" {
  log.Fatal("Error re-running flakey tests: `command` is required")
  }
@@ -68,7 +63,7 @@ func main() {
  }
 
  meta := flakeytests.GetGithubMetadata(*ghRepo, *ghEventName, *ghSHA, *ghEventPath, *ghRunID, runAttempt)
- rep := flakeytests.NewLokiReporter(*grafanaHost, *grafanaAuth, *grafanaOrgID, *command, meta)
+ rep := flakeytests.NewLokiReporter(*grafanaHost, *grafanaAuth, *command, meta)
  r := flakeytests.NewRunner(readers, rep, numReruns)
  err := r.Run(ctx)
  if err != nil {

tools/flakeytests/reporter.go

@@ -62,7 +62,6 @@ type Context struct {
 type LokiReporter struct {
  host string
  auth string
- orgId string
  command string
  now func() time.Time
  ctx Context
@@ -156,7 +155,6 @@ func (l *LokiReporter) makeRequest(ctx context.Context, pushReq pushRequest) err
  fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(l.auth))),
  )
  req.Header.Add("Content-Type", "application/json")
- req.Header.Add("X-Scope-OrgID", l.orgId)
  resp, err := http.DefaultClient.Do(req)
  if err != nil {
  return err
@@ -179,6 +177,6 @@ func (l *LokiReporter) Report(ctx context.Context, report *Report) error {
  return l.makeRequest(ctx, pushReq)
 }
 
-func NewLokiReporter(host, auth, orgId, command string, ctx Context) *LokiReporter {
- return &LokiReporter{host: host, auth: auth, orgId: orgId, command: command, now: time.Now, ctx: ctx}
+func NewLokiReporter(host, auth, command string, ctx Context) *LokiReporter {
+ return &LokiReporter{host: host, auth: auth, command: command, now: time.Now, ctx: ctx}
 }

tools/flakeytests/reporter_test.go

@@ -19,7 +19,7 @@ func TestMakeRequest_SingleTest(t *testing.T) {
  },
  },
  }
- lr := &LokiReporter{auth: "bla", host: "bla", orgId: "bla", command: "go_core_tests", now: func() time.Time { return now }}
+ lr := &LokiReporter{auth: "bla", host: "bla", command: "go_core_tests", now: func() time.Time { return now }}
  pr, err := lr.createRequest(r)
  require.NoError(t, err)
  assert.Len(t, pr.Streams, 1)
@@ -41,7 +41,7 @@ func TestMakeRequest_MultipleTests(t *testing.T) {
  },
  },
  }
- lr := &LokiReporter{auth: "bla", host: "bla", orgId: "bla", command: "go_core_tests", now: func() time.Time { return now }}
+ lr := &LokiReporter{auth: "bla", host: "bla", command: "go_core_tests", now: func() time.Time { return now }}
  pr, err := lr.createRequest(r)
  require.NoError(t, err)
  assert.Len(t, pr.Streams, 1)
@@ -58,7 +58,7 @@ func TestMakeRequest_NoTests(t *testing.T) {
  now := time.Now()
  ts := fmt.Sprintf("%d", now.UnixNano())
  r := NewReport()
- lr := &LokiReporter{auth: "bla", host: "bla", orgId: "bla", command: "go_core_tests", now: func() time.Time { return now }}
+ lr := &LokiReporter{auth: "bla", host: "bla", command: "go_core_tests", now: func() time.Time { return now }}
  pr, err := lr.createRequest(r)
  require.NoError(t, err)
  assert.Len(t, pr.Streams, 1)
@@ -72,7 +72,7 @@ func TestMakeRequest_WithContext(t *testing.T) {
  now := time.Now()
  ts := fmt.Sprintf("%d", now.UnixNano())
  r := NewReport()
- lr := &LokiReporter{auth: "bla", host: "bla", orgId: "bla", command: "go_core_tests", now: func() time.Time { return now }, ctx: Context{CommitSHA: "42"}}
+ lr := &LokiReporter{auth: "bla", host: "bla", command: "go_core_tests", now: func() time.Time { return now }, ctx: Context{CommitSHA: "42"}}
  pr, err := lr.createRequest(r)
  require.NoError(t, err)
  assert.Len(t, pr.Streams, 1)
@@ -95,7 +95,7 @@ func TestMakeRequest_Panics(t *testing.T) {
  "core/assets": 1,
  },
  }
- lr := &LokiReporter{auth: "bla", host: "bla", orgId: "bla", command: "go_core_tests", now: func() time.Time { return now }}
+ lr := &LokiReporter{auth: "bla", host: "bla", command: "go_core_tests", now: func() time.Time { return now }}
  pr, err := lr.createRequest(r)
  require.NoError(t, err)
  assert.Len(t, pr.Streams, 1)