[staging] Studies per file (#1229)

brave · Oct 23, 2024 · f714541 · f714541
1 parent 53fbdfe
commit f714541
Show file tree

Hide file tree

Showing 84 changed files with 3,257 additions and 14 deletions.
diff --git a/.github/workflows/test-src.yml b/.github/workflows/test-src.yml
@@ -24,3 +24,14 @@ jobs:
 
       - name: build
         run: npm run build
+
+      - name: Setup Python
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5
+        with:
+          python-version: '3.11'
+
+      - name: Install python seed generator requirements
+        run: pip install -r seed/requirements.txt
+
+      - name: Compare python and typescript seed generator
+        run: npm run seed_tools compare_python_gen
diff --git a/src/scripts/lint.ts b/src/scripts/lint.ts
@@ -41,8 +41,7 @@ function getLintAllCommands(options: Options): string[] {
   return [
     'prettier . --ignore-unknown' + (options.fix ? ' --write' : ' --check'),
     'eslint . --config src/.eslintrc.js' + (options.fix ? ' --fix' : ''),
-    // TODO(goodov): Add a command to lint JSON studies when per-file structure
-    // appears.
+    'npm run seed_tools -- lint studies' + (options.fix ? ' --fix' : ''),
   ];
 }
 

diff --git a/src/seed_tools/commands/compare_python_gen.ts b/src/seed_tools/commands/compare_python_gen.ts
@@ -0,0 +1,74 @@
+// Copyright (c) 2024 The Brave Authors. All rights reserved.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+import { Command } from '@commander-js/extra-typings';
+import { execSync } from 'child_process';
+import { assert } from 'console';
+import { existsSync, promises as fs } from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+
+export default function createCommand() {
+  return new Command('compare_python_gen')
+    .description(
+      'Run python and typescript seed generators and compare results',
+    )
+    .option('--python <value>', 'Path to python executable', 'python3')
+    .action(main);
+}
+
+interface Options {
+  python: string;
+}
+
+async function main(options: Options) {
+  const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'seed-compare-'));
+  const mockSerialNumber = 'mock_serial_number';
+
+  try {
+    const pythonSeedFilePath = path.join(tempDir, 'python_seed.bin');
+    const typescriptSeedFilePath = path.join(tempDir, 'typescript_seed.bin');
+    const pythonSeedSerialNumberFilePath = path.join(
+      tempDir,
+      'python_serialnumber',
+    );
+    const typescriptSeedSerialNumberFilePath = path.join(
+      tempDir,
+      'typescript_serialnumber',
+    );
+
+    // Run Python seed generator
+    execSync(
+      `${options.python} ./seed/serialize.py ./seed/seed.json --mock_serial_number ${mockSerialNumber}`,
+      {
+        stdio: 'inherit',
+      },
+    );
+    // Move generated seed.bin and serialnumber to temporary directory.
+    await moveFile('./seed.bin', pythonSeedFilePath);
+    await moveFile('./serialnumber', pythonSeedSerialNumberFilePath);
+
+    // Run TypeScript seed generator
+    execSync(
+      `npm run seed_tools create ./studies ${typescriptSeedFilePath} -- --mock_serial_number ${mockSerialNumber} --output_serial_number_file ${typescriptSeedSerialNumberFilePath}`,
+      { stdio: 'inherit' },
+    );
+
+    // Run seed comparator
+    execSync(
+      `npm run seed_tools compare_seeds ${pythonSeedFilePath} ${typescriptSeedFilePath} ${pythonSeedSerialNumberFilePath} ${typescriptSeedSerialNumberFilePath}`,
+      { stdio: 'inherit' },
+    );
+  } finally {
+    // Clean up temporary directory
+    await fs.rm(tempDir, { recursive: true, force: true });
+  }
+}
+
+async function moveFile(src: string, dest: string) {
+  await fs.copyFile(src, dest);
+  await fs.unlink(src);
+  assert(!existsSync(src));
+}
diff --git a/src/seed_tools/commands/compare_seeds.ts b/src/seed_tools/commands/compare_seeds.ts
@@ -13,17 +13,19 @@ export default function createCommand() {
     .description('Compare two seed.bin')
     .argument('<seed1_file>', 'seed1 file')
     .argument('<seed2_file>', 'seed2 file')
+    .argument('<seed1_serialnumber>', 'seed1 serialnumber file')
+    .argument('<seed2_serialnumber>', 'seed2 serialnumber file')
     .action(main);
 }
 
-async function main(seed1FilePath: string, seed2FilePath: string) {
+async function main(
+  seed1FilePath: string,
+  seed2FilePath: string,
+  seed1SerialnumberFilePath: string,
+  seed2SerialnumberFilePath: string,
+) {
   const seed1Binary: Buffer = await fs.readFile(seed1FilePath);
   const seed2Binary: Buffer = await fs.readFile(seed2FilePath);
-  if (seed1Binary.equals(seed2Binary)) {
-    console.log('Seeds are equal');
-    process.exit(0);
-  }
-
   const seed1Content = VariationsSeed.fromBinary(seed1Binary);
   const seed2Content = VariationsSeed.fromBinary(seed2Binary);
 
@@ -52,9 +54,30 @@ async function main(seed1FilePath: string, seed2FilePath: string) {
         seed2FilePath,
       ),
     );
-  } else {
+    process.exit(1);
+  }
+
+  if (!seed1Binary.equals(seed2Binary)) {
     console.error('Seeds semantically equal but binary different');
+    process.exit(1);
+  }
+
+  const seed1Serialnumber: string = await fs.readFile(
+    seed1SerialnumberFilePath,
+    'utf8',
+  );
+  const seed2Serialnumber: string = await fs.readFile(
+    seed2SerialnumberFilePath,
+    'utf8',
+  );
+  if (seed1Content.serial_number !== seed1Serialnumber) {
+    console.error('Seed1 serial number does not match');
+    process.exit(1);
+  }
+  if (seed2Content.serial_number !== seed2Serialnumber) {
+    console.error('Seed2 serial number does not match');
+    process.exit(1);
   }
 
-  process.exit(1);
+  console.log('Seeds are equal');
 }
diff --git a/src/seed_tools/commands/create.test.ts b/src/seed_tools/commands/create.test.ts
@@ -101,6 +101,35 @@ describe('create command', () => {
     );
   });
 
+  describe('serial number is equal in the seed and in the generated file', () => {
+    const validSeedsDir = path.join(testDataDir, 'valid_seeds');
+    it.each(fs_sync.readdirSync(validSeedsDir))(
+      'correctly creates %s',
+      async (testCase) => {
+        const testCaseDir = path.join(validSeedsDir, testCase);
+        const studiesDir = path.join(testCaseDir, 'studies');
+        const outputFile = path.join(tempDir, 'output.bin');
+        const serialNumberPath = path.join(tempDir, 'serial_number.txt');
+
+        await create().parseAsync([
+          'node',
+          'create',
+          studiesDir,
+          outputFile,
+          '--output_serial_number_file',
+          serialNumberPath,
+        ]);
+
+        const output = await fs.readFile(outputFile);
+        const outputSerialNumber = await fs.readFile(serialNumberPath, 'utf-8');
+        expect(outputSerialNumber).not.toEqual('1');
+        expect(VariationsSeed.fromBinary(output).serial_number).toEqual(
+          outputSerialNumber,
+        );
+      },
+    );
+  });
+
   describe('invalid studies', () => {
     const invalidStudiesDir = path.join(testDataDir, 'invalid_studies');
     it.each(fs_sync.readdirSync(invalidStudiesDir))(
@@ -117,8 +146,6 @@ describe('create command', () => {
             'create',
             studiesDir,
             outputFile,
-            '--mock_serial_number',
-            '1',
             '--output_serial_number_file',
             serialNumberPath,
           ]),
@@ -143,8 +170,6 @@ describe('create command', () => {
             'create',
             studiesDir,
             outputFile,
-            '--mock_serial_number',
-            '1',
             '--output_serial_number_file',
             serialNumberPath,
           ]),

diff --git a/src/seed_tools/seed_tools.ts b/src/seed_tools/seed_tools.ts
@@ -5,6 +5,7 @@
 
 import { program } from '@commander-js/extra-typings';
 
+import compare_python_gen from './commands/compare_python_gen';
 import compare_seeds from './commands/compare_seeds';
 import create from './commands/create';
 import lint from './commands/lint';
@@ -13,6 +14,7 @@ import split_seed_json from './commands/split_seed_json';
 program
   .name('seed_tools')
   .description('Seed tools for manipulating study files.')
+  .addCommand(compare_python_gen())
   .addCommand(compare_seeds())
   .addCommand(create())
   .addCommand(lint())

diff --git a/studies/AllowCertainClientHintsStudy.json5 b/studies/AllowCertainClientHintsStudy.json5
@@ -0,0 +1,34 @@
+[
+  {
+    name: 'AllowCertainClientHintsStudy',
+    experiment: [
+      {
+        name: 'Enabled',
+        probability_weight: 100,
+        feature_association: {
+          enable_feature: [
+            'AllowCertainClientHints',
+          ],
+        },
+      },
+      {
+        name: 'Default',
+        probability_weight: 0,
+      },
+    ],
+    filter: {
+      min_version: '104.1.44.59',
+      channel: [
+        'RELEASE',
+        'BETA',
+        'NIGHTLY',
+      ],
+      platform: [
+        'WINDOWS',
+        'MAC',
+        'LINUX',
+        'ANDROID',
+      ],
+    },
+  },
+]
diff --git a/studies/BraveAIChatEnabledStudy.json5 b/studies/BraveAIChatEnabledStudy.json5
@@ -0,0 +1,31 @@
+[
+  {
+    name: 'BraveAIChatEnabledStudy',
+    experiment: [
+      {
+        name: 'Enabled',
+        probability_weight: 100,
+        feature_association: {
+          enable_feature: [
+            'AIChat',
+          ],
+        },
+      },
+      {
+        name: 'Default',
+        probability_weight: 0,
+      },
+    ],
+    filter: {
+      min_version: '119.1.60.0',
+      channel: [
+        'RELEASE',
+      ],
+      platform: [
+        'WINDOWS',
+        'MAC',
+        'LINUX',
+      ],
+    },
+  },
+]
diff --git a/studies/BraveAdblockExperimentalListDefaultStudy.json5 b/studies/BraveAdblockExperimentalListDefaultStudy.json5
@@ -0,0 +1,33 @@
+[
+  {
+    name: 'BraveAdblockExperimentalListDefaultStudy',
+    experiment: [
+      {
+        name: 'Enabled',
+        probability_weight: 100,
+        feature_association: {
+          enable_feature: [
+            'BraveAdblockExperimentalListDefault',
+          ],
+        },
+      },
+      {
+        name: 'Default',
+        probability_weight: 0,
+      },
+    ],
+    filter: {
+      min_version: '123.1.66.53',
+      channel: [
+        'BETA',
+        'NIGHTLY',
+      ],
+      platform: [
+        'WINDOWS',
+        'MAC',
+        'LINUX',
+        'ANDROID',
+      ],
+    },
+  },
+]
diff --git a/studies/BraveAdblockMobileNotificationsListDefault.json5 b/studies/BraveAdblockMobileNotificationsListDefault.json5
@@ -0,0 +1,33 @@
+[
+  {
+    name: 'BraveAdblockMobileNotificationsListDefault',
+    experiment: [
+      {
+        name: 'Enabled',
+        probability_weight: 100,
+        feature_association: {
+          enable_feature: [
+            'BraveAdblockMobileNotificationsListDefault',
+          ],
+        },
+      },
+      {
+        name: 'Default',
+        probability_weight: 0,
+      },
+    ],
+    filter: {
+      channel: [
+        'RELEASE',
+        'BETA',
+        'NIGHTLY',
+      ],
+      platform: [
+        'WINDOWS',
+        'MAC',
+        'LINUX',
+        'ANDROID',
+      ],
+    },
+  },
+]