feat(common): precompile ajv schemas 🎺

common/web/types/.eslintrc.cjs

@@ -8,6 +8,8 @@ module.exports = {
     "coverage/*",
     "node_modules/*",
     "test/fixtures/*",
+    "tools/*",
+    "src/schemas/*"
   ],
   overrides: [
     {

common/web/types/.gitignore

@@ -1 +1,2 @@
-src/schemas/
+src/schemas/
+obj/

common/web/types/build.sh

@@ -40,16 +40,35 @@ function compile_schemas() {
     "$KEYMAN_ROOT/common/schemas/keyboard_info/keyboard_info.schema.json"
   )
 
+  rm -rf "$THIS_SCRIPT_PATH/obj/schemas"
+  mkdir -p "$THIS_SCRIPT_PATH/obj/schemas"
   rm -rf "$THIS_SCRIPT_PATH/src/schemas"
   mkdir -p "$THIS_SCRIPT_PATH/src/schemas"
   cp "${schemas[@]}" "$THIS_SCRIPT_PATH/src/schemas/"
 
   # TODO: use https://github.com/tc39/proposal-json-modules instead of this once it stablises
   for schema in "${schemas[@]}"; do
-    local fn="$THIS_SCRIPT_PATH/src/schemas/$(basename "$schema" .json)"
+    local schema_base="$(basename "$schema" .json)"
+    local fn="$THIS_SCRIPT_PATH/src/schemas/$schema_base"
+    local out="$THIS_SCRIPT_PATH/obj/schemas/$schema_base.validator.cjs"
+
+    # emit a .ts wrapper for the schema file
+
+    builder_echo "Compiling schema $schema_base.json"
     echo 'export default ' > "$fn.ts"
     cat "$fn.json" >> "$fn.ts"
+
+    # emit a compiled validator for the schema file
+
+    # While would seem obvious to just run 'ajv' directly here, somewhere node
+    # is picking up the wrong path for the build and breaking the formats
+    # imports. So it is essential to use `npm run` at this point, even though it
+    # is painfully slower, at least until we figure out the path discrepancy.
+    npm run build:schema -- -c ./tools/formats.cjs -s "$fn.json" --strict-types false -o "$out"
   done
+
+  # the validators now need to be compiled to esm
+  node tools/schema-bundler.js
 }
 
 function copy_cldr_imports() {

common/web/types/package.json

@@ -16,6 +16,7 @@
   ],
   "scripts": {
     "build": "tsc -b",
+    "build:schema": "ajv compile",
     "lint": "eslint .",
     "test": "npm run lint && cd test && tsc -b && cd .. && c8 --skip-full --reporter=lcov --reporter=text mocha",
     "prepublishOnly": "npm run build"
@@ -27,7 +28,6 @@
   },
   "dependencies": {
     "@keymanapp/keyman-version": "*",
-    "ajv": "^8.11.0",
     "restructure": "git+https://github.com/keymanapp/dependency-restructure.git#7a188a1e26f8f36a175d95b67ffece8702363dfc",
     "semver": "^7.5.2",
     "xml2js": "git+https://github.com/keymanapp/dependency-node-xml2js#535fe732dc408d697e0f847c944cc45f0baf0829"
@@ -39,6 +39,9 @@
     "@types/node": "^20.4.1",
     "@types/semver": "^7.3.12",
     "@types/xml2js": "^0.4.5",
+    "ajv": "^8.12.0",
+    "ajv-cli": "^5.0.0",
+    "ajv-formats": "^2.1.1",
     "c8": "^7.12.0",
     "chai": "^4.3.4",
     "chalk": "^2.4.2",
@@ -74,6 +77,7 @@
       "src/ldml-keyboard/unicodeset-parser-api.ts",
       "src/keyman-touch-layout/keyman-touch-layout-file-writer.ts",
       "src/osk/osk.ts",
+      "src/schemas/*",
       "test/"
     ]
   }

common/web/types/src/keyman-touch-layout/keyman-touch-layout-file-reader.ts

@@ -1,7 +1,5 @@
-import { default as AjvModule } from 'ajv';
-const Ajv = AjvModule.default; // The actual expected Ajv type.
 import { TouchLayoutFile } from "./keyman-touch-layout-file.js";
-import Schemas from '../../src/schemas.js';
+import SchemaValidators from '../schema-validators.js';
 
 export class TouchLayoutFileReader {
   public read(source: Uint8Array): TouchLayoutFile {
@@ -69,11 +67,10 @@ export class TouchLayoutFileReader {
   }
 
   public validate(source: TouchLayoutFile): void {
-    const ajv = new Ajv();
-    if(!ajv.validate(Schemas.touchLayoutClean, source))
+    if(!SchemaValidators.touchLayoutClean(source))
     /* c8 ignore next 3 */
     {
-      throw new Error(ajv.errorsText());
+      throw new Error((<any>SchemaValidators.touchLayoutClean).errors);
     }
   }
 

common/web/types/src/kpj/kpj-file-reader.ts

@@ -1,11 +1,9 @@
 import * as xml2js from 'xml2js';
 import { KPJFile, KPJFileProject } from './kpj-file.js';
-import { default as AjvModule } from 'ajv';
-const Ajv = AjvModule.default; // The actual expected Ajv type.
 import { boxXmlArray } from '../util/util.js';
 import { KeymanDeveloperProject, KeymanDeveloperProjectFile10, KeymanDeveloperProjectType } from './keyman-developer-project.js';
 import { CompilerCallbacks } from '../util/compiler-interfaces.js';
-import Schemas from '../schemas.js';
+import SchemaValidators from '../schema-validators.js';
 
 export class KPJFileReader {
   constructor(private callbacks: CompilerCallbacks) {
@@ -35,13 +33,11 @@ export class KPJFileReader {
   }
 
   public validate(source: KPJFile): void {
-    const ajv = new Ajv();
-    if(!ajv.validate(Schemas.kpj, source)) {
-      const ajvLegacy = new Ajv();
-      if(!ajvLegacy.validate(Schemas.kpj90, source)) {
+    if(!SchemaValidators.kpj(source)) {
+      if(!SchemaValidators.kpj90(source)) {
         // If the legacy schema also does not validate, then we will only report
         // the errors against the modern schema
-        throw new Error(ajv.errorsText());
+        throw new Error((<any>SchemaValidators.kpj).errors);
       }
     }
   }

common/web/types/src/kvk/kvks-file-reader.ts

@@ -1,12 +1,10 @@
 import * as xml2js from 'xml2js';
 import KVKSourceFile from './kvks-file.js';
-import { default as AjvModule } from 'ajv';
-const Ajv = AjvModule.default; // The actual expected Ajv type.
 import { boxXmlArray } from '../util/util.js';
 import { DEFAULT_KVK_FONT, VisualKeyboard, VisualKeyboardHeaderFlags, VisualKeyboardKey, VisualKeyboardKeyFlags, VisualKeyboardLegalShiftStates, VisualKeyboardShiftState } from './visual-keyboard.js';
 import { USVirtualKeyCodes } from '../consts/virtual-key-constants.js';
 import { BUILDER_KVK_HEADER_VERSION, KVK_HEADER_IDENTIFIER_BYTES } from './kvk-file.js';
-import Schemas from '../schemas.js';
+import SchemaValidators from '../schema-validators.js';
 
 
 export default class KVKSFileReader {
@@ -85,9 +83,8 @@ export default class KVKSFileReader {
   }
 
   public validate(source: KVKSourceFile): void {
-    const ajv = new Ajv();
-    if(!ajv.validate(Schemas.kvks, source)) {
-      throw new Error(ajv.errorsText());
+    if(!SchemaValidators.kvks(source)) {
+      throw new Error((<any>SchemaValidators.kvks).errorsText());
     }
   }
 

common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts

@@ -1,13 +1,11 @@
 import * as xml2js from 'xml2js';
 import { LDMLKeyboardXMLSourceFile, LKImport, ImportStatus } from './ldml-keyboard-xml.js';
-import { default as AjvModule } from 'ajv';
-const Ajv = AjvModule.default; // The actual expected Ajv type.
 import { boxXmlArray } from '../util/util.js';
 import { CompilerCallbacks } from '../util/compiler-interfaces.js';
 import { constants } from '@keymanapp/ldml-keyboard-constants';
 import { CommonTypesMessages } from '../util/common-events.js';
 import { LDMLKeyboardTestDataXMLSourceFile, LKTTest, LKTTests } from './ldml-keyboard-testdata-xml.js';
-import Schemas from '../schemas.js';
+import SchemaValidators from '../schema-validators.js';
 
 interface NameAndProps  {
   '$'?: any; // content
@@ -237,9 +235,8 @@ export class LDMLKeyboardXMLSourceFileReader {
    * @returns true if valid, false if invalid
    */
   public validate(source: LDMLKeyboardXMLSourceFile | LDMLKeyboardTestDataXMLSourceFile): boolean {
-    const ajv = new Ajv();
-    if(!ajv.validate(Schemas.ldmlKeyboard3, source)) {
-      for (let err of ajv.errors) {
+    if(!SchemaValidators.ldmlKeyboard3(source)) {
+      for (let err of (<any>SchemaValidators.ldmlKeyboard3).errors) {
         this.callbacks.reportMessage(CommonTypesMessages.Error_SchemaValidationError({
           instancePath: err.instancePath,
           keyword: err.keyword,

common/web/types/src/main.ts

@@ -47,4 +47,5 @@ export * as KeymanFileTypes from './util/file-types.js';
 
 export * as Osk from './osk/osk.js';
 
-export * as Schemas from './schemas.js';
+export * as Schemas from './schemas.js';
+export * as SchemaValidators from './schema-validators.js';

common/web/types/src/osk/osk.ts

@@ -1,8 +1,6 @@
 import { TouchLayoutFile, TouchLayoutFlick, TouchLayoutKey, TouchLayoutPlatform, TouchLayoutSubKey } from "src/keyman-touch-layout/keyman-touch-layout-file.js";
 import { VisualKeyboard } from "../kvk/visual-keyboard.js";
-import { default as AjvModule } from 'ajv';
-import Schemas from "../schemas.js";
-const Ajv = AjvModule.default; // The actual expected Ajv type.
+import SchemaValidators from "../schema-validators.js";
 
 export interface StringRefUsage {
   filename: string;
@@ -24,11 +22,10 @@ export interface StringResult {
 export type PuaMap = {[index:string]: string};
 
 export function parseMapping(mapping: any) {
-  const ajv = new Ajv();
-  if(!ajv.validate(Schemas.displayMap, <any>mapping))
+  if(!SchemaValidators.displayMap(<any>mapping))
   /* c8 ignore next 3 */
   {
-    throw new Error(ajv.errorsText());
+    throw new Error((<any>SchemaValidators.displayMap).errorsText());
   }
 
   let map: PuaMap = {};

common/web/types/src/schema-validators.ts

@@ -0,0 +1,23 @@
+import kpj from './schemas/kpj.schema.validator.mjs';
+import kpj90 from './schemas/kpj-9.0.schema.validator.mjs';
+import kvks from './schemas/kvks.schema.validator.mjs';
+import ldmlKeyboard3 from './schemas/ldml-keyboard3.schema.validator.mjs';
+import ldmlKeyboardTest3 from './schemas/ldml-keyboardtest3.schema.validator.mjs';
+import displayMap from './schemas/displaymap.schema.validator.mjs';
+import touchLayoutClean from './schemas/keyman-touch-layout.clean.spec.validator.mjs';
+import touchLayout from './schemas/keyman-touch-layout.spec.validator.mjs';
+import keyboard_info from './schemas/keyboard_info.schema.validator.mjs';
+
+const SchemaValidators = {
+  kpj,
+  kpj90,
+  kvks,
+  ldmlKeyboard3,
+  ldmlKeyboardTest3,
+  displayMap,
+  touchLayoutClean,
+  touchLayout,
+  keyboard_info,
+};
+
+export default SchemaValidators;

common/web/types/test/helpers/index.ts

@@ -1,5 +1,5 @@
-import path from "path";
-import fs from "fs";
+import * as path from "path";
+import * as fs from "fs";
 import { fileURLToPath } from "url";
 
 /**

common/web/types/test/tsconfig.json

@@ -7,7 +7,7 @@
     "outDir": "../build/test",
     "baseUrl": ".",
     "strictNullChecks": false, // TODO: get rid of this as some point
-    "allowSyntheticDefaultImports": true // for ajv
+    "allowSyntheticDefaultImports": true
   },
   "include": [
     "**/test-*.ts",

common/web/types/tools/formats.cjs

@@ -0,0 +1,10 @@
+/*
+ * This somewhat peculiar function is used in `build.sh configure` when
+ * precompiling the validators and makes it possible to use the extended formats
+ * in ajv-formats.
+ */
+function formats(ajv) {
+  require("ajv-formats")(ajv);
+}
+
+module.exports = formats;

common/web/types/tools/schema-bundler.js

@@ -0,0 +1,27 @@
+/*
+ * Bundle schema validation files (from .cjs) and make them available as ES modules
+ */
+
+import esbuild from 'esbuild';
+
+await esbuild.build({
+  entryPoints: [
+    'obj/schemas/kpj.schema.validator.cjs',
+    'obj/schemas/kpj-9.0.schema.validator.cjs',
+    'obj/schemas/kvks.schema.validator.cjs',
+    'obj/schemas/ldml-keyboard3.schema.validator.cjs',
+    'obj/schemas/ldml-keyboardtest3.schema.validator.cjs',
+    'obj/schemas/displaymap.schema.validator.cjs',
+    'obj/schemas/keyman-touch-layout.clean.spec.validator.cjs',
+    'obj/schemas/keyman-touch-layout.spec.validator.cjs',
+    'obj/schemas/keyboard_info.schema.validator.cjs',
+  ],
+  bundle: true,
+  format: 'esm',
+  target: 'es2022',
+  outdir: 'src/schemas/',
+  sourcemap: false,
+
+  // We want a .mjs extension to force node into ESM module mode
+  outExtension: { '.js': '.mjs' },
+});

common/web/types/tsconfig.json

@@ -5,11 +5,11 @@
     "outDir": "build/src/",
     "rootDir": "src/",
     "baseUrl": ".",
-    "allowSyntheticDefaultImports": true, // for ajv
-    "resolveJsonModule": true,
+    "allowSyntheticDefaultImports": true,
   },
   "include": [
-    "src/**/*.ts"
+    "src/**/*.ts",
+    "src/schemas/*.mjs", // Import the validators
   ],
   "references": [
     { "path": "../keyman-version" },

common/web/utils/package.json

@@ -23,13 +23,13 @@
   },
   "homepage": "https://github.com/keymanapp/keyman#readme",
   "devDependencies": {
-    "@keymanapp/resources-gosh": "*",
     "@keymanapp/keyman-version": "*",
+    "@keymanapp/resources-gosh": "*",
+    "@types/node": "^14.0.5",
     "c8": "^7.12.0",
     "chai": "^4.3.4",
     "mocha": "^10.0.0",
     "mocha-teamcity-reporter": "^4.0.0",
-    "@types/node": "^14.0.5",
     "typescript": "^4.9.5"
   },
   "type": "module",

developer/src/kmc-keyboard-info/build.sh

@@ -11,6 +11,7 @@ cd "$THIS_SCRIPT_PATH"
 
 builder_describe "Build Keyman kmc keyboard-info Compiler module" \
   "@/common/web/types" \
+  "@/developer/src/common/web/utils" \
   "clean" \
   "configure" \
   "build" \

developer/src/kmc-keyboard-info/package.json

@@ -25,10 +25,8 @@
   },
   "dependencies": {
     "@keymanapp/common-types": "*",
-    "@keymanapp/kmc-package": "*",
     "@keymanapp/developer-utils": "*",
-    "ajv": "^8.11.0",
-    "ajv-formats": "^2.1.1"
+    "@keymanapp/kmc-package": "*"
   },
   "bundleDependencies": [
     "@keymanapp/developer-utils"

developer/src/kmc-keyboard-info/src/index.ts

@@ -11,12 +11,7 @@ import langtags from "./imports/langtags.js";
 import { validateMITLicense } from "@keymanapp/developer-utils";
 import { KmpCompiler } from "@keymanapp/kmc-package";
 
-import AjvModule from 'ajv';
-import AjvFormatsModule from 'ajv-formats';
-const Ajv = AjvModule.default; // The actual expected Ajv type.
-const ajvFormats = AjvFormatsModule.default;
-
-import { Schemas } from "@keymanapp/common-types";
+import { SchemaValidators } from "@keymanapp/common-types";
 import { packageKeysExamplesToKeyboardInfo } from "./example-keys.js";
 
 const regionNames = new Intl.DisplayNames(['en'], { type: "region" });
@@ -290,17 +285,10 @@ export class KeyboardInfoCompiler {
 
     const jsonOutput = JSON.stringify(keyboard_info, null, 2);
 
-    // TODO: look at performance improvements by precompiling Ajv schemas on first use
-    const ajv = new Ajv({ logger: {
-      log: (message) => this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Hint_OutputValidation({message})),
-      warn: (message) => this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Warn_OutputValidation({message})),
-      error: (message) => this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Error_OutputValidation({message})),
-    }});
-    ajvFormats.default(ajv);
-    if(!ajv.validate(Schemas.default.keyboard_info, keyboard_info)) {
+    if(!SchemaValidators.default.keyboard_info(keyboard_info)) {
       // This is an internal fatal error; we should not be capable of producing
       // invalid output, so it is best to throw and die
-      throw new Error(ajv.errorsText());
+      throw new Error((<any>SchemaValidators.default.keyboard_info).errorsText());
     }
 
     return new TextEncoder().encode(jsonOutput);

developer/src/kmc-ldml/package.json

@@ -29,7 +29,6 @@
     "@keymanapp/keyman-version": "*",
     "@keymanapp/kmc-kmn": "*",
     "@keymanapp/ldml-keyboard-constants": "*",
-    "ajv": "^8.11.0",
     "restructure": "git+https://github.com/keymanapp/dependency-restructure.git#7a188a1e26f8f36a175d95b67ffece8702363dfc",
     "semver": "^7.5.2",
     "xml2js": "git+https://github.com/keymanapp/dependency-node-xml2js#535fe732dc408d697e0f847c944cc45f0baf0829"