From fa8febd6156f73d9ff61c685c8fb3885785b493a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ro=C5=BCek?= Date: Sun, 12 May 2024 23:12:57 +0200 Subject: [PATCH] feat: support more powerful custom shorthands --- src/__tests__/codegen.test.mjs | 72 +++-- src/__tests__/index.test.mjs | 280 ++++++++++++++---- src/codegen/__tests__/iterator.test.mjs | 27 ++ ...generate-filter-script-expression.test.mjs | 4 - src/codegen/baseline/generators.mjs | 32 +- src/codegen/baseline/index.mjs | 22 +- src/codegen/fast-paths/all-parents.mjs | 3 +- src/codegen/fast-paths/fixed.mjs | 15 +- src/codegen/guards.mjs | 4 + src/codegen/iterator.mjs | 33 ++- src/codegen/templates/fn-params.mjs | 5 - src/codegen/templates/scope.mjs | 1 + src/codegen/templates/tree-method-call.mjs | 22 +- src/codegen/templates/tree-method.mjs | 30 +- src/codegen/tree/consts.mjs | 3 + src/codegen/tree/tree.mjs | 178 +++++------ src/codegen/utils/jsonpath-hashes.mjs | 4 - src/core/index.mjs | 8 +- src/index.d.ts | 10 +- src/parser/__tests__/parser.test.mjs | 69 +++-- src/parser/parser.mjs | 64 +++- 21 files changed, 573 insertions(+), 313 deletions(-) delete mode 100644 src/codegen/templates/fn-params.mjs create mode 100644 src/codegen/tree/consts.mjs diff --git a/src/__tests__/codegen.test.mjs b/src/__tests__/codegen.test.mjs index 172d7d7..135fa18 100644 --- a/src/__tests__/codegen.test.mjs +++ b/src/__tests__/codegen.test.mjs @@ -1289,16 +1289,8 @@ export default function (input, callbacks) { describe('custom shorthands', () => { it('should be supported', () => { - const shorthands = { - schema: ['patternProperties', 'properties'] - .map(k => `scope.path[scope.path.length - 2] === '${k}'`) - .join(' || '), - }; - assert.equal( - generate(['$.components.schemas[*]..@@schema()'], { - customShorthands: shorthands, - }), + generate(['$.components.schemas[*]..@@schema(0)']), `import {Scope} from "nimma/runtime"; const zones = { keys: ["components"], @@ -1310,24 +1302,19 @@ const zones = { }] }; const tree = { - "$.components.schemas[*]..@@schema()": function (scope) { - if (scope.path.length < 4) return; + "$.components.schemas[*]..@@schema(0)": function (scope, shorthands) { + if (scope.path.length < 3) return; if (scope.path[0] !== "components") return; if (scope.path[1] !== "schemas") return; if (!shorthands.schema(scope)) return; - scope.emit("$.components.schemas[*]..@@schema()", 0, false); - } -}; -const shorthands = { - schema: function (scope, state) { - return scope.path[scope.path.length - 2] === 'patternProperties' || scope.path[scope.path.length - 2] === 'properties'; + scope.emit("$.components.schemas[*]..@@schema(0)", 0, false); } }; -export default function (input, callbacks) { +export default function (input, callbacks, shorthands) { const scope = new Scope(input, callbacks); try { scope.traverse(() => { - tree["$.components.schemas[*]..@@schema()"](scope); + tree["$.components.schemas[*]..@@schema(0)"](scope, shorthands); }, zones); } finally { scope.destroy(); @@ -1337,13 +1324,46 @@ export default function (input, callbacks) { ); }); - it('should refuse to use an undefined shorthand', () => { - assert.throws( - generate.bind(null, ['$.components.schemas[*]..@@schema()'], { - customShorthands: {}, - }), - ReferenceError, - "Shorthand 'schema' is not defined", + it('should adjust state', () => { + assert.deepEqual( + generate(['$.components.schemas[*]..abc..@@schema(2)..enum']), + `import {Scope} from "nimma/runtime"; +const zones = { + keys: ["components"], + zones: [{ + keys: ["schemas"], + zones: [{ + zone: null + }] + }] +}; +const tree = { + "$.components.schemas[*]..abc..@@schema(2)..enum": function (scope, state, shorthands) { + if (scope.path.length < 4) return; + if (scope.path[0] !== "components") return; + if (scope.path[1] !== "schemas") return; + if (state.initialValue >= 0) { + if (scope.path[scope.path.length - 1] === "abc") { + state.value |= 1 + } + } + if (!shorthands.schema(scope, state, 1)) return; + if (state.initialValue < 15 || !(scope.path[scope.path.length - 1] === "enum")) return; + scope.emit("$.components.schemas[*]..abc..@@schema(2)..enum", 0, false); + } +}; +export default function (input, callbacks, shorthands) { + const scope = new Scope(input, callbacks); + try { + const state0 = scope.allocState(); + scope.traverse(() => { + tree["$.components.schemas[*]..abc..@@schema(2)..enum"](scope, state0, shorthands); + }, zones); + } finally { + scope.destroy(); + } +} +`, ); }); }); diff --git a/src/__tests__/index.test.mjs b/src/__tests__/index.test.mjs index ef1772d..ae772d0 100644 --- a/src/__tests__/index.test.mjs +++ b/src/__tests__/index.test.mjs @@ -56,7 +56,7 @@ describe('Nimma', () => { }); }); - it('works #2', () => { + it('works#2', () => { const document = { info: { contact: { @@ -76,7 +76,7 @@ describe('Nimma', () => { }); }); - it('works #3', () => { + it('works#3', () => { const document = { info: { contact: { @@ -102,7 +102,7 @@ describe('Nimma', () => { }); }); - it('works #4', () => { + it('works#4', () => { const document = { info: { contact: { @@ -128,7 +128,7 @@ describe('Nimma', () => { }); }); - it('works #5', () => { + it('works#5', () => { const document = { paths: { bar: { @@ -162,7 +162,7 @@ describe('Nimma', () => { }); }); - it('works #6', () => { + it('works#6', () => { const document = { paths: { bar: { @@ -198,7 +198,7 @@ describe('Nimma', () => { }); }); - it('works #7', () => { + it('works#7', () => { const document = { paths: { bar: { @@ -234,7 +234,7 @@ describe('Nimma', () => { }); }); - it('works #8', () => { + it('works#8', () => { const document = { paths: { bar: { @@ -269,7 +269,7 @@ describe('Nimma', () => { }); }); - it('works #9', () => { + it('works#9', () => { const document = { paths: { bar: { @@ -300,7 +300,7 @@ describe('Nimma', () => { }); }); - it('works #10', () => { + it('works#10', () => { const document = { bar: { foo: { @@ -335,7 +335,7 @@ describe('Nimma', () => { }); }); - it('works #11', () => { + it('works#11', () => { const document = { bar: { 200: { @@ -366,7 +366,7 @@ describe('Nimma', () => { }); }); - it('works #12', () => { + it('works#12', () => { const document = { bar: { 200: { @@ -397,7 +397,7 @@ describe('Nimma', () => { }); }); - it('works #13', () => { + it('works#13', () => { const document = { bar: { 200: { @@ -430,7 +430,7 @@ describe('Nimma', () => { }); }); - it('works #14', () => { + it('works#14', () => { const document = { bar: { examples: { @@ -456,7 +456,7 @@ describe('Nimma', () => { }); }); - it('works #15', () => { + it('works#15', () => { const document = { info: { contact: { @@ -472,7 +472,7 @@ describe('Nimma', () => { }); }); - it('works #16', () => { + it('works#16', () => { const document = { parameters: [ { @@ -500,7 +500,7 @@ describe('Nimma', () => { }); }); - it('works #17', () => { + it('works#17', () => { const document = { bar: { user: { @@ -529,7 +529,7 @@ describe('Nimma', () => { }); }); - it('works #18', () => { + it('works#18', () => { const document = { example: 'test', examples: { @@ -559,7 +559,7 @@ describe('Nimma', () => { }); }); - it('works #19', () => { + it('works#19', () => { const document = { channels: { '/a': { @@ -592,7 +592,7 @@ describe('Nimma', () => { }); }); - it('works #20', () => { + it('works#20', () => { const document = { openapi: '3.0.2', components: { @@ -622,7 +622,7 @@ describe('Nimma', () => { }); }); - it('works #21', () => { + it('works#21', () => { const document = { firstName: 'John', lastName: 'doe', @@ -658,7 +658,7 @@ describe('Nimma', () => { }); }); - it('works #22', () => { + it('works#22', () => { const document = { test1: { example: true, @@ -672,7 +672,7 @@ describe('Nimma', () => { }); }); - it('works #24', () => { + it('works#24', () => { const document = { channels: [ { @@ -694,7 +694,7 @@ describe('Nimma', () => { }); }); - it('works #25', () => { + it('works#25', () => { const document = { continents: [ { @@ -755,7 +755,7 @@ describe('Nimma', () => { }); }); - it('works #26', () => { + it('works#26', () => { const document = [ 'Moscow', 'Saint Petersburg', @@ -798,7 +798,7 @@ describe('Nimma', () => { }); }); - it('works #27', () => { + it('works#27', () => { const document = { size: 'xl', }; @@ -811,7 +811,7 @@ describe('Nimma', () => { }); }); - it('works #28', () => { + it('works#28', () => { const document = { Europe: { East: { @@ -832,7 +832,7 @@ describe('Nimma', () => { }); }); - it('works #29', () => { + it('works#29', () => { const document = { paths: { '/some-url/{someId}': { @@ -862,7 +862,7 @@ describe('Nimma', () => { }); }); - it('works #30', () => { + it('works#30', () => { const document = { Europe: { East: { @@ -885,7 +885,7 @@ describe('Nimma', () => { }); }); - it('works #31', () => { + it('works#31', () => { const document = { Asia: ['Malaysia', 'Indonesia', 'Thailand', 'Laos', 'Myanmar', 'Vietnam'], Europe: ['Austria', 'Belgium', 'Czechia', 'France', 'Germany'], @@ -910,7 +910,7 @@ describe('Nimma', () => { }); }); - it('works #32', () => { + it('works#32', () => { const document = [ { country: 'Poland', @@ -923,7 +923,7 @@ describe('Nimma', () => { assert.deepEqual(collected, {}); }); - it('works #33', () => { + it('works#33', () => { const document = [ { country: 'Poland', @@ -949,7 +949,7 @@ describe('Nimma', () => { }); }); - it('works #34', () => { + it('works#34', () => { const document = { data: { geo: { @@ -981,9 +981,30 @@ describe('Nimma', () => { }); }); - describe('custom shorthands', () => { - it('should be supported', () => { + describe.only('custom shorthands', () => { + it('should support deep shorthands', () => { const document = { + paths: { + '/users': { + get: { + responses: { + 200: { + description: 'A list of users.', + content: { + 'application/json': { + schema: { + type: 'array', + items: { + $ref: '#/components/schemas/User', + }, + }, + }, + }, + }, + }, + }, + }, + }, components: { schemas: { User: { @@ -1007,46 +1028,170 @@ describe('Nimma', () => { patternProperties: { '^x-': true, }, + additionalProperties: false, + 'x-ignore': { + type: 'object', + properties: { + foo: { + type: 'string', + }, + }, + }, + }, + Name: { + oneOf: [ + { + type: 'string', + }, + { + type: 'null', + }, + ], }, }, }, }; const shorthands = { - schema: ['patternProperties', 'properties'] - .map(k => `scope.path[scope.path.length - 2] === '${k}'`) - .join(' || '), + schema: function (scope, state, initialValue) { + if (state.value < initialValue) return; + + const nextValue = (initialValue << 1) + 1; + if (state.initialValue === initialValue) { + if (isSchema(scope.sandbox.value)) { + state.value = nextValue; + return true; + } + + state.value = -1; + return false; + } + + if (state.initialValue === nextValue) { + const property = scope.path.at(-1); + switch (true) { + case ARRAY_ONLY_SCHEMA.includes(property): + if (Array.isArray(scope.sandbox.value)) { + state.value = initialValue; + } else { + state.value = -1; + } + + return false; + case OBJECT_ONLY_SCHEMA.includes(property): + if (isPlainObject(scope.sandbox.value)) { + state.value = initialValue; + } else { + state.value = -1; + } + + return false; + case property === 'items': + if (Array.isArray(scope.sandbox.value)) { + state.value = initialValue; + } else if (isSchema(scope.sandbox.value)) { + state.value = nextValue; + return true; + } else { + state.value = -1; + } + + return false; + case TOP_LEVEL.includes(property): + if (isSchema(scope.sandbox.value)) { + state.value = nextValue; + return true; + } + + state.value = -1; + return false; + default: + state.value = -1; + return false; + } + } + + return state.initialValue === initialValue; + }, }; + const ARRAY_ONLY_SCHEMA = ['allOf', 'oneOf', 'anyOf', 'prefixItems']; + + const OBJECT_ONLY_SCHEMA = [ + 'properties', + 'patternProperties', + '$defs', + 'definitions', + ]; + + const TOP_LEVEL = [ + 'if', + 'then', + 'else', + 'not', + 'additionalProperties', + 'unevaluatedProperties', + 'items', + 'contains', + 'additionalItems', + 'unevaluatedItems', + ]; + + function isSchema(value) { + return ( + (isPlainObject(value) && !Object.hasOwn(value, '$ref')) || + typeof value === 'boolean' + ); + } + + function isPlainObject(value) { + return ( + typeof value === 'object' && value !== null && !Array.isArray(value) + ); + } + const collected = collect( document, - ['$.components.schemas[*]..@@schema()'], + [ + '$.paths[*][get,put].responses[*].content[*].schema..@@schema(2)', + '$.components.schemas[*]..@@schema(2)', + ], { customShorthands: shorthands, }, ); assert.deepEqual(collected, { - '$.components.schemas[*]..@@schema()': [ + '$.paths[*][get,put].responses[*].content[*].schema..@@schema(2)': [ [ - { type: 'string' }, + document.paths['/users'].get.responses[200].content[ + 'application/json' + ].schema, + [ + 'paths', + '/users', + 'get', + 'responses', + '200', + 'content', + 'application/json', + 'schema', + ], + ], + ], + '$.components.schemas[*]..@@schema(2)': [ + [document.components.schemas.User, ['components', 'schemas', 'User']], + [ + document.components.schemas.User.properties.id, ['components', 'schemas', 'User', 'properties', 'id'], ], [ - { - type: 'object', - properties: { - street: { - type: 'string', - }, - }, - }, + document.components.schemas.User.properties.address, ['components', 'schemas', 'User', 'properties', 'address'], ], [ - { - type: 'string', - }, + document.components.schemas.User.properties.address.properties + .street, [ 'components', 'schemas', @@ -1058,15 +1203,32 @@ describe('Nimma', () => { ], ], [ - true, + document.components.schemas.Extensions, + ['components', 'schemas', 'Extensions'], + ], + [ + document.components.schemas.Extensions.patternProperties['^x-'], ['components', 'schemas', 'Extensions', 'patternProperties', '^x-'], ], + [ + document.components.schemas.Extensions.additionalProperties, + ['components', 'schemas', 'Extensions', 'additionalProperties'], + ], + [document.components.schemas.Name, ['components', 'schemas', 'Name']], + [ + document.components.schemas.Name.oneOf[0], + ['components', 'schemas', 'Name', 'oneOf', 0], + ], + [ + document.components.schemas.Name.oneOf[1], + ['components', 'schemas', 'Name', 'oneOf', 1], + ], ], }); }); }); - it('works #35', () => { + it('works#35', () => { const document = { definitions: { propA: { @@ -1101,7 +1263,7 @@ describe('Nimma', () => { }); }); - it('works #36', () => { + it('works#36', () => { const document = { foo: { bar: 'foo-bar', @@ -1137,7 +1299,7 @@ describe('Nimma', () => { }); }); - it('works #37', () => { + it('works#37', () => { const document = { paths: { '/pet': { @@ -1177,7 +1339,7 @@ describe('Nimma', () => { }); }); - it('works #38', () => { + it('works#38', () => { const document = { foo: { example: { @@ -1215,7 +1377,7 @@ describe('Nimma', () => { }); }); - it('works #39', () => { + it('works#39', () => { const document = { baz: { a: { @@ -1248,7 +1410,7 @@ describe('Nimma', () => { }); }); - it('works #40', () => { + it('works#40', () => { const document = { baz: { baz: { @@ -1278,7 +1440,7 @@ describe('Nimma', () => { }); }); - it('works #41', () => { + it('works#41', () => { const document = { baz: { baz: { diff --git a/src/codegen/__tests__/iterator.test.mjs b/src/codegen/__tests__/iterator.test.mjs index 2ff4934..d40cea4 100644 --- a/src/codegen/__tests__/iterator.test.mjs +++ b/src/codegen/__tests__/iterator.test.mjs @@ -13,6 +13,7 @@ describe('Iterator', () => { fixed: true, inverseOffset: -1, minimumDepth: 0, + shorthands: 0, stateOffset: -1, }); }); @@ -24,6 +25,7 @@ describe('Iterator', () => { fixed: false, inverseOffset: -1, minimumDepth: 1, + shorthands: 0, stateOffset: -1, }); }); @@ -35,6 +37,7 @@ describe('Iterator', () => { fixed: false, inverseOffset: 1, minimumDepth: 2, + shorthands: 0, stateOffset: -1, }); }); @@ -46,6 +49,7 @@ describe('Iterator', () => { fixed: false, inverseOffset: -1, minimumDepth: 1, + shorthands: 0, stateOffset: 1, }); }); @@ -57,6 +61,7 @@ describe('Iterator', () => { fixed: false, inverseOffset: -1, minimumDepth: 0, + shorthands: 0, stateOffset: 0, }); }); @@ -68,6 +73,7 @@ describe('Iterator', () => { fixed: true, inverseOffset: -1, minimumDepth: 2, + shorthands: 0, stateOffset: -1, }); }); @@ -79,6 +85,7 @@ describe('Iterator', () => { fixed: false, inverseOffset: -1, minimumDepth: 1, + shorthands: 0, stateOffset: 1, }); }); @@ -90,6 +97,7 @@ describe('Iterator', () => { fixed: false, inverseOffset: -1, minimumDepth: 0, + shorthands: 0, stateOffset: 0, }); }); @@ -101,6 +109,7 @@ describe('Iterator', () => { fixed: false, inverseOffset: 0, minimumDepth: 1, + shorthands: 0, stateOffset: -1, }); }); @@ -114,6 +123,7 @@ describe('Iterator', () => { fixed: true, inverseOffset: -1, minimumDepth: 3, + shorthands: 0, stateOffset: 3, }); }); @@ -125,6 +135,7 @@ describe('Iterator', () => { fixed: true, inverseOffset: -1, minimumDepth: 1, + shorthands: 0, stateOffset: 1, }); }); @@ -136,6 +147,7 @@ describe('Iterator', () => { fixed: false, inverseOffset: 2, minimumDepth: 3, + shorthands: 0, stateOffset: -1, }); }); @@ -147,6 +159,7 @@ describe('Iterator', () => { fixed: false, inverseOffset: 0, minimumDepth: 1, + shorthands: 0, stateOffset: -1, }); }); @@ -158,6 +171,7 @@ describe('Iterator', () => { fixed: false, inverseOffset: 0, minimumDepth: 2, + shorthands: 0, stateOffset: -1, }); }); @@ -169,8 +183,21 @@ describe('Iterator', () => { fixed: true, inverseOffset: -1, minimumDepth: 3, + shorthands: 0, stateOffset: -1, }); }); + + it('$.components.schemas[*]..@@schema(2)', () => { + const ast = parse('$.components.schemas[*]..@@schema(2)'); + + assert.deepEqual(Iterator.analyze(ast), { + fixed: false, + inverseOffset: -1, + minimumDepth: 2, + shorthands: 1, + stateOffset: 3, + }); + }); }); }); diff --git a/src/codegen/baseline/__tests__/generate-filter-script-expression.test.mjs b/src/codegen/baseline/__tests__/generate-filter-script-expression.test.mjs index b3c1997..c4a5500 100644 --- a/src/codegen/baseline/__tests__/generate-filter-script-expression.test.mjs +++ b/src/codegen/baseline/__tests__/generate-filter-script-expression.test.mjs @@ -148,10 +148,6 @@ describe('generateFilterScriptExpression', () => { ); }); - it('supports custom handlers', () => { - assert.equal(print(`?(@@schema())`), `!shorthands.schema(scope)`); - }); - it('throws upon unknown shorthand', () => { assert.throws( () => print(`?(@foo())`), diff --git a/src/codegen/baseline/generators.mjs b/src/codegen/baseline/generators.mjs index 51084bd..4c95520 100644 --- a/src/codegen/baseline/generators.mjs +++ b/src/codegen/baseline/generators.mjs @@ -254,6 +254,26 @@ export function generateWildcardExpression(branch, iterator) { } } +export function generateCustomShorthandExpression(branch, iterator, node) { + branch.push( + b.ifStatement( + b.unaryExpression( + '!', + b.callExpression( + b.memberExpression( + internalScope.shorthands, + b.identifier(node.value), + ), + iterator.state.usesState + ? [scope._, state._, b.numericLiteral(iterator.state.numbers[0])] + : [scope._], + ), + ), + b.returnStatement(), + ), + ); +} + export function generateFilterScriptExpression( branch, iterator, @@ -417,18 +437,6 @@ function processAtIdentifier(tree, name) { [sandbox.value], ); default: - if (name.startsWith('@@')) { - const shorthandName = name.slice(2); - tree.attachCustomShorthand(shorthandName); - return b.callExpression( - b.memberExpression( - internalScope.shorthands, - b.identifier(shorthandName), - ), - [scope._], - ); - } - throw Error(`Unsupported shorthand "${name}"`); } } diff --git a/src/codegen/baseline/index.mjs b/src/codegen/baseline/index.mjs index 43f941f..8d53e11 100644 --- a/src/codegen/baseline/index.mjs +++ b/src/codegen/baseline/index.mjs @@ -4,9 +4,15 @@ import { isDeep } from '../guards.mjs'; import Iterator from '../iterator.mjs'; import generateEmitCall from '../templates/emit-call.mjs'; import scope from '../templates/scope.mjs'; +import { + NEEDS_SHORTHANDS, + NEEDS_STATE, + NEEDS_TRAVERSAL, +} from '../tree/consts.mjs'; import ESTree from '../tree/tree.mjs'; import JsonPathHashes from '../utils/jsonpath-hashes.mjs'; import { + generateCustomShorthandExpression, generateFilterScriptExpression, generateMemberExpression, generateMultipleMemberExpression, @@ -14,10 +20,9 @@ import { generateWildcardExpression, } from './generators.mjs'; -export default function baseline(jsonPaths, opts) { +export default function baseline(jsonPaths) { const hashes = new JsonPathHashes(); const tree = new ESTree({ - customShorthands: opts.customShorthands, hashes, }); @@ -103,16 +108,23 @@ export default function baseline(jsonPaths, opts) { generateWildcardExpression(branch, iterator, node, tree); zone?.resize(); break; + case 'CustomShorthandExpression': + generateCustomShorthandExpression(branch, iterator, node, tree); + break; } } branch.push(generateEmitCall(ctx.id, iterator.modifiers)); + let feedback = NEEDS_TRAVERSAL; if (iterator.feedback.stateOffset !== -1) { - tree.addTreeMethod(ctx.id, b.blockStatement(branch), 'stateful-traverse'); - } else { - tree.addTreeMethod(ctx.id, b.blockStatement(branch), 'traverse'); + feedback |= NEEDS_STATE; } + if (iterator.feedback.shorthands > 0) { + feedback |= NEEDS_SHORTHANDS; + } + + tree.addTreeMethod(ctx.id, b.blockStatement(branch), feedback); } return tree; diff --git a/src/codegen/fast-paths/all-parents.mjs b/src/codegen/fast-paths/all-parents.mjs index 63d6dca..3346ac2 100644 --- a/src/codegen/fast-paths/all-parents.mjs +++ b/src/codegen/fast-paths/all-parents.mjs @@ -4,6 +4,7 @@ import * as b from '../ast/builders.mjs'; import generateEmitCall from '../templates/emit-call.mjs'; import sandbox from '../templates/sandbox.mjs'; +import { NEEDS_TRAVERSAL } from '../tree/consts.mjs'; const IS_OBJECT_IDENTIFIER = b.identifier('isObject'); const IS_NOT_OBJECT_IF_STATEMENT = b.ifStatement( @@ -32,7 +33,7 @@ export default (nodes, tree, ctx) => { IS_NOT_OBJECT_IF_STATEMENT, generateEmitCall(ctx.id, ctx.iterator.modifiers), ]), - 'traverse', + NEEDS_TRAVERSAL, ); tree.body.push(EMIT_ROOT_CALL_EXPRESSION); diff --git a/src/codegen/fast-paths/fixed.mjs b/src/codegen/fast-paths/fixed.mjs index 3f5c4f7..12dab07 100644 --- a/src/codegen/fast-paths/fixed.mjs +++ b/src/codegen/fast-paths/fixed.mjs @@ -1,16 +1,3 @@ -// Examples -// $.info -// $.info.foo -// $.foo.bar.baz -/** - * function (scope, fn) { - * const value = scope.sandbox.root?.info; - * if (isObject(value)) { - * fn(scope.fork(['info', 'foo']).emit()); - * } - * } - */ - import * as b from '../ast/builders.mjs'; import { isDeep, isMemberExpression } from '../guards.mjs'; import generateEmitCall from '../templates/emit-call.mjs'; @@ -74,7 +61,7 @@ export default (nodes, tree, ctx) => { IS_NULL_SCOPE_IF_STATEMENT, generateEmitCall(ctx.id, ctx.iterator.modifiers), ]), - 'body', + 0, ); return true; diff --git a/src/codegen/guards.mjs b/src/codegen/guards.mjs index a64b27d..704e667 100644 --- a/src/codegen/guards.mjs +++ b/src/codegen/guards.mjs @@ -6,6 +6,10 @@ export function isScriptFilterExpression(node) { return node.type === 'ScriptFilterExpression'; } +export function isShorthandExpression(node) { + return node.type === 'CustomShorthandExpression'; +} + export function isNegativeSliceExpression(node) { return node.type === 'SliceExpression' && node.value.some(isNegativeNumber); } diff --git a/src/codegen/iterator.mjs b/src/codegen/iterator.mjs index 4a1c5fa..c369cbe 100644 --- a/src/codegen/iterator.mjs +++ b/src/codegen/iterator.mjs @@ -3,6 +3,7 @@ import { isModifierExpression, isNegativeSliceExpression, isScriptFilterExpression, + isShorthandExpression, isWildcardExpression, } from './guards.mjs'; @@ -84,6 +85,7 @@ export default class Iterator { fixed: true, inverseOffset: -1, minimumDepth: -1, + shorthands: 0, stateOffset: -1, }; @@ -93,6 +95,16 @@ export default class Iterator { for (let i = 0; i < nodes.length; i++) { const node = nodes[i]; + if (isShorthandExpression(node)) { + if (node.arguments[0] > 0) { + feedback.stateOffset = i; + } + + feedback.minimumDepth = i - 1; + feedback.shorthands++; + feedback.fixed = false; + } + if (!isDeep(node)) { if (isScriptFilterExpression(node) || isNegativeSliceExpression(node)) { if (i === nodes.length - 1) { @@ -123,9 +135,11 @@ export default class Iterator { deep = i; } - feedback.fixed = deep === -1; - feedback.minimumDepth = - feedback.stateOffset === -1 ? nodes.length - 1 : feedback.stateOffset; + if (feedback.shorthands === 0) { + feedback.fixed = deep === -1; + feedback.minimumDepth = + feedback.stateOffset === -1 ? nodes.length - 1 : feedback.stateOffset; + } return feedback; } @@ -135,7 +149,6 @@ export default class Iterator { Object.assign(state, emptyState()); - let statePos = 0; for (let i = 0; i < nodes.length; i++) { state.absoluteOffset = i; @@ -160,21 +173,27 @@ export default class Iterator { } if (state.usesState) { - if (statePos === 0) { + if (state.numbers[0] === -1) { state.numbers[0] = 0; state.numbers[1] = 1; } else { state.numbers[0] = state.numbers[1]; - state.numbers[1] = state.numbers[1] + 2 ** statePos; + state.numbers[1] = (state.numbers[1] << 1) + 1; } state.groupNumbers.push(state.numbers[0]); - statePos++; } state.isLastNode = i === nodes.length - 1; yield nodes[i]; + + if (isShorthandExpression(nodes[i])) { + let depth = nodes[i].arguments[0]; + while (depth-- > 0) { + state.numbers[1] = (state.numbers[1] << 1) + 1; + } + } } } } diff --git a/src/codegen/templates/fn-params.mjs b/src/codegen/templates/fn-params.mjs deleted file mode 100644 index ecf3e0c..0000000 --- a/src/codegen/templates/fn-params.mjs +++ /dev/null @@ -1,5 +0,0 @@ -import scope from './scope.mjs'; -import state from './state.mjs'; - -export const statelessFnParams = [scope._]; -export const statefulFnParams = [scope._, state._]; diff --git a/src/codegen/templates/scope.mjs b/src/codegen/templates/scope.mjs index dfb58b1..74ce597 100644 --- a/src/codegen/templates/scope.mjs +++ b/src/codegen/templates/scope.mjs @@ -21,6 +21,7 @@ export default { true, ), sandbox: b.memberExpression(SCOPE_IDENTIFIER, b.identifier('sandbox')), + shorthands: b.memberExpression(SCOPE_IDENTIFIER, b.identifier('shorthands')), traverse: b.memberExpression(SCOPE_IDENTIFIER, b.identifier('traverse')), value: b.memberExpression(SCOPE_IDENTIFIER, b.identifier('value')), }; diff --git a/src/codegen/templates/tree-method-call.mjs b/src/codegen/templates/tree-method-call.mjs index 9d1be14..c4993a5 100644 --- a/src/codegen/templates/tree-method-call.mjs +++ b/src/codegen/templates/tree-method-call.mjs @@ -1,22 +1,16 @@ import * as b from '../ast/builders.mjs'; -import { statelessFnParams } from './fn-params.mjs'; import internalScope from './internal-scope.mjs'; import scope from './scope.mjs'; -export function generateTreeMethodCall(id) { - return b.expressionStatement( - b.callExpression( - b.memberExpression(internalScope.tree, id, true), - statelessFnParams, - ), - ); -} +export function generateTreeMethodCall(id, state, needsShorthand) { + const params = + state === null ? [scope._] : [scope._, state.declarations[0].id]; + + if (needsShorthand) { + params.push(internalScope.shorthands); + } -export function generateStatefulTreeMethodCall(id, state) { return b.expressionStatement( - b.callExpression(b.memberExpression(internalScope.tree, id, true), [ - scope._, - state.declarations[0].id, - ]), + b.callExpression(b.memberExpression(internalScope.tree, id, true), params), ); } diff --git a/src/codegen/templates/tree-method.mjs b/src/codegen/templates/tree-method.mjs index 0e40bf0..dda52c4 100644 --- a/src/codegen/templates/tree-method.mjs +++ b/src/codegen/templates/tree-method.mjs @@ -1,11 +1,25 @@ import * as b from '../ast/builders.mjs'; -import { statefulFnParams, statelessFnParams } from './fn-params.mjs'; +import { + NEEDS_SHORTHANDS, + NEEDS_STATE, + NEEDS_TRAVERSAL, +} from '../tree/consts.mjs'; +import internalScope from './internal-scope.mjs'; +import scope from './scope.mjs'; +import state from './state.mjs'; -export default function generateTreeMethod(id, branch, needsState) { - return b.objectMethod( - 'method', - id, - needsState ? statefulFnParams : statelessFnParams, - branch, - ); +const PARAMS = { + [0]: [scope._], + [NEEDS_TRAVERSAL]: [scope._], + [NEEDS_TRAVERSAL | NEEDS_STATE]: [scope._, state._], + [NEEDS_TRAVERSAL | NEEDS_SHORTHANDS]: [scope._, internalScope.shorthands], + [NEEDS_TRAVERSAL | NEEDS_STATE | NEEDS_SHORTHANDS]: [ + scope._, + state._, + internalScope.shorthands, + ], +}; + +export default function generateTreeMethod(id, branch, feedback) { + return b.objectMethod('method', id, PARAMS[feedback], branch); } diff --git a/src/codegen/tree/consts.mjs b/src/codegen/tree/consts.mjs new file mode 100644 index 0000000..6742456 --- /dev/null +++ b/src/codegen/tree/consts.mjs @@ -0,0 +1,3 @@ +export const NEEDS_TRAVERSAL = 1; +export const NEEDS_STATE = 2; +export const NEEDS_SHORTHANDS = 4; diff --git a/src/codegen/tree/tree.mjs b/src/codegen/tree/tree.mjs index b0f0254..42ceece 100644 --- a/src/codegen/tree/tree.mjs +++ b/src/codegen/tree/tree.mjs @@ -1,121 +1,86 @@ -import jsep from '../../parser/jsep.mjs'; import * as b from '../ast/builders.mjs'; import astring from '../dump.mjs'; import generateAllocState from '../templates/alloc-state.mjs'; -import { statefulFnParams } from '../templates/fn-params.mjs'; import internalScope from '../templates/internal-scope.mjs'; import scope from '../templates/scope.mjs'; import generateTreeMethod from '../templates/tree-method.mjs'; -import { - generateStatefulTreeMethodCall, - generateTreeMethodCall, -} from '../templates/tree-method-call.mjs'; +import { generateTreeMethodCall } from '../templates/tree-method-call.mjs'; +import { NEEDS_SHORTHANDS, NEEDS_STATE, NEEDS_TRAVERSAL } from './consts.mjs'; import commonjs from './modules/commonjs.mjs'; import esm from './modules/esm.mjs'; import TraversalZones from './traversal-zones.mjs'; -const params = [b.identifier('input'), b.identifier('callbacks')]; +const DEFAULT_PARAMS = [b.identifier('input'), b.identifier('callbacks')]; const NEW_SCOPE_VARIABLE_DECLARATION = b.variableDeclaration('const', [ - b.variableDeclarator(scope._, b.newExpression(b.identifier('Scope'), params)), + b.variableDeclarator( + scope._, + b.newExpression(b.identifier('Scope'), DEFAULT_PARAMS), + ), ]); -/* -import { - // deps -} from 'nimma/runtime'; -// placement: tree -const tree = {}; - -// placement: program - -export default function (input, callbacks) { - const scope = new Scope(input, callbacks); - - try { - // placement: body - - scope.traverse(() => { - // placement: traverse - }); - } finally { - scope.destroy(); - } -} -*/ - export default class ESTree { - #hashes; #tree = b.objectExpression([]); - #shorthands = b.objectExpression([]); + #hasShorthands = false; #runtimeDependencies; #traverse = []; - #availableShorthands; #states = -1; - constructor({ hashes, customShorthands }) { - this.#hashes = hashes; - this.cacheInfo = {}; + constructor() { this.body = []; this.traversalZones = new TraversalZones(); - this.#availableShorthands = customShorthands; this.#runtimeDependencies = new Map([['Scope', 'Scope']]); } + /** + * @param {string} specifier + */ addRuntimeDependency(specifier) { this.#runtimeDependencies.set(specifier, specifier); } - attachCustomShorthand(name) { - if ( - this.#availableShorthands === null || - !(name in this.#availableShorthands) - ) { - throw new ReferenceError(`Shorthand '${name}' is not defined`); - } - - this.#shorthands.properties.push( - b.objectMethod( - 'method', - b.identifier(name), - statefulFnParams, - b.blockStatement([ - b.returnStatement(jsep.parse(this.#availableShorthands[name])), - ]), - ), - ); - } - + /** + * @param hash + * @returns {*} + */ getMethodByHash(hash) { return this.#tree.properties.find(prop => prop.key.value === hash); } - addTreeMethod(id, block, scope) { - this.cacheInfo[id.value] = { - hash: this.#hashes.getHash(id.value), - scope, - }; - - if (scope === 'stateful-traverse') { - const state = generateAllocState(++this.#states); + /** + * @param {Object} id + * @param {'StringLiteral'} id.type + * @param {string} id.value + * @param {Object} block + * @param {number} feedback + */ + addTreeMethod(id, block, feedback) { + let state; + if ((feedback & NEEDS_STATE) > 0) { + state = generateAllocState(++this.#states); this.body.push(state); - this.#tree.properties.push(generateTreeMethod(id, block, true)); - this.#traverse.push(generateStatefulTreeMethodCall(id, state)); - return; + } else { + state = null; } - this.#tree.properties.push(generateTreeMethod(id, block, false)); + const needsShorthands = (feedback & NEEDS_SHORTHANDS) > 0; + this.#hasShorthands ||= needsShorthands; - const call = generateTreeMethodCall(id); - if (scope === 'traverse') { - this.#traverse.push(call); - } else { - this.body.push(call); - } + this.#tree.properties.push(generateTreeMethod(id, block, feedback)); + const call = generateTreeMethodCall(id, state, needsShorthands); + ((feedback & NEEDS_TRAVERSAL) > 0 ? this.#traverse : this.body).push(call); } + /** + * Generates JS code based on the underlying ESTree + * @param {'esm'|'commonjs'} format + * @returns {string} + */ export(format) { const traversalZones = this.traversalZones.build(); + const params = this.#hasShorthands + ? [...DEFAULT_PARAMS, internalScope.shorthands] + : DEFAULT_PARAMS; const program = b.program( [ @@ -125,43 +90,36 @@ export default class ESTree { : b.variableDeclaration('const', [ b.variableDeclarator(internalScope.tree, this.#tree), ]), - this.#shorthands.properties.length === 0 - ? null - : b.variableDeclaration('const', [ - b.variableDeclarator(internalScope.shorthands, this.#shorthands), - ]), b.functionDeclaration( null, params, - b.blockStatement( - [ - NEW_SCOPE_VARIABLE_DECLARATION, - b.tryStatement( - b.blockStatement( - [ - ...this.body, - this.#traverse.length === 0 - ? null - : b.expressionStatement( - b.callExpression(scope.traverse, [ - b.arrowFunctionExpression( - [], - b.blockStatement(Array.from(this.#traverse)), - ), - traversalZones === null - ? b.nullLiteral() - : traversalZones.declarations[0].id, - ]), - ), - ].filter(Boolean), - ), - null, - b.blockStatement([ - b.expressionStatement(b.callExpression(scope.destroy, [])), - ]), + b.blockStatement([ + NEW_SCOPE_VARIABLE_DECLARATION, + b.tryStatement( + b.blockStatement( + [ + ...this.body, + this.#traverse.length === 0 + ? null + : b.expressionStatement( + b.callExpression(scope.traverse, [ + b.arrowFunctionExpression( + [], + b.blockStatement(Array.from(this.#traverse)), + ), + traversalZones === null + ? b.nullLiteral() + : traversalZones.declarations[0].id, + ]), + ), + ].filter(Boolean), ), - ].filter(Boolean), - ), + null, + b.blockStatement([ + b.expressionStatement(b.callExpression(scope.destroy, [])), + ]), + ), + ]), ), ].filter(Boolean), ); diff --git a/src/codegen/utils/jsonpath-hashes.mjs b/src/codegen/utils/jsonpath-hashes.mjs index 9fd822e..da6e469 100644 --- a/src/codegen/utils/jsonpath-hashes.mjs +++ b/src/codegen/utils/jsonpath-hashes.mjs @@ -6,10 +6,6 @@ export default class JsonPathHashes { return this.#hashes.get(key); } - getHash(expression) { - return this.#expressions.get(expression); - } - set(key, value) { this.#hashes.set(key, value); this.#expressions.set(value, key); diff --git a/src/core/index.mjs b/src/core/index.mjs index 177af8a..eda6ad5 100644 --- a/src/core/index.mjs +++ b/src/core/index.mjs @@ -6,11 +6,13 @@ export default class Nimma { #compiledFn; #module; #sourceCode; + #customShorthands; constructor(expressions, { module = 'esm', customShorthands = null } = {}) { this.#compiledFn = null; this.#module = module; this.#sourceCode = null; + this.#customShorthands = customShorthands; this.tree = codegen(parseExpressions(expressions), { customShorthands, @@ -29,7 +31,11 @@ export default class Nimma { `${String(this.tree.export('commonjs'))};return module.exports`, )({}, () => runtime); - this.#compiledFn(input, callbacks); + if (this.#customShorthands === null) { + this.#compiledFn(input, callbacks); + } else { + this.#compiledFn(input, callbacks, this.#customShorthands); + } } static query(input, callbacks, options) { diff --git a/src/index.d.ts b/src/index.d.ts index f9baee3..7a3478f 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -7,8 +7,16 @@ export type EmittedScope = { readonly value: unknown; }; +export type State = { + value: number; + initialValue: number; +}; + export type Options = { - customShorthands?: Record | null; + customShorthands?: Record< + string, + (path: JsonPath, state?: State, initialValue?: number) => void + >; module?: 'esm' | 'commonjs'; }; diff --git a/src/parser/__tests__/parser.test.mjs b/src/parser/__tests__/parser.test.mjs index f411cae..ba799b1 100644 --- a/src/parser/__tests__/parser.test.mjs +++ b/src/parser/__tests__/parser.test.mjs @@ -510,8 +510,8 @@ describe('Parser', () => { }); } - it('parses @@', () => { - assert.deepEqual(parse('$.components.schemas..@@schema()'), [ + it.only('parses shorthand expressions', () => { + assert.deepEqual(parse('$.components.schemas..@@schema(0)'), [ { type: 'MemberExpression', value: 'components', @@ -523,21 +523,14 @@ describe('Parser', () => { deep: false, }, { - type: 'ScriptFilterExpression', - raw: '@@schema()', - value: { - type: 'CallExpression', - arguments: [], - callee: { - name: '@@schema', - type: 'Identifier', - }, - }, + type: 'CustomShorthandExpression', + value: 'schema', + arguments: [0], deep: true, }, ]); - assert.deepEqual(parse('$.components.schemas.@@schema()'), [ + assert.deepEqual(parse('$.components.schemas.@@schema(2)'), [ { type: 'MemberExpression', value: 'components', @@ -549,16 +542,9 @@ describe('Parser', () => { deep: false, }, { - type: 'ScriptFilterExpression', - raw: '@@schema()', - value: { - type: 'CallExpression', - arguments: [], - callee: { - name: '@@schema', - type: 'Identifier', - }, - }, + type: 'CustomShorthandExpression', + value: 'schema', + arguments: [2], deep: false, }, ]); @@ -574,7 +560,7 @@ describe('Parser', () => { } it('skips whitespaces', () => { - assert.deepEqual(parse('$.[ name ] [?( @.abc )]\t ..@@test( )'), [ + assert.deepEqual(parse('$.[ name ] [?( @.abc )]\t ..@@test ( 5 )'), [ { type: 'MemberExpression', value: 'name', @@ -598,16 +584,9 @@ describe('Parser', () => { deep: false, }, { - type: 'ScriptFilterExpression', - raw: '@@test( )', - value: { - type: 'CallExpression', - arguments: [], - callee: { - type: 'Identifier', - name: '@@test', - }, - }, + type: 'CustomShorthandExpression', + value: 'test', + arguments: [5], deep: true, }, ]); @@ -719,17 +698,37 @@ describe('Parser', () => { }); it('invalid shorthands', () => { + assert.throws( + () => parse('$..@()'), + SyntaxError('Expected [a-z] but "(" found at 4.'), + ); + assert.throws( + () => parse('$..@1()'), + SyntaxError('Expected [a-z] but "1" found at 4.'), + ); assert.throws( () => parse('$..@@()'), SyntaxError('Expected [a-z] but "(" found at 5.'), ); + assert.throws( + () => parse('$..@@1()'), + SyntaxError('Expected [a-z] but "1" found at 5.'), + ); assert.throws( () => parse('$..@@test)'), SyntaxError('Expected "(" but ")" found at 9.'), ); assert.throws( () => parse('$..@@test('), - SyntaxError('Expected ")" but end of input found at 10.'), + SyntaxError('Expected [0-9] but end of input found at 10.'), + ); + assert.throws( + () => parse('$..@@test(5'), + SyntaxError('Expected ")" but end of input found at 11.'), + ); + assert.throws( + () => parse('$..@@test()'), + SyntaxError('Expected [0-9] but ")" found at 10.'), ); assert.throws( () => parse('$..@'), diff --git a/src/parser/parser.mjs b/src/parser/parser.mjs index 2b6a2d2..d61dfea 100644 --- a/src/parser/parser.mjs +++ b/src/parser/parser.mjs @@ -30,6 +30,9 @@ import { * @typedef {Object} ScriptFilterExpression * @property {string} raw - The raw expression. * @property {*} value - The parsed expression. + * + * @typedef {Object} CustomShorthandExpression + * @property {string} name - The shorthand name. */ /* eslint-disable sort-keys */ @@ -83,7 +86,7 @@ function parseNode(ctx, nodes, jsep) { case expr.charCodeAt(i) === 0x2e /* "." */: ctx.i++; if (expr.charCodeAt(ctx.i) === 0x40 /* "@" */) { - nodes.push(parseCustomShorthand(ctx)); + nodes.push(parseShorthand(ctx)); } else { nodes.push(parseNamed(ctx)); } @@ -120,7 +123,7 @@ function parseNode(ctx, nodes, jsep) { break; case expr.charCodeAt(i) === 0x40 /* "@" */: - nodes.push(parseCustomShorthand(ctx)); + nodes.push(parseShorthand(ctx)); break; default: throw SyntaxError( @@ -135,7 +138,7 @@ function parseDeepNode(ctx, jsep) { ctx.i++; return parseBracket(ctx, jsep); } else if (expr.charCodeAt(ctx.i) === 0x40 /* "@" */) { - return parseCustomShorthand(ctx); + return parseShorthand(ctx); } else if (ctx.i === expr.length) { return { type: 'AllParentExpression' }; } else if ( @@ -384,16 +387,16 @@ function parseMember(ctx) { return hasOnlyDigits ? Number.parseInt(member, 10) : member; } -function parseCustomShorthand(ctx) { +function parseShorthand(ctx) { const { expr } = ctx; let { i } = ctx; - const start = i; + let start = i; ctx.i = ++i; if (i < expr.length && expr.charCodeAt(i) === 0x40 /* "@" */) { ctx.i = ++i; - assertNotEndOfInput(ctx, '[a-z]'); + return parseCustomShorthand(ctx); } ctx.i = i; @@ -405,8 +408,9 @@ function parseCustomShorthand(ctx) { while (++i < expr.length && isChar(expr.charCodeAt(i))); - ctx.i = i; const name = expr.slice(start, i); + ctx.i = i; + skipWhitespace(ctx); eat(ctx, 0x28 /* "(" */); skipWhitespace(ctx); eat(ctx, 0x29 /* ")" */); @@ -425,3 +429,49 @@ function parseCustomShorthand(ctx) { deep: false, }; } + +function parseCustomShorthand(ctx) { + assertNotEndOfInput(ctx, '[a-z]'); + + const { expr } = ctx; + let { i } = ctx; + let start = i; + + if (!isChar(expr.charCodeAt(i))) { + throw SyntaxError(`Expected [a-z] but "${expr[i]}" found at ${i}.`); + } + + while (++i < expr.length) { + const code = expr.charCodeAt(i); + if (!isDigit(code) && !isChar(code)) { + break; + } + } + + const name = expr.slice(start, i); + ctx.i = i; + skipWhitespace(ctx); + eat(ctx, 0x28 /* "(" */); + skipWhitespace(ctx); + assertNotEndOfInput(ctx, '[0-9]'); + + i = ctx.i; + start = i; + while (isDigit(expr.charCodeAt(i))) i++; + if (start === i) { + throw SyntaxError(`Expected [0-9] but "${expr[i]}" found at ${i}.`); + } + + const depth = Number.parseInt(expr.slice(start, i), 10); + ctx.i = i; + + skipWhitespace(ctx); + eat(ctx, 0x29 /* ")" */); + + return { + type: 'CustomShorthandExpression', + value: name, + arguments: [depth], + deep: false, + }; +}