From caf8c3ae2012bd6f2c29a19b99757a98b46fa1d6 Mon Sep 17 00:00:00 2001 From: Hans Delano <43413899+hanscau@users.noreply.github.com> Date: Sun, 24 Mar 2024 18:10:06 +0800 Subject: [PATCH] Refactor validator (#1583) * moved validation function to validator * update validator imports * linting * format * re-format with new version * fixing merge conflict fixes * linting --------- Co-authored-by: Martin Henz --- src/cse-machine/interpreter.ts | 2 +- src/stepper/stepper.ts | 2 +- src/transpiler/transpiler.ts | 242 ++------------------------------- src/utils/uniqueIds.ts | 25 ++++ src/validator/validator.ts | 126 +++++++++++++++++ 5 files changed, 161 insertions(+), 236 deletions(-) diff --git a/src/cse-machine/interpreter.ts b/src/cse-machine/interpreter.ts index 253a3ec6b..bbcbb83b7 100644 --- a/src/cse-machine/interpreter.ts +++ b/src/cse-machine/interpreter.ts @@ -18,13 +18,13 @@ import { UndefinedImportError } from '../modules/errors' import { initModuleContext, loadModuleBundle } from '../modules/moduleLoader' import { ImportTransformOptions } from '../modules/moduleTypes' import { checkEditorBreakpoints } from '../stdlib/inspector' -import { checkProgramForUndefinedVariables } from '../transpiler/transpiler' import { Context, ContiguousArrayElements, Result, StatementSequence, Value } from '../types' import assert from '../utils/assert' import { filterImportDeclarations } from '../utils/ast/helpers' import * as ast from '../utils/astCreator' import { evaluateBinaryExpression, evaluateUnaryExpression } from '../utils/operators' import * as rttc from '../utils/rttc' +import { checkProgramForUndefinedVariables } from '../validator/validator' import * as seq from '../utils/statementSeqTransform' import { Continuation, diff --git a/src/stepper/stepper.ts b/src/stepper/stepper.ts index b9fb93add..5ba6310b6 100644 --- a/src/stepper/stepper.ts +++ b/src/stepper/stepper.ts @@ -7,7 +7,6 @@ import { UndefinedImportError } from '../modules/errors' import { initModuleContextAsync, loadModuleBundleAsync } from '../modules/moduleLoaderAsync' import type { ImportTransformOptions } from '../modules/moduleTypes' import { parse } from '../parser/parser' -import { checkProgramForUndefinedVariables } from '../transpiler/transpiler' import { BlockExpression, Context, @@ -29,6 +28,7 @@ import { } from '../utils/dummyAstCreator' import { evaluateBinaryExpression, evaluateUnaryExpression } from '../utils/operators' import * as rttc from '../utils/rttc' +import { checkProgramForUndefinedVariables } from '../validator/validator' import { nodeToValue, objectToString, valueToExpression } from './converter' import * as builtin from './lib' import { diff --git a/src/transpiler/transpiler.ts b/src/transpiler/transpiler.ts index bce1d655f..9dd6aee14 100644 --- a/src/transpiler/transpiler.ts +++ b/src/transpiler/transpiler.ts @@ -5,7 +5,6 @@ import { partition } from 'lodash' import { RawSourceMap, SourceMapGenerator } from 'source-map' import { NATIVE_STORAGE_ID, REQUIRE_PROVIDER_ID, UNKNOWN_LOCATION } from '../constants' -import { UndefinedVariable } from '../errors/errors' import { ModuleNotFoundError } from '../errors/moduleErrors' import { ModuleInternalError, UndefinedImportError } from '../modules/errors' import { @@ -15,7 +14,6 @@ import { memoizedGetModuleManifestAsync } from '../modules/moduleLoaderAsync' import type { ImportTransformOptions } from '../modules/moduleTypes' -import { parse } from '../parser/parser' import { AllowedDeclarations, Chapter, @@ -23,7 +21,6 @@ import { NativeStorage, Node, RecursivePartial, - StatementSequence, Variant } from '../types' import assert from '../utils/assert' @@ -33,9 +30,12 @@ import { getFunctionDeclarationNamesInProgram, getIdentifiersInNativeStorage, getIdentifiersInProgram, - getUniqueId + getNativeIds, + getUniqueId, + NativeIds } from '../utils/uniqueIds' -import { ancestor, simple } from '../utils/walkers' +import { simple } from '../utils/walkers' +import { checkForUndefinedVariables } from '../validator/validator' /** * This whole transpiler includes many many many many hacks to get stuff working. @@ -43,22 +43,6 @@ import { ancestor, simple } from '../utils/walkers' * There should be an explanation on it coming up soon. */ -const globalIdNames = [ - 'native', - 'callIfFuncAndRightArgs', - 'boolOrErr', - 'wrap', - 'wrapSourceModule', - 'unaryOp', - 'binaryOp', - 'throwIfTimeout', - 'setProp', - 'getProp', - 'builtins' -] as const - -export type NativeIds = Record<(typeof globalIdNames)[number], es.Identifier> - export async function transformImportDeclarations( program: es.Program, usedIdentifiers: Set, @@ -361,209 +345,6 @@ function transformCallExpressionsToCheckIfFunction(program: es.Program, globalId }) } -export function checkForUndefinedVariables( - program: es.Program, - nativeStorage: NativeStorage, - globalIds: NativeIds, - skipUndefined: boolean -) { - const builtins = nativeStorage.builtins - const identifiersIntroducedByNode = new Map>() - function processBlock(node: es.Program | es.BlockStatement) { - const identifiers = new Set() - for (const statement of node.body) { - if (statement.type === 'VariableDeclaration') { - identifiers.add((statement.declarations[0].id as es.Identifier).name) - } else if (statement.type === 'FunctionDeclaration') { - if (statement.id === null) { - throw new Error( - 'Encountered a FunctionDeclaration node without an identifier. This should have been caught when parsing.' - ) - } - identifiers.add(statement.id.name) - } else if (statement.type === 'ImportDeclaration') { - for (const specifier of statement.specifiers) { - identifiers.add(specifier.local.name) - } - } - } - identifiersIntroducedByNode.set(node, identifiers) - } - function processFunction( - node: es.FunctionDeclaration | es.ArrowFunctionExpression, - _ancestors: Node[] - ) { - identifiersIntroducedByNode.set( - node, - new Set( - node.params.map(id => - id.type === 'Identifier' - ? id.name - : ((id as es.RestElement).argument as es.Identifier).name - ) - ) - ) - } - const identifiersToAncestors = new Map() - ancestor(program, { - Program: processBlock, - BlockStatement: processBlock, - FunctionDeclaration: processFunction, - ArrowFunctionExpression: processFunction, - ForStatement(forStatement: es.ForStatement, ancestors: Node[]) { - const init = forStatement.init! - if (init.type === 'VariableDeclaration') { - identifiersIntroducedByNode.set( - forStatement, - new Set([(init.declarations[0].id as es.Identifier).name]) - ) - } - }, - Identifier(identifier: es.Identifier, ancestors: Node[]) { - identifiersToAncestors.set(identifier, [...ancestors]) - }, - Pattern(node: es.Pattern, ancestors: Node[]) { - if (node.type === 'Identifier') { - identifiersToAncestors.set(node, [...ancestors]) - } else if (node.type === 'MemberExpression') { - if (node.object.type === 'Identifier') { - identifiersToAncestors.set(node.object, [...ancestors]) - } - } - } - }) - const nativeInternalNames = new Set(Object.values(globalIds).map(({ name }) => name)) - - for (const [identifier, ancestors] of identifiersToAncestors) { - const name = identifier.name - const isCurrentlyDeclared = ancestors.some(a => identifiersIntroducedByNode.get(a)?.has(name)) - if (isCurrentlyDeclared) { - continue - } - const isPreviouslyDeclared = nativeStorage.previousProgramsIdentifiers.has(name) - if (isPreviouslyDeclared) { - continue - } - const isBuiltin = builtins.has(name) - if (isBuiltin) { - continue - } - const isNativeId = nativeInternalNames.has(name) - if (!isNativeId && !skipUndefined) { - throw new UndefinedVariable(name, identifier) - } - } -} - -export function checkProgramForUndefinedVariables(program: es.Program, context: Context) { - const usedIdentifiers = new Set([ - ...getIdentifiersInProgram(program), - ...getIdentifiersInNativeStorage(context.nativeStorage) - ]) - const globalIds = getNativeIds(program, usedIdentifiers) - - const preludes = context.prelude - ? getFunctionDeclarationNamesInProgram(parse(context.prelude, context)!) - : new Set() - - const builtins = context.nativeStorage.builtins - const env = context.runtime.environments[0].head - - const identifiersIntroducedByNode = new Map>() - function processBlock(node: es.Program | es.BlockStatement | StatementSequence) { - const identifiers = new Set() - for (const statement of node.body) { - if (statement.type === 'VariableDeclaration') { - identifiers.add((statement.declarations[0].id as es.Identifier).name) - } else if (statement.type === 'FunctionDeclaration') { - if (statement.id === null) { - throw new Error( - 'Encountered a FunctionDeclaration node without an identifier. This should have been caught when parsing.' - ) - } - identifiers.add(statement.id.name) - } else if (statement.type === 'ImportDeclaration') { - for (const specifier of statement.specifiers) { - identifiers.add(specifier.local.name) - } - } - } - identifiersIntroducedByNode.set(node, identifiers) - } - function processFunction( - node: es.FunctionDeclaration | es.ArrowFunctionExpression, - _ancestors: Node[] - ) { - identifiersIntroducedByNode.set( - node, - new Set( - node.params.map(id => - id.type === 'Identifier' - ? id.name - : ((id as es.RestElement).argument as es.Identifier).name - ) - ) - ) - } - const identifiersToAncestors = new Map() - ancestor(program, { - Program: processBlock, - BlockStatement: processBlock, - StatementSequence: processBlock, - FunctionDeclaration: processFunction, - ArrowFunctionExpression: processFunction, - ForStatement(forStatement: es.ForStatement, ancestors: Node[]) { - const init = forStatement.init! - if (init.type === 'VariableDeclaration') { - identifiersIntroducedByNode.set( - forStatement, - new Set([(init.declarations[0].id as es.Identifier).name]) - ) - } - }, - Identifier(identifier: es.Identifier, ancestors: Node[]) { - identifiersToAncestors.set(identifier, [...ancestors]) - }, - Pattern(node: es.Pattern, ancestors: Node[]) { - if (node.type === 'Identifier') { - identifiersToAncestors.set(node, [...ancestors]) - } else if (node.type === 'MemberExpression') { - if (node.object.type === 'Identifier') { - identifiersToAncestors.set(node.object, [...ancestors]) - } - } - } - }) - const nativeInternalNames = new Set(Object.values(globalIds).map(({ name }) => name)) - - for (const [identifier, ancestors] of identifiersToAncestors) { - const name = identifier.name - const isCurrentlyDeclared = ancestors.some(a => identifiersIntroducedByNode.get(a)?.has(name)) - if (isCurrentlyDeclared) { - continue - } - const isPreviouslyDeclared = context.nativeStorage.previousProgramsIdentifiers.has(name) - if (isPreviouslyDeclared) { - continue - } - const isBuiltin = builtins.has(name) - if (isBuiltin) { - continue - } - const isPrelude = preludes.has(name) - if (isPrelude) { - continue - } - const isInEnv = name in env - if (isInEnv) { - continue - } - const isNativeId = nativeInternalNames.has(name) - if (!isNativeId) { - throw new UndefinedVariable(name, identifier) - } - } -} function transformSomeExpressionsToCheckIfBoolean(program: es.Program, globalIds: NativeIds) { function transform( node: @@ -593,14 +374,6 @@ function transformSomeExpressionsToCheckIfBoolean(program: es.Program, globalIds }) } -function getNativeIds(program: es.Program, usedIdentifiers: Set): NativeIds { - const globalIds = {} - for (const identifier of globalIdNames) { - globalIds[identifier] = create.identifier(getUniqueId(usedIdentifiers, identifier)) - } - return globalIds as NativeIds -} - function transformUnaryAndBinaryOperationsToFunctionCalls( program: es.Program, globalIds: NativeIds, @@ -767,7 +540,8 @@ async function transpileToSource( transformSomeExpressionsToCheckIfBoolean(program, globalIds) transformPropertyAssignment(program, globalIds) transformPropertyAccess(program, globalIds) - checkForUndefinedVariables(program, context.nativeStorage, globalIds, skipUndefined) + checkForUndefinedVariables(program, context, globalIds, skipUndefined) + // checkProgramForUndefinedVariables(program, context, skipUndefined) transformFunctionDeclarationsToArrowFunctions(program, functionsToStringMap) wrapArrowFunctionsToAllowNormalCallsAndNiceToString(program, functionsToStringMap, globalIds) addInfiniteLoopProtection(program, globalIds, usedIdentifiers) @@ -815,7 +589,7 @@ async function transpileToFullJS( ]) const globalIds = getNativeIds(program, usedIdentifiers) - checkForUndefinedVariables(program, context.nativeStorage, globalIds, skipUndefined) + checkForUndefinedVariables(program, context, globalIds, skipUndefined) const [modulePrefix, importNodes, otherNodes] = await transformImportDeclarations( program, diff --git a/src/utils/uniqueIds.ts b/src/utils/uniqueIds.ts index 3f7352453..03988a480 100644 --- a/src/utils/uniqueIds.ts +++ b/src/utils/uniqueIds.ts @@ -1,8 +1,33 @@ import * as es from 'estree' import { NativeStorage } from '../types' +import * as create from '../utils/astCreator' import { simple } from '../utils/walkers' +const globalIdNames = [ + 'native', + 'callIfFuncAndRightArgs', + 'boolOrErr', + 'wrap', + 'wrapSourceModule', + 'unaryOp', + 'binaryOp', + 'throwIfTimeout', + 'setProp', + 'getProp', + 'builtins' +] as const + +export type NativeIds = Record<(typeof globalIdNames)[number], es.Identifier> + +export function getNativeIds(program: es.Program, usedIdentifiers: Set): NativeIds { + const globalIds = {} + for (const identifier of globalIdNames) { + globalIds[identifier] = create.identifier(getUniqueId(usedIdentifiers, identifier)) + } + return globalIds as NativeIds +} + export function getUniqueId(usedIdentifiers: Set, uniqueId = 'unique') { while (usedIdentifiers.has(uniqueId)) { const start = uniqueId.slice(0, -1) diff --git a/src/validator/validator.ts b/src/validator/validator.ts index e26e9ab75..97f6cb1b7 100644 --- a/src/validator/validator.ts +++ b/src/validator/validator.ts @@ -1,9 +1,18 @@ import * as es from 'estree' import { ConstAssignment } from '../errors/errors' +import { UndefinedVariable } from '../errors/errors' import { NoAssignmentToForVariable } from '../errors/validityErrors' +import { parse } from '../parser/parser' import { Context, Node, NodeWithInferredType } from '../types' import { getVariableDecarationName } from '../utils/astCreator' +import { + getFunctionDeclarationNamesInProgram, + getIdentifiersInNativeStorage, + getIdentifiersInProgram, + getNativeIds, + NativeIds +} from '../utils/uniqueIds' import { ancestor, base, FullWalkerCallback } from '../utils/walkers' class Declaration { @@ -144,3 +153,120 @@ export function validateAndAnnotate( */ return program } + +export function checkProgramForUndefinedVariables(program: es.Program, context: Context) { + const usedIdentifiers = new Set([ + ...getIdentifiersInProgram(program), + ...getIdentifiersInNativeStorage(context.nativeStorage) + ]) + const globalIds = getNativeIds(program, usedIdentifiers) + return checkForUndefinedVariables(program, context, globalIds, false) +} + +export function checkForUndefinedVariables( + program: es.Program, + context: Context, + globalIds: NativeIds, + skipUndefined: boolean +) { + const preludes = context.prelude + ? getFunctionDeclarationNamesInProgram(parse(context.prelude, context)!) + : new Set() + + const env = context.runtime.environments[0].head || {} + + const builtins = context.nativeStorage.builtins + const identifiersIntroducedByNode = new Map>() + function processBlock(node: es.Program | es.BlockStatement) { + const identifiers = new Set() + for (const statement of node.body) { + if (statement.type === 'VariableDeclaration') { + identifiers.add((statement.declarations[0].id as es.Identifier).name) + } else if (statement.type === 'FunctionDeclaration') { + if (statement.id === null) { + throw new Error( + 'Encountered a FunctionDeclaration node without an identifier. This should have been caught when parsing.' + ) + } + identifiers.add(statement.id.name) + } else if (statement.type === 'ImportDeclaration') { + for (const specifier of statement.specifiers) { + identifiers.add(specifier.local.name) + } + } + } + identifiersIntroducedByNode.set(node, identifiers) + } + function processFunction( + node: es.FunctionDeclaration | es.ArrowFunctionExpression, + _ancestors: es.Node[] + ) { + identifiersIntroducedByNode.set( + node, + new Set( + node.params.map(id => + id.type === 'Identifier' + ? id.name + : ((id as es.RestElement).argument as es.Identifier).name + ) + ) + ) + } + const identifiersToAncestors = new Map() + ancestor(program, { + Program: processBlock, + BlockStatement: processBlock, + FunctionDeclaration: processFunction, + ArrowFunctionExpression: processFunction, + ForStatement(forStatement: es.ForStatement, ancestors: es.Node[]) { + const init = forStatement.init! + if (init.type === 'VariableDeclaration') { + identifiersIntroducedByNode.set( + forStatement, + new Set([(init.declarations[0].id as es.Identifier).name]) + ) + } + }, + Identifier(identifier: es.Identifier, ancestors: es.Node[]) { + identifiersToAncestors.set(identifier, [...ancestors]) + }, + Pattern(node: es.Pattern, ancestors: es.Node[]) { + if (node.type === 'Identifier') { + identifiersToAncestors.set(node, [...ancestors]) + } else if (node.type === 'MemberExpression') { + if (node.object.type === 'Identifier') { + identifiersToAncestors.set(node.object, [...ancestors]) + } + } + } + }) + const nativeInternalNames = new Set(Object.values(globalIds).map(({ name }) => name)) + + for (const [identifier, ancestors] of identifiersToAncestors) { + const name = identifier.name + const isCurrentlyDeclared = ancestors.some(a => identifiersIntroducedByNode.get(a)?.has(name)) + if (isCurrentlyDeclared) { + continue + } + const isPreviouslyDeclared = context.nativeStorage.previousProgramsIdentifiers.has(name) + if (isPreviouslyDeclared) { + continue + } + const isBuiltin = builtins.has(name) + if (isBuiltin) { + continue + } + const isPrelude = preludes.has(name) + if (isPrelude) { + continue + } + const isInEnv = name in env + if (isInEnv) { + continue + } + const isNativeId = nativeInternalNames.has(name) + if (!isNativeId && !skipUndefined) { + throw new UndefinedVariable(name, identifier) + } + } +}