Added support for binary operators, unary operators and math library (#…

…18) * Changes that implement the treatment of ints as BigInts. TODO: explicit type conversions of BigInt back to Number for expressions and function calls. * Added support for binary operators, unary operators and math library * Prevent multiple assignments in the same function * Changes that implement the treatment of ints as BigInts. TODO: explicit type conversions of BigInt back to Number for expressions and function calls. * Added support for binary operators, unary operators and math library --------- Co-authored-by: yizhak <[email protected]>
source-academy · Mar 23, 2024 · 979f54d · 979f54d
1 parent 215754a
commit 979f54d
Show file tree

Hide file tree

Showing 8 changed files with 141 additions and 41 deletions.
diff --git a/src/ast-types.ts b/src/ast-types.ts
diff --git a/src/generate-ast.ts b/src/generate-ast.ts
@@ -51,6 +51,7 @@ export class AstWriter {
     main() {
         this.setup();
         this.defineAst("Expr", [
+            "BigIntLiteral -> value: string",
             "Binary    -> left: Expr, operator: Token, right: Expr",
             // Semantically different from Binary - for logical comparisons.
             "Compare   -> left: Expr, operator: Token, right: Expr",

diff --git a/src/index.ts b/src/index.ts
@@ -150,6 +150,10 @@ export function parsePythonToEstreeAst(code: string,
     return translator.resolve(ast) as unknown as Program
 }
 
+const text = `
+-1
+`
+console.dir(parsePythonToEstreeAst(text, 1, false));
 export * from './errors';
 
 // import {ParserErrors, ResolverErrors, TokenizerErrors} from "./errors";

diff --git a/src/parser.ts b/src/parser.ts
@@ -141,9 +141,9 @@ export class Parser {
         const startToken = this.peek();
         const statements: Stmt[] = [];
         while (!this.isAtEnd()) {
-            if (this.match(TokenType.NEWLINE)) {
-                continue;
-            }
+            // if (this.match(TokenType.NEWLINE)) {
+            //     continue;
+            // }
             statements.push(this.stmt());
         }
         const endToken = this.peek();
@@ -154,9 +154,9 @@ export class Parser {
         if (this.check(TokenType.DEF, TokenType.FOR, TokenType.IF, TokenType.WHILE)) {
             return this.compound_stmt();
         } else if (this.check(TokenType.NAME, ...PSEUD_NAMES, TokenType.NUMBER,
-            TokenType.PASS, TokenType.BREAK, TokenType.CONTINUE,
+            TokenType.PASS, TokenType.BREAK, TokenType.CONTINUE, TokenType.MINUS, TokenType.PLUS, TokenType.INDENT, TokenType.DEDENT,
             TokenType.RETURN, TokenType.FROM, TokenType.GLOBAL, TokenType.NONLOCAL,
-            TokenType.ASSERT, TokenType.LPAR, TokenType.STRING, ...SPECIAL_IDENTIFIER_TOKENS)) {
+            TokenType.ASSERT, TokenType.LPAR, TokenType.STRING, TokenType.BIGINT, ...SPECIAL_IDENTIFIER_TOKENS)) {
             return this.simple_stmt();
         }
         const startToken = this.peek();
@@ -239,6 +239,10 @@ export class Parser {
         let res = null;
         if (this.match(TokenType.NAME)) {
             res = this.assign_stmt();
+        } else if (this.match(TokenType.INDENT)) {
+            res = new StmtNS.Indent(startToken, startToken);
+        } else if (this.match(TokenType.DEDENT)) {
+            res = new StmtNS.Dedent(startToken, startToken);
         } else if (this.match(TokenType.PASS)) {
             res = new StmtNS.Pass(startToken, startToken);
         } else if (this.match(TokenType.BREAK)) {
@@ -255,7 +259,8 @@ export class Parser {
             res = new StmtNS.NonLocal(startToken, startToken, this.advance());
         } else if (this.match(TokenType.ASSERT)) {
             res = new StmtNS.Assert(startToken, startToken, this.test());
-        } else if (this.check(TokenType.LPAR, TokenType.NUMBER, TokenType.STRING, ...SPECIAL_IDENTIFIER_TOKENS)) {
+        } else if (this.check(TokenType.LPAR, TokenType.NUMBER, TokenType.STRING,
+            TokenType.BIGINT, TokenType.MINUS, TokenType.PLUS, ...SPECIAL_IDENTIFIER_TOKENS)) {
             res = new StmtNS.SimpleExpr(startToken, startToken, this.test());
         } else {
             throw new Error("Unreachable code path");
@@ -501,6 +506,9 @@ export class Parser {
         if (this.match(TokenType.NUMBER)) {
             return new ExprNS.Literal(startToken, this.previous(), Number(this.previous().lexeme));
         }
+        if (this.match(TokenType.BIGINT)) {
+            return new ExprNS.BigIntLiteral(startToken, this.previous(), this.previous().lexeme);
+        }
 
         if (this.match(TokenType.NAME, ...PSEUD_NAMES)) {
             return new ExprNS.Variable(startToken, this.previous(), this.previous());

diff --git a/src/resolver.ts b/src/resolver.ts
@@ -1,9 +1,9 @@
-import {StmtNS, ExprNS} from "./ast-types";
+import { StmtNS, ExprNS } from "./ast-types";
 type Expr = ExprNS.Expr;
 type Stmt = StmtNS.Stmt;
-import {Token} from "./tokenizer";
-import {TokenType} from "./tokens";
-import {ResolverErrors} from "./errors";
+import { Token } from "./tokenizer";
+import { TokenType } from "./tokens";
+import { ResolverErrors } from "./errors";
 
 const levenshtein = require('fast-levenshtein');
 
@@ -35,7 +35,7 @@ class Environment {
         const name = identifier.lexeme;
         let distance = 0;
         let curr: Environment | null = this;
-        while(curr !== null) {
+        while (curr !== null) {
             if (curr.names.has(name)) {
                 break;
             }
@@ -49,7 +49,7 @@ class Environment {
     lookupNameCurrentEnv(identifier: Token): Token | undefined {
         return this.names.get(identifier.lexeme);
     }
-    lookupNameCurrentEnvWithError(identifier: Token){
+    lookupNameCurrentEnvWithError(identifier: Token) {
         if (this.lookupName(identifier) < 0) {
             throw new ResolverErrors.NameNotFoundError(identifier.line, identifier.col,
                 this.source,
@@ -104,7 +104,7 @@ class Environment {
         let minDistance = Infinity;
         let minName = null;
         let curr: Environment | null = this;
-        while(curr !== null) {
+        while (curr !== null) {
             for (const declName of curr.names.keys()) {
                 const dist = levenshtein.get(name, declName);
                 if (dist < minDistance) {
@@ -125,12 +125,13 @@ class Environment {
 export class Resolver implements StmtNS.Visitor<void>, ExprNS.Visitor<void> {
     source: string;
     ast: Stmt;
+    // change the environment to be suite scope as in python
     environment: Environment | null;
     constructor(source: string, ast: Stmt) {
         this.source = source;
         this.ast = ast;
         // The global environment
-        this.environment = new Environment(source,null, new Map([
+        this.environment = new Environment(source, null, new Map([
             ["range", new Token(TokenType.NAME, "range", 0, 0, 0)],
             ["display", new Token(TokenType.NAME, "display", 0, 0, 0)],
             ["stringify", new Token(TokenType.NAME, "stringify", 0, 0, 0)],
@@ -173,21 +174,40 @@ export class Resolver implements StmtNS.Visitor<void>, ExprNS.Visitor<void> {
         this.environment = oldEnv;
     }
 
+    visitIndentCreation(stmt: StmtNS.Indent): void {
+        // Create a new environment.
+        const oldEnv = this.environment;
+        this.environment = new Environment(this.source, this.environment, new Map());
+    }
+
+    visitDedentCreation(stmt: StmtNS.Dedent): void {
+        // Switch to the previous environment.
+        if (this.environment?.enclosing !== undefined) {
+            this.environment = this.environment.enclosing;
+        }
+    }
+
     visitFunctionDefStmt(stmt: StmtNS.FunctionDef) {
         this.environment?.declareName(stmt.name);
         this.environment?.functions.add(stmt.name.lexeme);
-        // Create a new environment.
-        const oldEnv = this.environment;
-        // Assign the parameters to the new environment.
-        const newEnv = new Map(
+        // // Create a new environment.
+        // const oldEnv = this.environment;
+        // // Assign the parameters to the new environment.
+        // const newEnv = new Map(
+        //     stmt.parameters.map(param => [param.lexeme, param])
+        // );
+        // this.environment = new Environment(this.source, this.environment, newEnv);
+        const params = new Map(
             stmt.parameters.map(param => [param.lexeme, param])
         );
-        this.environment = new Environment(this.source, this.environment, newEnv);
+        if (this.environment !== null) {
+            this.environment.names = params;
+        }
         this.resolve(stmt.body);
         // Grab identifiers from that new environment. That are NOT functions.
         // stmt.varDecls = this.varDeclNames(this.environment.names)
         // Restore old environment
-        this.environment = oldEnv;
+        // this.environment = oldEnv;
     }
 
     visitAnnAssignStmt(stmt: StmtNS.AnnAssign): void {
@@ -319,6 +339,7 @@ export class Resolver implements StmtNS.Visitor<void>, ExprNS.Visitor<void> {
     }
     visitLiteralExpr(expr: ExprNS.Literal): void {
     }
-
+    visitBigIntLiteralExpr(expr: ExprNS.BigIntLiteral): void {
+    }
 
 }
diff --git a/src/tokenizer.ts b/src/tokenizer.ts
@@ -291,6 +291,11 @@ export class Tokenizer {
         while (this.isDigit(this.peek())) {
             this.advance();
         }
+
+        if (this.peek() !== '.' && this.peek() !== 'e') {
+            this.addToken(TokenType.BIGINT);
+            return;
+        }
         // Fractional part
         if (this.peek() === '.') {
             this.advance();

diff --git a/src/tokens.ts b/src/tokens.ts
@@ -7,6 +7,7 @@ export enum TokenType {
     ENDMARKER,
     NAME,
     NUMBER,
+    BIGINT,
     STRING,
     NEWLINE,
     INDENT,