From 61a0233e8555bc04cb4dfc8044aea017cf59244f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilherme=20Paix=C3=A3o?= Date: Fri, 29 Dec 2023 10:31:45 -0300 Subject: [PATCH] feat: created linker for import others deps --- src/builder/builder.ts | 7 +- src/fsutil/fsutil.test.ts | 38 ++ src/fsutil/fsutil.ts | 11 +- src/importer/dependency.ts | 3 - src/importer/discover.test.ts | 57 --- src/importer/discover.ts | 54 --- src/index.ts | 36 +- src/interpreter/evaluator/evaluator.test.ts | 420 +++++++++----------- src/interpreter/evaluator/evaluator.ts | 10 + src/interpreter/interpreter.ts | 27 +- src/lexer/lexer.ts | 15 +- src/lexer/tokens/terminal.ts | 1 - src/parser/node.ts | 24 +- src/parser/parser.test.ts | 245 +++++++++--- src/parser/parser.ts | 191 +++++---- src/parser/tag.ts | 3 + 16 files changed, 638 insertions(+), 504 deletions(-) create mode 100644 src/fsutil/fsutil.test.ts delete mode 100644 src/importer/dependency.ts delete mode 100644 src/importer/discover.test.ts delete mode 100644 src/importer/discover.ts diff --git a/src/builder/builder.ts b/src/builder/builder.ts index e38e524..5d2425e 100644 --- a/src/builder/builder.ts +++ b/src/builder/builder.ts @@ -5,14 +5,15 @@ import { Parser } from "@/parser"; import Generator from "./generator"; import SymTable from "@/symtable/symtable"; +import { read } from "@/fsutil"; export default class Builder { constructor(private readonly _buffer: Buffer = Buffer.from("")) {} - public run(debug?: boolean): string[] { + public async run(debug?: boolean): Promise { const lexer = new Lexer(this._buffer); - const parser = new Parser(lexer, new SymTable("root")); - const tree = parser.parse(); + const parser = new Parser({ read }, new SymTable("root")); + const tree = await parser.parse(lexer); debug && console.log(colorize(JSON.stringify(tree, null, 2))); return Generator.run(tree); } diff --git a/src/fsutil/fsutil.test.ts b/src/fsutil/fsutil.test.ts new file mode 100644 index 0000000..d9b5bca --- /dev/null +++ b/src/fsutil/fsutil.test.ts @@ -0,0 +1,38 @@ +import { noext } from "./fsutil"; + +describe("fsutil test suite", () => { + test("Should return filename and empty extension", async () => { + const raw = "greeting"; + const expected = ["greeting", ""]; + const got = noext(raw); + expect(got).toStrictEqual(expected); + }); + + test("Should return filename and 'br' extension", async () => { + const raw = "greeting.br"; + const expected = ["greeting", "br"]; + const got = noext(raw); + expect(got).toStrictEqual(expected); + }); + + test("Should return filename and 'ar' extension", async () => { + const raw = "greeting.ar"; + const expected = ["greeting", "ar"]; + const got = noext(raw); + expect(got).toStrictEqual(expected); + }); + + test("Should return 'gree.ting' filename and 'ar' extension", async () => { + const raw = "gree.ting.ar"; + const expected = ["gree.ting", "ar"]; + const got = noext(raw); + expect(got).toStrictEqual(expected); + }); + + test("Should return empty filename and empty extension", async () => { + const raw = ""; + const expected = ["", ""]; + const got = noext(raw); + expect(got).toStrictEqual(expected); + }); +}); diff --git a/src/fsutil/fsutil.ts b/src/fsutil/fsutil.ts index f9ccad3..6b85e9f 100644 --- a/src/fsutil/fsutil.ts +++ b/src/fsutil/fsutil.ts @@ -1,10 +1,12 @@ import fs from "fs"; -export async function read(arg: string): Promise { +export async function read(raw: string): Promise { + const [filename] = noext(raw); + return new Promise((resolve, reject) => { let buffer = Buffer.from(""); - const reader = fs.createReadStream(arg); + const reader = fs.createReadStream(`${filename}.ar`); reader.on("data", (chunk: Buffer) => { buffer = Buffer.concat([buffer, chunk]); @@ -30,3 +32,8 @@ export async function write(filename: string, content: string[]) { writer.close(); } + +export function noext(raw: string): [string, string] { + const [ext, ...splitted] = raw.split(".").reverse(); + return splitted.length > 0 ? [splitted.reverse().join("."), ext] : [ext, ""]; +} diff --git a/src/importer/dependency.ts b/src/importer/dependency.ts deleted file mode 100644 index ba4960f..0000000 --- a/src/importer/dependency.ts +++ /dev/null @@ -1,3 +0,0 @@ -export default class Dependency { - constructor(public readonly id: string, public readonly alias: string) {} -} diff --git a/src/importer/discover.test.ts b/src/importer/discover.test.ts deleted file mode 100644 index 15522bf..0000000 --- a/src/importer/discover.test.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { Lexer } from "@/lexer"; -import Discover from "./discover"; -import Dependency from "./dependency"; - -describe("Discover test suite", () => { - test("Program that imports only one dependency", () => { - const program = ` - from "github.com/guiferpa/testing" as testing - `; - - const expected = [new Dependency("github.com/guiferpa/testing", "testing")]; - - const lexer = new Lexer(Buffer.from(program, "utf-8")); - const discover = new Discover(lexer); - const got = discover.run(); - expect(got).toStrictEqual(expected); - }); - - test("Program that imports some dependencies", () => { - const program = ` - from "github.com/guiferpa/testing" as testing - from "github.com/guiferpa/tester" as tester - from "github.com/guiferpa/test" as test - `; - - const expected = [ - new Dependency("github.com/guiferpa/testing", "testing"), - new Dependency("github.com/guiferpa/tester", "tester"), - new Dependency("github.com/guiferpa/test", "test"), - ]; - - const lexer = new Lexer(Buffer.from(program, "utf-8")); - const discover = new Discover(lexer); - const got = discover.run(); - expect(got).toStrictEqual(expected); - }); - - test("Program that imports some duplicated dependencies", () => { - const program = ` - from "github.com/guiferpa/testing" as testing - from "github.com/guiferpa/testing" as testing - from "github.com/guiferpa/tester" as tester - from "github.com/guiferpa/test" as test - `; - - const expected = [ - new Dependency("github.com/guiferpa/testing", "testing"), - new Dependency("github.com/guiferpa/tester", "tester"), - new Dependency("github.com/guiferpa/test", "test"), - ]; - - const lexer = new Lexer(Buffer.from(program, "utf-8")); - const discover = new Discover(lexer); - const got = discover.run(); - expect(got).toStrictEqual(expected); - }); -}); diff --git a/src/importer/discover.ts b/src/importer/discover.ts deleted file mode 100644 index 7c5db0b..0000000 --- a/src/importer/discover.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { Lexer } from "@/lexer"; -import { TokenTag } from "@/lexer/tokens/tag"; -import { Token } from "@/lexer/tokens/token"; - -import Dependency from "./dependency"; - -export default class Discover { - private _lookahead: Token | null = null; - - constructor(private readonly _lexer: Lexer) {} - - private _discovery(): Dependency[] { - const deps = new Map(); - - this._lookahead = this._lexer.getNextToken(); - - while (this._lookahead?.tag === TokenTag.FROM) { - this._lookahead = this._lexer.getNextToken(); - if (this._lookahead.tag !== TokenTag.STR) { - throw new Error( - `Missing dependency identifier at line ${this._lookahead.line}` - ); - } - - const id = this._lookahead.value; - - this._lookahead = this._lexer.getNextToken(); - - if (this._lookahead.tag !== TokenTag.AS) { - throw new SyntaxError( - `Unexpected token ${this._lookahead.tag} at line ${this._lookahead.line}` - ); - } - - this._lookahead = this._lexer.getNextToken(); - - if (this._lookahead.tag !== TokenTag.IDENT) { - throw new Error(`Invalid alias at line ${this._lookahead.line}`); - } - - const alias = this._lookahead.value; - - deps.set(id, new Dependency(id, alias)); - - this._lookahead = this._lexer.getNextToken(); - } - - return Array.from(deps.values()); - } - - public run(): Dependency[] { - return this._discovery(); - } -} diff --git a/src/index.ts b/src/index.ts index ba3dbf4..9d57479 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,10 @@ import { Interpreter } from "@/interpreter"; import { read } from "@/fsutil"; import { repl } from "@/repl"; import { Builder } from "./builder"; +import Environment from "./environ/environ"; +import { Parser } from "./parser"; +import SymTable from "./symtable/symtable"; +import { Lexer } from "./lexer"; function run() { const program = new Command(); @@ -15,17 +19,28 @@ function run() { program .option("-t, --tree", "tree flag to show AST", false) .option("-a, --args ", "pass arguments for runtime", "") - .action(function () { + .action(async function () { const options = program.opts(); const optArgs = options.args.split(","); const r = repl(); - const interpreter = new Interpreter(); - r.on("line", function (chunk) { - interpreter.write(Buffer.from(chunk)); - console.log(`= ${interpreter.run(options.tree as boolean, optArgs)}`); + const environ = new Environment("global"); + const interpreter = new Interpreter(environ); + + r.on("line", async function (chunk) { + const buffer = Buffer.from(chunk); + const lexer = new Lexer(buffer); + const symtable = new SymTable("global"); + const parser = new Parser({ read }, symtable); + const tree = await parser.parse(lexer); + const result = await interpreter.run( + tree, + options.tree as boolean, + optArgs + ); + console.log(`= ${result}`); r.prompt(true); }); @@ -47,8 +62,13 @@ function run() { const optArgs = options.args.split(","); const buffer = await read(arg); - const interpreter = new Interpreter(buffer); - interpreter.run(options.tree as boolean, optArgs); + const lexer = new Lexer(buffer); + const symtable = new SymTable("global"); + const parser = new Parser({ read }, symtable); + const tree = await parser.parse(lexer); + const environ = new Environment("global"); + const interpreter = new Interpreter(environ); + await interpreter.run(tree, options.tree as boolean, optArgs); } catch (err) { if (err instanceof SyntaxError) { console.log(err); @@ -69,7 +89,7 @@ function run() { const buffer = await read(arg); const builder = new Builder(buffer); - const ops = builder.run(options.tree as boolean); + const ops = await builder.run(options.tree as boolean); ops.forEach((op, idx) => { console.log(`${idx}: ${op}`); }); diff --git a/src/interpreter/evaluator/evaluator.test.ts b/src/interpreter/evaluator/evaluator.test.ts index cb5c28e..10dc9ff 100644 --- a/src/interpreter/evaluator/evaluator.test.ts +++ b/src/interpreter/evaluator/evaluator.test.ts @@ -5,351 +5,303 @@ import Evaluator from "./evaluator"; import SymTable from "@/symtable"; import Environment from "@/environ"; +const execEvaluator = async ( + bucket: Map, + args: string[] = [], + environ: Environment = new Environment("global") +) => { + const program = bucket.get("main") as string; + const lexer = new Lexer(Buffer.from(program, "utf-8")); + const symtable = new SymTable("global"); + const reader = { + read: async (entry: string) => Buffer.from(bucket.get(entry) as string), + }; + const parser = new Parser(reader, symtable); + const tree = await parser.parse(lexer); + const evaluator = new Evaluator(environ, args); + return evaluator.evaluate(tree); +}; + describe("Evaluator test suite", () => { - test("Program that sum two numbers", () => { - const program = ` - 1 + 1 - `; + test("Program that sum two numbers", async () => { + const bucket = new Map([["main", `1 + 1`]]); const expected = [2]; - - const lexer = new Lexer(Buffer.from(program)); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - const environ = new Environment("root"); - const evaluator = new Evaluator(environ); - const got = evaluator.evaluate(parser.parse()); + const got = await execEvaluator(bucket); expect(got).toStrictEqual(expected); }); - test("Program that calc precedence expression", () => { - const program = ` - 10 + 20 - 3 * 20 - `; + test("Program that calc precedence expression", async () => { + const bucket = new Map([["main", `10 + 20 - 3 * 20`]]); const expected = [-30]; - - const lexer = new Lexer(Buffer.from(program)); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - const environ = new Environment("root"); - const evaluator = new Evaluator(environ); - const got = evaluator.evaluate(parser.parse()); + const got = await execEvaluator(bucket); expect(got).toStrictEqual(expected); }); - test("Program 2 that calc precedence expression", () => { - const program = ` - 10 - 2 * 5 - `; + test("Program 2 that calc precedence expression", async () => { + const bucket = new Map([["main", `10 - 2 * 5`]]); const expected = [0]; - - const lexer = new Lexer(Buffer.from(program)); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - const environ = new Environment("root"); - const evaluator = new Evaluator(environ); - const got = evaluator.evaluate(parser.parse()); + const got = await execEvaluator(bucket); expect(got).toStrictEqual(expected); }); - test("Program 3 that calc precedence expression", () => { - const program = ` - 10 * 2 - 5 - `; + test("Program 3 that calc precedence expression", async () => { + const bucket = new Map([["main", `10 * 2 - 5`]]); const expected = [15]; - - const lexer = new Lexer(Buffer.from(program)); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - const environ = new Environment("root"); - const evaluator = new Evaluator(environ); - const got = evaluator.evaluate(parser.parse()); + const got = await execEvaluator(bucket); expect(got).toStrictEqual(expected); }); - test("Program that set a variable then sum it with another number", () => { - const program = ` - var value = 10; - value + 20; - `; + test("Program that set a variable then sum it with another number", async () => { + const bucket = new Map([ + [ + "main", + `var value = 10 + value + 20`, + ], + ]); const expected = [undefined, 30]; - - const lexer = new Lexer(Buffer.from(program)); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - const environ = new Environment("root"); - const evaluator = new Evaluator(environ); - const got = evaluator.evaluate(parser.parse()); + const got = await execEvaluator(bucket); expect(got).toStrictEqual(expected); }); - test('Program that set an "if" then it has condition with "false" value', () => { - const program = ` - var compare = false; + test('Program that set an "if" then it has condition with "false" value', async () => { + const bucket = new Map([ + [ + "main", + `var compare = false - if (compare) { - print("Testing"); - 20; - } + if (compare) { + print("Testing") + 20 + } - 10; - `; + 10`, + ], + ]); const expected = [undefined, undefined, 10]; - - const lexer = new Lexer(Buffer.from(program)); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - const environ = new Environment("root"); - const evaluator = new Evaluator(environ); - const got = evaluator.evaluate(parser.parse()); + const got = await execEvaluator(bucket); expect(got).toStrictEqual(expected); }); - test('Program that set an "if" then it has condition with "true" value', () => { - const program = ` - var compare = true; + test('Program that set an "if" then it has condition with "true" value', async () => { + const bucket = new Map([ + [ + "main", + `var compare = true - if (compare) { - print("Testing"); - 20; - } + if compare { + print("Testing") + 20 + } - 10; - `; + 10`, + ], + ]); const expected = [undefined, undefined, 10]; - - const lexer = new Lexer(Buffer.from(program)); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - const environ = new Environment("root"); - const evaluator = new Evaluator(environ); - const got = evaluator.evaluate(parser.parse()); + const got = await execEvaluator(bucket); expect(got).toStrictEqual(expected); }); - test("Program that get argument from CLI execution", () => { - const program = ` - func greeting(who) { - if who { - return who - } + test("Program that get argument from CLI execution", async () => { + const bucket = new Map([ + [ + "main", + `func greeting(who) { + if who { + return who + } - return "World" - } + return "World" + } - print("Hello") - greeting(greeting(arg(0))) - `; + print("Hello") + greeting(greeting(arg(0))) + `, + ], + ]); const expected = [undefined, undefined, "Testing"]; - - const lexer = new Lexer(Buffer.from(program)); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - const environ = new Environment("root"); - const evaluator = new Evaluator(environ, ["Testing"]); - const got = evaluator.evaluate(parser.parse()); + const got = await execEvaluator(bucket, ["Testing"]); expect(got).toStrictEqual(expected); }); - test("Program that get more than one argument from CLI execution", () => { - const program = ` - arg(1) - func greeting(who) { - if who { - return who - } + test("Program that get more than one argument from CLI execution", async () => { + const bucket = new Map([ + [ + "main", + `arg(1) + func greeting(who) { + if who { + return who + } - return "World" - } + return "World" + } - print("Hello") - greeting(greeting(arg(0))) - `; + print("Hello") + greeting(greeting(arg(0)))`, + ], + ]); const expected = ["Testing 2", undefined, undefined, "Testing"]; - - const lexer = new Lexer(Buffer.from(program)); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - const environ = new Environment("root"); - const evaluator = new Evaluator(environ, ["Testing", "Testing 2"]); - const got = evaluator.evaluate(parser.parse()); + const got = await execEvaluator(bucket, ["Testing", "Testing 2"]); expect(got).toStrictEqual(expected); }); - test("Program that get more than one argument from CLI execution", () => { + test("Program that get more than one argument from CLI execution", async () => { const program = ` - arg(1) - func greeting(who) { - if who { - return who - } - - return "World" - } - - print("Hello") - greeting(greeting(arg(0))) `; + const bucket = new Map([ + [ + "main", + `arg(1) + func greeting(who) { + if who { + return who + } + + return "World" + } + + print("Hello") + greeting(greeting(arg(0)))`, + ], + ]); const expected = ["Testing 2", undefined, undefined, "Testing"]; - - const lexer = new Lexer(Buffer.from(program)); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - const environ = new Environment("root"); - const evaluator = new Evaluator(environ, ["Testing", "Testing 2"]); - const got = evaluator.evaluate(parser.parse()); + const got = await execEvaluator(bucket, ["Testing", "Testing 2"]); expect(got).toStrictEqual(expected); }); - test('Program that set an "if" then it has condition with "true" value based on argument input', () => { - const program = ` - var a = arg(0) - var b = arg(1) + test('Program that set an "if" then it has condition with "true" value based on argument input', async () => { + const bucket = new Map([ + [ + "main", + `var a = arg(0) + var b = arg(1) - if a equal "print" { - return b - } - `; + if a equal "print" { + return b + }`, + ], + ]); const expected = [undefined, undefined, "Testing"]; - - const lexer = new Lexer(Buffer.from(program)); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - const environ = new Environment("root"); - const evaluator = new Evaluator(environ, ["print", "Testing"]); - const got = evaluator.evaluate(parser.parse()); + const got = await execEvaluator(bucket, ["print", "Testing"]); expect(got).toStrictEqual(expected); }); - test('Program that set an "if" then it has condition with "false" value based on argument input', () => { - const program = ` - var a = arg(0) - var b = arg(1) + test('Program that set an "if" then it has condition with "false" value based on argument input', async () => { + const bucket = new Map([ + [ + "main", + `var a = arg(0) + var b = arg(1) - if a equal "print" { - return b - } - `; + if a equal "print" { + return b + }`, + ], + ]); const expected = [undefined, undefined, undefined]; - - const lexer = new Lexer(Buffer.from(program)); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - const environ = new Environment("root"); - const evaluator = new Evaluator(environ, ["no-print", "Testing"]); - const got = evaluator.evaluate(parser.parse()); + const got = await execEvaluator(bucket); expect(got).toStrictEqual(expected); }); - test("Program execute fibonacci algorithm", () => { - const program = ` - func fib(n) - desc "This function calculate fibonacci number" { - if n less 1 or n equal 1 { - return n - } + test("Program that execute fibonacci algorithm", async () => { + const bucket = new Map([ + [ + "main", + `func fib(n) + desc "This function calculate fibonacci number" { + if n less 1 or n equal 1 { + return n + } - return fib(n - 1) + fib(n - 2) - } + return fib(n - 1) + fib(n - 2) + } - fib(25) - `; + fib(25)`, + ], + ]); const expected = [undefined, 75025]; - - const lexer = new Lexer(Buffer.from(program)); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - const environ = new Environment("root"); - const evaluator = new Evaluator(environ); - const got = evaluator.evaluate(parser.parse()); + const got = await execEvaluator(bucket); expect(got).toStrictEqual(expected); }); - test("Program that declare an array", () => { - const program = ` - var arr = [1, 2, 50] - arr - `; + test("Program that declare an array", async () => { + const bucket = new Map([ + [ + "main", + `var arr = [1, 2, 50] + arr`, + ], + ]); const expected = [undefined, [1, 2, 50]]; - - const lexer = new Lexer(Buffer.from(program)); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - const environ = new Environment("root"); - const evaluator = new Evaluator(environ); - const got = evaluator.evaluate(parser.parse()); + const got = await execEvaluator(bucket); expect(got).toStrictEqual(expected); }); - test("Program that call map to increment one in array's items", () => { - const program = ` - var arr = [1, 2, 50] + test("Program that call map to increment one in array's items", async () => { + const bucket = new Map([ + [ + "main", + `var arr = [1, 2, 50] - func increment(item) { - return item + 1 - } + func increment(item) { + return item + 1 + } - map(arr, increment) - `; + map(arr, increment)`, + ], + ]); const expected = [undefined, undefined, [2, 3, 51]]; - - const lexer = new Lexer(Buffer.from(program)); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - const environ = new Environment("root"); - const evaluator = new Evaluator(environ); - const got = evaluator.evaluate(parser.parse()); + const got = await execEvaluator(bucket); expect(got).toStrictEqual(expected); }); - test("Program that call filter to return all values from array less 'aquaman' string", () => { - const program = ` - var arr = ["aunt-man", "batman", "aquaman", "iron man"] + test("Program that call filter to return all values from array less 'aquaman' string", async () => { + const bucket = new Map([ + [ + "main", + `var arr = ["aunt-man", "batman", "aquaman", "iron man"] - func is_not_aquaman?(item) { - return not (item equal "aquaman") - } + func is_not_aquaman?(item) { + return not (item equal "aquaman") + } - filter(arr, is_not_aquaman?) - `; + filter(arr, is_not_aquaman?)`, + ], + ]); const expected = [undefined, undefined, ["aunt-man", "batman", "iron man"]]; - - const lexer = new Lexer(Buffer.from(program)); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - const environ = new Environment("root"); - const evaluator = new Evaluator(environ); - const got = evaluator.evaluate(parser.parse()); + const got = await execEvaluator(bucket); expect(got).toStrictEqual(expected); }); diff --git a/src/interpreter/evaluator/evaluator.ts b/src/interpreter/evaluator/evaluator.ts index 6a8eb23..d66f39e 100644 --- a/src/interpreter/evaluator/evaluator.ts +++ b/src/interpreter/evaluator/evaluator.ts @@ -25,6 +25,9 @@ import { ArrayNode, CallMapStmtNode, CallFilterStmtNode, + FromStmtNode, + AsStmtNode, + ImportStmtNode, } from "@/parser/node"; export default class Evaluator { @@ -46,6 +49,13 @@ export default class Evaluator { public evaluate(tree: ParserNode): any { if (tree instanceof ProgramNode) return this.compose(tree.children); + if (tree instanceof ImportStmtNode) + return this.compose(tree.program.children); + + if (tree instanceof FromStmtNode || tree instanceof AsStmtNode) { + return undefined; + } + if (tree instanceof BlockStmtNode) return this.compose(tree.children); if (tree instanceof AssignStmtNode) { diff --git a/src/interpreter/interpreter.ts b/src/interpreter/interpreter.ts index 519a7f2..0237581 100644 --- a/src/interpreter/interpreter.ts +++ b/src/interpreter/interpreter.ts @@ -1,30 +1,17 @@ import colorize from "json-colorizer"; import { Evaluator } from "./evaluator"; -import { Lexer } from "@/lexer"; -import { Parser } from "@/parser"; +import { ParserNode } from "@/parser"; import Environment from "@/environ/environ"; -import SymTable from "@/symtable"; export default class Interpreter { - private _lexer: Lexer; - private _parser: Parser; - private _symtable: SymTable; - private _environ: Environment; + constructor(private _environ: Environment) {} - constructor(buffer: Buffer = Buffer.from("")) { - this._environ = new Environment("global"); - this._symtable = new SymTable("global"); - this._lexer = new Lexer(buffer); - this._parser = new Parser(this._lexer, this._symtable); - } - - public write(buffer: Buffer) { - this._lexer.write(buffer); - } - - public run(debug?: boolean, args: string[] = []): string[] { - const tree = this._parser.parse(); + public async run( + tree: ParserNode, + debug?: boolean, + args: string[] = [] + ): Promise { debug && console.log(colorize(JSON.stringify(tree, null, 2))); const evaluator = new Evaluator(this._environ, args); return evaluator.evaluate(tree); diff --git a/src/lexer/lexer.ts b/src/lexer/lexer.ts index a43d16e..a3638ea 100644 --- a/src/lexer/lexer.ts +++ b/src/lexer/lexer.ts @@ -6,12 +6,13 @@ export default class Lexer { private _cursor = 0; private _line = 1; private _column = 1; - private _buffer: Buffer; private _isComment: boolean = false; + private _currentToken: Token | null = null; - constructor(buffer: Buffer = Buffer.from("")) { - this._buffer = buffer; - } + constructor( + private _buffer: Buffer = Buffer.from(""), + public readonly previous: Lexer | null = null + ) {} public write(buffer: Buffer) { this._buffer = Buffer.concat([this._buffer, buffer]); @@ -21,6 +22,10 @@ export default class Lexer { return this._cursor < this._buffer.length; } + public getCurrentToken(): Token | null { + return this._currentToken; + } + public getNextToken(): Token { if (!this.hasMoreTokens()) return new Token(this._line, this._column, TokenTag.EOF, "EOF"); @@ -81,6 +86,8 @@ export default class Lexer { ); } + this._currentToken = token; + return token; } diff --git a/src/lexer/tokens/terminal.ts b/src/lexer/tokens/terminal.ts index 40cca53..08fe634 100644 --- a/src/lexer/tokens/terminal.ts +++ b/src/lexer/tokens/terminal.ts @@ -23,7 +23,6 @@ export const Terminals: [RegExp, TokenTag][] = [ [new RegExp(/^return void/), TokenTag.RETURN_VOID], [new RegExp(/^return/), TokenTag.RETURN], [new RegExp(/^from/), TokenTag.FROM], - [new RegExp(/^as/), TokenTag.AS], [new RegExp(/^var [a-zA-Z_]+(\s?)=/), TokenTag.ASSIGN], [new RegExp(/^func [a-zA-Z_><\-!?]+/), TokenTag.DECL_FN], [new RegExp(/^\+/), TokenTag.OP_ADD], diff --git a/src/parser/node.ts b/src/parser/node.ts index 02712d5..c8cf318 100644 --- a/src/parser/node.ts +++ b/src/parser/node.ts @@ -162,7 +162,7 @@ export class CallArgStmtNode extends ParserNode { export class CallConcatStmtNode extends ParserNode { constructor(public readonly values: ParserNode[]) { - super(ParserNodeTag.CALL_ARG_STMT); + super(ParserNodeTag.CALL_CONCAT_STMT); } } @@ -184,6 +184,28 @@ export class CallFilterStmtNode extends ParserNode { } } +export class FromStmtNode extends ParserNode { + constructor(public readonly id: string) { + super(ParserNodeTag.FROM_STMT); + } +} + +export class AsStmtNode extends ParserNode { + constructor(public readonly alias: string) { + super(ParserNodeTag.AS_STMT); + } +} + +export class ImportStmtNode extends ParserNode { + constructor( + public readonly id: ParserNode, + public readonly alias: ParserNode, + public readonly program: ProgramNode + ) { + super(ParserNodeTag.IMPORT_STMT); + } +} + export class ProgramNode extends ParserNode { constructor(public readonly children: ParserNode[]) { super(ParserNodeTag.PROGRAM); diff --git a/src/parser/parser.test.ts b/src/parser/parser.test.ts index 76dc2e6..ea7f18b 100644 --- a/src/parser/parser.test.ts +++ b/src/parser/parser.test.ts @@ -1,62 +1,213 @@ import Lexer from "@/lexer/lexer"; import Parser from "./parser"; import SymTable from "@/symtable"; +import { + ArityStmtNode, + AsStmtNode, + AssignStmtNode, + BinaryOpNode, + BlockStmtNode, + CallFuncStmtNode, + CallPrintStmtNode, + DeclFuncStmtNode, + FromStmtNode, + IdentNode, + IfStmtNode, + ImportStmtNode, + LogicalNode, + NumericalNode, + ProgramNode, + ReturnStmtNode, + StringNode, +} from "./node"; +import { Token } from "@/lexer/tokens/token"; +import { TokenTag } from "@/lexer/tokens/tag"; + +const execParser = async (bucket: Map) => { + const program = bucket.get("main") as string; + const lexer = new Lexer(Buffer.from(program, "utf-8")); + const symtable = new SymTable("global"); + const reader = { + read: async (entry: string) => Buffer.from(bucket.get(entry) as string), + }; + const parser = new Parser(reader, symtable); + return await parser.parse(lexer); +}; describe("Parser test suite", () => { - test("Parse expression with __OP_ADD__ token", () => { - const program = `1 + 1_000`; - const lexer = new Lexer(Buffer.from(program, "utf-8")); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - parser.parse(); + test("Program that parse sum binary operation", async () => { + const bucket = new Map([["main", `1_000 + 10`]]); + const expected = new ProgramNode([ + new BinaryOpNode( + new NumericalNode(1000), + new NumericalNode(10), + new Token(1, 8, TokenTag.OP_ADD, "+") + ), + ]); + + const got = await execParser(bucket); + expect(got).toStrictEqual(expected); + }); + + test("Program that parse function declaration", async () => { + const bucket = new Map([ + [ + "main", + `var i = 0 + func hello() {}`, + ], + ]); + const expected = new ProgramNode([ + new AssignStmtNode("i", new NumericalNode(0)), + new DeclFuncStmtNode( + "hello", + null, + new ArityStmtNode([]), + new BlockStmtNode([]) + ), + ]); + + const got = await execParser(bucket); + expect(got).toStrictEqual(expected); + }); + + test("Program that parse function declaration using parameters", async () => { + const bucket = new Map([ + [ + "main", + `var i = 2 + func hello(world) {}`, + ], + ]); + const expected = new ProgramNode([ + new AssignStmtNode("i", new NumericalNode(2)), + new DeclFuncStmtNode( + "hello", + null, + new ArityStmtNode(["world"]), + new BlockStmtNode([]) + ), + ]); + + const got = await execParser(bucket); + expect(got).toStrictEqual(expected); }); - test("Get function token", () => { - const program = ` - var i = 0; - func hello() {} - `; - const lexer = new Lexer(Buffer.from(program, "utf-8")); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - parser.parse(); + test("Program that parse function declaration using parameters and body", async () => { + const bucket = new Map([ + [ + "main", + `var i = 2 + func hello(world) { + var a = 1 + }`, + ], + ]); + const expected = new ProgramNode([ + new AssignStmtNode("i", new NumericalNode(2)), + new DeclFuncStmtNode( + "hello", + null, + new ArityStmtNode(["world"]), + new BlockStmtNode([new AssignStmtNode("a", new NumericalNode(1))]) + ), + ]); + + const got = await execParser(bucket); + expect(got).toStrictEqual(expected); }); - test("Get function token using params", () => { - const program = ` - var i = 0; - func hello(world) {} - `; - const lexer = new Lexer(Buffer.from(program, "utf-8")); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - parser.parse(); + test("Program that parse function declaration using parameters and body calling another function", async () => { + const bucket = new Map([ + [ + "main", + `var i = 100; + + func hello(world) { + var a = 25; + print(world); + }`, + ], + ]); + const expected = new ProgramNode([ + new AssignStmtNode("i", new NumericalNode(100)), + new DeclFuncStmtNode( + "hello", + null, + new ArityStmtNode(["world"]), + new BlockStmtNode([ + new AssignStmtNode("a", new NumericalNode(25)), + new CallPrintStmtNode(new IdentNode("world")), + ]) + ), + ]); + + const got = await execParser(bucket); + expect(got).toStrictEqual(expected); }); - test("Get function token using body", () => { - const program = ` - var i = 0; - func hello(world) { - var a = 1; - } - `; - const lexer = new Lexer(Buffer.from(program, "utf-8")); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - parser.parse(); + test("Program that parse assign a variable", async () => { + const bucket = new Map([ + [ + "main", + `var compare = true + + if compare { + print("Testing") + 20 + } + + 10`, + ], + ]); + const expected = new ProgramNode([ + new AssignStmtNode("compare", new LogicalNode(true)), + new IfStmtNode( + new IdentNode("compare"), + new BlockStmtNode([ + new CallPrintStmtNode(new StringNode("Testing")), + new NumericalNode(20), + ]) + ), + new NumericalNode(10), + ]); + + const got = await execParser(bucket); + expect(got).toStrictEqual(expected); }); - test("Get function token using body calling another func", () => { - const program = ` - var i = 0; - func hello(world) { - var a = 1; - print(world); - } - `; - const lexer = new Lexer(Buffer.from(program, "utf-8")); - const symtable = new SymTable("root"); - const parser = new Parser(lexer, symtable); - parser.parse(); + test("Program that parse import syntax", async () => { + const bucket = new Map([ + [ + "main", + `from "testing" + + print(hello())`, + ], + [ + "testing", + `func hello() { + return 10 + }`, + ], + ]); + const expected = new ProgramNode([ + new ImportStmtNode( + new FromStmtNode("testing"), + new AsStmtNode(""), + new ProgramNode([ + new DeclFuncStmtNode( + "hello", + null, + new ArityStmtNode([]), + new BlockStmtNode([new ReturnStmtNode(new NumericalNode(10))]) + ), + ]) + ), + new CallPrintStmtNode(new CallFuncStmtNode("hello", [])), + ]); + + const got = await execParser(bucket); + expect(got).toStrictEqual(expected); }); }); diff --git a/src/parser/parser.ts b/src/parser/parser.ts index a1b582a..8103e75 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -27,23 +27,32 @@ import { ArrayNode, CallMapStmtNode, CallFilterStmtNode, + FromStmtNode, + AsStmtNode, + ImportStmtNode, } from "./node"; import SymTable from "@/symtable"; +import { Lexer } from "@/lexer"; -interface Lexer { - getNextToken(): Token | null; +export interface Reader { + read(entry: string): Promise; } export default class Parser { private _lookahead: Token | null = null; + private _lexer: Lexer | null = null; constructor( - private readonly _lexer: Lexer, - private _symtable: SymTable | null + private _reader: Reader, + private _symtable: SymTable | null = null ) {} private _eat(tokenTag: TokenTag): Token { + if (this._lexer === null) { + throw new Error("None source code at Lexer buffer"); + } + const token = this._lookahead; if (token?.tag === TokenTag.EOF) @@ -67,7 +76,7 @@ export default class Parser { * | __NUM__ * | __STR__ * **/ - private _fact(): ParserNode { + private async _fact(): Promise { if (this._lookahead?.tag === TokenTag.PAREN_O) { return this._pre(); } @@ -108,51 +117,82 @@ export default class Parser { return this._arr(); } + if (this._lookahead?.tag === TokenTag.FROM) { + return await this._import(); + } + throw new Error(`Unknwon token ${JSON.stringify(this._lookahead)}`); } + private _from(): ParserNode { + this._eat(TokenTag.FROM); + const id = this._str(); + return new FromStmtNode(id.value); + } + + private _as(): ParserNode { + this._eat(TokenTag.AS); + const id = this._eat(TokenTag.IDENT); + return new AsStmtNode(id.value); + } + + private async _import(): Promise { + const from = this._from(); + const alias = new AsStmtNode(""); + + if (!(from instanceof FromStmtNode)) { + throw new SyntaxError(`Unsupported identifier type for import`); + } + + const buffer = await this._reader.read(from.id); + const lexer = new Lexer(buffer, this._lexer); + const program = await this.parse(lexer); + + return new ImportStmtNode(from, alias, program); + } + // __CALL_MAP__ - private _map(): ParserNode { + private async _map(): Promise { this._eat(TokenTag.CALL_MAP); this._eat(TokenTag.PAREN_O); - const param = this._fact(); + const param = await this._fact(); this._eat(TokenTag.COMMA); - const fact = this._fact(); + const fact = await this._fact(); this._eat(TokenTag.PAREN_C); return new CallMapStmtNode(param, fact); } // __CALL_FILTER__ - private _filter(): ParserNode { + private async _filter(): Promise { this._eat(TokenTag.CALL_FILTER); this._eat(TokenTag.PAREN_O); - const param = this._fact(); + const param = await this._fact(); this._eat(TokenTag.COMMA); - const fact = this._fact(); + const fact = await this._fact(); this._eat(TokenTag.PAREN_C); return new CallFilterStmtNode(param, fact); } // __PAREN_O__ __PAREN_C__ - private _pre(): ParserNode { + private async _pre(): Promise { this._eat(TokenTag.PAREN_O); - const log = this._log(); + const log = await this._log(); this._eat(TokenTag.PAREN_C); return log; } // __S_BREAK_O__ __S_BREAK_C__ - private _arr(): ArrayNode { + private async _arr(): Promise { this._eat(TokenTag.S_BRACK_O); if (this._lookahead?.tag === TokenTag.S_BRACK_C) { this._eat(TokenTag.S_BRACK_C); return new ArrayNode([]); } - const items = [this._log()]; + const items = [await this._log()]; while (this._lookahead?.tag === TokenTag.COMMA) { this._eat(TokenTag.COMMA); - items.push(this._log()); + items.push(await this._log()); } this._eat(TokenTag.S_BRACK_C); @@ -174,19 +214,18 @@ export default class Parser { } // __STR__ - private _str(): ParserNode { + private _str(): StringNode { const str = this._eat(TokenTag.STR); return new StringNode(str.value); } // __IDENT__ or __CALL_FUNC__ - private _call(): ParserNode { + private async _call(): Promise { const ident = this._eat(TokenTag.IDENT); - this._symtable?.has(ident.value); // @ts-ignore if (this._lookahead.tag === TokenTag.PAREN_O) { - return this._callfn(ident); + return await this._callfn(ident); } return new IdentNode(ident.value); @@ -197,17 +236,17 @@ export default class Parser { * | __OP_SUB__ _uny * | _fact * **/ - private _uny(): ParserNode { + private async _uny(): Promise { if ( this._lookahead?.tag === TokenTag.OP_ADD || this._lookahead?.tag === TokenTag.OP_SUB ) { const op = this._eat(this._lookahead.tag); - const uny = this._uny(); + const uny = await this._uny(); return new UnaryOpNode(uny, op); } - return this._fact(); + return await this._fact(); } /** @@ -215,17 +254,19 @@ export default class Parser { * | _uny __OP_DIV__ _term * | _uny * **/ - private _term(): ParserNode { - const uny = this._uny(); + private async _term(): Promise { + const uny = await this._uny(); if (this._lookahead?.tag === TokenTag.OP_MUL) { const op = this._eat(TokenTag.OP_MUL); - return new BinaryOpNode(uny, this._term(), op); + const term = await this._term(); + return new BinaryOpNode(uny, term, op); } if (this._lookahead?.tag === TokenTag.OP_DIV) { const op = this._eat(TokenTag.OP_DIV); - return new BinaryOpNode(uny, this._term(), op); + const term = await this._term(); + return new BinaryOpNode(uny, term, op); } return uny; @@ -236,17 +277,19 @@ export default class Parser { * | _term __OP_SUB__ _expr * | _term * **/ - private _expr(): ParserNode { - const term = this._term(); + private async _expr(): Promise { + const term = await this._term(); if (this._lookahead?.tag === TokenTag.OP_ADD) { const op = this._eat(TokenTag.OP_ADD); - return new BinaryOpNode(term, this._expr(), op); + const expr = await this._expr(); + return new BinaryOpNode(term, expr, op); } if (this._lookahead?.tag === TokenTag.OP_SUB) { const op = this._eat(TokenTag.OP_SUB); - return new BinaryOpNode(term, this._expr(), op); + const expr = await this._expr(); + return new BinaryOpNode(term, expr, op); } return term; @@ -259,8 +302,8 @@ export default class Parser { * | _expr __REL_DIF__ _rel * | _expr * **/ - private _rel(): ParserNode { - const expr = this._expr(); + private async _rel(): Promise { + const expr = await this._expr(); if ( this._lookahead?.tag === TokenTag.REL_GT || @@ -269,7 +312,7 @@ export default class Parser { this._lookahead?.tag === TokenTag.REL_DIF ) { const op = this._eat(this._lookahead.tag); - const rel = this._rel(); + const rel = await this._rel(); return new RelativeExprNode(expr, rel, op); } @@ -280,13 +323,14 @@ export default class Parser { * _neg -> __NEG__ _rel * | _rel * **/ - private _neg(): ParserNode { + private async _neg(): Promise { if (this._lookahead?.tag === TokenTag.NEG) { this._eat(TokenTag.NEG); - return new NegativeExprNode(this._rel()); + const rel = await this._rel(); + return new NegativeExprNode(rel); } - return this._rel(); + return await this._rel(); } /** @@ -294,15 +338,15 @@ export default class Parser { * | _neg __LOG_OR__ _log * | _neg * **/ - private _log(): ParserNode { - const neg = this._neg(); + private async _log(): Promise { + const neg = await this._neg(); if ( this._lookahead?.tag === TokenTag.LOG_AND || this._lookahead?.tag === TokenTag.LOG_OR ) { const op = this._eat(this._lookahead.tag); - const log = this._log(); + const log = await this._log(); return new LogicExprNode(neg, log, op); } @@ -329,7 +373,7 @@ export default class Parser { * | __IDENT__ * | _log * **/ - private _callfn(id: Token): ParserNode { + private async _callfn(id: Token): Promise { this._eat(TokenTag.PAREN_O); // @ts-ignore @@ -338,12 +382,12 @@ export default class Parser { return new CallFuncStmtNode(id.value, []); } - const params: ParserNode[] = [this._log()]; + const params: ParserNode[] = [await this._log()]; // @ts-ignore while (this._lookahead.tag === TokenTag.COMMA) { this._eat(TokenTag.COMMA); - params.push(this._log()); + params.push(await this._log()); } this._eat(TokenTag.PAREN_C); @@ -385,13 +429,13 @@ export default class Parser { * _block -> __BRACK_O__ _statements __BRACK_C__ * | _callfn * **/ - private _block(): ParserNode { + private async _block(): Promise { if (this._lookahead?.tag === TokenTag.BRACK_O) { const envId = `BLOCK-${Date.now()}`; this._symtable = new SymTable(envId, this._symtable); this._eat(TokenTag.BRACK_O); - const statements = this._statements(TokenTag.BRACK_C); + const statements = await this._statements(TokenTag.BRACK_C); this._eat(TokenTag.BRACK_C); this._symtable.previous?.mergeRefs(this._symtable); @@ -400,26 +444,26 @@ export default class Parser { return new BlockStmtNode(statements); } - return this._log(); + return await this._log(); } /** * _if -> __IF__ _callfn _block * **/ - private _if(): ParserNode { + private async _if(): Promise { this._eat(TokenTag.IF); - const log = this._log(); - const block = this._block(); + const log = await this._log(); + const block = await this._block(); return new IfStmtNode(log, block); } /** * _ass -> __ASS__ _callfn * **/ - private _ass(): ParserNode { + private async _ass(): Promise { if (this._lookahead?.tag === TokenTag.ASSIGN) { const ass = this._eat(TokenTag.ASSIGN); - const callfn = this._log(); + const callfn = await this._log(); this._symtable?.set(ass.value); return new AssignStmtNode(ass.value, callfn); } @@ -430,7 +474,7 @@ export default class Parser { /** * _declfunc -> __DECL_FN__ __PAREN_O__ _arity __PAREN_C__ _block * **/ - private _declfunc(): ParserNode { + private async _declfunc(): Promise { const func = this._eat(TokenTag.DECL_FN); this._eat(TokenTag.PAREN_O); const arity = this._arity(); @@ -444,17 +488,17 @@ export default class Parser { arity.params.forEach((param) => { this._symtable?.set(param); }); - const block = this._block(); + const block = await this._block(); return new DeclFuncStmtNode(func.value, desc, arity, block); } /** * _print -> __CALL_PRINT__ __PAREN_O__ _log __PAREN_C__ * **/ - private _print(): ParserNode { + private async _print(): Promise { this._eat(TokenTag.CALL_PRINT); this._eat(TokenTag.PAREN_O); - const log = this._log(); + const log = await this._log(); this._eat(TokenTag.PAREN_C); return new CallPrintStmtNode(log); } @@ -462,10 +506,10 @@ export default class Parser { /** * _arg -> __CALL_ARG__ __PAREN_O__ _log __PAREN_C__ * **/ - private _arg(): ParserNode { + private async _arg(): Promise { this._eat(TokenTag.CALL_ARG); this._eat(TokenTag.PAREN_O); - const term = this._term(); + const term = await this._term(); this._eat(TokenTag.PAREN_C); return new CallArgStmtNode(term); } @@ -473,16 +517,16 @@ export default class Parser { /** * _arg -> __CALL_ARG__ __PAREN_O__ _log __PAREN_C__ * **/ - private _concat(): ParserNode { + private async _concat(): Promise { this._eat(TokenTag.CALL_CONCAT); this._eat(TokenTag.PAREN_O); - const params: ParserNode[] = [this._term()]; + const params: ParserNode[] = [await this._term()]; // @ts-ignore while (this._lookahead.tag === TokenTag.COMMA) { this._eat(TokenTag.COMMA); - params.push(this._term()); + params.push(await this._term()); } this._eat(TokenTag.PAREN_C); @@ -493,9 +537,9 @@ export default class Parser { /** * _return -> __RETURN__ _log * **/ - private _return(): ParserNode { + private async _return(): Promise { this._eat(TokenTag.RETURN); - const log = this._log(); + const log = await this._log(); return new ReturnStmtNode(log); } @@ -509,7 +553,7 @@ export default class Parser { * | _return * | __RETURN_VOID__ * **/ - private _statement(): ParserNode { + private async _statement(): Promise { if (this._lookahead?.tag === TokenTag.RETURN_VOID) { this._eat(TokenTag.RETURN_VOID); return new ReturnVoidStmtNode(); @@ -538,25 +582,32 @@ export default class Parser { return this._block(); } - private _statements(eot: TokenTag): ParserNode[] { + private async _statements(eot: TokenTag): Promise { const list = []; while (this._lookahead?.tag !== eot) { - list.push(this._statement()); + list.push(await this._statement()); } return list; } - private _program(): ProgramNode { - const program = new ProgramNode(this._statements(TokenTag.EOF)); - + private async _program(): Promise { + const program = new ProgramNode(await this._statements(TokenTag.EOF)); return program; } - public parse(): ProgramNode { + public async parse(lexer: Lexer): Promise { + this._lexer = lexer; this._lookahead = this._lexer.getNextToken(); - return this._program(); + const program = await this._program(); + + this._lexer = lexer.previous; + if (!!this._lexer) { + this._lookahead = this._lexer.getCurrentToken(); + } + + return program; } } diff --git a/src/parser/tag.ts b/src/parser/tag.ts index 2da88c1..af267f9 100644 --- a/src/parser/tag.ts +++ b/src/parser/tag.ts @@ -23,6 +23,9 @@ export enum ParserNodeTag { CALL_MAP_STMT = "CallMapStatement", CALL_FILTER_STMT = "CallFilterStatement", CALL_FUNC_STMT = "CallFuncStatement", + FROM_STMT = "FromStatement", + AS_STMT = "AsStatement", + IMPORT_STMT = "ImportStatement", DESC_FUNC_STMT = "DescFuncStatement", PROGRAM = "Program", }