From 6df8167669617a7db029ccaeb19e81414d4e5c1e Mon Sep 17 00:00:00 2001 From: bog Date: Sat, 23 Sep 2023 19:21:55 +0200 Subject: [PATCH] ADD: import modules. --- doc/grammar.bnf | 9 ++- examples/mod.fk | 4 ++ examples/mod2.fk | 2 + libstd/fun.hpp | 2 - libstd/lib.cpp | 4 -- src/Compiler.cpp | 114 +++++++++++++++++++++++++++++++++++-- src/Compiler.hpp | 14 ++++- src/Lexer.cpp | 13 ++++- src/Module.cpp | 24 ++++++++ src/Module.hpp | 10 +++- src/Node.hpp | 3 +- src/Parser.cpp | 95 +++++++++++++++++++++++++++++-- src/Parser.hpp | 4 ++ src/VM.cpp | 144 +++++++++++++++++++++++++++++++++++++---------- src/VM.hpp | 1 + src/opcodes.hpp | 9 +-- tests/Lexer.cpp | 12 ++++ tests/Parser.cpp | 29 +++++++++- 18 files changed, 435 insertions(+), 58 deletions(-) create mode 100644 examples/mod.fk create mode 100644 examples/mod2.fk diff --git a/doc/grammar.bnf b/doc/grammar.bnf index aec8570..427346d 100644 --- a/doc/grammar.bnf +++ b/doc/grammar.bnf @@ -5,9 +5,16 @@ EXPR ::= | CALL | LAMBDA | FUNDECL +| VARDECL +| NS +| IMPORT CALL ::= opar EXPR EXPR* cpar LAMBDA ::= opar rarrow opar PARAMS cpar BODY cpar PARAMS ::= ident* BODY ::= EXPR* FUNDECL ::= -| opar ident opar ident PARAMS cpar BODY cpar +| opar decl opar ident PARAMS cpar BODY cpar +VARDECL ::= opar decl ident EXPR cpar +NS ::= ident ns ident +IMPORT ::= opar import string cpar +SHORT_IMPORT ::= opar decl import ident string? cpar diff --git a/examples/mod.fk b/examples/mod.fk new file mode 100644 index 0000000..2a929d8 --- /dev/null +++ b/examples/mod.fk @@ -0,0 +1,4 @@ +($ a (@ './mod2')) + +(assert= 42 a::var) +(assert= 6 (a::fun 3)) \ No newline at end of file diff --git a/examples/mod2.fk b/examples/mod2.fk new file mode 100644 index 0000000..ab640bd --- /dev/null +++ b/examples/mod2.fk @@ -0,0 +1,2 @@ +($ var 42) +($ (fun x) (* 2 x)) \ No newline at end of file diff --git a/libstd/fun.hpp b/libstd/fun.hpp index 776ac77..9ff3b7d 100644 --- a/libstd/fun.hpp +++ b/libstd/fun.hpp @@ -4,8 +4,6 @@ namespace fkstd { - STDRET set(Loc loc, Module& mod, STDARGS args); - STDRET assert_eq(Loc loc, Module& mod, STDARGS args); STDRET println(Loc loc, Module& mod, STDARGS args); diff --git a/libstd/lib.cpp b/libstd/lib.cpp index b541584..1a42902 100644 --- a/libstd/lib.cpp +++ b/libstd/lib.cpp @@ -2,11 +2,8 @@ #include "fun.hpp" #include "macro.hpp" -Module* _module; - extern "C" void lib(Module& mod) { - _module = &mod; mod.register_function("assert=", fkstd::assert_eq); mod.register_function("println", fkstd::println); mod.register_function("+", fkstd::add_int); @@ -25,6 +22,5 @@ extern "C" void lib(Module& mod) mod.register_macro("!", fkstd::set_addr); mod.register_macro("assert-static-fail", fkstd::assert_static_fail); mod.register_macro(":", fkstd::block); - mod.register_macro("$", fkstd::decl); mod.register_macro("if", fkstd::if_macro); } diff --git a/src/Compiler.cpp b/src/Compiler.cpp index 98362d9..df01bb5 100644 --- a/src/Compiler.cpp +++ b/src/Compiler.cpp @@ -1,10 +1,14 @@ #include "Compiler.hpp" +#include "Module.hpp" #include "Lambda.hpp" +#include "commons.hpp" namespace fk { - /*explicit*/ Compiler::Compiler(std::shared_ptr sym) - : m_sym { sym } + /*explicit*/ Compiler::Compiler(Module& mod, + std::shared_ptr sym) + : m_mod { mod } + , m_sym { sym } { } @@ -18,6 +22,24 @@ namespace fk m_macros[name] = macro; } + std::shared_ptr Compiler::get_macro(std::string const& name) + { + assert(m_macros.find(name) != std::end(m_macros)); + return m_macros[name]; + } + + bool Compiler::has_macro(std::string const& name) const + { + return m_macros.find(name) != std::end(m_macros); + } + + std::unordered_map> + Compiler::macros() + { + return m_macros; + } + + std::shared_ptr Compiler::compile(std::shared_ptr node) { auto prog = std::make_shared(); @@ -50,6 +72,73 @@ namespace fk } } break; + case NODE_VARDECL: { + m_addr++; + + std::string ident = node->child(0)->repr(); + auto rhs = node->child(1); + + push_decl(ident); + compile_prog(rhs, prog); + pop_decl(); + + auto entry = sym()->declare_local(ident, + m_addr, + node->loc()) + .set_node(rhs); + + prog->add(OP_STORE_LOCAL, m_addr); + + } break; + + case NODE_NS: { + auto mod = node->child(0); + auto var = node->child(1); + + if (has_macro(var->repr())) + { + return; + } + + prog->load_const(std::make_shared(TYPE_STRING, + var->repr(), + node->loc())); + + compile_prog(mod, prog); + + prog->add(OP_LOAD_MOD); + + } break; + + case NODE_IMPORT: { + std::string val = node->child(0)->repr(); + + std::string path_str = val.substr(1, val.size() - 2); + std::filesystem::path path = path_str; + + if (path_str[0] == '.') + { + if (!path.has_extension()) + { + path += ".fk"; + } + + auto res = std::make_shared + (TYPE_STRING, path.string(), node->loc()); + prog->load_const(res); + prog->add(OP_IMPORT); + } + else + { + auto res = std::make_shared + (TYPE_STRING, path.string(), + node->loc()); + + prog->load_const(res); + prog->add(OP_IMPORT_SYS); + } + } break; + case NODE_LAMBDA: { auto params = node->child(0); auto body = node->child(1); @@ -70,7 +159,7 @@ namespace fk m_sym->declare_local(func_name, params->size(), node->loc()); - Compiler compiler {m_sym}; + Compiler compiler {m_mod, m_sym}; for (auto e: m_macros) { compiler.add_macro(e.first, e.second); @@ -92,13 +181,24 @@ namespace fk case NODE_CALL: { std::string ident = node->child(0)->repr(); - if (auto itr=m_macros.find(ident); itr != std::end(m_macros)) { auto macro = itr->second; macro->call(*this, node, prog); } + else if (node->child(0)->type() == NODE_NS + && has_macro(node->child(0)->child(1)->repr())) + { + std::string name = node->child(0)->child(1)->repr(); + + if (auto itr=m_macros.find(name); + itr != std::end(m_macros)) + { + auto macro = itr->second; + macro->call(*this, node, prog); + } + } else { for (size_t i=0; isize(); i++) @@ -106,8 +206,10 @@ namespace fk compile_prog(node->child(i), prog); } + if (node->child(0)->type() == NODE_LAMBDA - || node->child(0)->type() == NODE_CALL) + || node->child(0)->type() == NODE_CALL + || node->child(0)->type() == NODE_NS) { prog->add(OP_CALL_REF, node->size() - 1); } @@ -137,7 +239,7 @@ namespace fk } else { - prog->add(OP_CALL_NATIVE, node->size() - 1); + prog->add(OP_CALL_REF, node->size() - 1); } } } diff --git a/src/Compiler.hpp b/src/Compiler.hpp index c5cf4ec..84e45e2 100644 --- a/src/Compiler.hpp +++ b/src/Compiler.hpp @@ -15,10 +15,12 @@ namespace fk size_t arity; }; + class Module; + class Compiler { public: - explicit Compiler(std::shared_ptr sym); + explicit Compiler(Module& mod, std::shared_ptr sym); virtual ~Compiler(); std::shared_ptr sym() const { return m_sym; } @@ -26,6 +28,14 @@ namespace fk void add_macro(std::string const& name, std::shared_ptr macro); + std::shared_ptr get_macro(std::string const& name); + + bool has_macro(std::string const& name) const; + + std::unordered_map> + macros(); + + std::shared_ptr compile(std::shared_ptr node); void compile_prog(std::shared_ptr node, @@ -35,6 +45,8 @@ namespace fk void pop_decl(); private: + Module& m_mod; + addr_t m_addr = 0; std::shared_ptr m_sym; std::vector m_decl_stack; std::unordered_map> text = { + {NODE_IMPORT, "@", false}, + {NODE_DECL, "$", false}, + {NODE_NS, "::", false}, {NODE_RARROW, "->", false}, {NODE_OPAR, "(", false}, {NODE_CPAR, ")", false}, @@ -255,8 +258,16 @@ namespace fk std::string repr; while (cursor < m_source.size() - && !is_sep(cursor)) + && !is_sep(cursor) + && m_source[cursor]) { + if (cursor + 1 < m_source.size() + && m_source[cursor] == ':' + && m_source[cursor + 1] == ':') + { + break; + } + repr += m_source[cursor]; cursor++; } diff --git a/src/Module.cpp b/src/Module.cpp index 686a86c..c99292f 100644 --- a/src/Module.cpp +++ b/src/Module.cpp @@ -15,11 +15,27 @@ namespace fk { } + std::shared_ptr Module::get_mod(std::string const& name) + { + assert(m_native_modules.find(name) != std::end(m_native_modules)); + return m_native_modules[name]; + } + void Module::build() { import_std(); m_source = load_sources(); m_ast = m_parser->parse(m_source); + + // Add natives macros before compilation + for (auto mod: m_native_modules) + { + for (auto e: mod.second->compiler()->macros()) + { + m_compiler->add_macro(e.first, e.second); + } + } + m_program = m_compiler->compile(m_ast); m_vm->mount(m_program); @@ -43,6 +59,14 @@ namespace fk f(*this); } + std::shared_ptr Module::register_module(std::string const& name) + { + auto mod = std::make_shared(name); + m_native_modules[name] = mod; + + return mod; + } + void Module::register_function(std::string const& name, native_t native) { auto fun = std::make_shared(native); diff --git a/src/Module.hpp b/src/Module.hpp index 08a5de5..4363bff 100644 --- a/src/Module.hpp +++ b/src/Module.hpp @@ -18,15 +18,20 @@ namespace fk explicit Module(std::filesystem::path source_path); virtual ~Module(); + std::shared_ptr compiler() const { return m_compiler; } std::shared_ptr program() const { return m_program; } std::shared_ptr sym() const { return m_sym; } std::shared_ptr vm() const { return m_vm; } + std::shared_ptr get_mod(std::string const& name); + void build(); void import_std(); void import_library(std::filesystem::path lib_path); + std::shared_ptr register_module(std::string const& name); + void register_function(std::string const& name, native_t native); void register_macro(std::string const& name, macro_t macro); @@ -37,10 +42,13 @@ namespace fk std::shared_ptr m_lexer = std::make_shared(m_loc); std::shared_ptr m_parser = std::make_shared(*m_lexer); std::shared_ptr m_sym = std::make_shared(); - std::shared_ptr m_compiler = std::make_shared(m_sym); + std::shared_ptr m_compiler = std::make_shared(*this, + m_sym); std::shared_ptr m_ast; std::shared_ptr m_vm = std::make_shared(*this); std::shared_ptr m_program = std::make_shared(); + std::unordered_map> m_native_modules; + std::string load_sources(); }; } diff --git a/src/Node.hpp b/src/Node.hpp index 54b6e20..db712c1 100644 --- a/src/Node.hpp +++ b/src/Node.hpp @@ -7,7 +7,8 @@ #define NODE_TYPES(G) \ G(NODE_MODULE), G(NODE_INT), G(NODE_FLOAT), G(NODE_BOOL), G(NODE_STRING),\ G(NODE_IDENT), G(NODE_OPAR), G(NODE_CPAR), G(NODE_CALL), G(NODE_LAMBDA),\ - G(NODE_RARROW), G(NODE_PARAMS), G(NODE_BODY) + G(NODE_RARROW), G(NODE_PARAMS), G(NODE_BODY), G(NODE_NS), G(NODE_IMPORT),\ + G(NODE_VARDECL), G(NODE_DECL) namespace fk { diff --git a/src/Parser.cpp b/src/Parser.cpp index b21230e..d27ebf7 100644 --- a/src/Parser.cpp +++ b/src/Parser.cpp @@ -1,6 +1,7 @@ #include "Parser.hpp" #include "src/Loc.hpp" #include "src/Node.hpp" +#include namespace fk { @@ -49,6 +50,24 @@ namespace fk std::shared_ptr Parser::parse_expr() { + + if (type_all({NODE_IDENT, NODE_NS})) + { + return parse_ns(); + } + + if (type_all({NODE_OPAR, NODE_DECL, NODE_IMPORT, + NODE_IDENT, NODE_STRING, NODE_CPAR}) + || type_all({NODE_OPAR, NODE_DECL, NODE_IMPORT, + NODE_IDENT, NODE_CPAR})) + { + return parse_short_import(); + } + else if (type_all({NODE_OPAR, NODE_IMPORT})) + { + return parse_import(); + } + if (type_any({ NODE_INT, NODE_FLOAT, @@ -59,11 +78,14 @@ namespace fk return consume(); } - if (type_all({NODE_OPAR, NODE_IDENT, NODE_OPAR, NODE_IDENT}) - && m_tokens[m_cursor + 1]->repr() == "$") + if (type_all({NODE_OPAR, NODE_DECL, NODE_OPAR, NODE_IDENT})) { return parse_fundecl(); } + else if (type_all({NODE_OPAR, NODE_DECL})) + { + return parse_vardecl(); + } else if (type_all({NODE_OPAR, NODE_RARROW})) { return parse_lambda(); @@ -142,7 +164,7 @@ namespace fk std::shared_ptr Parser::parse_fundecl() { consume(NODE_OPAR); - auto dollar = consume(NODE_IDENT); + auto dollar = consume(NODE_DECL); consume(NODE_OPAR); auto ident = consume(NODE_IDENT); @@ -152,8 +174,7 @@ namespace fk auto body = parse_body(); consume(NODE_CPAR); - auto res = make_node(NODE_CALL); - res->add_child(dollar); + auto res = make_node(NODE_VARDECL); res->add_child(ident); auto lambda = make_node(NODE_LAMBDA); @@ -164,6 +185,70 @@ namespace fk return res; } + std::shared_ptr Parser::parse_vardecl() + { + auto node = make_node(NODE_VARDECL); + consume(NODE_OPAR); + consume(NODE_DECL); + + node->add_child(consume(NODE_IDENT)); + node->add_child(parse_expr()); + consume(NODE_CPAR); + + return node; + } + + std::shared_ptr Parser::parse_ns() + { + auto ns = make_node(NODE_NS); + ns->add_child(consume(NODE_IDENT)); + consume(NODE_NS); + ns->add_child(consume(NODE_IDENT)); + return ns; + } + + std::shared_ptr Parser::parse_import() + { + consume(NODE_OPAR); + auto node = consume(NODE_IMPORT); + node->add_child(consume(NODE_STRING)); + consume(NODE_CPAR); + + return node; + } + + std::shared_ptr Parser::parse_short_import() + { + consume(NODE_OPAR); + consume(NODE_DECL); + + auto node_import = consume(NODE_IMPORT); + auto ident = consume(NODE_IDENT); + std::shared_ptr str; + + if (type_is(NODE_CPAR)) + { + str = std::make_shared(NODE_STRING, + "'" + ident->repr() + "'", + ident->loc()); + } + else + { + str = consume(NODE_STRING); + } + + consume(NODE_CPAR); + + auto vardecl = make_node(NODE_VARDECL); + vardecl->add_child(ident); + + auto imp = make_node(NODE_IMPORT); + imp->add_child(str); + vardecl->add_child(imp); + + return vardecl; + } + std::shared_ptr Parser::make_node(NodeType type) { return std::make_shared(type, "", loc()); diff --git a/src/Parser.hpp b/src/Parser.hpp index cd7df9d..abb2f2f 100644 --- a/src/Parser.hpp +++ b/src/Parser.hpp @@ -27,6 +27,10 @@ namespace fk std::shared_ptr parse_params(); std::shared_ptr parse_body(); std::shared_ptr parse_fundecl(); + std::shared_ptr parse_vardecl(); + std::shared_ptr parse_ns(); + std::shared_ptr parse_import(); + std::shared_ptr parse_short_import(); std::shared_ptr make_node(NodeType type); Loc loc() const; diff --git a/src/VM.cpp b/src/VM.cpp index 9dc8521..8cbdb05 100644 --- a/src/VM.cpp +++ b/src/VM.cpp @@ -1,5 +1,7 @@ #include "VM.hpp" -#include "src/opcodes.hpp" +#include "opcodes.hpp" +#include "Module.hpp" +#include "src/NativeFunction.hpp" #include namespace fk @@ -29,6 +31,103 @@ namespace fk switch (instr.opcode) { + case OP_IMPORT: { + auto path_val = frame().program->get_const(pop()); + std::filesystem::path path = + std::get(path_val->value()); + + auto mod = std::make_shared(path); + mod->build(); + + addr_t addr = store_global(mod); + auto res = std::make_shared(TYPE_REF, addr, + path_val->loc()); + push(frame().program->add(res)); + m_pc++; + + } break; + + case OP_IMPORT_SYS: { + auto path_val = frame().program->get_const(pop()); + + std::filesystem::path path = + std::get(path_val->value()); + + auto mod = m_mod.get_mod(path); + + addr_t addr = store_global(mod); + auto res = std::make_shared(TYPE_REF, addr, + path_val->loc()); + push(frame().program->add(res)); + + m_pc++; + } break; + + case OP_LOAD_MOD: { + addr_t mod_addr = pop(); + auto mod_val = frame().program->get_const(mod_addr); + size_t mod_ref = + std::get(mod_val->value()); + + auto mod = std::get> + (load_global(mod_ref)); + + addr_t ident_addr = pop(); + auto ident = + std::get(frame().program-> + get_const(ident_addr)->value()); + + auto entry = mod->sym()->find_global(ident); + + if (mod->compiler()->has_macro(ident)) + { + m_pc++; + break; + } + + if (!entry) + { + std::cerr << "cannot load " << ident << std::endl; + abort(); + } + + if (entry->is_global()) + { + auto fun = std::get> + (mod->vm()->m_globals[entry->addr()]); + size_t addr = store_global(fun); + + auto ref = std::make_shared(TYPE_REF, + addr, + Loc {""}); + push(frame().program->add(ref)); + m_pc++; + break; + } + + auto var = mod->vm()->load_local(entry->addr()); + + if (var->type() == TYPE_REF) + { + auto value = mod->vm()->load_global + (std::get(var->value())); + + size_t new_addr = store_global(value); + + push(frame().program->add + (std::make_shared(TYPE_REF, + new_addr, + mod_val->loc()))); + } + else + { + auto value = mod->vm()->load_local(entry->addr()); + push(frame().program->add(value)); + } + + m_pc++; + } break; + case OP_STORE_CLOSURE: { auto val = frame().program->get_const(top()); auto lambda = frame().lambda; @@ -135,6 +234,19 @@ namespace fk addr_t ref = std::get(ref_val->value()); + auto val = load_global(ref); + + if (std::get_if>(&val)) + { + auto fun = std::get> + (load_global(ref)); + + push(frame().program->add(fun->call(ref_val->loc(), + m_mod, args))); + m_pc++; + break; + } + auto lambda = std::get>(load_global(ref)); @@ -159,36 +271,6 @@ namespace fk m_pc = 0; } break; - case OP_CALL_NATIVE: { - std::vector> args; - - for (size_t i=0; iget_const(addr)); - } - - auto ref_val = frame().program->get_const(pop()); - - Loc loc = ref_val->loc(); - - if (instr.param > 0) - { - loc = args.front()->loc(); - } - - auto ref = std::get(ref_val->value()); - - auto fun = std::get> - (load_global(ref)); - - push(frame().program->add(fun->call(loc, m_mod, args))); - - m_pc++; - } break; - case OP_LOAD_LOCAL: { auto value = frame().locals[instr.param]; diff --git a/src/VM.hpp b/src/VM.hpp index 48362f1..27cbe95 100644 --- a/src/VM.hpp +++ b/src/VM.hpp @@ -9,6 +9,7 @@ namespace fk { using global_t = std::variant, + std::shared_ptr, std::shared_ptr >; diff --git a/src/opcodes.hpp b/src/opcodes.hpp index 014b857..ef56a68 100644 --- a/src/opcodes.hpp +++ b/src/opcodes.hpp @@ -1,10 +1,11 @@ #ifndef fk_OPCODES_HPP #define fk_OPCODES_HPP -#define OPCODES(G) G(OP_LOAD_CONST), G(OP_POP), G(OP_CALL_NATIVE), \ - G(OP_LOAD_GLOBAL), G(OP_LOAD_LOCAL), G(OP_STORE_LOCAL), \ - G(OP_MAKE_FUNCTION), G(OP_CALL_REF), G(OP_RET), G(OP_BNE), \ - G(OP_BR), G(OP_LOAD_CLOSURE), G(OP_STORE_CLOSURE) +#define OPCODES(G) G(OP_LOAD_CONST), G(OP_POP), \ + G(OP_LOAD_GLOBAL), G(OP_LOAD_LOCAL), G(OP_STORE_LOCAL), \ + G(OP_MAKE_FUNCTION), G(OP_CALL_REF), G(OP_RET), G(OP_BNE), \ + G(OP_BR), G(OP_LOAD_CLOSURE), G(OP_STORE_CLOSURE), \ + G(OP_LOAD_MOD), G(OP_IMPORT), G(OP_IMPORT_SYS) #include "commons.hpp" diff --git a/tests/Lexer.cpp b/tests/Lexer.cpp index dd98df2..9c995bb 100644 --- a/tests/Lexer.cpp +++ b/tests/Lexer.cpp @@ -97,3 +97,15 @@ TEST_CASE_METHOD(LexerTest, "Lexer_lambda") test_next("CPAR"); test_end(); } + +TEST_CASE_METHOD(LexerTest, "Lexer_namespace") +{ + m_lexer.scan(" (a::x 3) "); + test_next("OPAR"); + test_next("IDENT[a]"); + test_next("NS"); + test_next("IDENT[x]"); + test_next("INT[3]"); + test_next("CPAR"); + test_end(); +} diff --git a/tests/Parser.cpp b/tests/Parser.cpp index f762b42..f767c9d 100644 --- a/tests/Parser.cpp +++ b/tests/Parser.cpp @@ -36,6 +36,12 @@ TEST_CASE_METHOD(ParserTest, "Parser_call") " (bim 2 '3' false) "); } +TEST_CASE_METHOD(ParserTest, "Parser_vardecl") +{ + test_parse("MODULE(VARDECL(IDENT[hello],INT[3]))", + " ($ hello 3) "); +} + TEST_CASE_METHOD(ParserTest, "Parser_lambda") { test_parse("MODULE(LAMBDA(PARAMS,BODY))", @@ -50,7 +56,28 @@ TEST_CASE_METHOD(ParserTest, "Parser_lambda") TEST_CASE_METHOD(ParserTest, "Parser_fundecl") { - test_parse("MODULE(CALL(IDENT[$],IDENT[f]," + test_parse("MODULE(VARDECL(IDENT[f]," "LAMBDA(PARAMS(IDENT[x]),BODY(INT[7]))))", " ($ (f x) 7) "); } + +TEST_CASE_METHOD(ParserTest, "Parser_namespace") +{ + test_parse("MODULE(CALL(NS(IDENT[x],IDENT[y]),IDENT[z]))", + " (x::y z) "); +} + +TEST_CASE_METHOD(ParserTest, "Parser_import") +{ + test_parse("MODULE(VARDECL(IDENT[x],IMPORT(STRING['y'])))", + " ($ x (@ 'y') )"); + + test_parse("MODULE(VARDECL(IDENT[x],IMPORT(STRING['y'])))", + " ($ @x 'y' )"); + + test_parse("MODULE(VARDECL(IDENT[bim],IMPORT(STRING['bam'])))", + " ($ @bim 'bam' )"); + + test_parse("MODULE(VARDECL(IDENT[example],IMPORT(STRING['example'])))", + " ($ @example )"); +}