🏗️ make interpreter more testable.

doc/grammar.bnf

@@ -1,7 +1,10 @@
-DOC ::= RULE*
+DOC ::= (VAR_DECL | RULE)*
 RULE ::= TARGET rarrow DEPS BLOCK
-TARGET ::= ident+
-DEPS ::= ident*
+TARGET ::= LITERAL+
+DEPS ::= LITERAL*
 BLOCK ::= obrace CMD_LST cbrace
 CMD_LST ::= (CMD (comma CMD)* comma?)?
-CMD ::= ident*
+CMD ::= LITERAL*
+LITERAL ::= ident | var
+
+VAR_DECL ::= var assign ident

meson.build

@@ -22,6 +22,9 @@ snake_lib = static_library('snake',
                              'src/Interpreter.cpp',
                              'src/State.cpp',
                              'src/DAG.cpp',
+                             'src/SymTable.cpp',
+                             'src/Loader.cpp',
+                             'src/Executor.cpp',
                            ],
                            dependencies: [
                              dependency('sqlite3')
@@ -45,6 +48,7 @@ executable('snake-tests',
              'tests/main.cpp',
              'tests/Lexer.cpp',
              'tests/Parser.cpp',
+             'tests/Interpreter.cpp',
            ],
            dependencies: [
              snake_dep,

src/Executor.cpp

@@ -0,0 +1,30 @@
+#include "Executor.hpp"
+
+namespace sn
+{
+  /*explicit*/ Executor::Executor()
+  {
+  }
+
+  /*virtual*/ Executor::~Executor()
+  {
+  }
+
+  /*virtual*/ std::string Executor::execute(std::string const& command)
+  {
+    std::stringstream ss;
+
+    FILE* out = popen(command.c_str(), "r");
+    assert(out);
+
+    char buf;
+
+    while ( fread(&buf, sizeof(char), 1, out) )
+      {
+        ss << buf;
+      }
+
+    pclose(out);
+    return ss.str();
+  }
+}

src/Executor.hpp

@@ -0,0 +1,20 @@
+#ifndef sn_EXECUTOR_HPP
+#define sn_EXECUTOR_HPP
+
+#include "commons.hpp"
+
+namespace sn
+{
+  class Executor
+  {
+  public:
+    explicit Executor();
+    virtual ~Executor();
+
+    virtual std::string execute(std::string const& command);
+
+  private:
+  };
+}
+
+#endif

src/Interpreter.cpp

@@ -5,7 +5,12 @@
 
 namespace sn
 {
-  /*explicit*/ Interpreter::Interpreter()
+  /*explicit*/ Interpreter::Interpreter(std::shared_ptr<State> state,
+                                        std::shared_ptr<Loader> loader,
+                                        std::shared_ptr<Executor> executor)
+    : m_state  { state }
+    , m_loader { loader }
+    , m_executor { executor }
   {
   }
 
@@ -17,7 +22,7 @@ namespace sn
   {
     // Process the document ast
     // ------------------------
-    auto node = parse_snakefile();
+    auto node = m_loader->parse_snakefile();
     process(node);
 
     // Get all files
@@ -47,11 +52,9 @@ namespace sn
 
     // Get modified files
     // ------------------
-    auto state = std::make_shared<State>();
+    m_state->init(files);
 
-    state->init(files);
-
-    auto modified_files = state->get_modified_files(files);
+    auto modified_files = m_state->get_modified_files(files);
 
     // Build DAG
     // ---------
@@ -120,7 +123,7 @@ namespace sn
               }
           }
 
-        state->update(file);
+        m_state->update(file);
       }
 
     if (sorted.empty())
@@ -129,36 +132,21 @@ namespace sn
       }
   }
 
-  std::filesystem::path Interpreter::find_snakefile()
+  std::string Interpreter::get_value(std::shared_ptr<Node> node)
   {
-    auto path = std::filesystem::current_path() / "Snakefile";
-
-    if (!std::filesystem::exists(path))
+    if (node->type() == NODE_IDENT)
       {
-        throw run_error {"Snakefile not found"};
+        return node->repr();
+      }
+    else if (node->type() == NODE_VAR)
+      {
+        auto var = *m_sym.get(node->repr());
+        return var.value;
       }
 
-    return path;
-  }
-
-  std::string Interpreter::load_snakefile()
-  {
-    std::ifstream file {find_snakefile()};
-    assert(file);
-    std::stringstream ss;
-    ss << file.rdbuf();
-    return ss.str();
-  }
-
-  std::shared_ptr<Node> Interpreter::parse_snakefile()
-  {
-    Lexer lexer;
-    lexer.scan(load_snakefile());
-
-    Parser parser;
-    auto node = parser.parse(lexer.all());
-
-    return node;
+    throw run_error {"cannot find value for '"
+                     + SN_GET(NodeTypeStr, node->type(), "NODE_")
+                     + "'"};
   }
 
   std::vector<std::filesystem::path>
@@ -191,6 +179,12 @@ namespace sn
           }
       } break;
 
+      case NODE_VAR_DECL: {
+        std::string name = (*node->get(0))->repr();
+        auto n = *node->get(1);
+        m_sym.declare(name, get_value(n));
+      } break;
+
       case NODE_RULE: {
         auto all_targets = *node->get(0);
         auto deps = *node->get(1);
@@ -203,11 +197,11 @@ namespace sn
             for (size_t j=0; j<deps->size(); j++)
               {
                 auto dep = *deps->get(j);
-                auto path = format_path(dep->repr());
+                auto path = format_path(get_value(dep));
 
-                m_dependencies[format_path(target->repr())].push_back(path);
+                m_dependencies[format_path(get_value(target))].push_back(path);
 
-                auto target_path = format_path(target->repr());
+                auto target_path = format_path(get_value(target));
 
                 m_scripts[target_path] = scripts(block);
               }
@@ -237,7 +231,18 @@ namespace sn
 
         for (size_t j=0; j<cmd->size(); j++)
           {
-            script += sep + (*cmd->get(j))->repr();
+            auto c = *cmd->get(j);
+
+            if (c->type() == NODE_IDENT)
+              {
+                script += sep + c->repr();
+              }
+            else
+              {
+                auto var = *m_sym.get(c->repr());
+                script += sep + var.value;
+              }
+
             sep = " ";
           }
 
@@ -249,20 +254,7 @@ namespace sn
 
   std::string Interpreter::execute(std::string const& script) const
   {
-    std::stringstream ss;
-
-    FILE* out = popen(script.c_str(), "r");
-    assert(out);
-
-    char buf;
-
-    while ( fread(&buf, sizeof(char), 1, out) )
-      {
-        ss << buf;
-      }
-
-    pclose(out);
-    return ss.str();
+    return m_executor->execute(script);
   }
 
   std::filesystem::path

src/Interpreter.hpp

@@ -5,6 +5,9 @@
 #include "Node.hpp"
 #include "State.hpp"
 #include "DAG.hpp"
+#include "SymTable.hpp"
+#include "Loader.hpp"
+#include "Executor.hpp"
 
 namespace sn
 {
@@ -18,21 +21,26 @@ namespace sn
   class Interpreter
   {
   public:
-    explicit Interpreter();
+    explicit Interpreter(std::shared_ptr<State> state,
+                         std::shared_ptr<Loader> loader,
+                         std::shared_ptr<Executor> executor);
     virtual ~Interpreter();
 
     void run();
 
-    std::filesystem::path find_snakefile();
-    std::string load_snakefile();
-    std::shared_ptr<Node> parse_snakefile();
-
   private:
+    std::shared_ptr<State> m_state;
+    std::shared_ptr<Loader> m_loader;
+    std::shared_ptr<Executor> m_executor;
+
     std::unordered_map<std::filesystem::path,
                        std::vector<std::filesystem::path>> m_dependencies;
 
     std::unordered_map<std::filesystem::path,
                        std::vector<std::string>> m_scripts;
+    SymTable m_sym;
+
+    std::string get_value(std::shared_ptr<Node> node);
 
     std::vector<std::filesystem::path>
     targets(std::filesystem::path path) const;
@@ -45,7 +53,6 @@ namespace sn
 
     std::filesystem::path
     format_path(std::filesystem::path path) const;
-
   };
 }
 

src/Lexer.cpp

@@ -9,11 +9,13 @@ namespace sn
     m_separators.push_back('\n');
     m_separators.push_back(' ');
 
+    add_text("=", NODE_ASSIGN);
     add_text("->", NODE_RARROW);
     add_text("{", NODE_OBRACE);
     add_text("}", NODE_CBRACE);
     add_text(",", NODE_COMMA);
 
+    m_scanners.push_back(std::bind(&Lexer::scan_var, this));
     m_scanners.push_back(std::bind(&Lexer::scan_ident, this));
   }
 
@@ -151,4 +153,39 @@ namespace sn
 
     return std::nullopt;
   }
+
+  std::optional<ScanInfo> Lexer::scan_var()
+  {
+    size_t cursor = m_cursor;
+    std::string repr;
+
+    if (cursor < m_source.size()
+           && m_source[cursor] == '$')
+      {
+        repr += '$';
+        cursor++;
+      }
+    else
+      {
+        return std::nullopt;
+      }
+
+    while (cursor < m_source.size()
+           && !is_separator(m_source[cursor]))
+      {
+        repr += m_source[cursor];
+        cursor++;
+      }
+
+    if (repr.empty() == false)
+      {
+        return ScanInfo {
+          cursor,
+          NODE_VAR,
+          repr.substr(1)
+        };
+      }
+
+    return std::nullopt;
+  }
 }

src/Lexer.hpp

@@ -56,6 +56,7 @@ namespace sn
                   bool has_repr=false);
 
     std::optional<ScanInfo> scan_ident();
+    std::optional<ScanInfo> scan_var();
   };
 }
 

src/Loader.cpp

@@ -0,0 +1,46 @@
+#include "Loader.hpp"
+#include "Lexer.hpp"
+#include "Parser.hpp"
+
+namespace sn
+{
+  /*explicit*/ Loader::Loader()
+  {
+  }
+
+  /*virtual*/ Loader::~Loader()
+  {
+  }
+
+  std::filesystem::path Loader::find_snakefile()
+  {
+    auto path = std::filesystem::current_path() / "Snakefile";
+
+    if (!std::filesystem::exists(path))
+      {
+        throw load_error {"Snakefile not found"};
+      }
+
+    return path;
+  }
+
+  std::string Loader::load_snakefile()
+  {
+    std::ifstream file {find_snakefile()};
+    assert(file);
+    std::stringstream ss;
+    ss << file.rdbuf();
+    return ss.str();
+  }
+
+  std::shared_ptr<Node> Loader::parse_snakefile()
+  {
+    Lexer lexer;
+    lexer.scan(load_snakefile());
+
+    Parser parser;
+    auto node = parser.parse(lexer.all());
+
+    return node;
+  }
+}

src/Loader.hpp

@@ -0,0 +1,30 @@
+#ifndef sn_LOADER_HPP
+#define sn_LOADER_HPP
+
+#include "commons.hpp"
+#include "Node.hpp"
+
+namespace sn
+{
+  SN_ERROR(load_error);
+
+  /**
+   * Find and compile snakefiles.
+   * @see Lexer
+   * @see Parser
+   **/
+  class Loader
+  {
+  public:
+    explicit Loader();
+    virtual ~Loader();
+
+    virtual std::filesystem::path find_snakefile();
+    virtual std::string load_snakefile();
+    virtual std::shared_ptr<Node> parse_snakefile();
+
+  private:
+  };
+}
+
+#endif

src/Node.hpp

@@ -6,7 +6,8 @@
 #define NODE_TYPES(G)                                                   \
   G(NODE_DOC), G(NODE_IDENT), G(NODE_RARROW), G(NODE_OBRACE),           \
     G(NODE_CBRACE), G(NODE_CMD), G(NODE_CMD_LST), G(NODE_COMMA),        \
-    G(NODE_BLOCK), G(NODE_TARGET), G(NODE_DEPS), G(NODE_RULE)
+    G(NODE_BLOCK), G(NODE_TARGET), G(NODE_DEPS), G(NODE_RULE), \
+    G(NODE_VAR), G(NODE_ASSIGN), G(NODE_VAR_DECL)
 
 namespace sn
 {

src/Parser.cpp

@@ -66,7 +66,16 @@ namespace sn
 
     while (m_cursor < m_tokens.size())
       {
-        node->add_child(parse_rule());
+        if (type_is(NODE_VAR)
+            && type_is(NODE_ASSIGN, 1))
+          {
+            node->add_child(parse_var_decl());
+          }
+        else
+          {
+            node->add_child(parse_rule());
+          }
+
       }
 
     return node;
@@ -91,11 +100,11 @@ namespace sn
   {
     auto node = std::make_shared<Node>(NODE_TARGET);
 
-    node->add_child(*consume(NODE_IDENT));
+    node->add_child(parse_literal());
 
     while (!type_is(NODE_RARROW))
       {
-        node->add_child(*consume(NODE_IDENT));
+        node->add_child(parse_literal());
       }
 
     consume();
@@ -109,7 +118,7 @@ namespace sn
 
     while (!type_is(NODE_OBRACE))
       {
-        node->add_child(*consume(NODE_IDENT));
+        node->add_child(parse_literal());
       }
 
     return node;
@@ -163,9 +172,34 @@ namespace sn
 
     while (!type_is(NODE_CBRACE) && !type_is(NODE_COMMA))
       {
-        node->add_child(*consume(NODE_IDENT));
+        node->add_child(parse_literal());
       }
 
     return node;
   }
+
+  std::shared_ptr<Node> Parser::parse_literal()
+  {
+    if (type_is(NODE_IDENT)
+        || type_is(NODE_VAR))
+      {
+        return *consume();
+      }
+
+    throw syntax_error {"unknown token '"
+                        + SN_GET(NodeTypeStr,
+                                 m_tokens[m_cursor]->type(),
+                                 "NODE_")
+                        + "'"};
+  }
+
+  std::shared_ptr<Node> Parser::parse_var_decl()
+  {
+    auto node = std::make_shared<Node>(NODE_VAR_DECL);
+    node->add_child(*consume(NODE_VAR));
+    consume(NODE_ASSIGN);
+    node->add_child(*consume(NODE_IDENT));
+
+    return node;
+  }
 }

src/Parser.hpp

@@ -37,6 +37,9 @@ namespace sn
     std::shared_ptr<Node> parse_block();
     std::shared_ptr<Node> parse_cmd_lst();
     std::shared_ptr<Node> parse_cmd();
+
+    std::shared_ptr<Node> parse_literal();
+    std::shared_ptr<Node> parse_var_decl();
   };
 }
 

src/State.hpp

@@ -22,10 +22,10 @@ namespace sn
     explicit State();
     virtual ~State();
 
-    void init(std::vector<std::filesystem::path> const& files);
-    void update(std::filesystem::path path);
+    virtual void init(std::vector<std::filesystem::path> const& files);
+    virtual void update(std::filesystem::path path);
 
-    std::vector<std::filesystem::path>
+    virtual std::vector<std::filesystem::path>
     get_modified_files(std::vector<std::filesystem::path> const& files);
 
     std::filesystem::path format_path(std::filesystem::path path) const;

src/SymTable.cpp

@@ -0,0 +1,29 @@
+#include "SymTable.hpp"
+
+namespace sn
+{
+  /*explicit*/ SymTable::SymTable()
+  {
+  }
+
+  /*virtual*/ SymTable::~SymTable()
+  {
+  }
+
+  void SymTable::declare(std::string const& name, std::string const& value)
+  {
+    m_vars[name] = Var { value };
+  }
+
+  std::optional<Var> SymTable::get(std::string const& name) const
+  {
+    if (auto var=m_vars.find(name);
+        var != std::end(m_vars))
+      {
+        return var->second;
+      }
+
+    return std::nullopt;
+  }
+
+}

src/SymTable.hpp

@@ -0,0 +1,34 @@
+#ifndef sn_SYMTABLE_HPP
+#define sn_SYMTABLE_HPP
+
+#include "commons.hpp"
+#include "Node.hpp"
+
+namespace sn
+{
+  /**
+   *  A variable of the snakefile.
+   **/
+  struct Var {
+    std::string value;
+  };
+
+  /**
+   * A repository to declare and get variables.
+   * @see Var
+   **/
+  class SymTable
+  {
+  public:
+    explicit SymTable();
+    virtual ~SymTable();
+
+    void declare(std::string const& name, std::string const& value);
+    std::optional<Var> get(std::string const& name) const;
+
+  private:
+    std::unordered_map<std::string, Var> m_vars;
+  };
+}
+
+#endif

src/main.cpp

@@ -3,10 +3,15 @@
 #include "snake.hpp"
 #include "Interpreter.hpp"
 #include "State.hpp"
+#include "Executor.hpp"
 
 int main(int, char**)
 {
-  sn::Interpreter interpreter;
+  auto state = std::make_shared<sn::State>();
+  auto loader = std::make_shared<sn::Loader>();
+  auto executor = std::make_shared<sn::Executor>();
+
+  sn::Interpreter interpreter {state, loader, executor};
   interpreter.run();
 
   return 0;

tests/Interpreter.cpp

@@ -0,0 +1,103 @@
+#include <catch2/catch.hpp>
+#include "../src/Interpreter.hpp"
+
+#define TEST_RUN(SRC, ...) \
+  test_run(SRC, {__VA_ARGS__})
+
+using namespace sn;
+
+struct StateMock: public State {
+  std::vector<std::filesystem::path> modified_files;
+
+  StateMock(std::vector<std::filesystem::path> const& p_modified_files)
+    : modified_files { p_modified_files }
+  {
+  }
+
+  virtual void init(std::vector<std::filesystem::path> const&) override
+  {
+  }
+
+  virtual void update(std::filesystem::path) override
+  {
+  }
+
+  virtual std::vector<std::filesystem::path>
+  get_modified_files(std::vector<std::filesystem::path> const&) override
+  {
+    return modified_files;
+  }
+};
+
+struct LoaderMock: public Loader {
+  std::string snakefile;
+
+  LoaderMock(std::string const& p_snakefile)
+    : snakefile { p_snakefile }
+  {
+  }
+
+  virtual std::filesystem::path find_snakefile() override { return ""; }
+
+  virtual std::string load_snakefile() override
+  {
+    return snakefile;
+  }
+};
+
+struct ExecutorMock: public Executor {
+  std::vector<std::string> history;
+
+  virtual std::string execute(std::string const& command) override
+  {
+    history.push_back(command);
+    return "";
+  }
+};
+
+class InterpreterTest
+{
+public:
+  explicit InterpreterTest() {}
+  virtual ~InterpreterTest() {}
+
+  std::vector<std::string>
+  test_run(std::string const& snakefile,
+           std::vector<std::filesystem::path> const& modified)
+  {
+    auto state = std::make_shared<StateMock>(modified);
+
+    auto loader = std::make_shared<LoaderMock>(snakefile);
+    auto executor = std::make_shared<ExecutorMock>();
+
+    Interpreter interpreter {state, loader, executor};
+    interpreter.run();
+
+    return executor->history;
+  }
+
+  void test_contains(std::string const& str, std::vector<std::string> vec)
+  {
+    INFO("'" << str << "' not found");
+    REQUIRE(std::find(std::begin(vec), std::end(vec), str)
+            != std::end(vec));
+  }
+
+protected:
+};
+
+TEST_CASE_METHOD(InterpreterTest, "Interpreter_simple_var")
+{
+  auto modified = std::vector<std::filesystem::path> {
+    std::filesystem::path("a"),
+  };
+
+  std::stringstream ss;
+  ss << "$X = a" << std::endl;
+  ss << "b -> $X {" << std::endl;
+  ss << "executed" << std::endl;
+  ss << "}" << std::endl;
+
+  auto res = TEST_RUN(ss.str(), std::filesystem::path("a"));
+  test_contains("executed", res);
+}

tests/Lexer.cpp

@@ -41,3 +41,13 @@ TEST_CASE_METHOD(LexerTest, "Lexer_rule")
   test_next("CBRACE");
   test_end();
 }
+
+TEST_CASE_METHOD(LexerTest, "Lexer_vars")
+{
+  m_lexer.scan("$hello $WORLD = ");
+
+  test_next("VAR[hello]");
+  test_next("VAR[WORLD]");
+  test_next("ASSIGN");
+  test_end();
+}

tests/Parser.cpp

@@ -46,3 +46,31 @@ TEST_CASE_METHOD(ParserTest, "Parser_rule")
              ")))",
              ss.str());
 }
+
+TEST_CASE_METHOD(ParserTest, "Parser_var_decl_and_use")
+{
+  std::stringstream ss;
+  ss << "$BIM = hello.cpp" << std::endl;
+  ss << "$BAM = world.hpp" << std::endl;
+  ss << " $BIM -> hello.cpp {" << std::endl;
+  ss << " g++ $BIM -o hello.elf, " << std::endl;
+  ss << " ls " << std::endl;
+  ss << " }" << std::endl;
+
+  test_parse("DOC("
+             "VAR_DECL(VAR[BIM],IDENT[hello.cpp]),"
+             "VAR_DECL(VAR[BAM],IDENT[world.hpp]),"
+             "RULE("
+             "TARGET("
+             "VAR[BIM]"
+             "),DEPS("
+             "IDENT[hello.cpp]"
+             "),BLOCK("
+             "CMD("
+             "IDENT[g++],VAR[BIM],IDENT[-o],IDENT[hello.elf]"
+             "),CMD("
+             "IDENT[ls]"
+             ")"
+             ")))",
+             ss.str());
+}