WIP, add custom MLIR type and step op

README.md

@@ -11,7 +11,9 @@ While working on [REANA](https://github.com/reanahub/reana) project at CERN, I n
 Please make sure to build LLVM and MLIR projects first according to [the instruction](https://mlir.llvm.org/getting_started/). For this particular project those commands were used to build LLVM and MLIR project (provided as an example):
 
 ```sh
-cmake -G Ninja ../llvm \
+$ mkdir build && cd build
+
+$ cmake -G Ninja ../llvm \
    -DLLVM_ENABLE_PROJECTS=mlir \
    -DLLVM_BUILD_EXAMPLES=ON \
    -DLLVM_TARGETS_TO_BUILD="host" \
@@ -23,7 +25,8 @@ cmake -G Ninja ../llvm \
    -DMLIR_INCLUDE_INTEGRATION_TESTS=ON \
    -DLLVM_INSTALL_UTILS=ON \
    -DLLVM_INCLUDE_TOOLS=ON
-cmake --build . --target check-mlir
+
+$ cmake --build . --target check-mlir
 ```
 
 To build PolyFlow, run the following commands:
@@ -48,4 +51,4 @@ cmake --build . --target mlir-doc
 
 # Credits
 
-- [MLIR Hello World repo](https://github.com/Lewuathe/mlir-hello)
+- [MLIR Hello World repo](https://github.com/Lewuathe/mlir-hello)

example_workflow.snakefile

@@ -0,0 +1,16 @@
+rule task1:
+    input:
+        "inputs/input1.txt",
+        "inputs/input2.txt"
+    output:
+        "./output_task1.txt"
+    shell:
+        "cat input1.txt input2.txt > outputs/task1.txt | echo Done >> outputs/task1.txt"
+
+rule task2:
+    input:
+        "outputs/task1.txt"
+    output:
+        "outputs/task2.txt"
+    shell:
+        "cat outputs/task1.txt >> outputs/task2.txt"

include/Parsers/SnakemakeParser.hpp

@@ -4,6 +4,16 @@
 #include <string>
 #include <vector>
 #include <memory>
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/MLIRContext.h"
+
+#include "PolyFlow/PolyFlowDialect.h"
+#include "PolyFlow/PolyFlowTypes.h"
+#include "PolyFlow/PolyFlowOps.h"
+
+
+namespace snakemake {
 
 enum TokenType {
     RULE,
@@ -26,9 +36,9 @@ struct Token {
     size_t column;
 };
 
-class SnakemakeLexer {
+class Lexer {
     public:
-        SnakemakeLexer(std::istream& input) : input_(input) {}
+        Lexer(std::istream& input) : input_(input) {}
         Token nextToken();
         bool eof() const { return input_.eof(); }
 
@@ -49,57 +59,89 @@ class SnakemakeLexer {
         Token readIdentifierOrKeyword();
 };
 
-class TreeNode {
+class BaseAST {
 public:
-    std::string name;
-    std::vector<std::shared_ptr<TreeNode>> children;
+    enum ASTKind {
+        Rule,
+    };
 
-    TreeNode(const std::string &name) : name(name) {}
+    BaseAST(ASTKind kind, Token location)
+        : kind(kind), location(std::move(location)) {}
+    virtual ~BaseAST() = default;
 
-    void add_child(const std::shared_ptr<TreeNode> &child) {
-        children.push_back(child);
-    }
+    ASTKind getKind() const { return kind; }
 
-    void print(int indent = 0) {
-        for (int i = 0; i < indent; ++i) {
-            std::cout << "  ";
-        }
-        std::cout << name << std::endl;
-        for (const auto &child : children) {
-            child->print(indent + 1);
-        }
-    }
+    const Token &loc() { return location; }
+
+private:
+    const ASTKind kind;
+    Token location;
+};
+
+class RuleAST : public BaseAST {
+    public:
+        std::string name;
+        std::vector<std::string> inputs;
+        std::vector<std::string> outputs;
+        std::string command;
+
+        RuleAST(Token loc,
+                std::string name,
+                std::vector<std::string> inputs,
+                std::vector<std::string> outputs,
+                std::string command)
+            : BaseAST(Rule, std::move(loc)), name(std::move(name)), inputs(std::move(inputs)), outputs(std::move(outputs)), command(std::move(command)) {}
+
+        static bool classof(const BaseAST *c) { return c->getKind() == Rule; }
 };
 
-class SnakemakeParser {
+class ModuleAST {
+    public:
+        std::vector<std::unique_ptr<RuleAST>> rules;
+        ModuleAST(std::vector<std::unique_ptr<RuleAST>> rules)
+            : rules(std::move(rules)) {}
+};
+
+void dumpAST(ModuleAST &);
+
+class Parser {
 public:
-    SnakemakeParser(std::istream& input): lexer_(input) {}
+    Parser(std::istream& input): lexer_(input) {}
 
-    void parse() {
+    std::unique_ptr<ModuleAST> parseModule() {
         advance();
-        snakemake();
-    }
-
-    void print_tree() {
-        root->print();
+        return snakemake();
     }
 
 private:
-    SnakemakeLexer lexer_;
+    Lexer lexer_;
     Token lookahead;
-    std::shared_ptr<TreeNode> root = std::make_shared<TreeNode>("snakemake");
 
     void advance();
 
     void throw_unexpected_token();
 
     void match(const TokenType tokenType);
 
-    std::shared_ptr<TreeNode> snakemake();
-    std::shared_ptr<TreeNode> rule();
+    std::unique_ptr<ModuleAST> snakemake();
+    std::unique_ptr<RuleAST> rule();
 
-    std::shared_ptr<TreeNode> input_rule();
-    std::shared_ptr<TreeNode> output_rule();
-    std::shared_ptr<TreeNode> shell_rule();
-    std::shared_ptr<TreeNode> parameter_list();
+    std::vector<std::string> input_rule();
+    std::vector<std::string> output_rule();
+    std::string shell_rule();
+    std::vector<std::string> parameter_list();
 };
+
+class MLIRGen {
+    public:
+        MLIRGen(mlir::MLIRContext &context) : builder(&context) {}
+        mlir::ModuleOp mlirGen(ModuleAST &);
+
+    private:
+        mlir::ModuleOp theModule;
+        mlir::OpBuilder builder;
+
+        polyflow::StepOp mlirGen(RuleAST &);
+};
+
+} // namespace snakemake

include/PolyFlow/PolyFlowDialect.h

@@ -2,11 +2,6 @@
 #define POLYFLOW_POLYFLOWDIALECT_H
 
 #include "mlir/IR/Dialect.h"
-#include "mlir/IR/BuiltinOps.h"
-#include "mlir/IR/BuiltinDialect.h"
-#include "mlir/IR/BuiltinTypes.h"
-#include "mlir/IR/Dialect.h"
-#include "mlir/Interfaces/SideEffectInterfaces.h"
 
 #include "PolyFlow/PolyFlowOpsDialect.h.inc"
 

include/PolyFlow/PolyFlowDialect.td

@@ -15,6 +15,7 @@ def PolyFlow_Dialect : Dialect {
     }];
     let cppNamespace = "::polyflow";
     let hasConstantMaterializer = 0;
+    let useDefaultTypePrinterParser = 1;
 }
 
 //===----------------------------------------------------------------------===//

include/PolyFlow/PolyFlowOps.h

@@ -4,8 +4,11 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpDefinition.h"
+#include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 
+#include "PolyFlow/PolyFlowTypes.h"
+
 #define GET_OP_CLASSES
 #include "PolyFlow/PolyFlowOps.h.inc"
 

include/PolyFlow/PolyFlowOps.td

@@ -2,16 +2,33 @@
 #define POLYFLOW_OPS
 
 include "PolyFlowDialect.td"
+include "PolyFlowTypes.td"
+include "mlir/Interfaces/InferTypeOpInterface.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
 
-def ConstantOp : PolyFlow_Op<"constant", [Pure]> {
-  let summary = "constant";
-  let description = "simple constant operation";
 
-  let arguments = (ins F64Attr:$value);
-  let results = (outs F64:$result);
+def FileLocation : PolyFlow_Op<"FileLocation", [Pure]> {
+  let summary = "An input value to a file.";
+  let description = "A single step in the workflow execution.";
 
-  let assemblyFormat = "attr-dict `(` $value `)` `:` type($result)";
+  let arguments = (ins StrAttr:$value);
+
+  let results = (outs PolyFlow_StringType:$result);
+  let assemblyFormat = "attr-dict `:` type($result)";
+}
+
+def StepOp : PolyFlow_Op<"Step", []> {
+  let summary = "A single step in the workflow execution.";
+  let description = "A single step in the workflow execution.";
+
+  let arguments = (ins 
+    StrAttr:$name,
+    Variadic<PolyFlow_StringType>:$inputs
+  );
+
+  let regions = (region AnyRegion:$body);
+
+  let assemblyFormat = "$name `(` $inputs `)` attr-dict `(` type($inputs) `)` `:` $body";
 }
 
 #endif // POLYFLOW_OPS

include/PolyFlow/PolyFlowTypes.h

@@ -0,0 +1,9 @@
+#ifndef POLYFLOW_POLYFLOWTYPES_H
+#define POLYFLOW_POLYFLOWTYPES_H
+
+#include "mlir/IR/BuiltinTypes.h"
+
+#define GET_TYPEDEF_CLASSES
+#include "PolyFlow/PolyFlowOpsTypes.h.inc"
+
+#endif // POLYFLOW_POLYFLOWTYPES_H

include/PolyFlow/PolyFlowTypes.td

@@ -0,0 +1,21 @@
+#ifndef STANDALONE_TYPES
+#define STANDALONE_TYPES
+
+include "mlir/IR/AttrTypeBase.td"
+include "PolyFlowDialect.td"
+
+
+class PolyFlow_Type<string name, string typeMnemonic, list<Trait> traits = []>
+    : TypeDef<PolyFlow_Dialect, name, traits> {
+  let mnemonic = typeMnemonic;
+}
+
+
+def PolyFlow_StringType : PolyFlow_Type<"Polyflow_Str", "polyflow_str"> {
+  let summary = "A string";
+  let description = "Custom type in standalone dialect";
+  //let parameters = (ins StringRefParameter<"the custom value">:$value);
+  //let assemblyFormat = "`<` $value `>`";
+}
+
+#endif