Merge pull request #19 from gf712/mlir-backend-improvements

Mlir backend improvements

Author: gf712

.github/workflows/premerge.yml

@@ -22,8 +22,8 @@ jobs:
       run: |
         wget https://apt.llvm.org/llvm.sh
         chmod +x llvm.sh
-        sudo ./llvm.sh 20
-        sudo apt install libmlir-20-dev mlir-20-tools
+        sudo ./llvm.sh 23
+        sudo apt install libmlir-23-dev mlir-23-tools
 
     - name: ccache
       uses: hendrikmuhs/ccache-action@v1.2

CMakePresets.json

@@ -20,6 +20,16 @@
         "CMAKE_BUILD_TYPE": "Release",
         "CPM_SOURCE_CACHE": ".cache/CPM"
       }
+    },
+    {
+      "name": "release-with-debug-info",
+      "displayName": "Release with Debug Info",
+      "generator": "Ninja",
+      "binaryDir": "build/release-with-debug-info",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "RelWithDebInfo",
+        "CPM_SOURCE_CACHE": ".cache/CPM"
+      }
     }
   ],
   "buildPresets": [
@@ -34,6 +44,12 @@
       "displayName": "Release Build",
       "configurePreset": "release",
       "configuration": "Release"
+    },
+    {
+      "name": "release-with-debug-info",
+      "displayName": "Release with Debug Info Build",
+      "configurePreset": "release-with-debug-info",
+      "configuration": "RelWithDebInfo"
     }
   ],
   "testPresets": [
@@ -46,6 +62,11 @@
       "name": "release",
       "displayName": "Test all in Release mode",
       "configurePreset": "release"
+    },
+    {
+      "name": "release-with-debug-info",
+      "displayName": "Test all in Release with Debug Info mode",
+      "configurePreset": "release-with-debug-info"
     }
   ]
 

src/CMakeLists.txt

@@ -298,7 +298,7 @@ target_link_libraries(python-cpp
 )
 
 # LLVM backend
-find_package(LLVM CONFIG 20.1)
+find_package(LLVM CONFIG 23)
 if(ENABLE_LLVM_BACKEND AND NOT LLVM_FOUND)
   message(FATAL_ERROR "Could not find LLVM in the local environment")
 elseif(ENABLE_LLVM_BACKEND AND LLVM_FOUND)

src/executable/bytecode/Bytecode.cpp

@@ -132,7 +132,9 @@ py::PyResult<py::Value> Bytecode::eval_loop(VirtualMachine &vm, Interpreter &int
 		ASSERT((*vm.instruction_pointer()).get());
 		const auto &current_ip = vm.instruction_pointer();
 		const auto &instruction = *current_ip;
-		spdlog::debug("{} {}", (void *)instruction.get(), instruction->to_string());
+		// spdlog::debug("{} {}", (void *)instruction.get(), instruction->to_string());
+		// std::cout << std::format("{} {}", (void *)instruction.get(), instruction->to_string())
+		// 		  << std::endl;
 		auto result = instruction->execute(vm, vm.interpreter());
 		// we left the current stack frame in the previous instruction
 		if (vm.stack().size() != stack_depth) {

src/executable/bytecode/instructions/ListToTuple.cpp

@@ -1,8 +1,11 @@
 #include "ListToTuple.hpp"
 #include "runtime/PyList.hpp"
+#include "runtime/PyString.hpp"
 #include "runtime/PyTuple.hpp"
 #include "vm/VM.hpp"
 
+#include <iostream>
+
 using namespace py;
 
 PyResult<Value> ListToTuple::execute(VirtualMachine &vm, Interpreter &) const
@@ -12,6 +15,14 @@ PyResult<Value> ListToTuple::execute(VirtualMachine &vm, Interpreter &) const
 	ASSERT(std::holds_alternative<PyObject *>(list));
 
 	auto *pylist = std::get<PyObject *>(list);
+	if (!as<PyList>(pylist)) {
+		std::cout << to_string() << std::endl;
+		if (!pylist) {
+			std::cout << "(null)" << std::endl;
+		} else {
+			std::cout << pylist->str().unwrap()->to_string() << std::endl;
+		}
+	}
 	ASSERT(as<PyList>(pylist));
 
 	auto result = PyTuple::create(as<PyList>(pylist)->elements());

src/executable/bytecode/instructions/YieldFrom.cpp

@@ -49,7 +49,7 @@ PyResult<Value> YieldFrom::execute(VirtualMachine &vm, Interpreter &interpreter)
 		vm.reg(m_dst) = result.unwrap();
 		vm.reg(0) = result.unwrap();
 		vm.set_instruction_pointer(vm.instruction_pointer() - 1);
-		vm.pop_frame(true);
+		vm.pop_frame(false);
 	}
 
 	return result;

src/executable/bytecode/instructions/YieldValue.cpp

@@ -23,7 +23,7 @@ PyResult<Value> YieldValue::execute(VirtualMachine &vm, Interpreter &interpreter
 
 	vm.reg(0) = result;
 
-	vm.pop_frame(true);
+	vm.pop_frame(false);
 
 	return Ok(result);
 }

src/executable/bytecode/serialization/deserialize.hpp

@@ -5,6 +5,7 @@
 #include "serialize.hpp"
 #include "utilities.hpp"
 
+#include <iostream>
 #include <span>
 #include <string>
 

src/executable/mlir/CMakeLists.txt

@@ -1,4 +1,4 @@
-find_package(LLVM 20.1 REQUIRED CONFIG)
+find_package(LLVM 23 REQUIRED CONFIG)
 message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
 
 find_package(MLIR CONFIG REQUIRED
@@ -20,6 +20,8 @@ set(PYTHON_MLIR_BINARY_DIR ${PROJECT_BINARY_DIR}/src/executable/mlir)
 add_subdirectory(Conversion)
 add_subdirectory(Dialect)
 add_subdirectory(Target)
+add_subdirectory(tools/python-mlir-opt)
+add_subdirectory(test)
 
 add_library(python-mlir compile.cpp)
 target_link_libraries(python-mlir PRIVATE PythonMLIRDialect TargetPythonBytecode PythonConversionPasses)

src/executable/mlir/Conversion/Passes.td

@@ -4,4 +4,30 @@ include "mlir/Pass/PassBase.td"
 def ConvertPythonToPythonBytecode : Pass<"convert-python-to-pythonbytecode"> {
   let summary = "Convert recognized Python ops to PythonCpp bytecode";
   let constructor = "mlir::py::createPythonToPythonBytecodePass()";
+}
+
+def ConvertPyForLoop : Pass<"convert-py-forloop"> {
+  let summary = "Lower py.for_loop to emitpybytecode control flow";
+  let constructor = "mlir::py::createConvertForLoopPass()";
+}
+
+def ConvertPyWhile : Pass<"convert-py-while"> {
+  let summary = "Lower py.while to emitpybytecode control flow";
+  let constructor = "mlir::py::createConvertWhileLoopPass()";
+}
+
+def ConvertPyTry : Pass<"convert-py-try"> {
+  let summary = "Lower py.try (and its handler scopes) to emitpybytecode control flow";
+  let constructor = "mlir::py::createConvertTryPass()";
+}
+
+def ConvertPyWith : Pass<"convert-py-with"> {
+  let summary = "Lower py.with to emitpybytecode control flow";
+  let constructor = "mlir::py::createConvertWithPass()";
+}
+
+def MaterialiseReturnNone : Pass<"materialise-return-none", "::mlir::func::FuncOp"> {
+  let summary = "Insert emitpybytecode.LOAD_CONST(None) before zero-operand func.return ops";
+  let constructor = "mlir::py::createMaterialiseReturnNonePass()";
+  let dependentDialects = ["::mlir::emitpybytecode::EmitPythonBytecodeDialect"];
 }