Merge pull request #588 from scratchcpp/new_compiler

LLVM: Add Compiler class

Author: adazem009

.github/workflows/utests-llvm.yml

@@ -0,0 +1,28 @@
+name: Unit tests (LLVM compiler)
+
+on:
+  push:
+    branches: '*'
+  pull_request:
+    branches: [ "master" ]
+
+env:
+  BUILD_TYPE: Debug
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+
+    - name: Configure CMake
+      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DLIBSCRATCHCPP_BUILD_UNIT_TESTS=ON -DLIBSCRATCHCPP_USE_LLVM=ON
+
+    - name: Build
+      run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j$(nproc --all)
+
+    - name: Run unit tests
+      run: ctest --test-dir build -V

CMakeLists.txt

@@ -9,6 +9,8 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 option(LIBSCRATCHCPP_BUILD_UNIT_TESTS "Build unit tests" ON)
 option(LIBSCRATCHCPP_NETWORK_SUPPORT "Support for downloading projects" ON)
 option(LIBSCRATCHCPP_COMPUTED_GOTO "Support for computed goto" ON)
+option(LIBSCRATCHCPP_USE_LLVM "Compile scripts to LLVM IR (work in progress)" OFF)
+option(LIBSCRATCHCPP_PRINT_LLVM_IR "Print LLVM IR of compiled Scratch scripts (for debugging)" OFF)
 
 if (NOT (CMAKE_CXX_COMPILER_ID STREQUAL "GNU"))
 	# Computed goto not supported on anything except GCC
@@ -45,7 +47,6 @@ target_sources(scratchcpp
     include/scratchcpp/field.h
     include/scratchcpp/script.h
     include/scratchcpp/broadcast.h
-    include/scratchcpp/compiler.h
     include/scratchcpp/virtualmachine.h
     include/scratchcpp/blockprototype.h
     include/scratchcpp/block.h
@@ -65,6 +66,25 @@ target_sources(scratchcpp
     include/scratchcpp/imonitorhandler.h
 )
 
+if (LIBSCRATCHCPP_USE_LLVM)
+    target_compile_definitions(scratchcpp PUBLIC USE_LLVM)
+    target_sources(scratchcpp
+      PUBLIC
+        include/scratchcpp/dev/compiler.h
+        include/scratchcpp/dev/executablecode.h
+        include/scratchcpp/dev/executioncontext.h
+    )
+
+    if(LIBSCRATCHCPP_PRINT_LLVM_IR)
+        target_compile_definitions(scratchcpp PRIVATE PRINT_LLVM_IR)
+    endif()
+else()
+    target_sources(scratchcpp
+      PUBLIC
+        include/scratchcpp/compiler.h
+    )
+endif()
+
 include(FetchContent)
 set(ZIP_SRC thirdparty/zip/src)
 set(UTFCPP_SRC thirdparty/utfcpp/source)
@@ -94,6 +114,12 @@ if (LIBSCRATCHCPP_NETWORK_SUPPORT)
     target_compile_definitions(scratchcpp PRIVATE LIBSCRATCHCPP_NETWORK_SUPPORT)
 endif()
 
+if (LIBSCRATCHCPP_USE_LLVM)
+    include(build/HunterPackages.cmake)
+    include(build/LLVM.cmake)
+    target_link_libraries(scratchcpp PRIVATE LLVM)
+endif()
+
 target_compile_definitions(scratchcpp PRIVATE LIBSCRATCHCPP_LIBRARY)
 target_compile_definitions(scratchcpp PRIVATE LIBSCRATCHCPP_VERSION="${PROJECT_VERSION}")
 target_compile_definitions(scratchcpp PRIVATE LIBSCRATCHCPP_VERSION_MAJOR=${PROJECT_VERSION_MAJOR})

build/HunterPackages.cmake

@@ -0,0 +1,31 @@
+# https://layle.me/posts/using-llvm-with-cmake/
+# HUNTER_URL is the URL to the latest source code archive on GitHub
+# HUNTER_SHA1 is the hash of the downloaded archive
+
+set(OLD_PROJECT_NAME ${PROJECT_NAME})
+set(PROJECT_NAME "")
+
+set(HUNTER_URL "https://github.com/scratchcpp/hunter/archive/ee768cdd2c027b5be346f114e726d4b0c4296de6.zip")
+set(HUNTER_SHA1 "4A018750743AC656A859C99C655723EAF68EE038")
+
+set(HUNTER_LLVM_VERSION 19.1.0)
+set(HUNTER_LLVM_CMAKE_ARGS
+    LLVM_ENABLE_CRASH_OVERRIDES=OFF
+    LLVM_ENABLE_ASSERTIONS=ON
+    LLVM_ENABLE_ZLIB=OFF
+    LLVM_ENABLE_RTTI=ON
+    LLVM_BUILD_EXAMPLES=OFF
+    LLVM_BUILD_TOOLS=OFF
+    LLVM_BUILD_LLVM_DYLIB=ON
+    LLVM_INCLUDE_EXAMPLES=OFF
+    LLVM_TARGETS_TO_BUILD=host
+)
+
+set(HUNTER_PACKAGES LLVM)
+
+include(FetchContent)
+message(STATUS "Fetching hunter...")
+FetchContent_Declare(SetupHunter GIT_REPOSITORY https://github.com/cpp-pm/gate)
+FetchContent_MakeAvailable(SetupHunter)
+
+set(PROJECT_NAME ${OLD_PROJECT_NAME})

build/LLVM.cmake

@@ -0,0 +1,52 @@
+# https://layle.me/posts/using-llvm-with-cmake/
+# This is an INTERFACE target for LLVM, usage:
+#   target_link_libraries(${PROJECT_NAME} <PRIVATE|PUBLIC|INTERFACE> LLVM)
+# The include directories and compile definitions will be properly handled.
+
+set(CMAKE_FOLDER_LLVM "${CMAKE_FOLDER}")
+if(CMAKE_FOLDER)
+    set(CMAKE_FOLDER "${CMAKE_FOLDER}/LLVM")
+else()
+    set(CMAKE_FOLDER "LLVM")
+endif()
+
+# Find LLVM
+find_package(LLVM REQUIRED CONFIG)
+
+message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}")
+message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
+
+# Split the definitions properly (https://weliveindetail.github.io/blog/post/2017/07/17/notes-setup.html)
+separate_arguments(LLVM_DEFINITIONS)
+
+# Some diagnostics (https://stackoverflow.com/a/17666004/1806760)
+message(STATUS "LLVM libraries: ${LLVM_LIBRARIES}")
+message(STATUS "LLVM includes: ${LLVM_INCLUDE_DIRS}")
+message(STATUS "LLVM definitions: ${LLVM_DEFINITIONS}")
+message(STATUS "LLVM tools: ${LLVM_TOOLS_BINARY_DIR}")
+
+if (NOT TARGET LLVM)
+    add_library(LLVM INTERFACE)
+endif()
+
+target_include_directories(LLVM SYSTEM INTERFACE ${LLVM_INCLUDE_DIRS})
+target_link_libraries(LLVM INTERFACE ${LLVM_AVAILABLE_LIBS})
+target_compile_definitions(LLVM INTERFACE ${LLVM_DEFINITIONS} -DNOMINMAX)
+
+set(CMAKE_FOLDER "${CMAKE_FOLDER_LLVM}")
+unset(CMAKE_FOLDER_LLVM)
+
+# MSVC-specific options
+if(MSVC)
+    # This assumes the installed LLVM was built in Release mode
+    set(CMAKE_C_FLAGS_RELWITHDEBINFO "/ZI /Od /Ob0 /DNDEBUG" CACHE STRING "" FORCE)
+    set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "/ZI /Od /Ob0 /DNDEBUG" CACHE STRING "" FORCE)
+
+    if(${LLVM_USE_CRT_RELEASE} STREQUAL "MD")
+        set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreadedDLL)
+    elseif(${LLVM_USE_CRT_RELEASE} STREQUAL "MT")
+        set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreaded)
+    else()
+        message(FATAL_ERROR "Unsupported LLVM_USE_CRT_RELEASE=${LLVM_USE_CRT_RELEASE}")
+    endif()
+endif()

include/scratchcpp/dev/compiler.h

@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include <unordered_set>
+
+#include "../global.h"
+#include "../spimpl.h"
+
+namespace libscratchcpp
+{
+
+class IEngine;
+class Target;
+class ExecutableCode;
+class Variable;
+class List;
+class Input;
+class Field;
+class CompilerPrivate;
+
+/*! \brief The Compiler class provides API for compiling Scratch scripts. */
+class LIBSCRATCHCPP_EXPORT Compiler
+{
+    public:
+        Compiler(IEngine *engine, Target *target);
+        Compiler(const Compiler &) = delete;
+
+        IEngine *engine() const;
+        Target *target() const;
+        std::shared_ptr<Block> block() const;
+
+        std::shared_ptr<ExecutableCode> compile(std::shared_ptr<Block> startBlock);
+
+        void addFunctionCall(const std::string &functionName, int argCount, bool returns);
+        void addConstValue(const Value &value);
+        void addVariableValue(Variable *variable);
+        void addListContents(List *list);
+        void addInput(const std::string &name);
+
+        void moveToIf(std::shared_ptr<Block> substack);
+        void moveToIfElse(std::shared_ptr<Block> substack1, std::shared_ptr<Block> substack2);
+        void moveToLoop(std::shared_ptr<Block> substack);
+        void warp();
+
+        Field *field(const std::string &name) const;
+
+        const std::unordered_set<std::string> &unsupportedBlocks() const;
+
+    private:
+        void addInput(Input *input);
+
+        spimpl::unique_impl_ptr<CompilerPrivate> impl;
+};
+
+} // namespace libscratchcpp

include/scratchcpp/dev/executablecode.h

@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include <memory>
+
+#include "../global.h"
+
+namespace libscratchcpp
+{
+
+class ExecutionContext;
+class Target;
+
+/*! \brief The ExecutableCode class represents the code of a compiled Scratch script. */
+class LIBSCRATCHCPP_EXPORT ExecutableCode
+{
+    public:
+        virtual ~ExecutableCode() { }
+
+        /*! Runs the script until it finishes or yields. */
+        virtual void run(ExecutionContext *context) = 0;
+
+        /*! Stops the code. isFinished() will return true. */
+        virtual void kill(ExecutionContext *context) = 0;
+
+        /*! Resets the code to run from the start. */
+        virtual void reset(ExecutionContext *context) = 0;
+
+        /*! Returns true if the code is stopped or finished. */
+        virtual bool isFinished(ExecutionContext *context) const = 0;
+
+        /*! Pauses the script (when it's executed using run() again) until resolvePromise() is called. */
+        virtual void promise() = 0;
+
+        /*! Resolves the promise and resumes the script. */
+        virtual void resolvePromise() = 0;
+
+        /*! Creates an execution context for the given Target. */
+        virtual std::shared_ptr<ExecutionContext> createExecutionContext(Target *target) const = 0;
+};
+
+} // namespace libscratchcpp

include/scratchcpp/dev/executioncontext.h

@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include "../global.h"
+#include "../spimpl.h"
+
+namespace libscratchcpp
+{
+
+class Target;
+class ExecutionContextPrivate;
+
+/*! \brief The ExecutionContext represents the execution context of a target (can be a clone) with variables, lists, etc. */
+class LIBSCRATCHCPP_EXPORT ExecutionContext
+{
+    public:
+        ExecutionContext(Target *target);
+        ExecutionContext(const ExecutionContext &) = delete;
+        virtual ~ExecutionContext() { }
+
+        Target *target() const;
+
+    private:
+        spimpl::unique_impl_ptr<ExecutionContextPrivate> impl;
+};
+
+} // namespace libscratchcpp

include/scratchcpp/inputvalue.h

@@ -11,7 +11,6 @@ namespace libscratchcpp
 {
 
 class Block;
-class Compiler;
 class Entity;
 class InputValuePrivate;
 

include/scratchcpp/script.h

@@ -14,6 +14,9 @@ namespace libscratchcpp
 class Target;
 class Block;
 class IEngine;
+#ifdef USE_LLVM
+class ExecutableCode;
+#endif
 class Value;
 class Thread;
 class Variable;
@@ -34,6 +37,11 @@ class LIBSCRATCHCPP_EXPORT Script
         const std::vector<unsigned int> &bytecodeVector() const;
         void setBytecode(const std::vector<unsigned int> &code);
 
+#ifdef USE_LLVM
+        ExecutableCode *code() const;
+        void setCode(std::shared_ptr<ExecutableCode> code);
+#endif
+
         void setHatPredicateBytecode(const std::vector<unsigned int> &code);
         bool runHatPredicate(Target *target);
 

include/scratchcpp/valuedata.h

@@ -32,6 +32,7 @@ extern "C"
     /*! \brief The ValueData struct holds the data of Value. It's used in compiled Scratch code for better performance. */
     struct LIBSCRATCHCPP_EXPORT ValueData
     {
+            // NOTE: Any changes must also be done in the LLVM code builder!
             union
             {
                     long intValue;