Scratch to C++ transpiler experiment

CMakeLists.txt

@@ -11,6 +11,7 @@ option(LIBSCRATCHCPP_NETWORK_SUPPORT "Support for downloading projects" ON)
 option(LIBSCRATCHCPP_PRINT_LLVM_IR "Print LLVM IR of compiled Scratch scripts (for debugging)" OFF)
 option(LIBSCRATCHCPP_ENABLE_CODE_ANALYZER "Analyze Scratch scripts to enable various optimizations" ON)
 option(LIBSCRATCHCPP_LLVM_INTEGER_SUPPORT "Use integers when possible to enable various optimizations" ON)
+option(LIBSCRATCHCPP_BUILD_TRANSPILER "Build the Scratch to C++ transpiler tool" ON)
 option(LIBSCRATCHCPP_ENABLE_SANITIZER "Enable sanitizer to detect memory issues" OFF)
 
 if (LIBSCRATCHCPP_ENABLE_SANITIZER)
@@ -153,6 +154,11 @@ target_compile_definitions(scratchcpp PUBLIC LIBSCRATCHCPP_VERSION_MAJOR=${PROJE
 target_compile_definitions(scratchcpp PUBLIC LIBSCRATCHCPP_VERSION_MINOR=${PROJECT_VERSION_MINOR})
 target_compile_definitions(scratchcpp PUBLIC LIBSCRATCHCPP_VERSION_PATCH=${PROJECT_VERSION_PATCH})
 
+# Scratch to C++ transpiler
+if (LIBSCRATCHCPP_BUILD_TRANSPILER)
+    add_subdirectory(transpiler)
+endif()
+
 # Unit tests
 target_link_libraries(scratchcpp PRIVATE scratchcpp-test-export)
 

src/engine/CMakeLists.txt

@@ -45,3 +45,4 @@ target_sources(scratchcpp
 )
 
 add_subdirectory(internal/llvm)
+add_subdirectory(internal/cpp)

src/engine/internal/cpp/CMakeLists.txt

@@ -0,0 +1,27 @@
+target_sources(scratchcpp
+  PRIVATE
+    cppprelude.h
+    cppruntime.cpp
+    cppruntime.h
+    cppcodebuilder.cpp
+    cppcodebuilder.h
+    cppcodebuilderfactory.cpp
+    cppcodebuilderfactory.h
+    cppregister.h
+  cppcodebasegenerator.cpp
+  cppcodebasegenerator.h
+  cppcompilercontext.cpp
+    cppcompilercontext.h
+    cppexecutablecode.cpp
+    cppexecutablecode.h
+    cppexecutioncontext.cpp
+    cppexecutioncontext.h
+)
+
+# Embed the runtime prelude into the library, it's a part of the generated code
+set(CPP_PRELUDE_HEADER ${CMAKE_CURRENT_BINARY_DIR}/cppprelude_source.h)
+file(READ ${CMAKE_CURRENT_SOURCE_DIR}/cppprelude.h CPP_PRELUDE_CONTENT)
+file(WRITE ${CPP_PRELUDE_HEADER} "// Generated from cppprelude.h\n#pragma once\n\nstatic const char *CPP_PRELUDE_SOURCE = R\"scpp_raw(${CPP_PRELUDE_CONTENT})scpp_raw\";\n")
+set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/cppprelude.h)
+
+target_include_directories(scratchcpp PRIVATE ${CMAKE_CURRENT_BINARY_DIR})

src/engine/internal/cpp/cppcodebasegenerator.cpp

@@ -0,0 +1,205 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#include <scratchcpp/project.h>
+#include <filesystem>
+#include <fstream>
+#include <sstream>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+
+#include "cppcodebasegenerator.h"
+#include "cppcodebuilderfactory.h"
+#include "cppcompilercontext.h"
+#include "../../compiler_p.h"
+
+using namespace libscratchcpp;
+
+namespace
+{
+
+// Creates C++ code builders and keeps the compiler contexts to collect the generated code
+class RecordingFactory : public ICodeBuilderFactory
+{
+    public:
+        std::shared_ptr<ICodeBuilder> create(CompilerContext *ctx, BlockPrototype *procedurePrototype, Compiler::CodeType codeType) const override
+        {
+            return CppCodeBuilderFactory::instance()->create(ctx, procedurePrototype, codeType);
+        }
+
+        std::shared_ptr<CompilerContext> createCtx(IEngine *engine, Target *target) const override
+        {
+            auto ctx = CppCodeBuilderFactory::instance()->createCtx(engine, target);
+            contexts.push_back(ctx);
+            return ctx;
+        }
+
+        mutable std::vector<std::shared_ptr<CompilerContext>> contexts;
+};
+
+bool writeFile(const std::filesystem::path &path, const std::string &content)
+{
+    std::ofstream file(path, std::ios::binary);
+
+    if (!file.is_open()) {
+        std::cerr << "error: failed to write " << path.string() << std::endl;
+        return false;
+    }
+
+    file << content;
+    return true;
+}
+
+} // namespace
+
+/*!
+ * Compiles the given project using the C++ backend and writes a CMake project to outputDir:
+ * - project.cpp - the generated code of all scripts
+ * - project_data.cpp - the embedded project file
+ * - main.cpp - loads and runs the embedded project
+ * - CMakeLists.txt
+ */
+bool CppCodebaseGenerator::generate(const std::string &projectFile, const std::string &outputDir, const std::string &projectName)
+{
+    std::ifstream input(projectFile, std::ios::binary);
+
+    if (!input.is_open()) {
+        std::cerr << "error: failed to read " << projectFile << std::endl;
+        return false;
+    }
+
+    std::stringstream data;
+    data << input.rdbuf();
+
+    RecordingFactory factory;
+    ICodeBuilderFactory *oldFactory = CompilerPrivate::builderFactory();
+    CompilerPrivate::setBuilderFactory(&factory);
+
+    std::stringstream code;
+    bool ret;
+
+    {
+        Project project(projectFile);
+        ret = project.load();
+
+        if (ret) {
+            code << "// Generated by libscratchcpp, do not edit\n\n" << CppCompilerContext::prelude() << "\n";
+
+            for (auto ctx : factory.contexts)
+                code << "\n" << static_cast<CppCompilerContext *>(ctx.get())->moduleSource();
+        }
+    }
+
+    CompilerPrivate::setBuilderFactory(oldFactory);
+
+    if (!ret)
+        return false;
+
+    std::filesystem::path dir(outputDir);
+    std::error_code err;
+    std::filesystem::create_directories(dir, err);
+
+    return writeFile(dir / "project.cpp", code.str()) && writeFile(dir / "project_data.cpp", dataSource(data.str())) && writeFile(dir / "main.cpp", mainSource()) &&
+           writeFile(dir / "CMakeLists.txt", cmakeSource(projectName));
+}
+
+std::string CppCodebaseGenerator::mainSource()
+{
+    return R"(// Generated by libscratchcpp, do not edit
+
+#include <scratchcpp/project.h>
+#include <filesystem>
+#include <fstream>
+#include <string>
+
+extern const unsigned char scratchcpp_project_data[];
+extern const unsigned long long scratchcpp_project_size;
+
+extern "C" void cpp_use_cpp_backend();
+
+int main(int argc, char **argv)
+{
+    // libscratchcpp loads projects from files, so the embedded project is extracted first
+    std::filesystem::path path = std::filesystem::temp_directory_path() / ("scratchcpp_project_" + std::to_string(std::hash<std::string>()(argv[0])) + ".sb3");
+
+    {
+        std::ofstream file(path, std::ios::binary);
+        file.write(reinterpret_cast<const char *>(scratchcpp_project_data), scratchcpp_project_size);
+    }
+
+    // Scripts are compiled by the C++ backend which uses the code in project.cpp
+    cpp_use_cpp_backend();
+
+    libscratchcpp::Project project(path.string());
+    bool loaded = project.load();
+    std::error_code err;
+    std::filesystem::remove(path, err);
+
+    if (!loaded)
+        return 1;
+
+    project.run();
+    return 0;
+}
+)";
+}
+
+std::string CppCodebaseGenerator::dataSource(const std::string &data)
+{
+    std::stringstream ss;
+    ss << "// Generated by libscratchcpp, do not edit\n\n";
+    ss << "extern const unsigned long long scratchcpp_project_size = " << data.size() << "ULL;\n\n";
+    ss << "extern const unsigned char scratchcpp_project_data[] = {";
+
+    for (size_t i = 0; i < data.size(); i++) {
+        if (i % 16 == 0)
+            ss << "\n   ";
+
+        ss << " 0x" << std::hex << std::setw(2) << std::setfill('0') << (int)(unsigned char)data[i] << ",";
+    }
+
+    ss << std::dec << "\n    0x00\n};\n";
+    return ss.str();
+}
+
+std::string CppCodebaseGenerator::cmakeSource(const std::string &projectName)
+{
+    std::string ret = R"(# Generated by libscratchcpp
+cmake_minimum_required(VERSION 3.14)
+project(@NAME@ LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+# Use either libscratchcpp sources (LIBSCRATCHCPP_SOURCE_DIR) or a prebuilt library (LIBSCRATCHCPP_INCLUDE_DIR and LIBSCRATCHCPP_LIBRARY)
+set(LIBSCRATCHCPP_SOURCE_DIR "" CACHE PATH "libscratchcpp source directory")
+set(LIBSCRATCHCPP_INCLUDE_DIR "" CACHE PATH "libscratchcpp include directory")
+set(LIBSCRATCHCPP_LIBRARY "" CACHE FILEPATH "libscratchcpp library")
+
+add_executable(@NAME@ main.cpp project.cpp project_data.cpp)
+
+if(LIBSCRATCHCPP_SOURCE_DIR)
+    set(LIBSCRATCHCPP_BUILD_UNIT_TESTS OFF CACHE BOOL "" FORCE)
+    add_subdirectory(${LIBSCRATCHCPP_SOURCE_DIR} libscratchcpp)
+    target_link_libraries(@NAME@ PRIVATE scratchcpp)
+elseif(LIBSCRATCHCPP_INCLUDE_DIR AND LIBSCRATCHCPP_LIBRARY)
+    target_include_directories(@NAME@ PRIVATE ${LIBSCRATCHCPP_INCLUDE_DIR})
+    target_link_libraries(@NAME@ PRIVATE ${LIBSCRATCHCPP_LIBRARY})
+else()
+    message(FATAL_ERROR "Set LIBSCRATCHCPP_SOURCE_DIR or LIBSCRATCHCPP_INCLUDE_DIR and LIBSCRATCHCPP_LIBRARY")
+endif()
+)";
+    std::string name = projectName;
+
+    for (char &c : name) {
+        if (!std::isalnum((unsigned char)c) && c != '_' && c != '-')
+            c = '_';
+    }
+
+    size_t pos;
+
+    while ((pos = ret.find("@NAME@")) != std::string::npos)
+        ret.replace(pos, 6, name);
+
+    return ret;
+}

src/engine/internal/cpp/cppcodebasegenerator.h

@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include <string>
+
+#include "../../../test_export.h"
+
+namespace libscratchcpp
+{
+
+/*! Generates a standalone C++ codebase (CMake project) from a Scratch project. */
+class LIBSCRATCHCPP_TEST_EXPORT CppCodebaseGenerator
+{
+    public:
+        static bool generate(const std::string &projectFile, const std::string &outputDir, const std::string &projectName = "project");
+
+        static std::string mainSource();
+        static std::string dataSource(const std::string &data);
+        static std::string cmakeSource(const std::string &projectName);
+};
+
+} // namespace libscratchcpp