Initial Commit

.gitignore

@@ -0,0 +1,14 @@
+out/
+
+.idea/
+.vs/
+.vscode/
+
+old/
+OLD/
+wip/
+WIP/
+cmake.lock
+diff.txt
+
+!.gitignore

CMakeLists.txt

@@ -0,0 +1,47 @@
+cmake_minimum_required(VERSION 3.19)
+project("SymbolGenerator")
+
+
+# Prevent concurrent CMake runs.
+file(LOCK ${CMAKE_SOURCE_DIR}/cmake.lock)
+
+
+# Allow loading of scripts from cmake folder.
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake")
+# Store builds in out folder.
+set(CMAKE_BINARY_DIR "${CMAKE_SOURCE_DIR}/out")
+# Allow including from the root directory.
+include_directories(${CMAKE_SOURCE_DIR})
+
+
+# Output to out directory.
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}/out/${CMAKE_BUILD_TYPE}/lib)
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}/out/${CMAKE_BUILD_TYPE}/lib)
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}/out/${CMAKE_BUILD_TYPE}/bin)
+
+
+# Force C++20. (Note this must be done manually with MSVC, see cmake/profiles/msvc.cmake)
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+# Install Dependencies
+include(run_conan)
+select_conan_profile(conan_profile)
+run_conan(${conan_profile})
+
+# Enable compiler-specific warnings
+include(compiler_profile)
+set_compiler_profile()
+
+# Copy shared libraries from conan folder to binary folder.
+if (EXISTS ${CMAKE_SOURCE_DIR}/out/conan/bin)
+    add_custom_target(
+        COPY_SHARED_LIBS ALL
+        COMMAND ${CMAKE_COMMAND} -E copy_directory
+            ${CMAKE_SOURCE_DIR}/out/conan/bin
+            ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}
+    )
+endif()
+
+# Add subprojects.
+add_subdirectory(SymbolGenerator)

README.md

@@ -0,0 +1,47 @@
+# SymbolGenerator
+A tool to automatically generate a module-exports definition file (`.def` file) 
+from one or more compiled C++ translation units (`.obj` files) based on user provided filters.  
+This removes the need to mark all exported symbols in a project with `__declspec(dllexport)`, 
+while simultaneously avoiding issues that CMake's `WINDOWS_EXPORT_ALL_SYMBOLS` property causes,
+like hitting the 64K symbol limit from unnecessarily exported symbols.
+
+### Building
+- This is a Windows-only library. Linux platforms do not have the same issues, since symbols are exported by default
+and there is no 64K symbol limit. This library will not build on non-Windows platforms.
+- [Conan](https://conan.io/) (and therefore [Python](https://www.python.org/downloads/)) is required to install the project's dependencies (`pip install conan`).
+- [CMake](https://cmake.org/download/) is required, together with some generator to build the project with (e.g. [Ninja](https://ninja-build.org/)).
+
+To build the project (with Ninja):
+```shell
+mkdir out
+cd out
+cmake -DCMAKE_BUILD_TYPE=[DEBUG|RELEASE] -G Ninja -DCMAKE_C_COMPILER=[MSVC|clang-cl] -DCMAKE_CXX_COMPILER=[MSVC|clang-cl] ../
+cmake --build ./[debug|release] --target all
+```
+
+### Usage
+The program takes as its input a set of `.obj` files and some regex filters, and produces a `.def` file containing all symbols that are matched by said filters.  
+The command line arguments are as follows:
+- `-lib`:     the name of the DLL that will be created using the generated `.def` file.
+- `-i`:       the directory containing the `.obj` to process. The provided path is searched recursively.
+- `-o`:       the path of the output `.def` file.
+- `-y`:       a list of regexes for namespaces to include. Includes all symbols in the given namespace and all subnamespaces.
+Namespace should be a top-level namespace, or its parent should already be included by another `-y` parameter.
+- `-n`:       a list of regexes for namespaces to exclude. Excludes all symbols in the given namespace and all subnamespaces.
+Namespace need not be a top-level namespace.
+- `-yo`:      a list of regexes of force-included symbols. These symbols are always included, even if they are in an excluded namespace.
+- `-no`:      a list of regexes of force-excluded symbols. These symbols are always excluded, even if they are in an included namespace.
+- `-cache`:   if provided, the results of the program will be cached so that it can run faster the next time it is invoked with the same arguments.
+Caching occurs on a per-obj-file basis.
+- `-verbose`: if provided, logs additional information to stdout.
+- `-j`:       if provided, the number of threads used to process objects. Defaults to number of threads of the current device.
+
+The `-lib`, `-i` and `-o` parameters are required. All other parameters are optional (Although you should provide at least one to match anything).
+
+Example:
+```shell
+SymbolGenerator.exe -cache -lib VoxelEngine -i ../out/debug/CMakeFiles/VoxelEngine.dir -o ./VoxelEngine.def -y ve -n .*detail.* .*impl.* meta -yo .*vertex_layout.*
+```
+This command creates `./VoxelEngine.def` which can be used to create `VoxelEngine.dll`, by taking symbols from all `.obj` files in the `VoxelEngine.dir` directory.
+All symbols in the `ve` namespace and nested namespaces therein are included, except nested namespaces containing the text `detail` or `impl` or named `meta`.
+Symbols containing the text `vertex_layout` are always included, even if they are in such an excluded namespace.

SymbolGenerator/CMakeLists.txt

@@ -0,0 +1,12 @@
+include(create_target)
+
+
+create_target(
+    SymbolGenerator
+    EXECUTABLE
+    0 0 1
+    # Dependencies:
+    CONAN_PKG::abseil
+    CONAN_PKG::COFFI
+    CONAN_PKG::range-v3
+)

SymbolGenerator/argument_parser.hpp

@@ -0,0 +1,126 @@
+#pragma once
+
+#include <SymbolGenerator/defs.hpp>
+#include <SymbolGenerator/utility.hpp>
+#include <SymbolGenerator/logger.hpp>
+
+#include <vector>
+#include <string>
+#include <string_view>
+#include <variant>
+#include <algorithm>
+
+
+namespace symgen {
+    class argument_parser {
+    public:
+        static argument_parser& instance(void) {
+            static argument_parser i;
+            return i;
+        }
+
+
+        struct none_t {
+            friend std::ostream& operator<<(std::ostream& stream, const none_t& self) {
+                stream << "[None]";
+                return stream;
+            }
+        };
+
+        using argument_t = std::variant<std::string, long long, bool, none_t>;
+
+
+        template <typename Rng> void add_arguments(const Rng& range) {
+            enum { NO_TOKEN, HAS_KEY, HAS_VAL } state = NO_TOKEN;
+            std::string pending_key, pending_value;
+
+
+            for (const auto& arg : range) {
+                std::string_view sv { arg };
+                bool has_prefix = false;
+
+                while (sv.starts_with('-')) sv.remove_prefix(1), has_prefix = true;
+
+                if (state == NO_TOKEN) {
+                    pending_key = sv;
+                    state = HAS_KEY;
+                    continue;
+                }
+
+                if (state == HAS_KEY || state == HAS_VAL) {
+                    if (has_prefix) {
+                        arguments.emplace(std::move(pending_key), to_argument(pending_value));
+
+                        pending_key = sv;
+                        pending_value.clear();
+
+                        state = HAS_KEY;
+                    } else {
+                        if (!pending_value.empty()) pending_value += " ";
+                        pending_value += sv;
+                        state = HAS_VAL;
+                    }
+
+                    continue;
+                }
+            }
+
+
+            if (state != NO_TOKEN) {
+                arguments.emplace(std::move(pending_key), to_argument(pending_value));
+            }
+        }
+
+
+        bool has_argument(std::string_view key) const {
+            return arguments.contains(key);
+        }
+
+
+        template <typename T> std::optional<T> get_argument(std::string_view key) const {
+            auto it = arguments.find(key);
+            if (it == arguments.end()) return std::nullopt;
+
+            if (std::holds_alternative<T>(it->second)) {
+                return std::get<T>(it->second);
+            } else {
+                return std::nullopt;
+            }
+        }
+
+
+        template <typename T> void require_argument(std::string_view key) {
+            logger::instance().assert_that(has_argument(key), "Missing required argument ", key);
+        }
+
+
+        void print_arguments(void) const {
+            logger::instance().normal("Argument list of size ", arguments.size(), ":");
+
+            for (const auto& [k, v] : arguments) {
+                std::visit([&, key = &k] (const auto& v) {
+                    logger::instance().normal(*key, ": ", v);
+                }, v);
+            }
+        }
+    private:
+        hash_map<std::string, argument_t> arguments;
+
+
+        static argument_t to_argument(std::string_view value) {
+            if (value.empty()) return none_t { };
+
+            std::string lower;
+            std::transform(value.begin(), value.end(), std::back_inserter(lower), [] (char c) { return char(std::tolower(c)); });
+
+            if (lower == "true" ) return true;
+            if (lower == "false") return false;
+
+            try {
+                return argument_t { std::stoll(lower) };
+            } catch (...) {}
+
+            return argument_t { std::string { value } };
+        }
+    };
+}

SymbolGenerator/defs.hpp

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <SymbolGenerator/legacy.hpp>
+
+#include <absl/container/flat_hash_map.h>
+#include <absl/container/flat_hash_set.h>
+#include <range/v3/all.hpp>
+
+#include <filesystem>
+#include <string>
+#include <string_view>
+#include <type_traits>
+
+
+namespace symgen {
+    namespace fs    = std::filesystem;
+    namespace views = ranges::views;
+    using namespace std::string_literals;
+    using namespace std::string_view_literals;
+
+
+    template <typename K> using default_hash = typename absl::flat_hash_set<K>::hasher;
+    template <typename K> using default_eq   = typename absl::flat_hash_set<K>::key_equal;
+
+
+    template <typename K, typename V, typename Hash = default_hash<K>, typename Eq = default_eq<K>>
+    using hash_map = absl::flat_hash_map<K, V, Hash, Eq>;
+
+    template <typename K, typename Hash = default_hash<K>, typename Eq = default_eq<K>>
+    using hash_set = absl::flat_hash_set<K, Hash, Eq>;
+}

SymbolGenerator/legacy.hpp

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <type_traits>
+
+
+// Abseil uses std::result_of, which was removed in C++20.
+// TODO: Remove this once Abseil is updated as this is UB!
+namespace std {
+    template <typename T> class result_of {};
+
+    template <typename Fn, typename... Args> struct result_of<Fn(Args...)> {
+        using type = std::invoke_result_t<Fn, Args...>;
+    };
+
+    template <typename T> using result_of_t = typename std::result_of<T>::type;
+}

SymbolGenerator/logger.hpp

@@ -0,0 +1,58 @@
+#pragma once
+
+#include <iostream>
+#include <mutex>
+#include <syncstream>
+
+
+namespace symgen {
+    class logger {
+    public:
+        enum logger_level { VERBOSE, NORMAL, WARNING, ERROR };
+
+
+        static logger& instance(void) {
+            static logger i { };
+            return i;
+        }
+
+
+        logger fork(std::string new_prefix) {
+            logger copy = *this;
+            copy.prefix = std::move(new_prefix);
+            return copy;
+        }
+
+
+        void assert_that(bool cond, const auto&... msg) {
+            if (!cond) [[unlikely]] {
+                message(ERROR, msg...);
+                std::exit(-1);
+            }
+        }
+
+
+        void message(logger_level level, const auto&... msg) {
+            if (level < this->level) return;
+
+            std::osyncstream stream { std::cout };
+            if (!prefix.empty()) stream << "[" << prefix << "] ";
+            (stream << ... << msg) << "\n";
+
+            #ifndef NDEBUG
+                stream << std::flush;
+            #endif
+        }
+
+        void verbose(const auto&... msg) { message(logger_level::VERBOSE, msg...); }
+        void normal (const auto&... msg) { message(logger_level::NORMAL,  msg...); }
+        void warning(const auto&... msg) { message(logger_level::WARNING, msg...); }
+        void error  (const auto&... msg) { message(logger_level::ERROR,   msg...); }
+
+
+        void set_level(logger_level level) { this->level = level; }
+    private:
+        std::string prefix = "";
+        logger_level level = NORMAL;
+    };
+}