From bf5aaed06e66caaa5af99c9b3b22b2126ea1df16 Mon Sep 17 00:00:00 2001 From: "Tarn W. Burton" Date: Wed, 11 Dec 2024 11:06:38 -0500 Subject: [PATCH] Add dynamic extensions (#1641) --- .github/workflows/pkg.yml | 4 +- .github/workflows/test.yml | 8 +- debian/control | 14 +- debian/rules | 4 +- guix/cando.scm | 8 +- guix/clasp.scm | 8 +- include/clasp/core/debugger.h | 23 +-- include/clasp/core/fli.h | 2 +- include/clasp/gctools/snapshotSaveLoad.h | 2 +- include/cscript.lisp | 4 +- src/analysis/cscript.lisp | 4 +- src/asttooling/cscript.lisp | 2 +- src/clbind/cscript.lisp | 2 +- src/core/cscript.lisp | 2 +- src/core/debug_macosx.cc | 7 +- src/core/debug_unixes.cc | 23 ++- src/core/debugger.cc | 61 +++++-- src/core/fli.cc | 24 ++- src/core/keywordPackage.cc | 2 + src/cscript.lisp | 4 + src/gctools/cscript.lisp | 2 +- src/gctools/gcFunctions.cc | 12 ++ src/gctools/snapshotSaveLoad.cc | 83 +++++---- src/gctools/startRunStop.cc | 5 + src/koga/config-header.lisp | 1 + src/koga/configure.lisp | 34 +++- src/koga/koga.asd | 3 +- src/koga/ninja.lisp | 211 ++++++++++++++++------- src/koga/scripts.lisp | 77 +++++++++ src/koga/setup.lisp | 2 + src/koga/target-sources.lisp | 10 +- src/koga/units.lisp | 48 ++++-- src/lisp/kernel/lsp/fli.lisp | 18 +- src/llvmo/cscript.lisp | 2 +- src/main/cscript.lisp | 1 - src/mpip/cscript.lisp | 2 +- src/serveEvent/cscript.lisp | 2 +- src/sockets/cscript.lisp | 2 +- 38 files changed, 509 insertions(+), 214 deletions(-) diff --git a/.github/workflows/pkg.yml b/.github/workflows/pkg.yml index 8d6e2a4ab5..af589d1248 100644 --- a/.github/workflows/pkg.yml +++ b/.github/workflows/pkg.yml @@ -66,7 +66,7 @@ jobs: run: | sudo apt-get update sudo apt install -y sbcl - ./tools-for-build/nightly-version-bump --extensions=cando,seqan-clasp + ./tools-for-build/nightly-version-bump --extensions=cando rm -rf dependencies extensions src/lisp/kernel/contrib src/lisp/modules/asdf src/mps src/bdwgc src/libatomic_ops - name: Build Ubuntu packages uses: jtdor/build-deb-action@v1 @@ -152,7 +152,7 @@ jobs: run: | sudo apt-get update sudo apt install -y sbcl - ./tools-for-build/nightly-version-bump --extensions=cando,seqan-clasp + ./tools-for-build/nightly-version-bump --extensions=cando echo "(:skip-sync t)" > config.sexp - name: Build Debian packages uses: jtdor/build-deb-action@v1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4f39fe5188..81ec1549d9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,7 +61,7 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt-get update - sudo apt install -y binutils-gold clang-15 libclang-15-dev libclang-cpp15-dev llvm-15 llvm-15-dev libelf-dev libgmp-dev libunwind-dev ninja-build sbcl libnetcdf-dev libexpat1-dev libfmt-dev libboost-all-dev + sudo apt install -y binutils-gold clang-18 libclang-18-dev libclang-cpp18-dev llvm-18 llvm-18-dev libelf-dev libgmp-dev libunwind-dev ninja-build sbcl libnetcdf-dev libexpat1-dev libfmt-dev libboost-all-dev - name: Install MacOS dependencies if: matrix.os == 'macos-13' || matrix.os == 'macos-14' @@ -87,7 +87,7 @@ jobs: - name: Cando koga @ Ubuntu if: ${{ matrix.os == 'ubuntu-latest' && matrix.build == 'cando' }} run: | - ./koga --build-mode=${{ matrix.mode }} --extensions=cando,seqan-clasp + ./koga --build-mode=${{ matrix.mode }} --extensions=cando - name: Clasp koga @ MacOS=13 if: ${{ matrix.os == 'macos-13' && matrix.build == 'clasp' }} run: | @@ -99,11 +99,11 @@ jobs: - name: Cando koga @ MacOS-13 if: ${{ matrix.os == 'macos-13' && matrix.build == 'cando' }} run: | - ./koga --build-mode=${{ matrix.mode }} --extensions=cando,seqan-clasp + ./koga --build-mode=${{ matrix.mode }} --extensions=cando - name: Cando koga @ MacOS-14 if: ${{ matrix.os == 'macos-14' && matrix.build == 'cando' }} run: | - ./koga --build-mode=${{ matrix.mode }} --extensions=cando,seqan-clasp --cflags=-I/opt/homebrew/include --cppflags=-I/opt/homebrew/include --cxxflags=-I/opt/homebrew/include + ./koga --build-mode=${{ matrix.mode }} --extensions=cando --cflags=-I/opt/homebrew/include --cppflags=-I/opt/homebrew/include --cxxflags=-I/opt/homebrew/include - name: Build run: | ninja -C build diff --git a/debian/control b/debian/control index ee0f78d6c4..5abf47a10e 100644 --- a/debian/control +++ b/debian/control @@ -2,9 +2,9 @@ Source: clasp-cl Section: lisp Priority: optional Maintainer: Tarn W. Burton -Build-Depends: debhelper-compat (= 13), libelf-dev, libgmp-dev, llvm-15, - clang-15, sbcl, libclang-15-dev, llvm-15-dev, ninja-build, libfmt-dev, - git, pkg-config, libboost-all-dev, libclang-cpp15, libclang-cpp15-dev, +Build-Depends: debhelper-compat (= 13), libelf-dev, libgmp-dev, llvm-18, + clang-18, sbcl, libclang-18-dev, llvm-18-dev, ninja-build, libfmt-dev, + git, pkg-config, libboost-all-dev, libclang-cpp18, libclang-cpp18-dev, libnetcdf-dev, libczmq-dev, libexpat1-dev Standards-Version: 4.5.1 Homepage: https://github.com/clasp-developers/clasp @@ -20,7 +20,7 @@ Provides: lisp-compiler Architecture: any Build-Profiles: Depends: ${shlibs:Depends}, ${misc:Depends}, libelf1, libgmp10, libgmpxx4ldbl, - llvm-15, clang-15, libclang-cpp15 + llvm-18, clang-18, libclang-cpp18 Description: Common Lisp implementation that brings Common Lisp and C++ Together Clasp is a new Common Lisp implementation that seamlessly interoperates with C++ libraries and programs using LLVM for compilation to native @@ -37,9 +37,9 @@ Conflicts: clasp-cl Provides: lisp-compiler, clasp-cl Architecture: any Build-Profiles: -Depends: ${shlibs:Depends}, ${misc:Depends}, libelf-dev, libgmp-dev, llvm-15, - clang-15, sbcl, libclang-15-dev, llvm-15-dev, libfmt-dev, - libclang-cpp15, libclang-cpp15-dev, libnetcdf-dev, libczmq-dev, +Depends: ${shlibs:Depends}, ${misc:Depends}, libelf-dev, libgmp-dev, llvm-18, + clang-18, sbcl, libclang-18-dev, llvm-18-dev, libfmt-dev, + libclang-cpp18, libclang-cpp18-dev, libnetcdf-dev, libczmq-dev, libexpat1-dev, ninja-build Description: Common Lisp implementation that brings Common Lisp and C++ Together Clasp is a new Common Lisp implementation that seamlessly interoperates diff --git a/debian/rules b/debian/rules index 1a373ffe99..9d5a601a55 100755 --- a/debian/rules +++ b/debian/rules @@ -18,10 +18,10 @@ override_dh_auto_test: override_dh_auto_configure: ifneq (,$(filter clasp-cl,$(DOPACKAGES))) - ./koga --build-mode=bytecode-faso --reproducible-build --build-path=clasp-build/ --ldlibs=-ldl --package-path=$$(pwd)/debian/clasp-cl/ --bin-path=/usr/bin/ --share-path=/usr/share/clasp/ --lib-path=/usr/lib/clasp/ + ./koga --build-mode=bytecode-faso --reproducible-build --build-path=clasp-build/ --ldlibs=-ldl --package-path=$$(pwd)/debian/clasp-cl/ --bin-path=/usr/bin/ --share-path=/usr/share/clasp/ --lib-path=/usr/lib/clasp/ --dylib-path=/usr/lib/ --pkgconfig-path=/usr/lib/pkgconfig/ endif ifneq (,$(filter cando,$(DOPACKAGES))) - ./koga --build-mode=bytecode-faso --reproducible-build --build-path=cando-build/ --extensions=cando,seqan-clasp --ldlibs=-ldl --package-path=$$(pwd)/debian/cando/ --bin-path=/usr/bin/ --share-path=/usr/share/clasp/ --lib-path=/usr/lib/clasp/ + ./koga --build-mode=bytecode-faso --reproducible-build --build-path=cando-build/ --extensions=cando --ldlibs=-ldl --package-path=$$(pwd)/debian/cando/ --bin-path=/usr/bin/ --share-path=/usr/share/clasp/ --lib-path=/usr/lib/clasp/ --dylib-path=/usr/lib/ --pkgconfig-path=/usr/lib/pkgconfig/ endif override_dh_auto_build: diff --git a/guix/cando.scm b/guix/cando.scm index 7b07c94e21..52782e36b4 100644 --- a/guix/cando.scm +++ b/guix/cando.scm @@ -36,13 +36,9 @@ (apply append (list (string-append source-dir "/src/bdwgc") (string-append source-dir "/src/libatomic_ops") - (string-append source-dir "/src/lisp/modules/asdf") - ;; This is a special case: the parent is omitted - ;; from external-dirs, but added to descend-dirs. - (string-append source-dir "/extensions/seqan-clasp/seqan")) + (string-append source-dir "/src/lisp/modules/asdf")) (map-in-order scandir-absolute external-dirs))) (descend-dirs `(,@git-dirs - ,(string-append source-dir "/extensions/seqan-clasp") ,@external-dirs)) (predicates (map-in-order git-predicate (cons source-dir git-dirs)))) (package @@ -60,7 +56,7 @@ (substitute-keyword-arguments (package-arguments clasp-cl) ((#:configure-flags flags '()) #~(cons* - "--extensions=cando,seqan-clasp" + "--extensions=cando" ;; Had trouble simplifying this using find. ;; This is prepended because Koga keeps the first instance ;; of repeated options. diff --git a/guix/clasp.scm b/guix/clasp.scm index 9333b2195c..089163f17f 100644 --- a/guix/clasp.scm +++ b/guix/clasp.scm @@ -89,9 +89,11 @@ #~(let ((clang #$(this-package-input "clang-toolchain"))) (list "--build-mode=bytecode-faso" "--reproducible-build" - (string-append "--bin-path=" #$output "/bin") - (string-append "--lib-path=" #$output "/lib/clasp") - (string-append "--share-path=" #$output "/share/clasp") + (string-append "--bin-path=" #$output "/bin/") + (string-append "--lib-path=" #$output "/lib/clasp/") + (string-append "--share-path=" #$output "/share/clasp/") + (string-append "--dylib-path=" #$output "/lib/") + (string-append "--pkgconfig-path=" #$output "/lib/pkgconfig/") ;; Without --cc and --cxx, Clang is searched for in ;; LLVM's /bin. (string-append "--cc=" clang "/bin/clang") diff --git a/include/clasp/core/debugger.h b/include/clasp/core/debugger.h index b7536988b8..a3942be195 100644 --- a/include/clasp/core/debugger.h +++ b/include/clasp/core/debugger.h @@ -81,10 +81,15 @@ struct OpenDynamicLibraryInfo { gctools::clasp_ptr_t _LibraryStart; gctools::clasp_ptr_t _TextStart; gctools::clasp_ptr_t _TextEnd; + bool _HasVtableSection; + gctools::clasp_ptr_t _VtableSectionStart; + gctools::clasp_ptr_t _VtableSectionEnd; OpenDynamicLibraryInfo(const std::string& f, void* h, const SymbolTable& symbol_table, gctools::clasp_ptr_t libstart, - gctools::clasp_ptr_t textStart, gctools::clasp_ptr_t textEnd) - : _Filename(f), _Handle(h), _SymbolTable(symbol_table), _LibraryStart(libstart), _TextStart(textStart), - _TextEnd(textEnd){ + gctools::clasp_ptr_t textStart, gctools::clasp_ptr_t textEnd, + bool hasVtableSection, + gctools::clasp_ptr_t vtableSectionStart, gctools::clasp_ptr_t vtableSectionEnd) + : _Filename(f), _Handle(h), _SymbolTable(symbol_table), _LibraryStart(libstart), _TextStart(textStart), _TextEnd(textEnd), + _HasVtableSection(hasVtableSection), _VtableSectionStart(vtableSectionStart), _VtableSectionEnd(vtableSectionEnd) { // printf("%s:%d:%s filename = %s\n", __FILE__, __LINE__, __FUNCTION__, f.c_str() ); }; virtual LoadableKind loadableKind() const { return Library; }; @@ -92,14 +97,10 @@ struct OpenDynamicLibraryInfo { }; struct ExecutableLibraryInfo : OpenDynamicLibraryInfo { - bool _HasVtableSection; - gctools::clasp_ptr_t _VtableSectionStart; - gctools::clasp_ptr_t _VtableSectionEnd; ExecutableLibraryInfo(const std::string& f, void* h, const SymbolTable& symbol_table, gctools::clasp_ptr_t libstart, - gctools::clasp_ptr_t textStart, gctools::clasp_ptr_t textEnd, bool hasVtableSection, - gctools::clasp_ptr_t vtableSectionStart, gctools::clasp_ptr_t vtableSectionEnd) - : OpenDynamicLibraryInfo(f, h, symbol_table, libstart, textStart, textEnd), _HasVtableSection(hasVtableSection), - _VtableSectionStart(vtableSectionStart), _VtableSectionEnd(vtableSectionEnd) { + gctools::clasp_ptr_t textStart, gctools::clasp_ptr_t textEnd, + bool hasVtableSection, gctools::clasp_ptr_t vtableSectionStart, gctools::clasp_ptr_t vtableSectionEnd) + : OpenDynamicLibraryInfo(f, h, symbol_table, libstart, textStart, textEnd, hasVtableSection, vtableSectionStart, vtableSectionEnd ) { #if 0 printf("%s:%d:%s filename = %s\n", __FILE__, __LINE__, __FUNCTION__, f.c_str() ); if (vtableSectionStart==NULL) { @@ -215,6 +216,6 @@ DebugInfo& debugInfo(); void executablePath(std::string& name); void executableTextSectionRange(gctools::clasp_ptr_t& start, gctools::clasp_ptr_t& end); -void executableVtableSectionRange(gctools::clasp_ptr_t& start, gctools::clasp_ptr_t& end); +void exclusiveVtableSectionRange(gctools::clasp_ptr_t& start, gctools::clasp_ptr_t& end); }; // namespace core diff --git a/include/clasp/core/fli.h b/include/clasp/core/fli.h index 62e5526d44..b9da54c693 100644 --- a/include/clasp/core/fli.h +++ b/include/clasp/core/fli.h @@ -214,7 +214,7 @@ CL_DEFUN core::T_sp PERCENTdlopen(core::T_sp path_designator); DOCGROUP(clasp) CL_DEFUN core::T_sp PERCENTdlclose(ForeignData_sp handle); DOCGROUP(clasp) -CL_DEFUN core::T_sp PERCENTdlsym(core::String_sp name); +CL_DEFUN core::T_sp PERCENTdlsym(core::T_sp library, core::String_sp name); }; // namespace clasp_ffi diff --git a/include/clasp/gctools/snapshotSaveLoad.h b/include/clasp/gctools/snapshotSaveLoad.h index 12a8c4e291..b9e1eac2e8 100644 --- a/include/clasp/gctools/snapshotSaveLoad.h +++ b/include/clasp/gctools/snapshotSaveLoad.h @@ -74,7 +74,7 @@ struct ISLLibrary { struct Fixup { FixupOperation_ _operation; uintptr_t _memoryStart; - std::vector _Libraries; + std::vector _ISLLibraries; bool _trackAddressName; map _addressName; diff --git a/include/cscript.lisp b/include/cscript.lisp index 2a248120ad..e86b45c91f 100644 --- a/include/cscript.lisp +++ b/include/cscript.lisp @@ -1,4 +1,4 @@ -(k:includes #~"" #~"clasp/main/") +(k:includes #~"") (k:sources :install-code #~"clasp/asttooling/" @@ -9,7 +9,6 @@ #~"clasp/external/" #~"clasp/gctools/" #~"clasp/llvmo/" - #~"clasp/main/" #~"clasp/mpip/" #~"clasp/serveEvent/" #~"clasp/sockets/" @@ -24,7 +23,6 @@ #~"clasp/external/" #~"clasp/gctools/" #~"clasp/llvmo/" - #~"clasp/main/" #~"clasp/mpip/" #~"clasp/serveEvent/" #~"clasp/sockets/" diff --git a/src/analysis/cscript.lisp b/src/analysis/cscript.lisp index 1ba8a2e8b9..b63d5cfc49 100644 --- a/src/analysis/cscript.lisp +++ b/src/analysis/cscript.lisp @@ -1,3 +1,3 @@ (if (member :cando k:*extensions*) - (k:sources :iclasp #~"clasp_gc_cando.sif") - (k:sources :iclasp #~"clasp_gc.sif")) \ No newline at end of file + (k:sources :libclasp #~"clasp_gc_cando.sif") + (k:sources :libclasp #~"clasp_gc.sif")) diff --git a/src/asttooling/cscript.lisp b/src/asttooling/cscript.lisp index bceb7bf89f..8d782f5b23 100644 --- a/src/asttooling/cscript.lisp +++ b/src/asttooling/cscript.lisp @@ -1,4 +1,4 @@ -(k:sources :iclasp +(k:sources :libclasp #~"astExpose0.cc" #~"astExpose1.cc" #~"clangTooling.cc" diff --git a/src/clbind/cscript.lisp b/src/clbind/cscript.lisp index 5b39c1b30a..3dafe8c867 100644 --- a/src/clbind/cscript.lisp +++ b/src/clbind/cscript.lisp @@ -1,4 +1,4 @@ -(k:sources :iclasp +(k:sources :libclasp #~"adapter.cc" #~"class_rep.cc" #~"open.cc" diff --git a/src/core/cscript.lisp b/src/core/cscript.lisp index 0532203c0f..2eb53a94ed 100644 --- a/src/core/cscript.lisp +++ b/src/core/cscript.lisp @@ -1,4 +1,4 @@ -(k:sources :iclasp +(k:sources :libclasp #~"dummy.cc" #~"mpPackage.cc" #~"nativeVector.cc" diff --git a/src/core/debug_macosx.cc b/src/core/debug_macosx.cc index a40fb175e9..fde07be9e2 100644 --- a/src/core/debug_macosx.cc +++ b/src/core/debug_macosx.cc @@ -350,12 +350,15 @@ void add_dynamic_library_impl(add_dynamic_library* callback, bool is_executable, if (is_executable) { odli = new ExecutableLibraryInfo(libraryName, handle, symbol_table, reinterpret_cast(library_origin), reinterpret_cast(library_origin), - reinterpret_cast(library_origin + text_segment_size), found, + reinterpret_cast(library_origin + text_segment_size), + found, (gctools::clasp_ptr_t)vtableRegionStart, (gctools::clasp_ptr_t)vtableRegionEnd); } else { odli = new OpenDynamicLibraryInfo(libraryName, handle, symbol_table, reinterpret_cast(library_origin), reinterpret_cast(library_origin), - reinterpret_cast(library_origin + text_segment_size)); + reinterpret_cast(library_origin + text_segment_size), + found, + (gctools::clasp_ptr_t)vtableRegionStart, (gctools::clasp_ptr_t)vtableRegionEnd); } if (callback) (*callback)(odli); diff --git a/src/core/debug_unixes.cc b/src/core/debug_unixes.cc index 7756ef842d..ec45a380cd 100644 --- a/src/core/debug_unixes.cc +++ b/src/core/debug_unixes.cc @@ -304,23 +304,21 @@ CL_DEFUN void core__walk_loaded_objects() { } int elf_startup_loaded_object_callback(struct dl_phdr_info* info, size_t size, void* data) { - // printf("%s:%d:%s Startup registering loaded object %s\n", __FILE__, __LINE__, __FUNCTION__, info->dlpi_name); +// printf("%s:%d:%s --------- Startup registering loaded object %s\n", __FILE__, __LINE__, __FUNCTION__, info->dlpi_name); core::General_O general; gctools::clasp_ptr_t vtablePtr = *(gctools::clasp_ptr_t*)&general; - // printf("%s:%d:%s one vtablePtr = %p\n", __FILE__, __LINE__, __FUNCTION__, vtablePtr ); +// printf("%s:%d:%s a KNOWN vtablePtr = %p\n", __FILE__, __LINE__, __FUNCTION__, vtablePtr ); ScanInfo* scan_callback_info = (ScanInfo*)data; bool is_executable = false; const char* libname; if (scan_callback_info->_Index == 0 && strlen(info->dlpi_name) == 0) { libname = getExecutablePath(); is_executable = true; - // printf("%s:%d:%s getExecutablePath() libname %s is_executable = %d\n", __FILE__, __LINE__, __FUNCTION__, libname, - // is_executable ); +// printf("%s:%d:%s getExecutablePath() libname %s is_executable = %d\n", __FILE__, __LINE__, __FUNCTION__, libname, is_executable ); } else { libname = info->dlpi_name; is_executable = false; - // printf("%s:%d:%s info->dlpi_name libname %s is_executable = %d\n", __FILE__, __LINE__, __FUNCTION__, libname, is_executable - // ); +// printf("%s:%d:%s info->dlpi_name libname %s is_executable = %d\n", __FILE__, __LINE__, __FUNCTION__, libname, is_executable ); } gctools::clasp_ptr_t text_start; gctools::clasp_ptr_t text_end; @@ -335,8 +333,7 @@ int elf_startup_loaded_object_callback(struct dl_phdr_info* info, size_t size, v if (p_type == PT_LOAD && (info->dlpi_phdr[j].p_flags & 0x1)) { // executable text_start = (gctools::clasp_ptr_t)(info->dlpi_addr + info->dlpi_phdr[j].p_vaddr); text_end = (gctools::clasp_ptr_t)(text_start + info->dlpi_phdr[j].p_memsz); - // printf("%s:%d:%s text_start = %p text_end = %p\n low = %p high = %p vtablePtr = %p\n", __FILE__, __LINE__, - // __FUNCTION__, (void*)text_start, (void*)text_end, low, high, vtablePtr ); +// printf("%s:%d:%s text_start = %p text_end = %p low = %p high = %p\n", __FILE__, __LINE__, __FUNCTION__, (void*)text_start, (void*)text_end, low, high ); } if (low <= vtablePtr && vtablePtr < high) { // @@ -354,9 +351,9 @@ int elf_startup_loaded_object_callback(struct dl_phdr_info* info, size_t size, v hasVtableSection = true; vtableSectionStart = low; vtableSectionEnd = high; - // printf("%s:%d:%s set vtableSection Start/End = %p/%p\n", __FILE__, __LINE__, __FUNCTION__, low, high ); +// printf("%s:%d:%s set vtableSection Start/End = %p/%p\n", __FILE__, __LINE__, __FUNCTION__, low, high ); } - // printf("%s:%d:%s Found vtableSection %p to %p\n", __FILE__, __LINE__, __FUNCTION__, vtableSectionStart, vtableSectionEnd ); +// printf("%s:%d:%s Found vtableSection %p to %p\n", __FILE__, __LINE__, __FUNCTION__, vtableSectionStart, vtableSectionEnd ); } } // printf("%s:%d:%s About to call add_dynamic_library_using_origin libname = %s is_executable = %d\n", __FILE__, __LINE__, @@ -377,8 +374,7 @@ void startup_register_loaded_objects(add_dynamic_library* callback) { otherwise it uses handle to look up the start of the library. */ void add_dynamic_library_impl(add_dynamic_library* callback, bool is_executable, const std::string& libraryName, bool use_origin, uintptr_t library_origin, void* handle, gctools::clasp_ptr_t text_start, - gctools::clasp_ptr_t text_end, bool hasVtableSection, gctools::clasp_ptr_t vtableSectionStart, - gctools::clasp_ptr_t vtableSectionEnd) { + gctools::clasp_ptr_t text_end, bool hasVtableSection, gctools::clasp_ptr_t vtableSectionStart, gctools::clasp_ptr_t vtableSectionEnd) { // printf("%s:%d:%s libraryName = %s is_executable = %d\n", __FILE__, __LINE__, __FUNCTION__, libraryName.c_str(), is_executable // ); BT_LOG(("Starting to load library: %s\n", libraryName.c_str())); @@ -421,7 +417,8 @@ void add_dynamic_library_impl(add_dynamic_library* callback, bool is_executable, } else { // printf("%s:%d:%s Creating an OpenDynamicLibraryInfo\n", __FILE__, __LINE__, __FUNCTION__ ); odli = - new OpenDynamicLibraryInfo(libraryName, handle, symbol_table, (gctools::clasp_ptr_t)library_origin, text_start, text_end); + new OpenDynamicLibraryInfo(libraryName, handle, symbol_table, (gctools::clasp_ptr_t)library_origin, text_start, text_end, + hasVtableSection, vtableSectionStart, vtableSectionEnd ); loadable = Library; } TrapProblem trap(is_executable, libraryName, odli->loadableKind()); diff --git a/src/core/debugger.cc b/src/core/debugger.cc index a45e447b60..7970d3049b 100644 --- a/src/core/debugger.cc +++ b/src/core/debugger.cc @@ -199,18 +199,42 @@ void executablePath(std::string& name) { } SIMPLE_ERROR("Could not find the executablePath"); } -void executableVtableSectionRange(gctools::clasp_ptr_t& start, gctools::clasp_ptr_t& end) { + +void exclusiveVtableSectionRange(gctools::clasp_ptr_t& start, gctools::clasp_ptr_t& end) { WITH_READ_LOCK(debugInfo()._OpenDynamicLibraryMutex); + size_t numberOfVtableRanges = 0; for (auto& entry : debugInfo()._OpenDynamicLibraryHandles) { - if (entry.second->loadableKind() == Executable) { - ExecutableLibraryInfo* eli = dynamic_cast(entry.second); - start = eli->_VtableSectionStart; - end = eli->_VtableSectionEnd; + if (entry.second->_HasVtableSection) { + // This assumes there is only one vtable section!!!!!!! + start = entry.second->_VtableSectionStart; + end = entry.second->_VtableSectionEnd; + numberOfVtableRanges++; +// printf("%s:%d:%s vtableRange# %lu library: %s vtable section start %p end %p\n", __FILE__, __LINE__, __FUNCTION__, numberOfVtableRanges, entry.first.c_str(), start, end ); + } + } + if (numberOfVtableRanges>1) { + printf("%s:%d:%s There can be only one vtable range in the executable or dynamic libraries - if there is more than one then we need to rearrange things and identify the vtable range that contains snapshot save/load-able objects\n", __FILE__, __LINE__, __FUNCTION__); + gctools::wait_for_user_signal("About to exit"); + } else if (numberOfVtableRanges==0) { + printf("%s:%d:%s Could not find any vtable sections\n", __FILE__, __LINE__, __FUNCTION__); + gctools::wait_for_user_signal("About to exit"); + } +} + +CL_DEFUN void core__openDynamicLibraries() { + WITH_READ_LOCK(debugInfo()._OpenDynamicLibraryMutex); + for (auto& entry : debugInfo()._OpenDynamicLibraryHandles) { + printf("%s:%d:%s lib: %s exec %d\n", __FILE__, __LINE__, __FUNCTION__, entry.first.c_str(), entry.second->loadableKind()==Executable ); + void* textstart = entry.second->_TextStart; + void* textend = entry.second->_TextEnd; + printf("%s:%d:%s textStart: %p textEnd: %p \n", __FILE__, __LINE__, __FUNCTION__, textstart, textend ); + if (entry.second->_HasVtableSection) { + void* start = entry.second->_VtableSectionStart; + void* end = entry.second->_VtableSectionEnd; + printf("%s:%d:%s vtableStart: %p vtableEnd: %p \n", __FILE__, __LINE__, __FUNCTION__, start, end ); // printf("%s:%d:%s start %p end %p\n", __FILE__, __LINE__, __FUNCTION__, start, end ); - return; } } - SIMPLE_ERROR("Could not find the executableVtableSectionRange"); } void executableTextSectionRange(gctools::clasp_ptr_t& start, gctools::clasp_ptr_t& end) { @@ -222,12 +246,14 @@ void executableTextSectionRange(gctools::clasp_ptr_t& start, gctools::clasp_ptr_ return; } } - SIMPLE_ERROR("Could not find the executableVtableSectionRange"); + SIMPLE_ERROR("Could not find the executableTextSectionRange"); } bool lookup_address_in_library(gctools::clasp_ptr_t address, gctools::clasp_ptr_t& start, gctools::clasp_ptr_t& end, std::string& libraryName, bool& executable, uintptr_t& vtableStart, uintptr_t& vtableEnd) { WITH_READ_LOCK(debugInfo()._OpenDynamicLibraryMutex); + vtableStart = 0; + vtableEnd = 0; for (auto entry : debugInfo()._OpenDynamicLibraryHandles) { // printf("%s:%d:%s Looking at entry: %s start: %p end: %p\n", __FILE__, __LINE__, __FUNCTION__, // entry.second._Filename.c_str(), entry.second._LibraryStart, entry.second._LibraryEnd ); @@ -238,17 +264,22 @@ bool lookup_address_in_library(gctools::clasp_ptr_t address, gctools::clasp_ptr_ start = eli->_TextStart; end = eli->_TextEnd; executable = true; - vtableStart = (uintptr_t)eli->_VtableSectionStart; - vtableEnd = (uintptr_t)eli->_VtableSectionEnd; + if (eli->_HasVtableSection) { + vtableStart = (uintptr_t)eli->_VtableSectionStart; + vtableEnd = (uintptr_t)eli->_VtableSectionEnd; + } return true; } - } else if (entry.second->_TextStart <= address && address < entry.second->_TextEnd) { + } else if ((entry.second->_TextStart <= address && address < entry.second->_TextEnd) + || (entry.second->_HasVtableSection && entry.second->_VtableSectionStart<=address && address_VtableSectionEnd) ) { libraryName = entry.second->_Filename; start = entry.second->_TextStart; end = entry.second->_TextEnd; executable = false; - vtableStart = 0; // libraries don't have vtables we care about - vtableEnd = 0; + if (entry.second->_HasVtableSection) { + vtableStart = (uintptr_t)entry.second->_VtableSectionStart; // libraries don't have vtables we care about + vtableEnd = (uintptr_t)entry.second->_VtableSectionEnd; + } return true; } } @@ -280,8 +311,8 @@ bool library_with_name(const std::string& name, bool isExecutable, std::string& libraryName = entry.second->_Filename; start = (uintptr_t)(entry.second->_TextStart); end = (uintptr_t)(entry.second->_TextEnd); - vtableStart = 0; // (uintptr_t)entry.second->_VtableSectionStart; - vtableEnd = 0; // (uintptr_t)entry.second->_VtableSectionEnd; + vtableStart = (uintptr_t)entry.second->_VtableSectionStart; + vtableEnd = (uintptr_t)entry.second->_VtableSectionEnd; // printf("%s:%d:%s isExecutable = %d name = %s libraryName = %s\n", __FILE__, __LINE__, __FUNCTION__, isExecutable, // name.c_str(), libraryName.c_str()); return true; diff --git a/src/core/fli.cc b/src/core/fli.cc index 389b54cc3d..41c214aea1 100644 --- a/src/core/fli.cc +++ b/src/core/fli.cc @@ -653,10 +653,17 @@ core::T_sp PERCENTdlclose(ForeignData_sp handle) { // --------------------------------------------------------------------------- // --------------------------------------------------------------------------- -core::T_sp PERCENTdlsym(core::String_sp name) { - +core::T_sp PERCENTdlsym(core::T_sp library, core::String_sp name) { + void *handle = NULL; + if (library == kw::_sym_rtld_default) { + handle = RTLD_DEFAULT; + } else if (library == kw::_sym_rtld_next) { + handle = RTLD_NEXT; + } else if (ForeignData_sp fd = library.asOrNull()) { + handle = fd->raw_data(); + } ForeignData_sp sp_sym; - auto result = core::do_dlsym(RTLD_DEFAULT, name->get_std_string().c_str()); + auto result = core::do_dlsym(handle, name->get_std_string().c_str()); void* p_sym = std::get<0>(result); if (!p_sym) { @@ -1167,7 +1174,7 @@ core::T_sp PERCENTmem_set_unsigned_char(core::Integer_sp address, core::T_sp val return mk_fixnum_uint8(tmp); } -const struct section_64* get_section_data(const char* segment_name, const char* section_name) { +/*const struct section_64* get_section_data(const char* segment_name, const char* section_name) { const struct section_64* p_section = (struct section_64*)NULL; #if defined(__APPLE__) @@ -1176,10 +1183,10 @@ const struct section_64* get_section_data(const char* segment_name, const char* #endif return p_section; -} +}*/ core::T_mv PERCENTget_section_data(core::String_sp sp_segment_name, core::String_sp sp_section_name) { -#if defined(__APPLE__) +/*#if defined(__APPLE__) const struct section_64* p_section = (struct section_64*)NULL; unsigned long section_size = 0; @@ -1193,11 +1200,10 @@ core::T_mv PERCENTget_section_data(core::String_sp sp_segment_name, core::String return Values(nil(), _lisp->_true()); } -#else - +#else*/ return Values(nil(), nil()); -#endif +//#endif } }; // namespace clasp_ffi diff --git a/src/core/keywordPackage.cc b/src/core/keywordPackage.cc index 683cd078f3..be8e618d87 100644 --- a/src/core/keywordPackage.cc +++ b/src/core/keywordPackage.cc @@ -133,6 +133,8 @@ SYMBOL_EXPORT_SC_(KeywordPkg, read_only); SYMBOL_EXPORT_SC_(KeywordPkg, relative); SYMBOL_EXPORT_SC_(KeywordPkg, rename); SYMBOL_EXPORT_SC_(KeywordPkg, rename_and_delete); +SYMBOL_EXPORT_SC_(KeywordPkg, rtld_default); +SYMBOL_EXPORT_SC_(KeywordPkg, rtld_next); SYMBOL_EXPORT_SC_(KeywordPkg, script); SYMBOL_EXPORT_SC_(KeywordPkg, seek_cur); SYMBOL_EXPORT_SC_(KeywordPkg, seek_end); diff --git a/src/cscript.lisp b/src/cscript.lisp index a3c1d355d4..2d26e02f8a 100644 --- a/src/cscript.lisp +++ b/src/cscript.lisp @@ -29,3 +29,7 @@ #@"source_info_inc.h" #@"symbols_scraped_inc.h" #@"terminators_inc.h") + +(k:sources :install-code + #~"bdwgc/include/" + #~"schubfach/") diff --git a/src/gctools/cscript.lisp b/src/gctools/cscript.lisp index 7713d02fb0..3c2ae2af05 100644 --- a/src/gctools/cscript.lisp +++ b/src/gctools/cscript.lisp @@ -1,4 +1,4 @@ -(k:sources :iclasp +(k:sources :libclasp #~"gc_interface.cc" #~"exposeFunctions0.cc" #~"exposeFunctions1.cc" diff --git a/src/gctools/gcFunctions.cc b/src/gctools/gcFunctions.cc index d45d4fddf8..79d31413d3 100644 --- a/src/gctools/gcFunctions.cc +++ b/src/gctools/gcFunctions.cc @@ -473,6 +473,18 @@ CL_DEFUN core::T_mv gctools__memory_profile_status() { core::make_fixnum(allocationSizeCounter), core::make_fixnum(allocationSizeThreshold)); } + +CL_DEFUN core::T_sp gctools__object_address(core::General_sp generalObject) { + void* address = (void*)&(*generalObject); + printf("%s:%d:%s address = %p\n", __FILE__, __LINE__, __FUNCTION__, address ); + return nil(); +} +CL_DEFUN core::T_sp gctools__vtable_address(core::General_sp generalObject) { + void* vtable_ptr = *(void**)&(*generalObject); + printf("%s:%d:%s vtable pointer = %p\n", __FILE__, __LINE__, __FUNCTION__, vtable_ptr ); + return nil(); +} + DOCGROUP(clasp); CL_DEFUN core::T_sp gctools__stack_depth() { int z = 0; diff --git a/src/gctools/snapshotSaveLoad.cc b/src/gctools/snapshotSaveLoad.cc index 68c7ebedee..3faeac00ea 100644 --- a/src/gctools/snapshotSaveLoad.cc +++ b/src/gctools/snapshotSaveLoad.cc @@ -39,6 +39,10 @@ #include #include +// Turn on debugging vtable pointer updates with libraries +//#define DEBUG_ISLLIBRARIES 1 + + #ifdef _TARGET_OS_LINUX #include #endif @@ -596,7 +600,7 @@ bool virtualMethodP(uintptr_t* ptrptr) { void Fixup::registerVtablePointer(size_t libraryIndex, core::T_O* vtablePtrPtr) { - this->_Libraries[libraryIndex]._InternalPointers.emplace_back(VtablePointer, (uintptr_t*)vtablePtrPtr, + this->_ISLLibraries[libraryIndex]._InternalPointers.emplace_back(VtablePointer, (uintptr_t*)vtablePtrPtr, *(uintptr_t*)vtablePtrPtr); }; @@ -606,7 +610,7 @@ void Fixup::registerFunctionPointer(size_t libraryIndex, uintptr_t* functionPtrP __FILE__, __LINE__, __FUNCTION__, libraryIndex); abort(); } - this->_Libraries[libraryIndex]._InternalPointers.emplace_back(FunctionPointer, (uintptr_t*)functionPtrPtr, *functionPtrPtr); + this->_ISLLibraries[libraryIndex]._InternalPointers.emplace_back(FunctionPointer, (uintptr_t*)functionPtrPtr, *functionPtrPtr); #ifdef DEBUG_ENTRY_POINTS printf("%s:%d:%s libraryIndex[%lu] functionPtrPtr @%p -> %p location: %s\n", __FILE__, __LINE__, __FUNCTION__, @@ -627,10 +631,10 @@ uintptr_t Fixup::fixedAddress(bool functionP, uintptr_t* ptrptr, const char* add uintptr_t codedAddress = *ptrptr; decodeRelocation_(codedAddress, firstByte, libidx, pointerIndex); // printf("%s:%d:%s libidx = %lu pointerIndex = %lu\n", __FILE__, __LINE__, __FUNCTION__, libidx, pointerIndex ); - uintptr_t address = this->_Libraries[libidx]._GroupedPointers[pointerIndex]._address; + uintptr_t address = this->_ISLLibraries[libidx]._GroupedPointers[pointerIndex]._address; // printf("%s:%d:%s address = %p @ %p\n", __FILE__, __LINE__, __FUNCTION__, (void*)address, // &this->_libraries[libidx]._GroupedPointers[pointerIndex]._address ); - uintptr_t addressOffset = this->_Libraries[libidx]._SymbolInfo[pointerIndex]._AddressOffset; + uintptr_t addressOffset = this->_ISLLibraries[libidx]._SymbolInfo[pointerIndex]._AddressOffset; // printf("%s:%d:%s addressOffset = %lu\n", __FILE__, __LINE__, __FUNCTION__, addressOffset ); uintptr_t ptr = address + addressOffset; if (functionP && *(uint8_t*)ptr != firstByte) { @@ -670,11 +674,11 @@ void* encodePointer(Fixup* fixup, gctools::clasp_ptr_t address,size_t idx, gctoo #endif size_t Fixup::ensureLibraryRegistered(uintptr_t address) { - for (size_t idx = 0; idx < this->_Libraries.size(); idx++) { - if (((uintptr_t)this->_Libraries[idx]._TextStart) <= address && address < ((uintptr_t)this->_Libraries[idx]._TextEnd)) { + for (size_t idx = 0; idx < this->_ISLLibraries.size(); idx++) { + if (((uintptr_t)this->_ISLLibraries[idx]._TextStart) <= address && address < ((uintptr_t)this->_ISLLibraries[idx]._TextEnd)) { return idx; } - if (this->_Libraries[idx]._VtableStart <= address && address < this->_Libraries[idx]._VtableEnd) { + if (this->_ISLLibraries[idx]._VtableStart <= address && address < this->_ISLLibraries[idx]._VtableEnd) { return idx; } } @@ -686,10 +690,12 @@ size_t Fixup::ensureLibraryRegistered(uintptr_t address) { bool isExecutable; core::lookup_address_in_library((gctools::clasp_ptr_t)address, start, end, libraryPath, isExecutable, vtableStart, vtableEnd); ISLLibrary lib(libraryPath, isExecutable, start, end, vtableStart, vtableEnd); - size_t idx = this->_Libraries.size(); - // printf("%s:%d:%s Registering library %s address: %p start: %p end: %p vtableStart: %p vtableEnd: %p \n", __FILE__, __LINE__, - // __FUNCTION__, libraryPath.c_str(), address, start, end, vtableStart, vtableEnd ); - this->_Libraries.push_back(lib); + size_t idx = this->_ISLLibraries.size(); +#ifdef DEBUG_ISLLIBRARIES + printf("%s:%d:%s Registering library %s address: %p start: %p end: %p vtableStart: %p vtableEnd: %p \n", __FILE__, __LINE__, + __FUNCTION__, libraryPath.c_str(), (void*)address, start, end, (void*)vtableStart, (void*)vtableEnd ); +#endif + this->_ISLLibraries.push_back(lib); return idx; }; @@ -2166,20 +2172,20 @@ void prepareRelocationTableForSave(Fixup* fixup, SymbolLookup& symbolLookup) { OrderByAddress() {} bool operator()(const PointerBase& x, const PointerBase& y) { return x._address <= y._address; } void addLibraries(Fixup* fixup, SymbolLookup& symbolLookup) { - for (size_t idx = 0; idx < fixup->_Libraries.size(); idx++) { + for (size_t idx = 0; idx < fixup->_ISLLibraries.size(); idx++) { DBG_SLS("Adding library #%lu: %s\n", idx, fixup->_Libraries[idx]._Name.c_str()); - symbolLookup.addLibrary(fixup->_Libraries[idx]._Name); - auto pointersBegin = fixup->_Libraries[idx]._InternalPointers.begin(); - auto pointersEnd = fixup->_Libraries[idx]._InternalPointers.end(); + symbolLookup.addLibrary(fixup->_ISLLibraries[idx]._Name); + auto pointersBegin = fixup->_ISLLibraries[idx]._InternalPointers.begin(); + auto pointersEnd = fixup->_ISLLibraries[idx]._InternalPointers.end(); if (pointersBegin < pointersEnd) { DBG_SLS("About to quickSortFirstCheckOrder _Pointers.size(): %lu\n", fixup->_Libraries[idx]._InternalPointers.size()); } } } void identifyUnique(Fixup* fixup, SymbolLookup& symbolLookup) { - for (size_t idx = 0; idx < fixup->_Libraries.size(); idx++) { + for (size_t idx = 0; idx < fixup->_ISLLibraries.size(); idx++) { int groupPointerIdx = -1; - ISLLibrary& curLib = fixup->_Libraries[idx]; + ISLLibrary& curLib = fixup->_ISLLibraries[idx]; // printf("%s:%d:%s Dealing with library#%lu: %s @%p\n", __FILE__, __LINE__, __FUNCTION__, idx, curLib._Name.c_str(), &curLib // ); printf("%s:%d:%s Number of pointers before extracting unique pointers: %lu\n", __FILE__, __LINE__, __FUNCTION__, // curLib._InternalPointers.size() ); @@ -2584,9 +2590,10 @@ void* snapshot_save_impl(void* data) { { DBG_SL_STEP(11, "snapshot_save fixing up vtable pointers\n"); + core::lisp_write(fmt::format("Fixup vtable pointers\n")); gctools::clasp_ptr_t start; gctools::clasp_ptr_t end; - core::executableVtableSectionRange(start, end); + core::exclusiveVtableSectionRange(start, end); fixup_vtables_t fixup_vtables(&fixup, (uintptr_t)start, (uintptr_t)end, &islInfo); walk_snapshot_save_load_objects((ISLHeader_s*)snapshot._Memory->_BufferStart, fixup_vtables); } @@ -2605,20 +2612,20 @@ void* snapshot_save_impl(void* data) { DBG_SL_STEP(13, "Calculating library sizes\n"); core::lisp_write(fmt::format("Calculate library sizes\n")); size_t librarySize = 0; - for (size_t idx = 0; idx < fixup._Libraries.size(); idx++) { - librarySize += fixup._Libraries[idx].writeSize(); + for (size_t idx = 0; idx < fixup._ISLLibraries.size(); idx++) { + librarySize += fixup._ISLLibraries[idx].writeSize(); } DBG_SL_STEP(14, "copy_buffer_t\n"); snapshot._Libraries = new copy_buffer_t(librarySize); core::lisp_write(fmt::format("Copy buffer\n")); - for (size_t idx = 0; idx < fixup._Libraries.size(); idx++) { - size_t alignedLen = fixup._Libraries[idx].nameSize(); + for (size_t idx = 0; idx < fixup._ISLLibraries.size(); idx++) { + size_t alignedLen = fixup._ISLLibraries[idx].nameSize(); char* buffer = (char*)malloc(alignedLen); - ISLLibrary& lib = fixup._Libraries[idx]; + ISLLibrary& lib = fixup._ISLLibraries[idx]; memset(buffer, '\0', alignedLen); strcpy(buffer, lib._Name.c_str()); ISLLibraryHeader_s libhead(Library, lib._Executable, lib.writeSize(), alignedLen, alignedLen + lib.symbolBufferSize(), - fixup._Libraries[idx]._SymbolInfo.size()); + fixup._ISLLibraries[idx]._SymbolInfo.size()); #if 0 printf("%s:%d:%s ------ &libhead = %p\n", __FILE__, __LINE__, __FUNCTION__, &libhead ); printf("%s:%d:%s buffer_offset = %p\n", __FILE__, __LINE__, __FUNCTION__, (void*)snapshot._Libraries->buffer_offset() ); @@ -2648,7 +2655,7 @@ void* snapshot_save_impl(void* data) { ISLFileHeader* fileHeader = snapshot._FileHeader; uintptr_t offset = snapshot._HeaderBuffer->_Size; fileHeader->_LibrariesOffset = offset; - fileHeader->_NumberOfLibraries = fixup._Libraries.size(); + fileHeader->_NumberOfLibraries = fixup._ISLLibraries.size(); offset += snapshot._Libraries->_Size; fileHeader->_SaveTimeMemoryAddress = (uintptr_t)snapshot._Memory->_BufferStart; fileHeader->_MemoryStart = offset; @@ -2720,12 +2727,20 @@ void* snapshot_save_impl(void* data) { } cmd = CXX_BINARY " " BUILD_LINKFLAGS " -L" + snapshot_data->_LibDir + " -o" + snapshot_data->_FileName + " " + obj_filename + - " -Wl,-whole-archive -lclasp -Wl,-no-whole-archive " BUILD_LIB; + " -Wl,-whole-archive -liclasp" +#ifndef CLASP_STATIC_LINKING + " -Wl,-no-whole-archive" +#endif + " -lclasp " +#ifdef CLASP_STATIC_LINKING + " -Wl,-no-whole-archive" +#endif + BUILD_LIB; #endif #ifdef _TARGET_OS_DARWIN cmd = CXX_BINARY " " BUILD_LINKFLAGS " -o" + snapshot_data->_FileName + " -sectcreate " SNAPSHOT_SEGMENT " " SNAPSHOT_SECTION " " + filename + " -Wl,-force_load," + snapshot_data->_LibDir + - "/libclasp.a " BUILD_LIB; + "/libiclasp.a -lclasp " BUILD_LIB; #endif std::cout << "Link command:" << std::endl << std::flush; @@ -3095,8 +3110,12 @@ void snapshot_load(void* maybeStartOfSnapshot, void* maybeEndOfSnapshot, const s if (fout) fflush(fout); // flush fout if it's defined. --arguments option was passed updateRelocationTableAfterLoad(lib, lookup); - // printf("%s:%d:%s Done updateRelocationTableAfterLoad\n", __FILE__, __LINE__, __FUNCTION__ ); - fixup._Libraries.push_back(lib); + //printf("%s:%d:%s Done updateRelocationTableAfterLoad pushing library with name: %s\n", __FILE__, __LINE__, __FUNCTION__, lib._Name.c_str() ); +#ifdef DEBUG_ISLLIBRARIES + printf("%s:%d:%s push_back lib: %s start: %p end: %p vtableStart: %p vtableEnd: %p\n", + __FILE__, __LINE__, __FUNCTION__, lib._Name.c_str(), lib._TextStart, lib._TextEnd, (void*)lib._VtableStart, (void*)lib._VtableEnd); +#endif + fixup._ISLLibraries.push_back(lib); } if (fout) fclose(fout); // Close fout if it's defined. --arguments option was passed @@ -3115,7 +3134,7 @@ void snapshot_load(void* maybeStartOfSnapshot, void* maybeEndOfSnapshot, const s DBG_SL("2 snapshot_load fixing up vtable pointers\n"); gctools::clasp_ptr_t start; gctools::clasp_ptr_t end; - core::executableVtableSectionRange(start, end); + core::exclusiveVtableSectionRange(start, end); if (start == NULL) { printf("%s:%d:%s vtable section range start is NULL\n", __FILE__, __LINE__, __FUNCTION__); } @@ -3190,9 +3209,11 @@ void snapshot_load(void* maybeStartOfSnapshot, void* maybeEndOfSnapshot, const s DBG_SL("6 Allocate objects\n"); { ISLHeader_s* start_header = reinterpret_cast(islbuffer); +#if 0 gctools::clasp_ptr_t startVtables; gctools::clasp_ptr_t end; - core::executableVtableSectionRange(startVtables, end); + core::exclusiveVtableSectionRange(startVtables, end); +#endif ISLHeader_s* next_header; ISLHeader_s* cur_header; diff --git a/src/gctools/startRunStop.cc b/src/gctools/startRunStop.cc index 2d7e018122..aee1933927 100644 --- a/src/gctools/startRunStop.cc +++ b/src/gctools/startRunStop.cc @@ -163,6 +163,11 @@ static void clasp_terminate_handler(void) { abort(); } +CL_DEFUN void gctools__register_loaded_objects() { + core::add_library addlib; + core::startup_register_loaded_objects(&addlib); +} + }; // namespace gctools #ifndef SCRAPING diff --git a/src/koga/config-header.lisp b/src/koga/config-header.lisp index d254f0ddf5..ff695aa842 100644 --- a/src/koga/config-header.lisp +++ b/src/koga/config-header.lisp @@ -132,6 +132,7 @@ "CLASP_DEV_GENERATED_PATH" (namestring (merge-pathnames (root :variant-generated) (build-path configuration))) "CLASP_DEV_INCLUDE_PATH" "include" + "CLASP_STATIC_LINKING" (static-linking-p configuration) "CLASP_INSTALL_SYS_PATH" (namestring (root :install-share)) "CLASP_INSTALL_LIB_PATH" (namestring (root :install-lib)) "CLASP_INSTALL_GENERATED_PATH" (namestring (root :install-generated)) diff --git a/src/koga/configure.lisp b/src/koga/configure.lisp index 9350e1414b..27c7d83351 100644 --- a/src/koga/configure.lisp +++ b/src/koga/configure.lisp @@ -141,11 +141,21 @@ :initarg :lib-path :type pathname :documentation "The directory under which to install the Clasp libraries.") + (dylib-path :accessor dylib-path + :initform #P"/usr/local/lib/" + :initarg :dylib-path + :type pathname + :documentation "The directory under which to install the Clasp dynamic libraries.") (share-path :accessor share-path :initform #P"/usr/local/share/clasp/" :initarg :share-path :type pathname :documentation "The directory under which to install shared Clasp files.") + (pkgconfig-path :accessor pkgconfig-path + :initform #P"/usr/lib/pkgconfig/" + :initarg :pkgconfig-path + :type pathname + :documentation "The directory under which to install the pkgconfig files.") (jupyter-path :accessor jupyter-path :initform nil :initarg :jupyter-path @@ -613,6 +623,11 @@ is not compatible with snapshots.") :initform t :type boolean :documentation "Enable long-float") + (static-linking :accessor static-linking-p + :initarg :static-linking + :initform nil + :type boolean + :documentation "Static link Clasp library.") (units :accessor units :initform '(:git :describe :cpu-count #+darwin :xcode :base :default-target :pkg-config :clang :llvm :ar :cc :cxx :dis :mpi :nm :etags :ctags :objcopy :jupyter @@ -659,9 +674,13 @@ is not compatible with snapshots.") (list (make-source #P"asdf-test.bash" :build)) :bench (list (make-source #P"bench.lisp" :build)) + :libclasp-pc + (list (make-source #P"libclasp.pc" :build)) + :libclasp-pc-variant + (list (make-source #P"libclasp.pc" :variant)) :ninja (list (make-source #P"build.ninja" :build) - :iclasp :cclasp :modules :eclasp + :libclasp :iclasp :cclasp :modules :eclasp :eclasp-link :sclasp :install-bin :install-code :clasp :regression-tests :analyzer :analyze :tags :install-extension-code :vm-header @@ -686,6 +705,9 @@ is not compatible with snapshots.") :iclasp))) :type hash-table :documentation "The configuration outputs with the associated targets.") + (libraries :accessor libraries + :initform nil + :type list) (targets :accessor targets :initform (make-hash-table) :type hash-table @@ -768,6 +790,14 @@ is not compatible with snapshots.") (not (member feature +core-features+))) *features*)))) +(defun lib-filename (configuration name &key (dynamic nil dynamicp)) + (make-pathname :name name + :type (if (or (static-linking-p configuration) + (not dynamicp)) + "a" + #+darwin "dylib" + #-darwin "so"))) + (defun build-name (name &key common (gc *variant-gc* gc-p) @@ -902,6 +932,8 @@ the function to the overall configuration." &key required min-version max-version &allow-other-keys) "Configure a library" (message :info "Configuring library ~a" library) + (when required + (push (list library min-version max-version) (libraries configuration))) (flet ((failure (control-string &rest args) (apply #'message (if required :err :warn) control-string args) nil)) diff --git a/src/koga/koga.asd b/src/koga/koga.asd index d34ff1fca5..70badefda6 100644 --- a/src/koga/koga.asd +++ b/src/koga/koga.asd @@ -6,7 +6,8 @@ "closer-mop" "ninja" "shasht" - "trivial-features") + "trivial-features" + "split-sequence") :serial t :components ((:file "packages") (:file "utilities") diff --git a/src/koga/ninja.lisp b/src/koga/ninja.lisp index 1ac7ea1384..e30e3b13b9 100644 --- a/src/koga/ninja.lisp +++ b/src/koga/ninja.lisp @@ -4,6 +4,12 @@ (declare (ignore configuration name)) (ninja:make-line-wrapping-stream (ninja:make-timestamp-preserving-stream path))) +(defun wrap-with-env (configuration executable-name) + (if (reproducible-build configuration) + (format nil "LD_LIBRARY_PATH=~a ~a" + (make-source "lib/" :variant) executable-name) + executable-name)) + (defun lisp-command (script &optional arguments) (concatenate 'string "$lisp " @@ -112,6 +118,10 @@ (ninja:write-rule output-stream :link :command "$cxx $variant-ldflags $ldflags -o$out $in $variant-ldlibs $ldlibs" :description "Linking $out") + (ninja:write-rule output-stream :link-lib + :command #+darwin "$cxx -dynamiclib $variant-ldflags $ldflags -install_name @rpath/$libname -o$out $in $variant-ldlibs $ldlibs" + #-darwin "$cxx -shared $variant-ldflags $ldflags -o$out $in $variant-ldlibs $ldlibs" + :description "Linking $out") (ninja:write-rule output-stream :load-cclasp :command "$clasp --norc --disable-mpi --ignore-image --feature clasp-min --load load-clasp.lisp -- base 0 $source" :description "Loading clasp $name" @@ -196,7 +206,7 @@ &aux (ll (make-source-output source :type "ll")) (bc (make-source-output source :type "bc")) (header (make-source "trampoline.h" :variant-generated)) - (installed-header (make-source "trampoline.h" :installed-generated))) + (installed-header (make-source "include/trampoline.h" :package-share))) (declare (ignore configuration)) (ninja:write-build output-stream :cxx-llvm :variant-cxxflags *variant-cxxflags* @@ -256,8 +266,12 @@ (defmethod print-target-sources (configuration (name (eql :ninja)) output-stream (target (eql :install-code)) sources &key outputs &allow-other-keys) + (declare (ignore configuration)) (ninja:write-build output-stream :phony - :inputs outputs + :inputs (list* (make-source "include/trampoline.h" :package-share) + (make-source "include/config.h" :package-share) + (make-source "include/virtualMachine.h" :package-share) + outputs) :outputs (list "install_code"))) (defmethod print-target-source @@ -298,17 +312,17 @@ (configuration (name (eql :ninja)) output-stream (target (eql :vm-header)) sources &key &allow-other-keys &aux (header (make-source "virtualMachine.h" :variant-generated)) - (installed-header (make-source "virtualMachine.h" :installed-generated))) + (installed-header (make-source "include/virtualMachine.h" :package-share))) (ninja:write-build output-stream :generate-vm-header :inputs sources :outputs (list header)) (when *variant-default* (ninja:write-build output-stream :install-file :inputs (list header) - :outputs (list installed-header)))) + :outputs (list installed-header)))) (defmethod print-variant-target-source - (configuration (name (eql :ninja)) output-stream (target (eql :iclasp)) + (configuration (name (eql :ninja)) output-stream (target (eql :libclasp)) (source c-source)) (let ((o (make-source-output source :type "o"))) (ninja:write-build output-stream :cc @@ -321,7 +335,7 @@ (list :objects o))) (defmethod print-variant-target-source - (configuration (name (eql :ninja)) output-stream (target (eql :iclasp)) + (configuration (name (eql :ninja)) output-stream (target (eql :libclasp)) (source cc-source)) (let ((pp (make-source-output source :type "pp")) (sif (make-source-output source :type "sif")) @@ -349,12 +363,27 @@ (defmethod print-variant-target-source (configuration (name (eql :ninja)) output-stream (target (eql :iclasp)) + (source cc-source)) + (let ((o (make-source-output source :type "o"))) + (ninja:write-build output-stream :cxx + :variant-cxxflags *variant-cxxflags* + :inputs (list source) + :order-only-inputs (list* (make-source "virtualMachine.h" :variant-generated) + (make-source "trampoline.h" :variant-generated) + (if *variant-precise* + (scraper-precise-headers configuration) + (scraper-headers configuration))) + :outputs (list o)) + (list :objects o))) + +(defmethod print-variant-target-source + (configuration (name (eql :ninja)) output-stream (target (eql :libclasp)) (source sif-source)) (declare (ignore configuration name output-stream target)) (list :sifs source)) (defmethod print-variant-target-sources - (configuration (name (eql :ninja)) output-stream (target (eql :iclasp)) sources + (configuration (name (eql :ninja)) output-stream (target (eql :libclasp)) sources &key objects sifs &allow-other-keys &aux (generated (append (if *variant-precise* (scraper-precise-headers configuration) @@ -363,22 +392,12 @@ (products (mapcar (lambda (source) (make-source (source-path source) :package-share)) generated)) - (exe (make-source "iclasp" :variant)) - (exe-installed (make-source "iclasp" :package-bin)) - (lib (make-source "libclasp.a" :variant-lib)) - (lib-installed (make-source "libclasp.a" :package-lib)) - (symlink (make-source (if (member :cando (extensions configuration)) - "cando" - "clasp") - :variant)) - (symlink-installed (make-source (if (member :cando (extensions configuration)) - "cando" - "clasp") - :package-bin)) - (clasp-sh (make-source "clasp" :variant)) - (clasp-sh-installed (make-source "clasp" :package-bin)) - (cleap-symlink (make-source "cleap" :variant)) - (cleap-symlink-installed (make-source "cleap" :package-bin)) + (libclasp-name (lib-filename configuration "libclasp" :dynamic t)) + (libclasp (make-source libclasp-name :variant-lib)) + (libclasp-installed (make-source libclasp-name (if (static-linking-p configuration) + :package-lib + :package-dylib))) + (libclasp-pc-installed (make-source "libclasp.pc" :package-pkgconfig)) (filtered-sifs (if *variant-precise* (sort sifs (lambda (x y) @@ -396,13 +415,77 @@ (ninja:write-build output-stream :phony :outputs (list (build-name "generated")) :inputs generated) + (if (static-linking-p configuration) + (ninja:write-build output-stream :ar + :inputs objects + :outputs (list libclasp)) + (ninja:write-build output-stream :link-lib + :variant-ldflags *variant-ldflags* + :variant-ldlibs *variant-ldlibs* + :libname libclasp-name + :inputs objects + :outputs (list libclasp))) + (ninja:write-build output-stream :phony + :inputs (list libclasp) + :outputs (list (build-name target))) + (when *variant-default* + (loop for input in generated + for output in products + do (ninja:write-build output-stream :install-file + :inputs (list input) + :outputs (list output))) + (ninja:write-build output-stream :install-file + :inputs (list (make-source "config.h" :variant)) + :outputs (list (make-source "include/config.h" :package-share))) + (ninja:write-build output-stream :install-file + :inputs (list libclasp) + :outputs (list libclasp-installed)) + (unless (static-linking-p configuration) + (ninja:write-build output-stream :install-file + :inputs (list (make-source "libclasp.pc" :build)) + :outputs (list libclasp-pc-installed))) + (ninja:write-build output-stream :phony + :inputs (list* libclasp-installed + (if (static-linking-p configuration) + products + (cons libclasp-pc-installed + products))) + :outputs (list "install_lib")))) + +(defmethod print-variant-target-sources + (configuration (name (eql :ninja)) output-stream (target (eql :iclasp)) sources + &key objects sifs &allow-other-keys + &aux (exe (make-source "iclasp" :variant)) + (exe-installed (make-source "iclasp" :package-bin)) + (libiclasp-name (lib-filename configuration "libiclasp")) + (libiclasp (make-source libiclasp-name :variant-lib)) + (libiclasp-installed (make-source libiclasp-name :package-lib)) + (libclasp (make-source (lib-filename configuration "libclasp" :dynamic t) :variant-lib)) + (symlink (make-source (if (member :cando (extensions configuration)) + "cando" + "clasp") + :variant)) + (symlink-installed (make-source (if (member :cando (extensions configuration)) + "cando" + "clasp") + :package-bin)) + (clasp-sh (make-source "clasp" :variant)) + (clasp-sh-installed (make-source "clasp" :package-bin)) + (cleap-symlink (make-source "cleap" :variant)) + (cleap-symlink-installed (make-source "cleap" :package-bin))) (ninja:write-build output-stream :ar :inputs objects - :outputs (list lib)) + :outputs (list libiclasp)) (ninja:write-build output-stream :link :variant-ldflags *variant-ldflags* - :variant-ldlibs *variant-ldlibs* + :variant-ldlibs (concatenate 'string + (if (static-linking-p configuration) + #+darwin "-Wl,-all_load -lclasp -Wl,-noall_load " + #-darwin "-Wl,-whole-archive -lclasp -Wl,-no-whole-archive" + "-lclasp ") + *variant-ldlibs*) :inputs objects + :order-only-inputs (list libclasp) :outputs (list exe)) (ninja:write-build output-stream :symbolic-link :inputs (list exe) @@ -414,7 +497,7 @@ :target (file-namestring (source-path exe)) :outputs (list cleap-symlink))) (ninja:write-build output-stream :phony - :inputs (append (list exe symlink lib) + :inputs (append (list exe symlink libiclasp) (when (member :cando (extensions configuration)) (list cleap-symlink)) (when (and *variant-default* @@ -424,17 +507,12 @@ (list "tags"))) :outputs (list (build-name target))) (when *variant-default* - (loop for input in generated - for output in products - do (ninja:write-build output-stream :install-file - :inputs (list input) - :outputs (list output))) (ninja:write-build output-stream :install-binary :inputs (list exe) :outputs (list exe-installed)) (ninja:write-build output-stream :install-file - :inputs (list lib) - :outputs (list lib-installed)) + :inputs (list libiclasp) + :outputs (list libiclasp-installed)) (ninja:write-build output-stream :symbolic-link :inputs (list exe-installed) :target (file-namestring (source-path exe-installed)) @@ -451,14 +529,12 @@ :inputs (append (list "install_code" "install_extension_code" "install_bin" - (make-source "virtualMachine.h" :installed-generated) - (make-source "trampoline.h" :installed-generated) + "install_lib" + libiclasp-installed exe-installed - lib-installed symlink-installed) (when (member :cando (extensions configuration)) - (list clasp-sh-installed cleap-symlink-installed)) - products) + (list clasp-sh-installed cleap-symlink-installed))) :outputs (list "install_iclasp")))) (defun make-kernel-source-list (configuration sources) @@ -496,9 +572,10 @@ name (fasl-extension configuration))) (output (make-source module-name :variant-lib)) (install-output (make-source module-name :package-lib)) - (iclasp (make-source "iclasp" :variant))) + (iclasp (make-source "iclasp" :variant)) + (clasp-with-env (wrap-with-env configuration iclasp))) (ninja:write-build output-stream :compile-module - :clasp iclasp + :clasp clasp-with-env :inputs (list source) :implicit-inputs (list iclasp image) :source (format nil "\"~/ninja:escape/\"" @@ -516,18 +593,19 @@ (defmethod print-variant-target-sources (configuration (name (eql :ninja)) output-stream (target (eql :cclasp)) sources &key &allow-other-keys) - (let ((vimage (image-source configuration nil)) - (vimage-installed (image-source configuration nil :package-lib)) - (iclasp (make-source "iclasp" :variant))) + (let* ((vimage (image-source configuration nil)) + (vimage-installed (image-source configuration nil :package-lib)) + (iclasp (make-source "iclasp" :variant)) + (clasp-with-env (wrap-with-env configuration iclasp))) (ninja:write-build output-stream :load-cclasp - :clasp iclasp + :clasp clasp-with-env :source (make-kernel-source-list configuration sources) :inputs sources :implicit-inputs (list iclasp (make-source "tools-for-build/character-names.sexp" :code)) :outputs (list (build-name "load_cclasp"))) (ninja:write-build output-stream :compile-cclasp - :clasp iclasp + :clasp clasp-with-env :source (make-kernel-source-list configuration sources) :inputs sources :implicit-inputs (list iclasp @@ -544,7 +622,7 @@ (otherwise "link-fasl-abc")) :variant-ldflags *variant-ldflags* :variant-ldlibs *variant-ldlibs* - :clasp iclasp + :clasp clasp-with-env :target "cclasp" :inputs (mapcar (lambda (x) (make-source-output x @@ -579,7 +657,7 @@ :name build-name :bin-path clasp :load-system 1 - :clasp clasp) + :clasp clasp-with-env) collect output))) (ninja:write-build output-stream :phony :inputs kernels @@ -610,7 +688,7 @@ :variant-path *variant-path* :bin-path name :load-system 1 - :clasp clasp) + :clasp clasp-with-env) collect output)))) (ninja:write-build output-stream :install-file :inputs (list vimage) @@ -637,13 +715,14 @@ (configuration (name (eql :ninja)) output-stream (target (eql :eclasp)) sources &key &allow-other-keys) (when (extensions configuration) - (let ((cimage (image-source configuration nil)) + (let* ((cimage (image-source configuration nil)) (eimage (image-source configuration t)) (eimage-installed (image-source configuration t :package-lib)) (iclasp (make-source "iclasp" :variant)) + (clasp-with-env (wrap-with-env configuration iclasp)) (eclasp-sources (member #P"src/lisp/kernel/stage/extension/0-begin.lisp" sources :key #'source-path :test #'equal))) (ninja:write-build output-stream :load-eclasp - :clasp iclasp + :clasp clasp-with-env :source (make-kernel-source-list configuration sources) :inputs sources :position (- (length sources) (length eclasp-sources)) @@ -653,7 +732,7 @@ (make-source "tools-for-build/character-names.sexp" :code)) :outputs (list (build-name "load_eclasp"))) (ninja:write-build output-stream :compile-eclasp - :clasp iclasp + :clasp clasp-with-env :image cimage :position (- (length sources) (length eclasp-sources)) :source (make-kernel-source-list configuration eclasp-sources) @@ -672,7 +751,7 @@ (otherwise "link-fasl-abc")) :variant-ldflags *variant-ldflags* :variant-ldlibs *variant-ldlibs* - :clasp iclasp + :clasp clasp-with-env :target "eclasp" :inputs (mapcar (lambda (x) (make-source-output x @@ -697,36 +776,37 @@ (defmethod print-variant-target-sources (configuration (name (eql :ninja)) output-stream (target (eql :regression-tests)) sources - &key &allow-other-keys) + &key &allow-other-keys + &aux (clasp (wrap-with-env configuration (make-source "iclasp" :variant)))) (ninja:write-build output-stream :regression-tests - :clasp (make-source "iclasp" :variant) + :clasp clasp :inputs (list (build-name "cclasp")) :outputs (list (build-name "test"))) (ninja:write-build output-stream :bench - :clasp (make-source "iclasp" :variant) + :clasp clasp :inputs (list (build-name "cclasp")) :outputs (list (build-name "bench"))) (ninja:write-build output-stream :ansi-test - :clasp (make-source "iclasp" :variant) + :clasp clasp :inputs (list (build-name "cclasp")) :outputs (list (build-name "ansi-test"))) (ninja:write-build output-stream :asdf-test - :clasp (make-source "iclasp" :variant) + :clasp clasp :target "t" :inputs (list (build-name "cclasp")) :outputs (list (build-name "asdf-test"))) (ninja:write-build output-stream :asdf-test - :clasp (make-source "iclasp" :variant) + :clasp clasp :target "u" :inputs (list (build-name "cclasp")) :outputs (list (build-name "asdf-test-upgrade"))) (ninja:write-build output-stream :test-random-integer - :clasp (make-source "iclasp" :variant) + :clasp clasp :inputs (list (build-name "cclasp")) :outputs (list (build-name "test-random-integer"))) (when (member :cando (extensions configuration)) (ninja:write-build output-stream :cando-regression-tests - :clasp (make-source "iclasp" :variant) + :clasp clasp :inputs (list (build-name "eclasp")) :outputs (list (build-name "cando-test")))) (when *variant-default* @@ -760,7 +840,7 @@ (configuration (name (eql :ninja)) output-stream (target (eql :analyzer)) sources &key &allow-other-keys) (ninja:write-build output-stream :compile-systems - :clasp (make-source "iclasp" :variant) + :clasp (wrap-with-env configuration (make-source "iclasp" :variant)) :inputs sources :implicit-inputs (list (build-name "cclasp")) :systems "clasp-analyzer" @@ -776,7 +856,7 @@ (declare (ignore configuration name target)) (unless (or *variant-prep* *variant-precise* *variant-debug*) (ninja:write-build output-stream :analyze-file - :clasp (make-source "iclasp" :variant) + :clasp (wrap-with-env configuration (make-source "iclasp" :variant)) :inputs (list source) :implicit-inputs (list (build-name "cclasp") (build-name "generated" :gc :boehm) @@ -796,7 +876,7 @@ :code))) (unless (or *variant-prep* *variant-precise* *variant-debug*) (ninja:write-build output-stream :analyze-generate - :clasp (make-source "iclasp" :variant) + :clasp (wrap-with-env configuration (make-source "iclasp" :variant)) :inputs outputs :implicit-inputs (list (build-name "cclasp") (build-name "generated" :gc :boehm) @@ -823,7 +903,8 @@ (configuration (name (eql :ninja)) output-stream (target (eql :sclasp)) sources &key objects &allow-other-keys &aux (cclasp (build-name (if (extensions configuration) :eclasp :cclasp))) - (iclasp (make-source "iclasp" :variant))) + (iclasp (make-source "iclasp" :variant)) + (clasp-with-env (wrap-with-env configuration iclasp))) (declare (ignore objects)) (flet ((snapshot (name &key ignore-extensions) (let* ((executable (make-source (build-name name) :variant)) @@ -848,7 +929,7 @@ "cando") (build-name name))))) (ninja:write-build output-stream :make-snapshot - :clasp iclasp + :clasp clasp-with-env :arguments (when ignore-extensions "--base --feature ignore-extensions") :variant-path *variant-path* diff --git a/src/koga/scripts.lisp b/src/koga/scripts.lisp index f7780eaf1e..8c9a7d7035 100644 --- a/src/koga/scripts.lisp +++ b/src/koga/scripts.lisp @@ -353,3 +353,80 @@ make $2 l=clasp CLASP=$CLASP")) (write-string (alexandria:read-file-into-string (second (uiop:command-line-arguments))) stream) (write-line \")trampoline\\\";\" stream))")) + +(defun path-flag-p (flag) + (and (> (length flag) 1) + (char= (char flag 0) #\-) + (find (char flag 1) "IL"))) + +(defun escape-pc-value (value) + (with-output-to-string (stream) + (loop for ch across value + when (find ch "#\\") + do (write-char #\\ stream) + do (write-char ch stream)))) + +(defun absolutify-flags (configuration flags) + (format nil "~{~a~^ ~}" + (mapcar (lambda (flag) + (if (path-flag-p flag) + (let ((path (uiop:ensure-directory-pathname (subseq flag 2)))) + (concatenate 'string (subseq flag 0 2) + (namestring (if (uiop:absolute-pathname-p path) + path + (uiop:ensure-absolute-pathname (merge-pathnames (uiop:ensure-directory-pathname path) + (build-path configuration)) + (uiop:getcwd)))))) + flag)) + (split-sequence:split-sequence #\space flags + :remove-empty-subseqs t)))) + +(defun write-pc (configuration output-stream cflags libs &key remove-include) + (flet ((normalize-version (version) + (if (char= #\. (char version 0)) + (concatenate 'string "0" version) + version))) + (format output-stream "~ +Name: Clasp library core +Description: Common dynamic core of Clasp +URL: https://github.com/clasp-developers/clasp +Version: ~a +~@[Requires: ~{~a~^, ~}~] +Cflags: ~a ~a +Libs: ~a -lclasp~%" + (version configuration) + (loop for (name min-version max-version) in (libraries configuration) + when (and (null min-version) + (null max-version)) + collect name + else when (equalp min-version max-version) + collect (format nil "~a = ~a" name (normalize-version min-version)) + else when min-version + collect (format nil "~a >= ~a" name (normalize-version min-version)) + else + collect (format nil "~a <= ~a" name (normalize-version max-version))) + (escape-pc-value + (if remove-include + (format nil "~{~a~^ ~}" + (remove-if #'path-flag-p + (split-sequence:split-sequence #\space (cxxflags configuration) + :remove-empty-subseqs t))) + (absolutify-flags configuration (cxxflags configuration)))) + (escape-pc-value cflags) + (escape-pc-value libs)))) + +(defmethod print-prologue (configuration (name (eql :libclasp-pc)) output-stream) + (write-pc configuration output-stream + (format nil "-I~a -I~a" + (root :install-share) + (make-source "include/" :install-share)) + "" + :remove-include t)) + +(defmethod print-prologue (configuration (name (eql :libclasp-pc-variant)) output-stream) + (write-pc configuration output-stream + (absolutify-flags configuration *variant-cxxflags*) + (absolutify-flags configuration + (format nil "~a ~a" + *variant-ldflags* + *variant-ldlibs*)))) diff --git a/src/koga/setup.lisp b/src/koga/setup.lisp index de1d5fcc65..fc0a6d2424 100644 --- a/src/koga/setup.lisp +++ b/src/koga/setup.lisp @@ -239,7 +239,9 @@ writing the build and variant outputs." :install-generated install-generated :package-bin (resolve-package-path (bin-path *configuration*)) :package-share (resolve-package-path (share-path *configuration*)) + :package-dylib (resolve-package-path (dylib-path *configuration*)) :package-lib (resolve-package-path (lib-path *configuration*)) + :package-pkgconfig (resolve-package-path (pkgconfig-path *configuration*)) :package-generated (resolve-package-path install-generated) *root-paths*)) (*extensions* (extensions *configuration*))) diff --git a/src/koga/target-sources.lisp b/src/koga/target-sources.lisp index 28ed1a0a51..632358f1c5 100644 --- a/src/koga/target-sources.lisp +++ b/src/koga/target-sources.lisp @@ -66,14 +66,20 @@ systems) do (add-target-source configuration target system)))) -;; Sources that are added to iclasp also need to be installed and scanned for tags. -(defmethod add-target-source :after (configuration (target (eql :iclasp)) (source source)) +;; Sources that are added to libclasp also need to be installed and scanned for tags. +(defmethod add-target-source :after (configuration (target (eql :libclasp)) (source source)) (when (eq :code (source-root source)) (add-target-source configuration :install-code source) (add-target-source configuration :tags source)) (add-target-source configuration :sclasp source) (add-target-source configuration :analyze source)) +;; Sources that are added to iclasp also need to be installed and scanned for tags. +(defmethod add-target-source :after (configuration (target (eql :iclasp)) (source source)) + (when (eq :code (source-root source)) + (add-target-source configuration :install-code source) + (add-target-source configuration :tags source))) + ;; Sources that are added to cclasp also need to be installed and scanned for tags. (defmethod add-target-source :after (configuration (target (eql :cclasp)) (source source)) (when (eq :code (source-root source)) diff --git a/src/koga/units.lisp b/src/koga/units.lisp index 2e6c2e8ce8..71b49c592d 100644 --- a/src/koga/units.lisp +++ b/src/koga/units.lisp @@ -200,7 +200,8 @@ -Wno-deprecated-anon-enum-enum-conversion")) (loop for variant in (variants configuration) do (append-cflags variant (format nil "-I~a" (variant-bitcode-name variant))) - (append-cflags variant (format nil "-I~a/generated" (variant-bitcode-name variant)))) + (append-cflags variant (format nil "-I~a/generated" (variant-bitcode-name variant))) + (append-ldflags variant (format nil "-L~a/lib" (variant-bitcode-name variant)))) (append-cflags configuration "-O3 -g -fPIC" :type :cxxflags :debug nil) (append-cflags configuration "-O3 -g -fPIC" :type :cflags :debug nil) (append-cflags configuration "-O0 -g" :type :cxxflags :debug t) @@ -303,22 +304,35 @@ has not been set." (defmethod configure-unit (configuration (unit (eql :reproducible))) "Configure for a reproducible build." - (when (reproducible-build configuration) - (message :emph "Configuring reproducible build") - (append-cflags configuration - (format nil "-ffile-prefix-map=..=~a" - (normalize-directory (root :install-share)))) - (loop for variant in (variants configuration) - do (append-cflags variant - (format nil "-ffile-prefix-map=~a=~a -ffile-prefix-map=~a=~a" - (normalize-directory (make-pathname :directory (list :relative - (variant-bitcode-name variant) - "generated"))) - (normalize-directory (root :install-generated)) - (normalize-directory (make-pathname :directory (list :relative - (variant-bitcode-name variant) - "lib"))) - (normalize-directory (root :install-lib))))))) + (cond ((reproducible-build configuration) + (message :emph "Configuring reproducible build") + (append-cflags configuration + (format nil "-ffile-prefix-map=..=~a" + (normalize-directory (root :install-share)))) + (loop for variant in (variants configuration) + do (append-cflags variant + (format nil "-ffile-prefix-map=~a=~a -ffile-prefix-map=~a=~a" + (normalize-directory (make-pathname :directory (list :relative + (variant-bitcode-name variant) + "generated"))) + (normalize-directory (root :install-generated)) + (normalize-directory (make-pathname :directory (list :relative + (variant-bitcode-name variant) + "lib"))) + (normalize-directory (root :install-lib)))))) + (t + (message :emph "Configuring non-reproducible build") + (loop for variant in (variants configuration) + do (append-ldflags variant + (format nil "-Wl,-rpath,~a" + (normalize-directory + (uiop:ensure-absolute-pathname + (merge-pathnames + (make-pathname :directory (list :relative + (variant-bitcode-name variant) + "lib")) + (root :build)) + (uiop:getcwd))))))))) (defmethod configure-unit (configuration (unit (eql :asdf))) "Configure ASDF" diff --git a/src/lisp/kernel/lsp/fli.lisp b/src/lisp/kernel/lsp/fli.lisp index 0cc2719182..ca35c31233 100644 --- a/src/lisp/kernel/lsp/fli.lisp +++ b/src/lisp/kernel/lsp/fli.lisp @@ -54,7 +54,7 @@ (format t "#'ensure-core-pointer *** Illegal argument value: PTR may not be NIL - fn = ~a info = ~a!~%" fn info) (gctools:wait-for-user-signal "Bad ensure-core-pointer Send SIGUSR1 to continue") (error "#'ensure-core-ptr *** Illegal argument value: PTR may not be NIL!")) - ((eql (type-of ptr) 'CLASP-FFI::FOREIGN-DATA) (%core-pointer-from-foreign-data ptr )) + ((eql (type-of ptr) 'CLASP-FFI::FOREIGN-DATA) (%core-pointer-from-foreign-data ptr)) ((eql (type-of ptr) 'CORE::POINTER) ptr) (t (error "#'ensure-core-pointer *** Illegal type ~S of ptr. Cannot convert to core::pointer." (type-of ptr))))) @@ -354,9 +354,14 @@ (declare (ignore name)) (multiple-value-bind (handle error) (%dlopen path) - (if (not handle) - (error "~A" error) - handle))) + (cond ((not handle) + (error "~A" error)) + (t + (gctools:register-loaded-objects) + (let ((ptr (%foreign-symbol-pointer "startup_clasp_extension" handle))) + (when ptr + (%foreign-funcall-pointer ptr :void))) + handle)))) (declaim (inline %close-foreign-library)) (defun %close-foreign-library (ptr) @@ -383,10 +388,9 @@ ;;; === F O R E I G N G L O B A L S === (declaim (inline %foreign-symbol-pointer)) -(defun %foreign-symbol-pointer (name module) +(defun %foreign-symbol-pointer (name &optional module) "Return a pointer (of type ForeignData_sp / FOREIGN_DATA to a foreign symbol." - (declare (ignore module)) - (%dlsym name)) + (%dlsym (or module :rtld-default) name)) ;;;---------------------------------------------------------------------------- ;;; diff --git a/src/llvmo/cscript.lisp b/src/llvmo/cscript.lisp index 80717975b5..ebcf9abee5 100644 --- a/src/llvmo/cscript.lisp +++ b/src/llvmo/cscript.lisp @@ -1,4 +1,4 @@ -(k:sources :iclasp +(k:sources :libclasp #~"debugInfoExpose.cc" #~"debugLoc.cc" #~"llvmoDwarf.cc" diff --git a/src/main/cscript.lisp b/src/main/cscript.lisp index feff33a98b..86d80e7af5 100644 --- a/src/main/cscript.lisp +++ b/src/main/cscript.lisp @@ -1,2 +1 @@ (k:sources :iclasp #~"main.cc") - diff --git a/src/mpip/cscript.lisp b/src/mpip/cscript.lisp index c930335e8d..2de34075ed 100644 --- a/src/mpip/cscript.lisp +++ b/src/mpip/cscript.lisp @@ -1 +1 @@ -(k:sources :iclasp #~"claspMpi.cc") +(k:sources :libclasp #~"claspMpi.cc") diff --git a/src/serveEvent/cscript.lisp b/src/serveEvent/cscript.lisp index def70a094e..c8e0cde645 100644 --- a/src/serveEvent/cscript.lisp +++ b/src/serveEvent/cscript.lisp @@ -1,3 +1,3 @@ -(k:sources :iclasp +(k:sources :libclasp #~"serveEvent.cc" #~"serveEventPackage.cc") diff --git a/src/sockets/cscript.lisp b/src/sockets/cscript.lisp index 5e56608f96..85494bd6ae 100644 --- a/src/sockets/cscript.lisp +++ b/src/sockets/cscript.lisp @@ -1,3 +1,3 @@ -(k:sources :iclasp +(k:sources :libclasp #~"sockets.cc" #~"socketsPackage.cc")