Skip to content

Commit

Permalink
Add dynamic extensions (#1641)
Browse files Browse the repository at this point in the history
  • Loading branch information
@yitzchak
yitzchak authored Dec 11, 2024
1 parent 7ab33f5 commit bf5aaed
Show file tree
Hide file tree
Showing 38 changed files with 509 additions and 214 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/pkg.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ jobs:
run: |
sudo apt-get update
sudo apt install -y sbcl
./tools-for-build/nightly-version-bump --extensions=cando,seqan-clasp
./tools-for-build/nightly-version-bump --extensions=cando
rm -rf dependencies extensions src/lisp/kernel/contrib src/lisp/modules/asdf src/mps src/bdwgc src/libatomic_ops
- name: Build Ubuntu packages
uses: jtdor/build-deb-action@v1
Expand Down Expand Up @@ -152,7 +152,7 @@ jobs:
run: |
sudo apt-get update
sudo apt install -y sbcl
./tools-for-build/nightly-version-bump --extensions=cando,seqan-clasp
./tools-for-build/nightly-version-bump --extensions=cando
echo "(:skip-sync t)" > config.sexp
- name: Build Debian packages
uses: jtdor/build-deb-action@v1
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ jobs:
if: matrix.os == 'ubuntu-latest'
run: |
sudo apt-get update
sudo apt install -y binutils-gold clang-15 libclang-15-dev libclang-cpp15-dev llvm-15 llvm-15-dev libelf-dev libgmp-dev libunwind-dev ninja-build sbcl libnetcdf-dev libexpat1-dev libfmt-dev libboost-all-dev
sudo apt install -y binutils-gold clang-18 libclang-18-dev libclang-cpp18-dev llvm-18 llvm-18-dev libelf-dev libgmp-dev libunwind-dev ninja-build sbcl libnetcdf-dev libexpat1-dev libfmt-dev libboost-all-dev
- name: Install MacOS dependencies
if: matrix.os == 'macos-13' || matrix.os == 'macos-14'
Expand All @@ -87,7 +87,7 @@ jobs:
- name: Cando koga @ Ubuntu
if: ${{ matrix.os == 'ubuntu-latest' && matrix.build == 'cando' }}
run: |
./koga --build-mode=${{ matrix.mode }} --extensions=cando,seqan-clasp
./koga --build-mode=${{ matrix.mode }} --extensions=cando
- name: Clasp koga @ MacOS=13
if: ${{ matrix.os == 'macos-13' && matrix.build == 'clasp' }}
run: |
Expand All @@ -99,11 +99,11 @@ jobs:
- name: Cando koga @ MacOS-13
if: ${{ matrix.os == 'macos-13' && matrix.build == 'cando' }}
run: |
./koga --build-mode=${{ matrix.mode }} --extensions=cando,seqan-clasp
./koga --build-mode=${{ matrix.mode }} --extensions=cando
- name: Cando koga @ MacOS-14
if: ${{ matrix.os == 'macos-14' && matrix.build == 'cando' }}
run: |
./koga --build-mode=${{ matrix.mode }} --extensions=cando,seqan-clasp --cflags=-I/opt/homebrew/include --cppflags=-I/opt/homebrew/include --cxxflags=-I/opt/homebrew/include
./koga --build-mode=${{ matrix.mode }} --extensions=cando --cflags=-I/opt/homebrew/include --cppflags=-I/opt/homebrew/include --cxxflags=-I/opt/homebrew/include
- name: Build
run: |
ninja -C build
Expand Down
14 changes: 7 additions & 7 deletions debian/control
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ Source: clasp-cl
Section: lisp
Priority: optional
Maintainer: Tarn W. Burton <[email protected]>
Build-Depends: debhelper-compat (= 13), libelf-dev, libgmp-dev, llvm-15,
clang-15, sbcl, libclang-15-dev, llvm-15-dev, ninja-build, libfmt-dev,
git, pkg-config, libboost-all-dev, libclang-cpp15, libclang-cpp15-dev,
Build-Depends: debhelper-compat (= 13), libelf-dev, libgmp-dev, llvm-18,
clang-18, sbcl, libclang-18-dev, llvm-18-dev, ninja-build, libfmt-dev,
git, pkg-config, libboost-all-dev, libclang-cpp18, libclang-cpp18-dev,
libnetcdf-dev, libczmq-dev, libexpat1-dev
Standards-Version: 4.5.1
Homepage: https://github.com/clasp-developers/clasp
Expand All @@ -20,7 +20,7 @@ Provides: lisp-compiler
Architecture: any
Build-Profiles: <!noclasp>
Depends: ${shlibs:Depends}, ${misc:Depends}, libelf1, libgmp10, libgmpxx4ldbl,
llvm-15, clang-15, libclang-cpp15
llvm-18, clang-18, libclang-cpp18
Description: Common Lisp implementation that brings Common Lisp and C++ Together
Clasp is a new Common Lisp implementation that seamlessly interoperates
with C++ libraries and programs using LLVM for compilation to native
Expand All @@ -37,9 +37,9 @@ Conflicts: clasp-cl
Provides: lisp-compiler, clasp-cl
Architecture: any
Build-Profiles: <!nocando>
Depends: ${shlibs:Depends}, ${misc:Depends}, libelf-dev, libgmp-dev, llvm-15,
clang-15, sbcl, libclang-15-dev, llvm-15-dev, libfmt-dev,
libclang-cpp15, libclang-cpp15-dev, libnetcdf-dev, libczmq-dev,
Depends: ${shlibs:Depends}, ${misc:Depends}, libelf-dev, libgmp-dev, llvm-18,
clang-18, sbcl, libclang-18-dev, llvm-18-dev, libfmt-dev,
libclang-cpp18, libclang-cpp18-dev, libnetcdf-dev, libczmq-dev,
libexpat1-dev, ninja-build
Description: Common Lisp implementation that brings Common Lisp and C++ Together
Clasp is a new Common Lisp implementation that seamlessly interoperates
Expand Down
4 changes: 2 additions & 2 deletions debian/rules
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ override_dh_auto_test:

override_dh_auto_configure:
ifneq (,$(filter clasp-cl,$(DOPACKAGES)))
./koga --build-mode=bytecode-faso --reproducible-build --build-path=clasp-build/ --ldlibs=-ldl --package-path=$$(pwd)/debian/clasp-cl/ --bin-path=/usr/bin/ --share-path=/usr/share/clasp/ --lib-path=/usr/lib/clasp/
./koga --build-mode=bytecode-faso --reproducible-build --build-path=clasp-build/ --ldlibs=-ldl --package-path=$$(pwd)/debian/clasp-cl/ --bin-path=/usr/bin/ --share-path=/usr/share/clasp/ --lib-path=/usr/lib/clasp/ --dylib-path=/usr/lib/ --pkgconfig-path=/usr/lib/pkgconfig/
endif
ifneq (,$(filter cando,$(DOPACKAGES)))
./koga --build-mode=bytecode-faso --reproducible-build --build-path=cando-build/ --extensions=cando,seqan-clasp --ldlibs=-ldl --package-path=$$(pwd)/debian/cando/ --bin-path=/usr/bin/ --share-path=/usr/share/clasp/ --lib-path=/usr/lib/clasp/
./koga --build-mode=bytecode-faso --reproducible-build --build-path=cando-build/ --extensions=cando --ldlibs=-ldl --package-path=$$(pwd)/debian/cando/ --bin-path=/usr/bin/ --share-path=/usr/share/clasp/ --lib-path=/usr/lib/clasp/ --dylib-path=/usr/lib/ --pkgconfig-path=/usr/lib/pkgconfig/
endif

override_dh_auto_build:
Expand Down
8 changes: 2 additions & 6 deletions guix/cando.scm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,9 @@
(apply append
(list (string-append source-dir "/src/bdwgc")
(string-append source-dir "/src/libatomic_ops")
(string-append source-dir "/src/lisp/modules/asdf")
;; This is a special case: the parent is omitted
;; from external-dirs, but added to descend-dirs.
(string-append source-dir "/extensions/seqan-clasp/seqan"))
(string-append source-dir "/src/lisp/modules/asdf"))
(map-in-order scandir-absolute external-dirs)))
(descend-dirs `(,@git-dirs
,(string-append source-dir "/extensions/seqan-clasp")
,@external-dirs))
(predicates (map-in-order git-predicate (cons source-dir git-dirs))))
(package
Expand All @@ -60,7 +56,7 @@
(substitute-keyword-arguments (package-arguments clasp-cl)
((#:configure-flags flags '())
#~(cons*
"--extensions=cando,seqan-clasp"
"--extensions=cando"
;; Had trouble simplifying this using find.
;; This is prepended because Koga keeps the first instance
;; of repeated options.
Expand Down
8 changes: 5 additions & 3 deletions guix/clasp.scm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,9 +89,11 @@
#~(let ((clang #$(this-package-input "clang-toolchain")))
(list "--build-mode=bytecode-faso"
"--reproducible-build"
(string-append "--bin-path=" #$output "/bin")
(string-append "--lib-path=" #$output "/lib/clasp")
(string-append "--share-path=" #$output "/share/clasp")
(string-append "--bin-path=" #$output "/bin/")
(string-append "--lib-path=" #$output "/lib/clasp/")
(string-append "--share-path=" #$output "/share/clasp/")
(string-append "--dylib-path=" #$output "/lib/")
(string-append "--pkgconfig-path=" #$output "/lib/pkgconfig/")
;; Without --cc and --cxx, Clang is searched for in
;; LLVM's /bin.
(string-append "--cc=" clang "/bin/clang")
Expand Down
23 changes: 12 additions & 11 deletions include/clasp/core/debugger.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,25 +81,26 @@ struct OpenDynamicLibraryInfo {
gctools::clasp_ptr_t _LibraryStart;
gctools::clasp_ptr_t _TextStart;
gctools::clasp_ptr_t _TextEnd;
bool _HasVtableSection;
gctools::clasp_ptr_t _VtableSectionStart;
gctools::clasp_ptr_t _VtableSectionEnd;
OpenDynamicLibraryInfo(const std::string& f, void* h, const SymbolTable& symbol_table, gctools::clasp_ptr_t libstart,
gctools::clasp_ptr_t textStart, gctools::clasp_ptr_t textEnd)
: _Filename(f), _Handle(h), _SymbolTable(symbol_table), _LibraryStart(libstart), _TextStart(textStart),
_TextEnd(textEnd){
gctools::clasp_ptr_t textStart, gctools::clasp_ptr_t textEnd,
bool hasVtableSection,
gctools::clasp_ptr_t vtableSectionStart, gctools::clasp_ptr_t vtableSectionEnd)
: _Filename(f), _Handle(h), _SymbolTable(symbol_table), _LibraryStart(libstart), _TextStart(textStart), _TextEnd(textEnd),
_HasVtableSection(hasVtableSection), _VtableSectionStart(vtableSectionStart), _VtableSectionEnd(vtableSectionEnd) {
// printf("%s:%d:%s filename = %s\n", __FILE__, __LINE__, __FUNCTION__, f.c_str() );
};
virtual LoadableKind loadableKind() const { return Library; };
OpenDynamicLibraryInfo(){};
};

struct ExecutableLibraryInfo : OpenDynamicLibraryInfo {
bool _HasVtableSection;
gctools::clasp_ptr_t _VtableSectionStart;
gctools::clasp_ptr_t _VtableSectionEnd;
ExecutableLibraryInfo(const std::string& f, void* h, const SymbolTable& symbol_table, gctools::clasp_ptr_t libstart,
gctools::clasp_ptr_t textStart, gctools::clasp_ptr_t textEnd, bool hasVtableSection,
gctools::clasp_ptr_t vtableSectionStart, gctools::clasp_ptr_t vtableSectionEnd)
: OpenDynamicLibraryInfo(f, h, symbol_table, libstart, textStart, textEnd), _HasVtableSection(hasVtableSection),
_VtableSectionStart(vtableSectionStart), _VtableSectionEnd(vtableSectionEnd) {
gctools::clasp_ptr_t textStart, gctools::clasp_ptr_t textEnd,
bool hasVtableSection, gctools::clasp_ptr_t vtableSectionStart, gctools::clasp_ptr_t vtableSectionEnd)
: OpenDynamicLibraryInfo(f, h, symbol_table, libstart, textStart, textEnd, hasVtableSection, vtableSectionStart, vtableSectionEnd ) {
#if 0
printf("%s:%d:%s filename = %s\n", __FILE__, __LINE__, __FUNCTION__, f.c_str() );
if (vtableSectionStart==NULL) {
Expand Down Expand Up @@ -215,6 +216,6 @@ DebugInfo& debugInfo();

void executablePath(std::string& name);
void executableTextSectionRange(gctools::clasp_ptr_t& start, gctools::clasp_ptr_t& end);
void executableVtableSectionRange(gctools::clasp_ptr_t& start, gctools::clasp_ptr_t& end);
void exclusiveVtableSectionRange(gctools::clasp_ptr_t& start, gctools::clasp_ptr_t& end);

}; // namespace core
2 changes: 1 addition & 1 deletion include/clasp/core/fli.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ CL_DEFUN core::T_sp PERCENTdlopen(core::T_sp path_designator);
DOCGROUP(clasp)
CL_DEFUN core::T_sp PERCENTdlclose(ForeignData_sp handle);
DOCGROUP(clasp)
CL_DEFUN core::T_sp PERCENTdlsym(core::String_sp name);
CL_DEFUN core::T_sp PERCENTdlsym(core::T_sp library, core::String_sp name);

}; // namespace clasp_ffi

Expand Down
2 changes: 1 addition & 1 deletion include/clasp/gctools/snapshotSaveLoad.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ struct ISLLibrary {
struct Fixup {
FixupOperation_ _operation;
uintptr_t _memoryStart;
std::vector<ISLLibrary> _Libraries;
std::vector<ISLLibrary> _ISLLibraries;
bool _trackAddressName;
map<void*, std::string> _addressName;

Expand Down
4 changes: 1 addition & 3 deletions include/cscript.lisp
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
(k:includes #~"" #~"clasp/main/")
(k:includes #~"")

(k:sources :install-code
#~"clasp/asttooling/"
Expand All @@ -9,7 +9,6 @@
#~"clasp/external/"
#~"clasp/gctools/"
#~"clasp/llvmo/"
#~"clasp/main/"
#~"clasp/mpip/"
#~"clasp/serveEvent/"
#~"clasp/sockets/"
Expand All @@ -24,7 +23,6 @@
#~"clasp/external/"
#~"clasp/gctools/"
#~"clasp/llvmo/"
#~"clasp/main/"
#~"clasp/mpip/"
#~"clasp/serveEvent/"
#~"clasp/sockets/"
Expand Down
4 changes: 2 additions & 2 deletions src/analysis/cscript.lisp
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
(if (member :cando k:*extensions*)
(k:sources :iclasp #~"clasp_gc_cando.sif")
(k:sources :iclasp #~"clasp_gc.sif"))
(k:sources :libclasp #~"clasp_gc_cando.sif")
(k:sources :libclasp #~"clasp_gc.sif"))
2 changes: 1 addition & 1 deletion src/asttooling/cscript.lisp
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
(k:sources :iclasp
(k:sources :libclasp
#~"astExpose0.cc"
#~"astExpose1.cc"
#~"clangTooling.cc"
Expand Down
2 changes: 1 addition & 1 deletion src/clbind/cscript.lisp
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
(k:sources :iclasp
(k:sources :libclasp
#~"adapter.cc"
#~"class_rep.cc"
#~"open.cc"
Expand Down
2 changes: 1 addition & 1 deletion src/core/cscript.lisp
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
(k:sources :iclasp
(k:sources :libclasp
#~"dummy.cc"
#~"mpPackage.cc"
#~"nativeVector.cc"
Expand Down
7 changes: 5 additions & 2 deletions src/core/debug_macosx.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -350,12 +350,15 @@ void add_dynamic_library_impl(add_dynamic_library* callback, bool is_executable,
if (is_executable) {
odli = new ExecutableLibraryInfo(libraryName, handle, symbol_table, reinterpret_cast<gctools::clasp_ptr_t>(library_origin),
reinterpret_cast<gctools::clasp_ptr_t>(library_origin),
reinterpret_cast<gctools::clasp_ptr_t>(library_origin + text_segment_size), found,
reinterpret_cast<gctools::clasp_ptr_t>(library_origin + text_segment_size),
found,
(gctools::clasp_ptr_t)vtableRegionStart, (gctools::clasp_ptr_t)vtableRegionEnd);
} else {
odli = new OpenDynamicLibraryInfo(libraryName, handle, symbol_table, reinterpret_cast<gctools::clasp_ptr_t>(library_origin),
reinterpret_cast<gctools::clasp_ptr_t>(library_origin),
reinterpret_cast<gctools::clasp_ptr_t>(library_origin + text_segment_size));
reinterpret_cast<gctools::clasp_ptr_t>(library_origin + text_segment_size),
found,
(gctools::clasp_ptr_t)vtableRegionStart, (gctools::clasp_ptr_t)vtableRegionEnd);
}
if (callback)
(*callback)(odli);
Expand Down
23 changes: 10 additions & 13 deletions src/core/debug_unixes.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,23 +304,21 @@ CL_DEFUN void core__walk_loaded_objects() {
}

int elf_startup_loaded_object_callback(struct dl_phdr_info* info, size_t size, void* data) {
// printf("%s:%d:%s Startup registering loaded object %s\n", __FILE__, __LINE__, __FUNCTION__, info->dlpi_name);
// printf("%s:%d:%s --------- Startup registering loaded object %s\n", __FILE__, __LINE__, __FUNCTION__, info->dlpi_name);
core::General_O general;
gctools::clasp_ptr_t vtablePtr = *(gctools::clasp_ptr_t*)&general;
// printf("%s:%d:%s one vtablePtr = %p\n", __FILE__, __LINE__, __FUNCTION__, vtablePtr );
// printf("%s:%d:%s a KNOWN vtablePtr = %p\n", __FILE__, __LINE__, __FUNCTION__, vtablePtr );
ScanInfo* scan_callback_info = (ScanInfo*)data;
bool is_executable = false;
const char* libname;
if (scan_callback_info->_Index == 0 && strlen(info->dlpi_name) == 0) {
libname = getExecutablePath();
is_executable = true;
// printf("%s:%d:%s getExecutablePath() libname %s is_executable = %d\n", __FILE__, __LINE__, __FUNCTION__, libname,
// is_executable );
// printf("%s:%d:%s getExecutablePath() libname %s is_executable = %d\n", __FILE__, __LINE__, __FUNCTION__, libname, is_executable );
} else {
libname = info->dlpi_name;
is_executable = false;
// printf("%s:%d:%s info->dlpi_name libname %s is_executable = %d\n", __FILE__, __LINE__, __FUNCTION__, libname, is_executable
// );
// printf("%s:%d:%s info->dlpi_name libname %s is_executable = %d\n", __FILE__, __LINE__, __FUNCTION__, libname, is_executable );
}
gctools::clasp_ptr_t text_start;
gctools::clasp_ptr_t text_end;
Expand All @@ -335,8 +333,7 @@ int elf_startup_loaded_object_callback(struct dl_phdr_info* info, size_t size, v
if (p_type == PT_LOAD && (info->dlpi_phdr[j].p_flags & 0x1)) { // executable
text_start = (gctools::clasp_ptr_t)(info->dlpi_addr + info->dlpi_phdr[j].p_vaddr);
text_end = (gctools::clasp_ptr_t)(text_start + info->dlpi_phdr[j].p_memsz);
// printf("%s:%d:%s text_start = %p text_end = %p\n low = %p high = %p vtablePtr = %p\n", __FILE__, __LINE__,
// __FUNCTION__, (void*)text_start, (void*)text_end, low, high, vtablePtr );
// printf("%s:%d:%s text_start = %p text_end = %p low = %p high = %p\n", __FILE__, __LINE__, __FUNCTION__, (void*)text_start, (void*)text_end, low, high );
}
if (low <= vtablePtr && vtablePtr < high) {
//
Expand All @@ -354,9 +351,9 @@ int elf_startup_loaded_object_callback(struct dl_phdr_info* info, size_t size, v
hasVtableSection = true;
vtableSectionStart = low;
vtableSectionEnd = high;
// printf("%s:%d:%s set vtableSection Start/End = %p/%p\n", __FILE__, __LINE__, __FUNCTION__, low, high );
// printf("%s:%d:%s set vtableSection Start/End = %p/%p\n", __FILE__, __LINE__, __FUNCTION__, low, high );
}
// printf("%s:%d:%s Found vtableSection %p to %p\n", __FILE__, __LINE__, __FUNCTION__, vtableSectionStart, vtableSectionEnd );
// printf("%s:%d:%s Found vtableSection %p to %p\n", __FILE__, __LINE__, __FUNCTION__, vtableSectionStart, vtableSectionEnd );
}
}
// printf("%s:%d:%s About to call add_dynamic_library_using_origin libname = %s is_executable = %d\n", __FILE__, __LINE__,
Expand All @@ -377,8 +374,7 @@ void startup_register_loaded_objects(add_dynamic_library* callback) {
otherwise it uses handle to look up the start of the library. */
void add_dynamic_library_impl(add_dynamic_library* callback, bool is_executable, const std::string& libraryName, bool use_origin,
uintptr_t library_origin, void* handle, gctools::clasp_ptr_t text_start,
gctools::clasp_ptr_t text_end, bool hasVtableSection, gctools::clasp_ptr_t vtableSectionStart,
gctools::clasp_ptr_t vtableSectionEnd) {
gctools::clasp_ptr_t text_end, bool hasVtableSection, gctools::clasp_ptr_t vtableSectionStart, gctools::clasp_ptr_t vtableSectionEnd) {
// printf("%s:%d:%s libraryName = %s is_executable = %d\n", __FILE__, __LINE__, __FUNCTION__, libraryName.c_str(), is_executable
// );
BT_LOG(("Starting to load library: %s\n", libraryName.c_str()));
Expand Down Expand Up @@ -421,7 +417,8 @@ void add_dynamic_library_impl(add_dynamic_library* callback, bool is_executable,
} else {
// printf("%s:%d:%s Creating an OpenDynamicLibraryInfo\n", __FILE__, __LINE__, __FUNCTION__ );
odli =
new OpenDynamicLibraryInfo(libraryName, handle, symbol_table, (gctools::clasp_ptr_t)library_origin, text_start, text_end);
new OpenDynamicLibraryInfo(libraryName, handle, symbol_table, (gctools::clasp_ptr_t)library_origin, text_start, text_end,
hasVtableSection, vtableSectionStart, vtableSectionEnd );
loadable = Library;
}
TrapProblem trap(is_executable, libraryName, odli->loadableKind());
Expand Down
Loading

0 comments on commit bf5aaed

Please sign in to comment.