GitHub actions for testing and release (#2)

* add executable bits

* github actions for tests

* fix line endings, print test logs to stdout

* add exec permissions to remaining scripts

* fix windows test env

* treat log files as binary

* print generated CMakeLists.txt in test_openssl

* skip test_clang_multiarch on Linux

* when generating target_link_options, don't join flags that span multiple arguments into one string

* add .gitignore

* fix test_def_file_generated_during_build for systems, where C:\Temp exists

* rename github actions

* github action that uploads release asset

.gitattributes

@@ -0,0 +1,3 @@
+# Preserve line endings in these files
+*.log binary
+*.log.in binary

.github/workflows/linux.yaml

@@ -0,0 +1,20 @@
+name: Linux
+
+on:
+  pull_request:
+  push:
+
+jobs:
+  test:
+    name: Test
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install dependencies
+      run: sudo apt-get install cmake build-essential ninja-build python
+
+    - name: Test
+      run: ./bin/test.sh

.github/workflows/macos.yaml

@@ -0,0 +1,20 @@
+name: macOS
+
+on:
+  pull_request:
+  push:
+
+jobs:
+  test:
+    name: Test
+
+    runs-on: macOS-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install dependencies
+      run: brew install cmake ninja python
+
+    - name: Test
+      run: ./bin/test.sh

.github/workflows/upload-release.yaml

@@ -0,0 +1,32 @@
+name: Linux
+
+on:
+  release:
+    types: published
+
+jobs:
+  upload_release:
+    name: Upload Release
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install dependencies
+      run: sudo apt-get install p7zip-full
+
+    - name: Create Archive
+      run: |
+        7z a -spf build-migrator.zip ./bin/build_migrator ./bin/build_migrator.bat ./bin/merge_cmake  ./bin/merge_cmake.bat build_migrator docs merge_cmake LICENSE README.md
+
+    - name: Upload Release Asset
+      if: github.event.action == 'published'
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: build-migrator.zip
+        asset_name: build-migrator.zip
+        asset_content_type: application/zip

.github/workflows/windows.yaml

@@ -0,0 +1,23 @@
+name: Windows
+
+on:
+  pull_request:
+  push:
+
+jobs:
+  test:
+    name: Test
+
+    runs-on: windows-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ilammy/msvc-dev-cmd@v1
+
+    - name: Install dependencies
+      shell: cmd
+      run: choco install cmake python ninja nasm yasm mingw llvm
+
+    - name: Test
+      shell: cmd
+      run: 'set "PATH=C:\Program Files\NASM;C:\Program Files\LLVM\bin;%PATH%" && bin\test.bat'

.gitignore

@@ -0,0 +1,3 @@
+.files
+.vscode
+*.pyc

bin/test.bat

@@ -4,4 +4,4 @@ if "%INCLUDE%" == "" (
     exit /b 1
 )
 set "SCRIPT_DIR=%~dp0"
-python -m unittest discover -v -s "%SCRIPT_DIR%\..\tests" >build_migrator_tests.log 2>&1
+python -m unittest discover -v -s "%SCRIPT_DIR%\..\tests"

bin/test.sh

@@ -2,4 +2,4 @@
 
 set -e
 SCRIPT_DIR=`dirname "$0"`
-python -m unittest discover -v -s "$SCRIPT_DIR/../tests" &> build_migrator_tests.log
+python -m unittest discover -v -s "$SCRIPT_DIR/../tests"

build_migrator/common/algorithm.py

@@ -90,6 +90,16 @@ def join_nested_lists(flags, delim=" "):
     return flags
 
 
+def flatten_list(lst):
+    result = []
+    for elem in lst:
+        if isinstance(elem, list):
+            result.extend(flatten_list(elem))
+        else:
+            result.append(elem)
+    return result
+
+
 # Replaces lists with tuples
 def make_hashable(v):
     if isinstance(v, Hashable):

build_migrator/generators/_cmake/cmake_class.py

@@ -1,7 +1,7 @@
 from __future__ import absolute_import
 
 from .cmake_module import system_library_map
-from build_migrator.helpers import flatten_list
+from build_migrator.common.algorithm import flatten_list
 from build_migrator.modules import Generator
 
 

build_migrator/generators/_cmake/cmake_cmd.py

@@ -1,4 +1,4 @@
-from build_migrator.helpers import flatten_list
+from build_migrator.common.algorithm import flatten_list
 from build_migrator.modules import Generator
 
 

build_migrator/generators/_cmake/cmake_module.py

@@ -3,7 +3,8 @@
 from build_migrator.common.algorithm import join_nested_lists
 from build_migrator.common.os_ext import Windows
 from build_migrator.generators._cmake.cmake_cmd import CMakeCmd
-from build_migrator.helpers import flatten_list, ModuleTypes, get_target_output_dir
+from build_migrator.common.algorithm import flatten_list
+from build_migrator.helpers import ModuleTypes, get_target_output_dir
 from build_migrator.modules import Generator
 
 

build_migrator/generators/cmake.py

@@ -11,7 +11,7 @@
 import sys
 import traceback
 
-from build_migrator.common.algorithm import join_nested_lists
+from build_migrator.common.algorithm import flatten_list
 from build_migrator.common.os_ext import Unix
 
 from build_migrator.parsers.build_log_parser import (
@@ -613,7 +613,7 @@ def format_interface(self, name, sources):
 
     def format_link_flags(self, target_name, flags):
         return self.format_call(
-            "target_link_options", [target_name, "PRIVATE"], join_nested_lists(flags)
+            "target_link_options", [target_name, "PRIVATE"], flatten_list(flags)
         )
 
     def format_header(

build_migrator/helpers.py

@@ -410,17 +410,6 @@ def filter_flags(delete_rxs, flags):
     return filtered_flags
 
 
-# TODO: improve
-def flatten_list(lst):
-    result = []
-    for elem in lst:
-        if isinstance(elem, list):
-            result.extend(flatten_list(elem))
-        else:
-            result.append(elem)
-    return result
-
-
 def format_flag_msvc_lowercase(lst):
     return "".join(format_flag_msvc(lst)).lower()
 

build_migrator/parsers/clang_gcc.py

@@ -4,12 +4,12 @@
 import platform
 import re
 import subprocess
+from build_migrator.common.algorithm import flatten_list
 from build_migrator.helpers import (
     get_module_target,
     ModuleTypes,
     get_source_file_reference,
     filter_flags,
-    flatten_list,
     format_flag_gnu,
 )
 from build_migrator.common.os_ext import get_platform

build_migrator/parsers/msvc_cl.py

@@ -3,6 +3,7 @@
 import os
 import re
 
+from build_migrator.common.algorithm import flatten_list
 from build_migrator.common.argument_parser_ex import ArgumentParserEx
 import build_migrator.common.os_ext as os_ext
 from build_migrator.common import subprocess_ex
@@ -14,7 +15,6 @@
     format_flag_gnu,
     format_flag_msvc,
     filter_flags,
-    flatten_list,
 )
 from .msvc_link import get_msvc_link_parser
 

tests/files/test_parse_and_generate/check_order_of_link_flags/CMakeLists.txt

@@ -8,6 +8,6 @@ include(extensions)
 set(SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}" CACHE PATH "")
 
 add_library(cap-ng SHARED ${SOURCE_DIR}/cap-ng.c)
-target_link_options(cap-ng PRIVATE -g "-Wl,-z -Wl,relro")
+target_link_options(cap-ng PRIVATE -g -Wl,-z -Wl,relro)
 set_target_properties(cap-ng PROPERTIES VERSION 0.0.0)
 set_target_properties(cap-ng PROPERTIES SOVERSION 0)

tests/files/test_parse_and_generate/clang_install_name/CMakeLists.txt

@@ -0,0 +1,16 @@
+cmake_minimum_required(VERSION 3.13)
+
+project(PROJECT C)
+
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR})
+include(extensions)
+
+set(SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}" CACHE PATH "")
+
+add_library(foo SHARED ${SOURCE_DIR}/foo.c)
+target_link_options(foo PRIVATE
+    -Wl,-headerpad_max_install_names
+    -Wl,-search_paths_first
+    -install_name
+    /usr/local/lib/libfoo.dylib
+)

tests/files/test_parse_and_generate/clang_install_name/build.log

@@ -0,0 +1 @@
+clang -c ../source/foo.c -o foo.oclang -dynamiclib -Wl,-headerpad_max_install_names -Wl,-search_paths_first -install_name /usr/local/lib/libfoo.dylib -o libfoo.dylib foo.o

tests/files/test_parse_and_generate/darwin/CMakeLists.txt

@@ -28,10 +28,12 @@ set_source_files_properties(${SOURCE_DIR}/bar.c PROPERTIES INCLUDE_DIRECTORIES "
 add_library(bar SHARED ${SOURCE_DIR}/bar.c)
 target_link_options(bar PRIVATE
     -all_load
-    "-install_name @executable_path/libbar.dylib"
+    -install_name
+    @executable_path/libbar.dylib
     -headerpad_max_install_names
     -stdlib=libc++
-    "-exported_symbols_list ${CMAKE_CURRENT_BINARY_DIR}/libbar.def"
+    -exported_symbols_list
+    ${CMAKE_CURRENT_BINARY_DIR}/libbar.def
     -Wl,-search_paths_first
     -Wl,-headerpad_max_install_names
 )

tests/files/test_parse_and_generate/def_file_generated_during_build/CMakeLists.txt

@@ -9,11 +9,11 @@ set(SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}" CACHE PATH "")
 add_compile_options(-DTEST=1)
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/include)
 configure_file(${CMAKE_CURRENT_LIST_DIR}/prebuilt/include/a.h ${CMAKE_CURRENT_BINARY_DIR}/include/a.h COPYONLY)
-configure_file(${CMAKE_CURRENT_LIST_DIR}/external/C_/temp/1.def C:/temp/1.def COPYONLY)
+configure_file(${CMAKE_CURRENT_LIST_DIR}/external/C_/somedir/1.def C:/somedir/1.def COPYONLY)
 configure_file(${CMAKE_CURRENT_LIST_DIR}/external/X_/a/b/c/2.tmp X:/a/b/c/2.tmp COPYONLY)
 
 add_library(test1 SHARED ${SOURCE_DIR}/1.c)
-target_link_options(test1 PRIVATE /def:C:/temp/1.def)
+target_link_options(test1 PRIVATE /def:C:/somedir/1.def)
 
 add_library(test2 SHARED ${SOURCE_DIR}/1.c)
 target_link_options(test2 PRIVATE /def:X:/a/b/c/2.tmp)

tests/files/test_parse_and_generate/def_file_generated_during_build/build.log

@@ -1,7 +1,7 @@
-cl.exe /c /Iinclude ..\source\1.c -DTEST=1 /Fo1.obj
-echo line1 > C:\temp\1.def
-echo line2 >> C:\temp\1.def
-echo line3 > X:\a\b\c\2.tmp
-echo line4 >> X:\a\b\c\2.tmp
-link /nologo /dll /out:test1.dll /def:C:\temp\1.def 1.obj
-link /nologo /dll /out:test2.dll /def:X:\a\b\c\2.tmp 1.obj
+cl.exe /c /Iinclude ..\source\1.c -DTEST=1 /Fo1.obj
+echo line1 > C:\somedir\1.def
+echo line2 >> C:\somedir\1.def
+echo line3 > X:\a\b\c\2.tmp
+echo line4 >> X:\a\b\c\2.tmp
+link /nologo /dll /out:test1.dll /def:C:\somedir\1.def 1.obj
+link /nologo /dll /out:test2.dll /def:X:\a\b\c\2.tmp 1.obj

tests/files/test_parse_and_generate/gcc_whole_archive/CMakeLists.txt

@@ -21,7 +21,7 @@ if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/empty.c)
     file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/empty.c "")
 endif()
 add_library(public SHARED ${CMAKE_CURRENT_BINARY_DIR}/empty.c)
-target_link_options(public PRIVATE "-z noexecstack" -Wl,-z,defs)
+target_link_options(public PRIVATE -z noexecstack -Wl,-z,defs)
 target_link_libraries(public PRIVATE
     m
     -Wl,-whole-archive

tests/files/test_parse_and_generate/icu/darwin/CMakeLists.txt

@@ -74,7 +74,7 @@ add_custom_target(lib_icudata ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/lib/libicu
 file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/stubdata)
 
 add_library(stubdata_icudata.58.2 SHARED ${ICU_SOURCE_DIR}/stubdata/stubdata.c)
-target_link_options(stubdata_icudata.58.2 PRIVATE "-install_name @loader_path/libicudata.58.dylib")
+target_link_options(stubdata_icudata.58.2 PRIVATE -install_name @loader_path/libicudata.58.dylib)
 set_target_properties(stubdata_icudata.58.2 PROPERTIES OUTPUT_NAME icudata)
 set_target_output_subdir(stubdata_icudata.58.2 LIBRARY_OUTPUT_DIRECTORY stubdata)
 set_target_output_subdir(stubdata_icudata.58.2 RUNTIME_OUTPUT_DIRECTORY stubdata)
@@ -89,7 +89,7 @@ add_library(icuuc.58.2 SHARED
     ${ICU_SOURCE_DIR}/common/unistr_case_locale.cpp
     ${ICU_SOURCE_DIR}/common/uniset.cpp
 )
-target_link_options(icuuc.58.2 PRIVATE "-install_name @loader_path/libicuuc.58.dylib")
+target_link_options(icuuc.58.2 PRIVATE -install_name @loader_path/libicuuc.58.dylib)
 target_link_libraries(icuuc.58.2 PRIVATE stubdata_icudata.58.2)
 target_compile_options(icuuc.58.2 PRIVATE -DU_COMMON_IMPLEMENTATION)
 target_language_compile_options(icuuc.58.2 C PRIVATE -fvisibility=hidden)
@@ -107,7 +107,7 @@ add_library(icui18n.58.2 SHARED
     ${ICU_SOURCE_DIR}/i18n/decContext.c
     ${ICU_SOURCE_DIR}/i18n/decimfmtimpl.cpp
 )
-target_link_options(icui18n.58.2 PRIVATE "-install_name @loader_path/libicui18n.58.dylib")
+target_link_options(icui18n.58.2 PRIVATE -install_name @loader_path/libicui18n.58.dylib)
 target_link_libraries(icui18n.58.2 PRIVATE icuuc.58.2 stubdata_icudata.58.2)
 target_compile_options(icui18n.58.2 PRIVATE -DU_I18N_IMPLEMENTATION)
 target_include_directories(icui18n.58.2 PRIVATE ${ICU_SOURCE_DIR}/i18n)
@@ -127,7 +127,7 @@ add_library(icuio.58.2 SHARED
     ${ICU_SOURCE_DIR}/io/ufile.c
     ${ICU_SOURCE_DIR}/io/ucln_io.cpp
 )
-target_link_options(icuio.58.2 PRIVATE "-install_name @loader_path/libicuio.58.dylib")
+target_link_options(icuio.58.2 PRIVATE -install_name @loader_path/libicuio.58.dylib)
 target_link_libraries(icuio.58.2 PRIVATE icuuc.58.2 stubdata_icudata.58.2 icui18n.58.2)
 target_compile_options(icuio.58.2 PRIVATE -DU_IO_IMPLEMENTATION)
 target_include_directories(icuio.58.2 PRIVATE ${ICU_SOURCE_DIR}/i18n)
@@ -147,7 +147,7 @@ add_library(icutu.58.2 SHARED
     ${ICU_SOURCE_DIR}/tools/toolutil/ucm.c
     ${ICU_SOURCE_DIR}/tools/toolutil/ucln_tu.cpp
 )
-target_link_options(icutu.58.2 PRIVATE "-install_name @loader_path/libicutu.58.dylib")
+target_link_options(icutu.58.2 PRIVATE -install_name @loader_path/libicutu.58.dylib)
 target_link_libraries(icutu.58.2 PRIVATE icui18n.58.2 icuuc.58.2 stubdata_icudata.58.2)
 target_compile_options(icutu.58.2 PRIVATE
     -DU_TOOLUTIL_IMPLEMENTATION

tests/files/test_parse_and_generate/install_name_with_symlink/CMakeLists.txt

@@ -10,7 +10,8 @@ set(SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}" CACHE PATH "")
 add_library(jsoncpp SHARED ${SOURCE_DIR}/libjsoncpp.cpp)
 target_link_options(jsoncpp PRIVATE
     -Wl,-headerpad_max_install_names
-    "-install_name libjsoncpp.1.dylib"
+    -install_name
+    libjsoncpp.1.dylib
 )
 target_compile_options(jsoncpp PRIVATE -std=c++11 -fstack-protector)
 set_target_properties(jsoncpp PROPERTIES VERSION 1.7.2)

tests/files/test_parse_and_generate/path_alias_rpath_gcc/CMakeLists.txt

@@ -9,4 +9,4 @@ set(SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}" CACHE PATH "")
 set(RPATH_PATH_ROOT /some/path CACHE STRING "")
 
 add_library(a SHARED ${SOURCE_DIR}/main.cpp)
-target_link_options(a PRIVATE "-Wl,-rpath -Wl,${RPATH_PATH_ROOT}/lib")
+target_link_options(a PRIVATE -Wl,-rpath -Wl,${RPATH_PATH_ROOT}/lib)

tests/files/test_parse_and_generate/rpath_flag_variations/CMakeLists.txt

@@ -9,15 +9,15 @@ set(SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}" CACHE PATH "")
 file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/1)
 
 add_library(a SHARED ${SOURCE_DIR}/1.c)
-target_link_options(a PRIVATE "-Wl,-rpath -Wl,${CMAKE_CURRENT_BINARY_DIR}/1")
+target_link_options(a PRIVATE -Wl,-rpath -Wl,${CMAKE_CURRENT_BINARY_DIR}/1)
 file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/2)
 
 add_library(b SHARED ${SOURCE_DIR}/2.c)
 target_link_options(b PRIVATE -Wl,-rpath,${CMAKE_CURRENT_BINARY_DIR}/2)
 file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/3)
 
 add_library(c SHARED ${SOURCE_DIR}/3.c)
-target_link_options(c PRIVATE "-Wl,--rpath -Wl,${CMAKE_CURRENT_BINARY_DIR}/3")
+target_link_options(c PRIVATE -Wl,--rpath -Wl,${CMAKE_CURRENT_BINARY_DIR}/3)
 file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/4)
 
 add_library(d SHARED ${SOURCE_DIR}/4.c)
@@ -33,4 +33,4 @@ target_link_options(f_so PRIVATE -Wl,-rpath=${CMAKE_CURRENT_BINARY_DIR}/6)
 file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/7)
 
 add_library(g SHARED ${SOURCE_DIR}/7.c)
-target_link_options(g PRIVATE "-rpath ${CMAKE_CURRENT_BINARY_DIR}/7")
+target_link_options(g PRIVATE -rpath ${CMAKE_CURRENT_BINARY_DIR}/7)

tests/files/test_parse_and_generate/shared_from_static_without_sources/CMakeLists.txt

@@ -15,5 +15,5 @@ if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/empty.c)
     file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/empty.c "")
 endif()
 add_library(impl SHARED ${CMAKE_CURRENT_BINARY_DIR}/empty.c)
-target_link_options(impl PRIVATE "-z noexecstack" -Wl,-z,defs)
+target_link_options(impl PRIVATE -z noexecstack -Wl,-z,defs)
 target_link_libraries(impl PRIVATE m -Wl,-whole-archive impl1.static -Wl,-no-whole-archive)