Fix packaging of source code (package whole workspace when needed)

.github/workflows/run_long_integration_tests.yml

@@ -22,7 +22,7 @@ jobs:
       - name: Build
         run: |
           export PYTHONPATH=.
-          python ./integration_tests/test_previous_builds_are_reproducible.py --selected-builds "a.1" "a.2"
+          python ./integration_tests/test_previous_builds_are_reproducible.py --selected-builds "a.1" "a.2" "a.3"
 
       - name: Save artifacts
         uses: actions/upload-artifact@v3

Dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:22.04
 
 # Constants
-ARG BUILDER_NAME="multiversx/sdk-rust-contract-builder:v5.3.0"
+ARG BUILDER_NAME="multiversx/sdk-rust-contract-builder:v5.4.0"
 ARG VERSION_RUST="nightly-2023-05-26"
 ARG VERSION_BINARYEN="version_112"
 ARG DOWNLOAD_URL_BINARYEN="https://github.com/WebAssembly/binaryen/releases/download/${VERSION_BINARYEN}/binaryen-${VERSION_BINARYEN}-x86_64-linux.tar.gz"
@@ -39,7 +39,7 @@ RUN wget -O rustup.sh https://sh.rustup.rs && \
     rm -rf /rust/registry
 
 # Install sc-tool
-RUN PATH="/rust/bin:${PATH}" CARGO_HOME=/rust RUSTUP_HOME=/rust cargo install multiversx-sc-meta --version ${VERSION_SC_META} && \
+RUN PATH="/rust/bin:${PATH}" CARGO_HOME=/rust RUSTUP_HOME=/rust cargo install multiversx-sc-meta --version ${VERSION_SC_META} --locked && \
     rm -rf /rust/registry
 
 COPY "multiversx_sdk_rust_contract_builder" "/multiversx_sdk_rust_contract_builder"

build_with_docker.py

@@ -20,6 +20,7 @@ def main(cli_args: List[str]):
     parser.add_argument("--packaged-src", type=str, help="source code packaged in a JSON file")
     parser.add_argument("--contract", type=str)
     parser.add_argument("--output", type=str, default=Path(os.getcwd()) / "output")
+    parser.add_argument("--package-whole-project-src", action="store_true", default=False, help="include all project files in *.source.json (default: %(default)s)")
     parser.add_argument("--cargo-target-dir", help="deprecated parameter, not used anymore")
     parser.add_argument("--no-wasm-opt", action="store_true", default=False, help="do not optimize wasm files after the build (default: %(default)s)")
     parser.add_argument("--build-root", type=str, required=False, help="root path (within container) for the build (default: %(default)s)")
@@ -34,6 +35,7 @@ def main(cli_args: List[str]):
     packaged_src_path = Path(parsed_args.packaged_src).expanduser().resolve() if parsed_args.packaged_src else None
     contract_path = parsed_args.contract
     output_path = Path(parsed_args.output).expanduser().resolve()
+    package_whole_project_src = parsed_args.package_whole_project_src
     no_wasm_opt = parsed_args.no_wasm_opt
     build_root = Path(parsed_args.build_root) if parsed_args.build_root else None
     cargo_verbose = parsed_args.cargo_verbose
@@ -95,6 +97,9 @@ def main(cli_args: List[str]):
     if build_root:
         entrypoint_args.extend(["--build-root", str(build_root)])
 
+    if package_whole_project_src:
+        entrypoint_args.append("--package-whole-project-src")
+
     # Run docker container
     args = docker_general_args + docker_mount_args + docker_env_args + [image] + entrypoint_args
     logger.info(f"Running docker: {args}")

integration_tests/previous_builds.py

@@ -22,29 +22,42 @@ def __init__(self, name: str,
 previous_builds: List[PreviousBuild] = [
     PreviousBuild(
         name="a.1",
-        project_archive_url="https://github.com/multiversx/mx-reproducible-contract-build-example-sc/archive/refs/tags/v0.4.0.zip",
-        project_relative_path_in_archive="mx-reproducible-contract-build-example-sc-0.4.0",
+        project_archive_url="https://github.com/multiversx/mx-exchange-sc/archive/refs/heads/reproducible-v2.1.1-staking-upgrade.zip",
+        project_relative_path_in_archive="mx-exchange-sc-reproducible-v2.1.1-staking-upgrade",
         packaged_src_url=None,
         contract_name=None,
         expected_code_hashes={
-            "adder": "9fd12f88f9474ba115fb75e9d18a8fdbc4f42147de005445048442d49c3aa725",
-            "multisig": "2101bc2a7a31ea42e5ffaadd86c1640009690e93b1cb46c3566ba5eac2984e36",
-            "multisig-full": "ef468403354b6d3a728f86101354359fe6864187d216f674d99b31fc05313a39",
-            "multisig-view": "3690af76be10c0520e3c3545cde8d9ef6a15c2d0af74dbd8704b4909644049c9"
+            "farm-staking": "6dc7c587b2cc4b177a192b709c092f3752b3dcf9ce1b484e69fe64dc333a9e0a",
+            "farm": "931ca233826ff9dacd889967365db1cde9ed8402eb553de2a3b9d58b6ff1098d",
+            "factory": "df06465b651594605466e817bfe9d8d7c68eef0f87df4a8d3266bcfb1bef6d83",
+            "pair": "f3f08ebd758fada871c113c18017d9761f157d00b19c4d3beaba530e6c53afc2",
+            "energy-factory": "241600c055df605cafd85b75d40b21316a6b35713485201b156d695b23c66a2f"
         },
-        docker_image="multiversx/sdk-rust-contract-builder:v5.1.0"
+        docker_image="multiversx/sdk-rust-contract-builder:v4.2.1"
     ),
     PreviousBuild(
         name="a.2",
-        project_archive_url="https://github.com/multiversx/mx-reproducible-contract-build-example-sc/archive/refs/tags/v0.4.3.zip",
-        project_relative_path_in_archive="mx-reproducible-contract-build-example-sc-0.4.3",
+        project_archive_url="https://github.com/multiversx/mx-metabonding-sc/archive/refs/heads/reproducible-v1.1.1.zip",
+        project_relative_path_in_archive="mx-metabonding-sc-reproducible-v1.1.1",
+        packaged_src_url=None,
+        contract_name=None,
+        expected_code_hashes={
+            "metabonding": "897b19e1990f7c487c99c12f50722febe1ee4468bcd3a7405641966dfff2791d"
+        },
+        docker_image="multiversx/sdk-rust-contract-builder:v4.2.1"
+    ),
+    PreviousBuild(
+        name="a.3",
+        project_archive_url="https://github.com/multiversx/mx-reproducible-contract-build-example-sc/archive/refs/tags/v0.4.6.zip",
+        project_relative_path_in_archive="mx-reproducible-contract-build-example-sc-0.4.6",
         packaged_src_url=None,
         contract_name=None,
         expected_code_hashes={
             "adder": "9fd12f88f9474ba115fb75e9d18a8fdbc4f42147de005445048442d49c3aa725",
             "multisig": "b73050629c11b1f1a20ca6232abcef07897624195691552e3f2e2fce47822166",
             "multisig-full": "37c3b90bdaa7d8d203385c91b0b5cb4d3c444ab9ec5263351978046a545854e3",
-            "multisig-view": "ebaf987b041fcda297da71291d76736e4e98a1e449e5ec37908cdc0198e8be37"
+            "multisig-view": "ebaf987b041fcda297da71291d76736e4e98a1e449e5ec37908cdc0198e8be37",
+            "lottery-esdt": "e06b1a5c7fb71181a79e9be6b86d8ad154e5c2def4da6d2f0aa5266163823291"
         },
         docker_image="multiversx/sdk-rust-contract-builder:v5.3.0"
     )

integration_tests/test_project_folder_and_packaged_src_are_equivalent.py

@@ -8,64 +8,78 @@
 
 
 def main(cli_args: List[str]):
-    # TODO: when possible, also add multiversx/mx-exchange-sc (as of May 2023, it references mx-sdk-rs < v0.41.0, thus cannot be used for testing reproducible builds v5).
-    project_path = download_project_repository("https://github.com/multiversx/mx-reproducible-contract-build-example-sc/archive/refs/tags/v0.4.0.zip", "mx-exchange-sc-main")
-    parent_output_using_project = PARENT_OUTPUT_FOLDER / "using-project"
-    parent_output_using_packaged_src = PARENT_OUTPUT_FOLDER / "using-packaged-src"
-
-    shutil.rmtree(parent_output_using_project, ignore_errors=True)
-    shutil.rmtree(parent_output_using_packaged_src, ignore_errors=True)
-
-    check_project_folder_and_packaged_src_are_equivalent(project_path, parent_output_using_project, parent_output_using_packaged_src, ["adder", "multisig"])
+    repository_url = "https://github.com/multiversx/mx-reproducible-contract-build-example-sc"
+    tag = "0.4.7-beta.1"
+    archve_subfolder = f"mx-reproducible-contract-build-example-sc-{tag}"
+    project_path = download_project_repository(f"{repository_url}/archive/refs/tags/v{tag}.zip", archve_subfolder)
+    project_path = project_path / archve_subfolder
+
+    # Only package_whole_project_src = True works.
+    # package_whole_project_src = False does not work, since a missing Cargo.lock at the workspace level leads to build errors.
+    check_project_folder_and_packaged_src_are_equivalent(
+        project_path=project_path,
+        package_whole_project_src=True,
+        parent_output_folder=PARENT_OUTPUT_FOLDER,
+        contracts=["adder", "multisig"],
+    )
 
 
 def check_project_folder_and_packaged_src_are_equivalent(
         project_path: Path,
-        parent_output_using_project: Path,
-        parent_output_using_packaged_src: Path,
+        package_whole_project_src: bool,
+        parent_output_folder: Path,
         contracts: List[str]):
     for contract in contracts:
-        for package_whole_project_src in [True, False]:
-            output_using_project = parent_output_using_project / contract / ("whole" if package_whole_project_src else "truncated")
-            output_using_packaged_src = parent_output_using_packaged_src / contract / ("whole" if package_whole_project_src else "truncated")
-
-            output_using_packaged_src.mkdir(parents=True, exist_ok=True)
-            output_using_project.mkdir(parents=True, exist_ok=True)
-
-            run_docker(
-                project_path=project_path,
-                package_whole_project_src=package_whole_project_src,
-                packaged_src_path=None,
-                contract_name=contract,
-                image="sdk-rust-contract-builder:next",
-                output_folder=output_using_project
-            )
-
-            packaged_src_path = output_using_project / f"{contract}/{contract}-0.0.0.source.json"
-
-            run_docker(
-                project_path=None,
-                package_whole_project_src=package_whole_project_src,
-                packaged_src_path=packaged_src_path,
-                contract_name=contract,
-                image="sdk-rust-contract-builder:next",
-                output_folder=output_using_packaged_src
-            )
-
-            # Check that output folders are identical
-            using_project_output_files = sorted((output_using_project / contract).rglob("*"))
-            using_packaged_src_output_files = sorted((output_using_packaged_src / contract).rglob("*"))
-
-            assert len(using_project_output_files) == len(using_packaged_src_output_files)
-
-            for index, file in enumerate(using_project_output_files):
-                if not file.is_file() or file.suffix == ".zip":
-                    continue
-                using_project_file_content = file.read_bytes()
-                using_packaged_src_file_content = using_packaged_src_output_files[index].read_bytes()
-
-                if using_project_file_content != using_packaged_src_file_content:
-                    raise Exception(f"Files differ ({contract}): {file.name}")
+        output_using_project = parent_output_folder / "using-project" / contract / ("whole" if package_whole_project_src else "truncated")
+        output_using_packaged_src = parent_output_folder / "using-packaged-src" / contract / ("whole" if package_whole_project_src else "truncated")
+
+        shutil.rmtree(output_using_project, ignore_errors=True)
+        shutil.rmtree(output_using_packaged_src, ignore_errors=True)
+
+        output_using_project.mkdir(parents=True, exist_ok=True)
+        output_using_packaged_src.mkdir(parents=True, exist_ok=True)
+
+        run_docker(
+            project_path=project_path,
+            package_whole_project_src=package_whole_project_src,
+            packaged_src_path=None,
+            contract_name=contract,
+            image="sdk-rust-contract-builder:next",
+            output_folder=output_using_project
+        )
+
+        packaged_src_path = output_using_project / f"{contract}/{contract}-0.0.0.source.json"
+
+        run_docker(
+            project_path=None,
+            package_whole_project_src=package_whole_project_src,
+            packaged_src_path=packaged_src_path,
+            contract_name=contract,
+            image="sdk-rust-contract-builder:next",
+            output_folder=output_using_packaged_src
+        )
+
+        # Check that output folders are identical
+        using_project_output_files = sorted((output_using_project / contract).rglob("*"))
+        using_packaged_src_output_files = sorted((output_using_packaged_src / contract).rglob("*"))
+
+        assert len(using_project_output_files) == len(using_packaged_src_output_files)
+
+        for index, file_using_project in enumerate(using_project_output_files):
+            file_using_packaged_src = using_packaged_src_output_files[index]
+
+            if not file_using_project.is_file() or file_using_project.suffix == ".zip":
+                continue
+            file_content_using_project = file_using_project.read_bytes()
+            file_content_using_packaged_src = file_using_packaged_src.read_bytes()
+
+            if file_content_using_project == file_content_using_packaged_src:
+                print(f"Files are identical ({contract}): {file_using_project.name}")
+            else:
+                print(f"Files differ ({contract}):")
+                print(f"  {file_using_project}")
+                print(f"  {file_using_packaged_src}")
+                raise Exception(f"Files differ ({contract}): {file_using_project.name}")
 
 
 if __name__ == "__main__":

multiversx_sdk_rust_contract_builder/source_code.py

@@ -50,7 +50,7 @@ def get_source_code_files(
     files_related_to_contract = set(file.path for file in source_code_files)
 
     if include_unrelated_to_contract:
-        all_files = get_all_files(contract_folder, _is_source_code_file)
+        all_files = get_all_files(project_folder, _is_source_code_file)
         for file in all_files:
             if file not in files_related_to_contract:
                 source_code_files.append(SourceCodeFile(file, contract_folder, sys.maxsize))