From 942a3301dc2b0117567cc5c6e8253f226e98fbbb Mon Sep 17 00:00:00 2001 From: semaraugusto Date: Wed, 29 Dec 2021 00:17:15 -0300 Subject: [PATCH 0001/1408] fix issue with delegating votes to people who cannot vote. That made it so that if wallet A delegated to wallet B and B had no rights to vote, than B would lose out on its own vote --- docs/examples/voting.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/examples/voting.rst b/docs/examples/voting.rst index 84191118db..24e1e5446d 100644 --- a/docs/examples/voting.rst +++ b/docs/examples/voting.rst @@ -130,9 +130,12 @@ of votes. // Since `sender` is a reference, this // modifies `voters[msg.sender].voted` + Voter storage delegate_ = voters[to]; + + // Voters cannot delegate to wallets that cannot vote. + require(delegate_.weight >= 1) sender.voted = true; sender.delegate = to; - Voter storage delegate_ = voters[to]; if (delegate_.voted) { // If the delegate already voted, // directly add to the number of votes From 6fbfa33486ba4bd1c6c3a9953566788df7546c78 Mon Sep 17 00:00:00 2001 From: Semar Augusto Date: Wed, 29 Dec 2021 07:04:13 -0300 Subject: [PATCH 0002/1408] Update docs/examples/voting.rst Co-authored-by: Bhargava Shastry --- docs/examples/voting.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples/voting.rst b/docs/examples/voting.rst index 24e1e5446d..0e8901c3dd 100644 --- a/docs/examples/voting.rst +++ b/docs/examples/voting.rst @@ -133,7 +133,7 @@ of votes. Voter storage delegate_ = voters[to]; // Voters cannot delegate to wallets that cannot vote. - require(delegate_.weight >= 1) + require(delegate_.weight >= 1); sender.voted = true; sender.delegate = to; if (delegate_.voted) { From 7fbf134ce3f40cd6f3248abffdaf3d14e4018a69 Mon Sep 17 00:00:00 2001 From: Marenz Date: Tue, 18 Jan 2022 13:36:54 +0100 Subject: [PATCH 0003/1408] Add test for fixed natspec ICE --- test/libsolidity/SolidityNatspecJSON.cpp | 49 ++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/test/libsolidity/SolidityNatspecJSON.cpp b/test/libsolidity/SolidityNatspecJSON.cpp index af04d07954..2b729205aa 100644 --- a/test/libsolidity/SolidityNatspecJSON.cpp +++ b/test/libsolidity/SolidityNatspecJSON.cpp @@ -2534,6 +2534,55 @@ BOOST_AUTO_TEST_CASE(dev_struct_getter_override) checkNatspec(sourceCode, "Thing", natspec2, false); } +BOOST_AUTO_TEST_CASE(dev_struct_getter_override_no_return_name) +{ + char const *sourceCode = R"( + interface IThing { + ///@return + function value(uint) external returns (uint128,uint128); + } + + contract Thing is IThing { + struct Value { + uint128 x; + uint128 A; + } + mapping(uint=>Value) public override value; + } + )"; + + char const *natspec = R"ABCDEF({ + "methods": + { + "value(uint256)": + { + "returns": + { + "_0": "" + } + } + } + })ABCDEF"; + + char const *natspec2 = R"ABCDEF({ + "methods": {}, + "stateVariables": + { + "value": + { + "return": "x ", + "returns": + { + "x": "" + } + } + } + })ABCDEF"; + + checkNatspec(sourceCode, "IThing", natspec, false); + checkNatspec(sourceCode, "Thing", natspec2, false); +} + BOOST_AUTO_TEST_CASE(dev_struct_getter_override_different_return_parameter_names) { char const *sourceCode = R"( From de28f317160f45d3d86c574dfcd43f161996c6fb Mon Sep 17 00:00:00 2001 From: Daniel Kirchner Date: Fri, 21 Jan 2022 15:24:18 +0100 Subject: [PATCH 0004/1408] Improved stack shuffling in corner cases. --- Changelog.md | 2 +- libyul/backends/evm/StackHelpers.h | 37 +++++++++++++++++++- test/CMakeLists.txt | 1 + test/libyul/StackShufflingTest.cpp | 54 ++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 test/libyul/StackShufflingTest.cpp diff --git a/Changelog.md b/Changelog.md index 93b9408390..3bdf50ffa8 100644 --- a/Changelog.md +++ b/Changelog.md @@ -19,7 +19,7 @@ Bugfixes: * IR Generator: Fix IR syntax error when copying storage arrays of structs containing functions. * Natspec: Fix ICE when overriding a struct getter with a Natspec-documented return value and the name in the struct is different. * TypeChecker: Fix ICE when a constant variable declaration forward references a struct. - + * Yul EVM Code Transform: Improved stack shuffling in corner cases. Solc-Js: * The wrapper now requires at least nodejs v10. diff --git a/libyul/backends/evm/StackHelpers.h b/libyul/backends/evm/StackHelpers.h index 38d0ddf4b2..a0a9428604 100644 --- a/libyul/backends/evm/StackHelpers.h +++ b/libyul/backends/evm/StackHelpers.h @@ -262,6 +262,11 @@ class Shuffler if (ops.sourceMultiplicity(ops.sourceSize() - 1 - swapDepth) < 0) { ops.swap(swapDepth); + if (ops.targetIsArbitrary(sourceTop)) + // Usually we keep a slot that is to-be-removed, if the current top is arbitrary. + // However, since we are in a stack-too-deep situation, pop it immediately + // to compress the stack (we can always push back junk in the end). + ops.pop(); return true; } // Otherwise we rely on stack compression or stack-to-memory. @@ -321,14 +326,44 @@ class Shuffler yulAssert(ops.sourceMultiplicity(i) == 0 && (ops.targetIsArbitrary(i) || ops.targetMultiplicity(i) == 0), ""); yulAssert(ops.isCompatible(sourceTop, sourceTop), ""); + auto swappableOffsets = ranges::views::iota(size > 17 ? size - 17 : 0u, size); + // If we find a lower slot that is out of position, but also compatible with the top, swap that up. + for (size_t offset: swappableOffsets) + if (!ops.isCompatible(offset, offset) && ops.isCompatible(sourceTop, offset)) + { + ops.swap(size - offset - 1); + return true; + } + // Swap up any reachable slot that is still out of position. + for (size_t offset: swappableOffsets) + if (!ops.isCompatible(offset, offset) && !ops.sourceIsSame(offset, sourceTop)) + { + ops.swap(size - offset - 1); + return true; + } + // We are in a stack-too-deep situation and try to reduce the stack size. + // If the current top is merely kept since the target slot is arbitrary, pop it. + if (ops.targetIsArbitrary(sourceTop) && ops.sourceMultiplicity(sourceTop) <= 0) + { + ops.pop(); + return true; + } + // If any reachable slot is merely kept, since the target slot is arbitrary, swap it up and pop it. + for (size_t offset: swappableOffsets) + if (ops.targetIsArbitrary(offset) && ops.sourceMultiplicity(offset) <= 0) + { + ops.swap(size - offset - 1); + ops.pop(); + return true; + } + // We cannot avoid a stack-too-deep error. Repeat the above without restricting to reachable slots. for (size_t offset: ranges::views::iota(0u, size)) if (!ops.isCompatible(offset, offset) && ops.isCompatible(sourceTop, offset)) { ops.swap(size - offset - 1); return true; } - // Swap up any slot that is still out of position. for (size_t offset: ranges::views::iota(0u, size)) if (!ops.isCompatible(offset, offset) && !ops.sourceIsSame(offset, sourceTop)) { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3a0b5f7b68..1ca9c33281 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -147,6 +147,7 @@ set(libyul_sources libyul/Parser.cpp libyul/StackLayoutGeneratorTest.cpp libyul/StackLayoutGeneratorTest.h + libyul/StackShufflingTest.cpp libyul/SyntaxTest.h libyul/SyntaxTest.cpp libyul/YulInterpreterTest.cpp diff --git a/test/libyul/StackShufflingTest.cpp b/test/libyul/StackShufflingTest.cpp new file mode 100644 index 0000000000..2a2c217490 --- /dev/null +++ b/test/libyul/StackShufflingTest.cpp @@ -0,0 +1,54 @@ +/* + This file is part of solidity. + + solidity is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + solidity is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with solidity. If not, see . +*/ +/** + * Unit tests for stack shuffling. + */ +#include +#include + +using namespace std; +using namespace solidity::langutil; + +namespace solidity::yul::test +{ + +BOOST_AUTO_TEST_SUITE(YulStackShuffling) + +BOOST_AUTO_TEST_CASE(swap_cycle) +{ + std::vector scopeVariables; + Scope::Function function; + std::vector v; + for (size_t i = 0; i < 17; ++i) + scopeVariables.emplace_back(Scope::Variable{""_yulstring, YulString{"v" + to_string(i)}}); + for (size_t i = 0; i < 17; ++i) + v.emplace_back(VariableSlot{scopeVariables[i]}); + + Stack sourceStack{ + v[0], v[1], v[2], v[3], v[4], v[5], v[6], v[7], v[9], v[10], v[11], v[12], v[13], v[14], v[15], v[16], + FunctionReturnLabelSlot{function}, FunctionReturnLabelSlot{function}, v[5]}; + Stack targetStack{ + v[1], v[0], v[2], v[3], v[4], v[5], v[6], v[7], v[9], v[10], v[11], v[12], v[13], v[14], v[15], v[16], + FunctionReturnLabelSlot{function}, JunkSlot{}, JunkSlot{} + }; + // Used to hit a swapping cycle. + createStackLayout(sourceStack, targetStack, [](auto){}, [](auto){}, [](){}); +} + +BOOST_AUTO_TEST_SUITE_END() + +} From e85bceb4178f82e05762dc3e7e7bcf3020ded091 Mon Sep 17 00:00:00 2001 From: nishant-sachdeva Date: Fri, 4 Feb 2022 02:08:36 +0530 Subject: [PATCH 0005/1408] Types.h:872 had a comment /// String is interpreted as a subtype of Bytes. - this was now incorrect after #12593 . That has been removed now. --- libsolidity/ast/Types.h | 1 - 1 file changed, 1 deletion(-) diff --git a/libsolidity/ast/Types.h b/libsolidity/ast/Types.h index 3f134df17d..c4aaf38ca1 100644 --- a/libsolidity/ast/Types.h +++ b/libsolidity/ast/Types.h @@ -864,7 +864,6 @@ class ArrayType: public ReferenceType std::vector decomposition() const override { return {m_baseType}; } private: - /// String is interpreted as a subtype of Bytes. enum class ArrayKind { Ordinary, Bytes, String }; bigint unlimitedStaticCalldataSize(bool _padded) const; From b925250705d5ba44cbf6a2b49bb78eacaa402d22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 28 Jan 2022 19:14:29 +0100 Subject: [PATCH 0006/1408] Make solc-js.sh run its tests directly - Its structure has diverged a lot from other external tests and there's not point in keeping it abstracted like this. --- test/externalTests/common.sh | 14 -------------- test/externalTests/solc-js/solc-js.sh | 8 ++++---- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/test/externalTests/common.sh b/test/externalTests/common.sh index 10df975256..3737069cb3 100644 --- a/test/externalTests/common.sh +++ b/test/externalTests/common.sh @@ -323,20 +323,6 @@ function hardhat_clean rm -rf artifacts/ cache/ } -function run_test -{ - local compile_fn="$1" - local test_fn="$2" - - replace_version_pragmas - - printLog "Running compile function..." - time $compile_fn - - printLog "Running test function..." - $test_fn -} - function settings_from_preset { local preset="$1" diff --git a/test/externalTests/solc-js/solc-js.sh b/test/externalTests/solc-js/solc-js.sh index 33b43e6430..3da03519b3 100755 --- a/test/externalTests/solc-js/solc-js.sh +++ b/test/externalTests/solc-js/solc-js.sh @@ -29,9 +29,6 @@ VERSION="$2" [[ $SOLJSON != "" && -f "$SOLJSON" && $VERSION != "" ]] || fail "Usage: $0 " -function compile_fn { echo "Nothing to compile."; } -function test_fn { npm test; } - function solcjs_test { TEST_DIR=$(pwd) @@ -60,7 +57,10 @@ function solcjs_test echo "Updating package.json to version $VERSION" npm version --allow-same-version --no-git-tag-version "$VERSION" - run_test compile_fn test_fn + replace_version_pragmas + + printLog "Running test function..." + npm test } external_test solc-js solcjs_test From 9e641e60e7eeff966f3cd33824cf42bdb5111493 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 28 Jan 2022 20:51:04 +0100 Subject: [PATCH 0007/1408] externalTests/solc-js: Allow using a local checkout of solc-js --- test/externalTests/common.sh | 10 +++++++++- test/externalTests/solc-js/solc-js.sh | 5 +++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/test/externalTests/common.sh b/test/externalTests/common.sh index 3737069cb3..ec782f0562 100644 --- a/test/externalTests/common.sh +++ b/test/externalTests/common.sh @@ -71,15 +71,23 @@ function setup_solc local binary_path="$3" local solcjs_branch="${4:-master}" local install_dir="${5:-solc/}" + local solcjs_dir="$6" [[ $binary_type == native || $binary_type == solcjs ]] || assertFail + [[ $binary_type == solcjs || $solcjs_dir == "" ]] || assertFail cd "$test_dir" if [[ $binary_type == solcjs ]] then printLog "Setting up solc-js..." - git clone --depth 1 -b "$solcjs_branch" https://github.com/ethereum/solc-js.git "$install_dir" + if [[ $solcjs_dir == "" ]]; then + printLog "Cloning branch ${solcjs_branch}..." + git clone --depth 1 -b "$solcjs_branch" https://github.com/ethereum/solc-js.git "$install_dir" + else + printLog "Using local solc-js from ${solcjs_dir}..." + cp -ra "$solcjs_dir" solc + fi pushd "$install_dir" npm install diff --git a/test/externalTests/solc-js/solc-js.sh b/test/externalTests/solc-js/solc-js.sh index 3da03519b3..bf7ca87628 100755 --- a/test/externalTests/solc-js/solc-js.sh +++ b/test/externalTests/solc-js/solc-js.sh @@ -26,8 +26,9 @@ source test/externalTests/common.sh SOLJSON="$1" VERSION="$2" +SOLCJS_CHECKOUT="$3" # optional -[[ $SOLJSON != "" && -f "$SOLJSON" && $VERSION != "" ]] || fail "Usage: $0 " +[[ $SOLJSON != "" && -f "$SOLJSON" && $VERSION != "" ]] || fail "Usage: $0 []" function solcjs_test { @@ -35,7 +36,7 @@ function solcjs_test SOLCJS_INPUT_DIR="$TEST_DIR"/test/externalTests/solc-js # set up solc-js on the branch specified - setup_solc "$DIR" solcjs "$SOLJSON" master solc/ + setup_solc "$DIR" solcjs "$SOLJSON" master solc/ "$SOLCJS_CHECKOUT" cd solc/ printLog "Updating index.js file..." From 653c1e68429937abec17c9242a191227db0202b9 Mon Sep 17 00:00:00 2001 From: Ayush Shukla Date: Thu, 3 Feb 2022 20:26:46 +0530 Subject: [PATCH 0008/1408] Fix slot calculation for bytes/string mapping --- docs/internals/layout_in_storage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/internals/layout_in_storage.rst b/docs/internals/layout_in_storage.rst index b1b89a6a3c..9afe97ddbf 100644 --- a/docs/internals/layout_in_storage.rst +++ b/docs/internals/layout_in_storage.rst @@ -90,7 +90,7 @@ The value corresponding to a mapping key ``k`` is located at ``keccak256(h(k) . where ``.`` is concatenation and ``h`` is a function that is applied to the key depending on its type: - for value types, ``h`` pads the value to 32 bytes in the same way as when storing the value in memory. -- for strings and byte arrays, ``h`` computes the ``keccak256`` hash of the unpadded data. +- for strings and byte arrays, ``h(k)`` is just the unpadded data. If the mapping value is a non-value type, the computed slot marks the start of the data. If the value is of struct type, From 4d65bfa95e28f3dc3b8ad6cd6aaf8578c15851c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 5 Feb 2022 00:05:39 +0100 Subject: [PATCH 0009/1408] CI: Remove notes about Hardhat failing on nodejs 17; that version is officially not supported --- .circleci/config.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 143a692fdb..3f2c661a0c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -521,35 +521,30 @@ defaults: name: t_native_test_ext_ens project: ens binary_type: native - # NOTE: Tests crash on nodejs 17: "Error: error:0308010C:digital envelope routines::unsupported" nodejs_version: '16' - job_native_test_ext_trident: &job_native_test_ext_trident <<: *workflow_ubuntu2004_static name: t_native_test_ext_trident project: trident binary_type: native - # NOTE: Tests crash on nodejs 17: "Error: error:0308010C:digital envelope routines::unsupported" nodejs_version: '16' - job_native_test_ext_euler: &job_native_test_ext_euler <<: *workflow_ubuntu2004_static name: t_native_test_ext_euler project: euler binary_type: native - # NOTE: Tests crash on nodejs 17: "Error: error:0308010C:digital envelope routines::unsupported" nodejs_version: '16' - job_native_test_ext_yield_liquidator: &job_native_test_ext_yield_liquidator <<: *workflow_ubuntu2004_static name: t_native_test_ext_yield_liquidator project: yield-liquidator binary_type: native - # NOTE: Tests crash on nodejs 17: "Error: error:0308010C:digital envelope routines::unsupported" nodejs_version: '16' - job_native_test_ext_bleeps: &job_native_test_ext_bleeps <<: *workflow_ubuntu2004_static name: t_native_test_ext_bleeps project: bleeps binary_type: native - # NOTE: Tests crash on nodejs 17: "Error: error:0308010C:digital envelope routines::unsupported" nodejs_version: '16' resource_class: medium - job_native_test_ext_pool_together: &job_native_test_ext_pool_together @@ -557,35 +552,30 @@ defaults: name: t_native_test_ext_pool_together project: pool-together binary_type: native - # NOTE: Tests crash on nodejs 17: "Error: error:0308010C:digital envelope routines::unsupported" nodejs_version: '16' - job_native_test_ext_perpetual_pools: &job_native_test_ext_perpetual_pools <<: *workflow_ubuntu2004_static name: t_native_test_ext_perpetual_pools project: perpetual-pools binary_type: native - # NOTE: Tests crash on nodejs 17: "Error: error:0308010C:digital envelope routines::unsupported" nodejs_version: '16' - job_native_test_ext_uniswap: &job_native_test_ext_uniswap <<: *workflow_ubuntu2004_static name: t_native_test_ext_uniswap project: uniswap binary_type: native - # NOTE: Tests crash on nodejs 17: "Error: error:0308010C:digital envelope routines::unsupported" nodejs_version: '16' - job_native_test_prb_math: &job_native_test_prb_math <<: *workflow_ubuntu2004_static name: t_native_test_ext_prb_math project: prb-math binary_type: native - # NOTE: Tests crash on nodejs 17: "Error: error:0308010C:digital envelope routines::unsupported" nodejs_version: '16' - job_native_test_ext_elementfi: &job_native_test_ext_elementfi <<: *workflow_ubuntu2004_static name: t_native_test_ext_elementfi project: elementfi binary_type: native - # NOTE: Tests crash on nodejs 17: "Error: error:0308010C:digital envelope routines::unsupported" nodejs_version: '16' resource_class: medium - job_ems_test_ext_colony: &job_ems_test_ext_colony From 247eab90567e36fb9d8e390ef530551316fc7a6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 5 Feb 2022 00:03:53 +0100 Subject: [PATCH 0010/1408] CI: Rename node_latest_small to node_small --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3f2c661a0c..db9baaa642 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -384,7 +384,7 @@ defaults: TERM: xterm MAKEFLAGS: -j 2 - - base_node_latest_small: &base_node_latest_small + - base_node_small: &base_node_small docker: - image: circleci/node resource_class: small @@ -605,7 +605,7 @@ jobs: - gitter_notify_failure_unless_pr chk_docs_examples: - <<: *base_node_latest_small + <<: *base_node_small steps: - checkout - attach_workspace: @@ -674,7 +674,7 @@ jobs: - gitter_notify_failure_unless_pr chk_buglist: - <<: *base_node_latest_small + <<: *base_node_small steps: - checkout - run: @@ -1136,7 +1136,7 @@ jobs: - gitter_notify_failure_unless_pr t_ems_ext_hardhat: - <<: *base_node_latest_small + <<: *base_node_small environment: TERM: xterm HARDHAT_TESTS_SOLC_PATH: /tmp/workspace/soljson.js @@ -1339,7 +1339,7 @@ jobs: - gitter_notify_failure_unless_pr b_bytecode_ems: - <<: *base_node_latest_small + <<: *base_node_small environment: SOLC_EMSCRIPTEN: "On" steps: From 2f0ccb21bef0ddee12b37b52289ca7081b6a5d31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 5 Feb 2022 00:04:20 +0100 Subject: [PATCH 0011/1408] CI: Switch t_ems_ext_hardhat to nodejs 16 --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index db9baaa642..58a4c27428 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1137,6 +1137,8 @@ jobs: t_ems_ext_hardhat: <<: *base_node_small + docker: + - image: circleci/node:16 environment: TERM: xterm HARDHAT_TESTS_SOLC_PATH: /tmp/workspace/soljson.js From 7634fc4ea9537d60fdf686f43a4e54e7ab0f43fe Mon Sep 17 00:00:00 2001 From: a3d4 Date: Fri, 4 Feb 2022 17:54:21 +0100 Subject: [PATCH 0012/1408] Clarify symlink handling on Windows --- docs/contributing.rst | 7 +++++++ test/FilesystemUtils.cpp | 1 + 2 files changed, 8 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index 2895744ef4..ba5a0a0f8f 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -94,6 +94,13 @@ dependencies (`evmone `_, On macOS some of the testing scripts expect GNU coreutils to be installed. This can be easiest accomplished using Homebrew: ``brew install coreutils``. +On Windows systems make sure that you have a privilege to create symlinks, +otherwise several tests may fail. +Administrators should have that privilege, but you may also +`grant it to other users `_ +or +`enable Developer Mode `_. + Running the Tests ----------------- diff --git a/test/FilesystemUtils.cpp b/test/FilesystemUtils.cpp index 0235b11476..0d1de3af95 100644 --- a/test/FilesystemUtils.cpp +++ b/test/FilesystemUtils.cpp @@ -81,5 +81,6 @@ bool solidity::test::createSymlinkIfSupportedByFilesystem( BOOST_THROW_EXCEPTION(runtime_error( "Failed to create a symbolic link: \"" + _linkName.string() + "\"" " -> " + _targetPath.string() + "\"." + " " + symlinkCreationError.message() + "." )); } From 0a17495cf94016318ee76f299419595d6bfdd718 Mon Sep 17 00:00:00 2001 From: a3d4 Date: Sat, 5 Feb 2022 11:51:11 +0100 Subject: [PATCH 0013/1408] Treat root path in normalizeCLIPathForVFS as case insensitive on Windows --- libsolidity/interface/FileReader.cpp | 3 ++- test/libsolidity/interface/FileReader.cpp | 28 +++++++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/libsolidity/interface/FileReader.cpp b/libsolidity/interface/FileReader.cpp index 47f4d2b5aa..adb4b01cae 100644 --- a/libsolidity/interface/FileReader.cpp +++ b/libsolidity/interface/FileReader.cpp @@ -269,7 +269,8 @@ boost::filesystem::path FileReader::normalizeCLIPathForVFS( if (!isUNCPath(normalizedPath)) { boost::filesystem::path workingDirRootPath = canonicalWorkDir.root_path(); - if (normalizedRootPath == workingDirRootPath) + // Ignore drive letter case on Windows (C:\ <=> c:\). + if (boost::filesystem::equivalent(normalizedRootPath, workingDirRootPath)) normalizedRootPath = "/"; } diff --git a/test/libsolidity/interface/FileReader.cpp b/test/libsolidity/interface/FileReader.cpp index 866a4435ac..39355e536f 100644 --- a/test/libsolidity/interface/FileReader.cpp +++ b/test/libsolidity/interface/FileReader.cpp @@ -25,6 +25,7 @@ #include #include +#include #include #include @@ -192,8 +193,8 @@ BOOST_AUTO_TEST_CASE(normalizeCLIPathForVFS_root_name_only) #if defined(_WIN32) boost::filesystem::path driveLetter = boost::filesystem::current_path().root_name(); - solAssert(!driveLetter.empty(), ""); - solAssert(driveLetter.is_relative(), ""); + soltestAssert(!driveLetter.empty(), ""); + soltestAssert(driveLetter.is_relative(), ""); BOOST_CHECK_EQUAL(FileReader::normalizeCLIPathForVFS(driveLetter, resolveSymlinks), expectedWorkDir); #endif @@ -212,13 +213,32 @@ BOOST_AUTO_TEST_CASE(normalizeCLIPathForVFS_stripping_root_name) for (SymlinkResolution resolveSymlinks: {SymlinkResolution::Enabled, SymlinkResolution::Disabled}) { + boost::filesystem::path workDir = boost::filesystem::current_path(); + boost::filesystem::path normalizedPath = FileReader::normalizeCLIPathForVFS( - boost::filesystem::current_path(), + workDir, resolveSymlinks ); - BOOST_CHECK_EQUAL(normalizedPath, "/" / boost::filesystem::current_path().relative_path()); + BOOST_CHECK_EQUAL(normalizedPath, "/" / workDir.relative_path()); BOOST_TEST(normalizedPath.root_name().empty()); BOOST_CHECK_EQUAL(normalizedPath.root_directory(), "/"); + +#if defined(_WIN32) + string root = workDir.root_path().string(); + soltestAssert(root.length() == 3 && root[1] == ':' && root[2] == '\\', ""); + + for (auto convert: {boost::to_lower_copy, boost::to_upper_copy}) + { + boost::filesystem::path workDirWin = convert(root, locale()) / workDir.relative_path(); + normalizedPath = FileReader::normalizeCLIPathForVFS( + workDirWin, + resolveSymlinks + ); + BOOST_CHECK_EQUAL(normalizedPath, "/" / workDir.relative_path()); + BOOST_TEST(normalizedPath.root_name().empty()); + BOOST_CHECK_EQUAL(normalizedPath.root_directory(), "/"); + } +#endif } } From a0dd2cd1ff59d06425c78180a2a8a6e2c8ff506e Mon Sep 17 00:00:00 2001 From: a3d4 Date: Sat, 5 Feb 2022 15:12:49 +0100 Subject: [PATCH 0014/1408] Remove a reference to unused SolidityFixedFeeRegistrar --- test/boostTest.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/boostTest.cpp b/test/boostTest.cpp index bfb546736c..1cc512c325 100644 --- a/test/boostTest.cpp +++ b/test/boostTest.cpp @@ -57,7 +57,7 @@ void removeTestSuite(std::string const& _name) { master_test_suite_t& master = framework::master_test_suite(); auto id = master.get(_name); - assert(id != INV_TEST_UNIT_ID); + soltestAssert(id != INV_TEST_UNIT_ID, "Removing non-existent test suite!"); master.remove(id); } @@ -279,7 +279,6 @@ test_suite* init_unit_test_suite( int /*argc*/, char* /*argv*/[] ) "ABIDecoderTest", "ABIEncoderTest", "SolidityAuctionRegistrar", - "SolidityFixedFeeRegistrar", "SolidityWallet", "GasMeterTests", "GasCostTests", From 6bd38aa4ef2ca036775896b77909b361ed3d8855 Mon Sep 17 00:00:00 2001 From: William Entriken Date: Sat, 5 Feb 2022 23:59:09 -0500 Subject: [PATCH 0015/1408] Fix signature of pop member --- docs/types/reference-types.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/types/reference-types.rst b/docs/types/reference-types.rst index 126440321c..88e23a47eb 100644 --- a/docs/types/reference-types.rst +++ b/docs/types/reference-types.rst @@ -337,9 +337,9 @@ Array Members Dynamic storage arrays and ``bytes`` (not ``string``) have a member function called ``push(x)`` that you can use to append a given element at the end of the array. The function returns nothing. -**pop**: +**pop()**: Dynamic storage arrays and ``bytes`` (not ``string``) have a member - function called ``pop`` that you can use to remove an element from the + function called ``pop()`` that you can use to remove an element from the end of the array. This also implicitly calls :ref:`delete` on the removed element. .. note:: From c3145979fc18c104479379916b17364e94fccb16 Mon Sep 17 00:00:00 2001 From: Hakeem Almidan Date: Sun, 6 Feb 2022 20:16:04 +0300 Subject: [PATCH 0016/1408] Update cheatsheet.rst Add more description to the bullet point of 'block.timestamp' (under 'Global Variables') --- docs/cheatsheet.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cheatsheet.rst b/docs/cheatsheet.rst index eab1e3d52a..cd9ba5c149 100644 --- a/docs/cheatsheet.rst +++ b/docs/cheatsheet.rst @@ -92,7 +92,7 @@ Global Variables - ``block.difficulty`` (``uint``): current block difficulty - ``block.gaslimit`` (``uint``): current block gaslimit - ``block.number`` (``uint``): current block number -- ``block.timestamp`` (``uint``): current block timestamp +- ``block.timestamp`` (``uint``): current block timestamp in seconds since Unix epoch - ``gasleft() returns (uint256)``: remaining gas - ``msg.data`` (``bytes``): complete calldata - ``msg.sender`` (``address``): sender of the message (current call) From 4715fafb829a156e25a027027d8632689ee3d04b Mon Sep 17 00:00:00 2001 From: chriseth Date: Mon, 7 Feb 2022 11:31:59 +0100 Subject: [PATCH 0017/1408] Re-enable preset for poolTogether. --- test/externalTests/pool-together.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/externalTests/pool-together.sh b/test/externalTests/pool-together.sh index 1cabd6a501..a7f5933c0e 100755 --- a/test/externalTests/pool-together.sh +++ b/test/externalTests/pool-together.sh @@ -45,7 +45,7 @@ function pool_together_test "${compile_only_presets[@]}" #ir-no-optimize # Compilation fails with "YulException: Variable var_amount_205 is 9 slot(s) too deep inside the stack." #ir-optimize-evm-only # Compilation fails with "YulException: Variable var_amount_205 is 9 slot(s) too deep inside the stack." - #ir-optimize-evm+yul # FIXME: ICE due to https://github.com/ethereum/solidity/issues/12558 + ir-optimize-evm+yul legacy-no-optimize legacy-optimize-evm-only legacy-optimize-evm+yul From b9fe628b70b0f118f02822ace678a69c39f5843c Mon Sep 17 00:00:00 2001 From: Marenz Date: Thu, 27 Jan 2022 14:07:50 +0100 Subject: [PATCH 0018/1408] Emit immutable references for pure yul code --- Changelog.md | 1 + libsolidity/interface/StandardCompiler.cpp | 22 ++++---- .../standard_yul_immutable_references/args | 1 + .../input.json | 22 ++++++++ .../output.json | 52 +++++++++++++++++++ .../standard_yul_object_name/output.json | 2 +- 6 files changed, 88 insertions(+), 12 deletions(-) create mode 100644 test/cmdlineTests/standard_yul_immutable_references/args create mode 100644 test/cmdlineTests/standard_yul_immutable_references/input.json create mode 100644 test/cmdlineTests/standard_yul_immutable_references/output.json diff --git a/Changelog.md b/Changelog.md index e75e2c710e..c938c44b0f 100644 --- a/Changelog.md +++ b/Changelog.md @@ -7,6 +7,7 @@ Language Features: Compiler Features: * Yul Optimizer: Remove ``mstore`` and ``sstore`` operations if the slot already contains the same value. + * Yul: Emit immutable references for pure yul code when requested. diff --git a/libsolidity/interface/StandardCompiler.cpp b/libsolidity/interface/StandardCompiler.cpp index 39ed7c10fb..5ccf2d447b 100644 --- a/libsolidity/interface/StandardCompiler.cpp +++ b/libsolidity/interface/StandardCompiler.cpp @@ -1461,36 +1461,36 @@ Json::Value StandardCompiler::compileYul(InputsAndSettings _inputsAndSettings) stack.optimize(); MachineAssemblyObject object; - MachineAssemblyObject runtimeObject; - tie(object, runtimeObject) = stack.assembleWithDeployed(); + MachineAssemblyObject deployedObject; + tie(object, deployedObject) = stack.assembleWithDeployed(); if (object.bytecode) object.bytecode->link(_inputsAndSettings.libraries); - if (runtimeObject.bytecode) - runtimeObject.bytecode->link(_inputsAndSettings.libraries); + if (deployedObject.bytecode) + deployedObject.bytecode->link(_inputsAndSettings.libraries); - for (string const& objectKind: vector{"bytecode", "deployedBytecode"}) + for (auto&& [kind, isDeployed]: {make_pair("bytecode"s, false), make_pair("deployedBytecode"s, true)}) if (isArtifactRequested( _inputsAndSettings.outputSelection, sourceName, contractName, - evmObjectComponents(objectKind), + evmObjectComponents(kind), wildcardMatchesExperimental )) { - MachineAssemblyObject const& o = objectKind == "bytecode" ? object : runtimeObject; + MachineAssemblyObject const& o = isDeployed ? deployedObject : object; if (o.bytecode) - output["contracts"][sourceName][contractName]["evm"][objectKind] = + output["contracts"][sourceName][contractName]["evm"][kind] = collectEVMObject( *o.bytecode, o.sourceMappings.get(), Json::arrayValue, - false, - [&](string const& _element) { return isArtifactRequested( + isDeployed, + [&, kind = kind](string const& _element) { return isArtifactRequested( _inputsAndSettings.outputSelection, sourceName, contractName, - "evm." + objectKind + "." + _element, + "evm." + kind + "." + _element, wildcardMatchesExperimental ); } ); diff --git a/test/cmdlineTests/standard_yul_immutable_references/args b/test/cmdlineTests/standard_yul_immutable_references/args new file mode 100644 index 0000000000..d13a8ac44a --- /dev/null +++ b/test/cmdlineTests/standard_yul_immutable_references/args @@ -0,0 +1 @@ +--pretty-json diff --git a/test/cmdlineTests/standard_yul_immutable_references/input.json b/test/cmdlineTests/standard_yul_immutable_references/input.json new file mode 100644 index 0000000000..aedbae8683 --- /dev/null +++ b/test/cmdlineTests/standard_yul_immutable_references/input.json @@ -0,0 +1,22 @@ +{ + "language": "Yul", + "sources": + { + "A": + { + "content": "object \"YulTest\" { code { let size := datasize(\"runtime\") datacopy(0, dataoffset(\"runtime\"), size) setimmutable(0, \"test\", 1) return(0, size) } object \"runtime\" { code { mstore(0, loadimmutable(\"test\")) return(0, 0x20) } }}" + } + }, + "settings": + { + "outputSelection": { + "A": { + "*": [ + "evm.deployedBytecode.immutableReferences", + "evm.bytecode", + "evm.deployedBytecode" + ] + } + } + } +} diff --git a/test/cmdlineTests/standard_yul_immutable_references/output.json b/test/cmdlineTests/standard_yul_immutable_references/output.json new file mode 100644 index 0000000000..42361097de --- /dev/null +++ b/test/cmdlineTests/standard_yul_immutable_references/output.json @@ -0,0 +1,52 @@ +{ + "contracts": + { + "A": + { + "YulTest": + { + "evm": + { + "bytecode": + { + "functionDebugData": {}, + "generatedSources": [], + "linkReferences": {}, + "object": "60298060156000396001600060010152806000f3fe7f000000000000000000000000000000000000000000000000000000000000000060005260206000f3", + "opcodes": "PUSH1 0x29 DUP1 PUSH1 0x15 PUSH1 0x0 CODECOPY PUSH1 0x1 PUSH1 0x0 PUSH1 0x1 ADD MSTORE DUP1 PUSH1 0x0 RETURN INVALID PUSH32 0x0 PUSH1 0x0 MSTORE PUSH1 0x20 PUSH1 0x0 RETURN ", + "sourceMap": "42:19:0:-:0;100:4;77:21;74:1;65:40;133:1;122;109:26;;;149:4;146:1;139:15" + }, + "deployedBytecode": + { + "functionDebugData": {}, + "generatedSources": [], + "immutableReferences": + { + "test": + [ + { + "length": 32, + "start": 1 + } + ] + }, + "linkReferences": {}, + "object": "7f000000000000000000000000000000000000000000000000000000000000000060005260206000f3", + "opcodes": "PUSH32 0x0 PUSH1 0x0 MSTORE PUSH1 0x20 PUSH1 0x0 RETURN ", + "sourceMap": "203:21:0:-:0;200:1;193:32;241:4;238:1;231:15" + } + } + } + } + }, + "errors": + [ + { + "component": "general", + "formattedMessage": "Yul is still experimental. Please use the output with care.", + "message": "Yul is still experimental. Please use the output with care.", + "severity": "warning", + "type": "Warning" + } + ] +} diff --git a/test/cmdlineTests/standard_yul_object_name/output.json b/test/cmdlineTests/standard_yul_object_name/output.json index f2e85ed7cb..45280c8a1b 100644 --- a/test/cmdlineTests/standard_yul_object_name/output.json +++ b/test/cmdlineTests/standard_yul_object_name/output.json @@ -23,7 +23,7 @@ sub_0: assembly { /* \"A\":137:149 */ revert } -","bytecode":{"functionDebugData":{},"generatedSources":[],"linkReferences":{},"object":"","opcodes":"","sourceMap":""},"deployedBytecode":{"functionDebugData":{},"generatedSources":[],"linkReferences":{},"object":"","opcodes":"","sourceMap":""}},"ir":"object \"NamedObject\" { +","bytecode":{"functionDebugData":{},"generatedSources":[],"linkReferences":{},"object":"","opcodes":"","sourceMap":""},"deployedBytecode":{"functionDebugData":{},"generatedSources":[],"immutableReferences":{},"linkReferences":{},"object":"","opcodes":"","sourceMap":""}},"ir":"object \"NamedObject\" { code { let x := dataoffset(\"DataName\") sstore(add(x, 0), 0) From 2e2094ad820994161cf621c90a0e82f3d1c1d6f2 Mon Sep 17 00:00:00 2001 From: Naveen Sahu <42338831+theNvN@users.noreply.github.com> Date: Wed, 2 Feb 2022 02:02:31 +0530 Subject: [PATCH 0019/1408] plain `address` can be sent Ether too The docs state that a plain `address` cannot be sent Ether. But even though `send` and `transfer` members are not available for plain `address`, the `call` is. And `call` can be invoked upon a plain `address` type to send Ether to the address. For instance, the `someone` (`address` type) can be sent Ether by invoking `sendSomeone()` method in the following `Dummy` contract: ``` contract Dummy { address someone = 0xAb8...cb2; function balanceOf(address addr) public view returns (uint) { return addr.balance; } function sendToSomeone() public payable returns (bool) { (bool sent, ) = someone.call{value: msg.value}(""); return sent; } } ``` --- docs/types/value-types.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/types/value-types.rst b/docs/types/value-types.rst index 30a8577963..0d6a30cd62 100644 --- a/docs/types/value-types.rst +++ b/docs/types/value-types.rst @@ -188,7 +188,8 @@ The address type comes in two flavours, which are largely identical: - ``address payable``: Same as ``address``, but with the additional members ``transfer`` and ``send``. The idea behind this distinction is that ``address payable`` is an address you can send Ether to, -while a plain ``address`` cannot be sent Ether. +while you are not supposed to send Ether to a plain ``address``, for example because it might be a smart contract +that was not built to accept Ether. Type conversions: From 6225dad3328ef1d8a440e2710545795f07bf9844 Mon Sep 17 00:00:00 2001 From: joshuatarkwski Date: Wed, 10 Nov 2021 15:16:11 +0100 Subject: [PATCH 0020/1408] Output searched locations on import failure. --- libsolidity/interface/FileReader.cpp | 23 ++++++++++++++----- .../output.json | 4 ++-- test/solc/CommandLineInterfaceAllowPaths.cpp | 6 ++--- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/libsolidity/interface/FileReader.cpp b/libsolidity/interface/FileReader.cpp index 47f4d2b5aa..8733e91a0d 100644 --- a/libsolidity/interface/FileReader.cpp +++ b/libsolidity/interface/FileReader.cpp @@ -116,7 +116,6 @@ ReadCallback::Result FileReader::readFile(string const& _kind, string const& _so for (auto const& prefix: prefixes) { boost::filesystem::path canonicalPath = normalizeCLIPathForVFS(prefix / strippedSourceUnitName, SymlinkResolution::Enabled); - if (boost::filesystem::exists(canonicalPath)) candidates.push_back(std::move(canonicalPath)); } @@ -124,7 +123,12 @@ ReadCallback::Result FileReader::readFile(string const& _kind, string const& _so auto pathToQuotedString = [](boost::filesystem::path const& _path){ return "\"" + _path.string() + "\""; }; if (candidates.empty()) - return ReadCallback::Result{false, "File not found."}; + return ReadCallback::Result{ + false, + "File not found. Searched the following locations: " + + joinHumanReadable(prefixes | ranges::views::transform(pathToQuotedString), ", ") + + "." + }; if (candidates.size() >= 2) return ReadCallback::Result{ @@ -135,11 +139,13 @@ ReadCallback::Result FileReader::readFile(string const& _kind, string const& _so "." }; - FileSystemPathSet extraAllowedPaths = {m_basePath.empty() ? "." : m_basePath}; - extraAllowedPaths += m_includePaths; + FileSystemPathSet allowedPaths = + m_allowedDirectories + + decltype(allowedPaths){m_basePath.empty() ? "." : m_basePath} + + m_includePaths; bool isAllowed = false; - for (boost::filesystem::path const& allowedDir: m_allowedDirectories + extraAllowedPaths) + for (boost::filesystem::path const& allowedDir: allowedPaths) if (isPathPrefix(normalizeCLIPathForVFS(allowedDir, SymlinkResolution::Enabled), candidates[0])) { isAllowed = true; @@ -147,7 +153,12 @@ ReadCallback::Result FileReader::readFile(string const& _kind, string const& _so } if (!isAllowed) - return ReadCallback::Result{false, "File outside of allowed directories."}; + return ReadCallback::Result{ + false, + "File outside of allowed directories. The following are allowed: " + + joinHumanReadable(allowedPaths | ranges::views::transform(pathToQuotedString), ", ") + + "." + }; if (!boost::filesystem::is_regular_file(candidates[0])) return ReadCallback::Result{false, "Not a valid file."}; diff --git a/test/cmdlineTests/standard_yul_single_file_via_urls/output.json b/test/cmdlineTests/standard_yul_single_file_via_urls/output.json index 3f62fa0fe4..a97057c889 100644 --- a/test/cmdlineTests/standard_yul_single_file_via_urls/output.json +++ b/test/cmdlineTests/standard_yul_single_file_via_urls/output.json @@ -3,8 +3,8 @@ [ { "component": "general", - "formattedMessage": "Cannot import url (\"in.yul\"): File not found.", - "message": "Cannot import url (\"in.yul\"): File not found.", + "formattedMessage": "Cannot import url (\"in.yul\"): File not found. Searched the following locations: \"\".", + "message": "Cannot import url (\"in.yul\"): File not found. Searched the following locations: \"\".", "severity": "error", "type": "IOError" }, diff --git a/test/solc/CommandLineInterfaceAllowPaths.cpp b/test/solc/CommandLineInterfaceAllowPaths.cpp index 77f727014f..472f2184b4 100644 --- a/test/solc/CommandLineInterfaceAllowPaths.cpp +++ b/test/solc/CommandLineInterfaceAllowPaths.cpp @@ -100,7 +100,7 @@ ImportCheck checkImport( return ImportCheck::OK(); static regex const sourceNotFoundErrorRegex{ - R"(^Error \(6275\): Source ".+" not found: (.*)\.\n)" + R"(^Error \(6275\): Source "[^"]+" not found: (.*)\.\n)" R"(\s*--> .*:\d+:\d+:\n)" R"(\s*\|\n)" R"(\d+\s*\| import '.+';\n)" @@ -110,12 +110,12 @@ ImportCheck checkImport( smatch submatches; if (!regex_match(cliResult.stderrContent, submatches, sourceNotFoundErrorRegex)) return ImportCheck::Unknown("Unexpected stderr content: '" + cliResult.stderrContent + "'"); - if (submatches[1] != "File not found" && submatches[1] != "File outside of allowed directories") + if (submatches[1] != "File not found" && !boost::starts_with(string(submatches[1]), "File outside of allowed directories")) return ImportCheck::Unknown("Unexpected error message: '" + cliResult.stderrContent + "'"); if (submatches[1] == "File not found") return ImportCheck::FileNotFound(); - else if (submatches[1] == "File outside of allowed directories") + else if (boost::starts_with(string(submatches[1]), "File outside of allowed directories")) return ImportCheck::PathDisallowed(); else return ImportCheck::Unknown("Unexpected error message '" + submatches[1].str() + "'"); From 9e62f21b2586afb6544f6a03d9c4c2f088dd94f7 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Mon, 17 Jan 2022 21:28:58 +0000 Subject: [PATCH 0021/1408] Add event and error identifiers to cli hashes cmd --- Changelog.md | 1 + libsolidity/interface/CompilerStack.cpp | 32 +++++++++++++++++++++++++ libsolidity/interface/CompilerStack.h | 6 +++++ solc/CommandLineInterface.cpp | 20 ++++++++++++++-- test/cmdlineTests/hashes/args | 1 + test/cmdlineTests/hashes/input.sol | 28 ++++++++++++++++++++++ test/cmdlineTests/hashes/output | 23 ++++++++++++++++++ 7 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 test/cmdlineTests/hashes/args create mode 100644 test/cmdlineTests/hashes/input.sol create mode 100644 test/cmdlineTests/hashes/output diff --git a/Changelog.md b/Changelog.md index c938c44b0f..98515970f0 100644 --- a/Changelog.md +++ b/Changelog.md @@ -6,6 +6,7 @@ Language Features: Compiler Features: + * Commandline Interface: Event and error signatures are also returned when using ``--hashes``. * Yul Optimizer: Remove ``mstore`` and ``sstore`` operations if the slot already contains the same value. * Yul: Emit immutable references for pure yul code when requested. diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index 5292d5329f..a1c26ab374 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -75,6 +75,7 @@ #include #include #include +#include #include @@ -1024,6 +1025,37 @@ Json::Value CompilerStack::methodIdentifiers(string const& _contractName) const return methodIdentifiers; } +Json::Value CompilerStack::errorIdentifiers(string const& _contractName) const +{ + if (m_stackState < AnalysisPerformed) + solThrow(CompilerError, "Analysis was not successful."); + + Json::Value errorIdentifiers(Json::objectValue); + for (ErrorDefinition const* error: contractDefinition(_contractName).interfaceErrors()) + { + string signature = error->functionType(true)->externalSignature(); + errorIdentifiers[signature] = toHex(toCompactBigEndian(selectorFromSignature32(signature), 4)); + } + + return errorIdentifiers; +} + +Json::Value CompilerStack::eventIdentifiers(string const& _contractName) const +{ + if (m_stackState < AnalysisPerformed) + solThrow(CompilerError, "Analysis was not successful."); + + Json::Value eventIdentifiers(Json::objectValue); + for (EventDefinition const* event: contractDefinition(_contractName).interfaceEvents()) + if (!event->isAnonymous()) + { + string signature = event->functionType(true)->externalSignature(); + eventIdentifiers[signature] = toHex(u256(h256::Arith(keccak256(signature)))); + } + + return eventIdentifiers; +} + bytes CompilerStack::cborMetadata(string const& _contractName, bool _forIR) const { if (m_stackState < AnalysisPerformed) diff --git a/libsolidity/interface/CompilerStack.h b/libsolidity/interface/CompilerStack.h index 3609662e02..2e1e687f71 100644 --- a/libsolidity/interface/CompilerStack.h +++ b/libsolidity/interface/CompilerStack.h @@ -330,6 +330,12 @@ class CompilerStack: public langutil::CharStreamProvider /// @returns a JSON representing a map of method identifiers (hashes) to function names. Json::Value methodIdentifiers(std::string const& _contractName) const; + /// @returns a JSON representing a map of error identifiers (hashes) to error names. + Json::Value errorIdentifiers(std::string const& _contractName) const; + + /// @returns a JSON representing a map of event identifiers (hashes) to event names. + Json::Value eventIdentifiers(std::string const& _contractName) const; + /// @returns the Contract Metadata matching the pipeline selected using the viaIR setting. std::string const& metadata(std::string const& _contractName) const { return metadata(contract(_contractName)); } diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index 235059a02b..d96460fa1e 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -271,14 +271,30 @@ void CommandLineInterface::handleSignatureHashes(string const& _contract) return; Json::Value methodIdentifiers = m_compiler->methodIdentifiers(_contract); - string out; + string out = "Function signatures:\n"; for (auto const& name: methodIdentifiers.getMemberNames()) out += methodIdentifiers[name].asString() + ": " + name + "\n"; + Json::Value errorIdentifiers = m_compiler->errorIdentifiers(_contract); + if (!errorIdentifiers.empty()) + { + out += "\nError signatures:\n"; + for (auto const& name: errorIdentifiers.getMemberNames()) + out += errorIdentifiers[name].asString() + ": " + name + "\n"; + } + + Json::Value eventIdentifiers = m_compiler->eventIdentifiers(_contract); + if (!eventIdentifiers.empty()) + { + out += "\nEvent signatures:\n"; + for (auto const& name: eventIdentifiers.getMemberNames()) + out += eventIdentifiers[name].asString() + ": " + name + "\n"; + } + if (!m_options.output.dir.empty()) createFile(m_compiler->filesystemFriendlyName(_contract) + ".signatures", out); else - sout() << "Function signatures:" << endl << out; + sout() << out; } void CommandLineInterface::handleMetadata(string const& _contract) diff --git a/test/cmdlineTests/hashes/args b/test/cmdlineTests/hashes/args new file mode 100644 index 0000000000..40469d358c --- /dev/null +++ b/test/cmdlineTests/hashes/args @@ -0,0 +1 @@ +--hashes \ No newline at end of file diff --git a/test/cmdlineTests/hashes/input.sol b/test/cmdlineTests/hashes/input.sol new file mode 100644 index 0000000000..1f5c758f71 --- /dev/null +++ b/test/cmdlineTests/hashes/input.sol @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-3.0 +pragma solidity >=0.0; + +error fileLevelError(uint z); + +library L { + event libraryEvent(uint r); + error libraryError(uint r); + error libraryErrorUnused(uint u); + event libraryEventUnused(uint u); +} + +contract C { + struct S { uint x; } + + event ev(uint y); + event anon_ev(uint y) anonymous; + + error err(uint z, uint w); + + function f(S memory s) public { + emit L.libraryEvent(3); + if (s.x > 1) + revert fileLevelError(3); + else + revert L.libraryError(4); + } +} diff --git a/test/cmdlineTests/hashes/output b/test/cmdlineTests/hashes/output new file mode 100644 index 0000000000..5223d732a3 --- /dev/null +++ b/test/cmdlineTests/hashes/output @@ -0,0 +1,23 @@ + +======= hashes/input.sol:C ======= +Function signatures: +3fc03eeb: f((uint256)) + +Error signatures: +619a0bb7: err(uint256,uint256) +82b5f64f: fileLevelError(uint256) +8c41f45c: libraryError(uint256) + +Event signatures: +2d4dd5fe18ada5a020a9f5591539a8dc3010a5c074ba6a70e1c956659f02786a: ev(uint256) + +======= hashes/input.sol:L ======= +Function signatures: + +Error signatures: +8c41f45c: libraryError(uint256) +c61c03f5: libraryErrorUnused(uint256) + +Event signatures: +81f3fb02f88d32d3bb08c80c9a622ca3b3223292f131c6ad049811f9a8a606dc: libraryEvent(uint256) +0a994ad3600197f16ffe1ea1101caea3174efe5ebd9ba9a75d6d5524c5de28cd: libraryEventUnused(uint256) From 3e7c68d9b09b59a959f32975f21ba3ec19152afa Mon Sep 17 00:00:00 2001 From: Marenz Date: Mon, 31 Jan 2022 17:52:08 +0100 Subject: [PATCH 0022/1408] Merge identifier query methods into one --- libsolidity/interface/CompilerStack.cpp | 33 +++++-------------- libsolidity/interface/CompilerStack.h | 10 ++---- libsolidity/interface/StandardCompiler.cpp | 2 +- solc/CommandLineInterface.cpp | 22 ++++++------- .../input.json | 17 ++++++++++ .../output.json | 1 + test/libsolidity/SemanticTest.cpp | 2 +- .../tools/ossfuzz/SolidityEvmoneInterface.cpp | 2 +- test/tools/ossfuzz/SolidityEvmoneInterface.h | 2 +- 9 files changed, 43 insertions(+), 48 deletions(-) create mode 100644 test/cmdlineTests/standard_method_identifiers_requested_empty/input.json create mode 100644 test/cmdlineTests/standard_method_identifiers_requested_empty/output.json diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index a1c26ab374..acf1932956 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -1014,46 +1014,31 @@ Json::Value const& CompilerStack::natspecDev(Contract const& _contract) const return _contract.devDocumentation.init([&]{ return Natspec::devDocumentation(*_contract.contract); }); } -Json::Value CompilerStack::methodIdentifiers(string const& _contractName) const +Json::Value CompilerStack::contractIdentifiers(string const& _contractName) const { if (m_stackState < AnalysisPerformed) solThrow(CompilerError, "Analysis was not successful."); - Json::Value methodIdentifiers(Json::objectValue); - for (auto const& it: contractDefinition(_contractName).interfaceFunctions()) - methodIdentifiers[it.second->externalSignature()] = it.first.hex(); - return methodIdentifiers; -} + Json::Value contractIdentifiers(Json::objectValue); + // Always have a methods object + contractIdentifiers["methods"] = Json::objectValue; -Json::Value CompilerStack::errorIdentifiers(string const& _contractName) const -{ - if (m_stackState < AnalysisPerformed) - solThrow(CompilerError, "Analysis was not successful."); - - Json::Value errorIdentifiers(Json::objectValue); + for (auto const& it: contractDefinition(_contractName).interfaceFunctions()) + contractIdentifiers["methods"][it.second->externalSignature()] = it.first.hex(); for (ErrorDefinition const* error: contractDefinition(_contractName).interfaceErrors()) { string signature = error->functionType(true)->externalSignature(); - errorIdentifiers[signature] = toHex(toCompactBigEndian(selectorFromSignature32(signature), 4)); + contractIdentifiers["errors"][signature] = toHex(toCompactBigEndian(selectorFromSignature32(signature), 4)); } - return errorIdentifiers; -} - -Json::Value CompilerStack::eventIdentifiers(string const& _contractName) const -{ - if (m_stackState < AnalysisPerformed) - solThrow(CompilerError, "Analysis was not successful."); - - Json::Value eventIdentifiers(Json::objectValue); for (EventDefinition const* event: contractDefinition(_contractName).interfaceEvents()) if (!event->isAnonymous()) { string signature = event->functionType(true)->externalSignature(); - eventIdentifiers[signature] = toHex(u256(h256::Arith(keccak256(signature)))); + contractIdentifiers["events"][signature] = toHex(u256(h256::Arith(keccak256(signature)))); } - return eventIdentifiers; + return contractIdentifiers; } bytes CompilerStack::cborMetadata(string const& _contractName, bool _forIR) const diff --git a/libsolidity/interface/CompilerStack.h b/libsolidity/interface/CompilerStack.h index 2e1e687f71..b1f85dd4df 100644 --- a/libsolidity/interface/CompilerStack.h +++ b/libsolidity/interface/CompilerStack.h @@ -327,14 +327,8 @@ class CompilerStack: public langutil::CharStreamProvider /// Prerequisite: Successful call to parse or compile. Json::Value const& natspecDev(std::string const& _contractName) const; - /// @returns a JSON representing a map of method identifiers (hashes) to function names. - Json::Value methodIdentifiers(std::string const& _contractName) const; - - /// @returns a JSON representing a map of error identifiers (hashes) to error names. - Json::Value errorIdentifiers(std::string const& _contractName) const; - - /// @returns a JSON representing a map of event identifiers (hashes) to event names. - Json::Value eventIdentifiers(std::string const& _contractName) const; + /// @returns a JSON object with the three members ``methods``, ``events``, ``errors``. Each is a map, mapping identifiers (hashes) to function names. + Json::Value contractIdentifiers(std::string const& _contractName) const; /// @returns the Contract Metadata matching the pipeline selected using the viaIR setting. std::string const& metadata(std::string const& _contractName) const { return metadata(contract(_contractName)); } diff --git a/libsolidity/interface/StandardCompiler.cpp b/libsolidity/interface/StandardCompiler.cpp index 5ccf2d447b..0e5b8aba46 100644 --- a/libsolidity/interface/StandardCompiler.cpp +++ b/libsolidity/interface/StandardCompiler.cpp @@ -1298,7 +1298,7 @@ Json::Value StandardCompiler::compileSolidity(StandardCompiler::InputsAndSetting if (compilationSuccess && isArtifactRequested(_inputsAndSettings.outputSelection, file, name, "evm.legacyAssembly", wildcardMatchesExperimental)) evmData["legacyAssembly"] = compilerStack.assemblyJSON(contractName); if (isArtifactRequested(_inputsAndSettings.outputSelection, file, name, "evm.methodIdentifiers", wildcardMatchesExperimental)) - evmData["methodIdentifiers"] = compilerStack.methodIdentifiers(contractName); + evmData["methodIdentifiers"] = compilerStack.contractIdentifiers(contractName)["methods"]; if (compilationSuccess && isArtifactRequested(_inputsAndSettings.outputSelection, file, name, "evm.gasEstimates", wildcardMatchesExperimental)) evmData["gasEstimates"] = compilerStack.gasEstimates(contractName); diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index d96460fa1e..ba557918c1 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -270,25 +270,23 @@ void CommandLineInterface::handleSignatureHashes(string const& _contract) if (!m_options.compiler.outputs.signatureHashes) return; - Json::Value methodIdentifiers = m_compiler->methodIdentifiers(_contract); + Json::Value contractIdentifiers = m_compiler->contractIdentifiers(_contract); string out = "Function signatures:\n"; - for (auto const& name: methodIdentifiers.getMemberNames()) - out += methodIdentifiers[name].asString() + ": " + name + "\n"; + for (auto const& name: contractIdentifiers["methods"].getMemberNames()) + out += contractIdentifiers["methods"][name].asString() + ": " + name + "\n"; - Json::Value errorIdentifiers = m_compiler->errorIdentifiers(_contract); - if (!errorIdentifiers.empty()) + if (contractIdentifiers.isMember("errors")) { out += "\nError signatures:\n"; - for (auto const& name: errorIdentifiers.getMemberNames()) - out += errorIdentifiers[name].asString() + ": " + name + "\n"; + for (auto const& name: contractIdentifiers["errors"].getMemberNames()) + out += contractIdentifiers["errors"][name].asString() + ": " + name + "\n"; } - Json::Value eventIdentifiers = m_compiler->eventIdentifiers(_contract); - if (!eventIdentifiers.empty()) + if (contractIdentifiers.isMember("events")) { out += "\nEvent signatures:\n"; - for (auto const& name: eventIdentifiers.getMemberNames()) - out += eventIdentifiers[name].asString() + ": " + name + "\n"; + for (auto const& name: contractIdentifiers["events"].getMemberNames()) + out += contractIdentifiers["events"][name].asString() + ": " + name + "\n"; } if (!m_options.output.dir.empty()) @@ -838,7 +836,7 @@ void CommandLineInterface::handleCombinedJSON() m_compiler->runtimeObject(contractName).functionDebugData ); if (m_options.compiler.combinedJsonRequests->signatureHashes) - contractData[g_strSignatureHashes] = m_compiler->methodIdentifiers(contractName); + contractData[g_strSignatureHashes] = m_compiler->contractIdentifiers(contractName)["methods"]; if (m_options.compiler.combinedJsonRequests->natspecDev) contractData[g_strNatspecDev] = m_compiler->natspecDev(contractName); if (m_options.compiler.combinedJsonRequests->natspecUser) diff --git a/test/cmdlineTests/standard_method_identifiers_requested_empty/input.json b/test/cmdlineTests/standard_method_identifiers_requested_empty/input.json new file mode 100644 index 0000000000..28b9699b51 --- /dev/null +++ b/test/cmdlineTests/standard_method_identifiers_requested_empty/input.json @@ -0,0 +1,17 @@ +{ + "language": "Solidity", + "sources": + { + "A": + { + "content": "// SPDX-License-Identifier: GPL-3.0\npragma solidity >=0.0; contract C { }" + } + }, + "settings": + { + "outputSelection": + { + "*": { "*": ["evm.methodIdentifiers"] } + } + } +} diff --git a/test/cmdlineTests/standard_method_identifiers_requested_empty/output.json b/test/cmdlineTests/standard_method_identifiers_requested_empty/output.json new file mode 100644 index 0000000000..7953dc1572 --- /dev/null +++ b/test/cmdlineTests/standard_method_identifiers_requested_empty/output.json @@ -0,0 +1 @@ +{"contracts":{"A":{"C":{"evm":{"methodIdentifiers":{}}}}},"sources":{"A":{"id":0}}} diff --git a/test/libsolidity/SemanticTest.cpp b/test/libsolidity/SemanticTest.cpp index 4fbb16614c..342cf24fc2 100644 --- a/test/libsolidity/SemanticTest.cpp +++ b/test/libsolidity/SemanticTest.cpp @@ -435,7 +435,7 @@ TestCase::TestResult SemanticTest::runTest( { soltestAssert( m_allowNonExistingFunctions || - m_compiler.methodIdentifiers(m_compiler.lastContractName(m_sources.mainSourceFile)).isMember(test.call().signature), + m_compiler.contractIdentifiers(m_compiler.lastContractName(m_sources.mainSourceFile))["methods"].isMember(test.call().signature), "The function " + test.call().signature + " is not known to the compiler" ); diff --git a/test/tools/ossfuzz/SolidityEvmoneInterface.cpp b/test/tools/ossfuzz/SolidityEvmoneInterface.cpp index 4d536183c3..49b17930a3 100644 --- a/test/tools/ossfuzz/SolidityEvmoneInterface.cpp +++ b/test/tools/ossfuzz/SolidityEvmoneInterface.cpp @@ -58,7 +58,7 @@ optional SolidityCompilationFramework::compileContract() else contractName = m_compilerInput.contractName; evmasm::LinkerObject obj = m_compiler.object(contractName); - Json::Value methodIdentifiers = m_compiler.methodIdentifiers(contractName); + Json::Value methodIdentifiers = m_compiler.contractIdentifiers(contractName)["methods"]; return CompilerOutput{obj.bytecode, methodIdentifiers}; } } diff --git a/test/tools/ossfuzz/SolidityEvmoneInterface.h b/test/tools/ossfuzz/SolidityEvmoneInterface.h index 980eb9a842..51f6d5bdcc 100644 --- a/test/tools/ossfuzz/SolidityEvmoneInterface.h +++ b/test/tools/ossfuzz/SolidityEvmoneInterface.h @@ -91,7 +91,7 @@ class SolidityCompilationFramework /// @returns method identifiers in contract called @param _contractName. Json::Value methodIdentifiers(std::string const& _contractName) { - return m_compiler.methodIdentifiers(_contractName); + return m_compiler.contractIdentifiers(_contractName)["methods"]; } /// @returns Compilation output comprising EVM bytecode and list of /// method identifiers in contract if compilation is successful, From 0f05b1485e0fa8f54119ba0f48c3e7b7bcac3044 Mon Sep 17 00:00:00 2001 From: yatharthagoenka Date: Sat, 4 Sep 2021 15:37:18 +0530 Subject: [PATCH 0023/1408] CI: shared build and dependency installation steps --- .circleci/config.yml | 62 +++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 58a4c27428..be4940c636 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -214,6 +214,14 @@ defaults: command: ./test/lsp.py ./build/solc/solc - gitter_notify_failure_unless_pr + - steps_build: &steps_build + steps: + - checkout + - run: *run_build + - store_artifacts: *artifacts_solc + - persist_to_workspace: *artifacts_executables + - gitter_notify_failure_unless_pr + - steps_soltest_all: &steps_soltest_all steps: - checkout @@ -234,6 +242,14 @@ defaults: - store_artifacts: *artifacts_test_results - gitter_notify_failure_unless_pr + - steps_install_dependencies_osx: &steps_install_dependencies_osx + steps: + - restore_cache: + keys: + - dependencies-osx-{{ arch }}-{{ checksum ".circleci/osx_install_dependencies.sh" }} + - attach_workspace: + at: . + # -------------------------------------------------------------------------- # Base Image Templates @@ -748,12 +764,7 @@ jobs: CMAKE_OPTIONS: -DSANITIZE=address MAKEFLAGS: -j 3 CMAKE_BUILD_TYPE: Release - steps: - - checkout - - run: *run_build - - store_artifacts: *artifacts_solc - - persist_to_workspace: *artifacts_executables - - gitter_notify_failure_unless_pr + <<: *steps_build b_ubu_clang: &b_ubu_clang <<: *base_ubuntu2004_clang_large @@ -762,12 +773,7 @@ jobs: CC: clang CXX: clang++ MAKEFLAGS: -j 10 - steps: - - checkout - - run: *run_build - - store_artifacts: *artifacts_solc - - persist_to_workspace: *artifacts_executables - - gitter_notify_failure_unless_pr + <<: *steps_build b_ubu_asan_clang: &b_ubu_asan_clang # This runs a bit faster on large and xlarge but on nightly efficiency matters more. @@ -777,12 +783,7 @@ jobs: CXX: clang++ CMAKE_OPTIONS: -DSANITIZE=address MAKEFLAGS: -j 3 - steps: - - checkout - - run: *run_build - - store_artifacts: *artifacts_solc - - persist_to_workspace: *artifacts_executables - - gitter_notify_failure_unless_pr + <<: *steps_build b_ubu_ubsan_clang: &b_ubu_ubsan_clang # This runs a bit faster on large and xlarge but on nightly efficiency matters more. @@ -792,12 +793,7 @@ jobs: CXX: clang++ CMAKE_OPTIONS: -DSANITIZE=undefined MAKEFLAGS: -j 3 - steps: - - checkout - - run: *run_build - - store_artifacts: *artifacts_solc - - persist_to_workspace: *artifacts_executables - - gitter_notify_failure_unless_pr + <<: *steps_build b_ubu_release: &b_ubu_release <<: *b_ubu @@ -949,7 +945,7 @@ jobs: - build/test/tools/solfuzzer - gitter_notify_failure_unless_pr - t_osx_soltest: + t_osx_soltest: &t_osx_soltest <<: *base_osx environment: EVM: << pipeline.parameters.evm-version >> @@ -957,11 +953,9 @@ jobs: TERM: xterm steps: - checkout - - restore_cache: - keys: - - dependencies-osx-{{ arch }}-{{ checksum ".circleci/osx_install_dependencies.sh" }} - - attach_workspace: - at: . + - when: + condition: true + <<: *steps_install_dependencies_osx - run: *run_soltest - store_test_results: *store_test_results - store_artifacts: *artifacts_test_results @@ -971,11 +965,9 @@ jobs: <<: *base_osx steps: - checkout - - restore_cache: - keys: - - dependencies-osx-{{ arch }}-{{ checksum ".circleci/osx_install_dependencies.sh" }} - - attach_workspace: - at: . + - when: + condition: true + <<: *steps_install_dependencies_osx - run: *run_cmdline_tests - store_artifacts: *artifacts_test_results - gitter_notify_failure_unless_pr From bce5f9e0da5d96b4f0ed715d3138775016e6337f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Tue, 8 Feb 2022 14:55:49 +0100 Subject: [PATCH 0024/1408] CI: Store all artifacts in steps_build so that we can use it for b_ubu --- .circleci/config.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index be4940c636..6069ba4fbb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -219,6 +219,8 @@ defaults: - checkout - run: *run_build - store_artifacts: *artifacts_solc + - store_artifacts: *artifact_solidity_upgrade + - store_artifacts: *artifact_yul_phaser - persist_to_workspace: *artifacts_executables - gitter_notify_failure_unless_pr @@ -747,14 +749,7 @@ jobs: # this runs 2x faster on xlarge but takes 4x more resources (compared to medium). # Enough other jobs depend on it that it's worth it though. <<: *base_ubuntu2004_xlarge - steps: - - checkout - - run: *run_build - - store_artifacts: *artifacts_solc - - store_artifacts: *artifact_solidity_upgrade - - store_artifacts: *artifact_yul_phaser - - persist_to_workspace: *artifacts_executables - - gitter_notify_failure_unless_pr + <<: *steps_build # x64 ASAN build, for testing for memory related bugs b_ubu_asan: &b_ubu_asan From bdf84c991fe6902cce3bac52a4a970ae5a1bca3f Mon Sep 17 00:00:00 2001 From: yatharthagoenka Date: Sat, 4 Sep 2021 15:37:18 +0530 Subject: [PATCH 0025/1408] Ci: parameterize the clang sanitizer jobs --- .circleci/config.yml | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6069ba4fbb..39c219bcec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -604,6 +604,15 @@ defaults: nodejs_version: '14' resource_class: medium + - job_b_ubu_asan_clang: &job_b_ubu_asan_clang + <<: *workflow_trigger_on_tags + name: b_ubu_asan_clang + cmake_options: -DSANITIZE=address + - job_b_ubu_ubsan_clang: &job_b_ubu_ubsan_clang + <<: *workflow_trigger_on_tags + name: b_ubu_ubsan_clang + cmake_options: -DSANITIZE=address + # ----------------------------------------------------------------------------------------------- jobs: @@ -770,24 +779,18 @@ jobs: MAKEFLAGS: -j 10 <<: *steps_build - b_ubu_asan_clang: &b_ubu_asan_clang - # This runs a bit faster on large and xlarge but on nightly efficiency matters more. - <<: *base_ubuntu2004_clang - environment: - CC: clang - CXX: clang++ - CMAKE_OPTIONS: -DSANITIZE=address - MAKEFLAGS: -j 3 - <<: *steps_build - - b_ubu_ubsan_clang: &b_ubu_ubsan_clang + b_ubu_san_clang: # This runs a bit faster on large and xlarge but on nightly efficiency matters more. + parameters: + cmake_options: + type: string <<: *base_ubuntu2004_clang environment: + TERM: xterm CC: clang CXX: clang++ - CMAKE_OPTIONS: -DSANITIZE=undefined MAKEFLAGS: -j 3 + CMAKE_OPTIONS: << parameters.cmake_options >> <<: *steps_build b_ubu_release: &b_ubu_release @@ -1506,13 +1509,13 @@ workflows: # ASan build and tests - b_ubu_asan: *workflow_trigger_on_tags - - b_ubu_asan_clang: *workflow_trigger_on_tags + - b_ubu_san_clang: *job_b_ubu_asan_clang - t_ubu_asan_soltest: *workflow_ubuntu2004_asan - t_ubu_asan_clang_soltest: *workflow_ubuntu2004_asan_clang - t_ubu_asan_cli: *workflow_ubuntu2004_asan # UBSan build and tests - - b_ubu_ubsan_clang: *workflow_trigger_on_tags + - b_ubu_san_clang: *job_b_ubu_ubsan_clang - t_ubu_ubsan_clang_soltest: *workflow_ubuntu2004_ubsan_clang - t_ubu_ubsan_clang_cli: *workflow_ubuntu2004_ubsan_clang From 46075d04d953981119f6376851051334076cf946 Mon Sep 17 00:00:00 2001 From: Marenz Date: Tue, 1 Feb 2022 15:42:08 +0100 Subject: [PATCH 0026/1408] Include used events in ``--hashes`` output --- libsolidity/ast/AST.cpp | 20 ++++++++++++------ libsolidity/ast/AST.h | 3 ++- libsolidity/interface/ABI.cpp | 2 +- libsolidity/interface/CompilerStack.cpp | 21 ++++++++++++------- libsolidity/interface/CompilerStack.h | 2 +- libsolidity/interface/Natspec.cpp | 2 +- libsolidity/interface/StandardCompiler.cpp | 2 +- solc/CommandLineInterface.cpp | 20 +++++++++--------- test/cmdlineTests/hashes/output | 1 + test/libsolidity/SemanticTest.cpp | 2 +- .../tools/ossfuzz/SolidityEvmoneInterface.cpp | 2 +- test/tools/ossfuzz/SolidityEvmoneInterface.h | 2 +- 12 files changed, 47 insertions(+), 32 deletions(-) diff --git a/libsolidity/ast/AST.cpp b/libsolidity/ast/AST.cpp index f2b7369b71..2a7609de85 100644 --- a/libsolidity/ast/AST.cpp +++ b/libsolidity/ast/AST.cpp @@ -192,7 +192,7 @@ FunctionDefinition const* ContractDefinition::receiveFunction() const return nullptr; } -vector const& ContractDefinition::interfaceEvents() const +vector const& ContractDefinition::definedInterfaceEvents() const { return m_interfaceEvents.init([&]{ set eventsSeen; @@ -213,11 +213,20 @@ vector const& ContractDefinition::interfaceEvents() cons interfaceEvents.push_back(e); } } - return interfaceEvents; }); } +vector const ContractDefinition::usedInterfaceEvents() const +{ + solAssert(annotation().creationCallGraph.set(), ""); + + return convertContainer>( + (*annotation().creationCallGraph)->emittedEvents + + (*annotation().deployedCallGraph)->emittedEvents + ); +} + vector ContractDefinition::interfaceErrors(bool _requireCallGraph) const { set result; @@ -227,10 +236,9 @@ vector ContractDefinition::interfaceErrors(bool _require if (_requireCallGraph) solAssert(annotation().creationCallGraph.set(), ""); if (annotation().creationCallGraph.set()) - { - result += (*annotation().creationCallGraph)->usedErrors; - result += (*annotation().deployedCallGraph)->usedErrors; - } + result += + (*annotation().creationCallGraph)->usedErrors + + (*annotation().deployedCallGraph)->usedErrors; return convertContainer>(move(result)); } diff --git a/libsolidity/ast/AST.h b/libsolidity/ast/AST.h index b7d5db8058..c35c69c9d2 100644 --- a/libsolidity/ast/AST.h +++ b/libsolidity/ast/AST.h @@ -519,7 +519,8 @@ class ContractDefinition: public Declaration, public StructurallyDocumented, pub return ranges::subrange(b, e) | ranges::views::values; } std::vector events() const { return filteredNodes(m_subNodes); } - std::vector const& interfaceEvents() const; + std::vector const& definedInterfaceEvents() const; + std::vector const usedInterfaceEvents() const; /// @returns all errors defined in this contract or any base contract /// and all errors referenced during execution. /// @param _requireCallGraph if false, do not fail if the call graph has not been computed yet. diff --git a/libsolidity/interface/ABI.cpp b/libsolidity/interface/ABI.cpp index 9c7a645b6f..eab1801c6a 100644 --- a/libsolidity/interface/ABI.cpp +++ b/libsolidity/interface/ABI.cpp @@ -101,7 +101,7 @@ Json::Value ABI::generate(ContractDefinition const& _contractDef) method["stateMutability"] = stateMutabilityToString(externalFunctionType->stateMutability()); abi.emplace(std::move(method)); } - for (auto const& it: _contractDef.interfaceEvents()) + for (auto const& it: _contractDef.definedInterfaceEvents()) { Json::Value event{Json::objectValue}; event["type"] = "event"; diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index acf1932956..7a8bc4ddbf 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -81,6 +81,8 @@ #include +#include + #include #include #include @@ -1014,31 +1016,34 @@ Json::Value const& CompilerStack::natspecDev(Contract const& _contract) const return _contract.devDocumentation.init([&]{ return Natspec::devDocumentation(*_contract.contract); }); } -Json::Value CompilerStack::contractIdentifiers(string const& _contractName) const +Json::Value CompilerStack::interfaceSymbols(string const& _contractName) const { if (m_stackState < AnalysisPerformed) solThrow(CompilerError, "Analysis was not successful."); - Json::Value contractIdentifiers(Json::objectValue); + Json::Value interfaceSymbols(Json::objectValue); // Always have a methods object - contractIdentifiers["methods"] = Json::objectValue; + interfaceSymbols["methods"] = Json::objectValue; for (auto const& it: contractDefinition(_contractName).interfaceFunctions()) - contractIdentifiers["methods"][it.second->externalSignature()] = it.first.hex(); + interfaceSymbols["methods"][it.second->externalSignature()] = it.first.hex(); for (ErrorDefinition const* error: contractDefinition(_contractName).interfaceErrors()) { string signature = error->functionType(true)->externalSignature(); - contractIdentifiers["errors"][signature] = toHex(toCompactBigEndian(selectorFromSignature32(signature), 4)); + interfaceSymbols["errors"][signature] = toHex(toCompactBigEndian(selectorFromSignature32(signature), 4)); } - for (EventDefinition const* event: contractDefinition(_contractName).interfaceEvents()) + for (EventDefinition const* event: ranges::concat_view( + contractDefinition(_contractName).definedInterfaceEvents(), + contractDefinition(_contractName).usedInterfaceEvents() + )) if (!event->isAnonymous()) { string signature = event->functionType(true)->externalSignature(); - contractIdentifiers["events"][signature] = toHex(u256(h256::Arith(keccak256(signature)))); + interfaceSymbols["events"][signature] = toHex(u256(h256::Arith(keccak256(signature)))); } - return contractIdentifiers; + return interfaceSymbols; } bytes CompilerStack::cborMetadata(string const& _contractName, bool _forIR) const diff --git a/libsolidity/interface/CompilerStack.h b/libsolidity/interface/CompilerStack.h index b1f85dd4df..c1f15a4804 100644 --- a/libsolidity/interface/CompilerStack.h +++ b/libsolidity/interface/CompilerStack.h @@ -328,7 +328,7 @@ class CompilerStack: public langutil::CharStreamProvider Json::Value const& natspecDev(std::string const& _contractName) const; /// @returns a JSON object with the three members ``methods``, ``events``, ``errors``. Each is a map, mapping identifiers (hashes) to function names. - Json::Value contractIdentifiers(std::string const& _contractName) const; + Json::Value interfaceSymbols(std::string const& _contractName) const; /// @returns the Contract Metadata matching the pipeline selected using the viaIR setting. std::string const& metadata(std::string const& _contractName) const { return metadata(contract(_contractName)); } diff --git a/libsolidity/interface/Natspec.cpp b/libsolidity/interface/Natspec.cpp index 533cbaa3dd..4496904937 100644 --- a/libsolidity/interface/Natspec.cpp +++ b/libsolidity/interface/Natspec.cpp @@ -78,7 +78,7 @@ Json::Value Natspec::userDocumentation(ContractDefinition const& _contractDef) doc["methods"][it.second->externalSignature()]["notice"] = value; } - for (auto const& event: _contractDef.interfaceEvents()) + for (auto const& event: _contractDef.definedInterfaceEvents()) { string value = extractDoc(event->annotation().docTags, "notice"); if (!value.empty()) diff --git a/libsolidity/interface/StandardCompiler.cpp b/libsolidity/interface/StandardCompiler.cpp index 0e5b8aba46..75876d9353 100644 --- a/libsolidity/interface/StandardCompiler.cpp +++ b/libsolidity/interface/StandardCompiler.cpp @@ -1298,7 +1298,7 @@ Json::Value StandardCompiler::compileSolidity(StandardCompiler::InputsAndSetting if (compilationSuccess && isArtifactRequested(_inputsAndSettings.outputSelection, file, name, "evm.legacyAssembly", wildcardMatchesExperimental)) evmData["legacyAssembly"] = compilerStack.assemblyJSON(contractName); if (isArtifactRequested(_inputsAndSettings.outputSelection, file, name, "evm.methodIdentifiers", wildcardMatchesExperimental)) - evmData["methodIdentifiers"] = compilerStack.contractIdentifiers(contractName)["methods"]; + evmData["methodIdentifiers"] = compilerStack.interfaceSymbols(contractName)["methods"]; if (compilationSuccess && isArtifactRequested(_inputsAndSettings.outputSelection, file, name, "evm.gasEstimates", wildcardMatchesExperimental)) evmData["gasEstimates"] = compilerStack.gasEstimates(contractName); diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index ba557918c1..bafd3a7356 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -270,23 +270,23 @@ void CommandLineInterface::handleSignatureHashes(string const& _contract) if (!m_options.compiler.outputs.signatureHashes) return; - Json::Value contractIdentifiers = m_compiler->contractIdentifiers(_contract); + Json::Value interfaceSymbols = m_compiler->interfaceSymbols(_contract); string out = "Function signatures:\n"; - for (auto const& name: contractIdentifiers["methods"].getMemberNames()) - out += contractIdentifiers["methods"][name].asString() + ": " + name + "\n"; + for (auto const& name: interfaceSymbols["methods"].getMemberNames()) + out += interfaceSymbols["methods"][name].asString() + ": " + name + "\n"; - if (contractIdentifiers.isMember("errors")) + if (interfaceSymbols.isMember("errors")) { out += "\nError signatures:\n"; - for (auto const& name: contractIdentifiers["errors"].getMemberNames()) - out += contractIdentifiers["errors"][name].asString() + ": " + name + "\n"; + for (auto const& name: interfaceSymbols["errors"].getMemberNames()) + out += interfaceSymbols["errors"][name].asString() + ": " + name + "\n"; } - if (contractIdentifiers.isMember("events")) + if (interfaceSymbols.isMember("events")) { out += "\nEvent signatures:\n"; - for (auto const& name: contractIdentifiers["events"].getMemberNames()) - out += contractIdentifiers["events"][name].asString() + ": " + name + "\n"; + for (auto const& name: interfaceSymbols["events"].getMemberNames()) + out += interfaceSymbols["events"][name].asString() + ": " + name + "\n"; } if (!m_options.output.dir.empty()) @@ -836,7 +836,7 @@ void CommandLineInterface::handleCombinedJSON() m_compiler->runtimeObject(contractName).functionDebugData ); if (m_options.compiler.combinedJsonRequests->signatureHashes) - contractData[g_strSignatureHashes] = m_compiler->contractIdentifiers(contractName)["methods"]; + contractData[g_strSignatureHashes] = m_compiler->interfaceSymbols(contractName)["methods"]; if (m_options.compiler.combinedJsonRequests->natspecDev) contractData[g_strNatspecDev] = m_compiler->natspecDev(contractName); if (m_options.compiler.combinedJsonRequests->natspecUser) diff --git a/test/cmdlineTests/hashes/output b/test/cmdlineTests/hashes/output index 5223d732a3..b010456d39 100644 --- a/test/cmdlineTests/hashes/output +++ b/test/cmdlineTests/hashes/output @@ -10,6 +10,7 @@ Error signatures: Event signatures: 2d4dd5fe18ada5a020a9f5591539a8dc3010a5c074ba6a70e1c956659f02786a: ev(uint256) +81f3fb02f88d32d3bb08c80c9a622ca3b3223292f131c6ad049811f9a8a606dc: libraryEvent(uint256) ======= hashes/input.sol:L ======= Function signatures: diff --git a/test/libsolidity/SemanticTest.cpp b/test/libsolidity/SemanticTest.cpp index 342cf24fc2..1f07f177d4 100644 --- a/test/libsolidity/SemanticTest.cpp +++ b/test/libsolidity/SemanticTest.cpp @@ -435,7 +435,7 @@ TestCase::TestResult SemanticTest::runTest( { soltestAssert( m_allowNonExistingFunctions || - m_compiler.contractIdentifiers(m_compiler.lastContractName(m_sources.mainSourceFile))["methods"].isMember(test.call().signature), + m_compiler.interfaceSymbols(m_compiler.lastContractName(m_sources.mainSourceFile))["methods"].isMember(test.call().signature), "The function " + test.call().signature + " is not known to the compiler" ); diff --git a/test/tools/ossfuzz/SolidityEvmoneInterface.cpp b/test/tools/ossfuzz/SolidityEvmoneInterface.cpp index 49b17930a3..ff9fcf2d8d 100644 --- a/test/tools/ossfuzz/SolidityEvmoneInterface.cpp +++ b/test/tools/ossfuzz/SolidityEvmoneInterface.cpp @@ -58,7 +58,7 @@ optional SolidityCompilationFramework::compileContract() else contractName = m_compilerInput.contractName; evmasm::LinkerObject obj = m_compiler.object(contractName); - Json::Value methodIdentifiers = m_compiler.contractIdentifiers(contractName)["methods"]; + Json::Value methodIdentifiers = m_compiler.interfaceSymbols(contractName)["methods"]; return CompilerOutput{obj.bytecode, methodIdentifiers}; } } diff --git a/test/tools/ossfuzz/SolidityEvmoneInterface.h b/test/tools/ossfuzz/SolidityEvmoneInterface.h index 51f6d5bdcc..da0fed58f2 100644 --- a/test/tools/ossfuzz/SolidityEvmoneInterface.h +++ b/test/tools/ossfuzz/SolidityEvmoneInterface.h @@ -91,7 +91,7 @@ class SolidityCompilationFramework /// @returns method identifiers in contract called @param _contractName. Json::Value methodIdentifiers(std::string const& _contractName) { - return m_compiler.contractIdentifiers(_contractName)["methods"]; + return m_compiler.interfaceSymbols(_contractName)["methods"]; } /// @returns Compilation output comprising EVM bytecode and list of /// method identifiers in contract if compilation is successful, From d511fe93ab480f22a55e8cb6329abb20ad5c8712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Mon, 24 Jan 2022 14:01:58 +0100 Subject: [PATCH 0027/1408] CI: Fix job name for PRBMath external test --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 39c219bcec..af58d830f1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -583,7 +583,7 @@ defaults: project: uniswap binary_type: native nodejs_version: '16' - - job_native_test_prb_math: &job_native_test_prb_math + - job_native_test_ext_prb_math: &job_native_test_ext_prb_math <<: *workflow_ubuntu2004_static name: t_native_test_ext_prb_math project: prb-math @@ -1454,7 +1454,7 @@ workflows: - t_ems_ext: *job_native_test_ext_pool_together - t_ems_ext: *job_native_test_ext_perpetual_pools - t_ems_ext: *job_native_test_ext_uniswap - - t_ems_ext: *job_native_test_prb_math + - t_ems_ext: *job_native_test_ext_prb_math - t_ems_ext: *job_native_test_ext_elementfi # Windows build and tests From 3e1aee17455b1d58fa485587df52db4ee7a2fc8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 18 Dec 2021 00:08:59 +0100 Subject: [PATCH 0028/1408] externalTests: Clean the build/ dir for Hardhat too --- test/externalTests/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/externalTests/common.sh b/test/externalTests/common.sh index 10df975256..cd0afb8a49 100644 --- a/test/externalTests/common.sh +++ b/test/externalTests/common.sh @@ -320,7 +320,7 @@ function truffle_clean function hardhat_clean { - rm -rf artifacts/ cache/ + rm -rf build/ artifacts/ cache/ } function run_test From 7fc225384132fff74cda971eaf7927fb07c921b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 22 Dec 2021 17:06:52 +0100 Subject: [PATCH 0029/1408] externalTests: Make comments about failing presets less terse --- test/externalTests/gnosis-v2.sh | 6 +++--- test/externalTests/gnosis.sh | 8 ++++---- test/externalTests/zeppelin.sh | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/externalTests/gnosis-v2.sh b/test/externalTests/gnosis-v2.sh index 8ad070d113..a48dddf9c1 100755 --- a/test/externalTests/gnosis-v2.sh +++ b/test/externalTests/gnosis-v2.sh @@ -40,12 +40,12 @@ function gnosis_safe_test local config_file="truffle-config.js" local compile_only_presets=( - legacy-no-optimize # "Error: while migrating GnosisSafe: Returned error: base fee exceeds gas limit" + legacy-no-optimize # Compiles but migrations run out of gas: "Error: while migrating GnosisSafe: Returned error: base fee exceeds gas limit" ) local settings_presets=( "${compile_only_presets[@]}" - #ir-no-optimize # "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack." - #ir-optimize-evm-only # "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack." + #ir-no-optimize # Compilation fails with "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack." + #ir-optimize-evm-only # Compilation fails with "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack." ir-optimize-evm+yul legacy-optimize-evm-only legacy-optimize-evm+yul diff --git a/test/externalTests/gnosis.sh b/test/externalTests/gnosis.sh index edada2fef3..1b430a9ae5 100755 --- a/test/externalTests/gnosis.sh +++ b/test/externalTests/gnosis.sh @@ -42,11 +42,11 @@ function gnosis_safe_test local compile_only_presets=() local settings_presets=( "${compile_only_presets[@]}" - #ir-no-optimize # "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack." - #ir-optimize-evm-only # "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack." + #ir-no-optimize # Compilation fails with "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack." + #ir-optimize-evm-only # Compilation fails with "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack." ir-optimize-evm+yul - #legacy-no-optimize # "Stack too deep" error - #legacy-optimize-evm-only # "Stack too deep" error + #legacy-no-optimize # Compilation fails with "Stack too deep" error + #legacy-optimize-evm-only # Compilation fails with "Stack too deep" error legacy-optimize-evm+yul ) diff --git a/test/externalTests/zeppelin.sh b/test/externalTests/zeppelin.sh index 5c54ceb8b2..d365dfe2a8 100755 --- a/test/externalTests/zeppelin.sh +++ b/test/externalTests/zeppelin.sh @@ -44,8 +44,8 @@ function zeppelin_test ) local settings_presets=( "${compile_only_presets[@]}" - #ir-no-optimize # "YulException: Variable var_account_852 is 4 slot(s) too deep inside the stack." - #ir-optimize-evm-only # "YulException: Variable var_account_852 is 4 slot(s) too deep inside the stack." + #ir-no-optimize # Compilation fails with "YulException: Variable var_account_852 is 4 slot(s) too deep inside the stack." + #ir-optimize-evm-only # Compilation fails with "YulException: Variable var_account_852 is 4 slot(s) too deep inside the stack." legacy-no-optimize legacy-optimize-evm-only legacy-optimize-evm+yul From a7852cba75e58b86955a0b8a5015b8282ca406b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 17 Dec 2021 22:42:45 +0100 Subject: [PATCH 0030/1408] Python script for parsing eth-gas-reporter output --- scripts/externalTests/parse_eth_gas_report.py | 269 +++++++++++++ .../fixtures/eth_gas_report_gnosis.rst | 381 ++++++++++++++++++ ...test_externalTests_parse_eth_gas_report.py | 219 ++++++++++ 3 files changed, 869 insertions(+) create mode 100755 scripts/externalTests/parse_eth_gas_report.py create mode 100644 test/scripts/fixtures/eth_gas_report_gnosis.rst create mode 100644 test/scripts/test_externalTests_parse_eth_gas_report.py diff --git a/scripts/externalTests/parse_eth_gas_report.py b/scripts/externalTests/parse_eth_gas_report.py new file mode 100755 index 0000000000..c5cc58e587 --- /dev/null +++ b/scripts/externalTests/parse_eth_gas_report.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +from dataclasses import asdict, dataclass, field +from typing import Dict, Optional, Tuple +import json +import re +import sys + +REPORT_HEADER_REGEX = re.compile(r''' + ^[|\s]+ Solc[ ]version:\s*(?P[\w\d.]+) + [|\s]+ Optimizer[ ]enabled:\s*(?P[\w]+) + [|\s]+ Runs:\s*(?P[\d]+) + [|\s]+ Block[ ]limit:\s*(?P[\d]+)\s*gas + [|\s]+$ +''', re.VERBOSE) +METHOD_HEADER_REGEX = re.compile(r'^[|\s]+Methods[|\s]+$') +METHOD_COLUMN_HEADERS_REGEX = re.compile(r''' + ^[|\s]+ Contract + [|\s]+ Method + [|\s]+ Min + [|\s]+ Max + [|\s]+ Avg + [|\s]+ \#[ ]calls + [|\s]+ \w+[ ]\(avg\) + [|\s]+$ +''', re.VERBOSE) +METHOD_ROW_REGEX = re.compile(r''' + ^[|\s]+ (?P[^|]+) + [|\s]+ (?P[^|]+) + [|\s]+ (?P[^|]+) + [|\s]+ (?P[^|]+) + [|\s]+ (?P[^|]+) + [|\s]+ (?P[^|]+) + [|\s]+ (?P[^|]+) + [|\s]+$ +''', re.VERBOSE) +FRAME_REGEX = re.compile(r'^[-|\s]+$') +DEPLOYMENT_HEADER_REGEX = re.compile(r'^[|\s]+Deployments[|\s]+% of limit[|\s]+$') +DEPLOYMENT_ROW_REGEX = re.compile(r''' + ^[|\s]+ (?P[^|]+) + [|\s]+ (?P[^|]+) + [|\s]+ (?P[^|]+) + [|\s]+ (?P[^|]+) + [|\s]+ (?P[^|]+)\s*% + [|\s]+ (?P[^|]+) + [|\s]+$ +''', re.VERBOSE) + + +class ReportError(Exception): + pass + +class ReportValidationError(ReportError): + pass + +class ReportParsingError(Exception): + def __init__(self, message: str, line: str, line_number: int): + # pylint: disable=useless-super-delegation # It's not useless, it adds type annotations. + super().__init__(message, line, line_number) + + def __str__(self): + return f"Parsing error on line {self.args[2] + 1}: {self.args[0]}\n{self.args[1]}" + + +@dataclass(frozen=True) +class MethodGasReport: + min_gas: int + max_gas: int + avg_gas: int + call_count: int + total_gas: int = field(init=False) + + def __post_init__(self): + object.__setattr__(self, 'total_gas', self.avg_gas * self.call_count) + + +@dataclass(frozen=True) +class ContractGasReport: + min_deployment_gas: Optional[int] + max_deployment_gas: Optional[int] + avg_deployment_gas: Optional[int] + methods: Optional[Dict[str, MethodGasReport]] + total_method_gas: int = field(init=False, default=0) + + def __post_init__(self): + if self.methods is not None: + object.__setattr__(self, 'total_method_gas', sum(method.total_gas for method in self.methods.values())) + + +@dataclass(frozen=True) +class GasReport: + solc_version: str + optimize: bool + runs: int + block_limit: int + contracts: Dict[str, ContractGasReport] + total_method_gas: int = field(init=False) + total_deployment_gas: int = field(init=False) + + def __post_init__(self): + object.__setattr__(self, 'total_method_gas', sum( + total_method_gas + for total_method_gas in (contract.total_method_gas for contract in self.contracts.values()) + if total_method_gas is not None + )) + object.__setattr__(self, 'total_deployment_gas', sum( + contract.avg_deployment_gas + for contract in self.contracts.values() + if contract.avg_deployment_gas is not None + )) + + def to_json(self): + return json.dumps(asdict(self), indent=4, sort_keys=True) + + +def parse_bool(input_string: str) -> bool: + if input_string == 'true': + return True + elif input_string == 'false': + return True + else: + raise ValueError(f"Invalid boolean value: '{input_string}'") + + +def parse_optional_int(input_string: str, default: Optional[int] = None) -> Optional[int]: + if input_string.strip() == '-': + return default + + return int(input_string) + + +def parse_report_header(line: str) -> Optional[dict]: + match = REPORT_HEADER_REGEX.match(line) + if match is None: + return None + + return { + 'solc_version': match.group('solc_version'), + 'optimize': parse_bool(match.group('optimize')), + 'runs': int(match.group('runs')), + 'block_limit': int(match.group('block_limit')), + } + + +def parse_method_row(line: str, line_number: int) -> Optional[Tuple[str, str, MethodGasReport]]: + match = METHOD_ROW_REGEX.match(line) + if match is None: + raise ReportParsingError("Expected a table row with method details.", line, line_number) + + avg_gas = parse_optional_int(match['avg']) + call_count = int(match['call_count']) + + if avg_gas is None and call_count == 0: + # No calls, no gas values. Uninteresting. Skip the row. + return None + + return ( + match['contract'].strip(), + match['method'].strip(), + MethodGasReport( + min_gas=parse_optional_int(match['min'], avg_gas), + max_gas=parse_optional_int(match['max'], avg_gas), + avg_gas=avg_gas, + call_count=call_count, + ) + ) + + +def parse_deployment_row(line: str, line_number: int) -> Tuple[str, int, int, int]: + match = DEPLOYMENT_ROW_REGEX.match(line) + if match is None: + raise ReportParsingError("Expected a table row with deployment details.", line, line_number) + + return ( + match['contract'].strip(), + parse_optional_int(match['min'].strip()), + parse_optional_int(match['max'].strip()), + int(match['avg'].strip()), + ) + + +def preprocess_unicode_frames(input_string: str) -> str: + # The report has a mix of normal pipe chars and its unicode variant. + # Let's just replace all frame chars with normal pipes for easier parsing. + return input_string.replace('\u2502', '|').replace('·', '|') + + +def parse_report(rst_report: str) -> GasReport: + report_params = None + methods_by_contract = {} + deployment_costs = {} + expected_row_type = None + + for line_number, line in enumerate(preprocess_unicode_frames(rst_report).splitlines()): + try: + if ( + line.strip() == "" or + FRAME_REGEX.match(line) is not None or + METHOD_COLUMN_HEADERS_REGEX.match(line) is not None + ): + continue + if METHOD_HEADER_REGEX.match(line) is not None: + expected_row_type = 'method' + continue + if DEPLOYMENT_HEADER_REGEX.match(line) is not None: + expected_row_type = 'deployment' + continue + + new_report_params = parse_report_header(line) + if new_report_params is not None: + if report_params is not None: + raise ReportParsingError("Duplicate report header.", line, line_number) + + report_params = new_report_params + continue + + if expected_row_type == 'method': + parsed_row = parse_method_row(line, line_number) + if parsed_row is None: + continue + + (contract, method, method_report) = parsed_row + + if contract not in methods_by_contract: + methods_by_contract[contract] = {} + + if method in methods_by_contract[contract]: + # Report must be generated with full signatures for method names to be unambiguous. + raise ReportParsingError(f"Duplicate method row for '{contract}.{method}'.", line, line_number) + + methods_by_contract[contract][method] = method_report + elif expected_row_type == 'deployment': + (contract, min_gas, max_gas, avg_gas) = parse_deployment_row(line, line_number) + + if contract in deployment_costs: + raise ReportParsingError(f"Duplicate contract deployment row for '{contract}'.", line, line_number) + + deployment_costs[contract] = (min_gas, max_gas, avg_gas) + else: + assert expected_row_type is None + raise ReportParsingError("Found data row without a section header.", line, line_number) + + except ValueError as error: + raise ReportParsingError(error.args[0], line, line_number) from error + + if report_params is None: + raise ReportValidationError("Report header not found.") + + report_params['contracts'] = { + contract: ContractGasReport( + min_deployment_gas=deployment_costs.get(contract, (None, None, None))[0], + max_deployment_gas=deployment_costs.get(contract, (None, None, None))[1], + avg_deployment_gas=deployment_costs.get(contract, (None, None, None))[2], + methods=methods_by_contract.get(contract), + ) + for contract in methods_by_contract.keys() | deployment_costs.keys() + } + + return GasReport(**report_params) + + +if __name__ == "__main__": + try: + report = parse_report(sys.stdin.read()) + print(report.to_json()) + except ReportError as exception: + print(f"{exception}", file=sys.stderr) + sys.exit(1) diff --git a/test/scripts/fixtures/eth_gas_report_gnosis.rst b/test/scripts/fixtures/eth_gas_report_gnosis.rst new file mode 100644 index 0000000000..c2e77f9bed --- /dev/null +++ b/test/scripts/fixtures/eth_gas_report_gnosis.rst @@ -0,0 +1,381 @@ +·----------------------------------------------------------------------------------------------------------------------------------------|---------------------------|-------------|------------------------------· +| Solc version: 0.8.10 · Optimizer enabled: true · Runs: 200 · Block limit: 100000000 gas │ +·········································································································································|···························|·············|······························· +| Methods │ +·································|·······································································································|·············|·············|·············|···············|··············· +| Contract · Method · Min · Max · Avg · # calls · eur (avg) │ +·································|·······································································································|·············|·············|·············|···············|··············· +| CompatibilityFallbackHandler · getMessageHash(bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| CompatibilityFallbackHandler · getMessageHashForSafe(address,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| CompatibilityFallbackHandler · getModules() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| CompatibilityFallbackHandler · isValidSignature(bytes,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| CompatibilityFallbackHandler · isValidSignature(bytes32,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| CompatibilityFallbackHandler · NAME() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| CompatibilityFallbackHandler · onERC1155BatchReceived(address,address,uint256[],uint256[],bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| CompatibilityFallbackHandler · onERC1155Received(address,address,uint256,uint256,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| CompatibilityFallbackHandler · onERC721Received(address,address,uint256,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| CompatibilityFallbackHandler · simulate(address,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| CompatibilityFallbackHandler · supportsInterface(bytes4) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| CompatibilityFallbackHandler · tokensReceived(address,address,address,uint256,bytes,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| CompatibilityFallbackHandler · VERSION() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| CreateCall · performCreate(uint256,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| CreateCall · performCreate2(uint256,bytes,bytes32) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| DebugTransactionGuard · checkAfterExecution(bytes32,bool) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| DebugTransactionGuard · checkTransaction(address,uint256,bytes,uint8,uint256,uint256,uint256,address,address,bytes,address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| DebugTransactionGuard · supportsInterface(bytes4) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| DebugTransactionGuard · txNonces(bytes32) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| DefaultCallbackHandler · NAME() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| DefaultCallbackHandler · onERC1155BatchReceived(address,address,uint256[],uint256[],bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| DefaultCallbackHandler · onERC1155Received(address,address,uint256,uint256,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| DefaultCallbackHandler · onERC721Received(address,address,uint256,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| DefaultCallbackHandler · supportsInterface(bytes4) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| DefaultCallbackHandler · tokensReceived(address,address,address,uint256,bytes,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| DefaultCallbackHandler · VERSION() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| DelegateCallTransactionGuard · allowedTarget() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| DelegateCallTransactionGuard · checkAfterExecution(bytes32,bool) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| DelegateCallTransactionGuard · checkTransaction(address,uint256,bytes,uint8,uint256,uint256,uint256,address,address,bytes,address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| DelegateCallTransactionGuard · supportsInterface(bytes4) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC1155Token · balanceOf(address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC1155Token · mint(address,uint256,uint256,bytes) · 47934 · 59804 · 57826 · 6 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC1155Token · safeTransferFrom(address,address,uint256,uint256,bytes) · - · - · 53900 · 2 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20 · allowance(address,address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20 · approve(address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20 · balanceOf(address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20 · decimals() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20 · decreaseAllowance(address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20 · increaseAllowance(address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20 · name() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20 · symbol() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20 · totalSupply() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20 · transfer(address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20 · transferFrom(address,address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20Token · allowance(address,address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20Token · approve(address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20Token · balanceOf(address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20Token · decimals() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20Token · decreaseAllowance(address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20Token · increaseAllowance(address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20Token · name() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20Token · symbol() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20Token · totalSupply() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20Token · transfer(address,uint256) · - · - · 51567 · 8 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ERC20Token · transferFrom(address,address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| FallbackManager · setFallbackHandler(address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · addOwnerWithThreshold(address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · approvedHashes(address,bytes32) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · approveHash(bytes32) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · changeThreshold(uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · checkNSignatures(bytes32,bytes,bytes,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · checkSignatures(bytes32,bytes,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · disableModule(address,address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · domainSeparator() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · enableModule(address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · encodeTransactionData(address,uint256,bytes,uint8,uint256,uint256,uint256,address,address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · execTransaction(address,uint256,bytes,uint8,uint256,uint256,uint256,address,address,bytes) · 59563 · 151736 · 94816 · 85 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · execTransactionFromModule(address,uint256,bytes,uint8) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · execTransactionFromModuleReturnData(address,uint256,bytes,uint8) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · getChainId() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · getModulesPaginated(address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · getOwners() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · getStorageAt(uint256,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · getThreshold() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · getTransactionHash(address,uint256,bytes,uint8,uint256,uint256,uint256,address,address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · isModuleEnabled(address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · isOwner(address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · nonce() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · removeOwner(address,address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · requiredTxGas(address,uint256,bytes,uint8) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · setFallbackHandler(address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · setGuard(address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · setup(address[],uint256,address,bytes,address,address,uint256,address) · 167642 · 263690 · 201944 · 49 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · signedMessages(bytes32) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · simulateAndRevert(address,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · swapOwner(address,address,address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafe · VERSION() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · addOwnerWithThreshold(address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · approvedHashes(address,bytes32) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · approveHash(bytes32) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · changeThreshold(uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · checkNSignatures(bytes32,bytes,bytes,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · checkSignatures(bytes32,bytes,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · disableModule(address,address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · domainSeparator() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · enableModule(address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · encodeTransactionData(address,uint256,bytes,uint8,uint256,uint256,uint256,address,address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · execTransaction(address,uint256,bytes,uint8,uint256,uint256,uint256,address,address,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · execTransactionFromModule(address,uint256,bytes,uint8) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · execTransactionFromModuleReturnData(address,uint256,bytes,uint8) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · getChainId() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · getModulesPaginated(address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · getOwners() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · getStorageAt(uint256,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · getThreshold() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · getTransactionHash(address,uint256,bytes,uint8,uint256,uint256,uint256,address,address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · isModuleEnabled(address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · isOwner(address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · nonce() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · removeOwner(address,address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · requiredTxGas(address,uint256,bytes,uint8) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · setFallbackHandler(address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · setGuard(address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · setup(address[],uint256,address,bytes,address,address,uint256,address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · signedMessages(bytes32) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · simulateAndRevert(address,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · swapOwner(address,address,address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeL2 · VERSION() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeProxyFactory · calculateCreateProxyWithNonceAddress(address,bytes,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeProxyFactory · createProxy(address,bytes) · 105568 · 105580 · 105568 · 52 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeProxyFactory · createProxyWithCallback(address,bytes,uint256,address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeProxyFactory · createProxyWithNonce(address,bytes,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeProxyFactory · proxyCreationCode() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GnosisSafeProxyFactory · proxyRuntimeCode() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| GuardManager · setGuard(address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| Migration · migrate() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| Migration · migrationSingleton() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| Migration · safe120Singleton() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · DEFAULT_FALLBACK_VALUE() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenAnyReturn(bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenAnyReturnAddress(address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenAnyReturnBool(bool) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenAnyReturnUint(uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenAnyRevert() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenAnyRevertWithMessage(string) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenAnyRunOutOfGas() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenCalldataReturn(bytes,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenCalldataReturnAddress(bytes,address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenCalldataReturnBool(bytes,bool) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenCalldataReturnUint(bytes,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenCalldataRevert(bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenCalldataRevertWithMessage(bytes,string) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenCalldataRunOutOfGas(bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenMethodReturn(bytes,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenMethodReturnAddress(bytes,address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenMethodReturnBool(bytes,bool) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenMethodReturnUint(bytes,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenMethodRevert(bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenMethodRevertWithMessage(bytes,string) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · givenMethodRunOutOfGas(bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · invocationCount() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · invocationCountForCalldata(bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · invocationCountForMethod(bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · MOCKS_LIST_END_HASH() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · MOCKS_LIST_END() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · MOCKS_LIST_START() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · reset() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · SENTINEL_ANY_MOCKS() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MockContract · updateInvocationCount(bytes4,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ModuleManager · disableModule(address,address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ModuleManager · enableModule(address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ModuleManager · execTransactionFromModule(address,uint256,bytes,uint8) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ModuleManager · execTransactionFromModuleReturnData(address,uint256,bytes,uint8) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ModuleManager · getModulesPaginated(address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ModuleManager · isModuleEnabled(address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MultiSend · multiSend(bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| MultiSendCallOnly · multiSend(bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| OwnerManager · addOwnerWithThreshold(address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| OwnerManager · changeThreshold(uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| OwnerManager · getOwners() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| OwnerManager · getThreshold() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| OwnerManager · isOwner(address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| OwnerManager · removeOwner(address,address,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| OwnerManager · swapOwner(address,address,address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ReentrancyTransactionGuard · checkAfterExecution(bytes32,bool) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ReentrancyTransactionGuard · checkTransaction(address,uint256,bytes,uint8,uint256,uint256,uint256,address,address,bytes,address) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| ReentrancyTransactionGuard · supportsInterface(bytes4) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| SignMessageLib · getMessageHash(bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| SignMessageLib · signMessage(bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| SimulateTxAccessor · simulate(address,uint256,bytes,uint8) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| StorageAccessible · getStorageAt(uint256,uint256) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| StorageAccessible · simulateAndRevert(address,bytes) · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| TestHandler · dudududu() · - · - · - · 0 · - │ +·································|·······································································································|·············|·············|·············|···············|··············· +| Deployments · · % of limit · │ +·········································································································································|·············|·············|·············|···············|··············· +| DelegateCallTransactionGuard · 283510 · 283522 · 283516 · 0.3 % · - │ +·········································································································································|·············|·············|·············|···············|··············· +| ERC1155Token · - · - · 525869 · 0.5 % · - │ +·········································································································································|·············|·············|·············|···············|··············· +| ERC20Token · - · - · 733462 · 0.7 % · - │ +·----------------------------------------------------------------------------------------------------------------------------------------|-------------|-------------|-------------|---------------|--------------· diff --git a/test/scripts/test_externalTests_parse_eth_gas_report.py b/test/scripts/test_externalTests_parse_eth_gas_report.py new file mode 100644 index 0000000000..cefa0487a2 --- /dev/null +++ b/test/scripts/test_externalTests_parse_eth_gas_report.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 + +from dataclasses import asdict +import unittest + +from textwrap import dedent + +from unittest_helpers import FIXTURE_DIR, load_fixture + +# NOTE: This test file file only works with scripts/ added to PYTHONPATH so pylint can't find the imports +# pragma pylint: disable=import-error +from externalTests.parse_eth_gas_report import parse_report, ReportParsingError, ReportValidationError +# pragma pylint: enable=import-error + +ETH_GAS_REPORT_GNOSIS_RST_PATH = FIXTURE_DIR / 'eth_gas_report_gnosis.rst' +ETH_GAS_REPORT_GNOSIS_RST_CONTENT = load_fixture(ETH_GAS_REPORT_GNOSIS_RST_PATH) + + +class TestEthGasReport(unittest.TestCase): + def setUp(self): + self.maxDiff = 10000 + + def test_parse_report(self): + parsed_report = parse_report(ETH_GAS_REPORT_GNOSIS_RST_CONTENT) + + expected_report = { + 'solc_version': '0.8.10', + 'optimize': True, + 'runs': 200, + 'block_limit': 100000000, + 'total_method_gas': 57826 * 6 + 53900 * 2 + 51567 * 8 + 94816 * 85 + 201944 * 49 + 105568 * 52, + 'total_deployment_gas': 283516 + 525869 + 733462, + 'contracts': { + 'DelegateCallTransactionGuard': { + 'total_method_gas': 0, + 'min_deployment_gas': 283510, + 'max_deployment_gas': 283522, + 'avg_deployment_gas': 283516, + 'methods': None, + }, + 'ERC1155Token': { + 'total_method_gas': 57826 * 6 + 53900 * 2, + 'min_deployment_gas': None, + 'max_deployment_gas': None, + 'avg_deployment_gas': 525869, + 'methods': { + 'mint(address,uint256,uint256,bytes)': { + 'total_gas': 57826 * 6, + 'min_gas': 47934, + 'max_gas': 59804, + 'avg_gas': 57826, + 'call_count': 6 + }, + 'safeTransferFrom(address,address,uint256,uint256,bytes)': { + 'total_gas': 53900 * 2, + 'min_gas': 53900, + 'max_gas': 53900, + 'avg_gas': 53900, + 'call_count': 2, + }, + }, + }, + 'ERC20Token': { + 'total_method_gas': 51567 * 8, + 'min_deployment_gas': None, + 'max_deployment_gas': None, + 'avg_deployment_gas': 733462, + 'methods': { + 'transfer(address,uint256)': { + 'total_gas': 51567 * 8, + 'min_gas': 51567, + 'max_gas': 51567, + 'avg_gas': 51567, + 'call_count': 8, + }, + }, + }, + 'GnosisSafe': { + 'total_method_gas': 94816 * 85 + 201944 * 49, + 'min_deployment_gas': None, + 'max_deployment_gas': None, + 'avg_deployment_gas': None, + 'methods': { + 'execTransaction(address,uint256,bytes,uint8,uint256,uint256,uint256,address,address,bytes)': { + 'total_gas': 94816 * 85, + 'min_gas': 59563, + 'max_gas': 151736, + 'avg_gas': 94816, + 'call_count': 85, + }, + 'setup(address[],uint256,address,bytes,address,address,uint256,address)': { + 'total_gas': 201944 * 49, + 'min_gas': 167642, + 'max_gas': 263690, + 'avg_gas': 201944, + 'call_count': 49, + }, + }, + }, + 'GnosisSafeProxyFactory': { + 'total_method_gas': 105568 * 52, + 'min_deployment_gas': None, + 'max_deployment_gas': None, + 'avg_deployment_gas': None, + 'methods': { + 'createProxy(address,bytes)': { + 'total_gas': 105568 * 52, + 'min_gas': 105568, + 'max_gas': 105580, + 'avg_gas': 105568, + 'call_count': 52, + }, + }, + }, + } + } + self.assertEqual(asdict(parsed_report), expected_report) + + def test_parse_report_should_fail_if_report_is_empty(self): + text_report = "" + with self.assertRaises(ReportValidationError) as manager: + parse_report(text_report) + self.assertEqual(str(manager.exception), "Report header not found.") + + def test_parse_report_should_fail_if_report_has_no_header(self): + text_report = dedent(""" + | Methods | + | ERC1155Token · mint() · 1 · 3 · 2 · 6 · - | + | Deployments · · % of limit · │ + | ERC1155Token · - · - · 5 · 1 % · - | + """).strip('\n') + with self.assertRaises(ReportValidationError) as manager: + parse_report(text_report) + self.assertEqual(str(manager.exception), "Report header not found.") + + def test_parse_report_should_fail_if_data_rows_have_no_headers(self): + text_report = dedent(""" + | ERC1155Token · mint() · 1 · 3 · 2 · 6 · - | + """).strip('\n') + expected_message = dedent(""" + Parsing error on line 1: Found data row without a section header. + | ERC1155Token | mint() | 1 | 3 | 2 | 6 | - | + """).strip('\n') + + with self.assertRaises(ReportParsingError) as manager: + parse_report(text_report) + self.assertEqual(str(manager.exception), expected_message) + + def test_parse_report_should_fail_if_report_has_more_than_one_header(self): + text_report = dedent(""" + | Solc version: 0.8.10 · Optimizer enabled: true · Runs: 200 · Block limit: 100000000 gas | + | Solc version: 0.8.9 · Optimizer enabled: false · Runs: 111 · Block limit: 999999999 gas | + """).strip('\n') + expected_message = dedent(""" + Parsing error on line 2: Duplicate report header. + | Solc version: 0.8.9 | Optimizer enabled: false | Runs: 111 | Block limit: 999999999 gas | + """).strip('\n') + + with self.assertRaises(ReportParsingError) as manager: + parse_report(text_report) + self.assertEqual(str(manager.exception), expected_message) + + def test_parse_report_should_fail_if_row_matching_same_method_call_appears_twice(self): + text_report = dedent(""" + | Methods | + | ERC1155Token · mint() · 47934 · 59804 · 57826 · 6 · - | + | ERC1155Token · mint() · 11111 · 22222 · 33333 · 4 · - | + """).strip('\n') + expected_message = dedent(""" + Parsing error on line 3: Duplicate method row for 'ERC1155Token.mint()'. + | ERC1155Token | mint() | 11111 | 22222 | 33333 | 4 | - | + """).strip('\n') + + with self.assertRaises(ReportParsingError) as manager: + parse_report(text_report) + self.assertEqual(str(manager.exception), expected_message) + + def test_parse_report_should_fail_if_row_matching_same_contract_deployment_appears_twice(self): + text_report = dedent(""" + | Deployments · · % of limit · │ + | ERC1155Token · - · - · 525869 · 0.5 % · - | + | ERC1155Token · - · - · 111111 · 0.6 % · - | + """).strip('\n') + expected_message = dedent(""" + Parsing error on line 3: Duplicate contract deployment row for 'ERC1155Token'. + | ERC1155Token | - | - | 111111 | 0.6 % | - | + """).strip('\n') + + with self.assertRaises(ReportParsingError) as manager: + parse_report(text_report) + self.assertEqual(str(manager.exception), expected_message) + + def test_parse_report_should_fail_if_method_row_appears_under_deployments_header(self): + text_report = dedent(""" + | Deployments · · % of limit · │ + | ERC1155Token · mint() · 47934 · 59804 · 57826 · 6 · - | + """).strip('\n') + expected_message = dedent(""" + Parsing error on line 2: Expected a table row with deployment details. + | ERC1155Token | mint() | 47934 | 59804 | 57826 | 6 | - | + """).strip('\n') + + with self.assertRaises(ReportParsingError) as manager: + parse_report(text_report) + self.assertEqual(str(manager.exception), expected_message) + + def test_parse_report_should_fail_if_deployment_row_appears_under_methods_header(self): + text_report = dedent(""" + | Methods | + | ERC1155Token · - · - · 525869 · 5 · - | + """).strip('\n') + expected_message = dedent(""" + Parsing error on line 2: Expected a table row with method details. + | ERC1155Token | - | - | 525869 | 5 | - | + """).strip('\n') + + with self.assertRaises(ReportParsingError) as manager: + parse_report(text_report) + self.assertEqual(str(manager.exception), expected_message) From c6094bb0c28a6735b0868ad9373c0240404db881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 18 Dec 2021 00:36:30 +0100 Subject: [PATCH 0031/1408] externalTests: Benchmark reports --- .circleci/config.yml | 9 ++ test/externalTests.sh | 32 ++--- test/externalTests/bleeps.sh | 6 +- test/externalTests/colony.sh | 3 + test/externalTests/common.sh | 166 +++++++++++++++++++++++-- test/externalTests/elementfi.sh | 3 + test/externalTests/ens.sh | 3 + test/externalTests/euler.sh | 3 + test/externalTests/gnosis-v2.sh | 4 + test/externalTests/gnosis.sh | 4 + test/externalTests/perpetual-pools.sh | 3 + test/externalTests/pool-together.sh | 3 + test/externalTests/prb-math.sh | 7 +- test/externalTests/trident.sh | 3 + test/externalTests/uniswap.sh | 4 + test/externalTests/yield-liquidator.sh | 3 + test/externalTests/zeppelin.sh | 3 + 17 files changed, 232 insertions(+), 27 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index af58d830f1..44837d8319 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1205,6 +1205,15 @@ jobs: name: External <> tests (native) command: | test/externalTests/<>.sh native /tmp/workspace/solc/solc + - store_artifacts: + path: reports/externalTests/ + # persist_to_workspace fails if the directory does not exist and the test script will create + # it only if it actually has benchmark results. + - run: mkdir -p reports/externalTests/ + - persist_to_workspace: + root: . + paths: + - reports/externalTests/ - gitter_notify_failure_unless_pr b_win: &b_win diff --git a/test/externalTests.sh b/test/externalTests.sh index a5e52c8524..b3b46460bf 100755 --- a/test/externalTests.sh +++ b/test/externalTests.sh @@ -28,26 +28,26 @@ set -e -REPO_ROOT="$(dirname "$0")" - source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/..") + verify_input "$@" printTask "Running external tests..." -"$REPO_ROOT/externalTests/zeppelin.sh" "$@" -"$REPO_ROOT/externalTests/gnosis.sh" "$@" -"$REPO_ROOT/externalTests/gnosis-v2.sh" "$@" -"$REPO_ROOT/externalTests/colony.sh" "$@" -"$REPO_ROOT/externalTests/ens.sh" "$@" -"$REPO_ROOT/externalTests/trident.sh" "$@" -"$REPO_ROOT/externalTests/euler.sh" "$@" -"$REPO_ROOT/externalTests/yield-liquidator.sh" "$@" -"$REPO_ROOT/externalTests/bleeps.sh" "$@" -"$REPO_ROOT/externalTests/pool-together.sh" "$@" -"$REPO_ROOT/externalTests/perpetual-pools.sh" "$@" -"$REPO_ROOT/externalTests/uniswap.sh" "$@" -"$REPO_ROOT/externalTests/prb-math.sh" "$@" -"$REPO_ROOT/externalTests/elementfi.sh" "$@" +"{$REPO_ROOT}/test/externalTests/zeppelin.sh" "$@" +"{$REPO_ROOT}/test/externalTests/gnosis.sh" "$@" +"{$REPO_ROOT}/test/externalTests/gnosis-v2.sh" "$@" +"{$REPO_ROOT}/test/externalTests/colony.sh" "$@" +"{$REPO_ROOT}/test/externalTests/ens.sh" "$@" +"{$REPO_ROOT}/test/externalTests/trident.sh" "$@" +"{$REPO_ROOT}/test/externalTests/euler.sh" "$@" +"{$REPO_ROOT}/test/externalTests/yield-liquidator.sh" "$@" +"{$REPO_ROOT}/test/externalTests/bleeps.sh" "$@" +"{$REPO_ROOT}/test/externalTests/pool-together.sh" "$@" +"{$REPO_ROOT}/test/externalTests/perpetual-pools.sh" "$@" +"{$REPO_ROOT}/test/externalTests/uniswap.sh" "$@" +"{$REPO_ROOT}/test/externalTests/prb-math.sh" "$@" +"{$REPO_ROOT}/test/externalTests/elementfi.sh" "$@" diff --git a/test/externalTests/bleeps.sh b/test/externalTests/bleeps.sh index 561200e219..bd3b0255b1 100755 --- a/test/externalTests/bleeps.sh +++ b/test/externalTests/bleeps.sh @@ -24,13 +24,16 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" SELECTED_PRESETS="$3" function compile_fn { npm run compile; } -function test_fn { npm run test; } +# NOTE: `npm run test` runs `mocha` which seems to disable the gas reporter. +function test_fn { HARDHAT_DEPLOY_FIXTURE=true npx --no hardhat --no-compile test; } function bleeps_test { @@ -87,6 +90,7 @@ function bleeps_test for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var" + store_benchmark_report hardhat bleeps "$repo" "$preset" done popd diff --git a/test/externalTests/colony.sh b/test/externalTests/colony.sh index d659899535..e23ce11350 100755 --- a/test/externalTests/colony.sh +++ b/test/externalTests/colony.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -73,6 +75,7 @@ function colony_test for preset in $SELECTED_PRESETS; do truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc/dist" "$preset" "${compile_only_presets[*]}" compile_fn test_fn + store_benchmark_report truffle colony "$repo" "$preset" done } diff --git a/test/externalTests/common.sh b/test/externalTests/common.sh index cd0afb8a49..8f7b09eb52 100644 --- a/test/externalTests/common.sh +++ b/test/externalTests/common.sh @@ -20,7 +20,7 @@ #------------------------------------------------------------------------------ set -e -# Requires "${REPO_ROOT}/scripts/common.sh" to be included before. +# Requires $REPO_ROOT to be defined and "${REPO_ROOT}/scripts/common.sh" to be included before. CURRENT_EVM_VERSION=london @@ -207,9 +207,19 @@ function force_truffle_compiler_settings echo "Compiler version (full): ${SOLCVERSION}" echo "-------------------------------------" - # Forcing the settings should always work by just overwriting the solc object. Forcing them by using a - # dedicated settings objects should only be the fallback. - echo "module.exports['compilers'] = $(truffle_compiler_settings "$solc_path" "$preset" "$evm_version");" >> "$config_file" + local compiler_settings gas_reporter_settings + compiler_settings=$(truffle_compiler_settings "$solc_path" "$preset" "$evm_version") + gas_reporter_settings=$(eth_gas_reporter_settings "$preset") + + { + echo "require('eth-gas-reporter');" + echo "module.exports['mocha'] = {" + echo " reporter: 'eth-gas-reporter'," + echo " reporterOptions: ${gas_reporter_settings}" + echo "};" + + echo "module.exports['compilers'] = ${compiler_settings};" + } >> "$config_file" } function name_hardhat_default_export @@ -278,16 +288,21 @@ function force_hardhat_compiler_settings echo "Compiler version (full): ${SOLCVERSION}" echo "-------------------------------------" - local settings - settings=$(hardhat_compiler_settings "$SOLCVERSION_SHORT" "$preset" "$evm_version") + local compiler_settings gas_reporter_settings + compiler_settings=$(hardhat_compiler_settings "$SOLCVERSION_SHORT" "$preset" "$evm_version") + gas_reporter_settings=$(eth_gas_reporter_settings "$preset") if [[ $config_file == *\.js ]]; then [[ $config_var_name == "" ]] || assertFail - echo "module.exports['solidity'] = ${settings}" >> "$config_file" + echo "require('hardhat-gas-reporter');" + echo "module.exports.gasReporter = ${gas_reporter_settings};" + echo "module.exports.solidity = ${compiler_settings};" else [[ $config_file == *\.ts ]] || assertFail [[ $config_var_name != "" ]] || assertFail - echo "${config_var_name}.solidity = {compilers: [${settings}]}" >> "$config_file" - fi + echo 'import "hardhat-gas-reporter";' + echo "${config_var_name}.gasReporter = ${gas_reporter_settings};" + echo "${config_var_name}.solidity = {compilers: [${compiler_settings}]};" + fi >> "$config_file" } function truffle_verify_compiler_version @@ -368,6 +383,21 @@ function replace_global_solc export PATH="$PWD:$PATH" } +function eth_gas_reporter_settings +{ + local preset="$1" + + echo "{" + echo " enabled: true," + echo " gasPrice: 1," # Gas price does not matter to us at all. Set to whatever to avoid API call. + echo " noColors: true," + echo " showTimeSpent: false," # We're not interested in test timing + echo " onlyCalledMethods: true," # Exclude entries with no gas for shorter report + echo " showMethodSig: true," # Should make diffs more stable if there are overloaded functions + echo " outputFile: \"$(gas_report_path "$preset")\"" + echo "}" +} + function truffle_compiler_settings { local solc_path="$1" @@ -495,3 +525,121 @@ function external_test rm -rf "$DIR" echo "Done." } + +function gas_report_path +{ + local preset="$1" + + echo "${DIR}/gas-report-${preset}.rst" +} + +function gas_report_to_json +{ + cat - | "${REPO_ROOT}/scripts/externalTests/parse_eth_gas_report.py" | jq '{gas: .}' +} + +function detect_hardhat_artifact_dir +{ + if [[ -e build/ && -e artifacts/ ]]; then + fail "Cannot determine Hardhat artifact location. Both build/ and artifacts/ exist" + elif [[ -e build/ ]]; then + echo -n build/artifacts + elif [[ -e artifacts/ ]]; then + echo -n artifacts + else + fail "Hardhat build artifacts not found." + fi +} + +function bytecode_size_json_from_truffle_artifacts +{ + # NOTE: The output of this function is a series of concatenated JSON dicts rather than a list. + + for artifact in build/contracts/*.json; do + if [[ $(jq '. | has("unlinked_binary")' "$artifact") == false ]]; then + # Each artifact represents compilation output for a single contract. Some top-level keys contain + # bits of Standard JSON output while others are generated by Truffle. Process it into a dict + # of the form `{"": {"": }}`. + # NOTE: The `bytecode` field starts with 0x, which is why we subtract 1 from size. + jq '{ + (.ast.absolutePath): { + (.contractName): (.bytecode | length / 2 - 1) + } + }' "$artifact" + fi + done +} + +function bytecode_size_json_from_hardhat_artifacts +{ + # NOTE: The output of this function is a series of concatenated JSON dicts rather than a list. + + for artifact in "$(detect_hardhat_artifact_dir)"/build-info/*.json; do + # Each artifact contains Standard JSON output under the `output` key. + # Process it into a dict of the form `{"": {"": }}`, + # Note that one Hardhat artifact often represents multiple input files. + jq '.output.contracts | to_entries[] | { + "\(.key)": .value | to_entries[] | { + "\(.key)": (.value.evm.bytecode.object | length / 2) + } + }' "$artifact" + done +} + +function combine_artifact_json +{ + # Combine all dicts into a list with `jq --slurp` and then use `reduce` to merge them into one + # big dict with keys of the form `":"`. Then run jq again to filter out items + # with zero size and put the rest under under a top-level `bytecode_size` key. Also add another + # key with total bytecode size. + # NOTE: The extra inner `bytecode_size` key is there only to make diffs more readable. + cat - | + jq --slurp 'reduce (.[] | to_entries[]) as {$key, $value} ({}; . + { + ($key + ":" + ($value | to_entries[].key)): { + bytecode_size: $value | to_entries[].value + } + })' | + jq --indent 4 --sort-keys '{ + bytecode_size: [. | to_entries[] | select(.value.bytecode_size > 0)] | from_entries, + total_bytecode_size: (reduce (. | to_entries[]) as {$key, $value} (0; . + $value.bytecode_size)) + }' +} + +function project_info_json +{ + local project_url="$1" + + echo "{" + echo " \"project\": {" + # NOTE: Given that we clone with `--depth 1`, we'll only get useful output out of `git describe` + # if we directly check out a tag. Still better than nothing. + echo " \"version\": \"$(git describe --always)\"," + echo " \"commit\": \"$(git rev-parse HEAD)\"," + echo " \"url\": \"${project_url}\"" + echo " }" + echo "}" +} + +function store_benchmark_report +{ + local framework="$1" + local project_name="$2" + local project_url="$3" + local preset="$4" + + [[ $framework == truffle || $framework == hardhat ]] || assertFail + [[ " ${AVAILABLE_PRESETS[*]} " == *" $preset "* ]] || assertFail + + local report_dir="${REPO_ROOT}/reports/externalTests" + local output_file="${report_dir}/benchmark-${project_name}-${preset}.json" + mkdir -p "$report_dir" + + { + if [[ -e $(gas_report_path "$preset") ]]; then + gas_report_to_json < "$(gas_report_path "$preset")" + fi + + "bytecode_size_json_from_${framework}_artifacts" | combine_artifact_json + project_info_json "$project_url" + } | jq --slurp "{\"${project_name}\": {\"${preset}\": add}}" --indent 4 --sort-keys > "$output_file" +} diff --git a/test/externalTests/elementfi.sh b/test/externalTests/elementfi.sh index f52a13ca67..3641eac2b4 100755 --- a/test/externalTests/elementfi.sh +++ b/test/externalTests/elementfi.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -96,6 +98,7 @@ function elementfi_test for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var" + store_benchmark_report hardhat elementfi "$repo" "$preset" done } diff --git a/test/externalTests/ens.sh b/test/externalTests/ens.sh index d3fcb0b13d..734c5d8ae8 100755 --- a/test/externalTests/ens.sh +++ b/test/externalTests/ens.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -68,6 +70,7 @@ function ens_test for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn + store_benchmark_report hardhat ens "$repo" "$preset" done } diff --git a/test/externalTests/euler.sh b/test/externalTests/euler.sh index 485d7ae56a..b3b505051c 100755 --- a/test/externalTests/euler.sh +++ b/test/externalTests/euler.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -68,6 +70,7 @@ function euler_test for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn + store_benchmark_report hardhat euler "$repo" "$preset" done } diff --git a/test/externalTests/gnosis-v2.sh b/test/externalTests/gnosis-v2.sh index a48dddf9c1..6b0915b6a0 100755 --- a/test/externalTests/gnosis-v2.sh +++ b/test/externalTests/gnosis-v2.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -65,12 +67,14 @@ function gnosis_safe_test neutralize_package_json_hooks force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc/dist" "$(first_word "$SELECTED_PRESETS")" npm install --package-lock + npm install eth-gas-reporter replace_version_pragmas [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc/dist" for preset in $SELECTED_PRESETS; do truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc/dist" "$preset" "${compile_only_presets[*]}" compile_fn test_fn + store_benchmark_report truffle gnosis2 "$repo" "$preset" done } diff --git a/test/externalTests/gnosis.sh b/test/externalTests/gnosis.sh index 1b430a9ae5..82d1892f2d 100755 --- a/test/externalTests/gnosis.sh +++ b/test/externalTests/gnosis.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -63,12 +65,14 @@ function gnosis_safe_test neutralize_package_json_hooks force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc/dist" "$(first_word "$SELECTED_PRESETS")" npm install --package-lock + npm install eth-gas-reporter replace_version_pragmas [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc/dist" for preset in $SELECTED_PRESETS; do truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc/dist" "$preset" "${compile_only_presets[*]}" compile_fn test_fn + store_benchmark_report truffle gnosis "$repo" "$preset" done } diff --git a/test/externalTests/perpetual-pools.sh b/test/externalTests/perpetual-pools.sh index e1ee9af8f5..8f90ed0b4d 100755 --- a/test/externalTests/perpetual-pools.sh +++ b/test/externalTests/perpetual-pools.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -68,6 +70,7 @@ function perpetual_pools_test for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var" + store_benchmark_report hardhat perpetual-pools "$repo" "$preset" done } diff --git a/test/externalTests/pool-together.sh b/test/externalTests/pool-together.sh index a7f5933c0e..b24cd6f4b2 100755 --- a/test/externalTests/pool-together.sh +++ b/test/externalTests/pool-together.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -72,6 +74,7 @@ function pool_together_test for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var" + store_benchmark_report hardhat pool-together "$repo" "$preset" done } diff --git a/test/externalTests/prb-math.sh b/test/externalTests/prb-math.sh index 53fad2c255..01bdb76f18 100755 --- a/test/externalTests/prb-math.sh +++ b/test/externalTests/prb-math.sh @@ -24,13 +24,16 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" SELECTED_PRESETS="$3" function compile_fn { yarn compile; } -function test_fn { yarn test; } +# NOTE: `yarn test` runs `mocha` which seems to disable the gas reporter. +function test_fn { npx --no hardhat --no-compile test; } function prb_math_test { @@ -70,11 +73,13 @@ function prb_math_test force_hardhat_compiler_binary "$config_file" "$BINARY_TYPE" "$BINARY_PATH" force_hardhat_compiler_settings "$config_file" "$(first_word "$SELECTED_PRESETS")" "$config_var" yarn install --no-lock-file + yarn add hardhat-gas-reporter replace_version_pragmas for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var" + store_benchmark_report hardhat prb-math "$repo" "$preset" done } diff --git a/test/externalTests/trident.sh b/test/externalTests/trident.sh index 8e6f35ae33..5e230560d5 100755 --- a/test/externalTests/trident.sh +++ b/test/externalTests/trident.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -92,6 +94,7 @@ function trident_test for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var" + store_benchmark_report hardhat trident "$repo" "$preset" done } diff --git a/test/externalTests/uniswap.sh b/test/externalTests/uniswap.sh index 96a5d2b63b..2f94c6abde 100755 --- a/test/externalTests/uniswap.sh +++ b/test/externalTests/uniswap.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -73,11 +75,13 @@ function uniswap_test yarn add @ethereumjs/tx@3.1.3 yarn install + yarn add hardhat-gas-reporter replace_version_pragmas for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var" + store_benchmark_report hardhat uniswap "$repo" "$preset" done } diff --git a/test/externalTests/yield-liquidator.sh b/test/externalTests/yield-liquidator.sh index 6712941e74..1b5b9f1462 100755 --- a/test/externalTests/yield-liquidator.sh +++ b/test/externalTests/yield-liquidator.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -68,6 +70,7 @@ function yield_liquidator_test for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn "$config_var" + store_benchmark_report hardhat yield_liquidator "$repo" "$preset" done } diff --git a/test/externalTests/zeppelin.sh b/test/externalTests/zeppelin.sh index d365dfe2a8..4094c69364 100755 --- a/test/externalTests/zeppelin.sh +++ b/test/externalTests/zeppelin.sh @@ -24,6 +24,8 @@ set -e source scripts/common.sh source test/externalTests/common.sh +REPO_ROOT=$(realpath "$(dirname "$0")/../..") + verify_input "$@" BINARY_TYPE="$1" BINARY_PATH="$2" @@ -66,6 +68,7 @@ function zeppelin_test for preset in $SELECTED_PRESETS; do hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn + store_benchmark_report hardhat zeppelin "$repo" "$preset" done } From 60d9aa0d4f34da1f0ccd022e69ca4c45f378ce20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Wed, 22 Dec 2021 17:45:20 +0100 Subject: [PATCH 0032/1408] Benchmark report collector job + summary --- .circleci/config.yml | 35 +++++++++++ scripts/externalTests/merge_benchmarks.sh | 60 +++++++++++++++++++ scripts/externalTests/summarize_benchmarks.sh | 53 ++++++++++++++++ 3 files changed, 148 insertions(+) create mode 100755 scripts/externalTests/merge_benchmarks.sh create mode 100755 scripts/externalTests/summarize_benchmarks.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 44837d8319..a1fca20ceb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1216,6 +1216,23 @@ jobs: - reports/externalTests/ - gitter_notify_failure_unless_pr + c_ext_benchmarks: + <<: *base_node_small + steps: + - checkout + - attach_workspace: + at: . + - run: + name: Combine benchmark reports + command: cat reports/externalTests/benchmark-*.json | scripts/externalTests/merge_benchmarks.sh > reports/externalTests/all-benchmarks.json + - run: + name: Summarize reports + command: cat reports/externalTests/all-benchmarks.json | scripts/externalTests/summarize_benchmarks.sh > reports/externalTests/summarized-benchmarks.json + - store_artifacts: + path: reports/externalTests/all-benchmarks.json + - store_artifacts: + path: reports/externalTests/summarized-benchmarks.json + b_win: &b_win <<: *base_win_powershell_large steps: @@ -1466,6 +1483,24 @@ workflows: - t_ems_ext: *job_native_test_ext_prb_math - t_ems_ext: *job_native_test_ext_elementfi + - c_ext_benchmarks: + <<: *workflow_trigger_on_tags + requires: + - t_ems_compile_ext_colony + - t_native_compile_ext_gnosis + - t_native_test_ext_gnosis_v2 + - t_native_test_ext_zeppelin + - t_native_test_ext_ens + - t_native_test_ext_trident + - t_native_test_ext_euler + - t_native_test_ext_yield_liquidator + - t_native_test_ext_bleeps + - t_native_test_ext_pool_together + - t_native_test_ext_perpetual_pools + - t_native_test_ext_uniswap + - t_native_test_ext_prb_math + - t_native_test_ext_elementfi + # Windows build and tests - b_win: *workflow_trigger_on_tags - b_win_release: *workflow_trigger_on_tags diff --git a/scripts/externalTests/merge_benchmarks.sh b/scripts/externalTests/merge_benchmarks.sh new file mode 100755 index 0000000000..3e2d7318c7 --- /dev/null +++ b/scripts/externalTests/merge_benchmarks.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +# ------------------------------------------------------------------------------ +# Reads multiple individual benchmark reports produced by scripts from +# test/externalTests/ from standard input and creates a combined report. +# +# Usage: +#