From 513005c0ea23f103416b32edbcf87e2a3ab64eab Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 21 Jul 2023 10:02:33 +0100 Subject: [PATCH] Add makefile auto download and sed on proto --- Makefile | 20 ++++ ibc/types/proto/proofs.proto | 176 +++++++++++++++++++---------------- 2 files changed, 116 insertions(+), 80 deletions(-) diff --git a/Makefile b/Makefile index 003c72b11..192cd67a6 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,15 @@ CWD ?= CURRENT_WORKING_DIRECTIONRY_NOT_SUPPLIED # `VERBOSE_TEST="" make test_persistence` is an easy way to run the same tests without verbose output VERBOSE_TEST ?= -v +# Detect OS using the $(shell uname -s) command +ifeq ($(shell uname -s),Darwin) + # Add macOS-specific commands here + SEDI = sed -i '' +else ifeq ($(shell uname -s),Linux) + # Add Linux-specific commands here + SEDI = sed -i +endif + .SILENT: .PHONY: list ## List all make targets @@ -309,6 +318,7 @@ protogen_local: go_protoc-go-inject-tag ## Generate go structures for all of the $(PROTOC_SHARED) -I=./p2p/types/proto --go_out=./p2p/types ./p2p/types/proto/*.proto # IBC + make copy_ics23_proto $(PROTOC_SHARED) -I=./ibc/types/proto --go_out=./ibc/types ./ibc/types/proto/*.proto # echo "View generated proto files by running: make protogen_show" @@ -316,6 +326,16 @@ protogen_local: go_protoc-go-inject-tag ## Generate go structures for all of the # CONSIDERATION: Some proto files contain unused gRPC services so we may need to add the following # if/when we decide to include it: `grpc--go-grpc_opt=paths=source_relative --go-grpc_out=./output/path` +.PHONY: copy_ics23_proto +copy_ics23_proto: + echo "Downloading cosmos/ics23 proto definitions..." + curl -s -o ./ibc/types/proto/proofs.proto https://raw.githubusercontent.com/cosmos/ics23/master/proto/cosmos/ics23/v1/proofs.proto && \ + $(SEDI) \ + -e '/^package/{N;d;}' \ + -e 's@github.com/.*"@github.com/pokt-network/pocket/ibc/types"@g' \ + ./ibc/types/proto/proofs.proto && \ + awk 'BEGIN { print "// ===== !! THIS IS CLONED FROM cosmos/ics23 !! =====\n" } { print }' ./ibc/types/proto/proofs.proto > tmpfile && mv tmpfile ./ibc/types/proto/proofs.proto + .PHONY: protogen_docker_m1 ## TECHDEBT: Test, validate & update. protogen_docker_m1: docker_check diff --git a/ibc/types/proto/proofs.proto b/ibc/types/proto/proofs.proto index e79881ed4..e3659215e 100644 --- a/ibc/types/proto/proofs.proto +++ b/ibc/types/proto/proofs.proto @@ -1,8 +1,6 @@ -syntax = "proto3"; +// ===== !! THIS IS CLONED FROM cosmos/ics23 !! ===== -// This file is a clone from the github.com/cosmos/ics23 repo, it has been -// cloned to be compiled with protoc generic. As such it will produce valid -// protobuf messages that can be serialised and cloned +syntax = "proto3"; option go_package = "github.com/pokt-network/pocket/ibc/types"; @@ -17,10 +15,12 @@ enum HashOp { SHA512_256 = 6; } -// LengthOp defines how to process the key and value of the LeafOp -// to include length information. After encoding the length with the given -// algorithm, the length will be prepended to the key and value bytes. -// (Each one with it's own encoded length) +/** +LengthOp defines how to process the key and value of the LeafOp +to include length information. After encoding the length with the given +algorithm, the length will be prepended to the key and value bytes. +(Each one with it's own encoded length) +*/ enum LengthOp { // NO_PREFIX don't include any length info NO_PREFIX = 0; @@ -42,25 +42,27 @@ enum LengthOp { REQUIRE_64_BYTES = 8; } -// ExistenceProof takes a key and a value and a set of steps to perform on it. -// The result of peforming all these steps will provide a "root hash", which can -// be compared to the value in a header. -// -// Since it is computationally infeasible to produce a hash collission for any of the used -// cryptographic hash functions, if someone can provide a series of operations to transform -// a given key and value into a root hash that matches some trusted root, these key and values -// must be in the referenced merkle tree. -// -// The only possible issue is maliablity in LeafOp, such as providing extra prefix data, -// which should be controlled by a spec. Eg. with lengthOp as NONE, -// prefix = FOO, key = BAR, value = CHOICE -// and -// prefix = F, key = OOBAR, value = CHOICE -// would produce the same value. -// -// With LengthOp this is tricker but not impossible. Which is why the "leafPrefixEqual" field -// in the ProofSpec is valuable to prevent this mutability. And why all trees should -// length-prefix the data before hashing it. +/** +ExistenceProof takes a key and a value and a set of steps to perform on it. +The result of peforming all these steps will provide a "root hash", which can +be compared to the value in a header. + +Since it is computationally infeasible to produce a hash collission for any of the used +cryptographic hash functions, if someone can provide a series of operations to transform +a given key and value into a root hash that matches some trusted root, these key and values +must be in the referenced merkle tree. + +The only possible issue is maliablity in LeafOp, such as providing extra prefix data, +which should be controlled by a spec. Eg. with lengthOp as NONE, + prefix = FOO, key = BAR, value = CHOICE +and + prefix = F, key = OOBAR, value = CHOICE +would produce the same value. + +With LengthOp this is tricker but not impossible. Which is why the "leafPrefixEqual" field +in the ProofSpec is valuable to prevent this mutability. And why all trees should +length-prefix the data before hashing it. +*/ message ExistenceProof { bytes key = 1; bytes value = 2; @@ -68,16 +70,20 @@ message ExistenceProof { repeated InnerOp path = 4; } -// NonExistenceProof takes a proof of two neighbors, one left of the desired key, -// one right of the desired key. If both proofs are valid AND they are neighbors, -// then there is no valid proof for the given key. +/* +NonExistenceProof takes a proof of two neighbors, one left of the desired key, +one right of the desired key. If both proofs are valid AND they are neighbors, +then there is no valid proof for the given key. +*/ message NonExistenceProof { bytes key = 1; // TODO: remove this as unnecessary??? we prove a range ExistenceProof left = 2; ExistenceProof right = 3; } -// CommitmentProof is either an ExistenceProof or a NonExistenceProof, or a Batch of such messages +/* +CommitmentProof is either an ExistenceProof or a NonExistenceProof, or a Batch of such messages +*/ message CommitmentProof { oneof proof { ExistenceProof exist = 1; @@ -87,20 +93,22 @@ message CommitmentProof { } } -// LeafOp represents the raw key-value data we wish to prove, and -// must be flexible to represent the internal transformation from -// the original key-value pairs into the basis hash, for many existing -// merkle trees. -// -// key and value are passed in. So that the signature of this operation is: -// leafOp(key, value) -> output -// -// To process this, first prehash the keys and values if needed (ANY means no hash in this case): -// hkey = prehashKey(key) -// hvalue = prehashValue(value) -// -// Then combine the bytes, and hash it -// output = hash(prefix || length(hkey) || hkey || length(hvalue) || hvalue) +/** +LeafOp represents the raw key-value data we wish to prove, and +must be flexible to represent the internal transformation from +the original key-value pairs into the basis hash, for many existing +merkle trees. + +key and value are passed in. So that the signature of this operation is: + leafOp(key, value) -> output + +To process this, first prehash the keys and values if needed (ANY means no hash in this case): + hkey = prehashKey(key) + hvalue = prehashValue(value) + +Then combine the bytes, and hash it + output = hash(prefix || length(hkey) || hkey || length(hvalue) || hvalue) +*/ message LeafOp { HashOp hash = 1; HashOp prehash_key = 2; @@ -111,37 +119,41 @@ message LeafOp { bytes prefix = 5; } -// InnerOp represents a merkle-proof step that is not a leaf. -// It represents concatenating two children and hashing them to provide the next result. -// -// The result of the previous step is passed in, so the signature of this op is: -// innerOp(child) -> output -// -// The result of applying InnerOp should be: -// output = op.hash(op.prefix || child || op.suffix) -// -// where the || operator is concatenation of binary data, -// and child is the result of hashing all the tree below this step. -// -// Any special data, like prepending child with the length, or prepending the entire operation with -// some value to differentiate from leaf nodes, should be included in prefix and suffix. -// If either of prefix or suffix is empty, we just treat it as an empty string +/** +InnerOp represents a merkle-proof step that is not a leaf. +It represents concatenating two children and hashing them to provide the next result. + +The result of the previous step is passed in, so the signature of this op is: + innerOp(child) -> output + +The result of applying InnerOp should be: + output = op.hash(op.prefix || child || op.suffix) + + where the || operator is concatenation of binary data, +and child is the result of hashing all the tree below this step. + +Any special data, like prepending child with the length, or prepending the entire operation with +some value to differentiate from leaf nodes, should be included in prefix and suffix. +If either of prefix or suffix is empty, we just treat it as an empty string +*/ message InnerOp { HashOp hash = 1; bytes prefix = 2; bytes suffix = 3; } -// ProofSpec defines what the expected parameters are for a given proof type. -// This can be stored in the client and used to validate any incoming proofs. -// -// verify(ProofSpec, Proof) -> Proof | Error -// -// As demonstrated in tests, if we don't fix the algorithm used to calculate the -// LeafHash for a given tree, there are many possible key-value pairs that can -// generate a given hash (by interpretting the preimage differently). -// We need this for proper security, requires client knows a priori what -// tree format server uses. But not in code, rather a configuration object. +/** +ProofSpec defines what the expected parameters are for a given proof type. +This can be stored in the client and used to validate any incoming proofs. + + verify(ProofSpec, Proof) -> Proof | Error + +As demonstrated in tests, if we don't fix the algorithm used to calculate the +LeafHash for a given tree, there are many possible key-value pairs that can +generate a given hash (by interpretting the preimage differently). +We need this for proper security, requires client knows a priori what +tree format server uses. But not in code, rather a configuration object. +*/ message ProofSpec { // any field in the ExistenceProof must be the same as in this spec. // except Prefix, which is just the first bytes of prefix (spec can be longer) @@ -157,14 +169,16 @@ message ProofSpec { bool prehash_key_before_comparison = 5; } -// InnerSpec contains all store-specific structure info to determine if two proofs from a -// given store are neighbors. -// -// This enables: -// -// isLeftMost(spec: InnerSpec, op: InnerOp) -// isRightMost(spec: InnerSpec, op: InnerOp) -// isLeftNeighbor(spec: InnerSpec, left: InnerOp, right: InnerOp) +/* +InnerSpec contains all store-specific structure info to determine if two proofs from a +given store are neighbors. + +This enables: + + isLeftMost(spec: InnerSpec, op: InnerOp) + isRightMost(spec: InnerSpec, op: InnerOp) + isLeftNeighbor(spec: InnerSpec, left: InnerOp, right: InnerOp) +*/ message InnerSpec { // Child order is the ordering of the children node, must count from 0 // iavl tree is [0, 1] (left then right) @@ -179,7 +193,9 @@ message InnerSpec { HashOp hash = 6; } -// BatchProof is a group of multiple proof types than can be compressed +/* +BatchProof is a group of multiple proof types than can be compressed +*/ message BatchProof { repeated BatchEntry entries = 1; } @@ -192,7 +208,7 @@ message BatchEntry { } } -// ====== all items here are compressed forms ======= +/****** all items here are compressed forms *******/ message CompressedBatchProof { repeated CompressedBatchEntry entries = 1;