Problem: missing grpc integration test (fix crypto-org-chain#194) (cr…

…ypto-org-chain#253)

Solution: add grpc query, tx broadcast integration test
generate proto+grpc code on the fly

.flake8

@@ -1,3 +1,4 @@
 [flake8]
 max-line-length = 88
 extend-ignore = E203
+exclude = .git,__pycache__,./pystarport/pystarport/cosmos-sdk,./pystarport/pystarport/tendermint,./pystarport/pystarport/proto_python

.golangci.yml

@@ -40,6 +40,7 @@ linters:
     # - gocognit
 
 linters-settings:
+  exclude: ./pystarport/pystarport/*
   govet:
     check-shadowing: true
   errcheck:

Makefile

@@ -144,11 +144,13 @@ clean-docker-compose: localnet-stop
 create-systemd:
 	./networks/create-service.sh
 
+make-proto:
+	./makeproto.sh
 ###############################################################################
 ###                                Nix                                      ###
 ###############################################################################
 # nix installation: https://nixos.org/download.html
-nix-integration-test: check-network
+nix-integration-test: check-network make-proto
 	nix-shell ./. -A ci-shell --run "pytest -v -n 7 -m 'not ledger' --dist loadscope"
 	nix-shell ./. -A ci-shell --run "pytest -v -m ledger"
 

default.nix

@@ -84,6 +84,7 @@ rec {
       (import ./pystarport { inherit pkgs; chaind = "${chain-maind-zemu}/bin/chain-maind"; })
       cosmovisor
     ];
+
   };
 
   # test in dev-shell will use the chain-maind in PATH
@@ -98,7 +99,7 @@ rec {
 
     shellHook = ''
       # prefer local pystarport directory for development
-      export PYTHONPATH=./pystarport:$PYTHONPATH
+      export PYTHONPATH=$PWD/pystarport:$PYTHONPATH
     '';
   };
 }

integration_tests/pyproject.toml

@@ -14,6 +14,8 @@ flake8-black = "^0.2.1"
 flake8-isort = "^4.0.0"
 pep8-naming = "^0.11.1"
 pytest-github-actions-annotate-failures = "^0.1.1"
+protobuf = "^3.13.0"
+grpcio = "^1.33.2"
 
 [tool.poetry.dev-dependencies]
 

integration_tests/test_grpc.py

@@ -0,0 +1,47 @@
+import json
+
+from pystarport.proto_python.grpc_util import GrpcUtil, GrpcUtilTxBroadcast
+
+from .utils import wait_for_new_blocks
+
+
+def test_query_validators(cluster):
+    wait_for_new_blocks(cluster, 5)
+    grpc_ip_port = cluster.ipport_grpc(0)
+    grpc = GrpcUtil(grpc_ip_port)
+    grpc_response = grpc.get_validators()
+    grpc_validators = grpc_response.validators
+    validators = cluster.validators()
+    operators = {}
+    for cli_validator in validators:
+        operator_address = cli_validator["operator_address"]
+        operators[operator_address] = cli_validator
+
+    for grpc_validator in grpc_validators:
+        assert grpc_validator.operator_address in operators
+
+
+def test_tx_broadcast(cluster, tmp_path):
+    tx_txt = tmp_path / "tx.txt"
+    sign_txt = tmp_path / "sign.txt"
+    encode_txt = tmp_path / "encode.txt"
+    signer1_addr = cluster.address("signer1")
+    signer2_addr = cluster.address("signer2")
+    signer2_balance = cluster.balance(signer2_addr)
+    tx_amount = 5
+    single_tx = cluster.transfer(
+        signer1_addr, signer2_addr, f"{tx_amount}basecro", generate_only=True
+    )
+    json.dump(single_tx, tx_txt.open("w"))
+    signed_tx = cluster.sign_single_tx(tx_txt, signer1_addr)
+    json.dump(signed_tx, sign_txt.open("w"))
+    encoded_tx = cluster.encode_signed_tx(sign_txt)
+    encode_file = open(encode_txt, "wb")
+    encode_file.write(encoded_tx)
+    encode_file.close()
+    grpc_ip_port = cluster.ipport_grpc_tx(0)
+    grpc = GrpcUtilTxBroadcast(grpc_ip_port)
+    grpc.send_tx_in_base64(encoded_tx)
+    wait_for_new_blocks(cluster, 2)
+    new_signer2_balance = cluster.balance(signer2_addr)
+    assert new_signer2_balance == signer2_balance + tx_amount

makeproto.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+cd ./pystarport/pystarport
+nix-shell . --run ""

pystarport/pystarport/__init__.py

@@ -0,0 +1,6 @@
+import os
+import sys
+from pathlib import Path
+
+proto_folder = Path(os.path.abspath(__file__)).parent.joinpath("proto_python")
+sys.path.append(str(proto_folder))

pystarport/pystarport/cluster.py

@@ -303,6 +303,16 @@ def node_rpc(self, i):
         "rpc url of i-th node"
         return "tcp://127.0.0.1:%d" % ports.rpc_port(self.base_port(i))
 
+    # for query
+    def ipport_grpc(self, i):
+        "grpc url of i-th node"
+        return "127.0.0.1:%d" % ports.grpc_port(self.base_port(i))
+
+    # tx broadcast only
+    def ipport_grpc_tx(self, i):
+        "grpc url of i-th node"
+        return "127.0.0.1:%d" % ports.grpc_port_tx_only(self.base_port(i))
+
     def node_id(self, i):
         "get i-th node's tendermint node id"
         output = self.raw("tendermint", "show-node-id", home=self.home(i))
@@ -645,6 +655,27 @@ def sign_multisig_tx(self, tx_file, multi_addr, signer_name, i=0):
             )
         )
 
+    def encode_signed_tx(self, signed_tx):
+        return self.raw(
+            "tx",
+            "encode",
+            signed_tx,
+        )
+
+    def sign_single_tx(self, tx_file, signer_name, i=0):
+        return json.loads(
+            self.raw(
+                "tx",
+                "sign",
+                tx_file,
+                from_=signer_name,
+                home=self.home(i),
+                keyring_backend="test",
+                chain_id=self.chain_id,
+                node=self.node_rpc(0),
+            )
+        )
+
     def combine_multisig_tx(self, tx_file, multi_name, signer1_file, signer2_file, i=0):
         return json.loads(
             self.raw(
@@ -1119,6 +1150,7 @@ def edit_tm_cfg(path, base_port, peers, *, custom_edit=None):
     # doc['proxy_app'] = 'tcp://127.0.0.1:%d' % abci_port(base_port)
     doc["rpc"]["laddr"] = "tcp://0.0.0.0:%d" % ports.rpc_port(base_port)
     doc["rpc"]["pprof_laddr"] = "localhost:%d" % ports.pprof_port(base_port)
+    doc["rpc"]["grpc_laddr"] = "tcp://0.0.0.0:%d" % ports.grpc_port_tx_only(base_port)
     doc["p2p"]["laddr"] = "tcp://0.0.0.0:%d" % ports.p2p_port(base_port)
     doc["p2p"]["persistent_peers"] = peers
     doc["p2p"]["addr_book_strict"] = False

pystarport/pystarport/convert.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+OUTPUT=./proto_python
+COSMOS=./cosmos-sdk
+TENDERMINT=./tendermint
+TMP=$(whereis grpc_python_plugin)
+PLUGIN="$(cut -d' ' -f2 <<<"$TMP")"
+mkdir $OUTPUT
+git clone --branch v0.40.0-rc2 https://github.com/cosmos/cosmos-sdk.git
+git clone --branch v0.34.0-rc5 https://github.com/tendermint/tendermint.git
+cp -Rf $COSMOS/third_party/proto/* $COSMOS/proto/ 
+# cosmos
+protoc --proto_path=$COSMOS/proto --proto_path=$COSMOS/third_party/proto --python_out=$OUTPUT $(find $COSMOS/proto/cosmos -iname "*.proto") --grpc_python_out=$OUTPUT  --plugin=protoc-gen-grpc_python=$PLUGIN
+# cosmos third-party
+protoc --proto_path=$COSMOS/third_party/proto --proto_path=$COSMOS/proto --python_out=$OUTPUT $(find $COSMOS/third_party/proto -iname "*.proto") --grpc_python_out=$OUTPUT  --plugin=protoc-gen-grpc_python=$PLUGIN
+# tendermint
+protoc --proto_path=$TENDERMINT/proto --proto_path=$TENDERMINT/third_party/proto --python_out=$OUTPUT $(find $TENDERMINT/proto/tendermint -iname "*.proto") --grpc_python_out=$OUTPUT  --plugin=protoc-gen-grpc_python=$PLUGIN

pystarport/pystarport/default.nix

@@ -0,0 +1,12 @@
+{ pkgs ? import ../../nix { }, commit ? "" }:
+with pkgs;
+pkgs.mkShell {
+  buildInputs = [
+    python38
+    protobuf3_13
+    grpc
+    git
+  ];
+  shellHook = ''                                                                                       
+  ./convert.sh                                                                                          '';
+}

pystarport/pystarport/ports.py

@@ -16,3 +16,7 @@ def api_port(base_port):
 
 def pprof_port(base_port):
     return base_port + 5
+
+
+def grpc_port_tx_only(base_port):
+    return base_port + 6

pystarport/pystarport/proto_python/grpc_util.py

@@ -0,0 +1,39 @@
+import base64
+
+import grpc
+
+import cosmos.bank.v1beta1.tx_pb2
+import cosmos.bank.v1beta1.tx_pb2_grpc
+import cosmos.crypto.ed25519.keys_pb2
+import cosmos.staking.v1beta1.query_pb2
+import cosmos.staking.v1beta1.query_pb2_grpc
+import tendermint.rpc.grpc.types_pb2_grpc
+
+
+# for query only
+class GrpcUtil:
+    def __init__(self, ip_port):
+        self.ip_port = ip_port
+
+    def get_validators(self):
+        channel = grpc.insecure_channel(self.ip_port)
+        stub = cosmos.staking.v1beta1.query_pb2_grpc.QueryStub(channel)
+        response = stub.Validators(
+            cosmos.staking.v1beta1.query_pb2.QueryValidatorsRequest()
+        )
+        return response
+
+
+# for tx broadcast only
+class GrpcUtilTxBroadcast:
+    def __init__(self, ip_port):
+        self.ip_port = ip_port
+
+    def send_tx_in_base64(self, tx_base64):
+        tx_raw_bytes = base64.b64decode(tx_base64)
+        channel = grpc.insecure_channel(self.ip_port)
+        stub = tendermint.rpc.grpc.types_pb2_grpc.BroadcastAPIStub(channel)
+        request = tendermint.rpc.grpc.types_pb2.RequestBroadcastTx()
+        request.tx = tx_raw_bytes
+        response = stub.BroadcastTx(request)
+        return response