diff --git a/.github/workflows/check-update-widget-index.yaml b/.github/workflows/check-update-widget-index.yaml index 4925c211..d75cd8b1 100644 --- a/.github/workflows/check-update-widget-index.yaml +++ b/.github/workflows/check-update-widget-index.yaml @@ -27,6 +27,13 @@ jobs: with: path: backend + - name: Extract branch name + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + id: extract_branch + + - name: Show branch name + run: echo "Branch name is ${{ steps.extract_branch.outputs.branch }}" + - id: 'auth' uses: 'google-github-actions/auth@v1' with: @@ -58,25 +65,9 @@ jobs: echo "SERVER_HOST=" >> "$GITHUB_ENV" echo "SERVER_ORIGINS=" >> "$GITHUB_ENV" echo "SERVER_SECRET_KEY=" >> "$GITHUB_ENV" - - if [[ ${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} == 'master' ]]; then - echo "ENV_TAG=prod" >> "$GITHUB_ENV" - else - echo "ENV_TAG=dev" >> "$GITHUB_ENV" - echo "WEAVIATE_URL=${{ secrets.DEV_WEAVIATE_URL }}" >> "$GITHUB_ENV" - echo "WEAVIATE_API_KEY=" >> "$GITHUB_ENV" - echo "CHATDB_URL=${{ secrets.DEV_CHATDB_URL }}" >> "$GITHUB_ENV" - fi - - - name: Set environment for branch - run: | - echo "SERVER_HOST=" >> "$GITHUB_ENV" - echo "SERVER_ORIGINS=" >> "$GITHUB_ENV" - echo "SERVER_SECRET_KEY=" >> "$GITHUB_ENV" - echo "CHATDB_URL=" >> "$GITHUB_ENV" echo "WEAVIATE_URL=${{ secrets.WEAVIATE_URL }}" >> "$GITHUB_ENV" - if [[ ${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} == 'master' ]]; then + if [[ ${{ steps.extract_branch.outputs.branch }} == 'master' ]]; then echo "ENV_TAG=prod" >> "$GITHUB_ENV" echo "WEAVIATE_API_KEY=${{ secrets.PROD_WEAVIATE_API_KEY }}" >> "$GITHUB_ENV" echo "CHATDB_URL=${{ secrets.PROD_CHATDB_URL }}" >> "$GITHUB_ENV" @@ -91,10 +82,10 @@ jobs: - name: Check and update widget index run: | GCP_SSH_CMD="gcloud compute ssh cacti-bastion-server --zone us-east1-b --ssh-key-file /tmp/gcp/google_compute_engine --quiet --tunnel-through-iap --ssh-flag" - if [[ ${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} == 'master' ]]; then - $GCP_SSH_CMD '-vvv -fN -L 8080:${{ secrets.PROD_WEAVIATE_INTERNAL_IP }}' + if [[ ${{ steps.extract_branch.outputs.branch }} == 'master' ]]; then + $GCP_SSH_CMD '-fN -L 8080:${{ secrets.PROD_WEAVIATE_INTERNAL_IP }}' else - $GCP_SSH_CMD '-vvv -fN -L 8080:${{ secrets.DEV_WEAVIATE_INTERNAL_IP }}' + $GCP_SSH_CMD '-fN -L 8080:${{ secrets.DEV_WEAVIATE_INTERNAL_IP }}' fi cd backend diff --git a/.github/workflows/check-widget-translation.yaml b/.github/workflows/check-widget-translation.yaml index 95b3133f..cbdbb115 100644 --- a/.github/workflows/check-widget-translation.yaml +++ b/.github/workflows/check-widget-translation.yaml @@ -29,6 +29,13 @@ jobs: with: path: backend + - name: Extract branch name + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + id: extract_branch + + - name: Show branch name + run: echo "Branch name is ${{ steps.extract_branch.outputs.branch }}" + - id: 'auth' uses: 'google-github-actions/auth@v1' with: @@ -63,7 +70,7 @@ jobs: echo "CHATDB_URL=" >> "$GITHUB_ENV" echo "WEAVIATE_URL=${{ secrets.WEAVIATE_URL }}" >> "$GITHUB_ENV" - if [[ ${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} == 'master' ]]; then + if [[ ${{ steps.extract_branch.outputs.branch }} == 'master' ]]; then echo "ENV_TAG=prod" >> "$GITHUB_ENV" echo "WEAVIATE_API_KEY=${{ secrets.PROD_WEAVIATE_API_KEY }}" >> "$GITHUB_ENV" echo "CHATDB_URL=${{ secrets.PROD_CHATDB_URL }}" >> "$GITHUB_ENV" @@ -76,10 +83,10 @@ jobs: - name: Check widget translation run: | GCP_SSH_CMD="gcloud compute ssh cacti-bastion-server --zone us-east1-b --ssh-key-file /tmp/gcp/google_compute_engine --quiet --tunnel-through-iap --ssh-flag" - if [[ ${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} == 'master' ]]; then - $GCP_SSH_CMD '-vvv -fN -L 8080:${{ secrets.PROD_WEAVIATE_INTERNAL_IP }}' + if [[ ${{ steps.extract_branch.outputs.branch }} == 'master' ]]; then + $GCP_SSH_CMD '-fN -L 8080:${{ secrets.PROD_WEAVIATE_INTERNAL_IP }}' else - $GCP_SSH_CMD '-vvv -fN -L 8080:${{ secrets.DEV_WEAVIATE_INTERNAL_IP }}' + $GCP_SSH_CMD '-fN -L 8080:${{ secrets.DEV_WEAVIATE_INTERNAL_IP }}' fi cd backend diff --git a/integrations/center.py b/integrations/center.py index c7f1ad74..027a7bea 100644 --- a/integrations/center.py +++ b/integrations/center.py @@ -5,6 +5,7 @@ from urllib.parse import urlencode import requests +import web3 import env import utils @@ -31,6 +32,8 @@ MAX_RESULTS = 12 PAGE_LIMIT = 12 +MAX_RESULTS_FOR_SEARCH = 12 + MAX_RESULTS_FOR_TRAITS = 100 PAGE_LIMIT_FOR_TRAITS = 100 @@ -39,7 +42,7 @@ class NFTCollection(ContainerMixin): network: str address: str name: str - num_assets: int + num_assets: Optional[int] preview_image_url: str def container_name(self) -> str: @@ -202,13 +205,16 @@ def container_params(self) -> Dict: ) def fetch_nft_search(search_str: str) -> Generator[Union[NFTCollection, NFTAsset], None, None]: + limit = MAX_RESULTS_FOR_SEARCH + offset = 0 q = urlencode(dict( query=search_str, - type='collection', # too noisy otherwise + limit=limit, + offset=offset, )) count = 0 for network in NETWORKS: - url = f"{API_URL}/{network}/search?{q}" + url = f"{API_V2_URL}/{network}/search?{q}" timing.log('search_begin') response = requests.get(url, headers=HEADERS) try: @@ -218,17 +224,24 @@ def fetch_nft_search(search_str: str) -> Generator[Union[NFTCollection, NFTAsset break timing.log('search_done') obj = response.json() - for r in obj['results']: - if not r.get('previewImageUrl'): + for item in obj['items']: + if 'collection' not in item: + continue + collection = item['collection'] + if 'featuredImageURL' not in collection or not collection['featuredImageURL']: + continue + count += 1 # increment pre-filtered count, to determine filtering cost to speed + # v2 endpoint might return a non-checksum address, convert it for compatibility with elsewhere + address = web3.utils.address.to_checksum_address(collection['address']) + result = NFTCollection( + network=network, + address=address, + name=collection['name'], + num_assets=collection['totalSupply'], + preview_image_url=collection['featuredImageURL'], + ) + if not _is_valid_collection(result): continue - count += 1 - network = r['id'].split('/')[0] - if r['type'].lower() == 'collection': - result = fetch_nft_collection(network, r['address']) - if not _is_valid_collection(result): - continue - else: - result = fetch_nft_asset(network, r['address'], r['tokenId']) yield result timing.log('first_result_done') timing.log('%d_results_done' % count) @@ -236,13 +249,9 @@ def fetch_nft_search(search_str: str) -> Generator[Union[NFTCollection, NFTAsset def _is_valid_collection(collection: NFTCollection) -> bool: """Check if this NFT collection is a valid search result.""" - # there should be traits - collection_traits = fetch_nft_collection_traits(collection.network, collection.address) - if not collection_traits.traits: - return False # should have listed and valid assets if collection.network == "ethereum-mainnet": - token_prices = opensea.fetch_contract_listing_prices_with_retries(collection.address) + token_prices = opensea.fetch_contract_listing_prices_with_retries(collection.address, max_results=1) if not token_prices: return False return True @@ -328,16 +337,25 @@ def fetch_nft_search_collection_by_trait(network: str, address: str, trait_name: def fetch_nft_collection(network: str, address: str) -> NFTCollection: - url = f"{API_URL}/{network}/{address}" + url = f"{API_V2_URL}/{network}/{address}/nft/metadata" response = requests.get(url, headers=HEADERS) response.raise_for_status() obj = response.json() + num_assets = obj['totalSupply'] + if num_assets == 0: # seems to return 0 incorrectly + # use the asset endpoint with dummy token + token_id = 1 + url = f"{API_V2_URL}/{network}/{address}/nft/{token_id}/metadata" + response = requests.get(url, headers=HEADERS) + if response.status_code == 200: + token_obj = response.json() + num_assets = token_obj['collection']['totalSupply'] return NFTCollection( network=network, address=address, name=obj['name'], - num_assets=obj['numAssets'], - preview_image_url=obj['smallPreviewImageUrl'], + num_assets=num_assets, + preview_image_url=obj['featuredImageURL'], ) diff --git a/integrations/opensea.py b/integrations/opensea.py index d1bea6ae..a1c65cab 100644 --- a/integrations/opensea.py +++ b/integrations/opensea.py @@ -118,18 +118,18 @@ def _exec_request(): return ret -def fetch_all_listings(address: str) -> List[NFTListing]: +def fetch_all_listings(address: str, max_results: Optional[int] = None) -> List[NFTListing]: """Fetch all listings for a collection.""" # NOTE: a given token ID might have more than one listing contract = fetch_contract(address) slug = contract.slug - limit = PAGE_LIMIT next_cursor = None ret = [] # Arbitary limit to optimize for latency, based on hueristics related to observed number of NFTs listed for blue-chip collections. - max_results = 300 - max_queries = 5 + max_results = 300 if max_results is None else max_results + max_queries = 3 queries = 0 + limit = min(PAGE_LIMIT, max_results) while len(ret) < max_results and queries < max_queries: queries += 1 q = urlencode(dict( @@ -170,7 +170,7 @@ def _exec_request(): next_cursor = obj.get("next") if not next_cursor: break - return ret + return ret[:max_results] def fetch_asset_listing_prices_with_retries(address: str, token_id: str) -> Optional[Dict[str, Union[str, int]]]: @@ -183,8 +183,8 @@ def fetch_asset_listing_with_retries(address: str, token_id: str) -> Optional[NF listings = fetch_listings(address, token_id) return listings[0] if len(listings) > 0 else None -def fetch_contract_listing_prices_with_retries(address: str) -> Dict[str, Dict[str, Union[str, int]]]: - listings = fetch_all_listings(address) +def fetch_contract_listing_prices_with_retries(address: str, max_results: Optional[int] = None) -> Dict[str, Dict[str, Union[str, int]]]: + listings = fetch_all_listings(address, max_results=max_results) ret = {} for listing in listings: if listing.token_id not in ret or ret[listing.token_id].price_value > listing.price_value: diff --git a/utils/common.py b/utils/common.py index d08988d9..fd5fb455 100644 --- a/utils/common.py +++ b/utils/common.py @@ -162,7 +162,7 @@ def wrapped_fn(*args, **kwargs): return str(e) except Exception as e: traceback.print_exc() - return "An error occurred. Please try again." + return f'Got exception evaluating {fn.__name__}(args={args}, kwargs={kwargs}): {e}' @functools.wraps(fn) def wrapped_generator_fn(*args, **kwargs):