Merge pull request #245 from yieldprotocol/dev

release: 2023-08-25

.github/workflows/check-update-widget-index.yaml

@@ -27,6 +27,13 @@ jobs:
         with:
           path: backend
 
+      - name: Extract branch name
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+
+      - name: Show branch name
+        run: echo "Branch name is ${{ steps.extract_branch.outputs.branch }}"
+
       - id: 'auth'
         uses: 'google-github-actions/auth@v1'
         with:
@@ -58,25 +65,9 @@ jobs:
           echo "SERVER_HOST=" >> "$GITHUB_ENV"
           echo "SERVER_ORIGINS=" >> "$GITHUB_ENV"
           echo "SERVER_SECRET_KEY=" >> "$GITHUB_ENV"
-          
-          if [[ ${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} == 'master' ]]; then
-            echo "ENV_TAG=prod" >> "$GITHUB_ENV"
-          else
-            echo "ENV_TAG=dev" >> "$GITHUB_ENV"
-            echo "WEAVIATE_URL=${{ secrets.DEV_WEAVIATE_URL }}" >> "$GITHUB_ENV"
-            echo "WEAVIATE_API_KEY=" >> "$GITHUB_ENV"
-            echo "CHATDB_URL=${{ secrets.DEV_CHATDB_URL }}" >> "$GITHUB_ENV"
-          fi
-
-      - name: Set environment for branch
-        run: |
-          echo "SERVER_HOST=" >> "$GITHUB_ENV"
-          echo "SERVER_ORIGINS=" >> "$GITHUB_ENV"
-          echo "SERVER_SECRET_KEY=" >> "$GITHUB_ENV"
-          echo "CHATDB_URL=" >> "$GITHUB_ENV"
           echo "WEAVIATE_URL=${{ secrets.WEAVIATE_URL }}" >> "$GITHUB_ENV"
 
-          if [[ ${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} == 'master' ]]; then
+          if [[ ${{ steps.extract_branch.outputs.branch }} == 'master' ]]; then
             echo "ENV_TAG=prod" >> "$GITHUB_ENV"
             echo "WEAVIATE_API_KEY=${{ secrets.PROD_WEAVIATE_API_KEY }}" >> "$GITHUB_ENV"
             echo "CHATDB_URL=${{ secrets.PROD_CHATDB_URL }}" >> "$GITHUB_ENV"
@@ -91,10 +82,10 @@ jobs:
       - name: Check and update widget index
         run: |
           GCP_SSH_CMD="gcloud compute ssh cacti-bastion-server --zone us-east1-b --ssh-key-file /tmp/gcp/google_compute_engine --quiet --tunnel-through-iap --ssh-flag"
-          if [[ ${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} == 'master' ]]; then
-            $GCP_SSH_CMD '-vvv -fN -L 8080:${{ secrets.PROD_WEAVIATE_INTERNAL_IP }}'
+          if [[ ${{ steps.extract_branch.outputs.branch }} == 'master' ]]; then
+            $GCP_SSH_CMD '-fN -L 8080:${{ secrets.PROD_WEAVIATE_INTERNAL_IP }}'
           else
-            $GCP_SSH_CMD '-vvv -fN -L 8080:${{ secrets.DEV_WEAVIATE_INTERNAL_IP }}'
+            $GCP_SSH_CMD '-fN -L 8080:${{ secrets.DEV_WEAVIATE_INTERNAL_IP }}'
           fi
           
           cd backend

.github/workflows/check-widget-translation.yaml

@@ -29,6 +29,13 @@ jobs:
         with:
           path: backend
 
+      - name: Extract branch name
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+
+      - name: Show branch name
+        run: echo "Branch name is ${{ steps.extract_branch.outputs.branch }}"
+
       - id: 'auth'
         uses: 'google-github-actions/auth@v1'
         with:
@@ -63,7 +70,7 @@ jobs:
           echo "CHATDB_URL=" >> "$GITHUB_ENV"
           echo "WEAVIATE_URL=${{ secrets.WEAVIATE_URL }}" >> "$GITHUB_ENV"
 
-          if [[ ${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} == 'master' ]]; then
+          if [[ ${{ steps.extract_branch.outputs.branch }} == 'master' ]]; then
             echo "ENV_TAG=prod" >> "$GITHUB_ENV"
             echo "WEAVIATE_API_KEY=${{ secrets.PROD_WEAVIATE_API_KEY }}" >> "$GITHUB_ENV"
             echo "CHATDB_URL=${{ secrets.PROD_CHATDB_URL }}" >> "$GITHUB_ENV"
@@ -76,10 +83,10 @@ jobs:
       - name: Check widget translation
         run: |
           GCP_SSH_CMD="gcloud compute ssh cacti-bastion-server --zone us-east1-b --ssh-key-file /tmp/gcp/google_compute_engine --quiet --tunnel-through-iap --ssh-flag"
-          if [[ ${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} == 'master' ]]; then
-            $GCP_SSH_CMD '-vvv -fN -L 8080:${{ secrets.PROD_WEAVIATE_INTERNAL_IP }}'
+          if [[ ${{ steps.extract_branch.outputs.branch }} == 'master' ]]; then
+            $GCP_SSH_CMD '-fN -L 8080:${{ secrets.PROD_WEAVIATE_INTERNAL_IP }}'
           else
-            $GCP_SSH_CMD '-vvv -fN -L 8080:${{ secrets.DEV_WEAVIATE_INTERNAL_IP }}'
+            $GCP_SSH_CMD '-fN -L 8080:${{ secrets.DEV_WEAVIATE_INTERNAL_IP }}'
           fi
 
           cd backend

integrations/center.py

@@ -5,6 +5,7 @@
 
 from urllib.parse import urlencode
 import requests
+import web3
 
 import env
 import utils
@@ -31,6 +32,8 @@
 MAX_RESULTS = 12
 PAGE_LIMIT = 12
 
+MAX_RESULTS_FOR_SEARCH = 12
+
 MAX_RESULTS_FOR_TRAITS = 100
 PAGE_LIMIT_FOR_TRAITS = 100
 
@@ -39,7 +42,7 @@ class NFTCollection(ContainerMixin):
     network: str
     address: str
     name: str
-    num_assets: int
+    num_assets: Optional[int]
     preview_image_url: str
 
     def container_name(self) -> str:
@@ -202,13 +205,16 @@ def container_params(self) -> Dict:
         )
 
 def fetch_nft_search(search_str: str) -> Generator[Union[NFTCollection, NFTAsset], None, None]:
+    limit = MAX_RESULTS_FOR_SEARCH
+    offset = 0
     q = urlencode(dict(
         query=search_str,
-        type='collection',  # too noisy otherwise
+        limit=limit,
+        offset=offset,
     ))
     count = 0
     for network in NETWORKS:
-        url = f"{API_URL}/{network}/search?{q}"
+        url = f"{API_V2_URL}/{network}/search?{q}"
         timing.log('search_begin')
         response = requests.get(url, headers=HEADERS)
         try:
@@ -218,31 +224,34 @@ def fetch_nft_search(search_str: str) -> Generator[Union[NFTCollection, NFTAsset
             break
         timing.log('search_done')
         obj = response.json()
-        for r in obj['results']:
-            if not r.get('previewImageUrl'):
+        for item in obj['items']:
+            if 'collection' not in item:
+                continue
+            collection = item['collection']
+            if 'featuredImageURL' not in collection or not collection['featuredImageURL']:
+                continue
+            count += 1  # increment pre-filtered count, to determine filtering cost to speed
+            # v2 endpoint might return a non-checksum address, convert it for compatibility with elsewhere
+            address = web3.utils.address.to_checksum_address(collection['address'])
+            result = NFTCollection(
+                network=network,
+                address=address,
+                name=collection['name'],
+                num_assets=collection['totalSupply'],
+                preview_image_url=collection['featuredImageURL'],
+            )
+            if not _is_valid_collection(result):
                 continue
-            count += 1
-            network = r['id'].split('/')[0]
-            if r['type'].lower() == 'collection':
-                result = fetch_nft_collection(network, r['address'])
-                if not _is_valid_collection(result):
-                    continue
-            else:
-                result = fetch_nft_asset(network, r['address'], r['tokenId'])
             yield result
             timing.log('first_result_done')
     timing.log('%d_results_done' % count)
 
 
 def _is_valid_collection(collection: NFTCollection) -> bool:
     """Check if this NFT collection is a valid search result."""
-    # there should be traits
-    collection_traits = fetch_nft_collection_traits(collection.network, collection.address)
-    if not collection_traits.traits:
-        return False
     # should have listed and valid assets
     if collection.network == "ethereum-mainnet":
-        token_prices = opensea.fetch_contract_listing_prices_with_retries(collection.address)
+        token_prices = opensea.fetch_contract_listing_prices_with_retries(collection.address, max_results=1)
         if not token_prices:
             return False
     return True
@@ -328,16 +337,25 @@ def fetch_nft_search_collection_by_trait(network: str, address: str, trait_name:
 
 
 def fetch_nft_collection(network: str, address: str) -> NFTCollection:
-    url = f"{API_URL}/{network}/{address}"
+    url = f"{API_V2_URL}/{network}/{address}/nft/metadata"
     response = requests.get(url, headers=HEADERS)
     response.raise_for_status()
     obj = response.json()
+    num_assets = obj['totalSupply']
+    if num_assets == 0:  # seems to return 0 incorrectly
+        # use the asset endpoint with dummy token
+        token_id = 1
+        url = f"{API_V2_URL}/{network}/{address}/nft/{token_id}/metadata"
+        response = requests.get(url, headers=HEADERS)
+        if response.status_code == 200:
+            token_obj = response.json()
+            num_assets = token_obj['collection']['totalSupply']
     return NFTCollection(
         network=network,
         address=address,
         name=obj['name'],
-        num_assets=obj['numAssets'],
-        preview_image_url=obj['smallPreviewImageUrl'],
+        num_assets=num_assets,
+        preview_image_url=obj['featuredImageURL'],
     )
 
 

integrations/opensea.py

@@ -118,18 +118,18 @@ def _exec_request():
     return ret
 
 
-def fetch_all_listings(address: str) -> List[NFTListing]:
+def fetch_all_listings(address: str, max_results: Optional[int] = None) -> List[NFTListing]:
     """Fetch all listings for a collection."""
     # NOTE: a given token ID might have more than one listing
     contract = fetch_contract(address)
     slug = contract.slug
-    limit = PAGE_LIMIT
     next_cursor = None
     ret = []
     # Arbitary limit to optimize for latency, based on hueristics related to observed number of NFTs listed for blue-chip collections.
-    max_results = 300
-    max_queries = 5
+    max_results = 300 if max_results is None else max_results
+    max_queries = 3
     queries = 0
+    limit = min(PAGE_LIMIT, max_results)
     while len(ret) < max_results and queries < max_queries:
         queries += 1
         q = urlencode(dict(
@@ -170,7 +170,7 @@ def _exec_request():
         next_cursor = obj.get("next")
         if not next_cursor:
             break
-    return ret
+    return ret[:max_results]
 
 
 def fetch_asset_listing_prices_with_retries(address: str, token_id: str) -> Optional[Dict[str, Union[str, int]]]:
@@ -183,8 +183,8 @@ def fetch_asset_listing_with_retries(address: str, token_id: str) -> Optional[NF
     listings = fetch_listings(address, token_id)
     return listings[0] if len(listings) > 0 else None
 
-def fetch_contract_listing_prices_with_retries(address: str) -> Dict[str, Dict[str, Union[str, int]]]:
-    listings = fetch_all_listings(address)
+def fetch_contract_listing_prices_with_retries(address: str, max_results: Optional[int] = None) -> Dict[str, Dict[str, Union[str, int]]]:
+    listings = fetch_all_listings(address, max_results=max_results)
     ret = {}
     for listing in listings:
         if listing.token_id not in ret or ret[listing.token_id].price_value > listing.price_value:

utils/common.py

@@ -162,7 +162,7 @@ def wrapped_fn(*args, **kwargs):
             return str(e)
         except Exception as e:
             traceback.print_exc()
-            return "An error occurred. Please try again."
+            return f'Got exception evaluating {fn.__name__}(args={args}, kwargs={kwargs}): {e}'
 
     @functools.wraps(fn)
     def wrapped_generator_fn(*args, **kwargs):