diff --git a/offchain/metadata/adapters/arweave.py b/offchain/metadata/adapters/arweave.py index 36f5414..586ea11 100644 --- a/offchain/metadata/adapters/arweave.py +++ b/offchain/metadata/adapters/arweave.py @@ -60,7 +60,7 @@ def parse_ar_url(self, url: str) -> str: return url async def gen_send(self, url: str, sess: httpx.AsyncClient(), *args, **kwargs) -> httpx.Response: # type: ignore[no-untyped-def, valid-type] # noqa: E501 - """Format and send async request to ARWeave host. + """Format and send an async `GET` request to ARWeave host at parsed url. Args: url (str): url to send request to @@ -72,7 +72,7 @@ async def gen_send(self, url: str, sess: httpx.AsyncClient(), *args, **kwargs) - return await sess.get(self.parse_ar_url(url), timeout=self.timeout, follow_redirects=True) # type: ignore[no-any-return] # noqa: E501 def send(self, request: PreparedRequest, *args, **kwargs) -> Response: # type: ignore[no-untyped-def] # noqa: E501 - """Format and send request to ARWeave host. + """Format and send a `GET` request to ARWeave host at parsed url. Args: request (PreparedRequest): incoming request @@ -83,3 +83,15 @@ def send(self, request: PreparedRequest, *args, **kwargs) -> Response: # type: request.url = self.parse_ar_url(request.url) # type: ignore[arg-type] kwargs["timeout"] = self.timeout return super().send(request, *args, **kwargs) + + async def gen_head(self, url: str, sess: httpx.AsyncClient(), *args, **kwargs) -> httpx.Response: # type: ignore[no-untyped-def, valid-type] # noqa: E501 + """Format and send an async `HEAD` request to ARWeave host at parsed url. + + Args: + url (str): url to send request to + sess (httpx.AsyncClient()): async client + + Returns: + httpx.Response: response from ARWeave host. + """ + return await sess.head(self.parse_ar_url(url), timeout=self.timeout, follow_redirects=True) # type: ignore[no-any-return] # noqa: E501 diff --git a/offchain/metadata/adapters/base_adapter.py b/offchain/metadata/adapters/base_adapter.py index 82fe2c1..133beba 100644 --- a/offchain/metadata/adapters/base_adapter.py +++ b/offchain/metadata/adapters/base_adapter.py @@ -14,7 +14,22 @@ def __init__(self, *args, **kwargs): # type: ignore[no-untyped-def] super().__init__() async def gen_send(self, url: str, *args, **kwargs) -> httpx.Response: # type: ignore[no-untyped-def] # noqa: E501 - """Format and send async request to url host. + """ + Format and send an async `GET` request to url host. + Abstract method, implemented in subclasses. + + Args: + url (str): url to send request to + + Returns: + httpx.Response: response from host. + """ + raise NotImplementedError + + async def gen_head(self, url: str, *args, **kwargs) -> httpx.Response: # type: ignore[no-untyped-def] # noqa: E501 + """ + Format and send an async `HEAD` request to url host. + Abstract method, implemented in subclasses. Args: url (str): url to send request to @@ -40,7 +55,7 @@ def __init__( # type: ignore[no-untyped-def] super().__init__(pool_connections, pool_maxsize, max_retries, pool_block) async def gen_send(self, url: str, sess: httpx.AsyncClient(), *args, **kwargs) -> httpx.Response: # type: ignore[no-untyped-def, valid-type] # noqa: E501 - """Format and send async request to url host. + """Format and send an async `GET` request to url host. Args: url (str): url to send request to @@ -50,6 +65,18 @@ async def gen_send(self, url: str, sess: httpx.AsyncClient(), *args, **kwargs) - """ return await sess.get(url, follow_redirects=True) # type: ignore[no-any-return] + async def gen_head(self, url: str, sess: httpx.AsyncClient(), *args, **kwargs) -> httpx.Response: # type: ignore[no-untyped-def, valid-type] # noqa: E501 + """Format and send an async `HEAD` request to url host. + + Args: + url (str): url to send request to + sess (httpx.AsyncClient()): async client + + Returns: + httpx.Response: response from host. + """ + return await sess.head(url, follow_redirects=True) # type: ignore[no-any-return] + Adapter = Union[BaseAdapter, HTTPAdapter] diff --git a/offchain/metadata/adapters/data_uri.py b/offchain/metadata/adapters/data_uri.py index a577e1f..a7e72ab 100644 --- a/offchain/metadata/adapters/data_uri.py +++ b/offchain/metadata/adapters/data_uri.py @@ -1,4 +1,5 @@ import base64 +from email.message import Message from urllib.request import urlopen import httpx @@ -28,7 +29,7 @@ def __init__(self, *args, **kwargs): # type: ignore[no-untyped-def] super().__init__(*args, **kwargs) # type: ignore[no-untyped-call] async def gen_send(self, url: str, *args, **kwargs) -> httpx.Response: # type: ignore[no-untyped-def] # noqa: E501 - """Handle async data uri request. + """Handle async data uri `GET` request. Args: url (str): url @@ -44,7 +45,7 @@ async def gen_send(self, url: str, *args, **kwargs) -> httpx.Response: # type: return response def send(self, request: PreparedRequest, *args, **kwargs): # type: ignore[no-untyped-def] # noqa: E501 - """Handle data uri request. + """Handle data uri `GET` request. Args: request (PreparedRequest): incoming request @@ -66,5 +67,25 @@ def send(self, request: PreparedRequest, *args, **kwargs): # type: ignore[no-un finally: return newResponse + async def gen_head(self, url: str, *args, **kwargs) -> httpx.Response: # type: ignore[no-untyped-def] # noqa: E501 + """Handle async data uri `HEAD` request. + + Args: + url (str): url + + Returns: + httpx.Response: encoded data uri response. + """ + response_headers = {} + with urlopen(url) as r: + message: Message = r.info() + response_headers = dict(message._headers) # type: ignore[attr-defined] + response = httpx.Response( + status_code=200, + headers=response_headers, + request=httpx.Request(method="HEAD", url=url), + ) + return response + def close(self): # type: ignore[no-untyped-def] self.response.close() diff --git a/offchain/metadata/adapters/ipfs.py b/offchain/metadata/adapters/ipfs.py index 91fafe3..29f6929 100644 --- a/offchain/metadata/adapters/ipfs.py +++ b/offchain/metadata/adapters/ipfs.py @@ -97,7 +97,7 @@ def make_request_url(self, request_url: str, gateway: Optional[str] = None) -> s return build_request_url(gateway=gateway, request_url=request_url) async def gen_send(self, url: str, sess: httpx.AsyncClient(), *args, **kwargs) -> httpx.Response: # type: ignore[no-untyped-def, valid-type] # noqa: E501 - """Format and send async request to IPFS host. + """Format and send an async `GET` request to IPFS host. Args: url (str): url to send request to @@ -121,3 +121,15 @@ def send(self, request: PreparedRequest, *args, **kwargs) -> Response: # type: kwargs["timeout"] = self.timeout return super().send(request, *args, **kwargs) + + async def gen_head(self, url: str, sess: httpx.AsyncClient(), *args, **kwargs) -> httpx.Response: # type: ignore[no-untyped-def, valid-type] # noqa: E501 + """Format and send an async `HEAD` request to IPFS host. + + Args: + url (str): url to send request to + sess (httpx.AsyncClient()): async client session + + Returns: + httpx.Response: response from IPFS host. + """ + return await sess.head(self.make_request_url(url), timeout=self.timeout, follow_redirects=True) # type: ignore[no-any-return] # noqa: E501 diff --git a/offchain/metadata/fetchers/metadata_fetcher.py b/offchain/metadata/fetchers/metadata_fetcher.py index 9a5c7a4..4ee38de 100644 --- a/offchain/metadata/fetchers/metadata_fetcher.py +++ b/offchain/metadata/fetchers/metadata_fetcher.py @@ -63,7 +63,7 @@ def _head(self, uri: str): # type: ignore[no-untyped-def] def _get(self, uri: str): # type: ignore[no-untyped-def] return self.sess.get(uri, timeout=self.timeout, allow_redirects=True) - async def _gen(self, uri: str) -> httpx.Response: + async def _gen(self, uri: str, method: Optional[str] = "GET") -> httpx.Response: from offchain.metadata.pipelines.metadata_pipeline import ( DEFAULT_ADAPTER_CONFIGS, ) @@ -78,13 +78,21 @@ async def _gen(self, uri: str) -> httpx.Response: adapter = adapter_config.adapter_cls( host_prefixes=adapter_config.host_prefixes, **adapter_config.kwargs ) - return await adapter.gen_send( - url=uri, timeout=self.timeout, sess=self.async_sess - ) + if method == "HEAD": + return await adapter.gen_head( + url=uri, timeout=self.timeout, sess=self.async_sess + ) + else: + return await adapter.gen_send( + url=uri, timeout=self.timeout, sess=self.async_sess + ) return await self.async_sess.get( uri, timeout=self.timeout, follow_redirects=True ) + async def _gen_head(self, uri: str) -> httpx.Response: + return await self._gen(uri=uri, method="HEAD") + def fetch_mime_type_and_size(self, uri: str) -> Tuple[str, int]: """Fetch the mime type and size of the content at a given uri. @@ -123,11 +131,10 @@ async def gen_fetch_mime_type_and_size(self, uri: str) -> Tuple[str, int]: tuple[str, int]: mime type and size """ try: - # try skip head request - # res = await self._gen_head(uri) - # # For any error status, try a get - # if 300 <= res.status_code < 600: - res = await self._gen(uri) + res = await self._gen_head(uri) + # For any error status, try a get + if 300 <= res.status_code < 600: + res = await self._gen(uri) res.raise_for_status() headers = res.headers size = headers.get("content-length", 0) diff --git a/tests/conftest.py b/tests/conftest.py index 0ee4b7e..11bc990 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -131,3 +131,34 @@ def raw_crypto_coven_metadata(): # type: ignore[no-untyped-def] "hash": "nyx", }, } + + +@pytest.fixture +def mock_video_rawdata(): # type: ignore[no-untyped-def] + return { + "name": "πŸ€‘πŸ‘‰πŸ˜πŸ‘‘πŸ’", + "description": "Yats πŸ–– are emoji usernames that become your universal Internet identity πŸ—Ώ, website URL πŸ’», payment address πŸ€‘, and more. By owning a Yat – let’s say πŸ€‘πŸ‘‰πŸ˜πŸ‘‘πŸ’ – it’s yours forever. Get inspired and join our amazingly creative Yat Community at Y.at.", + "image": "https://y.at/viz/money-mouth/money-mouth.point.heart-eyes.crown.ring-2ba3b7.png", + "thumbnail_image": "https://y.at/viz/money-mouth/money-mouth.point.heart-eyes.crown.ring-2ba3b7.png", + "animation_url": "https://y.at/viz/money-mouth/money-mouth.point.heart-eyes.crown.ring-2ba3b7.mp4", + "icon_url": "", + "token_id": "", + "owner_name": "", + "external_link": "https://y.at/%F0%9F%A4%91%F0%9F%91%89%F0%9F%98%8D%F0%9F%91%91%F0%9F%92%8D", + "attributes": [ + {"trait_type": "Length", "value": "Five-Emoji"}, + {"trait_type": "Rhythm Score", "value": "0-25"}, + {"trait_type": "Generation", "value": "Gen One"}, + {"trait_type": "Visualizer Theme", "value": "ribbons"}, + ], + } + + +@pytest.fixture +def mock_image_rawdata(): # type: ignore[no-untyped-def] + return { + "description": "The tax man came, and old gregson was left with nothing. They took it all, his house, his possessions, every ounce of savings in his bank account..and most of all, his beloved farm with the many animals he cherished and adored…”. Gregson wanted to end it all, instead by some miracle he ended up on the mysterious continent of crypto befriending hammond the punk, michelangelo the ape & kafka the cat changing their lives forever. James di martino turns reality into fiction in this Orwellian journey of trust, betrayal and the limitless power of ethereum.", + "external_url": "", + "image": "https://ipfs.io/ipfs/QmQaYaf3Q2oCBaUfUvV6mBP58EjbUTbMk6dC1o4YGjeWCo", + "name": "CryptoFarm", + } diff --git a/tests/metadata/adapters/test_arweave_adapter.py b/tests/metadata/adapters/test_arweave_adapter.py new file mode 100644 index 0000000..187d841 --- /dev/null +++ b/tests/metadata/adapters/test_arweave_adapter.py @@ -0,0 +1,43 @@ +import httpx +import pytest +from pytest_httpx import HTTPXMock + +from offchain.metadata.adapters import ARWeaveAdapter # type: ignore[attr-defined] + + +class TestARWeaveAdapter: + def test_arweave_adapter_make_request_url(self): # type: ignore[no-untyped-def] + adapter = ARWeaveAdapter() + arweave_url = "ar://-G92LjB-wFj-FCGx040NgniW_Ypy_Cbh3Jq1HUD6l7A" # noqa + assert ( + adapter.parse_ar_url(arweave_url) + == "https://arweave.net/-G92LjB-wFj-FCGx040NgniW_Ypy_Cbh3Jq1HUD6l7A" + ) + + @pytest.mark.asyncio + async def test_gen_head(self, httpx_mock: HTTPXMock): + # mocker responds to HEAD requests only + httpx_mock.add_response(method="HEAD") + + adapter = ARWeaveAdapter() + arweave_url = "ar://-G92LjB-wFj-FCGx040NgniW_Ypy_Cbh3Jq1HUD6l7A" # noqa + async with httpx.AsyncClient() as client: + await adapter.gen_head(url=arweave_url, sess=client) + outgoing_get_request = httpx_mock.get_request(method="GET") + assert not outgoing_get_request + outgoing_head_request = httpx_mock.get_request(method="HEAD") + assert outgoing_head_request + + @pytest.mark.asyncio + async def test_gen_send(self, httpx_mock: HTTPXMock): + # mocker responds to GET requests only + httpx_mock.add_response(method="GET") + + adapter = ARWeaveAdapter() + arweave_url = "ar://-G92LjB-wFj-FCGx040NgniW_Ypy_Cbh3Jq1HUD6l7A" # noqa + async with httpx.AsyncClient() as client: + await adapter.gen_send(url=arweave_url, sess=client) + outgoing_get_request = httpx_mock.get_request(method="GET") + assert outgoing_get_request + outgoing_head_request = httpx_mock.get_request(method="HEAD") + assert not outgoing_head_request diff --git a/tests/metadata/adapters/test_data_adapter.py b/tests/metadata/adapters/test_data_adapter.py new file mode 100644 index 0000000..d21d409 --- /dev/null +++ b/tests/metadata/adapters/test_data_adapter.py @@ -0,0 +1,39 @@ +import httpx +import pytest +from pytest_httpx import HTTPXMock + +from offchain.metadata.adapters import DataURIAdapter # type: ignore[attr-defined] + + +class TestDataURIAdapter: + @pytest.mark.asyncio + async def test_gen_head(self, httpx_mock: HTTPXMock): + adapter = DataURIAdapter() + data_url = "data:image/svg+xml;base64,PHN2ZyBpZD0iaDNpNXR6IiB2aWV3Qm94PSIwIDAgNDggNDgiIHdpZHRoPSIxMDAlIiBoZWlnaHQ9IjEwMCUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiIHZlcnNpb249IjEuMSIgPgoKPGltYWdlIHg9IjAiIHk9IjAiIHdpZHRoPSI0OCIgaGVpZ2h0PSI0OCIgaW1hZ2UtcmVuZGVyaW5nPSJwaXhlbGF0ZWQiIHByZXNlcnZlQXNwZWN0UmF0aW89InhNaWRZTWlkIiB4bGluazpocmVmPSJkYXRhOmltYWdlL3BuZztiYXNlNjQsaVZCT1J3MEtHZ29BQUFBTlNVaEVVZ0FBQUZBQUFBQlFDQU1BQUFDNXp3S2ZBQUFBWUZCTVZFVlFZWHIveUFFK1QxNy9rZ0I0YVdJUkVST3RwcHNhSGl3QUFBQVhtc3pKeGJ4MFhCUndxY2IvN2dKbGJIOWZiNGE1NWQ3L3RBRFBsdzFyWEZmOTVpMFBEQXVBa3F5bGhqTFQ4KzdLeU1jL1BUamh3ai8yeHk0VmFvcU1yc0t4MTlDSW1CMllBQUFENUVsRVFWUjRBZTJVaVpLak9CQkVzUllKdWZFMUFsbzJxSGYrL3k4M2xTQUt4Z1NodnZaKzl1QjB0WGxSTlFVVXh4dzBPZWJ3VndqZnRQYXRjNjJuOHROQ3JmczJ1RGJpUXFYMXA0VlYxWWVCd3VDSHZxbytLOVFoQk9kYmp1emNFSUwrckhCd2JoYUNyeEhDQ0w1VzZCeU1meFBoMjRwbjRmcnZHVUt0dXlYWXNnanhwVnVpZFphd1hLSjFQL2cyNG9laDBsMjVKRmRvU29NM1hnME9UU08zWHNNcUQvekZ4NFR4UkVLWENKdHM0V0NDTVVOOEJZUE1aUHhjR0ppWU00V3RPWnVoTlhpZERUTFRNSnhUb1dYQnhFS204R0VleHNURFN6dzhYaGI1bDBKMmg2M0JtNzFJUTVLSHVmRFJwVWpoZzF2ZW90bW81QXFiYkRLRm9OTnJ1bzBLeUJJU3JWNWVTdklTa2ZoSXNWTkhJVk40dTkvdm8xRGlnL0VEd2g1YnZOOXV0OUtBcmZpUkRwdXlhV0piaHBFalB3eHFqQjhRR2hLRkVwa1l1MG9wdFNWVUU4ZlQ4U1FaUWdGM1dZb1BrMkxYRjBYQnM4YVhPaDFWcEpoUXNhb3VCSm4vaDhTQXJZZ09TVExDemRObllheE9RdVNNTGZmcFBPbVF3bGVDRG9HNi9IN0JHNWxDN3BOQ2lZOFVPeVhuUlJSUGx3N1hRcFV4OGxxb2tFUjRuWVZFcVNJS2Nha0FXaHJDeUNLRWFlVGtROXdhbWVDdjNMS1p0N3dSSUdRanlRZHcrdWJJcXNnU3lzajAwWTdUeDdRV29tdU0vSXhlOElzUUNXOEl0MGFtVUsyYll0VGxqT2JJWUMza3lQWUpyZTBtdXJ6ZDd2RXl2RVdoQnJOd1JJRTlZZmNrL0hrREN5R2FJbHBYVlRVSlgrMTV4dHJ4T0grZUZ5UWhMMnh0dXFydm81Q2QxYlgzM3RweEtmWXdBd09QbHJYeFk5M2g2R09IVVFnWHNiWHRPdS8xaFVJNVAwTkkzOCtGOEFRZ3JQRjNYMWZUbHUwUE1BcW53QUkvVmtMNjBLV01EQ0ZNdnF2eHo5WTloSkYwdnFLUWlQQjVLVEx5cTZvNnVHcThmVHpPUWs1STRVSGd5SW5uTFhOa0gyMzFBVG9MMUNXTi9Cc1loUWdhVEFVZUU3TGxKTFNqejBaaHo1dDVzOE9kcFNRbW9jZWM0T0FHNSsxOFlZTjFoeFpJaDV1WERaZGlhL2JYTzNEQXpKT1FJK2NMNmJ1elE0ditJaitDRFhYd2ZubzQ3SGU0WUxwc1pDbnNUOGVKd3dGeEpSenBsTjFsTGJSUmlONEc2MXpnYmdyWjhqc3BTN3ZBKytDQ0NLWERmTFJlQ2V2Z1hBMm1od01XdmZ1UTRRMXVsMVg1RXJIOVdRY0lBWVNLdTk2OS9KTHdJSnlYMzZ3N09EY0tMVnp3dldVTDdlSTNFZ2JzcEZvSXdlNURCc0tOb2xMekl3cjlVamNKSXhTU2ZTRklGbmxFMWRRUlcwenNQMlE0OHJwS0lkTzJjUGNoQStIcTYvZ2J1YjNxbWI1SVpDeEZPSUJWaHdrdnd0MW5Bb1NwdU5laGw0RnpoWUpkQ24yMDRTQSt0U0VVWnFGVXJRaVpjRWltQ2Z0dVlKQzBnZnhxZzJMMUlSYUpHK1FJcDArUmZLOVFGZC9GRlUvWjYrWDZQaTQ0YTVtRmExRjlNZjhML3hmK0hZUi9BRVJPMzlYOE5Fb1VBQUFBQUVsRlRrU3VRbUNDIi8+Cgo8L3N2Zz4=" # noqa + async with httpx.AsyncClient() as client: + result = await adapter.gen_head(url=data_url, sess=client) + + expected = httpx.Response( + status_code=200, + headers={"content-type": "image/svg+xml", "content-length": "1853"}, + request=httpx.Request(method="HEAD", url=data_url), + ) + assert result.status_code == 200 + assert result.request.method == "HEAD" + assert result.headers == expected.headers + # no real request was made + outgoing_request = httpx_mock.get_requests() + assert not outgoing_request + + @pytest.mark.asyncio + async def test_gen_send(self, httpx_mock: HTTPXMock): + adapter = DataURIAdapter() + data_url = "data:image/svg+xml;base64,PHN2ZyBpZD0iaDNpNXR6IiB2aWV3Qm94PSIwIDAgNDggNDgiIHdpZHRoPSIxMDAlIiBoZWlnaHQ9IjEwMCUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiIHZlcnNpb249IjEuMSIgPgoKPGltYWdlIHg9IjAiIHk9IjAiIHdpZHRoPSI0OCIgaGVpZ2h0PSI0OCIgaW1hZ2UtcmVuZGVyaW5nPSJwaXhlbGF0ZWQiIHByZXNlcnZlQXNwZWN0UmF0aW89InhNaWRZTWlkIiB4bGluazpocmVmPSJkYXRhOmltYWdlL3BuZztiYXNlNjQsaVZCT1J3MEtHZ29BQUFBTlNVaEVVZ0FBQUZBQUFBQlFDQU1BQUFDNXp3S2ZBQUFBWUZCTVZFVlFZWHIveUFFK1QxNy9rZ0I0YVdJUkVST3RwcHNhSGl3QUFBQVhtc3pKeGJ4MFhCUndxY2IvN2dKbGJIOWZiNGE1NWQ3L3RBRFBsdzFyWEZmOTVpMFBEQXVBa3F5bGhqTFQ4KzdLeU1jL1BUamh3ai8yeHk0VmFvcU1yc0t4MTlDSW1CMllBQUFENUVsRVFWUjRBZTJVaVpLak9CQkVzUllKdWZFMUFsbzJxSGYrL3k4M2xTQUt4Z1NodnZaKzl1QjB0WGxSTlFVVXh4dzBPZWJ3VndqZnRQYXRjNjJuOHROQ3JmczJ1RGJpUXFYMXA0VlYxWWVCd3VDSHZxbytLOVFoQk9kYmp1emNFSUwrckhCd2JoYUNyeEhDQ0w1VzZCeU1meFBoMjRwbjRmcnZHVUt0dXlYWXNnanhwVnVpZFphd1hLSjFQL2cyNG9laDBsMjVKRmRvU29NM1hnME9UU08zWHNNcUQvekZ4NFR4UkVLWENKdHM0V0NDTVVOOEJZUE1aUHhjR0ppWU00V3RPWnVoTlhpZERUTFRNSnhUb1dYQnhFS204R0VleHNURFN6dzhYaGI1bDBKMmg2M0JtNzFJUTVLSHVmRFJwVWpoZzF2ZW90bW81QXFiYkRLRm9OTnJ1bzBLeUJJU3JWNWVTdklTa2ZoSXNWTkhJVk40dTkvdm8xRGlnL0VEd2g1YnZOOXV0OUtBcmZpUkRwdXlhV0piaHBFalB3eHFqQjhRR2hLRkVwa1l1MG9wdFNWVUU4ZlQ4U1FaUWdGM1dZb1BrMkxYRjBYQnM4YVhPaDFWcEpoUXNhb3VCSm4vaDhTQXJZZ09TVExDemRObllheE9RdVNNTGZmcFBPbVF3bGVDRG9HNi9IN0JHNWxDN3BOQ2lZOFVPeVhuUlJSUGx3N1hRcFV4OGxxb2tFUjRuWVZFcVNJS2Nha0FXaHJDeUNLRWFlVGtROXdhbWVDdjNMS1p0N3dSSUdRanlRZHcrdWJJcXNnU3lzajAwWTdUeDdRV29tdU0vSXhlOElzUUNXOEl0MGFtVUsyYll0VGxqT2JJWUMza3lQWUpyZTBtdXJ6ZDd2RXl2RVdoQnJOd1JJRTlZZmNrL0hrREN5R2FJbHBYVlRVSlgrMTV4dHJ4T0grZUZ5UWhMMnh0dXFydm81Q2QxYlgzM3RweEtmWXdBd09QbHJYeFk5M2g2R09IVVFnWHNiWHRPdS8xaFVJNVAwTkkzOCtGOEFRZ3JQRjNYMWZUbHUwUE1BcW53QUkvVmtMNjBLV01EQ0ZNdnF2eHo5WTloSkYwdnFLUWlQQjVLVEx5cTZvNnVHcThmVHpPUWs1STRVSGd5SW5uTFhOa0gyMzFBVG9MMUNXTi9Cc1loUWdhVEFVZUU3TGxKTFNqejBaaHo1dDVzOE9kcFNRbW9jZWM0T0FHNSsxOFlZTjFoeFpJaDV1WERaZGlhL2JYTzNEQXpKT1FJK2NMNmJ1elE0ditJaitDRFhYd2ZubzQ3SGU0WUxwc1pDbnNUOGVKd3dGeEpSenBsTjFsTGJSUmlONEc2MXpnYmdyWjhqc3BTN3ZBKytDQ0NLWERmTFJlQ2V2Z1hBMm1od01XdmZ1UTRRMXVsMVg1RXJIOVdRY0lBWVNLdTk2OS9KTHdJSnlYMzZ3N09EY0tMVnp3dldVTDdlSTNFZ2JzcEZvSXdlNURCc0tOb2xMekl3cjlVamNKSXhTU2ZTRklGbmxFMWRRUlcwenNQMlE0OHJwS0lkTzJjUGNoQStIcTYvZ2J1YjNxbWI1SVpDeEZPSUJWaHdrdnd0MW5Bb1NwdU5laGw0RnpoWUpkQ24yMDRTQSt0U0VVWnFGVXJRaVpjRWltQ2Z0dVlKQzBnZnhxZzJMMUlSYUpHK1FJcDArUmZLOVFGZC9GRlUvWjYrWDZQaTQ0YTVtRmExRjlNZjhML3hmK0hZUi9BRVJPMzlYOE5Fb1VBQUFBQUVsRlRrU3VRbUNDIi8+Cgo8L3N2Zz4=" # noqa + async with httpx.AsyncClient() as client: + result = await adapter.gen_send(url=data_url, sess=client) + + assert result.status_code == 200 + assert result.request.method == "GET" + # no real request was made + outgoing_request = httpx_mock.get_requests() + assert not outgoing_request diff --git a/tests/metadata/adapters/test_http_adapter.py b/tests/metadata/adapters/test_http_adapter.py new file mode 100644 index 0000000..4aa6917 --- /dev/null +++ b/tests/metadata/adapters/test_http_adapter.py @@ -0,0 +1,35 @@ +import httpx +import pytest +from pytest_httpx import HTTPXMock + +from offchain.metadata.adapters import HTTPAdapter # type: ignore[attr-defined] + + +class TestHTTPAdapter: + @pytest.mark.asyncio + async def test_gen_head(self, httpx_mock: HTTPXMock): + # mocker responds to HEAD requests only + httpx_mock.add_response(method="HEAD") + + adapter = HTTPAdapter() + url = "https://meta.sadgirlsbar.io/8403.json" # noqa + async with httpx.AsyncClient() as client: + await adapter.gen_head(url=url, sess=client) + outgoing_get_request = httpx_mock.get_request(method="GET") + assert not outgoing_get_request + outgoing_head_request = httpx_mock.get_request(method="HEAD") + assert outgoing_head_request + + @pytest.mark.asyncio + async def test_gen_send(self, httpx_mock: HTTPXMock): + # mocker responds to GET requests only + httpx_mock.add_response(method="GET") + + adapter = HTTPAdapter() + url = "https://meta.sadgirlsbar.io/8403.json" # noqa + async with httpx.AsyncClient() as client: + await adapter.gen_send(url=url, sess=client) + outgoing_get_request = httpx_mock.get_request(method="GET") + assert outgoing_get_request + outgoing_head_request = httpx_mock.get_request(method="HEAD") + assert not outgoing_head_request diff --git a/tests/metadata/adapters/test_ipfs_adapter.py b/tests/metadata/adapters/test_ipfs_adapter.py index e62b742..d69fe07 100644 --- a/tests/metadata/adapters/test_ipfs_adapter.py +++ b/tests/metadata/adapters/test_ipfs_adapter.py @@ -1,4 +1,6 @@ +import httpx import pytest +from pytest_httpx import HTTPXMock from offchain.metadata.adapters import IPFSAdapter # type: ignore[attr-defined] @@ -19,3 +21,35 @@ def test_ipfs_adapter_make_request_url(self): # type: ignore[no-untyped-def] adapter.make_request_url(url) == "https://gateway.pinata.cloud/ipfs/QmSr3vdMuP2fSxWD7S26KzzBWcAN1eNhm4hk1qaR3x3vmj/1.json" ) + + @pytest.mark.asyncio + async def test_gen_head(self, httpx_mock: HTTPXMock): + # mocker responds to HEAD requests only + httpx_mock.add_response(method="HEAD") + + adapter = IPFSAdapter() + ipfs_url = ( + "ipfs://bafkreiboyxwytfyufln3uzyzaixslzvmrqs5ezjo2cio2fymfqf6u57u6u" # noqa + ) + async with httpx.AsyncClient() as client: + await adapter.gen_head(url=ipfs_url, sess=client) + outgoing_get_request = httpx_mock.get_request(method="GET") + assert not outgoing_get_request + outgoing_head_request = httpx_mock.get_request(method="HEAD") + assert outgoing_head_request + + @pytest.mark.asyncio + async def test_gen_send(self, httpx_mock: HTTPXMock): + # mocker responds to GET requests only + httpx_mock.add_response(method="GET") + + adapter = IPFSAdapter() + ipfs_url = ( + "ipfs://bafkreiboyxwytfyufln3uzyzaixslzvmrqs5ezjo2cio2fymfqf6u57u6u" # noqa + ) + async with httpx.AsyncClient() as client: + await adapter.gen_send(url=ipfs_url, sess=client) + outgoing_get_request = httpx_mock.get_request(method="GET") + assert outgoing_get_request + outgoing_head_request = httpx_mock.get_request(method="HEAD") + assert not outgoing_head_request diff --git a/tests/metadata/fetchers/test_metadata_fetcher.py b/tests/metadata/fetchers/test_metadata_fetcher.py index c811235..09f405b 100644 --- a/tests/metadata/fetchers/test_metadata_fetcher.py +++ b/tests/metadata/fetchers/test_metadata_fetcher.py @@ -1,5 +1,7 @@ import pytest +from pytest_httpx import HTTPXMock + from offchain.metadata.adapters.ipfs import IPFSAdapter from offchain.metadata.fetchers.metadata_fetcher import MetadataFetcher @@ -56,3 +58,58 @@ async def test_gen_fetch_mime_type_and_size(): # type: ignore[no-untyped-def] ) assert result == ("image/png", "2887641") # type: ignore[comparison-overlap] print(result) + + +@pytest.mark.asyncio +async def test_gen_fetch_mime_type_and_size_http(httpx_mock: HTTPXMock): # type: ignore[no-untyped-def] + expected_headers = {"content-type": "image/png", "content-length": "99639"} + httpx_mock.add_response(method="HEAD", headers=expected_headers) + fetcher = MetadataFetcher() + result = await fetcher.gen_fetch_mime_type_and_size( + "https://d4ldbtmwfs9ii.cloudfront.net/7273.png" # noqa + ) + assert result == ( + expected_headers["content-type"], + expected_headers["content-length"], + ) + + +@pytest.mark.asyncio +async def test_gen_fetch_mime_type_and_size_ipfs(httpx_mock: HTTPXMock): # type: ignore[no-untyped-def] + expected_headers = {"content-type": "image/png", "content-length": "1251767"} + httpx_mock.add_response(method="HEAD", headers=expected_headers) + fetcher = MetadataFetcher() + result = await fetcher.gen_fetch_mime_type_and_size( + "ipfs://QmV4MseQF2QDDYbmxtg7eEQ9vMuYNntPQrR3arXHnK4yGX/150.png" + ) + assert result == ( + expected_headers["content-type"], + expected_headers["content-length"], + ) + + +@pytest.mark.asyncio +async def test_gen_fetch_mime_type_and_size_arweave(httpx_mock: HTTPXMock): # type: ignore[no-untyped-def] + expected_headers = {"content-type": "image/png", "content-length": "235779"} + httpx_mock.add_response(method="HEAD", headers=expected_headers) + fetcher = MetadataFetcher() + result = await fetcher.gen_fetch_mime_type_and_size( + "ar://veLMprs2c--Rl6nXCeakR5FG9K8y4WXt62iLxayrflo/1032.png" + ) + assert result == ( + expected_headers["content-type"], + expected_headers["content-length"], + ) + + +@pytest.mark.asyncio +async def test_gen_fetch_mime_type_and_size_data(httpx_mock: HTTPXMock): # type: ignore[no-untyped-def] + expected_headers = {"content-type": "image/svg+xml", "content-length": "1853"} + fetcher = MetadataFetcher() + result = await fetcher.gen_fetch_mime_type_and_size( + "data:image/svg+xml;base64,PHN2ZyBpZD0iaDNpNXR6IiB2aWV3Qm94PSIwIDAgNDggNDgiIHdpZHRoPSIxMDAlIiBoZWlnaHQ9IjEwMCUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiIHZlcnNpb249IjEuMSIgPgoKPGltYWdlIHg9IjAiIHk9IjAiIHdpZHRoPSI0OCIgaGVpZ2h0PSI0OCIgaW1hZ2UtcmVuZGVyaW5nPSJwaXhlbGF0ZWQiIHByZXNlcnZlQXNwZWN0UmF0aW89InhNaWRZTWlkIiB4bGluazpocmVmPSJkYXRhOmltYWdlL3BuZztiYXNlNjQsaVZCT1J3MEtHZ29BQUFBTlNVaEVVZ0FBQUZBQUFBQlFDQU1BQUFDNXp3S2ZBQUFBWUZCTVZFVlFZWHIveUFFK1QxNy9rZ0I0YVdJUkVST3RwcHNhSGl3QUFBQVhtc3pKeGJ4MFhCUndxY2IvN2dKbGJIOWZiNGE1NWQ3L3RBRFBsdzFyWEZmOTVpMFBEQXVBa3F5bGhqTFQ4KzdLeU1jL1BUamh3ai8yeHk0VmFvcU1yc0t4MTlDSW1CMllBQUFENUVsRVFWUjRBZTJVaVpLak9CQkVzUllKdWZFMUFsbzJxSGYrL3k4M2xTQUt4Z1NodnZaKzl1QjB0WGxSTlFVVXh4dzBPZWJ3VndqZnRQYXRjNjJuOHROQ3JmczJ1RGJpUXFYMXA0VlYxWWVCd3VDSHZxbytLOVFoQk9kYmp1emNFSUwrckhCd2JoYUNyeEhDQ0w1VzZCeU1meFBoMjRwbjRmcnZHVUt0dXlYWXNnanhwVnVpZFphd1hLSjFQL2cyNG9laDBsMjVKRmRvU29NM1hnME9UU08zWHNNcUQvekZ4NFR4UkVLWENKdHM0V0NDTVVOOEJZUE1aUHhjR0ppWU00V3RPWnVoTlhpZERUTFRNSnhUb1dYQnhFS204R0VleHNURFN6dzhYaGI1bDBKMmg2M0JtNzFJUTVLSHVmRFJwVWpoZzF2ZW90bW81QXFiYkRLRm9OTnJ1bzBLeUJJU3JWNWVTdklTa2ZoSXNWTkhJVk40dTkvdm8xRGlnL0VEd2g1YnZOOXV0OUtBcmZpUkRwdXlhV0piaHBFalB3eHFqQjhRR2hLRkVwa1l1MG9wdFNWVUU4ZlQ4U1FaUWdGM1dZb1BrMkxYRjBYQnM4YVhPaDFWcEpoUXNhb3VCSm4vaDhTQXJZZ09TVExDemRObllheE9RdVNNTGZmcFBPbVF3bGVDRG9HNi9IN0JHNWxDN3BOQ2lZOFVPeVhuUlJSUGx3N1hRcFV4OGxxb2tFUjRuWVZFcVNJS2Nha0FXaHJDeUNLRWFlVGtROXdhbWVDdjNMS1p0N3dSSUdRanlRZHcrdWJJcXNnU3lzajAwWTdUeDdRV29tdU0vSXhlOElzUUNXOEl0MGFtVUsyYll0VGxqT2JJWUMza3lQWUpyZTBtdXJ6ZDd2RXl2RVdoQnJOd1JJRTlZZmNrL0hrREN5R2FJbHBYVlRVSlgrMTV4dHJ4T0grZUZ5UWhMMnh0dXFydm81Q2QxYlgzM3RweEtmWXdBd09QbHJYeFk5M2g2R09IVVFnWHNiWHRPdS8xaFVJNVAwTkkzOCtGOEFRZ3JQRjNYMWZUbHUwUE1BcW53QUkvVmtMNjBLV01EQ0ZNdnF2eHo5WTloSkYwdnFLUWlQQjVLVEx5cTZvNnVHcThmVHpPUWs1STRVSGd5SW5uTFhOa0gyMzFBVG9MMUNXTi9Cc1loUWdhVEFVZUU3TGxKTFNqejBaaHo1dDVzOE9kcFNRbW9jZWM0T0FHNSsxOFlZTjFoeFpJaDV1WERaZGlhL2JYTzNEQXpKT1FJK2NMNmJ1elE0ditJaitDRFhYd2ZubzQ3SGU0WUxwc1pDbnNUOGVKd3dGeEpSenBsTjFsTGJSUmlONEc2MXpnYmdyWjhqc3BTN3ZBKytDQ0NLWERmTFJlQ2V2Z1hBMm1od01XdmZ1UTRRMXVsMVg1RXJIOVdRY0lBWVNLdTk2OS9KTHdJSnlYMzZ3N09EY0tMVnp3dldVTDdlSTNFZ2JzcEZvSXdlNURCc0tOb2xMekl3cjlVamNKSXhTU2ZTRklGbmxFMWRRUlcwenNQMlE0OHJwS0lkTzJjUGNoQStIcTYvZ2J1YjNxbWI1SVpDeEZPSUJWaHdrdnd0MW5Bb1NwdU5laGw0RnpoWUpkQ24yMDRTQSt0U0VVWnFGVXJRaVpjRWltQ2Z0dVlKQzBnZnhxZzJMMUlSYUpHK1FJcDArUmZLOVFGZC9GRlUvWjYrWDZQaTQ0YTVtRmExRjlNZjhML3hmK0hZUi9BRVJPMzlYOE5Fb1VBQUFBQUVsRlRrU3VRbUNDIi8+Cgo8L3N2Zz4=" # noqa + ) + assert result == ( + expected_headers["content-type"], + expected_headers["content-length"], + ) diff --git a/tests/metadata/pipelines/test_metadata_pipeline.py b/tests/metadata/pipelines/test_metadata_pipeline.py index a25478a..e007f0f 100644 --- a/tests/metadata/pipelines/test_metadata_pipeline.py +++ b/tests/metadata/pipelines/test_metadata_pipeline.py @@ -1,5 +1,6 @@ # flake8: noqa: E501 +from typing import Tuple from unittest.mock import AsyncMock, MagicMock import pytest @@ -21,6 +22,8 @@ AdapterConfig, MetadataPipeline, ) +from offchain.web3.contract_caller import ContractCaller +from offchain.web3.jsonrpc import EthereumJSONRPC class TestMetadataPipeline: @@ -159,6 +162,76 @@ def test_metadata_pipeline_fetch_token_metadata(self, raw_crypto_coven_metadata) ], ) + @pytest.mark.asyncio + async def test_metadata_pipeline_async_fetch_video_mime_type_and_size( + self, mock_video_rawdata + ): + token = Token( + chain_identifier="ZORA-MAINNET", + collection_address="0x7d256d82b32d8003d1ca1a1526ed211e6e0da9e2", + token_id="11539", + uri="https://a.y.at/nft_transfers/metadata/11539", + ) + + fetcher = MetadataFetcher() + fetcher.gen_fetch_content = AsyncMock(return_value=mock_video_rawdata) + + def mock_video_mime_type_and_size(uri: str) -> Tuple[str, int]: + if uri == "https://a.y.at/nft_transfers/metadata/11539": + return ("application/json", 0) + elif ( + uri + == "https://y.at/viz/money-mouth/money-mouth.point.heart-eyes.crown.ring-2ba3b7.mp4" + ): + return ("video/mp4", 5065775) + else: + return ("image/png", 207474) + + fetcher.gen_fetch_mime_type_and_size = AsyncMock( + side_effect=mock_video_mime_type_and_size + ) + pipeline = MetadataPipeline(fetcher=fetcher) + metadata = await pipeline.async_run(tokens=[token]) + + assert metadata[0].mime_type == "video/mp4" + assert metadata[0].content.mime_type == "video/mp4" + assert metadata[0].content.size == 5065775 + assert metadata[0].image.mime_type == "image/png" + assert metadata[0].image.size == 207474 + + @pytest.mark.asyncio + async def test_metadata_pipeline_async_fetch_image_mime_type_and_size( + self, mock_image_rawdata + ): + token = Token( + chain_identifier="ZORA-MAINNET", + collection_address="0x2781b28934943f51a8a08375d6f95e0208d7150e", + token_id="929", + uri="https://ipfs.io/ipfs/QmRVtPDV4JSsPRhec2djxr2qpx6ex6xrgYTWKKfAbS3u3b", + ) + + fetcher = MetadataFetcher() + fetcher.gen_fetch_content = AsyncMock(return_value=mock_image_rawdata) + + def mock_image_mime_type_and_size(uri: str) -> Tuple[str, int]: + if ( + uri + == "https://ipfs.io/ipfs/QmRVtPDV4JSsPRhec2djxr2qpx6ex6xrgYTWKKfAbS3u3b" + ): + return ("application/json", 0) + else: + return ("image/png", 2887641) + + fetcher.gen_fetch_mime_type_and_size = AsyncMock( + side_effect=mock_image_mime_type_and_size + ) + pipeline = MetadataPipeline(fetcher=fetcher) + metadata = await pipeline.async_run(tokens=[token]) + + assert metadata[0].mime_type == "image/png" + assert metadata[0].image.mime_type == "image/png" + assert metadata[0].image.size == 2887641 + def test_metadata_pipeline_run(self, raw_crypto_coven_metadata): # type: ignore[no-untyped-def] token = Token( chain_identifier="ETHEREUM-MAINNET",