feat: --crawl for common crawl

cdx_toolkit/__init__.py

@@ -246,6 +246,8 @@ def get(self, url, **kwargs):
         validate_timestamps(params)
         params['url'] = url
         params['output'] = 'json'
+        if 'crawl' not in params:
+            params['crawl'] = self.crawl
         if 'filter' in params:
             if isinstance(params['filter'], str):
                 params['filter'] = (params['filter'],)
@@ -275,6 +277,8 @@ def iter(self, url, **kwargs):
         validate_timestamps(params)
         params['url'] = url
         params['output'] = 'json'
+        if 'crawl' not in params:
+            params['crawl'] = self.crawl
         if 'filter' in params:
             if isinstance(params['filter'], str):
                 params['filter'] = (params['filter'],)

cdx_toolkit/commoncrawl.py

@@ -133,6 +133,8 @@ def match_cc_crawls(crawls, raw_index_list):
     # match crawls requested on the command line to actual crawls
     # note that from/to are not considered here
     # crawls should be normalized so it's supposed to be a list of str
+    if crawls is None:
+        return raw_index_list
     if len(crawls) == 1 and crawls[0].isdigit():
         num = int(crawls[0])
         raw_index_list = raw_index_list[-num:]

tests/test_cc_crawl.py

@@ -0,0 +1,12 @@
+def test_cc_crawl():
+    # 'crawl' can be specified for a CDXFetcher, and later change in
+    # an iter() or get() call.
+
+    # init no crawl, iter/get no crawl
+    # init crawl, iter/get no crawl
+    # init no crawl, iter/get crawl
+    # init crawl, iter/get crawl
+
+    # this is a lot of work to test 4 lines of code :/
+    pass
+