From a59d7c99402ff042d7fe53a4e974c12cabaea75d Mon Sep 17 00:00:00 2001 From: secynic Date: Thu, 13 Feb 2014 15:07:14 -0600 Subject: [PATCH 1/2] v0.8.0 --- CHANGES.rst | 11 ++ README.rst | 6 +- ipwhois/__init__.py | 4 +- ipwhois/ipwhois.py | 342 +++++++++++++++++++++------------- ipwhois/tests/test_ipwhois.py | 12 +- 5 files changed, 234 insertions(+), 141 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 72077ad..026b92d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,17 @@ Changelog ========= +0.8.0 (TBD) +------------------ + +- Added ASNRegistryError to handle unknown ASN registry return values. +- Added ASN registry lookup third tier fallback to ARIN. +- Fixed variable naming to avoid shadows built-in confusion. +- Fixed some type errors: Expected type 'str', got 'dict[str, dict]' instead. +- Fixed RIPE RWS links, since they changed their API. +- Temporarily removed RIPE RWS functionality until they fix their API. +- Removed RADB RIPE fallback, since they appeared to have removed it. + 0.7.0 (2014-01-14) ------------------ diff --git a/README.rst b/README.rst index eb2bda1..da3890a 100644 --- a/README.rst +++ b/README.rst @@ -152,4 +152,8 @@ IPWhois.lookup_rws() should be faster than IPWhois.lookup(), but may not be as reliable. AFRINIC does not have a Whois-RWS service yet. We have to rely on the Ripe RWS service, which does not contain all of the data we need. The LACNIC RWS service is supported, but is in beta v2. This may result in availability -or performance issues. \ No newline at end of file +or performance issues. + +**NOTE** +RIPE RWS functionality is currently disabled until their API is fixed: +https://github.com/RIPE-NCC/whois/issues/114 \ No newline at end of file diff --git a/ipwhois/__init__.py b/ipwhois/__init__.py index 730c24a..d10ea43 100644 --- a/ipwhois/__init__.py +++ b/ipwhois/__init__.py @@ -22,7 +22,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -__version__ = '0.7.0' +__version__ = '0.8.0' from .ipwhois import (IPWhois, IPDefinedError, ASNLookupError, - WhoisLookupError, HostLookupError) + ASNRegistryError, WhoisLookupError, HostLookupError) diff --git a/ipwhois/ipwhois.py b/ipwhois/ipwhois.py index a5b1007..5e884a4 100644 --- a/ipwhois/ipwhois.py +++ b/ipwhois/ipwhois.py @@ -107,10 +107,7 @@ }, 'ripencc': { 'server': 'whois.ripe.net', - 'url': ( - 'http://apps.db.ripe.net/whois/grs-search?' - 'query-string={0}&source=ripe-grs' - ), + 'url': 'http://rest.db.ripe.net/search.json?query-string={0}', 'fields': { 'name': r'^(netname):[^\S\n]+(?P.+)$', 'description': r'^(descr):[^\S\n]+(?P.+)$', @@ -172,10 +169,7 @@ }, 'afrinic': { 'server': 'whois.afrinic.net', - 'url': ( - 'http://apps.db.ripe.net/whois/grs-search?' - 'query-string={0}&source=afrinic-grs' - ), + 'url': 'http://rest.db.ripe.net/search.json?query-string={0}', 'fields': { 'name': r'^(netname):[^\S\n]+(?P.+)$', 'description': r'^(descr):[^\S\n]+(?P.+)$', @@ -194,6 +188,13 @@ } } +ASN_REFERRALS = { + 'whois://whois.ripe.net': 'ripencc', + 'whois://whois.apnic.net': 'apnic', + 'whois://whois.lacnic.net': 'lacnic', + 'whois://whois.afrinic.net': 'afrinic', +} + CYMRU_WHOIS = 'whois.cymru.com' IPV4_DNS_ZONE = '{0}.origin.asn.cymru.com' @@ -229,6 +230,13 @@ class ASNLookupError(Exception): """ +class ASNRegistryError(Exception): + """ + An Exception for when the ASN registry does not match one of the five + expected values (arin, ripencc, apnic, lacnic, afrinic). + """ + + class WhoisLookupError(Exception): """ An Exception for when the Whois lookup failed. @@ -289,7 +297,8 @@ def __init__(self, address, timeout=5, proxy_opener=None): if is_defined[0]: raise IPDefinedError( - 'IPv4 address %r is already defined as %r via %r.' % ( + 'IPv4 address %r is already defined as %r via ' + '%r.' % ( self.address_str, is_defined[1], is_defined[2] ) ) @@ -309,7 +318,8 @@ def __init__(self, address, timeout=5, proxy_opener=None): if is_defined[0]: raise IPDefinedError( - 'IPv6 address %r is already defined as %r via %r.' % ( + 'IPv6 address %r is already defined as %r via ' + '%r.' % ( self.address_str, is_defined[1], is_defined[2] ) ) @@ -359,6 +369,7 @@ def get_asn_dns(self): asn_country_code (String) - The assigned ASN country code. Raises: + ASNRegistryError: The ASN registry is not known. ASNLookupError: The ASN lookup failed. """ @@ -373,7 +384,9 @@ def get_asn_dns(self): if ret['asn_registry'] not in NIC_WHOIS.keys(): - return None + raise ASNRegistryError( + 'ASN registry %r is not known.' % ret['asn_registry'] + ) ret['asn'] = temp[0].strip(' "\n') ret['asn_cidr'] = temp[1].strip(' \n') @@ -382,6 +395,10 @@ def get_asn_dns(self): return ret + except ASNRegistryError: + + raise + except: raise ASNLookupError( @@ -406,6 +423,7 @@ def get_asn_whois(self, retry_count=3): asn_country_code (String) - The assigned ASN country code. Raises: + ASNRegistryError: The ASN registry is not known. ASNLookupError: The ASN lookup failed. """ @@ -440,7 +458,9 @@ def get_asn_whois(self, retry_count=3): if ret['asn_registry'] not in NIC_WHOIS.keys(): - return None + raise ASNRegistryError( + 'ASN registry %r is not known.' % ret['asn_registry'] + ) ret['asn'] = temp[0].strip(' \n') ret['asn_cidr'] = temp[2].strip(' \n') @@ -461,6 +481,10 @@ def get_asn_whois(self, retry_count=3): 'ASN lookup failed for %r.' % self.address_str ) + except ASNRegistryError: + + raise + except: raise ASNLookupError( @@ -575,7 +599,8 @@ def get_rws(self, url=None, retry_count=3): else: - raise WhoisLookupError('Whois RWS lookup failed for %r.' % url) + raise WhoisLookupError('Whois RWS lookup failed for %r.' % + url) except: @@ -657,14 +682,58 @@ def lookup(self, inc_raw=False, retry_count=3): is True. """ + #Initialize the response. + response = None + #Attempt to resolve ASN info via Cymru. DNS is faster, try that first. try: asn_data = self.get_asn_dns() - except ASNLookupError: + except (ASNLookupError, ASNRegistryError): + + try: + + asn_data = self.get_asn_whois(retry_count) + + except (ASNLookupError, ASNRegistryError): - asn_data = self.get_asn_whois(retry_count) + #Lets attempt to get the ASN registry information from ARIN. + response = self.get_whois('arin', retry_count) + + asn_data = { + 'asn_registry': None, + 'asn': None, + 'asn_cidr': None, + 'asn_country_code': None, + 'asn_date': None + } + + matched = False + for match in re.finditer( + r'^ReferralServer:[^\S\n]+(.+)$', + response, + re.MULTILINE + ): + + matched = True + + try: + + referral = match.group(1) + referral = referral.replace(':43', '') + + asn_data['asn_registry'] = ASN_REFERRALS[referral] + + except KeyError: + + raise ASNRegistryError('ASN registry lookup failed.') + + break + + if not matched: + + asn_data['asn_registry'] = 'arin' #Create the return dictionary. results = { @@ -676,8 +745,11 @@ def lookup(self, inc_raw=False, retry_count=3): #Add the ASN information to the return dictionary. results.update(asn_data) - #Retrieve the whois data. - response = self.get_whois(results['asn_registry'], retry_count) + #Only fetch the response if we haven't already. + if response is None or results['asn_registry'] is not 'arin': + + #Retrieve the whois data. + response = self.get_whois(results['asn_registry'], retry_count) #If inc_raw parameter is True, add the response to return dictionary. if inc_raw: @@ -788,37 +860,37 @@ def lookup(self, inc_raw=False, retry_count=3): #appropriate fields for each. for index, net in enumerate(nets): - end = None + section_end = None if index + 1 < len(nets): - end = nets[index + 1]['start'] + section_end = nets[index + 1]['start'] for field in NIC_WHOIS[results['asn_registry']]['fields']: pattern = re.compile( - NIC_WHOIS[results['asn_registry']]['fields'][field], + str(NIC_WHOIS[results['asn_registry']]['fields'][field]), re.MULTILINE ) - if end is not None: + if section_end is not None: - match = pattern.finditer(response, net['end'], end) + match = pattern.finditer(response, net['end'], section_end) else: match = pattern.finditer(response, net['end']) values = [] - sub_end = None + sub_section_end = None for m in match: - if sub_end: + if sub_section_end: if field not in ( 'abuse_emails', 'tech_emails', 'misc_emails' - ) and (sub_end != (m.start() - 1)): + ) and (sub_section_end != (m.start() - 1)): break @@ -830,7 +902,7 @@ def lookup(self, inc_raw=False, retry_count=3): values.append(m.group('val2').strip()) - sub_end = m.end() + sub_section_end = m.end() if len(values) > 0: @@ -844,8 +916,8 @@ def lookup(self, inc_raw=False, retry_count=3): value = datetime.strptime( values[0], - NIC_WHOIS[results['asn_registry']] - ['dt_format']).isoformat('T') + str(NIC_WHOIS[results['asn_registry']] + ['dt_format'])).isoformat('T') else: @@ -921,29 +993,19 @@ def _lookup_rws_arin(self, response=None, retry_count=3): pass - try: - - net['created'] = str(n['registrationDate']['$']).strip() - - except KeyError: + for k, v in { + 'created': 'registrationDate', + 'updated': 'updateDate', + 'name': 'name' + }.items(): - pass - - try: - - net['updated'] = str(n['updateDate']['$']).strip() - - except KeyError: - - pass - - try: + try: - net['name'] = str(n['name']['$']).strip() + net[k] = str(n[v]['$']).strip() - except KeyError: + except KeyError: - pass + pass ref = None if 'customerRef' in n: @@ -992,23 +1054,19 @@ def _lookup_rws_arin(self, response=None, retry_count=3): pass - try: - - net['postal_code'] = ( - str(ref_response[ref[1]]['postalCode']['$']) - ) - - except KeyError: - - pass + for k, v in { + 'postal_code': 'postalCode', + 'city': 'city', + 'state': 'iso3166-2' + }.items(): - try: + try: - net['city'] = str(ref_response[ref[1]]['city']['$']) + net[k] = str(ref_response[ref[1]][v]['$']) - except KeyError: + except KeyError: - pass + pass try: @@ -1020,16 +1078,6 @@ def _lookup_rws_arin(self, response=None, retry_count=3): pass - try: - - net['state'] = ( - str(ref_response[ref[1]]['iso3166-2']['$']) - ) - - except KeyError: - - pass - try: for poc in ( @@ -1075,6 +1123,11 @@ def _lookup_rws_ripe(self, response=None): The function for retrieving and parsing whois information for a RIPE IP address via HTTP (Whois-RWS). + *** + THIS FUNCTION IS TEMPORARILY BROKEN UNTIL RIPE FIXES THEIR API: + https://github.com/RIPE-NCC/whois/issues/114 + *** + Args: response: The dictionary containing whois information to parse. @@ -1087,13 +1140,9 @@ def _lookup_rws_ripe(self, response=None): nets = [] - try: - - object_list = response['whois-resources']['objects']['object'] + '''try: - if not isinstance(object_list, list): - - object_list = [object_list] + object_list = response['objects'] except KeyError: @@ -1102,11 +1151,13 @@ def _lookup_rws_ripe(self, response=None): ripe_abuse_emails = [] ripe_misc_emails = [] + net = BASE_NET.copy() + for n in object_list: try: - if n['type'] == 'organisation': + if n['type'] == 'role': for attr in n['attributes']['attribute']: @@ -1120,9 +1171,19 @@ def _lookup_rws_ripe(self, response=None): ripe_misc_emails.append(str(attr['value']).strip()) - elif n['type'] in ('inetnum', 'inet6num', 'route', 'route6'): + elif attr['name'] == 'address': - net = BASE_NET.copy() + if net['address'] is not None: + + net['address'] += '\n%s' % ( + str(attr['value']).strip() + ) + + else: + + net['address'] = str(attr['value']).strip() + + elif n['type'] in ('inetnum', 'inet6num'): for attr in n['attributes']['attribute']: @@ -1158,21 +1219,6 @@ def _lookup_rws_ripe(self, response=None): pass - elif attr['name'] in ('route', 'route6'): - - ipr = str(attr['value']).strip() - ip_ranges = ipr.split(', ') - - try: - - net['cidr'] = ', '.join( - ip_network(r).__str__() for r in ip_ranges - ) - - except ValueError: - - pass - elif attr['name'] == 'netname': net['name'] = str(attr['value']).strip() @@ -1193,24 +1239,12 @@ def _lookup_rws_ripe(self, response=None): net['country'] = str(attr['value']).strip().upper() - elif attr['name'] == 'address': - - if net['address'] is not None: - - net['address'] += '\n%s' % ( - str(attr['value']).strip() - ) - - else: - - net['address'] = str(attr['value']).strip() - - nets.append(net) - except KeyError: pass + nets.append(net) + #This is nasty. Since RIPE RWS doesn't provide a granular #contact to network relationship, we apply to all networks. if len(ripe_abuse_emails) > 0 or len(ripe_misc_emails) > 0: @@ -1227,7 +1261,7 @@ def _lookup_rws_ripe(self, response=None): for net in nets: net['abuse_emails'] = abuse - net['misc_emails'] = misc + net['misc_emails'] = misc''' return nets @@ -1452,7 +1486,7 @@ def _lookup_rws_lacnic(self, response=None): value = datetime.strptime( tmp, - NIC_WHOIS['lacnic']['dt_rws_format'] + str(NIC_WHOIS['lacnic']['dt_rws_format']) ).isoformat('T') net['created'] = value @@ -1463,7 +1497,7 @@ def _lookup_rws_lacnic(self, response=None): value = datetime.strptime( tmp, - NIC_WHOIS['lacnic']['dt_rws_format'] + str(NIC_WHOIS['lacnic']['dt_rws_format']) ).isoformat('T') net['updated'] = value @@ -1568,14 +1602,72 @@ def lookup_rws(self, inc_raw=False, retry_count=3): inc_raw parameter is True. """ + #Initialize the response. + response = None + #Attempt to resolve ASN info via Cymru. DNS is faster, try that first. try: asn_data = self.get_asn_dns() - except ASNLookupError: + except (ASNLookupError, ASNRegistryError): + + try: + + asn_data = self.get_asn_whois(retry_count) + + except (ASNLookupError, ASNRegistryError): + + #Lets attempt to get the ASN registry information from ARIN. + response = self.get_rws( + str(NIC_WHOIS['arin']['url']).format(self.address_str), + retry_count + ) + + asn_data = { + 'asn_registry': None, + 'asn': None, + 'asn_cidr': None, + 'asn_country_code': None, + 'asn_date': None + } + + try: + + net_list = response['nets']['net'] + + if not isinstance(net_list, list): + + net_list = [net_list] + + except KeyError: + + net_list = [] + + for n in net_list: + + try: + + if n['orgRef']['@handle'] in ('ARIN', 'VR-ARIN'): + + asn_data['asn_registry'] = 'arin' + + elif n['orgRef']['@handle'] == 'RIPE': + + asn_data['asn_registry'] = 'ripencc' + + else: + + test = NIC_WHOIS[n['orgRef']['@handle'].lower()] + asn_data['asn_registry'] = ( + n['orgRef']['@handle'].lower() + ) + + except KeyError: - asn_data = self.get_asn_whois(retry_count) + raise ASNRegistryError('ASN registry lookup failed.') + + break #Create the return dictionary. results = { @@ -1587,36 +1679,22 @@ def lookup_rws(self, inc_raw=False, retry_count=3): #Add the ASN information to the return dictionary. results.update(asn_data) - #Create the boolean for if the response is a radb-grs search. - is_radb = False - - #Retrieve the whois data. - try: + #Only fetch the response if we haven't already. + if response is None or results['asn_registry'] is not 'arin': + #Retrieve the whois data. response = self.get_rws( - NIC_WHOIS[results['asn_registry']]['url'].format( + str(NIC_WHOIS[results['asn_registry']]['url']).format( self.address_str), retry_count ) - #If the query failed, try the radb-grs source. - except WhoisLookupError: - - is_radb = True - - response = self.get_rws(( - 'http://apps.db.ripe.net/whois/grs-search' - '?query-string={0}&source=radb-grs').format(self.address_str), - retry_count - ) - #If inc_raw parameter is True, add the response to return dictionary. if inc_raw: results['raw'] = response - if (results['asn_registry'] in ('ripencc', 'afrinic') or - is_radb is True): + if results['asn_registry'] in ('ripencc', 'afrinic'): nets = self._lookup_rws_ripe(response) diff --git a/ipwhois/tests/test_ipwhois.py b/ipwhois/tests/test_ipwhois.py index 769077a..d69be00 100644 --- a/ipwhois/tests/test_ipwhois.py +++ b/ipwhois/tests/test_ipwhois.py @@ -1,6 +1,6 @@ import unittest -from ipwhois import (IPWhois, IPDefinedError, ASNLookupError, WhoisLookupError, - HostLookupError) +from ipwhois import (IPWhois, IPDefinedError, ASNLookupError, ASNRegistryError, + WhoisLookupError, HostLookupError) class TestIPWhois(unittest.TestCase): @@ -49,7 +49,7 @@ def test_get_asn_dns(self): result = IPWhois('74.125.225.229') try: self.assertIsInstance(result.get_asn_dns(), dict) - except ASNLookupError: + except (ASNLookupError, ASNRegistryError): pass except AssertionError as e: raise e @@ -60,7 +60,7 @@ def test_get_asn_whois(self): result = IPWhois('74.125.225.229') try: self.assertIsInstance(result.get_asn_whois(), dict) - except ASNLookupError: + except (ASNLookupError, ASNRegistryError): pass except AssertionError as e: raise e @@ -122,7 +122,7 @@ def test_lookup(self): result = IPWhois(ip) try: self.assertIsInstance(result.lookup(), dict) - except (ASNLookupError, WhoisLookupError): + except (ASNLookupError, ASNRegistryError, WhoisLookupError): pass except AssertionError as e: raise e @@ -153,7 +153,7 @@ def test_lookup_rws(self): result = IPWhois(ip) try: self.assertIsInstance(result.lookup_rws(), dict) - except (ASNLookupError, WhoisLookupError): + except (ASNLookupError, ASNRegistryError, WhoisLookupError): pass except AssertionError as e: raise e From 99ba8de10a8150f6c25efa1f3c2e9fb57c7c647f Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 18 Feb 2014 08:45:29 -0600 Subject: [PATCH 2/2] v0.8.0 --- CHANGES.rst | 4 ++-- README.rst | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 026b92d..be4c833 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,7 +1,7 @@ Changelog ========= -0.8.0 (TBD) +0.8.0 (2014-02-18) ------------------ - Added ASNRegistryError to handle unknown ASN registry return values. @@ -10,7 +10,7 @@ Changelog - Fixed some type errors: Expected type 'str', got 'dict[str, dict]' instead. - Fixed RIPE RWS links, since they changed their API. - Temporarily removed RIPE RWS functionality until they fix their API. -- Removed RADB RIPE fallback, since they appeared to have removed it. +- Removed RADB fallback, since RIPE removed it. 0.7.0 (2014-01-14) ------------------ diff --git a/README.rst b/README.rst index da3890a..d8ea324 100644 --- a/README.rst +++ b/README.rst @@ -154,6 +154,8 @@ Ripe RWS service, which does not contain all of the data we need. The LACNIC RWS service is supported, but is in beta v2. This may result in availability or performance issues. -**NOTE** +*****NOTE***** + RIPE RWS functionality is currently disabled until their API is fixed: + https://github.com/RIPE-NCC/whois/issues/114 \ No newline at end of file