Merge pull request #19 from secynic/dev

v0.4.0

.gitignore

@@ -33,4 +33,6 @@ nosetests.xml
 # Mr Developer
 .mr.developer.cfg
 .project
-.pydevproject
+.pydevproject
+
+MANIFEST

CHANGES.rst

@@ -1,6 +1,13 @@
 Changelog
 =========
 
+0.4.0 (2013-10-17)
+------------------
+
+- Added support for network registered and updated time stamps (keys: created, updated). Value in ISO 8601 format.
+- Added value assertion to test_utils.py.
+- Fixed IPWhois.lookup() handling of processed values. If processing throws an exception, discard the value and not the net dictionary.
+
 0.3.0 (2013-09-30)
 ------------------
 

README.rst

@@ -31,12 +31,14 @@ Typical usage::
 	          'cidr': '74.125.0.0/16',
 	          'city': 'Mountain View',
 	          'country': 'US',
+	          'created': '2007-03-13T00:00:00',
 	          'description': 'Google Inc.',
 	          'misc_emails': None,
 	          'name': 'GOOGLE',
 	          'postal_code': '94043',
 	          'state': 'CA',
-	          'tech_emails': '[email protected]'}],
+	          'tech_emails': '[email protected]',
+	          'updated': '2012-02-24T00:00:00'}],
 	'query': '74.125.225.229',
 	'raw': None
 	}
@@ -61,12 +63,14 @@ REST (HTTP)::
 	          'cidr': '74.125.0.0/16',
 	          'city': 'Mountain View',
 	          'country': 'US',
+	          'created': '2007-03-13T12:09:54-04:00',
 	          'description': 'Google Inc.',
 	          'misc_emails': None,
 	          'name': 'GOOGLE',
 	          'postal_code': '94043',
 	          'state': 'CA',
-	          'tech_emails': '[email protected]'}],
+	          'tech_emails': '[email protected]',
+	          'updated': '2012-02-24T09:44:34-05:00'}],
 	'query': '74.125.225.229',
 	'raw': None
 	}
@@ -116,11 +120,11 @@ Latest version from GitHub::
 Parsing
 =======
 
-Parsing is currently limited to CIDR, country, name, description, state, city, address, postal_code, abuse_emails, tech_emails, and misc_emails fields. This is assuming that those fields are present.
+Parsing is currently limited to CIDR, country, name, description, state, city, address, postal_code, abuse_emails, tech_emails, misc_emails, created and updated fields. This is assuming that those fields are present.
 
 Some IPs have parent networks listed. The parser attempts to recognize this, and break the networks into individual dictionaries. If a single network has multiple CIDRs, they will be separated by ', '.
 
-Sometimes, you will see whois information with multiple consecutive same name fields, e.g., Description: some text\\nDescription: more text. The parser will recognize this and the returned result will have these separated by '\\n'.
+Sometimes, you will see whois information with multiple consecutive same name fields, e.g., Description: some text\\nDescription: more text. The parser will recognize this and the returned result will have the values separated by '\\n'.
 
 REST (HTTP)
 ===========

ipwhois/__init__.py

@@ -21,6 +21,6 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-__version__ = '0.3.0'
+__version__ = '0.4.0'
 
 from .ipwhois import IPWhois, IPDefinedError, ASNLookupError, WhoisLookupError, HostLookupError

ipwhois/ipwhois.py

@@ -25,6 +25,7 @@
 from .utils import ipv4_is_defined, ipv6_is_defined
 from urllib import request
 from time import sleep
+from datetime import datetime
 
 #Import the dnspython3 rdtypes to resolve the dynamic import problem when frozen to exe.
 import dns.rdtypes.ANY.TXT
@@ -63,8 +64,12 @@
                                 'address': '^(Address):[^\S\n]+(?P<val>.+)$',
                                 'postal_code': '^(PostalCode):[^\S\n]+(?P<val>.+)$',
                                 'abuse_emails': '^(OrgAbuseEmail):[^\S\n]+(?P<val>.+)$',
-                                'tech_emails': '^(OrgTechEmail):[^\S\n]+(?P<val>.+)$'
-                                }
+                                'tech_emails': '^(OrgTechEmail):[^\S\n]+(?P<val>.+)$',
+                                'created': '^(RegDate):[^\S\n]+(?P<val>.+)$',
+                                'updated': '^(Updated):[^\S\n]+(?P<val>.+)$'
+                                },
+                     'dt_format': '%Y-%m-%d',
+                     'dt_rws_format': '%Y-%m-%dT%H:%M:%S%z'
                      },
             'ripencc': {
                      'server': 'whois.ripe.net',
@@ -87,8 +92,10 @@
                                 'country': '^(country):[^\S\n]+(?P<val>.+)$',
                                 'address': '^(address):[^\S\n]+(?P<val>.+)$',
                                 'abuse_emails': '^(abuse-mailbox:[^\S\n]+(?P<val>.+))|((?!abuse-mailbox).+?:.*[^\S\n]+(?P<val2>[\w\-\.]*abuse[\w\-\.]*@[\w\-\.]+\.[\w\-]+)([^\S\n]+.*)*)$',
-                                'misc_emails': '^(?!abuse-mailbox).+?:.*[^\S\n]+(?P<val>(?!abuse)[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)([^\S\n]+.*)*$'
-                                }
+                                'misc_emails': '^(?!abuse-mailbox).+?:.*[^\S\n]+(?P<val>(?!abuse)[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)([^\S\n]+.*)*$',
+                                'updated': '^(changed):[^\S\n]+.*?(?P<val>[0-9]{8})$'
+                                },
+                     'dt_format': '%Y%m%d'
                      },
             'lacnic': {
                      'server': 'whois.lacnic.net',
@@ -97,8 +104,11 @@
                                 'description': '^(owner):[^\S\n]+(?P<val>.+)$',
                                 'country': '^(country):[^\S\n]+(?P<val>.+)$',
                                 'abuse_emails': '^(abuse-mailbox:[^\S\n]+(?P<val>.+))|((?!abuse-mailbox).+?:.*[^\S\n]+(?P<val2>[\w\-\.]*abuse[\w\-\.]*@[\w\-\.]+\.[\w\-]+)([^\S\n]+.*)*)$',
-                                'misc_emails': '^(?!abuse-mailbox).+?:.*[^\S\n]+(?P<val>(?!abuse)[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)([^\S\n]+.*)*$'
-                                }
+                                'misc_emails': '^(?!abuse-mailbox).+?:.*[^\S\n]+(?P<val>(?!abuse)[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)([^\S\n]+.*)*$',
+                                'created': '^(created):[^\S\n]+(?P<val>[0-9]{8}).*$',
+                                'updated': '^(changed):[^\S\n]+(?P<val>[0-9]{8}).*$'
+                                },
+                     'dt_format': '%Y%m%d'
                      },
             'afrinic': {
                      'server': 'whois.afrinic.net',
@@ -536,6 +546,8 @@ def lookup(self, inc_raw = False, retry_count = 3):
               'abuse_emails': None,
               'tech_emails': None,
               'misc_emails': None,
+              'created': None,
+              'updated': None,
               'start': None,
               'end': None
               }
@@ -660,14 +672,25 @@ def lookup(self, inc_raw = False, retry_count = 3):
 
                 if len(values) > 0:
 
-                    if field == 'country':
-
-                        value = values[0].upper()
+                    try:
 
-                    else:
+                        if field == 'country':
+
+                            value = values[0].upper()
+
+                        elif field in ['created', 'updated']:
+
+                            value = datetime.strptime(values[0], NIC_WHOIS[results['asn_registry']]['dt_format']).isoformat('T')
+
+                        else:
+
+                            values = list(set(values))
+                            value = '\n'.join(values)
+
+                    except:
 
-                        values = list(set(values))
-                        value = '\n'.join(values)
+                        value = None
+                        pass
 
                     net[field] = value
 
@@ -751,7 +774,9 @@ def lookup_rws(self, inc_raw = False, retry_count = 3):
               'postal_code': None,
               'abuse_emails': None,
               'tech_emails': None,
-              'misc_emails': None
+              'misc_emails': None,
+              'created': None,
+              'updated': None
               }
 
         nets = []
@@ -778,6 +803,14 @@ def lookup_rws(self, inc_raw = False, retry_count = 3):
                     net = base_net.copy()
                     net['cidr'] = ', '.join([i.__str__() for i in ipaddress.collapse_addresses(addrs)])
 
+                    if 'registrationDate' in n:
+
+                        net['created'] = n['registrationDate']['$'].strip()
+
+                    if 'updateDate' in n:
+
+                        net['updated'] = n['updateDate']['$'].strip()
+
                     if 'name' in n:
 
                         net['name'] = n['name']['$'].strip()

ipwhois/tests/test_utils.py

@@ -4,7 +4,9 @@
 class TestFunctions(unittest.TestCase):
 
     def test_get_countries(self):
-        self.assertIsInstance(get_countries(), dict)
+        countries = get_countries()
+        self.assertIsInstance(countries, dict)
+        self.assertEqual(countries['US'], 'United States')
 
     def test_ipv4_is_defined(self):
         from ipaddress import AddressValueError