Update whitelisting and blacklisting behavior (#53)

To avoid dropping useful header names this updates the SDK to allow headers by default, and that headers are scrubbed instead of dropped if a blacklist or whitelist is applied.

Note that User-Agent is always passed, since it is required by the API. Cookie and Authorization are always scrubbed for security reasons.

README.rst

@@ -21,7 +21,7 @@ import and configure the library with your Castle API secret.
 
 .. code:: python
 
-    from castle.configuration import configuration
+    from castle.configuration import configuration, WHITELISTED
 
     # Same as setting it through Castle.api_secret
     configuration.api_secret = ':YOUR-API-SECRET'
@@ -33,15 +33,15 @@ import and configure the library with your Castle API secret.
     configuration.request_timeout = 1000
 
     # Whitelisted and Blacklisted headers are case insensitive and allow to use _ and - as a separator, http prefixes are removed
+    # By default all headers are passed, but some are automatically scrubbed.
+    # If you need to apply a whitelist, we recommend using the minimum set of
+    # standard headers that we've exposed in the `WHITELISTED` constant.
     # Whitelisted headers
-    configuration.whitelisted = ['X_HEADER']
-    # or append to default
-    configuration.whitelisted = configuration.whitelisted + ['http-x-header']
+    configuration.whitelisted = WHITELISTED + ['X_HEADER']
 
-    # Blacklisted headers take advantage over whitelisted elements
+    # Blacklisted headers take advantage over whitelisted elements. Note that
+    # some headers are always scrubbed, for security reasons.
     configuration.blacklisted = ['HTTP-X-header']
-    # or append to default
-    configuration.blacklisted = configuration.blacklisted + ['X_HEADER']
 
 Tracking
 --------

castle/configuration.py

@@ -2,22 +2,26 @@
 from castle.headers_formatter import HeadersFormatter
 
 WHITELISTED = [
-    'User-Agent',
-    'Accept-Language',
-    'Accept-Encoding',
-    'Accept-Charset',
-    'Accept',
-    'Accept-Datetime',
-    'Forwarded',
-    'X-Forwarded',
-    'X-Real-IP',
-    'REMOTE_ADDR',
-    'X-Forwarded-For',
-    'CF_CONNECTING_IP'
+    "Accept",
+    "Accept-Charset",
+    "Accept-Datetime",
+    "Accept-Encoding",
+    "Accept-Language",
+    "Cache-Control",
+    "Connection",
+    "Content-Length",
+    "Content-Type",
+    "Cookie",
+    "Host",
+    "Origin",
+    "Pragma",
+    "Referer",
+    "TE",
+    "Upgrade-Insecure-Requests",
+    "User-Agent",
+    "X-Castle-Client-Id",
 ]
 
-BLACKLISTED = ['HTTP_COOKIE']
-
 # 500 milliseconds
 REQUEST_TIMEOUT = 500
 FAILOVER_STRATEGIES = ['allow', 'deny', 'challenge', 'throw']
@@ -29,8 +33,8 @@ def __init__(self):
         self.host = 'api.castle.io'
         self.port = 443
         self.url_prefix = '/v1'
-        self.whitelisted = WHITELISTED
-        self.blacklisted = BLACKLISTED
+        self.whitelisted = []
+        self.blacklisted = []
         self.request_timeout = REQUEST_TIMEOUT
         self.failover_strategy = 'allow'
 

castle/extractors/headers.py

@@ -1,6 +1,9 @@
 from castle.headers_formatter import HeadersFormatter
 from castle.configuration import configuration
 
+DEFAULT_BLACKLIST = ['Cookie', 'Authorization']
+DEFAULT_WHITELIST = ['User-Agent']
+
 
 class ExtractorsHeaders(object):
     def __init__(self, environ):
@@ -9,12 +12,15 @@ def __init__(self, environ):
 
     def call(self):
         headers = dict()
+        has_whitelist = len(configuration.whitelisted) > 0
 
         for key, value in self.environ.items():
             name = self.formatter.call(key)
-            if name not in configuration.whitelisted:
+            if has_whitelist and name not in configuration.whitelisted and name not in DEFAULT_WHITELIST:
+                headers[name] = True
                 continue
-            if name in configuration.blacklisted:
+            if name in configuration.blacklisted or name in DEFAULT_BLACKLIST:
+                headers[name] = True
                 continue
             headers[name] = value
 

castle/test/client_test.py

@@ -31,7 +31,11 @@ def test_init(self):
         context = {
             'active': True,
             'client_id': '1234',
-            'headers': {'User-Agent': 'test', 'X-Forwarded-For': '217.144.192.112'},
+            'headers': {
+                'User-Agent': 'test',
+                'X-Forwarded-For': '217.144.192.112',
+                'X-Castle-Client-Id': '1234'
+            },
             'ip': '217.144.192.112',
             'library': {'name': 'castle-python', 'version': VERSION},
             'origin': 'web',
@@ -189,7 +193,11 @@ def test_to_context(self):
         context = {
             'active': True,
             'client_id': '1234',
-            'headers': {'User-Agent': 'test', 'X-Forwarded-For': '217.144.192.112'},
+            'headers': {
+                'User-Agent': 'test',
+                'X-Forwarded-For': '217.144.192.112',
+                'X-Castle-Client-Id': '1234'
+            },
             'ip': '217.144.192.112',
             'library': {'name': 'castle-python', 'version': VERSION},
             'origin': 'web',

castle/test/configuration_test.py

@@ -1,6 +1,6 @@
 from castle.test import unittest
 from castle.exceptions import ConfigurationError
-from castle.configuration import Configuration, WHITELISTED, BLACKLISTED
+from castle.configuration import Configuration
 from castle.headers_formatter import HeadersFormatter
 
 
@@ -11,10 +11,8 @@ def test_default_values(self):
         self.assertEqual(config.host, 'api.castle.io')
         self.assertEqual(config.port, 443)
         self.assertEqual(config.url_prefix, '/v1')
-        self.assertEqual(config.whitelisted, [
-                         HeadersFormatter.call(v) for v in WHITELISTED])
-        self.assertEqual(config.blacklisted, [
-                         HeadersFormatter.call(v) for v in BLACKLISTED])
+        self.assertEqual(config.whitelisted, [])
+        self.assertEqual(config.blacklisted, [])
         self.assertEqual(config.request_timeout, 500)
         self.assertEqual(config.failover_strategy, 'allow')
 

castle/test/context/default_test.py

@@ -46,7 +46,7 @@ def test_default_context(self):
         self.assertEqual(context['client_id'], client_id())
         self.assertEqual(context['active'], True)
         self.assertEqual(context['origin'], 'web')
-        self.assertEqual(context['headers'], {'X-Forwarded-For': request_ip()})
+        self.assertEqual(context['headers'], {'X-Forwarded-For': request_ip(), 'Cookie': True})
         self.assertEqual(context['ip'], request_ip())
         self.assertDictEqual(context['library'], {
                              'name': 'castle-python', 'version': __version__})
@@ -59,8 +59,12 @@ def test_default_context_with_extras(self):
         self.assertEqual(context['origin'], 'web')
         self.assertEqual(
             context['headers'],
-            {'X-Forwarded-For': request_ip(), 'Accept-Language': 'en',
-             'User-Agent': 'test'}
+            {
+                'X-Forwarded-For': request_ip(),
+                'Accept-Language': 'en',
+                'User-Agent': 'test',
+                'Cookie': True
+            }
         )
         self.assertEqual(context['ip'], request_ip())
         self.assertDictEqual(

castle/test/extractors/headers_test.py

@@ -1,5 +1,5 @@
 from castle.test import unittest
-from castle.configuration import configuration
+from castle.configuration import configuration, WHITELISTED
 from castle.extractors.headers import ExtractorsHeaders
 
 
@@ -9,7 +9,7 @@ def client_id():
 
 def environ():
     return {
-        'HTTP_X_FORWARDED_FOR': '1.2.3.4',
+        'HTTP_USER_AGENT': 'requests',
         'HTTP_OK': 'OK',
         'TEST': '1',
         'HTTP_COOKIE': "__cid={client_id};other=efgh".format(client_id=client_id)
@@ -18,13 +18,14 @@ def environ():
 
 class ExtractorsHeadersTestCase(unittest.TestCase):
     def test_extract_headers(self):
+        configuration.whitelisted = []
         self.assertEqual(ExtractorsHeaders(environ()).call(),
-                         {'X-Forwarded-For': '1.2.3.4'})
+                         {'User-Agent': 'requests', 'Ok': 'OK', 'Test': '1', 'Cookie': True})
 
-    def test_extend_whitelisted_headers(self):
-        configuration.whitelisted += ['TEST']
+    def test_add_whitelisted_headers(self):
+        configuration.whitelisted = WHITELISTED + ['TEST']
         self.assertEqual(
             ExtractorsHeaders(environ()).call(),
-            {'X-Forwarded-For': '1.2.3.4', 'Test': '1'}
+            {'User-Agent': 'requests', 'Test': '1', 'Cookie': True, 'Ok': True}
         )
-        configuration.whitelisted.remove('Test')
+        configuration.whitelisted = []