Merge branch 'master' into production

v-2019-07-31.1

config.py.example

@@ -34,6 +34,11 @@ REDIS_PORT = 6379
 REDIS_NAMESPACE = "AB"
 REDIS_NS_VERSIONS_LOCATION = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cache_namespaces')
 
+# RATE LIMITING
+# set a limit of per_ip requests per window seconds per unique ip address
+RATELIMIT_PER_IP = 100
+RATELIMIT_WINDOW = 10
+
 # LOGGING
 # Uncomment any of the following logging stubs if you want to enable logging
 # during development. In general you shouldn't need to do this.
@@ -68,3 +73,5 @@ FILE_STORAGE_DIR = "/data/files"
 
 #Feature Flags
 FEATURE_EVAL_LOCATION = False
+
+DEBUG_TB_INTERCEPT_REDIRECTS = False

consul_config.py.ctmpl

@@ -49,3 +49,6 @@ DATASET_DIR = '''{{template "KEY" "dataset_dir"}}'''
 FILE_STORAGE_DIR = '''{{template "KEY" "file_storage_dir"}}'''
 
 FEATURE_EVAL_LOCATION = False
+
+RATELIMIT_PER_IP = {{template "KEY" "ratelimit_per_ip"}} # number of requests per ip
+RATELIMIT_WINDOW = {{template "KEY" "ratelimit_window"}} # window size in seconds

db/testing.py

@@ -18,8 +18,9 @@ class DatabaseTestCase(TestCase):
 
     @staticmethod
     def create_app():
-        app = create_app()
+        app = create_app(debug=False)
         app.config['WTF_CSRF_ENABLED'] = False
+        app.config['TESTING'] = True
         return app
 
     def setUp(self):

docs/api.rst

@@ -24,9 +24,42 @@ Datasets
    :include-empty-docstring:
    :undoc-static:
 
+Rate limiting
+^^^^^^^^^^^^^
+
+The AcousticBrainz API is rate limited via the use of rate limiting headers that
+are sent as part of the HTTP response headers. Each call will include the
+following headers:
+
+- **X-RateLimit-Limit**: Number of requests allowed in given time window
+
+- **X-RateLimit-Remaining**: Number of requests remaining in current time
+  window
+
+- **X-RateLimit-Reset-In**: Number of seconds when current time window expires
+  (*recommended*: this header is resilient against clients with incorrect
+  clocks)
+
+- **X-RateLimit-Reset**: UNIX epoch number of seconds (without timezone) when
+  current time window expires [#]_
+
+We typically set the limit to 10 queries every 10 seconds per IP address,
+but these values may change. Make sure you check the response headers
+if you want to know the specific values.
+
+Rate limiting is automatic and the client must use these headers to determine
+the rate to make API calls. If the client exceeds the number of requests
+allowed, the server will respond with error code ``429: Too Many Requests``.
+Requests that provide the *Authorization* header with a valid user token may
+receive higher rate limits than those without valid user tokens.
+
+.. [#] Provided for compatibility with other APIs, but we still recommend using
+   ``X-RateLimit-Reset-In`` wherever possible
+
 Constants
 ^^^^^^^^^
 
 Constants that are relevant to using the API:
 
 .. autodata:: webserver.views.api.v1.core.MAX_ITEMS_PER_BULK_REQUEST
+

docs/dev/deployment.rst

@@ -0,0 +1,16 @@
+Deployment Practices
+====================
+
+We run two instances of AcousticBrainz on the same data.
+
+* AcousticBrainz (https://acousticbrainz.org)
+* AcousticBrainz beta (https://beta.acousticbrainz.org)
+
+
+The beta instance is used to test new features and bugfixes on the actual
+dataset before being officially deployed into production.
+
+We follow the general `MetaBrainz guidelines`_ to deploy, test and release
+code into beta and production.
+
+.. _MetaBrainz guidelines: https://github.com/metabrainz/guidelines/blob/master/Deployment.md

docs/index.rst

@@ -7,6 +7,7 @@ Contents:
    :maxdepth: 2
 
    api
+   dev/deployment
 
 Indices and tables
 ==================

manage.py

@@ -5,7 +5,7 @@
 import sys
 
 import click
-from brainzutils import cache
+from brainzutils import cache, ratelimit
 import flask.cli
 from flask import current_app
 from flask.cli import FlaskGroup
@@ -278,7 +278,40 @@ def remove_failed_rows():
         sys.exit(1)
 
 
-# Keep additional sets of commands down here
+@cli.command(name='set_rate_limits')
+@click.argument('per_ip', type=click.IntRange(1, None), required=False)
+@click.argument('window_size', type=click.IntRange(1, None), required=False)
+def set_rate_limits(per_ip, window_size):
+    """Set rate limit parameters for the AcousticBrainz webserver. If no arguments
+    are provided, print the current limits. To set limits, specify PER_IP and WINDOW_SIZE
+
+    \b
+    PER_IP: the number of requests allowed per IP address
+    WINDOW_SIZE: the window in number of seconds for how long the limit is applied
+    """
+
+    current_limit_per_ip = cache.get(ratelimit.ratelimit_per_ip_key)
+    current_limit_window = cache.get(ratelimit.ratelimit_window_key)
+
+    click.echo("Current values:")
+    if current_limit_per_ip is None and current_limit_window is None:
+        click.echo("No values set, showing limit defaults")
+        current_limit_per_ip = ratelimit.ratelimit_per_ip_default
+        current_limit_window = ratelimit.ratelimit_window_default
+    click.echo("Requests per IP: %s" % current_limit_per_ip)
+    click.echo("Window size (s): %s" % current_limit_window)
+
+    if per_ip is not None and window_size is not None:
+        if per_ip / float(window_size) < 1:
+            click.echo("Warning: Effective rate limit is less than 1 query per second")
+
+        ratelimit.set_rate_limits(per_ip, per_ip, window_size)
+        print("New ratelimit parameters set:")
+        click.echo("Requests per IP: %s" % per_ip)
+        click.echo("Window size (s): %s" % window_size)
+
+
+# Please keep additional sets of commands down there
 cli.add_command(db.dump_manage.cli, name="dump")
 
 if __name__ == '__main__':

package.json

@@ -22,7 +22,7 @@
     "highcharts": "6.1.0",
     "jquery": "3.4.0",
     "less-plugin-clean-css": "1.5.1",
-    "lodash": "4.17.11",
+    "lodash": "4.17.15",
     "q": "1.5.1",
     "react": "0.14.7",
     "react-dom": "0.14.7",

webserver/__init__.py

@@ -1,16 +1,17 @@
 from __future__ import print_function
 
 from brainzutils.flask import CustomFlask
+from brainzutils.ratelimit import set_rate_limits, inject_x_rate_headers
 from flask import request, url_for, redirect
 from flask_login import current_user
 from pprint import pprint
 
 import os
 import time
+import urlparse
 
 API_PREFIX = '/api/'
 
-
 # Check to see if we're running under a docker deployment. If so, don't second guess
 # the config file setup and just wait for the correct configuration to be generated.
 deploy_env = os.environ.get('DEPLOY_ENV', '')
@@ -81,6 +82,15 @@ def create_app(debug=None):
     else:
         raise Exception('One or more redis cache configuration options are missing from config.py')
 
+    # Add rate limiting support
+    @app.after_request
+    def after_request_callbacks(response):
+        return inject_x_rate_headers(response)
+
+    # check for ratelimit config values and set them if present
+    if 'RATELIMIT_PER_IP' in app.config and 'RATELIMIT_WINDOW' in app.config:
+        set_rate_limits(app.config['RATELIMIT_PER_IP'], app.config['RATELIMIT_PER_IP'], app.config['RATELIMIT_WINDOW'])
+
     # MusicBrainz
     import musicbrainzngs
     from db import SCHEMA_VERSION
@@ -117,7 +127,19 @@ def create_app(debug=None):
     admin = Admin(app, index_view=admin_views.HomeView(name='Admin'))
     admin.add_view(admin_views.AdminsView(name='Admins'))
 
-    @ app.before_request
+    @app.before_request
+    def prod_https_login_redirect():
+        """ Redirect to HTTPS in production except for the API endpoints
+        """
+        if urlparse.urlsplit(request.url).scheme == 'http' \
+                and app.config['DEBUG'] == False \
+                and app.config['TESTING'] == False \
+                and request.blueprint not in ('api', 'api_v1_core', 'api_v1_datasets', 'api_v1_dataset_eval'):
+            url = request.url[7:] # remove http:// from url
+            return redirect('https://{}'.format(url))
+
+
+    @app.before_request
     def before_request_gdpr_check():
         # skip certain pages, static content and the API
         if request.path == url_for('index.gdpr_notice') \