Use Notify API endpoint to automatically revoke a key (#316)

* Changing logic to use Notify's endpoint

* Adding new notify integration

* Addng notify tests

* Adding additional unti tests

* Writing additional unit tests

* Adding additional secrets to the github actions

* Adding a timeout to the post request

* Fixing up a typo

* Removing the api key in the message

* Adding service_id to the message plus posting to the right notifications channel

* Adding the Notify ops channel id to the unit tests env.varialbes

.github/workflows/ci_code.yml

@@ -52,4 +52,7 @@ jobs:
           GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }}
           GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }}
           SESSION_SECRET_KEY: ${{ secrets.SESSION_SECRET_KEY }}
-          NOTIFY_TEST_KEY: ${{ secrets.NOTIFY_TEST_KEY }}
+          NOTIFY_TEST_KEY: ${{ secrets.NOTIFY_TEST_KEY }}
+          NOTIFY_SRE_USER_NAME: ${{ secrets.NOTIFY_SRE_USER_NAME }}
+          NOTIFY_SRE_CLIENT_SECRET: ${{ secrets.NOTIFY_SRE_CLIENT_SECRET }}
+          NOTIFY_OPS_CHANNEL_ID: ${{ secrets.NOTIFY_OPS_CHANNEL_ID }}

app/integrations/notify.py

@@ -0,0 +1,104 @@
+import os
+import jwt
+import time
+import calendar
+import logging
+import requests
+import json
+
+
+# generate the epoch seconds for the jwt token
+def epoch_seconds():
+    return calendar.timegm(time.gmtime())
+
+
+def create_jwt_token(secret, client_id):
+    """
+    Generate a JWT Token for the Notify API
+
+    Tokens have a header consisting of:
+    {
+        "typ": "JWT",
+        "alg": "HS256"
+    }
+
+    Parameters:
+    secret: Application signing secret
+    client_id: Identifier for the client
+
+    Claims are:
+    iss: identifier for the client
+    iat: epoch seconds for the token (UTC)
+
+    Returns a JWT token for this request
+    """
+    assert secret, "Missing secret key"
+    assert client_id, "Missing client id"
+
+    headers = {"typ": "JWT", "alg": "HS256"}
+
+    claims = {"iss": client_id, "iat": epoch_seconds()}
+    t = jwt.encode(payload=claims, key=secret, headers=headers)
+    if isinstance(t, str):
+        return t
+    else:
+        return t.decode()
+
+
+# Function to create the authorization header for the Notify API
+def create_authorization_header():
+    # get the client_id and secret from the environment variables
+    client_id = os.getenv("NOTIFY_SRE_USER_NAME")
+    secret = os.getenv("NOTIFY_SRE_CLIENT_SECRET")
+
+    # If the client_id or secret is missing, raise an assertion error
+    assert client_id, "NOTIFY_SRE_USER_NAME is missing"
+    assert secret, "NOTIFY_SRE_CLIENT_SECRET is missing"
+
+    # Create the jwt token and return the authorization header
+    token = create_jwt_token(secret=secret, client_id=client_id)
+    return "Authorization", "Bearer {}".format(token)
+
+
+# Function to post an api call to Notify
+def post_event(url, payload):
+    # Create the authorization headers
+    header_key, header_value = create_authorization_header()
+    header = {header_key: header_value, "Content-Type": "application/json"}
+
+    # Post the response
+    response = requests.post(url, data=json.dumps(payload), headers=header, timeout=60)
+    return response
+
+
+# Function to revoke an api key by calling Notify's revoke api endpoint
+def revoke_api_key(api_key, api_type, github_repo, source):
+    # get the url and jwt_token
+    url = os.getenv("NOTIFY_API_URL")
+
+    if url is None:
+        logging.error("NOTIFY_API_URL is missing")
+        return False
+
+    # append the revoke-endpoint to the url
+    url = url + "/sre-tools/api-key-revoke"
+
+    # generate the payload
+    payload = {
+        "token": api_key,
+        "type": api_type,
+        "url": github_repo,
+        "source": source,
+    }
+
+    # post the event (ie call the api)
+    response = post_event(url, payload)
+    # A successful response has a status code of 201
+    if response.status_code == 201:
+        logging.info(f"API key {api_key} has been successfully revoked")
+        return True
+    else:
+        logging.error(
+            f"API key {api_key} could not be revoked. Response code: {response.status_code}"
+        )
+        return False

app/requirements.txt

@@ -12,6 +12,7 @@ google-auth-oauthlib==0.8.0
 httpx==0.25.1
 itsdangerous==2.1.2
 Jinja2==3.1.2
+PyJWT==2.8.0
 PyYAML!=6.0.0,!=5.4.0,!=5.4.1
 python-dotenv==0.21.1
 python-i18n==0.3.9

app/requirements_dev.txt

@@ -1,6 +1,7 @@
 black==22.12.0
 coverage==6.5.0
 flake8==6.1.0
+freezegun==1.2.2
 pytest==7.4.3
 pytest-asyncio==0.21.1
 pytest-env==0.8.2

app/server/event_handlers/aws.py

@@ -1,8 +1,9 @@
 import json
 import re
+import os
 import urllib.parse
 from commands.utils import log_ops_message
-from integrations import google_drive, opsgenie
+from integrations import notify
 
 
 def parse(payload, client):
@@ -41,47 +42,6 @@ def nested_get(dictionary, keys):
     return dictionary
 
 
-def alert_on_call(product, client, api_key_name, github_repo):
-    # get the list of folders
-    folders = google_drive.list_folders()
-    # get the folder id for the Product
-    for folder in folders:
-        if folder["name"] == product:
-            folder = folder["id"]
-            break
-    # Get folder metadata
-    folder_metadata = google_drive.list_metadata(folder).get("appProperties", {})
-    oncall = []
-    message = ""
-    private_message = ""
-
-    # Generate the opsgenie message. We don't want to expose the api key value exposed so we will just provide the name of the key.
-    opsgenie_message = f"Notify key with name {api_key_name} has been leaked and needs to be revoked. Please check Slack in #internal-sre-alerts or on-call staff can check your private messages for more detailed information. "
-
-    # Get OpsGenie users on call and construct string
-    if "genie_schedule" in folder_metadata:
-        for email in opsgenie.get_on_call_users(folder_metadata["genie_schedule"]):
-            r = client.users_lookupByEmail(email=email)
-            if r.get("ok"):
-                oncall.append(r["user"])
-        message = f"{product} on-call staff "
-        for user in oncall:
-            # send a private message to the people on call.
-            message += f"<@{user['id']}> "
-            private_message = f"Hello {user['profile']['first_name']}!\nA Notify API key has been leaked and needs to be revoked. 🙀 \nThe key name is *{api_key_name}* and it is exposed in file {github_repo}. You can see the message in #internal-sre-alerts to start an incident."
-            # send the private message
-            client.chat_postMessage(
-                channel=user["id"], text=private_message, as_user=True
-            )
-        message += "have been notified."
-
-        # create an alert in OpsGenie
-        result = opsgenie.create_alert(opsgenie_message)
-        message += f"\nAn alert has been created in OpsGenie with result: {result}."
-
-    return message
-
-
 def format_abuse_notification(payload, msg):
     regex = r"arn:aws:sns:\w.*:(\d.*):\w.*"
     account = re.search(regex, payload.TopicArn).groups()[0]
@@ -244,45 +204,73 @@ def format_cloudwatch_alarm(msg):
     return blocks
 
 
-# If the message contains an api key it will be parsed by the format_api_key_detected function.
+# Function to send the message to the Notify ops channel fo alerting. Right now it is set to #notification-ops channel
+def send_message_to_notify_chanel(client, blocks):
+    NOTIFY_OPS_CHANNEL_ID = os.environ.get("NOTIFY_OPS_CHANNEL_ID")
+
+    # Raise an exception if the NOTIFY_OPS_CHANNEL_ID is not set
+    assert NOTIFY_OPS_CHANNEL_ID, "NOTIFY_OPS_CHANNEL_ID is not set in the environment"
+
+    # post the message to the notification channel
+    client.chat_postMessage(channel=NOTIFY_OPS_CHANNEL_ID, blocks=blocks)
 
 
+# If the message contains an api key it will be parsed by the format_api_key_detected function.
 def format_api_key_detected(payload, client):
     msg = payload.Message
-    regex = r"API Key with value token='(\w.+)' has been detected in url='(\w.+)'"
+    regex = r"API Key with value token='(\w.+)', type='(\w.+)' and source='(\w.+)' has been detected in url='(\w.+)'!"
     # extract the api key and the github repo from the message
     api_key = re.search(regex, msg).groups()[0]
-    github_repo = re.search(regex, msg).groups()[1]
+    type = re.search(regex, msg).groups()[1]
+    source = re.search(regex, msg).groups()[2]
+    github_repo = re.search(regex, msg).groups()[3]
+
+    # Extract the service id so that we can include it in the message
+    api_regex = r"(?P<prefix>gcntfy-)(?P<keyname>.*)(?P<service_id>[-A-Za-z0-9]{36})-(?P<key_id>[-A-Za-z0-9]{36})"
+    pattern = re.compile(api_regex)
+    match = pattern.search(api_key)
+    if match:
+        service_id = match.group("service_id")
 
     # We don't want to send the actual api-key through Slack, but we do want the name to be given,
     # so therefore extract the api key name by following the format of a Notify api key
     api_key_name = api_key[7 : len(api_key) - 74]
 
-    # send a private message with the api-key and github repo to the people on call.
-    on_call_message = alert_on_call("Notify", client, api_key_name, github_repo)
+    # call the revoke api endpoint to revoke the api key
+    if notify.revoke_api_key(api_key, type, github_repo, source):
+        revoke_api_key_message = (
+            f"API key {api_key_name} has been successfully revoked."
+        )
+        header_text = "🙀 Notify API Key has been exposed and revoked! 😌"
+    else:
+        revoke_api_key_message = (
+            f"API key {api_key_name} could not be revoked due to an error."
+        )
+        header_text = "🙀 Notify API Key has been exposed but could not be revoked! 😱"
 
     # Format the message displayed in Slack
-    return [
+    blocks = [
         {"type": "section", "text": {"type": "mrkdwn", "text": " "}},
         {
             "type": "header",
-            "text": {
-                "type": "plain_text",
-                "text": "🙀 Notify API Key has been compromised! 🔑",
-            },
+            "text": {"type": "plain_text", "text": f"{header_text}"},
         },
         {
             "type": "section",
             "text": {
                 "type": "mrkdwn",
-                "text": f"Notify API Key Name *{api_key_name}* has been committed in github file {github_repo}. The key needs to be revoked!",
+                "text": f"Notify API Key Name {api_key_name} from service id {service_id} was committed in github file {github_repo}.\n",
             },
         },
         {
             "type": "section",
             "text": {
                 "type": "mrkdwn",
-                "text": f"{on_call_message}",
+                "text": f"*{revoke_api_key_message}*",
             },
         },
     ]
+    # send the message to the notify ops channel
+    send_message_to_notify_chanel(client, blocks)
+
+    return blocks

app/tests/intergrations/test_notify.py

@@ -0,0 +1,81 @@
+import jwt
+import os
+import pytest
+from integrations import notify
+from unittest.mock import patch
+from freezegun import freeze_time
+
+
+# helper function to decode the token for testing
+def decode_token(token, secret):
+    return jwt.decode(
+        token, key=secret, options={"verify_signature": True}, algorithms=["HS256"]
+    )
+
+
+# Test that an exception is raised if the secret is missing
+def test_create_jwt_token_secret_missing():
+    with pytest.raises(AssertionError) as err:
+        notify.create_jwt_token(None, "client_id")
+    assert str(err.value) == "Missing secret key"
+
+
+# Test that an exception is raised if the client_id is missing
+def test_create_jwt_token_client_id_missing():
+    with pytest.raises(AssertionError) as err:
+        notify.create_jwt_token("secret", None)
+    assert str(err.value) == "Missing client id"
+
+
+# Test that the token is created correctly and the type and alg headers are set correctly
+def test_create_jwt_token_contains_correct_headers():
+    token = notify.create_jwt_token("secret", "client_id")
+    headers = jwt.get_unverified_header(token)
+    assert headers["typ"] == "JWT"
+    assert headers["alg"] == "HS256"
+
+
+# Test that the claims headers are set correctly
+def test_create_jwt_token_contains_correct_claims_headers():
+    token = notify.create_jwt_token("secret", "client_id")
+    decoded_token = decode_token(token, "secret")
+    assert decoded_token["iss"] == "client_id"
+    assert "iat" in decoded_token
+    assert "req" not in decoded_token
+    assert "pay" not in decoded_token
+
+
+# Test that the correct iat time in epoch seconds is set correctly
+@freeze_time("2020-01-01 00:00:00")
+def test_token_contains_correct_iat():
+    token = notify.create_jwt_token("secret", "client_id")
+    decoded_token = decode_token(token, "secret")
+    assert decoded_token["iat"] == 1577836800
+
+
+# Test that an assertion error is raised if the NOTIFY_SRE_USER_NAME is missing
+@patch.dict(os.environ, {"NOTIFY_SRE_USER_NAME": "", "NOTIFY_SRE_CLIENT_SECRET": "foo"})
+@patch("integrations.notify.create_jwt_token")
+def test_authorization_header_missing_client_id(jwt_token_mock):
+    with pytest.raises(AssertionError) as err:
+        notify.create_authorization_header()
+    assert str(err.value) == "NOTIFY_SRE_USER_NAME is missing"
+
+
+# Test that an assertion error is raised if the NOTIFY_SRE_CLIENT_SECRET is missing
+@patch.dict(os.environ, {"NOTIFY_SRE_USER_NAME": "foo", "NOTIFY_SRE_CLIENT_SECRET": ""})
+@patch("integrations.notify.create_jwt_token")
+def test_authorization_header_missing_secret(jwt_token_mock):
+    with pytest.raises(AssertionError) as err:
+        notify.create_authorization_header()
+    assert str(err.value) == "NOTIFY_SRE_CLIENT_SECRET is missing"
+
+
+# Test that the authorization header is created correctly and the correct header is generated
+@patch("integrations.notify.create_jwt_token")
+def test_successful_creation_of_header(mock_jwt_token):
+    mock_jwt_token.return_value = "mocked_jwt_token"
+    header_key, header_value = notify.create_authorization_header()
+
+    assert header_key == "Authorization"
+    assert header_value == "Bearer mocked_jwt_token"