Feat: global filter for github log sources (#705)

* feat: global filter for github events

* docs: cloudflare filter ended up as a single function, docs mentioned two functions

global_helpers/global_filter_cloudflare.yml

@@ -2,11 +2,10 @@ AnalysisType: global
 Filename: global_filter_cloudflare.py
 GlobalID: global_filter_cloudflare
 Description: >
-  This module provides two filters that can apply to all cloudflare detections. 
-  The two functions in this module are included in every cloudflare detection, and 
-  define if we should include or exclude events. By default, all events are passed
-  through the filters. 
+  This module provides one filter that is included in all cloudflare detections.
+  
+  This filter defines if events should be included or excluded.
 
-  You can change the definition of these filters to work in your own environment. 
-  Panther will not change the definition of these, and should not create 
-  merge conflicts.
+  You can change the definition of this filter to work in your own environment. 
+  Panther will not change the filter definition, and should not create 
+  merge conflicts.

global_helpers/global_filter_github.py

@@ -0,0 +1,25 @@
+from panther_base_helpers import deep_get  # pylint: disable=unused-import
+
+
+def filter_include_event(event) -> bool:  # pylint: disable=unused-argument
+    """
+    filter_include_event provides a global include filter for all github detections
+    Panther will not update this filter, and you can edit it without creating
+    merge conflicts in the future.
+
+    return True to include events, and False to exclude events
+    """
+    # This commented-out example would have the effect of
+    #   ignoring all github organization logs except for
+    #   the production github organization.
+    # If you're ingesting GitHub Enterprise Logs with multiple
+    #   orgs, this may help to keep your detections running
+    #   on the production orgs
+    #
+    #
+    # # not all github enterprise events have org
+    # # example: enterprise.self_hosted_runner_online
+    # org = deep_get(event, "org", default="")
+    # return org in ["my-prod-org", ""]
+    #
+    return True

global_helpers/global_filter_github.yml

@@ -0,0 +1,11 @@
+AnalysisType: global
+Filename: global_filter_github.py
+GlobalID: global_filter_github
+Description: >
+  This module provides one filter that is included in all github detections.
+  
+  This filter defines if events should be included or excluded.
+
+  You can change the definition of this filter to work in your own environment. 
+  Panther will not change the filter definition, and should not create 
+  merge conflicts.

packs/github.yml

@@ -28,3 +28,4 @@ PackDefinition:
     - panther_event_type_helpers
     - panther_base_helpers
     - panther_oss_helpers
+    - global_filter_github

rules/github_rules/github_action_failed.py

@@ -1,6 +1,7 @@
 import json
 from unittest.mock import MagicMock
 
+from global_filter_github import filter_include_event
 from panther_base_helpers import deep_get, github_alert_context
 
 # The keys for MONITORED_ACTIONS are gh_org/repo_name
@@ -9,6 +10,8 @@
 
 
 def rule(event):
+    if not filter_include_event(event):
+        return False
     global MONITORED_ACTIONS  # pylint: disable=global-statement
     if isinstance(MONITORED_ACTIONS, MagicMock):
         MONITORED_ACTIONS = json.loads(MONITORED_ACTIONS())  # pylint: disable=not-callable

rules/github_rules/github_action_failed.yml

@@ -61,6 +61,41 @@ Tests:
           {
             "your-org/panther-analysis-copy": [ "sync-panther-analysis-from-upstream"]
           }
+    Log:
+      {
+        "_document_id": "pWWWWWWWWWWWWWWWWWWWWW",
+        "action": "workflows.completed_workflow_run",
+        "actor": "github_handle",
+        "at_sign_timestamp": "2023-01-31 18:58:27.638",
+        "business": "github-business-only-if-enterprise-audit-log",
+        "completed_at": "2023-01-31T18:58:27.000Z",
+        "conclusion": "failure",
+        "created_at": "2023-01-31 18:58:27.638",
+        "event": "schedule",
+        "head_branch": "master",
+        "head_sha": "66dddddddddddddddddddddddddddddddddddddd",
+        "name": "sync-panther-analysis-from-upstream",
+        "operation_type": "modify",
+        "org": "panther-labs",
+        "p_log_type": "GitHub.Audit",
+        "public_repo": false,
+        "repo": "your-org/panther-analysis-copy",
+        "run_attempt": 3,
+        "run_number": 99,
+        "started_at": "2023-01-31 18:58:04",
+        "workflow_id": 44444444,
+        "workflow_run_id": 5555555555
+      }
+  - Name: GitHub Action Failed - Monitored Action Configured - GLOBAL FILTER
+    ExpectedResult: false
+    Mocks:
+      - objectName: MONITORED_ACTIONS
+        returnValue: >-
+          {
+            "your-org/panther-analysis-copy": [ "sync-panther-analysis-from-upstream"]
+          }
+      - objectName: filter_include_event
+        returnValue: False
     Log:
       {
         "_document_id": "pWWWWWWWWWWWWWWWWWWWWW",

rules/github_rules/github_advanced_security_change.py

@@ -1,3 +1,4 @@
+from global_filter_github import filter_include_event
 from panther_base_helpers import github_alert_context
 
 # List of actions in markdown format
@@ -70,6 +71,8 @@
 
 
 def rule(event):
+    if not filter_include_event(event):
+        return False
     return event.get("action", "") in ADV_SEC_ACTIONS
 
 

rules/github_rules/github_branch_policy_override.py

@@ -1,4 +1,9 @@
+from global_filter_github import filter_include_event
+
+
 def rule(event):
+    if not filter_include_event(event):
+        return False
     return event.get("action") == "protected_branch.policy_override"
 
 

rules/github_rules/github_branch_protection_disabled.py

@@ -1,4 +1,9 @@
+from global_filter_github import filter_include_event
+
+
 def rule(event):
+    if not filter_include_event(event):
+        return False
     return event.get("action") == "protected_branch.destroy"
 
 

rules/github_rules/github_org_auth_modified.py

@@ -1,3 +1,5 @@
+from global_filter_github import filter_include_event
+
 AUTH_CHANGE_EVENTS = [
     "org.saml_disabled",
     "org.saml_enabled",
@@ -10,6 +12,8 @@
 
 
 def rule(event):
+    if not filter_include_event(event):
+        return False
 
     if not event.get("action").startswith("org."):
         return False

rules/github_rules/github_org_ip_allowlist.py

@@ -1,3 +1,5 @@
+from global_filter_github import filter_include_event
+
 ALLOWLIST_ACTIONS = [
     "ip_allow_list.enable",
     "ip_allow_list.disable",
@@ -10,6 +12,8 @@
 
 
 def rule(event):
+    if not filter_include_event(event):
+        return False
     return (
         event.get("action").startswith("ip_allow_list") and event.get("action") in ALLOWLIST_ACTIONS
     )

rules/github_rules/github_org_modified.py

@@ -1,4 +1,9 @@
+from global_filter_github import filter_include_event
+
+
 def rule(event):
+    if not filter_include_event(event):
+        return False
     return event.get("action") == "org.add_member" or event.get("action") == "org.remove_member"
 
 

rules/github_rules/github_organization_app_integration_installed.py

@@ -1,7 +1,10 @@
+from global_filter_github import filter_include_event
 from panther_base_helpers import github_alert_context
 
 
 def rule(event):
+    if not filter_include_event(event):
+        return False
     # Return True to match the log event and trigger an alert.
     # Creates a new alert if the event's action was ""
     return event.get("action") == "integration_installation.create"

rules/github_rules/github_public_repository_created.py

@@ -1,7 +1,10 @@
+from global_filter_github import filter_include_event
 from panther_base_helpers import github_alert_context
 
 
 def rule(event):
+    if not filter_include_event(event):
+        return False
     # Return True if a public repository was created
     return event.get("action", "") == "repo.create" and event.get("visibility", "") == "public"
 

rules/github_rules/github_repo_collaborator_change.py

@@ -1,4 +1,9 @@
+from global_filter_github import filter_include_event
+
+
 def rule(event):
+    if not filter_include_event(event):
+        return False
     return event.get("action") == "repo.add_member" or event.get("action") == "repo.remove_member"
 
 

rules/github_rules/github_repo_created.py

@@ -1,4 +1,9 @@
+from global_filter_github import filter_include_event
+
+
 def rule(event):
+    if not filter_include_event(event):
+        return False
     return event.get("action") == "repo.create"
 
 

rules/github_rules/github_repo_hook_modified.py

@@ -1,4 +1,9 @@
+from global_filter_github import filter_include_event
+
+
 def rule(event):
+    if not filter_include_event(event):
+        return False
     return event.get("action").startswith("hook.")
 
 

rules/github_rules/github_repo_initial_access.py

@@ -1,3 +1,4 @@
+from global_filter_github import filter_include_event
 from panther_oss_helpers import get_string_set, put_string_set
 
 CODE_ACCESS_ACTIONS = [
@@ -8,6 +9,8 @@
 
 
 def rule(event):
+    if not filter_include_event(event):
+        return False
     # if the actor field is empty, short circuit the rule
     if not event.udm("actor_user"):
         return False

rules/github_rules/github_repo_visibility_change.py

@@ -1,4 +1,9 @@
+from global_filter_github import filter_include_event
+
+
 def rule(event):
+    if not filter_include_event(event):
+        return False
     return event.get("action") == "repo.access"
 
 

rules/github_rules/github_repository_transfer.py

@@ -1,7 +1,10 @@
+from global_filter_github import filter_include_event
 from panther_base_helpers import github_alert_context
 
 
 def rule(event):
+    if not filter_include_event(event):
+        return False
     # Return True to match the log event and trigger an alert.
     return event.get("action", "") in (
         "repo.transfer",

rules/github_rules/github_secret_scanning_alert_created.py

@@ -1,4 +1,9 @@
+from global_filter_github import filter_include_event
+
+
 def rule(event):
+    if not filter_include_event(event):
+        return False
     return event.get("action") == "secret_scanning_alert.create"
 
 

rules/github_rules/github_team_modified.py

@@ -1,4 +1,9 @@
+from global_filter_github import filter_include_event
+
+
 def rule(event):
+    if not filter_include_event(event):
+        return False
     if not event.get("action").startswith("team"):
         return False
     return (

rules/github_rules/github_user_access_key_created.py

@@ -1,4 +1,9 @@
+from global_filter_github import filter_include_event
+
+
 def rule(event):
+    if not filter_include_event(event):
+        return False
     return event.get("action") == "public_key.create"
 
 

rules/github_rules/github_user_role_updated.py

@@ -1,4 +1,9 @@
+from global_filter_github import filter_include_event
+
+
 def rule(event):
+    if not filter_include_event(event):
+        return False
     return event.get("action") == "org.update_member"