Add related.url field

[chrisberkhout]

Summary

Add a related.url field to facilitate searching for URLs that appear in various other fields of an event.

Motivation:

This was requested by a user in order to improve mappings for data sources that have multiple URL fields, such as data from the o365 integration.

The closest existing field is related.hosts, which is for "All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases."

The addition of related.domain was suggested as an alternative to related.url. I have focused here on related.url as it is more distinct from the related.hosts use case.

Detailed Design:

A related.url field could be populated with the same kind of values as url.full when possible, or the same kind as url.original if that is the most complete value available.

Setting a field type of wildcard would match the url.full and url.original fields. A .text multi-field could be added.

Examples from o365 integration - not the most compelling, but this is what was readily available in test data

{
  "@timestamp": "2020-02-14T19:00:00.000Z",
  "ecs": {
    "version": "8.11.0"
  },
  "event": {
    "action": "AlertEntityGenerated",
    "category": [
      "web"
    ],
    "code": "SecurityComplianceAlerts",
    "id": "448854d7-81f6-4a06-d31a-08d7b1c1fb2f",
    "kind": "alert",
    "outcome": "success",
    "provider": "SecurityComplianceCenter",
    "type": [
      "info"
    ]
  },
  "host": {
    "id": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd",
    "name": "mytenant.onmicrosoft.com"
  },
  "message": "New alert",
  "o365": {
    "audit": {
      "AlertId": "5ba6e029-8b6e-13bd-b800-08d7b180173c",
      "AlertType": "System",
      "CreationTime": "2020-02-14T19:00:00",
      "Data": {
        "eid": "[email protected]",
        "etype": "User",
        "flattened": {
          "eid": "[email protected]",
          "etype": "User",
          "lon": "GrantAdminPermission",
          "op": "GrantAdminPermission",
          "suid": "[email protected]",
          "tdc": "1",
          "te": "2020-02-14T18:54:45.0000000Z",
          "tid": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd",
          "ts": "2020-02-14T18:54:45.0000000Z",
          "ut": "Admin"
        },
        "lon": "GrantAdminPermission",
        "op": "GrantAdminPermission",
        "suid": "[email protected]",
        "tdc": "1",
        "te": "2020-02-14T18:54:45.000Z",
        "tid": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd",
        "ts": "2020-02-14T18:54:45.000Z",
        "ut": "Admin"
      },
      "ObjectId": "[email protected]",
      "RecordType": "40",
      "ResultStatus": "Succeeded",
      "Severity": "Low",
      "Source": "Office 365 Security & Compliance",
      "Status": "Active",
      "UserId": "SecurityComplianceAlerts",
      "UserKey": "SecurityComplianceAlerts",
      "UserType": "4",
      "Version": "1"
    }
  },
  "organization": {
    "id": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd",
    "name": "mytenant.onmicrosoft.com"
  },
  "rule": {
    "category": "AccessGovernance",
    "description": "[email protected]",
    "id": "17d51759-88e1-40c1-8df3-20bcf2e43057",
    "name": "Elevation of Exchange admin privilege",
    "reference": [
      "http://example.net/alert",  // URL 1
      "http://example.net/info"    // URL 2
    ],
    "ruleset": "User"
  },
  "tags": [
    "preserve_original_event"
  ],
  "user": {
    "id": "SecurityComplianceAlerts"
  }
}

{
  "o365audit": {
    "ClientIP": "67.43.156.13",
    "CorrelationId": "fe71359f-005f-9000-7cb1-ccf5124703db",
    "CreationTime": "2020-02-14T18:25:45",
    "EventData": "<Permissions granted>Contribute</Permissions granted>",
    "EventSource": "SharePoint",
    "Id": "a8c23ab8-9447-4824-3208-08d7b17b4e5e",
    "ItemType": "File",
    "ListId": "2b6ad2bd-0fd7-4556-9c89-a97847085b85",
    "ListItemUniqueId": "7f06ab3a-bd98-41d3-a0b2-ad270d71e4d8",
    "ObjectId": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", // URL 1
    "Operation": "SharingSet",
    "OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd",
    "RecordType": 14,
    "Site": "d5180cfc-3479-44d6-b410-8c985ac894e3",
    "SiteUrl": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com",                           // URL 2
    "SourceFileExtension": "png",
    "SourceFileName": "Screenshot.png",
    "SourceRelativeUrl": "Documents/Screenshot.png",                                                                 // URL 3
    "TargetUserOrGroupName": "SharingLinks.7f06ab3a-bd98-41d3-a0b2-ad270d71e4d8.AnonymousEdit.d323b5ea-ceca-4d65-a628-e22ca9296a76",
    "TargetUserOrGroupType": "SharePointGroup",
    "UserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:73.0) Gecko/20100101 Firefox/73.0",
    "UserId": "[email protected]",
    "UserKey": "i:0h.f|membership|[email protected]",
    "UserType": 0,
    "Version": 1,
    "WebId": "8c5c94bb-8396-470c-87d7-8999f440cd30",
    "Workload": "OneDrive"
  }
}

{
  "o365audit": {
    "CreationTime": "2020-02-26T10:13:48",
    "Id": "d69c6758-f210-43bd-bac1-563adef4b4cf",
    "IncidentId": "f7295114-e601-f2b6-8800-08d7baa56f8b",
    "ObjectId": "f026407b-090a-4c15-99b5-09851842d96d",
    "Operation": "DLPRuleMatch",
    "OrganizationId": "0e1dddce-163e-4b0b-9e33-87ba56ac4655",
    "PolicyDetails": [
      {
        "PolicyId": "08745d02-5d45-48bd-98e1-8199ab1efdbe",
        "PolicyName": "Financial Data Detection",
        "Rules": [
          {
            "ActionParameters": [
              "GenerateIncidentReport:SiteAdmin"
            ],
            "Actions": [
              "BlockAccess",
              "NotifyUser",
              "GenerateIncidentReport"
            ],
            "ConditionsMatched": {
              "SensitiveInformation": [
                {
                  "Confidence": 85,
                  "Count": 42,
                  "SensitiveType": "50842eb7-edc8-4019-85dd-5a5c1f2bb085"
                },
                {
                  "Confidence": 85,
                  "Count": 23,
                  "SensitiveType": "0e9b3178-9678-47dd-a509-37222ca96b42"
                }
              ]
            },
            "RuleId": "bc4d376f-b038-4695-9362-609d32f963cf",
            "RuleMode": "Enable",
            "RuleName": "High volume of content detected France Financial",
            "Severity": "High"
          }
        ]
      }
    ],
    "RecordType": 11,
    "SensitiveInfoDetectionIsIncluded": false,
    "SharePointMetaData": {
      "FileName": "INTERNAL CREDIT CARD NUMBERS.docx",
      "FileOwner": "Alan Smithee",
      "FilePathUrl": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/INTERNAL%20CREDIT%20CARD%20NUMBERS.docx", // URL 1
      "From": "[email protected]",
      "ItemCreationTime": "2020-02-26T09:44:40",
      "ItemLastModifiedTime": "2020-02-26T09:46:23",
      "SiteCollectionGuid": "eae3edad-a192-43a9-b317-98d7ea5e3939",
      "SiteCollectionUrl": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com",                                             // URL 2
      "UniqueID": "f026407b-090a-4c15-99b5-09851842d96d"
    },
    "UserId": "DlpPolicyEventBasedAssistantOneDriveForBusiness",
    "UserKey": "DlpPolicyEventBasedAssistantOneDriveForBusiness",
    "UserType": 4,
    "Version": 1,
    "Workload": "OneDrive"
  }
}