Detection of Malicious URLs in Syslog Events

// Identifies a match in Syslog from the Recorded Future URLs Recently Reported by Insikt Group
let dt_lookBack = 1h;
let ioc_lookBack = 1d;
ThreatIntelligenceIndicator
// Picking up only Recorded Future IOC's that have been recently reported as malicious by Insikt Group
| where Description == 'Recorded Future - URL - Recently Reported by Insikt Group'
| where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
| where Active == true
// Picking up only IOC's that contain the entities we want
| where isnotempty(Url)
| join (
  Syslog
  | where TimeGenerated >= ago(dt_lookBack)
  // Extract URL from the Syslog message but only take messages that include URLs
  | extend Url = extract("(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)", 1,SyslogMessage)
  | where isnotempty(Url)
  | extend Syslog_TimeGenerated = TimeGenerated
  ) 
on Url
| where Syslog_TimeGenerated >= TimeGenerated and Syslog_TimeGenerated < ExpirationDateTime
| summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
| project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Syslog_TimeGenerated, SyslogMessage, Computer, ProcessName, Url, AdditionalInformation, HostIP
| extend timestamp = Syslog_TimeGenerated, HostCustomEntity = Computer, IPCustomEntity = HostIP, URLCustomEntity = Url

triggerOperator: gt
queryFrequency: 1h
description: |
    'Identifies a match in Syslog from Recorded Future URLs Recently Reported as malicious by Insikt Group.'
status: Available
kind: Scheduled
triggerThreshold: 0
requiredDataConnectors:
- connectorId: Syslog
  dataTypes:
  - Syslog
version: 1.0.0
queryPeriod: 1d
name: Detection of Malicious URLs in Syslog Events
OriginalUri: https://github.com/Azure/Azure-Sentinel/blob/master/Solutions/Recorded Future/Analytic Rules/RecordedFutureUrlReportedbyInsiktGroupinSyslogEvents.yaml
id: 9acb3664-72c4-4676-80fa-9f81912e347e
tactics:
- Impact
relevantTechniques: 
severity: Medium
entityMappings:
- fieldMappings:
  - identifier: FullName
    columnName: HostCustomEntity
  entityType: Host
- fieldMappings:
  - identifier: Address
    columnName: IPCustomEntity
  entityType: IP
- fieldMappings:
  - identifier: Url
    columnName: URLCustomEntity
  entityType: URL
query: |
  // Identifies a match in Syslog from the Recorded Future URLs Recently Reported by Insikt Group
  let dt_lookBack = 1h;
  let ioc_lookBack = 1d;
  ThreatIntelligenceIndicator
  // Picking up only Recorded Future IOC's that have been recently reported as malicious by Insikt Group
  | where Description == 'Recorded Future - URL - Recently Reported by Insikt Group'
  | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
  | where Active == true
  // Picking up only IOC's that contain the entities we want
  | where isnotempty(Url)
  | join (
    Syslog
    | where TimeGenerated >= ago(dt_lookBack)
    // Extract URL from the Syslog message but only take messages that include URLs
    | extend Url = extract("(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)", 1,SyslogMessage)
    | where isnotempty(Url)
    | extend Syslog_TimeGenerated = TimeGenerated
    ) 
  on Url
  | where Syslog_TimeGenerated >= TimeGenerated and Syslog_TimeGenerated < ExpirationDateTime
  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Syslog_TimeGenerated, SyslogMessage, Computer, ProcessName, Url, AdditionalInformation, HostIP
  | extend timestamp = Syslog_TimeGenerated, HostCustomEntity = Computer, IPCustomEntity = HostIP, URLCustomEntity = Url  

{
  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
  "contentVersion": "1.0.0.0",
  "parameters": {
    "workspace": {
      "type": "String"
    }
  },
  "resources": [
    {
      "apiVersion": "2023-02-01-preview",
      "id": "[concat(resourceId('Microsoft.OperationalInsights/workspaces/providers', parameters('workspace'), 'Microsoft.SecurityInsights'),'/alertRules/9acb3664-72c4-4676-80fa-9f81912e347e')]",
      "kind": "Scheduled",
      "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/9acb3664-72c4-4676-80fa-9f81912e347e')]",
      "properties": {
        "alertRuleTemplateName": "9acb3664-72c4-4676-80fa-9f81912e347e",
        "customDetails": null,
        "description": "'Identifies a match in Syslog from Recorded Future URLs Recently Reported as malicious by Insikt Group.'\n",
        "displayName": "Detection of Malicious URLs in Syslog Events",
        "enabled": true,
        "entityMappings": [
          {
            "entityType": "Host",
            "fieldMappings": [
              {
                "columnName": "HostCustomEntity",
                "identifier": "FullName"
              }
            ]
          },
          {
            "entityType": "IP",
            "fieldMappings": [
              {
                "columnName": "IPCustomEntity",
                "identifier": "Address"
              }
            ]
          },
          {
            "entityType": "URL",
            "fieldMappings": [
              {
                "columnName": "URLCustomEntity",
                "identifier": "Url"
              }
            ]
          }
        ],
        "OriginalUri": "https://github.com/Azure/Azure-Sentinel/blob/master/Solutions/Recorded Future/Analytic Rules/RecordedFutureUrlReportedbyInsiktGroupinSyslogEvents.yaml",
        "query": "// Identifies a match in Syslog from the Recorded Future URLs Recently Reported by Insikt Group\nlet dt_lookBack = 1h;\nlet ioc_lookBack = 1d;\nThreatIntelligenceIndicator\n// Picking up only Recorded Future IOC's that have been recently reported as malicious by Insikt Group\n| where Description == 'Recorded Future - URL - Recently Reported by Insikt Group'\n| where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()\n| where Active == true\n// Picking up only IOC's that contain the entities we want\n| where isnotempty(Url)\n| join (\n  Syslog\n  | where TimeGenerated >= ago(dt_lookBack)\n  // Extract URL from the Syslog message but only take messages that include URLs\n  | extend Url = extract(\"(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\\\(\\\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)\", 1,SyslogMessage)\n  | where isnotempty(Url)\n  | extend Syslog_TimeGenerated = TimeGenerated\n  ) \non Url\n| where Syslog_TimeGenerated >= TimeGenerated and Syslog_TimeGenerated < ExpirationDateTime\n| summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId\n| project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Syslog_TimeGenerated, SyslogMessage, Computer, ProcessName, Url, AdditionalInformation, HostIP\n| extend timestamp = Syslog_TimeGenerated, HostCustomEntity = Computer, IPCustomEntity = HostIP, URLCustomEntity = Url\n",
        "queryFrequency": "PT1H",
        "queryPeriod": "P1D",
        "severity": "Medium",
        "status": "Available",
        "suppressionDuration": "PT1H",
        "suppressionEnabled": false,
        "tactics": [
          "Impact"
        ],
        "techniques": null,
        "templateVersion": "1.0.0",
        "triggerOperator": "GreaterThan",
        "triggerThreshold": 0
      },
      "type": "Microsoft.OperationalInsights/workspaces/providers/alertRules"
    }
  ]
}