Detect port misuse by anomaly based detection ASIM Network Session schema

let lookback = 14d;
let mapping = _GetWatchlist('NetworkSession_Monitor_Configuration')
| where Type == "Detection" and ThresholdType == "Anomaly" and Severity != "Disabled"  
| extend Ports = split(Ports,","),
        App = split(App,","),
        Protocol = split(Protocol,","),
        Direction = split(Direction,","),
        Action = split(Action,",")
| project Ports, App, Protocol, Direction, Action, Type, ThresholdType, Threshold, Severity, Tactic, Name, Description
| mv-expand Ports
| mv-expand App
| mv-expand Protocol
| mv-expand Direction
| mv-expand Action
| extend Ports = tostring(Ports), App = tostring(App), Protocol = tostring(Protocol), Direction = tostring(Direction), Action = tostring(Action), Threshold = toint(Threshold)
;
let AnomalyThreshold = 2.5;
let eps = materialize (_Im_NetworkSession | project TimeGenerated | where TimeGenerated > ago(5m) | count | extend Count = Count/300);
let maxSummarizedTime = toscalar (
    union isfuzzy=true 
        (
            NetworkCustomAnalytics_protocol_CL
                | where EventTime_t > ago(lookback)
                | summarize max_TimeGenerated=max(EventTime_t)
                | extend max_TimeGenerated = datetime_add('minute',10,max_TimeGenerated)
        ),
        (
            print(ago(lookback))
            | project max_TimeGenerated = print_0
        )
      | summarize maxTimeGenerated = max(max_TimeGenerated) 
    );
let nosummary = materialize(
              union isfuzzy=true 
                (
                    NetworkCustomAnalytics_protocol_CL
                    | where EventTime_t > ago(1d) 
                    | project v = int(2)
                ),
                (
                    print int(1) 
                    | project v = print_0
                )
                | summarize maxv = max(v)
                | extend nosum = (maxv > 1)
              );
let allData = union isfuzzy=true 
    (
        (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) > 1000 | join (nosummary) on nosum) | join (
        _Im_NetworkSession(starttime=todatetime(ago(2d)), endtime=now())
        | where TimeGenerated > maxSummarizedTime
        | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)
        | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)
        ) on exists
        | project-away exists, maxv, nosum*
    ),
    (
        (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) between (501 .. 1000) | join (nosummary) on nosum) | join (
        _Im_NetworkSession(starttime=todatetime(ago(3d)), endtime=now())
        | where TimeGenerated > maxSummarizedTime
        | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)
        | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)
        ) on exists
        | project-away exists, maxv, nosum*
    ),
    (
        (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) <= 500 | join (nosummary) on nosum) | join (
        _Im_NetworkSession(starttime=todatetime(ago(4d)), endtime=now())
        | where TimeGenerated > maxSummarizedTime
        | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)
        | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)
        ) on exists
        | project-away exists, maxv, nosum*
    ),
    (
        NetworkCustomAnalytics_protocol_CL
        | where EventTime_t > ago(lookback)
        | project-rename NetworkProtocol=NetworkProtocol_s, DstPortNumber=DstPortNumber_d, DstAppName=DstAppName_s, NetworkDirection=NetworkDirection_s, DvcAction=DvcAction_s, Count=count__d, EventTime=EventTime_t
        | extend Count = toint(Count),DstPortNumber = toint(DstPortNumber) 
    )
;
allData
| where isnotempty(DstPortNumber)
| make-series Total=sum(Count) on EventTime from ago(lookback) to now() step 1d by DstPortNumber, NetworkProtocol, NetworkDirection, DvcAction
| extend (anomalies, score, baseline) = series_decompose_anomalies(Total, AnomalyThreshold, -1, 'linefit')
| mv-expand anomalies, score, baseline, EventTime, Total
| extend anomalies = toint(anomalies), score = toint(score), baseline = toint(baseline), EventTime = todatetime(EventTime), Total = tolong(Total)
| where EventTime >= ago(1d)
| extend DstPortNumber = trim_end(".0",tostring(DstPortNumber))
| where score > 2*AnomalyThreshold
| join kind=inner ['mapping'] where Ports == DstPortNumber
| where (Protocol == "*" or Protocol has NetworkProtocol)
                and (Direction == "*" or Direction has NetworkDirection)
                and (Action == "*" or Action has DvcAction)  
| project Name, Description, NetworkProtocol, DstPortNumber, NetworkDirection, DvcAction, Severity, Tactic
| summarize NetworkProtocols=make_set_if(NetworkProtocol,isnotempty(NetworkProtocol),20), 
                    NetworkDirections=make_set_if(NetworkDirection,isnotempty(NetworkDirection),5), 
                    DvcActions=make_set_if(DvcAction,isnotempty(DvcAction),10) by Name, Severity, Tactic, DstPortNumber, Description

customDetails:
  AllNetworkProtocols: NetworkProtocols
  AllDvcAction: DvcActions
  AllNetworkDirections: NetworkDirections
  DstPortNumber: DstPortNumber
name: Detect port misuse by anomaly based detection (ASIM Network Session schema)
tactics:
- CommandAndControl
- LateralMovement
- Execution
- InitialAccess
severity: Medium
triggerThreshold: 0
relevantTechniques:
- T1095
- T1059
- T1203
- T1190
description: |
    'This rule detects anomalous pattern in port usage. The rule utilize [ASIM](https://aka.ms/AboutASIM) normalization, and is applied to any source which supports the ASIM Network Session schema. To tune the rule to your environment configure it using the 'NetworkSession_Monitor_Configuration' watchlist. Note that to enhance performance, the rule uses summarized data generated from the summarization logic App.'
id: cbf07406-fa2a-48b0-82b8-efad58db14ec
version: 1.0.2
OriginalUri: https://github.com/Azure/Azure-Sentinel/blob/master/Solutions/Network Session Essentials/Analytic Rules/DetectPortMisuseByAnomalyBasedDetection.yaml
queryFrequency: 1d
eventGroupingSettings:
  aggregationKind: AlertPerResult
triggerOperator: gt
query: |
  let lookback = 14d;
  let mapping = _GetWatchlist('NetworkSession_Monitor_Configuration')
  | where Type == "Detection" and ThresholdType == "Anomaly" and Severity != "Disabled"  
  | extend Ports = split(Ports,","),
          App = split(App,","),
          Protocol = split(Protocol,","),
          Direction = split(Direction,","),
          Action = split(Action,",")
  | project Ports, App, Protocol, Direction, Action, Type, ThresholdType, Threshold, Severity, Tactic, Name, Description
  | mv-expand Ports
  | mv-expand App
  | mv-expand Protocol
  | mv-expand Direction
  | mv-expand Action
  | extend Ports = tostring(Ports), App = tostring(App), Protocol = tostring(Protocol), Direction = tostring(Direction), Action = tostring(Action), Threshold = toint(Threshold)
  ;
  let AnomalyThreshold = 2.5;
  let eps = materialize (_Im_NetworkSession | project TimeGenerated | where TimeGenerated > ago(5m) | count | extend Count = Count/300);
  let maxSummarizedTime = toscalar (
      union isfuzzy=true 
          (
              NetworkCustomAnalytics_protocol_CL
                  | where EventTime_t > ago(lookback)
                  | summarize max_TimeGenerated=max(EventTime_t)
                  | extend max_TimeGenerated = datetime_add('minute',10,max_TimeGenerated)
          ),
          (
              print(ago(lookback))
              | project max_TimeGenerated = print_0
          )
        | summarize maxTimeGenerated = max(max_TimeGenerated) 
      );
  let nosummary = materialize(
                union isfuzzy=true 
                  (
                      NetworkCustomAnalytics_protocol_CL
                      | where EventTime_t > ago(1d) 
                      | project v = int(2)
                  ),
                  (
                      print int(1) 
                      | project v = print_0
                  )
                  | summarize maxv = max(v)
                  | extend nosum = (maxv > 1)
                );
  let allData = union isfuzzy=true 
      (
          (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) > 1000 | join (nosummary) on nosum) | join (
          _Im_NetworkSession(starttime=todatetime(ago(2d)), endtime=now())
          | where TimeGenerated > maxSummarizedTime
          | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)
          | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)
          ) on exists
          | project-away exists, maxv, nosum*
      ),
      (
          (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) between (501 .. 1000) | join (nosummary) on nosum) | join (
          _Im_NetworkSession(starttime=todatetime(ago(3d)), endtime=now())
          | where TimeGenerated > maxSummarizedTime
          | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)
          | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)
          ) on exists
          | project-away exists, maxv, nosum*
      ),
      (
          (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) <= 500 | join (nosummary) on nosum) | join (
          _Im_NetworkSession(starttime=todatetime(ago(4d)), endtime=now())
          | where TimeGenerated > maxSummarizedTime
          | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)
          | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)
          ) on exists
          | project-away exists, maxv, nosum*
      ),
      (
          NetworkCustomAnalytics_protocol_CL
          | where EventTime_t > ago(lookback)
          | project-rename NetworkProtocol=NetworkProtocol_s, DstPortNumber=DstPortNumber_d, DstAppName=DstAppName_s, NetworkDirection=NetworkDirection_s, DvcAction=DvcAction_s, Count=count__d, EventTime=EventTime_t
          | extend Count = toint(Count),DstPortNumber = toint(DstPortNumber) 
      )
  ;
  allData
  | where isnotempty(DstPortNumber)
  | make-series Total=sum(Count) on EventTime from ago(lookback) to now() step 1d by DstPortNumber, NetworkProtocol, NetworkDirection, DvcAction
  | extend (anomalies, score, baseline) = series_decompose_anomalies(Total, AnomalyThreshold, -1, 'linefit')
  | mv-expand anomalies, score, baseline, EventTime, Total
  | extend anomalies = toint(anomalies), score = toint(score), baseline = toint(baseline), EventTime = todatetime(EventTime), Total = tolong(Total)
  | where EventTime >= ago(1d)
  | extend DstPortNumber = trim_end(".0",tostring(DstPortNumber))
  | where score > 2*AnomalyThreshold
  | join kind=inner ['mapping'] where Ports == DstPortNumber
  | where (Protocol == "*" or Protocol has NetworkProtocol)
                  and (Direction == "*" or Direction has NetworkDirection)
                  and (Action == "*" or Action has DvcAction)  
  | project Name, Description, NetworkProtocol, DstPortNumber, NetworkDirection, DvcAction, Severity, Tactic
  | summarize NetworkProtocols=make_set_if(NetworkProtocol,isnotempty(NetworkProtocol),20), 
                      NetworkDirections=make_set_if(NetworkDirection,isnotempty(NetworkDirection),5), 
                      DvcActions=make_set_if(DvcAction,isnotempty(DvcAction),10) by Name, Severity, Tactic, DstPortNumber, Description  
tags:
- SchemaVersion: 0.2.4
  Schema: ASimNetworkSessions
requiredDataConnectors:
- connectorId: AWSS3
  dataTypes:
  - AWSVPCFlow
- connectorId: MicrosoftThreatProtection
  dataTypes:
  - DeviceNetworkEvents
- connectorId: SecurityEvents
  dataTypes:
  - SecurityEvent
- connectorId: WindowsSecurityEvents
  dataTypes:
  - SecurityEvent
- connectorId: WindowsForwardedEvents
  dataTypes:
  - WindowsEvent
- connectorId: Zscaler
  dataTypes:
  - CommonSecurityLog
- connectorId: MicrosoftSysmonForLinux
  dataTypes:
  - Syslog
- connectorId: PaloAltoNetworks
  dataTypes:
  - CommonSecurityLog
- connectorId: AzureMonitor(VMInsights)
  dataTypes:
  - VMConnection
- connectorId: AzureFirewall
  dataTypes:
  - AzureDiagnostics
- connectorId: AzureNSG
  dataTypes:
  - AzureDiagnostics
- connectorId: CiscoASA
  dataTypes:
  - CommonSecurityLog
- connectorId: CiscoAsaAma
  dataTypes:
  - CommonSecurityLog
- connectorId: Corelight
  dataTypes:
  - Corelight_CL
- connectorId: AIVectraStream
  dataTypes:
  - VectraStream
- connectorId: CheckPoint
  dataTypes:
  - CommonSecurityLog
- connectorId: Fortinet
  dataTypes:
  - CommonSecurityLog
- connectorId: CiscoMeraki
  dataTypes:
  - Syslog
  - CiscoMerakiNativePoller
status: Available
queryPeriod: 14d
alertDetailsOverride:
  alertSeverityColumnName: Severity
  alertDescriptionFormat: '{{Description}}'
  alertTacticsColumnName: Tactic
  alertDisplayNameFormat: Detected {{Name}}
kind: Scheduled

{
  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
  "contentVersion": "1.0.0.0",
  "parameters": {
    "workspace": {
      "type": "String"
    }
  },
  "resources": [
    {
      "apiVersion": "2024-01-01-preview",
      "id": "[concat(resourceId('Microsoft.OperationalInsights/workspaces/providers', parameters('workspace'), 'Microsoft.SecurityInsights'),'/alertRules/cbf07406-fa2a-48b0-82b8-efad58db14ec')]",
      "kind": "Scheduled",
      "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/cbf07406-fa2a-48b0-82b8-efad58db14ec')]",
      "properties": {
        "alertDetailsOverride": {
          "alertDescriptionFormat": "{{Description}}",
          "alertDisplayNameFormat": "Detected {{Name}}",
          "alertSeverityColumnName": "Severity",
          "alertTacticsColumnName": "Tactic"
        },
        "alertRuleTemplateName": "cbf07406-fa2a-48b0-82b8-efad58db14ec",
        "customDetails": {
          "AllDvcAction": "DvcActions",
          "AllNetworkDirections": "NetworkDirections",
          "AllNetworkProtocols": "NetworkProtocols",
          "DstPortNumber": "DstPortNumber"
        },
        "description": "'This rule detects anomalous pattern in port usage. The rule utilize [ASIM](https://aka.ms/AboutASIM) normalization, and is applied to any source which supports the ASIM Network Session schema. To tune the rule to your environment configure it using the 'NetworkSession_Monitor_Configuration' watchlist. Note that to enhance performance, the rule uses summarized data generated from the summarization logic App.'\n",
        "displayName": "Detect port misuse by anomaly based detection (ASIM Network Session schema)",
        "enabled": true,
        "entityMappings": null,
        "eventGroupingSettings": {
          "aggregationKind": "AlertPerResult"
        },
        "OriginalUri": "https://github.com/Azure/Azure-Sentinel/blob/master/Solutions/Network Session Essentials/Analytic Rules/DetectPortMisuseByAnomalyBasedDetection.yaml",
        "query": "let lookback = 14d;\nlet mapping = _GetWatchlist('NetworkSession_Monitor_Configuration')\n| where Type == \"Detection\" and ThresholdType == \"Anomaly\" and Severity != \"Disabled\"  \n| extend Ports = split(Ports,\",\"),\n        App = split(App,\",\"),\n        Protocol = split(Protocol,\",\"),\n        Direction = split(Direction,\",\"),\n        Action = split(Action,\",\")\n| project Ports, App, Protocol, Direction, Action, Type, ThresholdType, Threshold, Severity, Tactic, Name, Description\n| mv-expand Ports\n| mv-expand App\n| mv-expand Protocol\n| mv-expand Direction\n| mv-expand Action\n| extend Ports = tostring(Ports), App = tostring(App), Protocol = tostring(Protocol), Direction = tostring(Direction), Action = tostring(Action), Threshold = toint(Threshold)\n;\nlet AnomalyThreshold = 2.5;\nlet eps = materialize (_Im_NetworkSession | project TimeGenerated | where TimeGenerated > ago(5m) | count | extend Count = Count/300);\nlet maxSummarizedTime = toscalar (\n    union isfuzzy=true \n        (\n            NetworkCustomAnalytics_protocol_CL\n                | where EventTime_t > ago(lookback)\n                | summarize max_TimeGenerated=max(EventTime_t)\n                | extend max_TimeGenerated = datetime_add('minute',10,max_TimeGenerated)\n        ),\n        (\n            print(ago(lookback))\n            | project max_TimeGenerated = print_0\n        )\n      | summarize maxTimeGenerated = max(max_TimeGenerated) \n    );\nlet nosummary = materialize(\n              union isfuzzy=true \n                (\n                    NetworkCustomAnalytics_protocol_CL\n                    | where EventTime_t > ago(1d) \n                    | project v = int(2)\n                ),\n                (\n                    print int(1) \n                    | project v = print_0\n                )\n                | summarize maxv = max(v)\n                | extend nosum = (maxv > 1)\n              );\nlet allData = union isfuzzy=true \n    (\n        (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) > 1000 | join (nosummary) on nosum) | join (\n        _Im_NetworkSession(starttime=todatetime(ago(2d)), endtime=now())\n        | where TimeGenerated > maxSummarizedTime\n        | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)\n        | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)\n        ) on exists\n        | project-away exists, maxv, nosum*\n    ),\n    (\n        (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) between (501 .. 1000) | join (nosummary) on nosum) | join (\n        _Im_NetworkSession(starttime=todatetime(ago(3d)), endtime=now())\n        | where TimeGenerated > maxSummarizedTime\n        | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)\n        | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)\n        ) on exists\n        | project-away exists, maxv, nosum*\n    ),\n    (\n        (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) <= 500 | join (nosummary) on nosum) | join (\n        _Im_NetworkSession(starttime=todatetime(ago(4d)), endtime=now())\n        | where TimeGenerated > maxSummarizedTime\n        | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)\n        | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)\n        ) on exists\n        | project-away exists, maxv, nosum*\n    ),\n    (\n        NetworkCustomAnalytics_protocol_CL\n        | where EventTime_t > ago(lookback)\n        | project-rename NetworkProtocol=NetworkProtocol_s, DstPortNumber=DstPortNumber_d, DstAppName=DstAppName_s, NetworkDirection=NetworkDirection_s, DvcAction=DvcAction_s, Count=count__d, EventTime=EventTime_t\n        | extend Count = toint(Count),DstPortNumber = toint(DstPortNumber) \n    )\n;\nallData\n| where isnotempty(DstPortNumber)\n| make-series Total=sum(Count) on EventTime from ago(lookback) to now() step 1d by DstPortNumber, NetworkProtocol, NetworkDirection, DvcAction\n| extend (anomalies, score, baseline) = series_decompose_anomalies(Total, AnomalyThreshold, -1, 'linefit')\n| mv-expand anomalies, score, baseline, EventTime, Total\n| extend anomalies = toint(anomalies), score = toint(score), baseline = toint(baseline), EventTime = todatetime(EventTime), Total = tolong(Total)\n| where EventTime >= ago(1d)\n| extend DstPortNumber = trim_end(\".0\",tostring(DstPortNumber))\n| where score > 2*AnomalyThreshold\n| join kind=inner ['mapping'] where Ports == DstPortNumber\n| where (Protocol == \"*\" or Protocol has NetworkProtocol)\n                and (Direction == \"*\" or Direction has NetworkDirection)\n                and (Action == \"*\" or Action has DvcAction)  \n| project Name, Description, NetworkProtocol, DstPortNumber, NetworkDirection, DvcAction, Severity, Tactic\n| summarize NetworkProtocols=make_set_if(NetworkProtocol,isnotempty(NetworkProtocol),20), \n                    NetworkDirections=make_set_if(NetworkDirection,isnotempty(NetworkDirection),5), \n                    DvcActions=make_set_if(DvcAction,isnotempty(DvcAction),10) by Name, Severity, Tactic, DstPortNumber, Description\n",
        "queryFrequency": "P1D",
        "queryPeriod": "P14D",
        "severity": "Medium",
        "status": "Available",
        "subTechniques": [],
        "suppressionDuration": "PT1H",
        "suppressionEnabled": false,
        "tactics": [
          "CommandAndControl",
          "Execution",
          "InitialAccess",
          "LateralMovement"
        ],
        "tags": [
          {
            "Schema": "ASimNetworkSessions",
            "SchemaVersion": "0.2.4"
          }
        ],
        "techniques": [
          "T1059",
          "T1095",
          "T1190",
          "T1203"
        ],
        "templateVersion": "1.0.2",
        "triggerOperator": "GreaterThan",
        "triggerThreshold": 0
      },
      "type": "Microsoft.OperationalInsights/workspaces/providers/alertRules"
    }
  ]
}