Anomaly found in Network Session Traffic (ASIM Network Session schema)

Back


Id	cd6def0d-3ef0-4d55-a7e3-faa96c46ba12
Rulename	Anomaly found in Network Session Traffic (ASIM Network Session schema)
Description	The rule identifies anomalous pattern in network session traffic based on previously seen data, different Device Action, Network Protocol, Network Direction or overall volume. The rule utilize ASIM normalization, and is applied to any source which supports the ASIM Network Session schema
Severity	Medium
Tactics	CommandAndControl Discovery Exfiltration LateralMovement
Techniques	T1095 T1071 T1046 T1030 T1210
Required data connectors	AIVectraStream AWSS3 AzureFirewall AzureMonitor(VMInsights) AzureNSG CheckPoint CiscoASA CiscoMeraki Corelight Fortinet MicrosoftSysmonForLinux MicrosoftThreatProtection PaloAltoNetworks SecurityEvents WindowsForwardedEvents Zscaler
Kind	Scheduled
Query frequency	1d
Query period	14d
Trigger threshold	0
Trigger operator	gt
Source Uri	https://github.com/Azure/Azure-Sentinel/blob/master/Solutions/Network Session Essentials/Analytic Rules/AnomalyFoundInNetworkSessionTraffic.yaml
Version	1.0.0
Arm template	cd6def0d-3ef0-4d55-a7e3-faa96c46ba12.json

KQL

let min_t = ago(14d);
let max_t = now();
let dt = 1d;
let fieldForDvcAction = "DvcAction";
let fieldForNetworkDirection = "NetworkDirection";
let fieldForNetworkProtocol = "NetworkProtocol";
let AnomalyThreshold = 2.5;
let eps = materialize (_Im_NetworkSession | project TimeGenerated | where TimeGenerated > ago(5m) | count | extend Count = Count/300);
let maxSummarizedTime = toscalar (
    union isfuzzy=true 
        (
            NetworkCustomAnalytics_protocol_CL
                | where EventTime_t > min_t
                | summarize max_TimeGenerated=max(EventTime_t)
                | extend max_TimeGenerated = datetime_add('minute',10,max_TimeGenerated)
        ),
        (
            print(min_t)
            | project max_TimeGenerated = print_0
        )
      | summarize maxTimeGenerated = max(max_TimeGenerated) 
    );
let nosummary = materialize(
              union isfuzzy=true 
                (
                    NetworkCustomAnalytics_protocol_CL
                    | where EventTime_t > ago(1d) 
                    | project v = int(2)
                ),
                (
                    print int(1) 
                    | project v = print_0
                )
                | summarize maxv = max(v)
                | extend nosum = (maxv > 1)
              );
let allData = union isfuzzy=true 
    (
        (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) > 1000 | join (nosummary) on nosum) | join (
        _Im_NetworkSession(starttime=todatetime(ago(2d)), endtime=now())
        | where TimeGenerated > maxSummarizedTime
        | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)
        | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)
        ) on exists
        | project-away exists, maxv, nosum*
    ),
    (
        (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) between (501 .. 1000) | join (nosummary) on nosum) | join (
        _Im_NetworkSession(starttime=todatetime(ago(3d)), endtime=now())
        | where TimeGenerated > maxSummarizedTime
        | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)
        | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)
        ) on exists
        | project-away exists, maxv, nosum*
    ),
    (
        (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) <= 500 | join (nosummary) on nosum) | join (
        _Im_NetworkSession(starttime=todatetime(ago(4d)), endtime=now())
        | where TimeGenerated > maxSummarizedTime
        | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)
        | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)
        ) on exists
        | project-away exists, maxv, nosum*
    ),
    (
        NetworkCustomAnalytics_protocol_CL
        | where EventTime_t > min_t
        | project-rename NetworkProtocol=NetworkProtocol_s, DstPortNumber=DstPortNumber_d, DstAppName=DstAppName_s, NetworkDirection=NetworkDirection_s, DvcAction=DvcAction_s, Count=count__d, EventTime=EventTime_t
        | extend Count = toint(Count),DstPortNumber = toint(DstPortNumber) 
    )
;
let findVolumneBasedAnomaly = allData
        | make-series total=sum(Count) on EventTime from min_t to max_t step dt
        | extend (anomalies, score, baseline) = series_decompose_anomalies(total, AnomalyThreshold, -1, 'linefit') 
        | mv-expand anomalies, score, baseline, EventTime, total
        | extend anomalies = toint(anomalies), score = toint(score), baseline = toint(baseline), EventTime = todatetime(EventTime), total = tolong(total)
        | where EventTime >= ago(1d)
        | where score >= 2*AnomalyThreshold
        ;
let findAnomalies = (field:string){
    allData
        | where isnotempty(column_ifexists(field,""))
        | make-series total=sum(Count) on EventTime from min_t to max_t step dt by column_ifexists(field,"")
        | extend (anomalies, score, baseline) = series_decompose_anomalies(total, AnomalyThreshold, -1, 'linefit')
        | mv-expand anomalies, score, baseline, EventTime, total
        | extend anomalies = toint(anomalies), score = toint(score), baseline = toint(baseline), EventTime = todatetime(EventTime), total = tolong(total)
        | where EventTime >= ago(1d)
        | where score >= 2*AnomalyThreshold
};
union findAnomalies(fieldForDvcAction), findAnomalies(fieldForNetworkDirection), findAnomalies(fieldForNetworkProtocol), findVolumneBasedAnomaly
| extend anomalyFieldType  = case (isnotempty(column_ifexists(fieldForDvcAction,"")), "DvcAction",
                                  isnotempty(column_ifexists(fieldForNetworkDirection,"")), "NetworkDirection",
                                  isnotempty(column_ifexists(fieldForNetworkProtocol,"")), "NetworkProtocol",
                                  "TotalVolume"
                                ),
        anomalyFieldValue  = case (isnotempty(column_ifexists(fieldForDvcAction,"")), column_ifexists(fieldForDvcAction,""),
                                  isnotempty(column_ifexists(fieldForNetworkDirection,"")), column_ifexists(fieldForNetworkDirection,""),
                                  isnotempty(column_ifexists(fieldForNetworkProtocol,"")), column_ifexists(fieldForNetworkProtocol,""),
                                  "Overall"
                                )

YAML

eventGroupingSettings:
  aggregationKind: AlertPerResult
OriginalUri: https://github.com/Azure/Azure-Sentinel/blob/master/Solutions/Network Session Essentials/Analytic Rules/AnomalyFoundInNetworkSessionTraffic.yaml
severity: Medium
tags:
- SchemaVersion: 0.2.4
  Schema: ASimNetworkSessions
name: Anomaly found in Network Session Traffic (ASIM Network Session schema)
relevantTechniques:
- T1095
- T1071
- T1046
- T1030
- T1210
queryFrequency: 1d
triggerThreshold: 0
queryPeriod: 14d
description: |
    'The rule identifies anomalous pattern in network session traffic based on previously seen data, different Device Action, Network Protocol, Network Direction or overall volume. The rule utilize [ASIM](https://aka.ms/AboutASIM) normalization, and is applied to any source which supports the ASIM Network Session schema'
id: cd6def0d-3ef0-4d55-a7e3-faa96c46ba12
alertDetailsOverride:
  alertDescriptionFormat: Based on past data, anomaly was observed in {{anomalyFieldValue}} Traffic with a score of {{score}}.
  alertDisplayNameFormat: Anomaly was observed with {{anomalyFieldValue}} Traffic
customDetails:
  AnomalyFieldType: anomalyFieldType
  AnomalyFieldValue: anomalyFieldValue
  Score: score
version: 1.0.0
tactics:
- CommandAndControl
- Discovery
- Exfiltration
- LateralMovement
query: |
  let min_t = ago(14d);
  let max_t = now();
  let dt = 1d;
  let fieldForDvcAction = "DvcAction";
  let fieldForNetworkDirection = "NetworkDirection";
  let fieldForNetworkProtocol = "NetworkProtocol";
  let AnomalyThreshold = 2.5;
  let eps = materialize (_Im_NetworkSession | project TimeGenerated | where TimeGenerated > ago(5m) | count | extend Count = Count/300);
  let maxSummarizedTime = toscalar (
      union isfuzzy=true 
          (
              NetworkCustomAnalytics_protocol_CL
                  | where EventTime_t > min_t
                  | summarize max_TimeGenerated=max(EventTime_t)
                  | extend max_TimeGenerated = datetime_add('minute',10,max_TimeGenerated)
          ),
          (
              print(min_t)
              | project max_TimeGenerated = print_0
          )
        | summarize maxTimeGenerated = max(max_TimeGenerated) 
      );
  let nosummary = materialize(
                union isfuzzy=true 
                  (
                      NetworkCustomAnalytics_protocol_CL
                      | where EventTime_t > ago(1d) 
                      | project v = int(2)
                  ),
                  (
                      print int(1) 
                      | project v = print_0
                  )
                  | summarize maxv = max(v)
                  | extend nosum = (maxv > 1)
                );
  let allData = union isfuzzy=true 
      (
          (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) > 1000 | join (nosummary) on nosum) | join (
          _Im_NetworkSession(starttime=todatetime(ago(2d)), endtime=now())
          | where TimeGenerated > maxSummarizedTime
          | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)
          | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)
          ) on exists
          | project-away exists, maxv, nosum*
      ),
      (
          (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) between (501 .. 1000) | join (nosummary) on nosum) | join (
          _Im_NetworkSession(starttime=todatetime(ago(3d)), endtime=now())
          | where TimeGenerated > maxSummarizedTime
          | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)
          | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)
          ) on exists
          | project-away exists, maxv, nosum*
      ),
      (
          (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) <= 500 | join (nosummary) on nosum) | join (
          _Im_NetworkSession(starttime=todatetime(ago(4d)), endtime=now())
          | where TimeGenerated > maxSummarizedTime
          | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)
          | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)
          ) on exists
          | project-away exists, maxv, nosum*
      ),
      (
          NetworkCustomAnalytics_protocol_CL
          | where EventTime_t > min_t
          | project-rename NetworkProtocol=NetworkProtocol_s, DstPortNumber=DstPortNumber_d, DstAppName=DstAppName_s, NetworkDirection=NetworkDirection_s, DvcAction=DvcAction_s, Count=count__d, EventTime=EventTime_t
          | extend Count = toint(Count),DstPortNumber = toint(DstPortNumber) 
      )
  ;
  let findVolumneBasedAnomaly = allData
          | make-series total=sum(Count) on EventTime from min_t to max_t step dt
          | extend (anomalies, score, baseline) = series_decompose_anomalies(total, AnomalyThreshold, -1, 'linefit') 
          | mv-expand anomalies, score, baseline, EventTime, total
          | extend anomalies = toint(anomalies), score = toint(score), baseline = toint(baseline), EventTime = todatetime(EventTime), total = tolong(total)
          | where EventTime >= ago(1d)
          | where score >= 2*AnomalyThreshold
          ;
  let findAnomalies = (field:string){
      allData
          | where isnotempty(column_ifexists(field,""))
          | make-series total=sum(Count) on EventTime from min_t to max_t step dt by column_ifexists(field,"")
          | extend (anomalies, score, baseline) = series_decompose_anomalies(total, AnomalyThreshold, -1, 'linefit')
          | mv-expand anomalies, score, baseline, EventTime, total
          | extend anomalies = toint(anomalies), score = toint(score), baseline = toint(baseline), EventTime = todatetime(EventTime), total = tolong(total)
          | where EventTime >= ago(1d)
          | where score >= 2*AnomalyThreshold
  };
  union findAnomalies(fieldForDvcAction), findAnomalies(fieldForNetworkDirection), findAnomalies(fieldForNetworkProtocol), findVolumneBasedAnomaly
  | extend anomalyFieldType  = case (isnotempty(column_ifexists(fieldForDvcAction,"")), "DvcAction",
                                    isnotempty(column_ifexists(fieldForNetworkDirection,"")), "NetworkDirection",
                                    isnotempty(column_ifexists(fieldForNetworkProtocol,"")), "NetworkProtocol",
                                    "TotalVolume"
                                  ),
          anomalyFieldValue  = case (isnotempty(column_ifexists(fieldForDvcAction,"")), column_ifexists(fieldForDvcAction,""),
                                    isnotempty(column_ifexists(fieldForNetworkDirection,"")), column_ifexists(fieldForNetworkDirection,""),
                                    isnotempty(column_ifexists(fieldForNetworkProtocol,"")), column_ifexists(fieldForNetworkProtocol,""),
                                    "Overall"
                                  )  
status: Available
kind: Scheduled
requiredDataConnectors:
- dataTypes:
  - AWSVPCFlow
  connectorId: AWSS3
- dataTypes:
  - DeviceNetworkEvents
  connectorId: MicrosoftThreatProtection
- dataTypes:
  - SecurityEvent
  connectorId: SecurityEvents
- dataTypes:
  - WindowsEvent
  connectorId: WindowsForwardedEvents
- dataTypes:
  - CommonSecurityLog
  connectorId: Zscaler
- dataTypes:
  - Syslog
  connectorId: MicrosoftSysmonForLinux
- dataTypes:
  - CommonSecurityLog
  connectorId: PaloAltoNetworks
- dataTypes:
  - VMConnection
  connectorId: AzureMonitor(VMInsights)
- dataTypes:
  - AzureDiagnostics
  connectorId: AzureFirewall
- dataTypes:
  - AzureDiagnostics
  connectorId: AzureNSG
- dataTypes:
  - CommonSecurityLog
  connectorId: CiscoASA
- dataTypes:
  - Corelight_CL
  connectorId: Corelight
- dataTypes:
  - VectraStream
  connectorId: AIVectraStream
- dataTypes:
  - CommonSecurityLog
  connectorId: CheckPoint
- dataTypes:
  - CommonSecurityLog
  connectorId: Fortinet
- dataTypes:
  - Syslog
  - CiscoMerakiNativePoller
  connectorId: CiscoMeraki
triggerOperator: gt

ARM

{
  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
  "contentVersion": "1.0.0.0",
  "parameters": {
    "workspace": {
      "type": "String"
    }
  },
  "resources": [
    {
      "id": "[concat(resourceId('Microsoft.OperationalInsights/workspaces/providers', parameters('workspace'), 'Microsoft.SecurityInsights'),'/alertRules/cd6def0d-3ef0-4d55-a7e3-faa96c46ba12')]",
      "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/cd6def0d-3ef0-4d55-a7e3-faa96c46ba12')]",
      "type": "Microsoft.OperationalInsights/workspaces/providers/alertRules",
      "kind": "Scheduled",
      "apiVersion": "2022-11-01-preview",
      "properties": {
        "displayName": "Anomaly found in Network Session Traffic (ASIM Network Session schema)",
        "description": "'The rule identifies anomalous pattern in network session traffic based on previously seen data, different Device Action, Network Protocol, Network Direction or overall volume. The rule utilize [ASIM](https://aka.ms/AboutASIM) normalization, and is applied to any source which supports the ASIM Network Session schema'\n",
        "severity": "Medium",
        "enabled": true,
        "query": "let min_t = ago(14d);\nlet max_t = now();\nlet dt = 1d;\nlet fieldForDvcAction = \"DvcAction\";\nlet fieldForNetworkDirection = \"NetworkDirection\";\nlet fieldForNetworkProtocol = \"NetworkProtocol\";\nlet AnomalyThreshold = 2.5;\nlet eps = materialize (_Im_NetworkSession | project TimeGenerated | where TimeGenerated > ago(5m) | count | extend Count = Count/300);\nlet maxSummarizedTime = toscalar (\n    union isfuzzy=true \n        (\n            NetworkCustomAnalytics_protocol_CL\n                | where EventTime_t > min_t\n                | summarize max_TimeGenerated=max(EventTime_t)\n                | extend max_TimeGenerated = datetime_add('minute',10,max_TimeGenerated)\n        ),\n        (\n            print(min_t)\n            | project max_TimeGenerated = print_0\n        )\n      | summarize maxTimeGenerated = max(max_TimeGenerated) \n    );\nlet nosummary = materialize(\n              union isfuzzy=true \n                (\n                    NetworkCustomAnalytics_protocol_CL\n                    | where EventTime_t > ago(1d) \n                    | project v = int(2)\n                ),\n                (\n                    print int(1) \n                    | project v = print_0\n                )\n                | summarize maxv = max(v)\n                | extend nosum = (maxv > 1)\n              );\nlet allData = union isfuzzy=true \n    (\n        (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) > 1000 | join (nosummary) on nosum) | join (\n        _Im_NetworkSession(starttime=todatetime(ago(2d)), endtime=now())\n        | where TimeGenerated > maxSummarizedTime\n        | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)\n        | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)\n        ) on exists\n        | project-away exists, maxv, nosum*\n    ),\n    (\n        (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) between (501 .. 1000) | join (nosummary) on nosum) | join (\n        _Im_NetworkSession(starttime=todatetime(ago(3d)), endtime=now())\n        | where TimeGenerated > maxSummarizedTime\n        | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)\n        | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)\n        ) on exists\n        | project-away exists, maxv, nosum*\n    ),\n    (\n        (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) <= 500 | join (nosummary) on nosum) | join (\n        _Im_NetworkSession(starttime=todatetime(ago(4d)), endtime=now())\n        | where TimeGenerated > maxSummarizedTime\n        | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)\n        | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)\n        ) on exists\n        | project-away exists, maxv, nosum*\n    ),\n    (\n        NetworkCustomAnalytics_protocol_CL\n        | where EventTime_t > min_t\n        | project-rename NetworkProtocol=NetworkProtocol_s, DstPortNumber=DstPortNumber_d, DstAppName=DstAppName_s, NetworkDirection=NetworkDirection_s, DvcAction=DvcAction_s, Count=count__d, EventTime=EventTime_t\n        | extend Count = toint(Count),DstPortNumber = toint(DstPortNumber) \n    )\n;\nlet findVolumneBasedAnomaly = allData\n        | make-series total=sum(Count) on EventTime from min_t to max_t step dt\n        | extend (anomalies, score, baseline) = series_decompose_anomalies(total, AnomalyThreshold, -1, 'linefit') \n        | mv-expand anomalies, score, baseline, EventTime, total\n        | extend anomalies = toint(anomalies), score = toint(score), baseline = toint(baseline), EventTime = todatetime(EventTime), total = tolong(total)\n        | where EventTime >= ago(1d)\n        | where score >= 2*AnomalyThreshold\n        ;\nlet findAnomalies = (field:string){\n    allData\n        | where isnotempty(column_ifexists(field,\"\"))\n        | make-series total=sum(Count) on EventTime from min_t to max_t step dt by column_ifexists(field,\"\")\n        | extend (anomalies, score, baseline) = series_decompose_anomalies(total, AnomalyThreshold, -1, 'linefit')\n        | mv-expand anomalies, score, baseline, EventTime, total\n        | extend anomalies = toint(anomalies), score = toint(score), baseline = toint(baseline), EventTime = todatetime(EventTime), total = tolong(total)\n        | where EventTime >= ago(1d)\n        | where score >= 2*AnomalyThreshold\n};\nunion findAnomalies(fieldForDvcAction), findAnomalies(fieldForNetworkDirection), findAnomalies(fieldForNetworkProtocol), findVolumneBasedAnomaly\n| extend anomalyFieldType  = case (isnotempty(column_ifexists(fieldForDvcAction,\"\")), \"DvcAction\",\n                                  isnotempty(column_ifexists(fieldForNetworkDirection,\"\")), \"NetworkDirection\",\n                                  isnotempty(column_ifexists(fieldForNetworkProtocol,\"\")), \"NetworkProtocol\",\n                                  \"TotalVolume\"\n                                ),\n        anomalyFieldValue  = case (isnotempty(column_ifexists(fieldForDvcAction,\"\")), column_ifexists(fieldForDvcAction,\"\"),\n                                  isnotempty(column_ifexists(fieldForNetworkDirection,\"\")), column_ifexists(fieldForNetworkDirection,\"\"),\n                                  isnotempty(column_ifexists(fieldForNetworkProtocol,\"\")), column_ifexists(fieldForNetworkProtocol,\"\"),\n                                  \"Overall\"\n                                )\n",
        "queryFrequency": "P1D",
        "queryPeriod": "P14D",
        "triggerOperator": "GreaterThan",
        "triggerThreshold": 0,
        "suppressionDuration": "PT1H",
        "suppressionEnabled": false,
        "tactics": [
          "CommandAndControl",
          "Discovery",
          "Exfiltration",
          "LateralMovement"
        ],
        "techniques": [
          "T1095",
          "T1071",
          "T1046",
          "T1030",
          "T1210"
        ],
        "alertRuleTemplateName": "cd6def0d-3ef0-4d55-a7e3-faa96c46ba12",
        "eventGroupingSettings": {
          "aggregationKind": "AlertPerResult"
        },
        "alertDetailsOverride": {
          "alertDisplayNameFormat": "Anomaly was observed with {{anomalyFieldValue}} Traffic",
          "alertDescriptionFormat": "Based on past data, anomaly was observed in {{anomalyFieldValue}} Traffic with a score of {{score}}."
        },
        "customDetails": {
          "AnomalyFieldType": "anomalyFieldType",
          "AnomalyFieldValue": "anomalyFieldValue",
          "Score": "score"
        },
        "entityMappings": null,
        "templateVersion": "1.0.0",
        "status": "Available",
        "tags": [
          {
            "Schema": "ASimNetworkSessions",
            "SchemaVersion": "0.2.4"
          }
        ],
        "OriginalUri": "https://github.com/Azure/Azure-Sentinel/blob/master/Solutions/Network Session Essentials/Analytic Rules/AnomalyFoundInNetworkSessionTraffic.yaml"
      }
    }
  ]
}