Anomaly found in Network Session Traffic (ASIM Network Session schema)

let min_t = ago(14d);
let max_t = now();
let dt = 1d;
let fieldForDvcAction = "DvcAction";
let fieldForNetworkDirection = "NetworkDirection";
let fieldForNetworkProtocol = "NetworkProtocol";
let AnomalyThreshold = 2.5;
let eps = materialize (_Im_NetworkSession | project TimeGenerated | where TimeGenerated > ago(5m) | count | extend Count = Count/300);
let maxSummarizedTime = toscalar (
    union isfuzzy=true 
        (
            NetworkCustomAnalytics_protocol_CL
                | where EventTime_t > min_t
                | summarize max_TimeGenerated=max(EventTime_t)
                | extend max_TimeGenerated = datetime_add('minute',10,max_TimeGenerated)
        ),
        (
            print(min_t)
            | project max_TimeGenerated = print_0
        )
      | summarize maxTimeGenerated = max(max_TimeGenerated) 
    );
let nosummary = materialize(
              union isfuzzy=true 
                (
                    NetworkCustomAnalytics_protocol_CL
                    | where EventTime_t > ago(1d) 
                    | project v = int(2)
                ),
                (
                    print int(1) 
                    | project v = print_0
                )
                | summarize maxv = max(v)
                | extend nosum = (maxv > 1)
              );
let allData = union isfuzzy=true 
    (
        (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) > 1000 | join (nosummary) on nosum) | join (
        _Im_NetworkSession(starttime=todatetime(ago(2d)), endtime=now())
        | where TimeGenerated > maxSummarizedTime
        | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)
        | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)
        ) on exists
        | project-away exists, maxv, nosum*
    ),
    (
        (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) between (501 .. 1000) | join (nosummary) on nosum) | join (
        _Im_NetworkSession(starttime=todatetime(ago(3d)), endtime=now())
        | where TimeGenerated > maxSummarizedTime
        | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)
        | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)
        ) on exists
        | project-away exists, maxv, nosum*
    ),
    (
        (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) <= 500 | join (nosummary) on nosum) | join (
        _Im_NetworkSession(starttime=todatetime(ago(4d)), endtime=now())
        | where TimeGenerated > maxSummarizedTime
        | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)
        | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)
        ) on exists
        | project-away exists, maxv, nosum*
    ),
    (
        NetworkCustomAnalytics_protocol_CL
        | where EventTime_t > min_t
        | project-rename NetworkProtocol=NetworkProtocol_s, DstPortNumber=DstPortNumber_d, DstAppName=DstAppName_s, NetworkDirection=NetworkDirection_s, DvcAction=DvcAction_s, Count=count__d, EventTime=EventTime_t
        | extend Count = toint(Count),DstPortNumber = toint(DstPortNumber) 
    )
;
let findVolumneBasedAnomaly = allData
        | make-series total=sum(Count) on EventTime from min_t to max_t step dt
        | extend (anomalies, score, baseline) = series_decompose_anomalies(total, AnomalyThreshold, -1, 'linefit') 
        | mv-expand anomalies, score, baseline, EventTime, total
        | extend anomalies = toint(anomalies), score = toint(score), baseline = toint(baseline), EventTime = todatetime(EventTime), total = tolong(total)
        | where EventTime >= ago(1d)
        | where score >= 2*AnomalyThreshold
        ;
let findAnomalies = (field:string){
    allData
        | where isnotempty(column_ifexists(field,""))
        | make-series total=sum(Count) on EventTime from min_t to max_t step dt by column_ifexists(field,"")
        | extend (anomalies, score, baseline) = series_decompose_anomalies(total, AnomalyThreshold, -1, 'linefit')
        | mv-expand anomalies, score, baseline, EventTime, total
        | extend anomalies = toint(anomalies), score = toint(score), baseline = toint(baseline), EventTime = todatetime(EventTime), total = tolong(total)
        | where EventTime >= ago(1d)
        | where score >= 2*AnomalyThreshold
};
union findAnomalies(fieldForDvcAction), findAnomalies(fieldForNetworkDirection), findAnomalies(fieldForNetworkProtocol), findVolumneBasedAnomaly
| extend anomalyFieldType  = case (isnotempty(column_ifexists(fieldForDvcAction,"")), "DvcAction",
                                  isnotempty(column_ifexists(fieldForNetworkDirection,"")), "NetworkDirection",
                                  isnotempty(column_ifexists(fieldForNetworkProtocol,"")), "NetworkProtocol",
                                  "TotalVolume"
                                ),
        anomalyFieldValue  = case (isnotempty(column_ifexists(fieldForDvcAction,"")), column_ifexists(fieldForDvcAction,""),
                                  isnotempty(column_ifexists(fieldForNetworkDirection,"")), column_ifexists(fieldForNetworkDirection,""),
                                  isnotempty(column_ifexists(fieldForNetworkProtocol,"")), column_ifexists(fieldForNetworkProtocol,""),
                                  "Overall"
                                )

query: |
  let min_t = ago(14d);
  let max_t = now();
  let dt = 1d;
  let fieldForDvcAction = "DvcAction";
  let fieldForNetworkDirection = "NetworkDirection";
  let fieldForNetworkProtocol = "NetworkProtocol";
  let AnomalyThreshold = 2.5;
  let eps = materialize (_Im_NetworkSession | project TimeGenerated | where TimeGenerated > ago(5m) | count | extend Count = Count/300);
  let maxSummarizedTime = toscalar (
      union isfuzzy=true 
          (
              NetworkCustomAnalytics_protocol_CL
                  | where EventTime_t > min_t
                  | summarize max_TimeGenerated=max(EventTime_t)
                  | extend max_TimeGenerated = datetime_add('minute',10,max_TimeGenerated)
          ),
          (
              print(min_t)
              | project max_TimeGenerated = print_0
          )
        | summarize maxTimeGenerated = max(max_TimeGenerated) 
      );
  let nosummary = materialize(
                union isfuzzy=true 
                  (
                      NetworkCustomAnalytics_protocol_CL
                      | where EventTime_t > ago(1d) 
                      | project v = int(2)
                  ),
                  (
                      print int(1) 
                      | project v = print_0
                  )
                  | summarize maxv = max(v)
                  | extend nosum = (maxv > 1)
                );
  let allData = union isfuzzy=true 
      (
          (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) > 1000 | join (nosummary) on nosum) | join (
          _Im_NetworkSession(starttime=todatetime(ago(2d)), endtime=now())
          | where TimeGenerated > maxSummarizedTime
          | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)
          | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)
          ) on exists
          | project-away exists, maxv, nosum*
      ),
      (
          (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) between (501 .. 1000) | join (nosummary) on nosum) | join (
          _Im_NetworkSession(starttime=todatetime(ago(3d)), endtime=now())
          | where TimeGenerated > maxSummarizedTime
          | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)
          | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)
          ) on exists
          | project-away exists, maxv, nosum*
      ),
      (
          (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) <= 500 | join (nosummary) on nosum) | join (
          _Im_NetworkSession(starttime=todatetime(ago(4d)), endtime=now())
          | where TimeGenerated > maxSummarizedTime
          | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)
          | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)
          ) on exists
          | project-away exists, maxv, nosum*
      ),
      (
          NetworkCustomAnalytics_protocol_CL
          | where EventTime_t > min_t
          | project-rename NetworkProtocol=NetworkProtocol_s, DstPortNumber=DstPortNumber_d, DstAppName=DstAppName_s, NetworkDirection=NetworkDirection_s, DvcAction=DvcAction_s, Count=count__d, EventTime=EventTime_t
          | extend Count = toint(Count),DstPortNumber = toint(DstPortNumber) 
      )
  ;
  let findVolumneBasedAnomaly = allData
          | make-series total=sum(Count) on EventTime from min_t to max_t step dt
          | extend (anomalies, score, baseline) = series_decompose_anomalies(total, AnomalyThreshold, -1, 'linefit') 
          | mv-expand anomalies, score, baseline, EventTime, total
          | extend anomalies = toint(anomalies), score = toint(score), baseline = toint(baseline), EventTime = todatetime(EventTime), total = tolong(total)
          | where EventTime >= ago(1d)
          | where score >= 2*AnomalyThreshold
          ;
  let findAnomalies = (field:string){
      allData
          | where isnotempty(column_ifexists(field,""))
          | make-series total=sum(Count) on EventTime from min_t to max_t step dt by column_ifexists(field,"")
          | extend (anomalies, score, baseline) = series_decompose_anomalies(total, AnomalyThreshold, -1, 'linefit')
          | mv-expand anomalies, score, baseline, EventTime, total
          | extend anomalies = toint(anomalies), score = toint(score), baseline = toint(baseline), EventTime = todatetime(EventTime), total = tolong(total)
          | where EventTime >= ago(1d)
          | where score >= 2*AnomalyThreshold
  };
  union findAnomalies(fieldForDvcAction), findAnomalies(fieldForNetworkDirection), findAnomalies(fieldForNetworkProtocol), findVolumneBasedAnomaly
  | extend anomalyFieldType  = case (isnotempty(column_ifexists(fieldForDvcAction,"")), "DvcAction",
                                    isnotempty(column_ifexists(fieldForNetworkDirection,"")), "NetworkDirection",
                                    isnotempty(column_ifexists(fieldForNetworkProtocol,"")), "NetworkProtocol",
                                    "TotalVolume"
                                  ),
          anomalyFieldValue  = case (isnotempty(column_ifexists(fieldForDvcAction,"")), column_ifexists(fieldForDvcAction,""),
                                    isnotempty(column_ifexists(fieldForNetworkDirection,"")), column_ifexists(fieldForNetworkDirection,""),
                                    isnotempty(column_ifexists(fieldForNetworkProtocol,"")), column_ifexists(fieldForNetworkProtocol,""),
                                    "Overall"
                                  )  
eventGroupingSettings:
  aggregationKind: AlertPerResult
triggerThreshold: 0
customDetails:
  AnomalyFieldType: anomalyFieldType
  AnomalyFieldValue: anomalyFieldValue
  Score: score
queryFrequency: 1d
requiredDataConnectors:
- connectorId: AWSS3
  dataTypes:
  - AWSVPCFlow
- connectorId: MicrosoftThreatProtection
  dataTypes:
  - DeviceNetworkEvents
- connectorId: SecurityEvents
  dataTypes:
  - SecurityEvent
- connectorId: WindowsForwardedEvents
  dataTypes:
  - WindowsEvent
- connectorId: Zscaler
  dataTypes:
  - CommonSecurityLog
- connectorId: MicrosoftSysmonForLinux
  dataTypes:
  - Syslog
- connectorId: PaloAltoNetworks
  dataTypes:
  - CommonSecurityLog
- connectorId: AzureMonitor(VMInsights)
  dataTypes:
  - VMConnection
- connectorId: AzureFirewall
  dataTypes:
  - AzureDiagnostics
- connectorId: AzureNSG
  dataTypes:
  - AzureDiagnostics
- connectorId: CiscoASA
  dataTypes:
  - CommonSecurityLog
- connectorId: Corelight
  dataTypes:
  - Corelight_CL
- connectorId: AIVectraStream
  dataTypes:
  - VectraStream
- connectorId: CheckPoint
  dataTypes:
  - CommonSecurityLog
- connectorId: Fortinet
  dataTypes:
  - CommonSecurityLog
- connectorId: CiscoMeraki
  dataTypes:
  - Syslog
  - CiscoMerakiNativePoller
id: cd6def0d-3ef0-4d55-a7e3-faa96c46ba12
version: 1.0.0
name: Anomaly found in Network Session Traffic (ASIM Network Session schema)
kind: Scheduled
status: Available
relevantTechniques:
- T1095
- T1071
- T1046
- T1030
- T1210
OriginalUri: https://github.com/Azure/Azure-Sentinel/blob/master/Solutions/Network Session Essentials/Analytic Rules/AnomalyFoundInNetworkSessionTraffic.yaml
queryPeriod: 14d
alertDetailsOverride:
  alertDescriptionFormat: Based on past data, anomaly was observed in {{anomalyFieldValue}} Traffic with a score of {{score}}.
  alertDisplayNameFormat: Anomaly was observed with {{anomalyFieldValue}} Traffic
severity: Medium
triggerOperator: gt
tactics:
- CommandAndControl
- Discovery
- Exfiltration
- LateralMovement
tags:
- Schema: ASimNetworkSessions
  SchemaVersion: 0.2.4
description: |
    'The rule identifies anomalous pattern in network session traffic based on previously seen data, different Device Action, Network Protocol, Network Direction or overall volume. The rule utilize [ASIM](https://aka.ms/AboutASIM) normalization, and is applied to any source which supports the ASIM Network Session schema'

{
  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
  "contentVersion": "1.0.0.0",
  "parameters": {
    "workspace": {
      "type": "String"
    }
  },
  "resources": [
    {
      "id": "[concat(resourceId('Microsoft.OperationalInsights/workspaces/providers', parameters('workspace'), 'Microsoft.SecurityInsights'),'/alertRules/cd6def0d-3ef0-4d55-a7e3-faa96c46ba12')]",
      "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/cd6def0d-3ef0-4d55-a7e3-faa96c46ba12')]",
      "type": "Microsoft.OperationalInsights/workspaces/providers/alertRules",
      "kind": "Scheduled",
      "apiVersion": "2022-11-01",
      "properties": {
        "displayName": "Anomaly found in Network Session Traffic (ASIM Network Session schema)",
        "description": "'The rule identifies anomalous pattern in network session traffic based on previously seen data, different Device Action, Network Protocol, Network Direction or overall volume. The rule utilize [ASIM](https://aka.ms/AboutASIM) normalization, and is applied to any source which supports the ASIM Network Session schema'\n",
        "severity": "Medium",
        "enabled": true,
        "query": "let min_t = ago(14d);\nlet max_t = now();\nlet dt = 1d;\nlet fieldForDvcAction = \"DvcAction\";\nlet fieldForNetworkDirection = \"NetworkDirection\";\nlet fieldForNetworkProtocol = \"NetworkProtocol\";\nlet AnomalyThreshold = 2.5;\nlet eps = materialize (_Im_NetworkSession | project TimeGenerated | where TimeGenerated > ago(5m) | count | extend Count = Count/300);\nlet maxSummarizedTime = toscalar (\n    union isfuzzy=true \n        (\n            NetworkCustomAnalytics_protocol_CL\n                | where EventTime_t > min_t\n                | summarize max_TimeGenerated=max(EventTime_t)\n                | extend max_TimeGenerated = datetime_add('minute',10,max_TimeGenerated)\n        ),\n        (\n            print(min_t)\n            | project max_TimeGenerated = print_0\n        )\n      | summarize maxTimeGenerated = max(max_TimeGenerated) \n    );\nlet nosummary = materialize(\n              union isfuzzy=true \n                (\n                    NetworkCustomAnalytics_protocol_CL\n                    | where EventTime_t > ago(1d) \n                    | project v = int(2)\n                ),\n                (\n                    print int(1) \n                    | project v = print_0\n                )\n                | summarize maxv = max(v)\n                | extend nosum = (maxv > 1)\n              );\nlet allData = union isfuzzy=true \n    (\n        (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) > 1000 | join (nosummary) on nosum) | join (\n        _Im_NetworkSession(starttime=todatetime(ago(2d)), endtime=now())\n        | where TimeGenerated > maxSummarizedTime\n        | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)\n        | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)\n        ) on exists\n        | project-away exists, maxv, nosum*\n    ),\n    (\n        (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) between (501 .. 1000) | join (nosummary) on nosum) | join (\n        _Im_NetworkSession(starttime=todatetime(ago(3d)), endtime=now())\n        | where TimeGenerated > maxSummarizedTime\n        | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)\n        | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)\n        ) on exists\n        | project-away exists, maxv, nosum*\n    ),\n    (\n        (datatable(exists:int, nosum:bool)[1,false] | where toscalar(eps) <= 500 | join (nosummary) on nosum) | join (\n        _Im_NetworkSession(starttime=todatetime(ago(4d)), endtime=now())\n        | where TimeGenerated > maxSummarizedTime\n        | summarize Count=count() by NetworkProtocol, DstPortNumber, DstAppName, NetworkDirection, DvcAction, bin(TimeGenerated,10m)\n        | extend EventTime = TimeGenerated, Count = toint(Count), DstPortNumber = toint(DstPortNumber), exists=int(1)\n        ) on exists\n        | project-away exists, maxv, nosum*\n    ),\n    (\n        NetworkCustomAnalytics_protocol_CL\n        | where EventTime_t > min_t\n        | project-rename NetworkProtocol=NetworkProtocol_s, DstPortNumber=DstPortNumber_d, DstAppName=DstAppName_s, NetworkDirection=NetworkDirection_s, DvcAction=DvcAction_s, Count=count__d, EventTime=EventTime_t\n        | extend Count = toint(Count),DstPortNumber = toint(DstPortNumber) \n    )\n;\nlet findVolumneBasedAnomaly = allData\n        | make-series total=sum(Count) on EventTime from min_t to max_t step dt\n        | extend (anomalies, score, baseline) = series_decompose_anomalies(total, AnomalyThreshold, -1, 'linefit') \n        | mv-expand anomalies, score, baseline, EventTime, total\n        | extend anomalies = toint(anomalies), score = toint(score), baseline = toint(baseline), EventTime = todatetime(EventTime), total = tolong(total)\n        | where EventTime >= ago(1d)\n        | where score >= 2*AnomalyThreshold\n        ;\nlet findAnomalies = (field:string){\n    allData\n        | where isnotempty(column_ifexists(field,\"\"))\n        | make-series total=sum(Count) on EventTime from min_t to max_t step dt by column_ifexists(field,\"\")\n        | extend (anomalies, score, baseline) = series_decompose_anomalies(total, AnomalyThreshold, -1, 'linefit')\n        | mv-expand anomalies, score, baseline, EventTime, total\n        | extend anomalies = toint(anomalies), score = toint(score), baseline = toint(baseline), EventTime = todatetime(EventTime), total = tolong(total)\n        | where EventTime >= ago(1d)\n        | where score >= 2*AnomalyThreshold\n};\nunion findAnomalies(fieldForDvcAction), findAnomalies(fieldForNetworkDirection), findAnomalies(fieldForNetworkProtocol), findVolumneBasedAnomaly\n| extend anomalyFieldType  = case (isnotempty(column_ifexists(fieldForDvcAction,\"\")), \"DvcAction\",\n                                  isnotempty(column_ifexists(fieldForNetworkDirection,\"\")), \"NetworkDirection\",\n                                  isnotempty(column_ifexists(fieldForNetworkProtocol,\"\")), \"NetworkProtocol\",\n                                  \"TotalVolume\"\n                                ),\n        anomalyFieldValue  = case (isnotempty(column_ifexists(fieldForDvcAction,\"\")), column_ifexists(fieldForDvcAction,\"\"),\n                                  isnotempty(column_ifexists(fieldForNetworkDirection,\"\")), column_ifexists(fieldForNetworkDirection,\"\"),\n                                  isnotempty(column_ifexists(fieldForNetworkProtocol,\"\")), column_ifexists(fieldForNetworkProtocol,\"\"),\n                                  \"Overall\"\n                                )\n",
        "queryFrequency": "P1D",
        "queryPeriod": "P14D",
        "triggerOperator": "GreaterThan",
        "triggerThreshold": 0,
        "suppressionDuration": "PT1H",
        "suppressionEnabled": false,
        "tactics": [
          "CommandAndControl",
          "Discovery",
          "Exfiltration",
          "LateralMovement"
        ],
        "techniques": [
          "T1095",
          "T1071",
          "T1046",
          "T1030",
          "T1210"
        ],
        "alertRuleTemplateName": "cd6def0d-3ef0-4d55-a7e3-faa96c46ba12",
        "eventGroupingSettings": {
          "aggregationKind": "AlertPerResult"
        },
        "alertDetailsOverride": {
          "alertDescriptionFormat": "Based on past data, anomaly was observed in {{anomalyFieldValue}} Traffic with a score of {{score}}.",
          "alertDisplayNameFormat": "Anomaly was observed with {{anomalyFieldValue}} Traffic"
        },
        "customDetails": {
          "AnomalyFieldType": "anomalyFieldType",
          "AnomalyFieldValue": "anomalyFieldValue",
          "Score": "score"
        },
        "entityMappings": null,
        "tags": [
          {
            "Schema": "ASimNetworkSessions",
            "SchemaVersion": "0.2.4"
          }
        ],
        "OriginalUri": "https://github.com/Azure/Azure-Sentinel/blob/master/Solutions/Network Session Essentials/Analytic Rules/AnomalyFoundInNetworkSessionTraffic.yaml",
        "templateVersion": "1.0.0",
        "status": "Available"
      }
    }
  ]
}