SFTP File transfer above threshold

let threshold = 50;
Syslog
| where ProcessName has "sftp"
    and SyslogMessage has "close "
    and SyslogMessage has " bytes read "
| parse SyslogMessage with "close \"" filepath "\" bytes read " readbytes: int " written " writtenbytes: int
| join kind=leftouter (
    Syslog
    | where ProcessName has "sftp" and SyslogMessage has "session opened for"
    | parse SyslogMessage with "session opened for local user " username: string " from [" src_ip "]"
    | project username, src_ip, ProcessID
    )
    on ProcessID
| project-away ProcessID1
| summarize count_distinct_filepath=dcount(filepath), fileslist=make_set(filepath) by Computer, username, src_ip, bin(TimeGenerated, 15m)
| where count_distinct_filepath >= threshold
| extend FileSample = iff(array_length(fileslist) == 1, tostring(fileslist[0]), strcat("SeeFilesListField","_", tostring(hash(tostring(fileslist)))))

relevantTechniques:
- T1020
incidentConfiguration:
  createIncident: true
  groupingConfiguration:
    lookbackDuration: 5h
    groupByCustomDetails: []
    groupByAlertDetails: []
    enabled: true
    reopenClosedIncident: false
    matchingMethod: Selected
    groupByEntities:
    - Account
    - Host
name: SFTP File transfer above threshold
requiredDataConnectors:
- dataTypes:
  - Syslog
  connectorId: Syslog
- dataTypes:
  - Syslog
  connectorId: SyslogAma
entityMappings:
- fieldMappings:
  - identifier: Name
    columnName: username
  entityType: Account
- fieldMappings:
  - identifier: Address
    columnName: src_ip
  entityType: IP
- fieldMappings:
  - identifier: HostName
    columnName: Computer
  entityType: Host
- fieldMappings:
  - identifier: Name
    columnName: FileSample
  entityType: File
triggerThreshold: 0
id: bb6a74c8-889d-4c6e-8412-7d5efe33f4ed
tactics:
- Exfiltration
version: 1.0.2
customDetails:
  FilesList: fileslist
  TransferCount: count_distinct_filepath
alertDetailsOverride: 
queryPeriod: 15m
kind: Scheduled
OriginalUri: https://github.com/Azure/Azure-Sentinel/blob/master/Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml
queryFrequency: 15m
severity: Medium
description: |
  'Identifies SFTP File Transfers above certain threshold in a 15min time period. It requires SFTP VERBOSE loglevel to be enabled.
   Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally, if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.'  
query: |
  let threshold = 50;
  Syslog
  | where ProcessName has "sftp"
      and SyslogMessage has "close "
      and SyslogMessage has " bytes read "
  | parse SyslogMessage with "close \"" filepath "\" bytes read " readbytes: int " written " writtenbytes: int
  | join kind=leftouter (
      Syslog
      | where ProcessName has "sftp" and SyslogMessage has "session opened for"
      | parse SyslogMessage with "session opened for local user " username: string " from [" src_ip "]"
      | project username, src_ip, ProcessID
      )
      on ProcessID
  | project-away ProcessID1
  | summarize count_distinct_filepath=dcount(filepath), fileslist=make_set(filepath) by Computer, username, src_ip, bin(TimeGenerated, 15m)
  | where count_distinct_filepath >= threshold
  | extend FileSample = iff(array_length(fileslist) == 1, tostring(fileslist[0]), strcat("SeeFilesListField","_", tostring(hash(tostring(fileslist)))))  
triggerOperator: gt

{
  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
  "contentVersion": "1.0.0.0",
  "parameters": {
    "workspace": {
      "type": "String"
    }
  },
  "resources": [
    {
      "apiVersion": "2024-01-01-preview",
      "id": "[concat(resourceId('Microsoft.OperationalInsights/workspaces/providers', parameters('workspace'), 'Microsoft.SecurityInsights'),'/alertRules/bb6a74c8-889d-4c6e-8412-7d5efe33f4ed')]",
      "kind": "Scheduled",
      "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/bb6a74c8-889d-4c6e-8412-7d5efe33f4ed')]",
      "properties": {
        "alertDetailsOverride": null,
        "alertRuleTemplateName": "bb6a74c8-889d-4c6e-8412-7d5efe33f4ed",
        "customDetails": {
          "FilesList": "fileslist",
          "TransferCount": "count_distinct_filepath"
        },
        "description": "'Identifies SFTP File Transfers above certain threshold in a 15min time period. It requires SFTP VERBOSE loglevel to be enabled.\n Please note that entity mapping for arrays is not supported, so when there is a single value in an array, we will pull that value from the array as a single string to populate the entity to support entity mapping features within Sentinel. Additionally, if the array is multivalued, we will input a string to indicate this with a unique hash so that matching will not occur.'\n",
        "displayName": "SFTP File transfer above threshold",
        "enabled": true,
        "entityMappings": [
          {
            "entityType": "Account",
            "fieldMappings": [
              {
                "columnName": "username",
                "identifier": "Name"
              }
            ]
          },
          {
            "entityType": "IP",
            "fieldMappings": [
              {
                "columnName": "src_ip",
                "identifier": "Address"
              }
            ]
          },
          {
            "entityType": "Host",
            "fieldMappings": [
              {
                "columnName": "Computer",
                "identifier": "HostName"
              }
            ]
          },
          {
            "entityType": "File",
            "fieldMappings": [
              {
                "columnName": "FileSample",
                "identifier": "Name"
              }
            ]
          }
        ],
        "incidentConfiguration": {
          "createIncident": true,
          "groupingConfiguration": {
            "enabled": true,
            "groupByAlertDetails": [],
            "groupByCustomDetails": [],
            "groupByEntities": [
              "Account",
              "Host"
            ],
            "lookbackDuration": "PT5H",
            "matchingMethod": "Selected",
            "reopenClosedIncident": false
          }
        },
        "OriginalUri": "https://github.com/Azure/Azure-Sentinel/blob/master/Solutions/Syslog/Analytic Rules/sftp_file_transfer_above_threshold.yaml",
        "query": "let threshold = 50;\nSyslog\n| where ProcessName has \"sftp\"\n    and SyslogMessage has \"close \"\n    and SyslogMessage has \" bytes read \"\n| parse SyslogMessage with \"close \\\"\" filepath \"\\\" bytes read \" readbytes: int \" written \" writtenbytes: int\n| join kind=leftouter (\n    Syslog\n    | where ProcessName has \"sftp\" and SyslogMessage has \"session opened for\"\n    | parse SyslogMessage with \"session opened for local user \" username: string \" from [\" src_ip \"]\"\n    | project username, src_ip, ProcessID\n    )\n    on ProcessID\n| project-away ProcessID1\n| summarize count_distinct_filepath=dcount(filepath), fileslist=make_set(filepath) by Computer, username, src_ip, bin(TimeGenerated, 15m)\n| where count_distinct_filepath >= threshold\n| extend FileSample = iff(array_length(fileslist) == 1, tostring(fileslist[0]), strcat(\"SeeFilesListField\",\"_\", tostring(hash(tostring(fileslist)))))\n",
        "queryFrequency": "PT15M",
        "queryPeriod": "PT15M",
        "severity": "Medium",
        "subTechniques": [],
        "suppressionDuration": "PT1H",
        "suppressionEnabled": false,
        "tactics": [
          "Exfiltration"
        ],
        "techniques": [
          "T1020"
        ],
        "templateVersion": "1.0.2",
        "triggerOperator": "GreaterThan",
        "triggerThreshold": 0
      },
      "type": "Microsoft.OperationalInsights/workspaces/providers/alertRules"
    }
  ]
}