Beacon Traffic Based on Common User Agents Visiting Limited Number of Domains

let timeframe = 1d; // Timeframe during which to search for beaconing behavior.
let lookback = 7d; // Look back period to find if browser was used for other domains by user.
let min_requests=50; // Minimum number of requests to consider it beacon traffic.
let min_hours=8; // Minimum number of different hours during which connections were made to consider it beacon traffic.
let trusted_user_count=10; // If visited by this many users a domain is considered 'trusted'.
let max_sites=3; // Maximum number of different sites visited using this user-agent.
// Client-specific query to obtain 'browser-like' traffic from proxy logs.
let BrowserTraffic = (p:timespan) {
CommonSecurityLog
| where DeviceVendor == "Zscaler" and DeviceProduct == "NSSWeblog"
| where TimeGenerated >ago(p)
| project TimeGenerated, SourceUserName, DestinationHostName, RequestClientApplication
| where (RequestClientApplication startswith "Mozilla/" and RequestClientApplication contains "Gecko")
};
let CommonDomains = BrowserTraffic(timeframe)
| summarize source_count=dcount(SourceUserName) by DestinationHostName
| where source_count>trusted_user_count
| project DestinationHostName;
let CommonUA = BrowserTraffic(timeframe)
| summarize source_count=dcount(SourceUserName), host_count=dcount(DestinationHostName) by RequestClientApplication
| where source_count>trusted_user_count and host_count > 100 // Normal browsers are browsers used by many people and visiting many different sites.
| project RequestClientApplication;
// Find browsers that are common, i.e. many users use them and they use them to visit many different sites,
// but some users only use the browser to visit a very limited set of sites.
// These are considered suspicious, since they might be an attacker masquerading a beacon as a legitimate browser.
let SuspiciousBrowers = BrowserTraffic(timeframe)
| where RequestClientApplication in(CommonUA)
| summarize BrowserHosts=make_set(DestinationHostName),request_count=count() by RequestClientApplication, SourceUserName
| where array_length(BrowserHosts) <= max_sites and request_count >= min_requests
| project RequestClientApplication, SourceUserName,BrowserHosts;
// Just reporting on suspicious browsers gives too many false positives.
// For example, users that have the browser open on the login screen of 1 specific application.
// In the suspicious browsers we can search for 'beacon-like' behavior.
// Get all browser traffic by the suspicious browsers.
let PotentialAlerts=SuspiciousBrowers
| join BrowserTraffic(timeframe) on RequestClientApplication, SourceUserName
// Find beaconing-like traffic - i.e. contacting the same host in many different hours.
| summarize hour_count=dcount(bin(TimeGenerated,1h)), BrowserHosts=any(BrowserHosts), request_count=count() by RequestClientApplication, SourceUserName, DestinationHostName
| where hour_count >= min_hours and request_count >= min_requests
// Remove common domains like login.microsoft.com.
| join kind=leftanti CommonDomains on DestinationHostName
| summarize RareHosts=make_set(DestinationHostName), TotalRequestCount=sum(request_count), BrowserHosts=any(BrowserHosts) by RequestClientApplication, SourceUserName
// Remove browsers that visit any common domains.
| where array_length(RareHosts) == array_length(BrowserHosts);
// Look back for X days to see if the browser was not used to visit more hosts.
// This is to get rid of someone that started up the browser a long time ago, and left only a single tab open.
PotentialAlerts
| join BrowserTraffic(lookback) on SourceUserName, RequestClientApplication
| summarize RareHosts=any(RareHosts),BrowserHosts1d=any(BrowserHosts),BrowserHostsLookback=make_set(DestinationHostName) by SourceUserName, RequestClientApplication
| where array_length(RareHosts) == array_length(BrowserHostsLookback)

queryPeriod: 7d
query: |
  let timeframe = 1d; // Timeframe during which to search for beaconing behavior.
  let lookback = 7d; // Look back period to find if browser was used for other domains by user.
  let min_requests=50; // Minimum number of requests to consider it beacon traffic.
  let min_hours=8; // Minimum number of different hours during which connections were made to consider it beacon traffic.
  let trusted_user_count=10; // If visited by this many users a domain is considered 'trusted'.
  let max_sites=3; // Maximum number of different sites visited using this user-agent.
  // Client-specific query to obtain 'browser-like' traffic from proxy logs.
  let BrowserTraffic = (p:timespan) {
  CommonSecurityLog
  | where DeviceVendor == "Zscaler" and DeviceProduct == "NSSWeblog"
  | where TimeGenerated >ago(p)
  | project TimeGenerated, SourceUserName, DestinationHostName, RequestClientApplication
  | where (RequestClientApplication startswith "Mozilla/" and RequestClientApplication contains "Gecko")
  };
  let CommonDomains = BrowserTraffic(timeframe)
  | summarize source_count=dcount(SourceUserName) by DestinationHostName
  | where source_count>trusted_user_count
  | project DestinationHostName;
  let CommonUA = BrowserTraffic(timeframe)
  | summarize source_count=dcount(SourceUserName), host_count=dcount(DestinationHostName) by RequestClientApplication
  | where source_count>trusted_user_count and host_count > 100 // Normal browsers are browsers used by many people and visiting many different sites.
  | project RequestClientApplication;
  // Find browsers that are common, i.e. many users use them and they use them to visit many different sites,
  // but some users only use the browser to visit a very limited set of sites.
  // These are considered suspicious, since they might be an attacker masquerading a beacon as a legitimate browser.
  let SuspiciousBrowers = BrowserTraffic(timeframe)
  | where RequestClientApplication in(CommonUA)
  | summarize BrowserHosts=make_set(DestinationHostName),request_count=count() by RequestClientApplication, SourceUserName
  | where array_length(BrowserHosts) <= max_sites and request_count >= min_requests
  | project RequestClientApplication, SourceUserName,BrowserHosts;
  // Just reporting on suspicious browsers gives too many false positives.
  // For example, users that have the browser open on the login screen of 1 specific application.
  // In the suspicious browsers we can search for 'beacon-like' behavior.
  // Get all browser traffic by the suspicious browsers.
  let PotentialAlerts=SuspiciousBrowers
  | join BrowserTraffic(timeframe) on RequestClientApplication, SourceUserName
  // Find beaconing-like traffic - i.e. contacting the same host in many different hours.
  | summarize hour_count=dcount(bin(TimeGenerated,1h)), BrowserHosts=any(BrowserHosts), request_count=count() by RequestClientApplication, SourceUserName, DestinationHostName
  | where hour_count >= min_hours and request_count >= min_requests
  // Remove common domains like login.microsoft.com.
  | join kind=leftanti CommonDomains on DestinationHostName
  | summarize RareHosts=make_set(DestinationHostName), TotalRequestCount=sum(request_count), BrowserHosts=any(BrowserHosts) by RequestClientApplication, SourceUserName
  // Remove browsers that visit any common domains.
  | where array_length(RareHosts) == array_length(BrowserHosts);
  // Look back for X days to see if the browser was not used to visit more hosts.
  // This is to get rid of someone that started up the browser a long time ago, and left only a single tab open.
  PotentialAlerts
  | join BrowserTraffic(lookback) on SourceUserName, RequestClientApplication
  | summarize RareHosts=any(RareHosts),BrowserHosts1d=any(BrowserHosts),BrowserHostsLookback=make_set(DestinationHostName) by SourceUserName, RequestClientApplication
  | where array_length(RareHosts) == array_length(BrowserHostsLookback)  
name: Beacon Traffic Based on Common User Agents Visiting Limited Number of Domains
entityMappings:
- fieldMappings:
  - columnName: SourceUserName
    identifier: FullName
  entityType: Account
queryFrequency: 1d
OriginalUri: https://github.com/Azure/Azure-Sentinel/blob/master/Solutions/FalconFriday/Analytic Rules/RecognizingBeaconingTraffic.yaml
requiredDataConnectors:
- connectorId: Zscaler
  dataTypes:
  - CommonSecurityLog
description: |
  This query searches web proxy logs for a specific type of beaconing behavior by joining a number of sources together: 
  - Traffic by actual web browsers - by looking at traffic generated by a UserAgent that looks like a browser and is used by multiple users to visit a large number of domains.
  - Users that make requests using one of these actual browsers, but only to a small set of domains, none of which are common domains.
  - The traffic is beacon-like; meaning that it occurs during many different hours of the day (i.e. periodic).   
kind: Scheduled
version: 1.0.1
status: Available
severity: Medium
relevantTechniques:
- T1071.001
triggerOperator: gt
triggerThreshold: 0
tactics:
- CommandAndControl
id: 6345c923-99eb-4a83-b11d-7af0ffa75577