feat(addon): Significantly improve and modernize CIM compliance

BREAKING CHANGE(addon): pan_traffic_start logs no longer included in CIM
BREAKING CHANGE(addon): pan_traffic_end logs moved from Network Session to Network Traffic datamodel
BREAKING CHANGE(addon): pan_threat event type now includes wildfire and data logs
BREAKING CHANGE(addon): pan_file logs moved from Web to IDS datamodel
BREAKING CHANGE(addon): pan_virus logs moved from Malware to IDS datamodel
BREAKING CHANGE(addon): pan_wildfire logs moved from Malware to IDS datamodel
BREAKING CHANGE(addon): pan_email removed from Email datamodel

Author: btorresgil

Splunk_TA_paloalto/app.manifest

@@ -17,11 +17,24 @@
     "releaseDate": null,
     "description": "The Palo Alto Networks Add-on allows a Splunk Enterprise administrator to collect data from Palo Alto Networks Next-Generation Firewall devices, Panorama, Advanced Endpoint Protection, Aperture SaaS Security, AutoFocus Threat Intelligence, and MineMeld.",
     "classification": {
-      "intendedAudience": null,
-      "categories": [],
+      "intendedAudience": "IT Professionals",
+      "categories": [
+        "IT Operations",
+        "Security, Fraud & Compliance"
+      ],
       "developmentStatus": "Production/Stable"
     },
-    "commonInformationModels": null,
+    "commonInformationModels": {
+      "Authentication":"4.19.0",
+      "Alert":"4.19.0",
+      "Change":"4.19.0",
+      "Endpoint":"4.19.0",
+      "Network Traffic":"4.19.0",
+      "Malware":"4.19.0",
+      "Intrusion Detection":"4.19.0",
+      "Network Sessions":"4.19.0",
+      "Web":"4.19.0"
+    },
     "license": {
       "name": null,
       "text": null,

Splunk_TA_paloalto/default/eventtypes.conf

@@ -15,27 +15,36 @@ search = sourcetype=pan_config OR sourcetype=pan:config
 
 [pan_traffic]
 search = sourcetype=pan_traffic OR sourcetype=pan:traffic OR (sourcetype=pan:firewall_cloud AND LogType="TRAFFIC")
-#tags = network communicate
 
 [pan_traffic_start]
 search = sourcetype=pan_traffic OR sourcetype=pan:traffic OR (sourcetype=pan:firewall_cloud AND LogType="TRAFFIC") AND log_subtype="start"
-#tags = network session start
 
 [pan_traffic_end]
 search = sourcetype=pan_traffic OR sourcetype=pan:traffic OR (sourcetype=pan:firewall_cloud AND LogType="TRAFFIC") AND log_subtype="end"
-#tags = network session end
+#tags = network communicate
 
 [pan_system]
 search = sourcetype=pan_system OR sourcetype=pan:system OR (sourcetype=pan:firewall_cloud AND LogType="SYSTEM")
-#tags = update status 
+
+[pan_system_auth]
+search = sourcetype=pan_system OR sourcetype=pan:system AND log_subtype="globalprotect" OR description="*Failed password*" NOT description="*client configuration released*" NOT description="*client configuration generated*"
+#tags = authentication default
+
+[pan_system_alert]
+search = sourcetype=pan_system OR sourcetype=pan:system AND log_subtype="url-filtering"
+#tags = alert
+
+[pan_system_change]
+search = sourcetype=pan_system OR sourcetype=pan:system description="*config cleared*" AND NOT (log_subtype IN ("routing", "ras", "vpn"))
+#tags = change
 
 [pan_threat]
-search = sourcetype=pan_threat OR sourcetype=pan:threat OR (sourcetype=pan:firewall_cloud AND LogType="THREAT") AND log_subtype != "url" log_subtype != "file" log_subtype != "wildfire" log_subtype != "data" 
+search = sourcetype=pan_threat OR sourcetype=pan:threat OR (sourcetype=pan:firewall_cloud AND LogType="THREAT") AND log_subtype != "url" log_subtype != "file"
 #tags = ids attack
 
 [pan_file]
 search = sourcetype=pan_threat OR sourcetype=pan:threat OR (sourcetype=pan:firewall_cloud AND LogType="THREAT") AND log_subtype = "file" 
-#tags = web
+#tags = ids attack
 
 [pan_url]
 search = sourcetype=pan_threat OR sourcetype=pan:threat OR (sourcetype=pan:firewall_cloud AND LogType="THREAT") AND log_subtype = "url" 
@@ -47,7 +56,7 @@ search = sourcetype=pan_threat OR sourcetype=pan:threat OR (sourcetype=pan:firew
 
 [pan_virus]
 search = (sourcetype=pan_threat OR sourcetype=pan:threat OR (sourcetype=pan:firewall_cloud AND LogType="THREAT")) AND (log_subtype = "virus" OR log_subtype = "wildfire-virus")
-#tags = malware attack
+#tags = ids attack
 
 [pan_spyware]
 search = sourcetype=pan_threat OR sourcetype=pan:threat OR (sourcetype=pan:firewall_cloud AND LogType="THREAT") AND log_subtype = "spyware" 
@@ -62,11 +71,11 @@ search = sourcetype=pan_decryption OR sourcetype=pan:decryption OR (sourcetype=p
 
 [pan_wildfire_malicious]
 search = sourcetype=pan_threat OR sourcetype=pan:threat OR (sourcetype=pan:firewall_cloud AND LogType="THREAT") AND log_subtype="wildfire" AND verdict=malicious
-#tags = malware attack
+#tags = ids attack
 
 [pan_wildfire]
 search = sourcetype=pan_threat OR sourcetype=pan:threat OR (sourcetype=pan:firewall_cloud AND LogType="THREAT") AND log_subtype = "wildfire" 
-#tags = malware operations
+#tags = ids attack
 
 [pan_malware_attacks]
 search = sourcetype=pan:threat_traps
@@ -86,6 +95,7 @@ search = sourcetype=pan:analytics_traps OR sourcetype=pan:threat_traps OR source
 
 [pan_correlation]
 search = sourcetype=pan_correlation OR sourcetype=pan:correlation OR (sourcetype=pan:firewall_cloud AND LogType="CORRELATION")
+#tags = alert
 
 [pan_email]
 search = sourcetype=pan_threat OR sourcetype=pan:threat OR (sourcetype=pan:firewall_cloud AND LogType="THREAT") AND recipient="*" AND sender="*"
@@ -95,18 +105,22 @@ search = sourcetype=pan_aperture OR sourcetype=pan:aperture
 
 [pan_aperture_incident]
 search = sourcetype=pan_aperture OR sourcetype=pan:aperture AND log_type="incident"
+#tags = alert
 
 [pan_aperture_remediation]
 search = sourcetype=pan_aperture OR sourcetype=pan:aperture AND log_type="remediation"
+#tags = alert
 
 [pan_aperture_policy_violation]
 search = sourcetype=pan_aperture OR sourcetype=pan:aperture AND log_type="policy_violation"
+#tags = alert
 
 [pan_aperture_activity_monitoring]
 search = sourcetype=pan_aperture OR sourcetype=pan:aperture AND log_type="activity_monitoring"
 
 [pan_aperture_admin_audit]
 search = sourcetype=pan_aperture OR sourcetype=pan:aperture AND log_type="admin_audit"
+#tags = authentication
 
 [pan_iot_alert]
 search = (sourcetype=pan:iot_alert)

Splunk_TA_paloalto/default/props.conf

@@ -173,10 +173,13 @@ rename = pan:threat
 
 [pan:threat]
 SHOULD_LINEMERGE = false
+EVENT_BREAKER_ENABLE = true
+KV_MODE = none
 TIME_PREFIX = ^(?:[^,]*,){6}
 MAX_TIMESTAMP_LOOKAHEAD = 32
+TIME_FORMAT = %Y/%m/%d %H:%M:%S
 
-REPORT-search = extract_threat, extract_threat_id, extract_threat_name, extract_dest_hostname, extract_http_referrer_name, extract_major_content_type, extract_filename
+REPORT-search = extract_threat, extract_threat_id, extract_threat_name, extract_dest_hostname, extract_http_referrer_name, extract_major_content_type, extract_filename, extract_url_domain
 
 FIELDALIAS-app                      = app as application
 FIELDALIAS-virtual_system           = vsys as virtual_system
@@ -191,6 +194,7 @@ FIELDALIAS-dvc_for_pan_threat       = host as dvc
 FIELDALIAS-http_content_type_for_pan_threat     = content_type as http_content_type
 FIELDALIAS-http_user_agent_for_pan_threat       = user_agent as http_user_agent
 FIELDALIAS-http_referrer_for_pan_threat         = referrer as http_referrer
+FIELDALIAS-http_referrer_domain_pan_threat      = http_referrer_name as http_referrer_domain
 EVAL-http_user_agent_length                     = len(user_agent)
 EVAL-url_length                                 = len(url)
 
@@ -205,14 +209,12 @@ EVAL-sender          = split(replace(sender,"^From: (.*?);?$","\1"),";")
 EVAL-recipient       = split(replace(recipient,"^To: (.*?);?$","\1"),";")
 EVAL-subject         = replace(subject,"^Sub: ","")
 
-EVAL-user                           = coalesce(src_user,dest_user,recipient,sender,"unknown")
+EVAL-user                           = coalesce(src_user,dest_user,recipient,sender)
 # All url logs have an http_method field, this makes ES and Web Proxy app work better. Also make method uppercase for CIM.
 EVAL-http_method                    = if(log_subtype == "url", if(isnull(http_method), "unknown", upper(http_method)), null)
 # Misc field is used by pan_url logs for full URL accessed.  Alias to url field.
 FIELDALIAS-url_for_pan_threat       = misc as url
 
-FIELDALIAS-protocol_for_pan_threat  = protocol as vendor_protocol
-
 # Determine client and server ip address based on direction of flow
 EVAL-server_ip       = if(version >= 2049 OR (version < 2049 AND (isnull(direction) OR direction="client-to-server")), dest_ip, src_ip)
 EVAL-client_ip       = if(version >= 2049 OR (version < 2049 AND (isnull(direction) OR direction="client-to-server")), src_ip, dest_ip)
@@ -226,7 +228,10 @@ EVAL-report_id       = if(log_subtype=="wildfire", coalesce(report_id,threat_id)
 EVAL-http_category   = if(log_subtype=="url", raw_category, null())
 EVAL-verdict         = if(log_subtype=="wildfire", raw_category, null())
 EVAL-threat_category = if(log_subtype!="url" AND log_subtype!="file", if(threat_category=="unknown",log_subtype,coalesce(threat_category,log_subtype)), null())
-EVAL-category        = if(log_subtype=="url" OR log_subtype=="file", raw_category, threat_category)
+EVAL-category        =  if(log_subtype=="url", if(raw_category!="unknown",raw_category,null()), if(threat_category=="unknown",log_subtype,coalesce(threat_category,log_subtype)))
+EVAL-protocol        = case(transport IN ("tcp", "udp"), "ip", transport=="icmp", "icmp", true(), protocol)
+EVAL-protocol_version = if(match(coalesce(src_ip,dest_ip),":"), "ipv6", if(match(coalesce(src_ip,dest_ip), "(?:\d+\.\d+\.\d+\.\d+)"),"ipv4", null))
+EVAL-vendor_protocol = case(transport IN ("tcp", "udp"), "ip", transport=="icmp", "icmp", true(), protocol)
 
 # Decode hex flags
 EVAL-flags           = mvappend(if(floor(tonumber(session_flags,16) / pow(2, 31))%2==0,null(),"pcap"),if(floor(tonumber(session_flags,16) / pow(2, 28))%2==0,null(),"credential_detected"),if(floor(tonumber(session_flags,16) / pow(2, 25))%2==0,null(),"ipv6"),if(floor(tonumber(session_flags,16) / pow(2, 24))%2==0,null(),"decrypted"),if(floor(tonumber(session_flags,16) / pow(2, 23))%2==0,null(),"denied_by_url_filtering"),if(floor(tonumber(session_flags,16) / pow(2, 22))%2==0,null(),"nat"),if(floor(tonumber(session_flags,16) / pow(2, 21))%2==0,null(),"captive_portal"),if(floor(tonumber(session_flags,16) / pow(2, 19))%2==0,null(),"x_forwarded_for"),if(floor(tonumber(session_flags,16) / pow(2, 18))%2==0,null(),"http_proxy"),if(floor(tonumber(session_flags,16) / pow(2, 15))%2==0,null(),"container_page"),if(floor(tonumber(session_flags,16) / pow(2, 13))%2==0,null(),"implicit_application"),if(floor(tonumber(session_flags,16) / pow(2, 11))%2==0,null(),"symmetric_return"))
@@ -247,8 +252,11 @@ rename = pan:traffic
 
 [pan:traffic]
 SHOULD_LINEMERGE = false
+EVENT_BREAKER_ENABLE = true
+KV_MODE = none
 TIME_PREFIX = ^(?:[^,]*,){6}
 MAX_TIMESTAMP_LOOKAHEAD = 32
+TIME_FORMAT = %Y/%m/%d %H:%M:%S
 
 REPORT-search = extract_traffic
 
@@ -259,9 +267,12 @@ EVAL-vendor_action                   = action
 LOOKUP-vendor_action                 = pan_vendor_action_lookup vendor_action OUTPUT action
 # bytes, bytes_in, bytes_out
 FIELDALIAS-dest_for_pan_traffic      = dest_ip as dest
-FIELDALIAS-dvc_for_pan_traffic       = host as dvc
-FIELDALIAS-protocol_for_pan_traffic  = protocol as vendor_protocol
+EVAL-dvc                             = coalesce(dvc_name, host)
 FIELDALIAS-src_for_pan_traffic       = src_ip as src
+FIELDALIAS-session_start_signature   = rule as signature
+EVAL-protocol                        = case(transport IN ("tcp", "udp"), "ip", transport=="icmp", "icmp", true(), protocol)
+EVAL-protocol_version                = if(match(coalesce(src_ip,dest_ip),":"), "ipv6", if(match(coalesce(src_ip,dest_ip), "(?:\d+\.\d+\.\d+\.\d+)"),"ipv4", null))
+EVAL-vendor_protocol                 = case(transport IN ("tcp", "udp"), "ip", transport=="icmp", "icmp", true(), protocol)
 
 # Set user field
 EVAL-user                            = coalesce(src_user,dest_user,"unknown")
@@ -291,17 +302,27 @@ rename = pan:system
 
 [pan:system]
 SHOULD_LINEMERGE = false
+EVENT_BREAKER_ENABLE = true
+KV_MODE = none
 TIME_PREFIX = ^(?:[^,]*,){6}
 MAX_TIMESTAMP_LOOKAHEAD = 32
 
-REPORT-search = extract_system, extract_globalprotect_user, extract_globalprotect_ip, extract_globalprotect_loginip, extract_globalprotect_clientversion, extract_globalprotect_message
+REPORT-search = extract_system, extract_globalprotect_user, extract_globalprotect_ip, extract_globalprotect_loginip, extract_globalprotect_clientversion, extract_globalprotect_message, extract_general_user, extract_system_alert_src, extract_system_auth
 
 FIELDALIAS-virtual_system         = vsys as virtual_system
 # Field Aliases to map specific fields to the Splunk Common Information Model - Update
 FIELDALIAS-dvc_for_pan_system     = host as dvc
 FIELDALIAS-dest_for_pan_system    = host as dest
 FIELDALIAS-signature              = event_id as signature
+FIELDALIAS-src_user               = user as src_user
+FIELDALIAS-reason                 = description as reason
+FIELDALIAS-body                   = description as body
 LOOKUP-vendor_info_for_pan_config = pan_vendor_info_lookup sourcetype OUTPUT vendor,product,vendor_product
+EVAL-action = case(match(description,"(?i)succeeded"),"success",match(description,"(?i)cleared"),"cleared",match(description,"(?i)GlobalProtect gateway agent message"),"success",match(description,"(?i)Failed"),"failure")
+EVAL-app = "Palo Alto Networks Firewall"
+EVAL-type = "event"
+EVAL-src = coalesce(src,src_ip)
+
 
 # GlobalProtect logs introduced in PANOS 9.1
 [pan_globalprotect]
@@ -338,6 +359,8 @@ rename = pan:config
 
 [pan:config]
 SHOULD_LINEMERGE = false
+EVENT_BREAKER_ENABLE = true
+KV_MODE = none
 TIME_PREFIX = ^(?:[^,]*,){6}
 MAX_TIMESTAMP_LOOKAHEAD = 32
 
@@ -350,6 +373,11 @@ FIELDALIAS-config              = configuration_path as path
 # Field Aliases to map specific fields to the Splunk Common Information Model - Change Analysis
 FIELDALIAS-dvc_for_pan_config  = host as dvc
 FIELDALIAS-dest_for_pan_config = host as dest
+FIELDALIAS-src_for_pan_config = host_name as src
+FIELDALIAS-user_for_pan_config = admin as user
+FIELDALIAS-src_user_for_pan_config = admin as src_user
+
+EVAL-status = if(result=="Succeeded" OR result=="Submitted", "success", null)
 
 # Manually set log_subtype because it isn't in the log
 EVAL-log_subtype = "config"
@@ -379,24 +407,28 @@ LOOKUP-src_class                    = classification_lookup cidr as src_ip OUTPU
 
 [pan:correlation]
 SHOULD_LINEMERGE = false
+EVENT_BREAKER_ENABLE = true
+KV_MODE = none
 TIME_PREFIX = ^(?:[^,]*,){6}
 MAX_TIMESTAMP_LOOKAHEAD = 32
+TIME_FORMAT = %Y/%m/%d %H:%M:%S
 REPORT-search = extract_correlation
 FIELDALIAS-virtual_system              = vsys as virtual_system
 FIELDALIAS-src_for_pan_correlation     = src_ip as src
-FIELDALIAS-dest_ip_for_pan_correlation = src_ip as dest_ip
 FIELDALIAS-client_ip                   = src_ip as client_ip
-FIELDALIAS-dest_for_pan_correlation    = src_ip as dest
+FIELDALIAS-dest_for_pan_correlation    = host as dest
 FIELDALIAS-dvc_for_pan_correlation     = host as dvc
-EVAL-user                              = coalesce(src_user,"unknown")
-FIELDALIAS-user_for_pan_correlation    = src_user as dest_user
+FIELDALIAS-user_for_pan_correlation    = src_user as user
 EVAL-log_subtype                       = "correlation"
 FIELDALIAS-category                    = object AS category
 FIELDALIAS-threat_category             = object AS threat_category
 FIELDALIAS-threat_name                 = evidence AS threat_name
 FIELDALIAS-signature                   = evidence AS signature
+FIELDALIAS-body                        = evidence AS body
 EVAL-vendor_action                     = "allowed"
 EVAL-action                            = "allowed"
+EVAL-type                              = case(severity IN ("informational","low","medium"),"event",severity IN ("high","critical"),"alert",1==1,log_type)
+EVAL-app                               = "Palo Alto Networks Firewall"
 
 
 [pan:userid]
@@ -505,10 +537,9 @@ TIME_FORMAT=%Y-%m-%dT%H:%M:%S%z
 MAX_TIMESTAMP_LOOKAHEAD=25
 KV_MODE                     = JSON
 SHOULD_LINEMERGE            = false
-EVAL-action                 = coalesce(action, action_taken, "unknown")
 EVAL-dvc                    = "aperture"
-EVAL-src_user               = coalesce(user, item_owner)
-EVAL-user                   = coalesce(user, item_owner)
+EVAL-src_user               = coalesce(user, item_owner, admin_id)
+EVAL-user                   = coalesce(user, item_owner, admin_id)
 FIELDALIAS-admin            = action_taken_by as admin
 FIELDALIAS-log_subtype      = log_type as log_subtype
 FIELDALIAS-object           = item_name as object
@@ -519,12 +550,20 @@ FIELDALIAS-object_creator   = item_creator as object_creator
 FIELDALIAS-object_category  = item_type as object_category
 FIELDALIAS-category         = incident_category as category
 FIELDALIAS-threat_category  = incident_category as threat_category
-FIELDALIAS-signature        = policy_rule_name as signature
-FIELDALIAS-src_ip           = source_ip as src_ip
-FIELDALIAS-client_ip        = source_ip as client_ip
 EVAL-threat_name            = coalesce(policy_rule_name,log_subtype)
-LOOKUP-vendor_info_for_pan_aperture = pan_vendor_info_lookup sourcetype OUTPUT vendor,product,vendor_product
-
+LOOKUP-vendor_info_for_pan_aperture = pan_vendor_info_lookup sourcetype OUTPUTNEW vendor,product,vendor_product, vendor_product as app
+
+EVAL-src                    = coalesce(ip, source_ip)
+EVAL-type                   = if(log_type IN ("incident", "remediation"), "alert", "event")
+FIELDALIAS-user_role        = admin_role as user_role
+FIELDALIAS-id               = incident_id as id
+FIELDALIAS-signature_id     = policy_rule_name as signature_id
+FIELDALIAS-severity_id      = severity as severity_id
+EVAL-action                 = coalesce(case(action IN ("sign_in", "sign_out"), "success"), action, action_taken)
+EVAL-client_ip              = coalesce(source_ip, ip)
+EVAL-src_ip                 = coalesce(source_ip, ip)
+EVAL-signature              = coalesce(policy_rule_name, event_type)
+EVAL-severity               = case(severity <= 1, "informational", severity <=2, "low", severity <=3, "medium", severity <=4, "high", severity <=5, "critical")
 
 [pan:minemeld]
 SHOULD_LINEMERGE = 0