feat(cortex-xdr): fetch incidents with their alerts, not a 21-field summary
incidents/get_incidents/ answers with a summary and nothing else: 21 fields,
no hosts, no users, no MITRE, no tags, and not one of the alerts the incident
aggregates. Ingesting through it leaves an incident whose raw payload says
almost nothing about what happened — and the shipped mapper had been written
for a richer shape than the endpoint ever returns, mapping hosts[0], users[0]
and mitre_* that simply are not in that response.
incidents/get_multiple_incidents_extra_data/ returns the same incidents with
39 fields, every alert in full — 156 fields each — and the file and network
artifacts. It is what the reference client fetches through (demisto/content,
CortexXDRIR.get_multiple_incidents_extra_data), and full_alert_fields must be
set or the nested alerts come back trimmed to a handful of fields.
Records arrive as {incident, alerts, network_artifacts, file_artifacts} with
each nested block wrapped as {total_count, data}. The script flattens them, so
every expression written against get_incidents keeps working — the summary's
21 fields are a subset of these 39 — while the alerts and artifacts land beside
them as plain lists, and their total_count says when a list is a sample rather
than the whole set. incident_sources is lifted into a scalar for the same
reason severity was on the alerts side: the incident-field mapper reads dotted
paths and cannot index a list.
get_incidents stays, for cheap polling, and now says in its description what it
does and does not carry.
The mapper maps the aggregate first and the first alert last, so the alert
fills in whatever the aggregate leaves silent — including the detection anchor,
since an XDR incident's detection_time is usually null while its alerts carry
theirs. Verified against the vendor's recorded response
(test_data/get_multiple_incidents_extra_data.json): 33 of 52 entries resolve,
severity critical lands on 5, source reads "XDR Agent", and the anchor falls
through to the alert's detection timestamp.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,174 @@
|
||||
import json, os, sys, time, hashlib, secrets, string, urllib.request, urllib.error
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def _client():
|
||||
s = json.loads(os.environ.get("INTEGRATION_SECRETS") or "{}")
|
||||
raw = str(s.get("url") or "").strip().rstrip("/")
|
||||
if not raw:
|
||||
raise ValueError("no url configured — paste the tenant API URL (Cortex XDR > Settings > Configurations > API Keys > Copy URL)")
|
||||
if "://" not in raw:
|
||||
raw = "https://" + raw
|
||||
# The tenant URL is a bare host. Drop whatever was pasted after it (a stray
|
||||
# /public_api/v1, a console path) so the API root is built exactly once.
|
||||
scheme, _, rest = raw.partition("://")
|
||||
base = scheme + "://" + rest.split("/")[0] + "/public_api/v1"
|
||||
key = s.get("api_key", "")
|
||||
kid = str(s.get("api_key_id", ""))
|
||||
if not key or not kid:
|
||||
raise ValueError("api_key and api_key_id are both required")
|
||||
# Nonce and timestamp ride along in both modes, as the reference client does.
|
||||
# A standard key travels as-is; an advanced one as sha256(key + nonce + ts).
|
||||
nonce = "".join(secrets.choice(string.ascii_letters + string.digits) for _ in range(64))
|
||||
ts = str(int(time.time()) * 1000)
|
||||
headers = {
|
||||
"x-xdr-auth-id": kid,
|
||||
"x-xdr-nonce": nonce,
|
||||
"x-xdr-timestamp": ts,
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
if str(s.get("auth_type") or "standard").lower() == "advanced":
|
||||
headers["Authorization"] = hashlib.sha256((key + nonce + ts).encode("utf-8")).hexdigest()
|
||||
else:
|
||||
headers["Authorization"] = key
|
||||
return base, headers
|
||||
|
||||
|
||||
def _not_json(r, raw):
|
||||
"""A 2xx that is not JSON means we are not talking to the XDR API at all."""
|
||||
ctype = (r.headers.get("Content-Type") or "unknown").split(";")[0].strip()
|
||||
head = raw[:160].decode("utf-8", "replace").replace("\n", " ").strip()
|
||||
return (
|
||||
"expected JSON from " + r.geturl() + ", got " + ctype + " (HTTP " + str(r.status) + "): " + head
|
||||
+ " — check the configured url is the tenant API host"
|
||||
+ " (https://api-<tenant>.xdr.<region>.paloaltonetworks.com), not the console URL"
|
||||
)
|
||||
|
||||
|
||||
def post(path, request_data):
|
||||
base, headers = _client()
|
||||
data = json.dumps({"request_data": request_data}).encode("utf-8")
|
||||
req = urllib.request.Request(base + path, data=data, headers=headers, method="POST")
|
||||
with urllib.request.urlopen(req, timeout=90) as r:
|
||||
raw = r.read()
|
||||
if not raw:
|
||||
return {}
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except ValueError:
|
||||
raise ValueError(_not_json(r, raw))
|
||||
|
||||
|
||||
def to_ms(v):
|
||||
if v in (None, ""):
|
||||
return None
|
||||
s = str(v)
|
||||
if s.isdigit():
|
||||
return int(s)
|
||||
try:
|
||||
return int(datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp() * 1000)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def csv(v):
|
||||
return [x.strip() for x in str(v or "").split(",") if x.strip()]
|
||||
|
||||
|
||||
# incidents/get_incidents/ answers with a 21-field summary: no hosts, no users,
|
||||
# no MITRE, and above all not one of the alerts the incident aggregates. This
|
||||
# endpoint returns the same incident with 39 fields, its alerts in full (156
|
||||
# fields each) and its file/network artifacts — which is why the reference
|
||||
# client fetches through it and not through get_incidents (demisto/content,
|
||||
# CortexXDRIR.get_multiple_incidents_extra_data).
|
||||
PAGE = 50
|
||||
|
||||
# Artifacts are dropped by name, not by omission — the API only understands
|
||||
# being told which blocks to leave out.
|
||||
ARTIFACT_BLOCKS = ["network_artifacts", "file_artifacts"]
|
||||
|
||||
|
||||
def flatten(item):
|
||||
"""One record -> one incident.
|
||||
|
||||
Records arrive as {incident, alerts, network_artifacts, file_artifacts},
|
||||
each nested block wrapped as {total_count, data}. Flattening the incident to
|
||||
the top level keeps every expression written against get_incidents working
|
||||
unchanged — the summary's 21 fields are a subset of these 39 — while the
|
||||
alerts and artifacts land beside them as plain lists.
|
||||
"""
|
||||
incident = dict(item.get("incident") or {})
|
||||
for key in ("alerts", "network_artifacts", "file_artifacts"):
|
||||
block = item.get(key)
|
||||
if not isinstance(block, dict):
|
||||
continue
|
||||
incident[key] = block.get("data") or []
|
||||
if block.get("total_count") is not None:
|
||||
# The tenant caps alerts per incident (50 by default), so the count
|
||||
# says when the list is a sample rather than the whole set.
|
||||
incident[key + "_total_count"] = block["total_count"]
|
||||
# The producing sensor is a list here, and the incident-field mapper reads
|
||||
# dotted paths only — no array indexing — so the first source is lifted out
|
||||
# for it. The list itself stays, for expressions that can index.
|
||||
sources = incident.get("incident_sources")
|
||||
if isinstance(sources, list) and sources:
|
||||
incident["incident_source"] = sources[0]
|
||||
return incident
|
||||
|
||||
|
||||
def main():
|
||||
inputs = json.loads(os.environ.get("INTEGRATION_INPUTS", "{}"))
|
||||
limit = max(1, int(inputs.get("limit") or 50))
|
||||
filters = []
|
||||
if inputs.get("status"):
|
||||
statuses = csv(inputs["status"])
|
||||
filters.append({"field": "status", "operator": "in", "value": statuses})
|
||||
created_ms = to_ms(inputs.get("created_after"))
|
||||
if created_ms is not None:
|
||||
filters.append({"field": "creation_time", "operator": "gte", "value": created_ms})
|
||||
# Oldest first on an incremental fetch, so that a window holding more
|
||||
# incidents than `limit` drops its most RECENT ones — the only ones the next
|
||||
# poll can still see. Newest first otherwise, for a hand-run command.
|
||||
keyword = "asc" if created_ms is not None else "desc"
|
||||
exclude = str(inputs.get("exclude_artifacts") or "").lower() in ("1", "true", "yes")
|
||||
|
||||
incidents, total = [], None
|
||||
while len(incidents) < limit:
|
||||
rd = {
|
||||
"search_from": len(incidents),
|
||||
"search_to": min(len(incidents) + PAGE, limit),
|
||||
"sort": {"field": "creation_time", "keyword": keyword},
|
||||
# Without this the nested alerts come back trimmed to a handful of
|
||||
# fields — the very thing this command exists to avoid.
|
||||
"full_alert_fields": True,
|
||||
}
|
||||
if filters:
|
||||
rd["filters"] = filters
|
||||
if exclude:
|
||||
rd["fields_to_exclude"] = ARTIFACT_BLOCKS
|
||||
reply = (post("/incidents/get_multiple_incidents_extra_data/", rd) or {}).get("reply") or {}
|
||||
page = reply.get("incidents") or []
|
||||
if total is None:
|
||||
total = reply.get("total_count")
|
||||
incidents.extend(flatten(i) for i in page)
|
||||
if len(page) < PAGE:
|
||||
break
|
||||
|
||||
out = {"result_count": len(incidents), "incidents": incidents}
|
||||
if total is not None:
|
||||
out["total_count"] = total
|
||||
# Say it when the window held more than the limit: those incidents are
|
||||
# not coming back on the next poll.
|
||||
out["truncated"] = total > len(incidents)
|
||||
print(json.dumps({"reply": out}))
|
||||
|
||||
|
||||
try:
|
||||
main()
|
||||
except urllib.error.HTTPError as e:
|
||||
print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")}))
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": str(e)}))
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user