diff --git a/integrations/rapid7-insightidr/manifest.yaml b/integrations/rapid7-insightidr/manifest.yaml index a939b53..43c4d55 100644 --- a/integrations/rapid7-insightidr/manifest.yaml +++ b/integrations/rapid7-insightidr/manifest.yaml @@ -1,8 +1,8 @@ id: rapid7_insightidr name: Rapid7 InsightIDR -version: 1.7.0 +version: 1.8.0 description: "Rapid7 InsightIDR (REST API) — investigation ingestion and full lifecycle (list/get/search/create/update/assign/set-status/bulk-close), investigation alerts and Rapid7 product alerts, custom threat indicators (add/replace), log management and LEQL log/log-set queries with downloads, and user directory search." -changelog: "1.7.0 — Map source/destination assets and accounts to src_endpoint, dst_endpoint, actor.user and user. A connection rule names both ends and neither is \"the host\", so an RDP-to-DC alert produced no entity at all and nothing a layout could show. | 1.6.0 — Surface region_failures: the Alert Triage API answers HTTP 200 with a partial result when it cannot reach a region, and that list was being dropped. Also report the regions seen in the RRNs against the configured one. | 1.5.0 — Flatten the whole of rule_keys_of_interest into the context. It is an open vocabulary — a process rule and an authentication rule share no keys — so the hardcoded list only covered rules already seen and left every other alert empty. | 1.4.1 — Read the alert RRN from `rrn` before `id` and drop anything not RRN-shaped: the investigation alert list also returns bare UUIDs, and one of those rejects the entire /alerts/ops/rrns batch (HTTP 400), losing every alert in the run. | 1.4.0 — Add get_investigations_full: the investigation with its alerts inlined, so it stays the unit that becomes an incident. Drop-in for list_investigations (results_path data, dedup rrn). | 1.3.0 — Add search_investigation_alerts: ingest only the alerts that belong to an investigation, with their full detection content, in 1+N+1 calls. Filtering search_alerts on investigation_rrn does not work — the term is accepted and silently matches nothing. | 1.2.1 — search_alerts: report a malformed terms_json with the input named and echoed, instead of json's bare \"Expecting value: line 1 column 1\". | 1.2.0 — Add the search_alerts OCSF mapper: rule_keys_of_interest carries the host, the process, its command line, its parent and the file hashes, so an alert-based feed indexes entities where the investigation feed indexed none. Validated against a live ABA alert. | 1.1.1 — list_alert_fields: default path_depth to 1 and expose the search filter. The V2 fields endpoint refuses an unbounded walk of the tree (\"root path with path depth greater than 1 is not supported for V2 without a search filter\"), so the bare call returned HTTP 400. | 1.1.0 — Add the Alert Triage API (/idr/at): search alerts, get an alert, its evidence, its actors and its process trees, plus the tenant's alert field catalogue. These carry the detection content the investigation endpoints do not — MITRE T-codes, matched rule keys, assets and accounts, command lines. MDR-gated. | 1.0.0 — Initial release: investigation ingestion (list_investigations) with an exhaustive OCSF mapper, 18 commands across investigations, alerts, custom threats, logs/LEQL search and users. API v1/v2 selectable per instance and per command." +changelog: "1.8.0 — Fetch each alert evidence at ingestion and parse it: the log line behind the alert lives in a JSON-encoded string no mapper can reach, and it is where the source and destination addresses, the account and the device actually are. | 1.7.0 — Map source/destination assets and accounts to src_endpoint, dst_endpoint, actor.user and user. A connection rule names both ends and neither is \"the host\", so an RDP-to-DC alert produced no entity at all and nothing a layout could show. | 1.6.0 — Surface region_failures: the Alert Triage API answers HTTP 200 with a partial result when it cannot reach a region, and that list was being dropped. Also report the regions seen in the RRNs against the configured one. | 1.5.0 — Flatten the whole of rule_keys_of_interest into the context. It is an open vocabulary — a process rule and an authentication rule share no keys — so the hardcoded list only covered rules already seen and left every other alert empty. | 1.4.1 — Read the alert RRN from `rrn` before `id` and drop anything not RRN-shaped: the investigation alert list also returns bare UUIDs, and one of those rejects the entire /alerts/ops/rrns batch (HTTP 400), losing every alert in the run. | 1.4.0 — Add get_investigations_full: the investigation with its alerts inlined, so it stays the unit that becomes an incident. Drop-in for list_investigations (results_path data, dedup rrn). | 1.3.0 — Add search_investigation_alerts: ingest only the alerts that belong to an investigation, with their full detection content, in 1+N+1 calls. Filtering search_alerts on investigation_rrn does not work — the term is accepted and silently matches nothing. | 1.2.1 — search_alerts: report a malformed terms_json with the input named and echoed, instead of json's bare \"Expecting value: line 1 column 1\". | 1.2.0 — Add the search_alerts OCSF mapper: rule_keys_of_interest carries the host, the process, its command line, its parent and the file hashes, so an alert-based feed indexes entities where the investigation feed indexed none. Validated against a live ABA alert. | 1.1.1 — list_alert_fields: default path_depth to 1 and expose the search filter. The V2 fields endpoint refuses an unbounded walk of the tree (\"root path with path depth greater than 1 is not supported for V2 without a search filter\"), so the bare call returned HTTP 400. | 1.1.0 — Add the Alert Triage API (/idr/at): search alerts, get an alert, its evidence, its actors and its process trees, plus the tenant's alert field catalogue. These carry the detection content the investigation endpoints do not — MITRE T-codes, matched rule keys, assets and accounts, command lines. MDR-gated. | 1.0.0 — Initial release: investigation ingestion (list_investigations) with an exhaustive OCSF mapper, 18 commands across investigations, alerts, custom threats, logs/LEQL search and users. API v1/v2 selectable per instance and per command." category: siem # Per-instance configuration. The Insight platform region selects the API host @@ -451,6 +451,8 @@ commands: statuses: { type: string, description: "Comma-separated investigation statuses to include (open, investigating, closed). Leaving out closed skips what the source has already triaged." } priorities: { type: string, description: "Comma-separated investigation priorities to include (Unspecified, Low, Medium, High, Critical)" } field_ids: { type: string, description: "Comma-separated additional alert field ids to include (see rapid7-insight-idr-list-alert-fields)" } + include_evidence: { type: string, description: "Also fetch each alert's evidence — the log line the alert was raised on, parsed and normalised into evidence_fields (true/false, default true). One extra call per alert." } + max_evidence_per_alert: { type: string, description: "Maximum evidence items per alert (default 5)" } max_alerts: { type: string, description: "Ceiling on alert RRNs collected in one run (default 500)" } time_budget_seconds: { type: string, description: "Stop walking investigations after this many seconds and report truncated=true, rather than exceed the server's 90-second command budget and lose the whole run (default 60)" } index: { type: string, description: "0-based page index" } diff --git a/integrations/rapid7-insightidr/mappers/get_investigations_full.yaml b/integrations/rapid7-insightidr/mappers/get_investigations_full.yaml index 9edba58..a88c67d 100644 --- a/integrations/rapid7-insightidr/mappers/get_investigations_full.yaml +++ b/integrations/rapid7-insightidr/mappers/get_investigations_full.yaml @@ -20,6 +20,8 @@ field_mappings: # alert rule's default severity. severity: "priority" source: "source" + # The log line's own sentence beats a generic title when there is one. + description: "first_evidence_message" # Rapid7's own triage state. It cannot reach the incident's status and verdict columns # — MapFields accepts only title, severity, source, description and custom:* — so it # rides as custom fields, where an analyst at least sees that the source already closed @@ -140,6 +142,30 @@ ocsf: - { source_path: "alerts[0].rule_keys_of_interest[key='destination_ip'].values[0]", ocsf_field: "dst_endpoint.ip" } - { source_path: "alerts[0].rule_keys_of_interest[key='geoip_country_code'].values[0]", ocsf_field: "src_endpoint.location.country" } - { source_path: "alerts[0].rule_keys_of_interest[key='result'].values[0]", ocsf_field: "status_detail" } + # ── The evidence: the log line the alert was raised on ──────────── + # An alert names what fired; the evidence is what actually happened. It lives + # in a JSON-encoded string the command parses and normalises into + # evidence_fields — see get_investigations_full.py. Vendor prefixes are + # stripped there (Forti_srcip -> srcip), so one entry serves every log source + # that names its fields the usual way. + - { source_path: "alerts[0].evidence_fields", ocsf_field: "evidence" } + - { source_path: "alerts[0].evidence_fields.srcip", ocsf_field: "src_endpoint.ip" } + - { source_path: "alerts[0].evidence_fields.dstip", ocsf_field: "dst_endpoint.ip" } + - { source_path: "alerts[0].evidence_fields.srcport", ocsf_field: "src_endpoint.port" } + - { source_path: "alerts[0].evidence_fields.dstport", ocsf_field: "dst_endpoint.port" } + - { source_path: "alerts[0].evidence_fields.user", ocsf_field: "actor.user.name" } + - { source_path: "alerts[0].evidence_fields.devname", ocsf_field: "device.name" } + - { source_path: "alerts[0].evidence_fields.devname", ocsf_field: "device.hostname" } + - { source_path: "alerts[0].evidence_fields.devid", ocsf_field: "device.uid" } + - { source_path: "alerts[0].evidence_fields.hostname", ocsf_field: "device.hostname" } + - { source_path: "alerts[0].evidence_fields.status", ocsf_field: "status_detail" } + # `action` is what the source system did ("login"), which says more than the + # investigation's own `source` ("ALERT"). Mapped after it so it wins. + - { source_path: "alerts[0].evidence_fields.action", ocsf_field: "activity_name" } + # The line itself, on the OCSF field meant for it — and one the layout builder + # offers, unlike `evidence` above, which only ever shows in the Context tab. + - { source_path: "alerts[0].evidence_raw_line", ocsf_field: "raw_data" } + - { source_path: "alerts[0].evidence_fields.msg ? alerts[0].evidence_fields.msg : alerts[0].evidence_fields.logdesc", ocsf_field: "message" } # ── Every alert, not just the first ─────────────────────────────── # An investigation aggregates alerts, and the fields above only ever read alerts[0]. # This entry lands the whole set as OCSF-shaped objects, so the host and the process of @@ -161,6 +187,7 @@ ocsf: "process": rule_keys_of_interest[key='process.name'].values[0], "cmd_line": rule_keys_of_interest[key='process.cmd_line'].values[0], "technique": rule.mitre_tcodes[$substring($, 0, 2) != 'TA'][0], - "detection": rule_keys_of_interest{key: values[0]} + "detection": rule_keys_of_interest{key: values[0]}, + "evidence": evidence_fields } ocsf_field: "evidences" diff --git a/integrations/rapid7-insightidr/scripts/get_investigations_full.py b/integrations/rapid7-insightidr/scripts/get_investigations_full.py index ab6171f..710f8ad 100644 --- a/integrations/rapid7-insightidr/scripts/get_investigations_full.py +++ b/integrations/rapid7-insightidr/scripts/get_investigations_full.py @@ -43,6 +43,7 @@ def request(method, path, params=None, body=None): raw = r.read() return json.loads(raw) if raw else {} +import re import time from datetime import datetime, timedelta, timezone @@ -112,6 +113,62 @@ def _collect_region_failures(out, sink): return out +# key="value" / key=value, the shape syslog-derived log lines take. Non-matching +# sources simply yield nothing, which is the right answer for them. +_KV = re.compile(r'([A-Za-z_][\w.\-]*)=("(?:[^"\\]|\\.)*"|\S+)') + + +def _clean(v): + """Strip the quoting Rapid7's parser leaves behind. + + custom_data values arrive as '10.70.20.66"' — a trailing quote from the + vendor parser. Left alone it produces an IP that matches no regex and an + entity nobody can pivot on, so the cleanup is not cosmetic. + """ + v = str(v).strip() + if len(v) >= 2 and v[0] == '"' and v[-1] == '"': + v = v[1:-1] + return v.replace('\\"', '"').strip().strip('"') + + +def _evidence_fields(ev): + """One evidence's log line as a flat, normalised dict. + + `data` is a JSON-encoded STRING holding {source_data, custom_data} — the + actual event behind the alert. JSONata cannot deserialise, so a mapper can + never reach into it; this is why the parsing has to happen here. + + custom_data is the vendor's normalised view, its keys prefixed per source + (Forti_user, Forti_srcip). The prefix is stripped so a mapper can name + `user` once rather than once per vendor. source_data is the raw line and is + overlaid on top: it carries far more keys (25 against 6 on the sample this + was written from) and it is not affected by the quoting artefact above. + """ + raw = ev.get("data") + if not isinstance(raw, str) or not raw.strip(): + return {}, None + try: + blob = json.loads(raw) + except ValueError: + # Not JSON: keep the string itself as the raw line rather than lose it. + return {}, raw + if not isinstance(blob, dict): + return {}, raw + fields = {} + custom = blob.get("custom_data") + if isinstance(custom, dict): + for k, v in custom.items(): + if not isinstance(v, (str, int, float)): + continue + key = k.split("_", 1)[1] if "_" in k else k + fields[key.lower()] = _clean(v) + source = blob.get("source_data") + if isinstance(source, str): + for k, v in _KV.findall(source): + fields[k.lower()] = _clean(v) + return fields, (source if isinstance(source, str) else raw) + + def run(): _, _, multi = _cfg() inp = _inputs() @@ -185,6 +242,34 @@ def run(): if a.get("rrn"): by_rrn[a["rrn"]] = a + # ④ The evidence behind each alert: the actual log line. An alert names what + # fired; the evidence is what happened. One call per alert — there is no + # bulk form — so it is optional, capped, and under the same time budget. + evidence_calls, evidence_errors = 0, 0 + if str(inp.get("include_evidence") or "true").strip().lower() in ("1", "true", "yes"): + per_alert = int(inp.get("max_evidence_per_alert") or 5) + for rrn, a in by_rrn.items(): + if time.monotonic() - started > budget: + truncated = True + break + try: + page = request("GET", AT + "alerts/" + urllib.parse.quote(rrn, safe="") + "/evidences", + params={"index": "0", "size": per_alert}) + evidence_calls += 1 + except urllib.error.HTTPError as e: + a["evidence_error"] = "HTTP " + str(e.code) + evidence_errors += 1 + continue + evs = (page.get("evidences") or [])[:per_alert] + for ev in evs: + ev["fields"], ev["raw_line"] = _evidence_fields(ev) + a["evidences"] = evs + if evs: + # Lifted for the mapper: MapFields cannot index an array, and the + # ocsf section reads alerts[0] only. + a["evidence_fields"] = evs[0].get("fields") or {} + a["evidence_raw_line"] = evs[0].get("raw_line") + fetched = 0 for inv in investigations: rrns = rrns_by_inv.get(inv.get("rrn") or "", []) @@ -218,6 +303,10 @@ def run(): "first_alert_logset_id": log.get("logset_id"), "first_alert_log_entry_id": log.get("log_entry_id"), }) + ef = a.get("evidence_fields") or {} + if ef: + inv["first_evidence_message"] = ef.get("msg") or ef.get("logdesc") or ef.get("message") + inv["first_evidence_raw_line"] = a.get("evidence_raw_line") print(json.dumps({ "data": investigations, @@ -225,6 +314,8 @@ def run(): "investigations_returned": len(investigations), "alert_rrns_collected": len(order), "alerts_fetched": fetched, + "evidence_calls": evidence_calls, + "evidence_errors": evidence_errors, "alert_ids_without_rrn": skipped, "truncated": truncated, # HTTP 200 with a non-empty region_failures is a PARTIAL result: