diff --git a/integrations/rapid7-insightidr/manifest.yaml b/integrations/rapid7-insightidr/manifest.yaml index a8314e5..59a35f7 100644 --- a/integrations/rapid7-insightidr/manifest.yaml +++ b/integrations/rapid7-insightidr/manifest.yaml @@ -1,8 +1,8 @@ id: rapid7_insightidr name: Rapid7 InsightIDR -version: 1.4.0 +version: 1.4.1 description: "Rapid7 InsightIDR (REST API) — investigation ingestion and full lifecycle (list/get/search/create/update/assign/set-status/bulk-close), investigation alerts and Rapid7 product alerts, custom threat indicators (add/replace), log management and LEQL log/log-set queries with downloads, and user directory search." -changelog: "1.4.0 — Add get_investigations_full: the investigation with its alerts inlined, so it stays the unit that becomes an incident. Drop-in for list_investigations (results_path data, dedup rrn). | 1.3.0 — Add search_investigation_alerts: ingest only the alerts that belong to an investigation, with their full detection content, in 1+N+1 calls. Filtering search_alerts on investigation_rrn does not work — the term is accepted and silently matches nothing. | 1.2.1 — search_alerts: report a malformed terms_json with the input named and echoed, instead of json's bare \"Expecting value: line 1 column 1\". | 1.2.0 — Add the search_alerts OCSF mapper: rule_keys_of_interest carries the host, the process, its command line, its parent and the file hashes, so an alert-based feed indexes entities where the investigation feed indexed none. Validated against a live ABA alert. | 1.1.1 — list_alert_fields: default path_depth to 1 and expose the search filter. The V2 fields endpoint refuses an unbounded walk of the tree (\"root path with path depth greater than 1 is not supported for V2 without a search filter\"), so the bare call returned HTTP 400. | 1.1.0 — Add the Alert Triage API (/idr/at): search alerts, get an alert, its evidence, its actors and its process trees, plus the tenant's alert field catalogue. These carry the detection content the investigation endpoints do not — MITRE T-codes, matched rule keys, assets and accounts, command lines. MDR-gated. | 1.0.0 — Initial release: investigation ingestion (list_investigations) with an exhaustive OCSF mapper, 18 commands across investigations, alerts, custom threats, logs/LEQL search and users. API v1/v2 selectable per instance and per command." +changelog: "1.4.1 — Read the alert RRN from `rrn` before `id` and drop anything not RRN-shaped: the investigation alert list also returns bare UUIDs, and one of those rejects the entire /alerts/ops/rrns batch (HTTP 400), losing every alert in the run. | 1.4.0 — Add get_investigations_full: the investigation with its alerts inlined, so it stays the unit that becomes an incident. Drop-in for list_investigations (results_path data, dedup rrn). | 1.3.0 — Add search_investigation_alerts: ingest only the alerts that belong to an investigation, with their full detection content, in 1+N+1 calls. Filtering search_alerts on investigation_rrn does not work — the term is accepted and silently matches nothing. | 1.2.1 — search_alerts: report a malformed terms_json with the input named and echoed, instead of json's bare \"Expecting value: line 1 column 1\". | 1.2.0 — Add the search_alerts OCSF mapper: rule_keys_of_interest carries the host, the process, its command line, its parent and the file hashes, so an alert-based feed indexes entities where the investigation feed indexed none. Validated against a live ABA alert. | 1.1.1 — list_alert_fields: default path_depth to 1 and expose the search filter. The V2 fields endpoint refuses an unbounded walk of the tree (\"root path with path depth greater than 1 is not supported for V2 without a search filter\"), so the bare call returned HTTP 400. | 1.1.0 — Add the Alert Triage API (/idr/at): search alerts, get an alert, its evidence, its actors and its process trees, plus the tenant's alert field catalogue. These carry the detection content the investigation endpoints do not — MITRE T-codes, matched rule keys, assets and accounts, command lines. MDR-gated. | 1.0.0 — Initial release: investigation ingestion (list_investigations) with an exhaustive OCSF mapper, 18 commands across investigations, alerts, custom threats, logs/LEQL search and users. API v1/v2 selectable per instance and per command." category: siem # Per-instance configuration. The Insight platform region selects the API host diff --git a/integrations/rapid7-insightidr/scripts/get_investigations_full.py b/integrations/rapid7-insightidr/scripts/get_investigations_full.py index 947f2aa..e18d7cf 100644 --- a/integrations/rapid7-insightidr/scripts/get_investigations_full.py +++ b/integrations/rapid7-insightidr/scripts/get_investigations_full.py @@ -61,6 +61,27 @@ DEFAULT_BUDGET_S = 60 # plain investigation feed. Only the command changes. +def _alert_rrn(a): + """The alert's RRN, or None when this alert has none. + + The investigation alert list is not consistent about where the RRN sits. + Some entries carry it in `id` (rrn:alerts:eu::alert:1:); others put + a plain UUID there and the RRN, if any, in `rrn`. `rrn` is therefore read + first, and anything that is not RRN-shaped is dropped rather than sent on. + + That last part is not defensive habit. /alerts/ops/rrns validates the whole + list and rejects the entire request when one entry is malformed, so a single + UUID slipping through costs every alert in the batch — which is exactly how + this failed: HTTP 400, eight invalid RRNs, zero alerts fetched, the whole run + lost. Dropping them here means the investigation still arrives, with its + unretrievable alerts counted rather than silently missing. + """ + for v in (a.get("rrn"), a.get("id")): + if isinstance(v, str) and v.startswith("rrn:"): + return v + return None + + def run(): _, _, multi = _cfg() inp = _inputs() @@ -92,7 +113,7 @@ def run(): # guards. An investigation whose alerts cannot be read keeps its place in the # output with an alerts_error: losing the whole run over one of them, or # dropping it silently, would both be worse. - rrns_by_inv, order, truncated = {}, [], False + rrns_by_inv, order, truncated, skipped = {}, [], False, 0 for inv in investigations: inv_rrn = inv.get("rrn") or "" if not inv_rrn: @@ -108,11 +129,13 @@ def run(): continue got = [] for a in (page.get("data") or []): - rrn = a.get("id") or a.get("rrn") - if rrn: - got.append(rrn) - if rrn not in order: - order.append(rrn) + rrn = _alert_rrn(a) + if rrn is None: + skipped += 1 + continue + got.append(rrn) + if rrn not in order: + order.append(rrn) rrns_by_inv[inv_rrn] = got # The alerts themselves, batched, then indexed so each investigation can be @@ -168,6 +191,7 @@ def run(): "investigations_returned": len(investigations), "alert_rrns_collected": len(order), "alerts_fetched": fetched, + "alert_ids_without_rrn": skipped, "truncated": truncated, "elapsed_seconds": round(time.monotonic() - started, 1), }, diff --git a/integrations/rapid7-insightidr/scripts/search_investigation_alerts.py b/integrations/rapid7-insightidr/scripts/search_investigation_alerts.py index 58c3e42..5485458 100644 --- a/integrations/rapid7-insightidr/scripts/search_investigation_alerts.py +++ b/integrations/rapid7-insightidr/scripts/search_investigation_alerts.py @@ -51,6 +51,27 @@ RRN_BATCH = 100 DEFAULT_BUDGET_S = 60 +def _alert_rrn(a): + """The alert's RRN, or None when this alert has none. + + The investigation alert list is not consistent about where the RRN sits. + Some entries carry it in `id` (rrn:alerts:eu::alert:1:); others put + a plain UUID there and the RRN, if any, in `rrn`. `rrn` is therefore read + first, and anything that is not RRN-shaped is dropped rather than sent on. + + That last part is not defensive habit. /alerts/ops/rrns validates the whole + list and rejects the entire request when one entry is malformed, so a single + UUID slipping through costs every alert in the batch — which is exactly how + this failed: HTTP 400, eight invalid RRNs, zero alerts fetched, the whole run + lost. Dropping them here means the investigation still arrives, with its + unretrievable alerts counted rather than silently missing. + """ + for v in (a.get("rrn"), a.get("id")): + if isinstance(v, str) and v.startswith("rrn:"): + return v + return None + + def run(): _, _, multi = _cfg() inp = _inputs() @@ -82,7 +103,7 @@ def run(): # 2) Their alert RRNs. One call per investigation — there is no bulk form — # so this is the step the time budget guards. - order, by_rrn, truncated = [], {}, False + order, by_rrn, truncated, skipped = [], {}, False, 0 for inv in investigations: if time.monotonic() - started > budget or len(order) >= max_alerts: truncated = True @@ -98,8 +119,11 @@ def run(): # alert simply does not appear; the counts below say how many did. continue for a in (page.get("data") or []): - rrn = a.get("id") or a.get("rrn") - if rrn and rrn not in by_rrn: + rrn = _alert_rrn(a) + if rrn is None: + skipped += 1 + continue + if rrn not in by_rrn: order.append(rrn) by_rrn[rrn] = inv if len(order) >= max_alerts: @@ -135,6 +159,7 @@ def run(): "investigations_read": len(investigations), "alert_rrns_collected": len(order), "alerts_returned": len(alerts), + "alert_ids_without_rrn": skipped, "truncated": truncated, "elapsed_seconds": round(time.monotonic() - started, 1), },