diff --git a/integrations/rapid7-insightidr/manifest.yaml b/integrations/rapid7-insightidr/manifest.yaml index ad2cc07..c944d0c 100644 --- a/integrations/rapid7-insightidr/manifest.yaml +++ b/integrations/rapid7-insightidr/manifest.yaml @@ -1,8 +1,8 @@ id: rapid7_insightidr name: Rapid7 InsightIDR -version: 1.5.0 +version: 1.6.0 description: "Rapid7 InsightIDR (REST API) — investigation ingestion and full lifecycle (list/get/search/create/update/assign/set-status/bulk-close), investigation alerts and Rapid7 product alerts, custom threat indicators (add/replace), log management and LEQL log/log-set queries with downloads, and user directory search." -changelog: "1.5.0 — Flatten the whole of rule_keys_of_interest into the context. It is an open vocabulary — a process rule and an authentication rule share no keys — so the hardcoded list only covered rules already seen and left every other alert empty. | 1.4.1 — Read the alert RRN from `rrn` before `id` and drop anything not RRN-shaped: the investigation alert list also returns bare UUIDs, and one of those rejects the entire /alerts/ops/rrns batch (HTTP 400), losing every alert in the run. | 1.4.0 — Add get_investigations_full: the investigation with its alerts inlined, so it stays the unit that becomes an incident. Drop-in for list_investigations (results_path data, dedup rrn). | 1.3.0 — Add search_investigation_alerts: ingest only the alerts that belong to an investigation, with their full detection content, in 1+N+1 calls. Filtering search_alerts on investigation_rrn does not work — the term is accepted and silently matches nothing. | 1.2.1 — search_alerts: report a malformed terms_json with the input named and echoed, instead of json's bare \"Expecting value: line 1 column 1\". | 1.2.0 — Add the search_alerts OCSF mapper: rule_keys_of_interest carries the host, the process, its command line, its parent and the file hashes, so an alert-based feed indexes entities where the investigation feed indexed none. Validated against a live ABA alert. | 1.1.1 — list_alert_fields: default path_depth to 1 and expose the search filter. The V2 fields endpoint refuses an unbounded walk of the tree (\"root path with path depth greater than 1 is not supported for V2 without a search filter\"), so the bare call returned HTTP 400. | 1.1.0 — Add the Alert Triage API (/idr/at): search alerts, get an alert, its evidence, its actors and its process trees, plus the tenant's alert field catalogue. These carry the detection content the investigation endpoints do not — MITRE T-codes, matched rule keys, assets and accounts, command lines. MDR-gated. | 1.0.0 — Initial release: investigation ingestion (list_investigations) with an exhaustive OCSF mapper, 18 commands across investigations, alerts, custom threats, logs/LEQL search and users. API v1/v2 selectable per instance and per command." +changelog: "1.6.0 — Surface region_failures: the Alert Triage API answers HTTP 200 with a partial result when it cannot reach a region, and that list was being dropped. Also report the regions seen in the RRNs against the configured one. | 1.5.0 — Flatten the whole of rule_keys_of_interest into the context. It is an open vocabulary — a process rule and an authentication rule share no keys — so the hardcoded list only covered rules already seen and left every other alert empty. | 1.4.1 — Read the alert RRN from `rrn` before `id` and drop anything not RRN-shaped: the investigation alert list also returns bare UUIDs, and one of those rejects the entire /alerts/ops/rrns batch (HTTP 400), losing every alert in the run. | 1.4.0 — Add get_investigations_full: the investigation with its alerts inlined, so it stays the unit that becomes an incident. Drop-in for list_investigations (results_path data, dedup rrn). | 1.3.0 — Add search_investigation_alerts: ingest only the alerts that belong to an investigation, with their full detection content, in 1+N+1 calls. Filtering search_alerts on investigation_rrn does not work — the term is accepted and silently matches nothing. | 1.2.1 — search_alerts: report a malformed terms_json with the input named and echoed, instead of json's bare \"Expecting value: line 1 column 1\". | 1.2.0 — Add the search_alerts OCSF mapper: rule_keys_of_interest carries the host, the process, its command line, its parent and the file hashes, so an alert-based feed indexes entities where the investigation feed indexed none. Validated against a live ABA alert. | 1.1.1 — list_alert_fields: default path_depth to 1 and expose the search filter. The V2 fields endpoint refuses an unbounded walk of the tree (\"root path with path depth greater than 1 is not supported for V2 without a search filter\"), so the bare call returned HTTP 400. | 1.1.0 — Add the Alert Triage API (/idr/at): search alerts, get an alert, its evidence, its actors and its process trees, plus the tenant's alert field catalogue. These carry the detection content the investigation endpoints do not — MITRE T-codes, matched rule keys, assets and accounts, command lines. MDR-gated. | 1.0.0 — Initial release: investigation ingestion (list_investigations) with an exhaustive OCSF mapper, 18 commands across investigations, alerts, custom threats, logs/LEQL search and users. API v1/v2 selectable per instance and per command." category: siem # Per-instance configuration. The Insight platform region selects the API host diff --git a/integrations/rapid7-insightidr/scripts/get_investigations_full.py b/integrations/rapid7-insightidr/scripts/get_investigations_full.py index e18d7cf..ab6171f 100644 --- a/integrations/rapid7-insightidr/scripts/get_investigations_full.py +++ b/integrations/rapid7-insightidr/scripts/get_investigations_full.py @@ -14,6 +14,10 @@ def _cfg(): return base, headers, multi +def _cfg_region(): + return str(json.loads(os.environ.get("INTEGRATION_SECRETS", "{}")).get("region") or "us").strip().lower() + + def _inputs(): return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}")) @@ -82,6 +86,32 @@ def _alert_rrn(a): return None +def _rrn_region(rrn): + """The region segment of an RRN, or None. + + An RRN is rrn:::::… — so a resource states which + region it lives in, while the base URL is built from the instance's + configured region. Those two agree on a single-region tenant and need not on + a multi-customer key, which spans organizations. + """ + parts = str(rrn or "").split(":") + return parts[2] if len(parts) > 4 and parts[0] == "rrn" and parts[2] else None + + +def _collect_region_failures(out, sink): + """Carry a partial failure out of a response instead of dropping it. + + The Alert Triage API answers HTTP 200 with a region_failures list when it + could not reach one of the regions behind the request — the spec says the + presence of items there IS the partial failure. Reading only the payload + turns that into missing alerts with no error anywhere, which for an + ingestion feed is indistinguishable from a quiet period. + """ + for f in (out.get("region_failures") or []): + sink.append(f) + return out + + def run(): _, _, multi = _cfg() inp = _inputs() @@ -114,6 +144,7 @@ def run(): # output with an alerts_error: losing the whole run over one of them, or # dropping it silently, would both be worse. rrns_by_inv, order, truncated, skipped = {}, [], False, 0 + region_failures, regions_seen = [], set() for inv in investigations: inv_rrn = inv.get("rrn") or "" if not inv_rrn: @@ -134,6 +165,9 @@ def run(): skipped += 1 continue got.append(rrn) + r = _rrn_region(rrn) + if r: + regions_seen.add(r) if rrn not in order: order.append(rrn) rrns_by_inv[inv_rrn] = got @@ -146,7 +180,7 @@ def run(): body = {"rrns": order[i:i + RRN_BATCH]} if field_ids: body["field_ids"] = field_ids - out = request("POST", AT + "alerts/ops/rrns", body=body) + out = _collect_region_failures(request("POST", AT + "alerts/ops/rrns", body=body), region_failures) for a in (out.get("alerts") or []): if a.get("rrn"): by_rrn[a["rrn"]] = a @@ -193,6 +227,13 @@ def run(): "alerts_fetched": fetched, "alert_ids_without_rrn": skipped, "truncated": truncated, + # HTTP 200 with a non-empty region_failures is a PARTIAL result: + # alerts from that region are simply absent. Surfaced so a shrinking + # feed has a cause attached to it. + "region_failures": region_failures, + "regions_seen": sorted(regions_seen), + "configured_region": _cfg_region(), + "region_mismatch": sorted(r for r in regions_seen if r != _cfg_region()), "elapsed_seconds": round(time.monotonic() - started, 1), }, })) diff --git a/integrations/rapid7-insightidr/scripts/search_alerts.py b/integrations/rapid7-insightidr/scripts/search_alerts.py index d07c009..80ddc23 100644 --- a/integrations/rapid7-insightidr/scripts/search_alerts.py +++ b/integrations/rapid7-insightidr/scripts/search_alerts.py @@ -79,6 +79,32 @@ def _terms(inp): return [{"field_ids": field_ids, "operator": op, "terms": values}] +def _rrn_region(rrn): + """The region segment of an RRN, or None. + + An RRN is rrn:::::… — so a resource states which + region it lives in, while the base URL is built from the instance's + configured region. Those two agree on a single-region tenant and need not on + a multi-customer key, which spans organizations. + """ + parts = str(rrn or "").split(":") + return parts[2] if len(parts) > 4 and parts[0] == "rrn" and parts[2] else None + + +def _collect_region_failures(out, sink): + """Carry a partial failure out of a response instead of dropping it. + + The Alert Triage API answers HTTP 200 with a region_failures list when it + could not reach one of the regions behind the request — the spec says the + presence of items there IS the partial failure. Reading only the payload + turns that into missing alerts with no error anywhere, which for an + ingestion feed is indistinguishable from a quiet period. + """ + for f in (out.get("region_failures") or []): + sink.append(f) + return out + + def run(): inp = _inputs() limit = int(inp.get("limit") or 50) @@ -108,13 +134,16 @@ def run(): rrns_only = str(inp.get("rrns_only") or "").strip().lower() in ("1", "true", "yes") alerts, rrns, meta = [], [], {} + region_failures = [] index = int(inp.get("index") or 0) while len(alerts) + len(rrns) < limit: size = min(PAGE, limit - len(alerts) - len(rrns)) - out = request("POST", AT + "alerts/ops/search", - params={"index": index, "size": size, - "rrns_only": "true" if rrns_only else None}, - body=body) + out = _collect_region_failures( + request("POST", AT + "alerts/ops/search", + params={"index": index, "size": size, + "rrns_only": "true" if rrns_only else None}, + body=body), + region_failures) meta = out.get("metadata") or {} page_alerts = out.get("alerts") or [] page_rrns = out.get("rrns") or [] @@ -124,6 +153,11 @@ def run(): break index += 1 + if not isinstance(meta, dict): + meta = {} + # A non-empty region_failures means alerts from that region are missing from + # this page, on an otherwise successful response. + meta["region_failures"] = region_failures result = {"metadata": meta} if rrns_only: result["rrns"] = rrns[:limit] diff --git a/integrations/rapid7-insightidr/scripts/search_investigation_alerts.py b/integrations/rapid7-insightidr/scripts/search_investigation_alerts.py index 5485458..bdd569c 100644 --- a/integrations/rapid7-insightidr/scripts/search_investigation_alerts.py +++ b/integrations/rapid7-insightidr/scripts/search_investigation_alerts.py @@ -14,6 +14,10 @@ def _cfg(): return base, headers, multi +def _cfg_region(): + return str(json.loads(os.environ.get("INTEGRATION_SECRETS", "{}")).get("region") or "us").strip().lower() + + def _inputs(): return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}")) @@ -72,6 +76,32 @@ def _alert_rrn(a): return None +def _rrn_region(rrn): + """The region segment of an RRN, or None. + + An RRN is rrn:::::… — so a resource states which + region it lives in, while the base URL is built from the instance's + configured region. Those two agree on a single-region tenant and need not on + a multi-customer key, which spans organizations. + """ + parts = str(rrn or "").split(":") + return parts[2] if len(parts) > 4 and parts[0] == "rrn" and parts[2] else None + + +def _collect_region_failures(out, sink): + """Carry a partial failure out of a response instead of dropping it. + + The Alert Triage API answers HTTP 200 with a region_failures list when it + could not reach one of the regions behind the request — the spec says the + presence of items there IS the partial failure. Reading only the payload + turns that into missing alerts with no error anywhere, which for an + ingestion feed is indistinguishable from a quiet period. + """ + for f in (out.get("region_failures") or []): + sink.append(f) + return out + + def run(): _, _, multi = _cfg() inp = _inputs() @@ -104,6 +134,7 @@ def run(): # 2) Their alert RRNs. One call per investigation — there is no bulk form — # so this is the step the time budget guards. order, by_rrn, truncated, skipped = [], {}, False, 0 + region_failures, regions_seen = [], set() for inv in investigations: if time.monotonic() - started > budget or len(order) >= max_alerts: truncated = True @@ -123,6 +154,9 @@ def run(): if rrn is None: skipped += 1 continue + r = _rrn_region(rrn) + if r: + regions_seen.add(r) if rrn not in by_rrn: order.append(rrn) by_rrn[rrn] = inv @@ -140,7 +174,7 @@ def run(): body = {"rrns": batch} if field_ids: body["field_ids"] = field_ids - out = request("POST", AT + "alerts/ops/rrns", body=body) + out = _collect_region_failures(request("POST", AT + "alerts/ops/rrns", body=body), region_failures) alerts.extend(out.get("alerts") or []) # The investigation each alert came from, carried alongside so a mapper can @@ -161,6 +195,10 @@ def run(): "alerts_returned": len(alerts), "alert_ids_without_rrn": skipped, "truncated": truncated, + "region_failures": region_failures, + "regions_seen": sorted(regions_seen), + "configured_region": _cfg_region(), + "region_mismatch": sorted(r for r in regions_seen if r != _cfg_region()), "elapsed_seconds": round(time.monotonic() - started, 1), }, }))