diff --git a/integrations/rapid7-insightidr/manifest.yaml b/integrations/rapid7-insightidr/manifest.yaml index 33a692b..16cb7fa 100644 --- a/integrations/rapid7-insightidr/manifest.yaml +++ b/integrations/rapid7-insightidr/manifest.yaml @@ -1,8 +1,8 @@ id: rapid7_insightidr name: Rapid7 InsightIDR -version: 1.2.1 +version: 1.3.0 description: "Rapid7 InsightIDR (REST API) — investigation ingestion and full lifecycle (list/get/search/create/update/assign/set-status/bulk-close), investigation alerts and Rapid7 product alerts, custom threat indicators (add/replace), log management and LEQL log/log-set queries with downloads, and user directory search." -changelog: "1.2.1 — search_alerts: report a malformed terms_json with the input named and echoed, instead of json's bare \"Expecting value: line 1 column 1\". | 1.2.0 — Add the search_alerts OCSF mapper: rule_keys_of_interest carries the host, the process, its command line, its parent and the file hashes, so an alert-based feed indexes entities where the investigation feed indexed none. Validated against a live ABA alert. | 1.1.1 — list_alert_fields: default path_depth to 1 and expose the search filter. The V2 fields endpoint refuses an unbounded walk of the tree (\"root path with path depth greater than 1 is not supported for V2 without a search filter\"), so the bare call returned HTTP 400. | 1.1.0 — Add the Alert Triage API (/idr/at): search alerts, get an alert, its evidence, its actors and its process trees, plus the tenant's alert field catalogue. These carry the detection content the investigation endpoints do not — MITRE T-codes, matched rule keys, assets and accounts, command lines. MDR-gated. | 1.0.0 — Initial release: investigation ingestion (list_investigations) with an exhaustive OCSF mapper, 18 commands across investigations, alerts, custom threats, logs/LEQL search and users. API v1/v2 selectable per instance and per command." +changelog: "1.3.0 — Add search_investigation_alerts: ingest only the alerts that belong to an investigation, with their full detection content, in 1+N+1 calls. Filtering search_alerts on investigation_rrn does not work — the term is accepted and silently matches nothing. | 1.2.1 — search_alerts: report a malformed terms_json with the input named and echoed, instead of json's bare \"Expecting value: line 1 column 1\". | 1.2.0 — Add the search_alerts OCSF mapper: rule_keys_of_interest carries the host, the process, its command line, its parent and the file hashes, so an alert-based feed indexes entities where the investigation feed indexed none. Validated against a live ABA alert. | 1.1.1 — list_alert_fields: default path_depth to 1 and expose the search filter. The V2 fields endpoint refuses an unbounded walk of the tree (\"root path with path depth greater than 1 is not supported for V2 without a search filter\"), so the bare call returned HTTP 400. | 1.1.0 — Add the Alert Triage API (/idr/at): search alerts, get an alert, its evidence, its actors and its process trees, plus the tenant's alert field catalogue. These carry the detection content the investigation endpoints do not — MITRE T-codes, matched rule keys, assets and accounts, command lines. MDR-gated. | 1.0.0 — Initial release: investigation ingestion (list_investigations) with an exhaustive OCSF mapper, 18 commands across investigations, alerts, custom threats, logs/LEQL search and users. API v1/v2 selectable per instance and per command." category: siem # Per-instance configuration. The Insight platform region selects the API host @@ -414,6 +414,31 @@ commands: required: [] outputs_schema: { properties: {} } + - id: search_investigation_alerts + name: rapid7-insight-idr-search-investigation-alerts + description: "Ingestion: the alerts that belong to an investigation, with their full detection content. Walks the investigations in a time window, collects their alert RRNs and retrieves the alerts in batches from the Alert Triage API — so an alert Rapid7 never folded into an investigation is never returned. Results path = alerts, dedup on rrn, incremental on start_time (the investigation's creation time). Use the search_alerts mapper: the alert objects are identical, with the parent investigation carried alongside under `investigation`." + risk: read + inputs_schema: + properties: + start_time: { type: string, description: "ISO-8601 lower bound on investigation created_time. Incremental fetch watermark." } + end_time: { type: string, description: "ISO-8601 upper bound on investigation created_time" } + default_days: { type: string, description: "Window size in days used when start_time is absent (default 1)" } + statuses: { type: string, description: "Comma-separated investigation statuses to include (open, investigating, closed). Leaving out closed skips what the source has already triaged." } + priorities: { type: string, description: "Comma-separated investigation priorities to include (Unspecified, Low, Medium, High, Critical)" } + field_ids: { type: string, description: "Comma-separated additional alert field ids to include (see rapid7-insight-idr-list-alert-fields)" } + include_investigation: { type: string, description: "Carry the parent investigation on each alert under `investigation` (true/false, default true)" } + max_alerts: { type: string, description: "Ceiling on alert RRNs collected in one run (default 500)" } + time_budget_seconds: { type: string, description: "Stop walking investigations after this many seconds and report truncated=true, rather than exceed the server's 90-second command budget and lose the whole run (default 60)" } + index: { type: string, description: "0-based page index of investigations" } + page_size: { type: string, description: "Investigation page size" } + limit: { type: number, description: "Maximum number of investigations to walk (default 50)" } + required: [] + outputs_schema: { properties: {} } + ingest: + results_path: alerts + dedup_key: rrn + incremental_field: start_time + # ── Connectivity test ───────────────────────────────────────────────────── - id: test_connection name: rapid7-insight-idr-test-connection diff --git a/integrations/rapid7-insightidr/mappers/search_alerts.yaml b/integrations/rapid7-insightidr/mappers/search_alerts.yaml index c7a1ae4..7699eff 100644 --- a/integrations/rapid7-insightidr/mappers/search_alerts.yaml +++ b/integrations/rapid7-insightidr/mappers/search_alerts.yaml @@ -1,7 +1,9 @@ name: "Rapid7 InsightIDR Alerts → OCSF" description: | - Maps one Rapid7 alert from the Alert Triage API (search_alerts, results_path = alerts) - to OCSF finding fields. + Maps one Rapid7 alert from the Alert Triage API to OCSF finding fields. Serves both + search_alerts and search_investigation_alerts (results_path = alerts on either): the + alert objects are identical, and the second command simply carries the parent + investigation alongside under `investigation`. This mapper exists because the investigation feed cannot carry a detection. An investigation, and the /investigations/{rrn}/alerts list behind it, hold a title, a @@ -113,3 +115,12 @@ ocsf: # Not investigable on its own, but it is the handle a LEQL query needs to pull the raw # event out of Log Search — the only bridge from an alert back to the line that made it. - { source_path: "log_details[0].log_entry_id", ocsf_field: "metadata.uid" } + # ── The parent investigation ────────────────────────────────────── + # Present only on the search_investigation_alerts feed, which carries the container + # beside the alert. Absent on search_alerts, where these resolve to nothing and are + # skipped — the same file serves both. + - { source_path: "investigation.title", ocsf_field: "finding_info.related_events.title" } + - { source_path: "investigation.status", ocsf_field: "finding_info.related_events.status" } + - { source_path: "investigation.priority", ocsf_field: "finding_info.related_events.priority" } + - { source_path: "investigation.disposition", ocsf_field: "finding_info.related_events.disposition" } + - { source_path: "investigation.rrn", ocsf_field: "finding_info.related_events.uid" } diff --git a/integrations/rapid7-insightidr/scripts/search_investigation_alerts.py b/integrations/rapid7-insightidr/scripts/search_investigation_alerts.py new file mode 100644 index 0000000..58c3e42 --- /dev/null +++ b/integrations/rapid7-insightidr/scripts/search_investigation_alerts.py @@ -0,0 +1,156 @@ +import json, os, sys, urllib.request, urllib.parse, urllib.error + +# The Alert Triage API lives under /idr/at, a base of its own — not /idr/v1 or +# /idr/v2 like the investigation endpoints. Same host, same API key. +AT = "idr/at/" + + +def _cfg(): + s = json.loads(os.environ.get("INTEGRATION_SECRETS", "{}")) + region = str(s.get("region") or "us").strip().lower() + base = "https://" + region + ".api.insight.rapid7.com/" + headers = {"X-Api-Key": s.get("api_key", ""), "Content-Type": "application/json", "Accept": "application/json"} + multi = str(s.get("is_multi_customer") or "").strip().lower() in ("1", "true", "yes") + return base, headers, multi + + +def _inputs(): + return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}")) + + +def _list(v): + if v in (None, ""): + return [] + if isinstance(v, list): + return [str(x).strip() for x in v if str(x).strip()] + return [p.strip() for p in str(v).split(",") if p.strip()] + + +def request(method, path, params=None, body=None): + base, headers, _ = _cfg() + url = base + path.lstrip("/") + if params: + clean = {k: v for k, v in params.items() if v not in (None, "")} + if clean: + url += ("&" if "?" in url else "?") + urllib.parse.urlencode(clean, doseq=True) + data = json.dumps(body).encode("utf-8") if body is not None else None + req = urllib.request.Request(url, data=data, headers=headers, method=method) + with urllib.request.urlopen(req, timeout=90) as r: + raw = r.read() + return json.loads(raw) if raw else {} + +import time +from datetime import datetime, timedelta, timezone + +ISO = "%Y-%m-%dT%H:%M:%SZ" +# /alerts/ops/rrns takes a list; batching keeps one URL and one body reasonable. +RRN_BATCH = 100 +# The server gives a command 90 seconds. Stop walking investigations well before +# that and report the truncation, rather than have the whole run killed and every +# alert in the window lost — a partial fetch that says so beats a silent zero. +DEFAULT_BUDGET_S = 60 + + +def run(): + _, _, multi = _cfg() + inp = _inputs() + limit = int(inp.get("limit") or 50) + max_alerts = int(inp.get("max_alerts") or 500) + budget = float(inp.get("time_budget_seconds") or DEFAULT_BUDGET_S) + started = time.monotonic() + + start_time = inp.get("start_time") + if not start_time: + days = int(inp.get("default_days") or 1) + start_time = (datetime.now(timezone.utc) - timedelta(days=days)).strftime(ISO) + + # 1) The investigations in the window. This is the unit that decides what is + # worth ingesting: an alert Rapid7 never folded into one is not returned by + # this command at all, which is the whole point of it. + inv_page = request("GET", "idr/v2/investigations", params={ + "index": inp.get("index") or "0", + "size": inp.get("page_size") or limit, + "statuses": inp.get("statuses"), + "priorities": inp.get("priorities"), + "start_time": start_time, + "end_time": inp.get("end_time"), + "sort_field": "Created time", + "sort_direction": "ASC", + "multi-customer": "true" if multi else None, + }) + investigations = (inv_page.get("data") or [])[:limit] + + # 2) Their alert RRNs. One call per investigation — there is no bulk form — + # so this is the step the time budget guards. + order, by_rrn, truncated = [], {}, False + for inv in investigations: + if time.monotonic() - started > budget or len(order) >= max_alerts: + truncated = True + break + inv_rrn = inv.get("rrn") or "" + if not inv_rrn: + continue + try: + page = request("GET", "idr/v2/investigations/" + urllib.parse.quote(inv_rrn, safe="") + "/alerts", + params={"multi-customer": "true" if multi else None}) + except urllib.error.HTTPError: + # One unreadable investigation must not cost the whole window. The + # alert simply does not appear; the counts below say how many did. + continue + for a in (page.get("data") or []): + rrn = a.get("id") or a.get("rrn") + if rrn and rrn not in by_rrn: + order.append(rrn) + by_rrn[rrn] = inv + if len(order) >= max_alerts: + truncated = True + break + + # 3) The alerts themselves, in batches. This is where the detection content + # lives — rule_keys_of_interest, mitre_tcodes, disposition — none of which + # the investigation or its alert list carries. + field_ids = _list(inp.get("field_ids")) + alerts = [] + for i in range(0, len(order), RRN_BATCH): + batch = order[i:i + RRN_BATCH] + body = {"rrns": batch} + if field_ids: + body["field_ids"] = field_ids + out = request("POST", AT + "alerts/ops/rrns", body=body) + alerts.extend(out.get("alerts") or []) + + # The investigation each alert came from, carried alongside so a mapper can + # read the container's own title, priority and disposition without a second + # feed. Alert fields are left untouched: `investigation_rrn` already exists + # on the alert and nothing here may shadow it. + if str(inp.get("include_investigation") or "true").strip().lower() in ("1", "true", "yes"): + for a in alerts: + inv = by_rrn.get(a.get("rrn")) + if inv: + a["investigation"] = inv + + print(json.dumps({ + "alerts": alerts, + "metadata": { + "investigations_read": len(investigations), + "alert_rrns_collected": len(order), + "alerts_returned": len(alerts), + "truncated": truncated, + "elapsed_seconds": round(time.monotonic() - started, 1), + }, + })) + + +try: + run() +except urllib.error.HTTPError as e: + detail = e.read().decode("utf-8", "replace") + if e.code in (401, 403): + detail += (" — the Alert Triage API (/idr/at) is only available to Managed Detection and" + " Response customers with access to the Alerts experience; a key that works on" + " the investigation endpoints can still be refused here") + print(json.dumps({"error": "HTTP " + str(e.code), "detail": detail})) + sys.exit(1) +except Exception as e: + print(json.dumps({"error": str(e)})) + sys.exit(1)