diff --git a/integrations/rapid7-insightidr/manifest.yaml b/integrations/rapid7-insightidr/manifest.yaml index 16cb7fa..a8314e5 100644 --- a/integrations/rapid7-insightidr/manifest.yaml +++ b/integrations/rapid7-insightidr/manifest.yaml @@ -1,8 +1,8 @@ id: rapid7_insightidr name: Rapid7 InsightIDR -version: 1.3.0 +version: 1.4.0 description: "Rapid7 InsightIDR (REST API) — investigation ingestion and full lifecycle (list/get/search/create/update/assign/set-status/bulk-close), investigation alerts and Rapid7 product alerts, custom threat indicators (add/replace), log management and LEQL log/log-set queries with downloads, and user directory search." -changelog: "1.3.0 — Add search_investigation_alerts: ingest only the alerts that belong to an investigation, with their full detection content, in 1+N+1 calls. Filtering search_alerts on investigation_rrn does not work — the term is accepted and silently matches nothing. | 1.2.1 — search_alerts: report a malformed terms_json with the input named and echoed, instead of json's bare \"Expecting value: line 1 column 1\". | 1.2.0 — Add the search_alerts OCSF mapper: rule_keys_of_interest carries the host, the process, its command line, its parent and the file hashes, so an alert-based feed indexes entities where the investigation feed indexed none. Validated against a live ABA alert. | 1.1.1 — list_alert_fields: default path_depth to 1 and expose the search filter. The V2 fields endpoint refuses an unbounded walk of the tree (\"root path with path depth greater than 1 is not supported for V2 without a search filter\"), so the bare call returned HTTP 400. | 1.1.0 — Add the Alert Triage API (/idr/at): search alerts, get an alert, its evidence, its actors and its process trees, plus the tenant's alert field catalogue. These carry the detection content the investigation endpoints do not — MITRE T-codes, matched rule keys, assets and accounts, command lines. MDR-gated. | 1.0.0 — Initial release: investigation ingestion (list_investigations) with an exhaustive OCSF mapper, 18 commands across investigations, alerts, custom threats, logs/LEQL search and users. API v1/v2 selectable per instance and per command." +changelog: "1.4.0 — Add get_investigations_full: the investigation with its alerts inlined, so it stays the unit that becomes an incident. Drop-in for list_investigations (results_path data, dedup rrn). | 1.3.0 — Add search_investigation_alerts: ingest only the alerts that belong to an investigation, with their full detection content, in 1+N+1 calls. Filtering search_alerts on investigation_rrn does not work — the term is accepted and silently matches nothing. | 1.2.1 — search_alerts: report a malformed terms_json with the input named and echoed, instead of json's bare \"Expecting value: line 1 column 1\". | 1.2.0 — Add the search_alerts OCSF mapper: rule_keys_of_interest carries the host, the process, its command line, its parent and the file hashes, so an alert-based feed indexes entities where the investigation feed indexed none. Validated against a live ABA alert. | 1.1.1 — list_alert_fields: default path_depth to 1 and expose the search filter. The V2 fields endpoint refuses an unbounded walk of the tree (\"root path with path depth greater than 1 is not supported for V2 without a search filter\"), so the bare call returned HTTP 400. | 1.1.0 — Add the Alert Triage API (/idr/at): search alerts, get an alert, its evidence, its actors and its process trees, plus the tenant's alert field catalogue. These carry the detection content the investigation endpoints do not — MITRE T-codes, matched rule keys, assets and accounts, command lines. MDR-gated. | 1.0.0 — Initial release: investigation ingestion (list_investigations) with an exhaustive OCSF mapper, 18 commands across investigations, alerts, custom threats, logs/LEQL search and users. API v1/v2 selectable per instance and per command." category: siem # Per-instance configuration. The Insight platform region selects the API host @@ -439,6 +439,30 @@ commands: dedup_key: rrn incremental_field: start_time + - id: get_investigations_full + name: rapid7-insight-idr-get-investigations-full + description: "Ingestion: investigations with their alerts inlined, so the INVESTIGATION stays the unit that becomes an incident. Walks the investigations in a time window, collects their alert RRNs and retrieves the alerts in batches from the Alert Triage API, then nests them under `alerts` with a few first-alert values lifted to the top level. Results path = data, dedup on rrn, incremental on start_time — a drop-in replacement for list_investigations, which carries no host, process or MITRE code. Use the get_investigations_full mapper." + risk: read + inputs_schema: + properties: + start_time: { type: string, description: "ISO-8601 lower bound on investigation created_time. Incremental fetch watermark." } + end_time: { type: string, description: "ISO-8601 upper bound on investigation created_time" } + default_days: { type: string, description: "Window size in days used when start_time is absent (default 1)" } + statuses: { type: string, description: "Comma-separated investigation statuses to include (open, investigating, closed). Leaving out closed skips what the source has already triaged." } + priorities: { type: string, description: "Comma-separated investigation priorities to include (Unspecified, Low, Medium, High, Critical)" } + field_ids: { type: string, description: "Comma-separated additional alert field ids to include (see rapid7-insight-idr-list-alert-fields)" } + max_alerts: { type: string, description: "Ceiling on alert RRNs collected in one run (default 500)" } + time_budget_seconds: { type: string, description: "Stop walking investigations after this many seconds and report truncated=true, rather than exceed the server's 90-second command budget and lose the whole run (default 60)" } + index: { type: string, description: "0-based page index" } + page_size: { type: string, description: "Investigation page size" } + limit: { type: number, description: "Maximum number of investigations to return (default 50)" } + required: [] + outputs_schema: { properties: {} } + ingest: + results_path: data + dedup_key: rrn + incremental_field: start_time + # ── Connectivity test ───────────────────────────────────────────────────── - id: test_connection name: rapid7-insight-idr-test-connection diff --git a/integrations/rapid7-insightidr/mappers/get_investigations_full.yaml b/integrations/rapid7-insightidr/mappers/get_investigations_full.yaml new file mode 100644 index 0000000..6aad5c7 --- /dev/null +++ b/integrations/rapid7-insightidr/mappers/get_investigations_full.yaml @@ -0,0 +1,124 @@ +name: "Rapid7 InsightIDR Investigations (full) → OCSF" +description: | + Maps one Rapid7 investigation fetched with its alerts inlined + (get_investigations_full, results_path = data) to OCSF finding fields. + + The plain investigation feed cannot carry a detection: idr/v2/investigations answers + with a title, a priority, a status, an assignee and timestamps, and nothing else. An + incident built from it indexes zero entities. This mapper reads the same investigation + plus the alerts the command inlined under `alerts`, where the detection content + actually lives. + + Investigation values are mapped first and the first alert's equivalents after, so the + alert wins wherever the container says nothing. `alerts` is empty on an investigation + whose alerts could not be read (the command sets alerts_error) — every alert path then + resolves to nothing and is skipped, and the incident degrades to what the investigation + alone says rather than failing. +field_mappings: + title: "title" + # toSeverity maps CRITICAL→5, HIGH→4, MEDIUM→3, LOW→2; anything else falls back to the + # alert rule's default severity. + severity: "priority" + source: "source" + # Rapid7's own triage state. It cannot reach the incident's status and verdict columns + # — MapFields accepts only title, severity, source, description and custom:* — so it + # rides as custom fields, where an analyst at least sees that the source already closed + # this and what it concluded. + "custom:rapid7_status": "status" + "custom:rapid7_disposition": "disposition" + # Rapid7's AI suggestion, exposed as a fact and never as a verdict. + # first_alert_* are lifted to the top level by the command on purpose: this + # mapper section resolves dotted paths only and cannot index an array, so + # `alerts[0].x` here would read as nothing — silently, leaving the custom field + # empty with no error. The ocsf section below is full JSONata and indexes freely. + "custom:rapid7_ai_disposition": "first_alert_ai_disposition" + "custom:rapid7_alert_count": "alert_count" + "custom:rapid7_log_id": "first_alert_log_id" + "custom:rapid7_logset_id": "first_alert_logset_id" + "custom:rapid7_rule_rrn": "first_alert_rule_rrn" + "custom:rapid7_event_type": "first_alert_event_type" +ocsf: + # ── Finding: the investigation ──────────────────────────────────── + - { source_path: "rrn", ocsf_field: "finding_info.uid" } + - { source_path: "title", ocsf_field: "finding_info.title" } + - { source_path: "created_time", ocsf_field: "finding_info.created_time" } + - { source_path: "last_accessed", ocsf_field: "finding_info.modified_time" } + - { source_path: "first_alert_time", ocsf_field: "finding_info.first_seen_time" } + - { source_path: "latest_alert_time", ocsf_field: "finding_info.last_seen_time" } + - { source_path: "tags", ocsf_field: "metadata.labels" } + - { source_path: "alert_count", ocsf_field: "count" } + # ── Incident state ──────────────────────────────────────────────── + - { source_path: "status", ocsf_field: "status" } + - { source_path: "priority", ocsf_field: "priority" } + - { source_path: "disposition", ocsf_field: "disposition" } + - { source_path: "source", ocsf_field: "activity_name" } + - { source_path: "organization_id", ocsf_field: "cloud.org.uid" } + - { source_path: "assignee.name", ocsf_field: "assignee.name" } + - { source_path: "assignee.email", ocsf_field: "assignee.email_addr" } + # ── Detection time: the MTTD anchor ─────────────────────────────── + # Weakest first, strongest last. The investigation is created after the alert that + # opened it; the alert's own alerted_at is when the event happened in the source system. + - { source_path: "created_time", ocsf_field: "time" } + - { source_path: "first_alert_time", ocsf_field: "time" } + - { source_path: "alerts[0].alerted_at", ocsf_field: "time" } + # ── The detection rule, from the first alert ────────────────────── + # `rule.name` is documented but absent from live ABA alerts, so `type` — which repeats + # the rule's display name — is the honest fallback. + - { source_path: "alerts[0].type", ocsf_field: "finding_info.analytic.name" } + - { source_path: "alerts[0].rule.rrn", ocsf_field: "finding_info.analytic.uid" } + - { source_path: "alerts[0].rule.version_rrn", ocsf_field: "finding_info.analytic.version" } + - { source_path: "alerts[0].rrn", ocsf_field: "finding_info.uid_alt" } + # ── Product identity ────────────────────────────────────────────── + - { source_path: "'Rapid7 InsightIDR'", ocsf_field: "metadata.product.name" } + - { source_path: "'Rapid7'", ocsf_field: "metadata.product.vendor_name" } + - { source_path: "alerts[0].external_source", ocsf_field: "metadata.log_source" } + - { source_path: "alerts[0].triggering_event_type", ocsf_field: "metadata.event_code" } + - { source_path: "alerts[0].log_details[0].log_entry_id", ocsf_field: "metadata.uid" } + # ── MITRE ATT&CK ────────────────────────────────────────────────── + # mitre_tcodes mixes both kinds in one array: "TA0040" is a tactic, "T1490" a technique. + # Splitting on the "TA" prefix keeps a tactic out of the technique field, where it would + # name a technique that does not exist. + - { source_path: "alerts[0].rule.mitre_tcodes[$substring($, 0, 2) = 'TA'][0]", ocsf_field: "attacks.tactic.uid" } + - { source_path: "alerts[0].rule.mitre_tcodes[$substring($, 0, 2) != 'TA'][0]", ocsf_field: "attacks.technique.uid" } + # ── The affected host ───────────────────────────────────────────── + - { source_path: "alerts[0].rule_keys_of_interest[key='hostname'].values[0]", ocsf_field: "device.hostname" } + - { source_path: "alerts[0].rule_keys_of_interest[key='hostname'].values[0]", ocsf_field: "src_endpoint.hostname" } + # ── The process the alert is about ──────────────────────────────── + - { source_path: "alerts[0].rule_keys_of_interest[key='process.name'].values[0]", ocsf_field: "process.name" } + - { source_path: "alerts[0].rule_keys_of_interest[key='process.name'].values[0]", ocsf_field: "process.file.name" } + - { source_path: "alerts[0].rule_keys_of_interest[key='process.cmd_line'].values[0]", ocsf_field: "process.cmd_line" } + - { source_path: "alerts[0].rule_keys_of_interest[key='process.exe_path'].values[0]", ocsf_field: "process.file.path" } + - { source_path: "alerts[0].rule_keys_of_interest[key='process.exe_file.hashes.md5'].values[0]", ocsf_field: "process.file.hashes.md5" } + - { source_path: "alerts[0].rule_keys_of_interest[key='process.exe_file.hashes.sha256'].values[0]", ocsf_field: "process.file.hashes.sha256" } + # ── Its parent, which says how it was launched ──────────────────── + - { source_path: "alerts[0].rule_keys_of_interest[key='parent_process.name'].values[0]", ocsf_field: "actor.process.parent_process.name" } + - { source_path: "alerts[0].rule_keys_of_interest[key='parent_process.cmd_line'].values[0]", ocsf_field: "actor.process.parent_process.cmd_line" } + - { source_path: "alerts[0].rule_keys_of_interest[key='parent_process.exe_path'].values[0]", ocsf_field: "actor.process.parent_process.file.path" } + - { source_path: "alerts[0].rule_keys_of_interest[key='parent_process.exe_file.hashes.md5'].values[0]", ocsf_field: "actor.process.parent_process.file.hashes.md5" } + # ── The acting account ──────────────────────────────────────────── + # This tenant's field catalogue has no canonical user namespace, so an account only + # appears when a rule names it among its keys of interest. Both spellings are mapped + # because rules are not consistent about which they emit. + - { source_path: "alerts[0].rule_keys_of_interest[key='user'].values[0]", ocsf_field: "actor.user.name" } + - { source_path: "alerts[0].rule_keys_of_interest[key='username'].values[0]", ocsf_field: "actor.user.name" } + # ── Every alert, not just the first ─────────────────────────────── + # An investigation aggregates alerts, and the fields above only ever read alerts[0]. + # This entry lands the whole set as OCSF-shaped objects, so the host and the process of + # the SECOND and THIRD alert are indexed as entities too — the extractor walks arrays + # and classifies on the path's suffix, so `.hostname` under here counts exactly as + # `device.hostname` does above. + - source_path: > + alerts.{ + "uid": rrn, + "title": title, + "time": alerted_at, + "status": status, + "disposition": disposition, + "device": { "hostname": rule_keys_of_interest[key='hostname'].values[0] }, + "process": { + "name": rule_keys_of_interest[key='process.name'].values[0], + "cmd_line": rule_keys_of_interest[key='process.cmd_line'].values[0] + }, + "attacks": { "technique": { "uid": rule.mitre_tcodes[$substring($, 0, 2) != 'TA'][0] } } + } + ocsf_field: "evidences" diff --git a/integrations/rapid7-insightidr/scripts/get_investigations_full.py b/integrations/rapid7-insightidr/scripts/get_investigations_full.py new file mode 100644 index 0000000..947f2aa --- /dev/null +++ b/integrations/rapid7-insightidr/scripts/get_investigations_full.py @@ -0,0 +1,189 @@ +import json, os, sys, urllib.request, urllib.parse, urllib.error + +# The Alert Triage API lives under /idr/at, a base of its own — not /idr/v1 or +# /idr/v2 like the investigation endpoints. Same host, same API key. +AT = "idr/at/" + + +def _cfg(): + s = json.loads(os.environ.get("INTEGRATION_SECRETS", "{}")) + region = str(s.get("region") or "us").strip().lower() + base = "https://" + region + ".api.insight.rapid7.com/" + headers = {"X-Api-Key": s.get("api_key", ""), "Content-Type": "application/json", "Accept": "application/json"} + multi = str(s.get("is_multi_customer") or "").strip().lower() in ("1", "true", "yes") + return base, headers, multi + + +def _inputs(): + return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}")) + + +def _list(v): + if v in (None, ""): + return [] + if isinstance(v, list): + return [str(x).strip() for x in v if str(x).strip()] + return [p.strip() for p in str(v).split(",") if p.strip()] + + +def request(method, path, params=None, body=None): + base, headers, _ = _cfg() + url = base + path.lstrip("/") + if params: + clean = {k: v for k, v in params.items() if v not in (None, "")} + if clean: + url += ("&" if "?" in url else "?") + urllib.parse.urlencode(clean, doseq=True) + data = json.dumps(body).encode("utf-8") if body is not None else None + req = urllib.request.Request(url, data=data, headers=headers, method=method) + with urllib.request.urlopen(req, timeout=90) as r: + raw = r.read() + return json.loads(raw) if raw else {} + +import time +from datetime import datetime, timedelta, timezone + +ISO = "%Y-%m-%dT%H:%M:%SZ" +RRN_BATCH = 100 +DEFAULT_BUDGET_S = 60 + +# idr/v2/investigations answers with 13 fields, three of them null: a title, a +# priority, a status, an assignee and timestamps. No host, no account, no +# process, no MITRE code — and not one of the alerts the investigation +# aggregates. An incident built from it indexes zero entities, so it cannot be +# pivoted on, cannot be correlated, and gives an analyst nothing to read. +# +# The detection content is two hops away: the investigation's alert list gives +# RRNs, and the Alert Triage API turns those into alerts carrying +# rule_keys_of_interest (the host, the process, its command line, its parent, +# the hashes) and rule.mitre_tcodes. This command walks both hops and returns +# the investigation with its alerts inlined, so the INVESTIGATION stays the unit +# that becomes an incident — results_path = data, dedup on rrn, exactly as the +# plain investigation feed. Only the command changes. + + +def run(): + _, _, multi = _cfg() + inp = _inputs() + limit = int(inp.get("limit") or 50) + max_alerts = int(inp.get("max_alerts") or 500) + budget = float(inp.get("time_budget_seconds") or DEFAULT_BUDGET_S) + started = time.monotonic() + + start_time = inp.get("start_time") + if not start_time: + days = int(inp.get("default_days") or 1) + start_time = (datetime.now(timezone.utc) - timedelta(days=days)).strftime(ISO) + + inv_page = request("GET", "idr/v2/investigations", params={ + "index": inp.get("index") or "0", + "size": inp.get("page_size") or limit, + "statuses": inp.get("statuses"), + "priorities": inp.get("priorities"), + "start_time": start_time, + "end_time": inp.get("end_time"), + "sort_field": "Created time", + "sort_direction": "ASC", + "multi-customer": "true" if multi else None, + }) + investigations = (inv_page.get("data") or [])[:limit] + + # Alert RRNs per investigation. One call each — there is no bulk form — so + # this is the step that grows with the window, and the one the time budget + # guards. An investigation whose alerts cannot be read keeps its place in the + # output with an alerts_error: losing the whole run over one of them, or + # dropping it silently, would both be worse. + rrns_by_inv, order, truncated = {}, [], False + for inv in investigations: + inv_rrn = inv.get("rrn") or "" + if not inv_rrn: + continue + if time.monotonic() - started > budget or len(order) >= max_alerts: + truncated = True + break + try: + page = request("GET", "idr/v2/investigations/" + urllib.parse.quote(inv_rrn, safe="") + "/alerts", + params={"multi-customer": "true" if multi else None}) + except urllib.error.HTTPError as e: + inv["alerts_error"] = "HTTP " + str(e.code) + continue + got = [] + for a in (page.get("data") or []): + rrn = a.get("id") or a.get("rrn") + if rrn: + got.append(rrn) + if rrn not in order: + order.append(rrn) + rrns_by_inv[inv_rrn] = got + + # The alerts themselves, batched, then indexed so each investigation can be + # handed its own. + field_ids = _list(inp.get("field_ids")) + by_rrn = {} + for i in range(0, len(order), RRN_BATCH): + body = {"rrns": order[i:i + RRN_BATCH]} + if field_ids: + body["field_ids"] = field_ids + out = request("POST", AT + "alerts/ops/rrns", body=body) + for a in (out.get("alerts") or []): + if a.get("rrn"): + by_rrn[a["rrn"]] = a + + fetched = 0 + for inv in investigations: + rrns = rrns_by_inv.get(inv.get("rrn") or "", []) + alerts = [by_rrn[r] for r in rrns if r in by_rrn] + inv["alerts"] = alerts + inv["alert_count"] = len(rrns) + # Say it when the alert list is a sample: alert_count is what the + # investigation actually has, len(alerts) what this run retrieved. + if len(alerts) < len(rrns): + inv["alerts_truncated"] = True + fetched += len(alerts) + # Lift a few first-alert values to the top level. The incident-field + # mapper resolves dotted paths only — it cannot index an array — so + # `alerts[0].ai_suggested_disposition` in a field_mappings entry reads as + # nothing, silently, and the custom field arrives empty with no error + # anywhere. The OCSF entries are evaluated by full JSONata and do not + # need this; only field_mappings does. + if alerts: + a = alerts[0] + log = (a.get("log_details") or [{}])[0] + inv.update({ + "first_alert_rrn": a.get("rrn"), + "first_alert_type": a.get("type"), + "first_alert_source": a.get("external_source"), + "first_alert_status": a.get("status"), + "first_alert_disposition": a.get("disposition"), + "first_alert_ai_disposition": a.get("ai_suggested_disposition"), + "first_alert_event_type": a.get("triggering_event_type"), + "first_alert_rule_rrn": (a.get("rule") or {}).get("rrn"), + "first_alert_log_id": log.get("log_id"), + "first_alert_logset_id": log.get("logset_id"), + "first_alert_log_entry_id": log.get("log_entry_id"), + }) + + print(json.dumps({ + "data": investigations, + "metadata": { + "investigations_returned": len(investigations), + "alert_rrns_collected": len(order), + "alerts_fetched": fetched, + "truncated": truncated, + "elapsed_seconds": round(time.monotonic() - started, 1), + }, + })) + + +try: + run() +except urllib.error.HTTPError as e: + detail = e.read().decode("utf-8", "replace") + if e.code in (401, 403): + detail += (" — the Alert Triage API (/idr/at) is only available to Managed Detection and" + " Response customers with access to the Alerts experience; a key that works on" + " the investigation endpoints can still be refused here") + print(json.dumps({"error": "HTTP " + str(e.code), "detail": detail})) + sys.exit(1) +except Exception as e: + print(json.dumps({"error": str(e)})) + sys.exit(1)