From 4bd468e67198d041577afc4469d5084c215c10a3 Mon Sep 17 00:00:00 2001 From: Guillaume BOURGEOIS Date: Sun, 11 Oct 2026 22:32:16 +0000 Subject: [PATCH] rapid7-insightidr 1.14.2: query a log set by name; Static widens UBA windows query_log_set accepts log_set_name and resolves the id itself (exact name, or the one set whose name contains it; otherwise refused with the existing names): on a real run the model mangled a 36-character id it copied. Static queries by name, retries a 0-event keyword search case-insensitively, takes the 24 hours before a behaviour alert (they aggregate hours of activity), and always looks for the user's other investigations. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015RhL2gy6Sx6Y3FEsyBE3mZ --- .../rapid7-insightidr/agents/operator.yaml | 34 ++++++++++++------- integrations/rapid7-insightidr/manifest.yaml | 7 ++-- .../scripts/list_investigation_alerts.py | 5 +-- .../scripts/query_log_set.py | 33 +++++++++++++++++- 4 files changed, 60 insertions(+), 19 deletions(-) diff --git a/integrations/rapid7-insightidr/agents/operator.yaml b/integrations/rapid7-insightidr/agents/operator.yaml index 4604e3f..d5ed070 100644 --- a/integrations/rapid7-insightidr/agents/operator.yaml +++ b/integrations/rapid7-insightidr/agents/operator.yaml @@ -38,22 +38,30 @@ instructions: | 3. Alerts with evidence_api true: get_alert_actors, then get_alert_evidence. 4. Alerts with evidence_api false, and any alert whose evidence did not answer: the logs. - - list_log_sets (1 call). Pick by the alert type: + - list_log_sets (1 call) to see the names. Then query_log_set with + log_set_name — the NAME, e.g. "Asset Authentication" — never a copied + log_set_id: a 36-character id gets mangled in the copy. + - Pick by the alert type: * authentication / lateral movement / "new assets" / "Domain - Credentials": the Asset Authentication log set first, then the Active - Directory ones. Never File Access or File Modification for these. - * process / endpoint alerts: the Endpoint Agent log sets. - - query_log_set with a keyword search, no field names: - where(""), e.g. where("jdoe"). Always parentheses, - always quotes; no groupby or calculate. - - Window: from 30 minutes before the alert's first seen time to 30 - minutes after its last seen time (start_time / end_time, ISO-8601). - The two can be inverted in the data: take the earlier as the start. + Credentials": Asset Authentication first. An Active Directory name + shared by several sets is refused as ambiguous: give the full name of + one domain controller's set. + * process / endpoint alerts: the Endpoint Agent log set. + * Never File Access or File Modification for an authentication alert. + - Keyword search, no field names: where(""), e.g. + where("jdoe"). If it returns 0 events, try once more + case-insensitively: where(/jdoe/i). Always parentheses; no groupby or + calculate. + - Window. A behaviour (UBA) alert aggregates activity over hours — its + count of "new assets" keeps growing after it is raised — so take the 24 + hours before its first seen time, up to 30 minutes after its last seen + time. Other alerts: 30 minutes on each side. First and last seen can be + inverted in the data: take the earlier as the start. - "Invalid Query Syntax": report the exact query and error, no variants. - Keep at most 6 calls for logs. - 5. Keep 1 call for search_investigation(actor_user_name=): other - investigations of the same user. It takes start_time / end_time, not - time_range; empty, it covers the last 28 days. + 5. ALWAYS call search_investigation(actor_user_name=) once, whatever + the logs gave: other investigations of the same user. It takes + start_time / end_time, not time_range; empty, it covers 28 days. In your answer, list the hosts and IPs the logs show for the user — that is what an analyst needs from a "new assets" alert — and say which log set and diff --git a/integrations/rapid7-insightidr/manifest.yaml b/integrations/rapid7-insightidr/manifest.yaml index 7d441ae..919235e 100644 --- a/integrations/rapid7-insightidr/manifest.yaml +++ b/integrations/rapid7-insightidr/manifest.yaml @@ -1,6 +1,6 @@ id: rapid7_insightidr name: Rapid7 InsightIDR -version: 1.14.1 +version: 1.14.2 description: "Rapid7 InsightIDR (REST API) — investigation ingestion and full lifecycle (list/get/search/create/update/assign/set-status/bulk-close), investigation alerts and Rapid7 product alerts, investigation comments (add/list), custom threat indicators (add/replace), log management and LEQL log/log-set queries with downloads, and user directory search." changelog: "1.13.2 — query_log and query_log_set: trim the quotes and spaces a pasted log or log set ID drags along, and explain the 404. A trailing quote was URL-encoded into the path and the API answered a bare \"HTTP 404 Not Found\" that named nothing. Also send both time bounds on every run: a scheduled run passes only the watermark, and the first run nothing — the window now ends now and, without a start, opens 24 hours back. | 1.13.1 — test_connection: accept an InsightIDR Essential key. Essential answers the Investigations API with a 403 that names the package, so a valid key failed its test; the test now falls back to Log Search and says which commands that package leaves out — the LEQL feeds (query_log, query_log_set) are its way to ingest detections. | 1.13.0 — Add add_comment and list_comments: the investigation comment thread is where an analyst’s reasoning lives, and a playbook had no way to write into it or read it back. The Comments API is v1-only and addresses its target by RRN — the short investigation ID the other v1 endpoints accept is rejected with a 400 that names nothing, so both commands refuse it up front. | 1.12.0 — Make the LEQL queries ingestible: query_log and query_log_set gain an ingest block, a stable event_id for dedup, a bounded page walk and a detection_name stamp, and they now read start_time as ISO-8601, UNIX seconds or milliseconds alike. A tenant whose basic detection rules only notify (email, Slack) creates no investigation and no alert an API can read, so replaying the rule's own query on a schedule is the only way its detections reach a SOAR. | 1.11.1 — Strip the whitespace out of the comma-separated filter params (statuses, priorities, sources, tags) before the URL. \"HIGH, CRITICAL\" as typed sent \" CRITICAL\" to the API, which rejects the whole call — so a priority filter written naturally fetched nothing. | 1.11.0 —Build one evidence row per alert evidence into custom:rapid7_evidence, for the table field: sortable and filterable where the Markdown summary only reads. | 1.10.0 — Render every detection key as Markdown into custom:rapid7_summary, so a layout can show what the OCSF catalogue has no field for. Map both ends of an account action, read source_json top-level scalars, drop placeholder values, and force evidences to stay an array. | 1.9.2 — Read the evidence data in both of its shapes. A parsed source states its event flat, with no source_data/custom_data wrapper, and the parser returned {} for all of them. | 1.9.1 — get_alert_process_trees: explain the 404, which the API returns with an empty body for three different situations. | 1.9.0 — Fetch each alert process tree and flatten it into ctx.process_tree, the shape the layout Process Tree block reads. Gated on the event type by default, since an authentication alert never has one. | 1.8.0 — Fetch each alert evidence at ingestion and parse it: the log line behind the alert lives in a JSON-encoded string no mapper can reach, and it is where the source and destination addresses, the account and the device actually are. | 1.7.0 — Map source/destination assets and accounts to src_endpoint, dst_endpoint, actor.user and user. A connection rule names both ends and neither is \"the host\", so an RDP-to-DC alert produced no entity at all and nothing a layout could show. | 1.6.0 — Surface region_failures: the Alert Triage API answers HTTP 200 with a partial result when it cannot reach a region, and that list was being dropped. Also report the regions seen in the RRNs against the configured one. | 1.5.0 — Flatten the whole of rule_keys_of_interest into the context. It is an open vocabulary — a process rule and an authentication rule share no keys — so the hardcoded list only covered rules already seen and left every other alert empty. | 1.4.1 — Read the alert RRN from `rrn` before `id` and drop anything not RRN-shaped: the investigation alert list also returns bare UUIDs, and one of those rejects the entire /alerts/ops/rrns batch (HTTP 400), losing every alert in the run. | 1.4.0 — Add get_investigations_full: the investigation with its alerts inlined, so it stays the unit that becomes an incident. Drop-in for list_investigations (results_path data, dedup rrn). | 1.3.0 — Add search_investigation_alerts: ingest only the alerts that belong to an investigation, with their full detection content, in 1+N+1 calls. Filtering search_alerts on investigation_rrn does not work — the term is accepted and silently matches nothing. | 1.2.1 — search_alerts: report a malformed terms_json with the input named and echoed, instead of json's bare \"Expecting value: line 1 column 1\". | 1.2.0 — Add the search_alerts OCSF mapper: rule_keys_of_interest carries the host, the process, its command line, its parent and the file hashes, so an alert-based feed indexes entities where the investigation feed indexed none. Validated against a live ABA alert. | 1.1.1 — list_alert_fields: default path_depth to 1 and expose the search filter. The V2 fields endpoint refuses an unbounded walk of the tree (\"root path with path depth greater than 1 is not supported for V2 without a search filter\"), so the bare call returned HTTP 400. | 1.1.0 — Add the Alert Triage API (/idr/at): search alerts, get an alert, its evidence, its actors and its process trees, plus the tenant's alert field catalogue. These carry the detection content the investigation endpoints do not — MITRE T-codes, matched rule keys, assets and accounts, command lines. MDR-gated. | 1.0.0 — Initial release: investigation ingestion (list_investigations) with an exhaustive OCSF mapper, 18 commands across investigations, alerts, custom threats, logs/LEQL search and users. API v1/v2 selectable per instance and per command." category: siem @@ -303,7 +303,8 @@ commands: risk: read inputs_schema: properties: - log_set_id: { type: string, description: "Log set ID to query" } + log_set_id: { type: string, description: "Log set ID to query (from list_log_sets). Prefer log_set_name when an agent calls this." } + log_set_name: { type: string, description: "Log set NAME to query instead of its ID, e.g. Asset Authentication: exact match ignoring case, or the one log set whose name contains it. An unknown or ambiguous name is refused with the list of existing names." } query: { type: string, description: "Valid LEQL query (no calculations: groupby/calculate are rejected by this endpoint)" } time_range: { type: string, description: "Relative time range (e.g. 1 week, 1 day). When set, start_time/end_time are not needed." } start_time: { type: string, description: "Lower bound: UNIX milliseconds, UNIX seconds or ISO-8601 — all three are accepted. Incremental fetch watermark." } @@ -312,7 +313,7 @@ commands: sequence_number: { type: string, description: "Earliest sequence number of a log entry to start from" } limit: { type: number, description: "Maximum events to gather across pages (default 1000). Bounds an unattended run against a query that matches more than it was meant to." } detection_name: { type: string, description: "Name of the detection rule this query stands for. Stamped on every event and used as the incident title — a log line does not know which rule it was replayed for." } - required: [log_set_id, query] + required: [query] outputs_schema: { properties: {} } ingest: results_path: events diff --git a/integrations/rapid7-insightidr/scripts/list_investigation_alerts.py b/integrations/rapid7-insightidr/scripts/list_investigation_alerts.py index 170de29..ce373c3 100644 --- a/integrations/rapid7-insightidr/scripts/list_investigation_alerts.py +++ b/integrations/rapid7-insightidr/scripts/list_investigation_alerts.py @@ -65,8 +65,9 @@ def _annotate(a): else: a["next"] = ("Behaviour (UBA) alert with no RRN: get_alert, get_alert_actors, get_alert_evidence and " "get_alert_process_trees answer 404 for it — do not call them. Its evidence is in the " - "authentication logs: list_log_sets, then query_log_set on the Asset Authentication log set " - "(or each Active Directory one) with where(\"\") over the alert's window.") + "authentication logs: query_log_set with log_set_name \"Asset Authentication\" and " + "where(\"\"), over the 24 hours before the alert's first seen time — a behaviour " + "alert aggregates hours of activity.") try: diff --git a/integrations/rapid7-insightidr/scripts/query_log_set.py b/integrations/rapid7-insightidr/scripts/query_log_set.py index f66caea..ea2655d 100644 --- a/integrations/rapid7-insightidr/scripts/query_log_set.py +++ b/integrations/rapid7-insightidr/scripts/query_log_set.py @@ -125,6 +125,31 @@ def _identify(events, fallback_log_id, detection_name): return out +def _log_sets(): + """Every log set of the account, as (id, name).""" + out = request("GET", "log_search/management/logsets") + sets = out.get("logsets", []) if isinstance(out, dict) else [] + return [(x.get("id", ""), x.get("name", "")) for x in sets if isinstance(x, dict)] + + +def _resolve_name(name): + """The id of the log set called name: exact match first, ignoring case, then + the one log set whose name contains it. Anything else is refused with the + names that do exist, so the next call can pick one rather than guess.""" + sets = _log_sets() + want = name.lower() + exact = [i for i, n in sets if n.lower() == want] + if len(exact) == 1: + return exact[0] + partial = [(i, n) for i, n in sets if want in n.lower()] + if len(partial) == 1: + return partial[0][0] + names = sorted(n for _, n in sets) + if partial: + raise ValueError("log_set_name %r matches several log sets: %s" % (name, ", ".join(sorted(n for _, n in partial)))) + raise ValueError("no log set is called %r. Log sets: %s" % (name, ", ".join(names))) + + def run(): inp = _inputs() start_time = _to_ms(inp.get("start_time")) @@ -152,8 +177,14 @@ def run(): # A pasted ID often drags a quote or a space along; quoted into the path it # names no log set, and the API answers a bare 404. log_set_id = str(inp.get("log_set_id") or "").strip().strip("\"'").strip() + name = str(inp.get("log_set_name") or "").strip() + if name: + # A model copying a 36-character id from one answer into the next call + # drops or swaps characters — seen on a real run, where the "Asset + # Authentication" query went to a mangled id. A name it copies right. + log_set_id = _resolve_name(name) if not log_set_id: - raise ValueError("log_set_id is required (list_log_sets gives it)") + raise ValueError("log_set_id or log_set_name is required (list_log_sets gives both)") first = request("GET", "log_search/query/logsets/" + urllib.parse.quote(log_set_id, safe=""), params=params) # No fallback log id: a log set spans several logs and every event names its own. events = _identify(_collect(first, limit), "", inp.get("detection_name"))