rapid7-insightidr 1.14.5: query_log_set can summarize; Static always does

summarize=true returns, over every matching event, each field's most frequent
values with counts, the first and last event time and five samples, instead of
the raw events. On a real run Static got 1000 raw events, could read about
ten, and reported the hosts and accounts of those ten; 300 events now fit in
about 1.5 KB. Static passes summarize always and quotes the returned window.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015RhL2gy6Sx6Y3FEsyBE3mZ

integrations/rapid7-insightidr/agents/operator.yaml

@@ -48,6 +48,10 @@ instructions: |
          one domain controller's set.
        * process / endpoint alerts: the Endpoint Agent log set.
        * Never File Access or File Modification for an authentication alert.
+     - ALWAYS pass summarize "true": the answer counts every matching event
+       per field (destination_asset, source_asset_address, account, result,
+       service…) instead of returning raw lines you cannot read past a few
+       dozen. Report the counts and say when "truncated" is true.
      - Keyword search, no field names: where("<user, host or IP>"), e.g.
        where("jdoe"). If it returns 0 events, try once more
        case-insensitively: where(/jdoe/i). Always parentheses; no groupby or
@@ -56,7 +60,8 @@ instructions: |
        first seen time exactly as the data gives it, and the window is
        computed for you. A behaviour (UBA) alert aggregates hours of activity:
        hours_before 24, minutes_after 30 (the defaults). Other alerts:
-       hours_before 0.5. Report the "window" the answer returns, not your own.
+       hours_before 0.5. Report the "window" field of the answer word for
+       word, never your own reading of it.
      - "Invalid Query Syntax": report the exact query and error, no variants.
      - Keep at most 6 calls for logs.
   5. ALWAYS call search_investigation(actor_user_name=<user>) once, whatever

integrations/rapid7-insightidr/scripts/query_log_set.py

@@ -137,6 +137,78 @@ def _identify(events, fallback_log_id, detection_name):
     return out
 
 
+# What a summary keeps: the values an analyst reads first, per field.
+_TOP_VALUES = 8
+_MAX_FIELDS = 30
+_SAMPLES = 5
+# Rapid7's whole vendor payload, kilobytes per event: it stays out of a summary.
+_SKIP_FIELDS = {"source_json", "source_data"}
+
+
+def _fields(e):
+    """One Log Search event as flat fields. `message` is a JSON-encoded string
+    on parsed sources; nested objects are flattened one level (a.b)."""
+    msg = e.get("message")
+    data = None
+    if isinstance(msg, str):
+        try:
+            data = json.loads(msg)
+        except ValueError:
+            data = None
+    elif isinstance(msg, dict):
+        data = msg
+    if not isinstance(data, dict):
+        return {"message": msg} if msg not in (None, "") else {}
+    flat = {}
+    for k, v in data.items():
+        if k in _SKIP_FIELDS:
+            continue
+        if isinstance(v, dict):
+            for k2, v2 in v.items():
+                if not isinstance(v2, (dict, list)) and v2 not in (None, ""):
+                    flat[k + "." + k2] = v2
+        elif not isinstance(v, list) and v not in (None, ""):
+            flat[k] = v
+    return flat
+
+
+def _summarize(events, limit):
+    """Counts instead of lines: per field, its most frequent values.
+
+    An agent asked who a user logged on to does not need a thousand raw lines —
+    on a real run it got 1000 events, could read about ten, and reported the
+    accounts and hosts of those ten. Top values per field answer the question
+    over the whole window and fit in a model's context.
+    """
+    counts = {}
+    first_ts, last_ts = None, None
+    for e in events:
+        ts = e.get("timestamp")
+        if isinstance(ts, (int, float)):
+            first_ts = ts if first_ts is None or ts < first_ts else first_ts
+            last_ts = ts if last_ts is None or ts > last_ts else last_ts
+        for k, v in _fields(e).items():
+            counts.setdefault(k, {})
+            key = str(v)[:200]
+            counts[k][key] = counts[k].get(key, 0) + 1
+    # Fields that vary least first: they are the ones that describe the activity
+    # (result, service, destination) rather than identify a line (timestamps, ids).
+    ranked = sorted(counts.items(), key=lambda kv: (len(kv[1]), kv[0]))
+    top = {}
+    for k, vals in ranked[:_MAX_FIELDS]:
+        top[k] = {"distinct": len(vals),
+                  "top": [[v, n] for v, n in sorted(vals.items(), key=lambda x: -x[1])[:_TOP_VALUES]]}
+    return {
+        "count": len(events),
+        "truncated": len(events) >= limit,
+        "first_event": _iso(first_ts) if first_ts is not None else None,
+        "last_event": _iso(last_ts) if last_ts is not None else None,
+        "fields": top,
+        "fields_omitted": max(0, len(counts) - _MAX_FIELDS),
+        "samples": [_fields(e) for e in events[:_SAMPLES]],
+    }
+
+
 def _log_sets():
     """Every log set of the account, as (id, name)."""
     out = request("GET", "log_search/management/logsets")
@@ -210,8 +282,14 @@ def run():
     first = request("GET", "log_search/query/logsets/" + urllib.parse.quote(log_set_id, safe=""), params=params)
     # No fallback log id: a log set spans several logs and every event names its own.
     events = _identify(_collect(first, limit), "", inp.get("detection_name"))
+    window = "%s to %s" % (_iso(start_time), _iso(end_time))
+    if str(inp.get("summarize") or "").strip().lower() in ("1", "true", "yes"):
+        out = _summarize(events, limit)
+        out.update({"from": start_time, "to": end_time, "window": window})
+        print(json.dumps(out))
+        return
     print(json.dumps({"events": events, "count": len(events), "from": start_time, "to": end_time,
-                      "window": "%s to %s" % (_iso(start_time), _iso(end_time))}))
+                      "window": window}))
 
 
 try: