feat(crtsh): Cert Spotter fallback and a 60 s time budget (1.1.0)

crt.sh often answers 502 for minutes at a time. When it is down,
sub-domain and exact searches are now answered by SSLMate Cert Spotter,
which reads the same CT logs; its rows are reshaped into crt.sh's so the
output keeps its shape, plus 'source' and 'complete'. starts_with has no
fallback (Cert Spotter cannot search by prefix).

Riposte kills an integration script at 60 s, and the old retry loop
(4 x 120 s) could never finish. Every call now takes its timeout from a
55 s budget, with 15 s kept for the fallback; only 5xx are retried.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AmWT9wtWHGtZpMS64Hp4q5

integrations/crtsh/scripts/test_connection.py

@@ -20,25 +20,40 @@ def _ctx(cfg):
     return None
 
 
-def search(cfg, q, exclude_expired):
+def _get(url, headers, cfg, timeout):
+    req = urllib.request.Request(url, headers=dict({"Accept": "application/json", "User-Agent": "riposte-soar"}, **headers))
+    with urllib.request.urlopen(req, timeout=timeout, context=_ctx(cfg)) as r:
+        return json.loads(r.read() or b"[]")
+
+
+def _crtsh(cfg):
     base = (str(cfg.get("base_url") or "").strip() or DEFAULT_BASE).rstrip("/")
-    params = {"q": q, "output": "json"}
-    if exclude_expired:
-        params["exclude"] = "expired"
-    req = urllib.request.Request(base + "/?" + urllib.parse.urlencode(params),
-                                 headers={"Accept": "application/json", "User-Agent": "riposte-soar"})
-    # crt.sh's database is shared by the whole internet and its front answers
-    # 502/503/504 when a query queues too long; the same query usually goes
-    # through a few seconds later.
-    for attempt in range(4):
+    # Riposte stops the script at 60 s: two tries at most, Cert Spotter keeps the rest.
+    last = ""
+    for attempt in range(2):
         try:
-            with urllib.request.urlopen(req, timeout=120, context=_ctx(cfg)) as r:
-                raw = r.read()
-                return json.loads(raw) if raw.strip() else []
+            return len(_get(base + "/?" + urllib.parse.urlencode({"q": "crt.sh", "output": "json", "exclude": "expired"}), {}, cfg, 20)), ""
         except urllib.error.HTTPError as e:
-            if e.code not in (502, 503, 504) or attempt == 3:
-                raise
-            time.sleep(3 * (attempt + 1))
+            last = "HTTP " + str(e.code)
+            if e.code not in (502, 503, 504):
+                break
+        except Exception as e:
+            last = str(getattr(e, "reason", e))
+        if attempt == 0:
+            time.sleep(3)
+    return None, last
+
+
+def _certspotter(cfg):
+    key = str(cfg.get("certspotter_api_key") or "").strip()
+    base = (str(cfg.get("certspotter_url") or "").strip() or "https://api.certspotter.com").rstrip("/")
+    try:
+        _get(base + "/v1/issuances?domain=crt.sh", {"Authorization": "Bearer " + key} if key else {}, cfg, 15)
+        return True, ""
+    except urllib.error.HTTPError as e:
+        return False, "HTTP " + str(e.code) + (" (quota reached)" if e.code == 429 else "")
+    except Exception as e:
+        return False, str(getattr(e, "reason", e))
 
 
 def _run(fn):
@@ -53,8 +68,17 @@ def _run(fn):
 
 
 def main(cfg, inputs):
-    rows = search(cfg, "crt.sh", True)
-    return {"ok": True, "certificates": len(rows)}
+    count, crt_err = _crtsh(cfg)
+    if count is not None:
+        return {"ok": True, "crtsh": "ok", "certificates": count}
+    if cfg.get("certspotter_fallback") in (False, "false", "False", 0, "0"):
+        raise Exception("crt.sh unavailable (" + crt_err + ") and the Cert Spotter fallback is off")
+    ok, cs_err = _certspotter(cfg)
+    if not ok:
+        raise Exception("crt.sh unavailable (" + crt_err + ") and Cert Spotter unreachable (" + cs_err + ")")
+    # crt.sh being down is common and the fallback answers: the instance works.
+    return {"ok": True, "crtsh": "unavailable (" + crt_err + ")", "certspotter": "ok",
+            "note": "crt.sh is down right now; searches will be answered by Cert Spotter"}
 
 
 _run(main)