feat: CIRCL hashlookup, RDAP and crt.sh — free IOC lookups, no API key
Three keyless sources for the questions an analyst asks first: - circl-hashlookup 1.0.0: is this hash known software (NSRL, distro packages, Windows updates)? `score` 0 for a known file, 100 when CIRCL flags it KnownMalicious, so an enrichment rule with benign 0 marks it benign; an unknown hash is an HTTP 404, read as "no record". Single and bulk lookups; base_url for a self-hosted instance. - rdap 1.0.0: who registered a domain, IP block or AS, and when — with age_days for the phishing question. Authoritative server from the IANA bootstrap, rdap.org as fallback; a host name walks up to its registered domain; GDPR placeholders are not reported as registrants; a TLD with no RDAP (.ch) is an error, not a fake "unregistered". - crtsh 1.0.0: Certificate Transparency — sub-domains with first/last seen, issuers, recent certificates, and a starts_with mode for look-alikes of a brand (crt.sh no longer searches with a leading wildcard). Retries the 502/503/504 its shared front throws under load. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AmWT9wtWHGtZpMS64Hp4q5
This commit is contained in:
@@ -0,0 +1,48 @@
|
||||
id: circl_hashlookup
|
||||
name: CIRCL hashlookup
|
||||
version: 1.0.0
|
||||
description: "CIRCL hashlookup — tells whether a file hash belongs to known software: NSRL, Linux distribution packages, Windows updates and other trusted sets (6.5 billion hashes). The fastest way to rule out a false positive on a file alert. Free public service, no API key; the server is open source and can be self-hosted (set base_url). Stdlib-only. As an enrichment rule: command lookup_hash, score field `score`, benign threshold 0 and malicious threshold 100 — a known file reads benign, a record CIRCL flags as KnownMalicious reads malicious, and an unknown hash is an answer (not found), never a failure."
|
||||
changelog: "1.0.0 — Initial release: single lookup (MD5, SHA-1, SHA-256), bulk lookup (MD5, SHA-1), test connection."
|
||||
category: enrichment
|
||||
|
||||
config_schema:
|
||||
properties:
|
||||
base_url:
|
||||
type: string
|
||||
description: "hashlookup server (default https://hashlookup.circl.lu — set it to a self-hosted instance to keep hashes in-house)"
|
||||
default: https://hashlookup.circl.lu
|
||||
insecure:
|
||||
type: boolean
|
||||
description: "Trust any TLS certificate (not secure — a self-hosted instance with a private CA only)"
|
||||
default: false
|
||||
required: []
|
||||
|
||||
commands:
|
||||
- id: lookup_hash
|
||||
name: hashlookup-lookup-hash
|
||||
description: "Look up one MD5, SHA-1 or SHA-256 hash. Returns the CIRCL record (file name, product, source set, hashlookup:trust) plus `known: true` and `score` (0 = known file, 100 = flagged KnownMalicious). An unknown hash returns HTTP 404, which Riposte's enrichment reads as 'no record'."
|
||||
risk: read
|
||||
inputs_schema:
|
||||
properties:
|
||||
hash: { type: string, description: "MD5 (32 hex), SHA-1 (40 hex) or SHA-256 (64 hex)" }
|
||||
required: [hash]
|
||||
outputs_schema: { properties: {} }
|
||||
|
||||
- id: bulk_lookup
|
||||
name: hashlookup-bulk-lookup
|
||||
description: "Look up many MD5 / SHA-1 hashes at once (the bulk API does not take SHA-256; those are looked up one by one). Returns `known` (records found) and `unknown` (hashes CIRCL has no record of)."
|
||||
risk: read
|
||||
inputs_schema:
|
||||
properties:
|
||||
hashes: { type: string, description: "Hashes separated by commas, spaces or new lines (a JSON array also works)" }
|
||||
required: [hashes]
|
||||
outputs_schema: { properties: {} }
|
||||
|
||||
- id: test_connection
|
||||
name: hashlookup-test-connection
|
||||
description: "Reach the server's /info endpoint (used by the Test button)."
|
||||
risk: read
|
||||
inputs_schema:
|
||||
properties: {}
|
||||
required: []
|
||||
outputs_schema: { properties: {} }
|
||||
@@ -0,0 +1,106 @@
|
||||
import json, os, re, ssl, sys, urllib.error, urllib.request
|
||||
|
||||
DEFAULT_BASE = "https://hashlookup.circl.lu"
|
||||
|
||||
|
||||
def _cfg():
|
||||
return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}"))
|
||||
|
||||
|
||||
def _inputs():
|
||||
return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}"))
|
||||
|
||||
|
||||
def _base(cfg):
|
||||
return (str(cfg.get("base_url") or "").strip() or DEFAULT_BASE).rstrip("/")
|
||||
|
||||
|
||||
def _ctx(cfg):
|
||||
if cfg.get("insecure") in (True, "true", "True", 1, "1"):
|
||||
c = ssl.create_default_context()
|
||||
c.check_hostname = False
|
||||
c.verify_mode = ssl.CERT_NONE
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def call(cfg, path, body=None):
|
||||
data = json.dumps(body).encode("utf-8") if body is not None else None
|
||||
headers = {"Accept": "application/json", "User-Agent": "riposte-soar"}
|
||||
if data is not None:
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(_base(cfg) + path, data=data, headers=headers, method="POST" if data is not None else "GET")
|
||||
with urllib.request.urlopen(req, timeout=60, context=_ctx(cfg)) as r:
|
||||
raw = r.read()
|
||||
return json.loads(raw) if raw else {}
|
||||
|
||||
|
||||
def _run(fn):
|
||||
try:
|
||||
print(json.dumps(fn(_cfg(), _inputs())))
|
||||
except urllib.error.HTTPError as e:
|
||||
print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")}))
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": str(e)}))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
KINDS = {32: "md5", 40: "sha1", 64: "sha256"}
|
||||
|
||||
|
||||
def kind_of(h):
|
||||
h = h.strip().lower()
|
||||
if not re.fullmatch(r"[0-9a-f]+", h) or len(h) not in KINDS:
|
||||
raise Exception("not an MD5, SHA-1 or SHA-256 hash: " + h)
|
||||
return KINDS[len(h)], h
|
||||
|
||||
|
||||
def annotate(rec):
|
||||
# score: 0 for a known file, 100 when CIRCL itself flags the record as
|
||||
# malicious — what an enrichment rule maps with benign 0 / malicious 100.
|
||||
rec = dict(rec)
|
||||
rec["known"] = True
|
||||
rec["score"] = 100 if rec.get("KnownMalicious") else 0
|
||||
return rec
|
||||
|
||||
|
||||
def parse_list(v):
|
||||
if isinstance(v, list):
|
||||
return [str(x) for x in v]
|
||||
s = str(v or "").strip()
|
||||
if s.startswith("["):
|
||||
return [str(x) for x in json.loads(s)]
|
||||
return [x for x in re.split(r"[\s,;]+", s) if x]
|
||||
|
||||
|
||||
def main(cfg, inputs):
|
||||
groups = {"md5": [], "sha1": [], "sha256": []}
|
||||
for raw in parse_list(inputs.get("hashes")):
|
||||
kind, h = kind_of(raw)
|
||||
if h not in groups[kind]:
|
||||
groups[kind].append(h)
|
||||
known, unknown = [], []
|
||||
for kind in ("md5", "sha1"):
|
||||
hs = groups[kind]
|
||||
for i in range(0, len(hs), 500):
|
||||
chunk = hs[i:i + 500]
|
||||
res = call(cfg, "/bulk/" + kind, {"hashes": chunk})
|
||||
found = set()
|
||||
for rec in res if isinstance(res, list) else []:
|
||||
key = str(rec.get("MD5" if kind == "md5" else "SHA-1", "")).lower()
|
||||
if key and "message" not in rec:
|
||||
found.add(key)
|
||||
known.append(annotate(rec))
|
||||
unknown.extend(h for h in chunk if h not in found)
|
||||
for h in groups["sha256"]:
|
||||
try:
|
||||
known.append(annotate(call(cfg, "/lookup/sha256/" + h)))
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code != 404:
|
||||
raise
|
||||
unknown.append(h)
|
||||
return {"known": known, "unknown": unknown, "known_count": len(known), "unknown_count": len(unknown)}
|
||||
|
||||
|
||||
_run(main)
|
||||
@@ -0,0 +1,73 @@
|
||||
import json, os, re, ssl, sys, urllib.error, urllib.request
|
||||
|
||||
DEFAULT_BASE = "https://hashlookup.circl.lu"
|
||||
|
||||
|
||||
def _cfg():
|
||||
return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}"))
|
||||
|
||||
|
||||
def _inputs():
|
||||
return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}"))
|
||||
|
||||
|
||||
def _base(cfg):
|
||||
return (str(cfg.get("base_url") or "").strip() or DEFAULT_BASE).rstrip("/")
|
||||
|
||||
|
||||
def _ctx(cfg):
|
||||
if cfg.get("insecure") in (True, "true", "True", 1, "1"):
|
||||
c = ssl.create_default_context()
|
||||
c.check_hostname = False
|
||||
c.verify_mode = ssl.CERT_NONE
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def call(cfg, path, body=None):
|
||||
data = json.dumps(body).encode("utf-8") if body is not None else None
|
||||
headers = {"Accept": "application/json", "User-Agent": "riposte-soar"}
|
||||
if data is not None:
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(_base(cfg) + path, data=data, headers=headers, method="POST" if data is not None else "GET")
|
||||
with urllib.request.urlopen(req, timeout=60, context=_ctx(cfg)) as r:
|
||||
raw = r.read()
|
||||
return json.loads(raw) if raw else {}
|
||||
|
||||
|
||||
def _run(fn):
|
||||
try:
|
||||
print(json.dumps(fn(_cfg(), _inputs())))
|
||||
except urllib.error.HTTPError as e:
|
||||
print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")}))
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": str(e)}))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
KINDS = {32: "md5", 40: "sha1", 64: "sha256"}
|
||||
|
||||
|
||||
def kind_of(h):
|
||||
h = h.strip().lower()
|
||||
if not re.fullmatch(r"[0-9a-f]+", h) or len(h) not in KINDS:
|
||||
raise Exception("not an MD5, SHA-1 or SHA-256 hash: " + h)
|
||||
return KINDS[len(h)], h
|
||||
|
||||
|
||||
def annotate(rec):
|
||||
# score: 0 for a known file, 100 when CIRCL itself flags the record as
|
||||
# malicious — what an enrichment rule maps with benign 0 / malicious 100.
|
||||
rec = dict(rec)
|
||||
rec["known"] = True
|
||||
rec["score"] = 100 if rec.get("KnownMalicious") else 0
|
||||
return rec
|
||||
|
||||
|
||||
def main(cfg, inputs):
|
||||
kind, h = kind_of(str(inputs.get("hash", "")))
|
||||
return annotate(call(cfg, "/lookup/%s/%s" % (kind, h)))
|
||||
|
||||
|
||||
_run(main)
|
||||
@@ -0,0 +1,73 @@
|
||||
import json, os, re, ssl, sys, urllib.error, urllib.request
|
||||
|
||||
DEFAULT_BASE = "https://hashlookup.circl.lu"
|
||||
|
||||
|
||||
def _cfg():
|
||||
return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}"))
|
||||
|
||||
|
||||
def _inputs():
|
||||
return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}"))
|
||||
|
||||
|
||||
def _base(cfg):
|
||||
return (str(cfg.get("base_url") or "").strip() or DEFAULT_BASE).rstrip("/")
|
||||
|
||||
|
||||
def _ctx(cfg):
|
||||
if cfg.get("insecure") in (True, "true", "True", 1, "1"):
|
||||
c = ssl.create_default_context()
|
||||
c.check_hostname = False
|
||||
c.verify_mode = ssl.CERT_NONE
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def call(cfg, path, body=None):
|
||||
data = json.dumps(body).encode("utf-8") if body is not None else None
|
||||
headers = {"Accept": "application/json", "User-Agent": "riposte-soar"}
|
||||
if data is not None:
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(_base(cfg) + path, data=data, headers=headers, method="POST" if data is not None else "GET")
|
||||
with urllib.request.urlopen(req, timeout=60, context=_ctx(cfg)) as r:
|
||||
raw = r.read()
|
||||
return json.loads(raw) if raw else {}
|
||||
|
||||
|
||||
def _run(fn):
|
||||
try:
|
||||
print(json.dumps(fn(_cfg(), _inputs())))
|
||||
except urllib.error.HTTPError as e:
|
||||
print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")}))
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": str(e)}))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
KINDS = {32: "md5", 40: "sha1", 64: "sha256"}
|
||||
|
||||
|
||||
def kind_of(h):
|
||||
h = h.strip().lower()
|
||||
if not re.fullmatch(r"[0-9a-f]+", h) or len(h) not in KINDS:
|
||||
raise Exception("not an MD5, SHA-1 or SHA-256 hash: " + h)
|
||||
return KINDS[len(h)], h
|
||||
|
||||
|
||||
def annotate(rec):
|
||||
# score: 0 for a known file, 100 when CIRCL itself flags the record as
|
||||
# malicious — what an enrichment rule maps with benign 0 / malicious 100.
|
||||
rec = dict(rec)
|
||||
rec["known"] = True
|
||||
rec["score"] = 100 if rec.get("KnownMalicious") else 0
|
||||
return rec
|
||||
|
||||
|
||||
def main(cfg, inputs):
|
||||
info = call(cfg, "/info")
|
||||
return {"ok": True, "info": info}
|
||||
|
||||
|
||||
_run(main)
|
||||
Reference in New Issue
Block a user