feat(feed): add generic RSS/Atom IOC feed connector (URL/IP/hash extraction, defang-aware)

2026-07-13 12:15:49 +02:00
parent ac2203f9a5
commit 40252dadcb
3 changed files with 212 additions and 0 deletions
@@ -0,0 +1,108 @@
+import json, os, sys, re, ssl, urllib.request, urllib.error
+import xml.etree.ElementTree as ET
+
+_URL = re.compile(r"https?://[^\s<>\"'\]\)]+", re.I)
+_IP = re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b")
+_HASH = re.compile(r"\b[a-fA-F0-9]{64}\b|\b[a-fA-F0-9]{40}\b|\b[a-fA-F0-9]{32}\b")
+
+
+def _defang(t):
+    for a, b in (("[.]", "."), ("(.)", "."), ("{.}", "."), ("[dot]", "."),
+                 ("(dot)", "."), ("[:]", ":"), ("[://]", "://"),
+                 ("hxxp", "http"), ("hXXp", "http"), ("hxxps", "https")):
+        t = t.replace(a, b)
+    return t
+
+
+def _valid_ip(ip):
+    parts = ip.split(".")
+    return len(parts) == 4 and all(p.isdigit() and 0 <= int(p) <= 255 for p in parts)
+
+
+def _localname(tag):
+    return tag.split("}")[-1] if "}" in tag else tag
+
+
+def _cfg():
+    return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}"))
+
+
+def _inputs():
+    return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}"))
+
+
+def _ctx(cfg):
+    if cfg.get("insecure"):
+        c = ssl.create_default_context()
+        c.check_hostname = False
+        c.verify_mode = ssl.CERT_NONE
+        return c
+    return None
+
+
+def _headers(cfg):
+    h = {"User-Agent": "Riposte-SOAR", "Accept": "application/rss+xml, application/xml, text/xml"}
+    t = cfg.get("api_token")
+    if t:
+        h["Authorization"] = "Bearer " + str(t)
+    return h
+
+
+def _get(url, cfg):
+    req = urllib.request.Request(url, headers=_headers(cfg))
+    with urllib.request.urlopen(req, timeout=120, context=_ctx(cfg)) as r:
+        return r.read()
+
+
+def _run(fn):
+    try:
+        print(json.dumps(fn(_cfg(), _inputs())))
+    except urllib.error.HTTPError as e:
+        print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")}))
+        sys.exit(1)
+    except Exception as e:
+        print(json.dumps({"error": str(e)}))
+        sys.exit(1)
+
+
+def main(cfg, inputs):
+    url = cfg.get("feed_url")
+    if not url:
+        raise Exception("feed_url is required")
+
+    raw = _get(url, cfg)
+    root = ET.fromstring(raw)
+
+    texts = []
+    for el in root.iter():
+        if _localname(el.tag) in ("item", "entry"):
+            texts.append(_defang(" ".join(el.itertext())))
+    blob = " ".join(texts) if texts else _defang(ET.tostring(root, encoding="unicode"))
+
+    maxn = int(inputs.get("max_indicators") or 0)
+    seen = set()
+    out = []
+
+    def add(value, typ):
+        key = (typ, value)
+        if key in seen:
+            return
+        seen.add(key)
+        out.append({"value": value, "type": typ})
+
+    for m in _URL.finditer(blob):
+        add(m.group(0).rstrip(".,);]’"), "url")
+    for m in _HASH.finditer(blob):
+        add(m.group(0), "hash")
+    for m in _IP.finditer(blob):
+        v = m.group(0)
+        if _valid_ip(v):
+            add(v, "ip")
+
+    if maxn and len(out) > maxn:
+        out = out[:maxn]
+
+    return {"source": url, "count": len(out), "indicators": out}
+
+
+_run(main)