From 40252dadcba08b6f0a1b60946c24a0460c2cc7e7 Mon Sep 17 00:00:00 2001 From: Guillaume BOURGEOIS Date: Mon, 13 Jul 2026 12:15:49 +0200 Subject: [PATCH] feat(feed): add generic RSS/Atom IOC feed connector (URL/IP/hash extraction, defang-aware) --- integrations/feed-rss/manifest.yaml | 43 +++++++ .../feed-rss/scripts/fetch_indicators.py | 108 ++++++++++++++++++ .../feed-rss/scripts/test_connection.py | 61 ++++++++++ 3 files changed, 212 insertions(+) create mode 100644 integrations/feed-rss/manifest.yaml create mode 100644 integrations/feed-rss/scripts/fetch_indicators.py create mode 100644 integrations/feed-rss/scripts/test_connection.py diff --git a/integrations/feed-rss/manifest.yaml b/integrations/feed-rss/manifest.yaml new file mode 100644 index 0000000..2e1d1c0 --- /dev/null +++ b/integrations/feed-rss/manifest.yaml @@ -0,0 +1,43 @@ +id: feed_rss +name: RSS/Atom IOC Feed +version: 1.0.0 +description: "Generic RSS/Atom threat-intel feed connector — fetch a security blog or advisory feed and extract indicators (URLs, IPs and file hashes, including common defanged forms like hxxp and 1[.]2[.]3[.]4) from the item titles and bodies, emitting normalized IOCs for import into the Threat Indicator Manager. One connector, many feeds. Optional bearer token; stdlib-only, no extra Python dependencies." +changelog: "1.0.0 — Initial release: extract URL/IP/hash IOCs from an RSS or Atom feed." +category: feed + +# The feed is fetched over HTTP(S) from feed_url and parsed as RSS or Atom XML. +config_schema: + properties: + feed_url: + type: string + description: "URL of the RSS/Atom feed" + api_token: + type: string + description: "Optional bearer token (if the feed requires auth)" + x-soar-sensitive: true + insecure: + type: boolean + description: "Trust any TLS certificate (not secure)" + default: false + required: + - feed_url + +commands: + - id: fetch_indicators + name: feed-rss-fetch-indicators + description: "Fetch the RSS/Atom feed and return IOCs extracted from its items." + risk: read + inputs_schema: + properties: + max_indicators: { type: number, description: "Max indicators to return (0 = no limit, default 0)" } + required: [] + outputs_schema: { properties: {} } + + - id: test_connection + name: feed-rss-test-connection + description: "Verify the feed URL returns parseable XML (used by the Test button)." + risk: read + inputs_schema: + properties: {} + required: [] + outputs_schema: { properties: {} } diff --git a/integrations/feed-rss/scripts/fetch_indicators.py b/integrations/feed-rss/scripts/fetch_indicators.py new file mode 100644 index 0000000..b9155e4 --- /dev/null +++ b/integrations/feed-rss/scripts/fetch_indicators.py @@ -0,0 +1,108 @@ +import json, os, sys, re, ssl, urllib.request, urllib.error +import xml.etree.ElementTree as ET + +_URL = re.compile(r"https?://[^\s<>\"'\]\)]+", re.I) +_IP = re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b") +_HASH = re.compile(r"\b[a-fA-F0-9]{64}\b|\b[a-fA-F0-9]{40}\b|\b[a-fA-F0-9]{32}\b") + + +def _defang(t): + for a, b in (("[.]", "."), ("(.)", "."), ("{.}", "."), ("[dot]", "."), + ("(dot)", "."), ("[:]", ":"), ("[://]", "://"), + ("hxxp", "http"), ("hXXp", "http"), ("hxxps", "https")): + t = t.replace(a, b) + return t + + +def _valid_ip(ip): + parts = ip.split(".") + return len(parts) == 4 and all(p.isdigit() and 0 <= int(p) <= 255 for p in parts) + + +def _localname(tag): + return tag.split("}")[-1] if "}" in tag else tag + + +def _cfg(): + return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}")) + + +def _inputs(): + return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}")) + + +def _ctx(cfg): + if cfg.get("insecure"): + c = ssl.create_default_context() + c.check_hostname = False + c.verify_mode = ssl.CERT_NONE + return c + return None + + +def _headers(cfg): + h = {"User-Agent": "Riposte-SOAR", "Accept": "application/rss+xml, application/xml, text/xml"} + t = cfg.get("api_token") + if t: + h["Authorization"] = "Bearer " + str(t) + return h + + +def _get(url, cfg): + req = urllib.request.Request(url, headers=_headers(cfg)) + with urllib.request.urlopen(req, timeout=120, context=_ctx(cfg)) as r: + return r.read() + + +def _run(fn): + try: + print(json.dumps(fn(_cfg(), _inputs()))) + except urllib.error.HTTPError as e: + print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")})) + sys.exit(1) + except Exception as e: + print(json.dumps({"error": str(e)})) + sys.exit(1) + + +def main(cfg, inputs): + url = cfg.get("feed_url") + if not url: + raise Exception("feed_url is required") + + raw = _get(url, cfg) + root = ET.fromstring(raw) + + texts = [] + for el in root.iter(): + if _localname(el.tag) in ("item", "entry"): + texts.append(_defang(" ".join(el.itertext()))) + blob = " ".join(texts) if texts else _defang(ET.tostring(root, encoding="unicode")) + + maxn = int(inputs.get("max_indicators") or 0) + seen = set() + out = [] + + def add(value, typ): + key = (typ, value) + if key in seen: + return + seen.add(key) + out.append({"value": value, "type": typ}) + + for m in _URL.finditer(blob): + add(m.group(0).rstrip(".,);]’"), "url") + for m in _HASH.finditer(blob): + add(m.group(0), "hash") + for m in _IP.finditer(blob): + v = m.group(0) + if _valid_ip(v): + add(v, "ip") + + if maxn and len(out) > maxn: + out = out[:maxn] + + return {"source": url, "count": len(out), "indicators": out} + + +_run(main) diff --git a/integrations/feed-rss/scripts/test_connection.py b/integrations/feed-rss/scripts/test_connection.py new file mode 100644 index 0000000..cd3ba1f --- /dev/null +++ b/integrations/feed-rss/scripts/test_connection.py @@ -0,0 +1,61 @@ +import json, os, sys, ssl, urllib.request, urllib.error +import xml.etree.ElementTree as ET + + +def _localname(tag): + return tag.split("}")[-1] if "}" in tag else tag + + +def _cfg(): + return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}")) + + +def _inputs(): + return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}")) + + +def _ctx(cfg): + if cfg.get("insecure"): + c = ssl.create_default_context() + c.check_hostname = False + c.verify_mode = ssl.CERT_NONE + return c + return None + + +def _headers(cfg): + h = {"User-Agent": "Riposte-SOAR", "Accept": "application/rss+xml, application/xml, text/xml"} + t = cfg.get("api_token") + if t: + h["Authorization"] = "Bearer " + str(t) + return h + + +def _get(url, cfg): + req = urllib.request.Request(url, headers=_headers(cfg)) + with urllib.request.urlopen(req, timeout=60, context=_ctx(cfg)) as r: + return r.read() + + +def _run(fn): + try: + print(json.dumps(fn(_cfg(), _inputs()))) + except urllib.error.HTTPError as e: + print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")})) + sys.exit(1) + except Exception as e: + print(json.dumps({"error": str(e)})) + sys.exit(1) + + +def main(cfg, inputs): + url = cfg.get("feed_url") + if not url: + raise Exception("feed_url is required") + raw = _get(url, cfg) + root = ET.fromstring(raw) + n = sum(1 for el in root.iter() if _localname(el.tag) in ("item", "entry")) + return {"ok": True, "items": n} + + +_run(main)