feat(feed): add Botvrij.eu OSINT feed connector (IOC lists by type)

integrations/feed-botvrij/manifest.yaml

@@ -0,0 +1,40 @@
+id: feed_botvrij
+name: Botvrij.eu Feed
+version: 1.0.0
+description: "Botvrij.eu OSINT feed connector — pull the free community IOC lists (destination IPs, domains, hostnames, URLs, file hashes, filenames, e-mails) and emit normalized IOCs (value + type) for import into the Threat Indicator Manager. Free, no authentication required; stdlib-only, no extra Python dependencies."
+changelog: "1.0.0 — Initial release: fetch a botvrij.eu IOC list by type."
+category: feed
+
+# Botvrij.eu publishes free plain-text IOC lists over HTTPS. No key required.
+config_schema:
+  properties:
+    list:
+      type: string
+      description: "Which list: ip-dst, domain, hostname, url, md5, sha1, sha256, filename, email-src. Default ip-dst"
+      default: "ip-dst"
+    insecure:
+      type: boolean
+      description: "Trust any TLS certificate (not secure)"
+      default: false
+  required: []
+
+commands:
+  - id: fetch_indicators
+    name: feed-botvrij-fetch-indicators
+    description: "Fetch a botvrij.eu IOC list and return normalized indicators."
+    risk: read
+    inputs_schema:
+      properties:
+        list: { type: string, description: "ip-dst | domain | hostname | url | md5 | sha1 | sha256 | filename | email-src (overrides config)" }
+        max_indicators: { type: number, description: "Max indicators to return (0 = no limit, default 0)" }
+      required: []
+    outputs_schema: { properties: {} }
+
+  - id: test_connection
+    name: feed-botvrij-test-connection
+    description: "Verify the botvrij.eu feed is reachable (used by the Test button)."
+    risk: read
+    inputs_schema:
+      properties: {}
+      required: []
+    outputs_schema: { properties: {} }

integrations/feed-botvrij/scripts/fetch_indicators.py

@@ -0,0 +1,74 @@
+import json, os, sys, ssl, urllib.request, urllib.error
+
+BASE = "https://www.botvrij.eu/data/ioclist."
+
+# botvrij list name -> normalized indicator type
+TYPES = {
+    "ip-dst": "ip",
+    "domain": "domain",
+    "hostname": "domain",
+    "url": "url",
+    "md5": "hash",
+    "sha1": "hash",
+    "sha256": "hash",
+    "filename": "filename",
+    "email-src": "email",
+}
+
+
+def _cfg():
+    return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}"))
+
+
+def _inputs():
+    return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}"))
+
+
+def _ctx(cfg):
+    if cfg.get("insecure"):
+        c = ssl.create_default_context()
+        c.check_hostname = False
+        c.verify_mode = ssl.CERT_NONE
+        return c
+    return None
+
+
+def _get(url, cfg):
+    req = urllib.request.Request(url, headers={"User-Agent": "Riposte-SOAR"})
+    with urllib.request.urlopen(req, timeout=120, context=_ctx(cfg)) as r:
+        return r.read()
+
+
+def _run(fn):
+    try:
+        print(json.dumps(fn(_cfg(), _inputs())))
+    except urllib.error.HTTPError as e:
+        print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")}))
+        sys.exit(1)
+    except Exception as e:
+        print(json.dumps({"error": str(e)}))
+        sys.exit(1)
+
+
+def main(cfg, inputs):
+    key = str(inputs.get("list") or cfg.get("list") or "ip-dst").lower()
+    if key not in TYPES:
+        key = "ip-dst"
+    typ = TYPES[key]
+
+    raw = _get(BASE + key, cfg).decode("utf-8", "replace")
+
+    maxn = int(inputs.get("max_indicators") or 0)
+    out = []
+    for line in raw.splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        out.append({"value": line, "type": typ, "list": key})
+        if maxn and len(out) >= maxn:
+            break
+
+    return {"source": "botvrij:" + key, "count": len(out), "indicators": out}
+
+
+_run(main)

integrations/feed-botvrij/scripts/test_connection.py

@@ -0,0 +1,47 @@
+import json, os, sys, ssl, urllib.request, urllib.error
+
+BASE = "https://www.botvrij.eu/data/ioclist."
+
+
+def _cfg():
+    return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}"))
+
+
+def _inputs():
+    return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}"))
+
+
+def _ctx(cfg):
+    if cfg.get("insecure"):
+        c = ssl.create_default_context()
+        c.check_hostname = False
+        c.verify_mode = ssl.CERT_NONE
+        return c
+    return None
+
+
+def _get(url, cfg):
+    req = urllib.request.Request(url, headers={"User-Agent": "Riposte-SOAR"})
+    with urllib.request.urlopen(req, timeout=60, context=_ctx(cfg)) as r:
+        return r.read()
+
+
+def _run(fn):
+    try:
+        print(json.dumps(fn(_cfg(), _inputs())))
+    except urllib.error.HTTPError as e:
+        print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")}))
+        sys.exit(1)
+    except Exception as e:
+        print(json.dumps({"error": str(e)}))
+        sys.exit(1)
+
+
+def main(cfg, inputs):
+    key = str(cfg.get("list") or "ip-dst").lower()
+    raw = _get(BASE + key, cfg).decode("utf-8", "replace")
+    n = sum(1 for ln in raw.splitlines() if ln.strip() and not ln.startswith("#"))
+    return {"ok": True, "sample_count": n, "list": key}
+
+
+_run(main)

integrations/feed-office365/manifest.yaml

@@ -0,0 +1,46 @@
+id: feed_office365
+name: Microsoft 365 Endpoints Feed
+version: 1.0.0
+description: "Microsoft 365 endpoints feed connector — pull the official published IP ranges and URLs for Microsoft 365 / Office 365 services and emit normalized IOCs (CIDR and domain + type, with service area) for import into the Threat Indicator Manager. Intended as an allowlist / known-infrastructure feed (mark benign in TIM). Free public endpoint, no authentication required; stdlib-only, no extra Python dependencies."
+changelog: "1.0.0 — Initial release: fetch the Microsoft 365 worldwide endpoints (IPs + URLs)."
+category: feed
+
+# The endpoints.office.com service is free. instance selects the cloud
+# (Worldwide, USGovDoD, USGovGCCHigh, China, Germany).
+config_schema:
+  properties:
+    instance:
+      type: string
+      description: "Which M365 cloud instance: Worldwide, USGovDoD, USGovGCCHigh, China, Germany (default Worldwide)"
+      default: "Worldwide"
+    include_urls:
+      type: boolean
+      description: "Also emit the published service URLs as domain indicators (default true)"
+      default: true
+    insecure:
+      type: boolean
+      description: "Trust any TLS certificate (not secure)"
+      default: false
+  required: []
+
+commands:
+  - id: fetch_indicators
+    name: feed-office365-fetch-indicators
+    description: "Fetch the Microsoft 365 endpoints and return normalized indicators."
+    risk: read
+    inputs_schema:
+      properties:
+        instance: { type: string, description: "Cloud instance (overrides config)" }
+        include_urls: { type: boolean, description: "Emit URLs as domain indicators (overrides config)" }
+        max_indicators: { type: number, description: "Max indicators to return (0 = no limit, default 0)" }
+      required: []
+    outputs_schema: { properties: {} }
+
+  - id: test_connection
+    name: feed-office365-test-connection
+    description: "Verify the Microsoft 365 endpoints service is reachable (used by the Test button)."
+    risk: read
+    inputs_schema:
+      properties: {}
+      required: []
+    outputs_schema: { properties: {} }

integrations/feed-office365/scripts/fetch_indicators.py

@@ -0,0 +1,89 @@
+import json, os, sys, ssl, urllib.parse, urllib.request, urllib.error
+
+BASE = "https://endpoints.office.com/endpoints/"
+# A fixed client request id is acceptable for this public, unauthenticated API.
+CLIENT_ID = "b10c5ed1-bad1-445f-b386-b919946339a7"
+
+
+def _cfg():
+    return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}"))
+
+
+def _inputs():
+    return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}"))
+
+
+def _ctx(cfg):
+    if cfg.get("insecure"):
+        c = ssl.create_default_context()
+        c.check_hostname = False
+        c.verify_mode = ssl.CERT_NONE
+        return c
+    return None
+
+
+def _get(url, cfg):
+    req = urllib.request.Request(
+        url, headers={"User-Agent": "Riposte-SOAR", "Accept": "application/json"}
+    )
+    with urllib.request.urlopen(req, timeout=120, context=_ctx(cfg)) as r:
+        return r.read()
+
+
+def _run(fn):
+    try:
+        print(json.dumps(fn(_cfg(), _inputs())))
+    except urllib.error.HTTPError as e:
+        print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")}))
+        sys.exit(1)
+    except Exception as e:
+        print(json.dumps({"error": str(e)}))
+        sys.exit(1)
+
+
+def main(cfg, inputs):
+    instance = str(inputs.get("instance") or cfg.get("instance") or "Worldwide")
+    include_urls = inputs.get("include_urls")
+    if include_urls is None:
+        include_urls = cfg.get("include_urls")
+    if include_urls is None:
+        include_urls = True
+
+    url = BASE + urllib.parse.quote(instance, safe="") + "?" + urllib.parse.urlencode({"clientrequestid": CLIENT_ID})
+    raw = _get(url, cfg)
+    data = json.loads(raw) if raw else []
+    if not isinstance(data, list):
+        data = []
+
+    maxn = int(inputs.get("max_indicators") or 0)
+    seen = set()
+    out = []
+
+    def add(value, typ, area):
+        key = (typ, value)
+        if key in seen:
+            return False
+        seen.add(key)
+        out.append({"value": value, "type": typ, "service_area": area, "provider": "microsoft365"})
+        return not (maxn and len(out) >= maxn)
+
+    for ep in data:
+        if not isinstance(ep, dict):
+            continue
+        area = ep.get("serviceArea")
+        for cidr in ep.get("ips", []) or []:
+            typ = "cidr" if "/" in str(cidr) else "ip"
+            if not add(str(cidr), typ, area):
+                return {"source": "microsoft365:" + instance, "count": len(out), "indicators": out}
+        if include_urls:
+            for u in ep.get("urls", []) or []:
+                dom = str(u).lstrip("*.").strip()
+                if not dom:
+                    continue
+                if not add(dom, "domain", area):
+                    return {"source": "microsoft365:" + instance, "count": len(out), "indicators": out}
+
+    return {"source": "microsoft365:" + instance, "count": len(out), "indicators": out}
+
+
+_run(main)

integrations/feed-office365/scripts/test_connection.py

@@ -0,0 +1,52 @@
+import json, os, sys, ssl, urllib.parse, urllib.request, urllib.error
+
+BASE = "https://endpoints.office.com/endpoints/"
+CLIENT_ID = "b10c5ed1-bad1-445f-b386-b919946339a7"
+
+
+def _cfg():
+    return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}"))
+
+
+def _inputs():
+    return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}"))
+
+
+def _ctx(cfg):
+    if cfg.get("insecure"):
+        c = ssl.create_default_context()
+        c.check_hostname = False
+        c.verify_mode = ssl.CERT_NONE
+        return c
+    return None
+
+
+def _get(url, cfg):
+    req = urllib.request.Request(
+        url, headers={"User-Agent": "Riposte-SOAR", "Accept": "application/json"}
+    )
+    with urllib.request.urlopen(req, timeout=60, context=_ctx(cfg)) as r:
+        return r.read()
+
+
+def _run(fn):
+    try:
+        print(json.dumps(fn(_cfg(), _inputs())))
+    except urllib.error.HTTPError as e:
+        print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")}))
+        sys.exit(1)
+    except Exception as e:
+        print(json.dumps({"error": str(e)}))
+        sys.exit(1)
+
+
+def main(cfg, inputs):
+    instance = str(cfg.get("instance") or "Worldwide")
+    url = BASE + urllib.parse.quote(instance, safe="") + "?" + urllib.parse.urlencode({"clientrequestid": CLIENT_ID})
+    raw = _get(url, cfg)
+    data = json.loads(raw) if raw else []
+    n = len(data) if isinstance(data, list) else 0
+    return {"ok": True, "endpoint_sets": n, "instance": instance}
+
+
+_run(main)

integrations/feed-rss/manifest.yaml

@@ -0,0 +1,43 @@
+id: feed_rss
+name: RSS/Atom IOC Feed
+version: 1.0.0
+description: "Generic RSS/Atom threat-intel feed connector — fetch a security blog or advisory feed and extract indicators (URLs, IPs and file hashes, including common defanged forms like hxxp and 1[.]2[.]3[.]4) from the item titles and bodies, emitting normalized IOCs for import into the Threat Indicator Manager. One connector, many feeds. Optional bearer token; stdlib-only, no extra Python dependencies."
+changelog: "1.0.0 — Initial release: extract URL/IP/hash IOCs from an RSS or Atom feed."
+category: feed
+
+# The feed is fetched over HTTP(S) from feed_url and parsed as RSS or Atom XML.
+config_schema:
+  properties:
+    feed_url:
+      type: string
+      description: "URL of the RSS/Atom feed"
+    api_token:
+      type: string
+      description: "Optional bearer token (if the feed requires auth)"
+      x-soar-sensitive: true
+    insecure:
+      type: boolean
+      description: "Trust any TLS certificate (not secure)"
+      default: false
+  required:
+    - feed_url
+
+commands:
+  - id: fetch_indicators
+    name: feed-rss-fetch-indicators
+    description: "Fetch the RSS/Atom feed and return IOCs extracted from its items."
+    risk: read
+    inputs_schema:
+      properties:
+        max_indicators: { type: number, description: "Max indicators to return (0 = no limit, default 0)" }
+      required: []
+    outputs_schema: { properties: {} }
+
+  - id: test_connection
+    name: feed-rss-test-connection
+    description: "Verify the feed URL returns parseable XML (used by the Test button)."
+    risk: read
+    inputs_schema:
+      properties: {}
+      required: []
+    outputs_schema: { properties: {} }

integrations/feed-rss/scripts/fetch_indicators.py

@@ -0,0 +1,108 @@
+import json, os, sys, re, ssl, urllib.request, urllib.error
+import xml.etree.ElementTree as ET
+
+_URL = re.compile(r"https?://[^\s<>\"'\]\)]+", re.I)
+_IP = re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b")
+_HASH = re.compile(r"\b[a-fA-F0-9]{64}\b|\b[a-fA-F0-9]{40}\b|\b[a-fA-F0-9]{32}\b")
+
+
+def _defang(t):
+    for a, b in (("[.]", "."), ("(.)", "."), ("{.}", "."), ("[dot]", "."),
+                 ("(dot)", "."), ("[:]", ":"), ("[://]", "://"),
+                 ("hxxp", "http"), ("hXXp", "http"), ("hxxps", "https")):
+        t = t.replace(a, b)
+    return t
+
+
+def _valid_ip(ip):
+    parts = ip.split(".")
+    return len(parts) == 4 and all(p.isdigit() and 0 <= int(p) <= 255 for p in parts)
+
+
+def _localname(tag):
+    return tag.split("}")[-1] if "}" in tag else tag
+
+
+def _cfg():
+    return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}"))
+
+
+def _inputs():
+    return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}"))
+
+
+def _ctx(cfg):
+    if cfg.get("insecure"):
+        c = ssl.create_default_context()
+        c.check_hostname = False
+        c.verify_mode = ssl.CERT_NONE
+        return c
+    return None
+
+
+def _headers(cfg):
+    h = {"User-Agent": "Riposte-SOAR", "Accept": "application/rss+xml, application/xml, text/xml"}
+    t = cfg.get("api_token")
+    if t:
+        h["Authorization"] = "Bearer " + str(t)
+    return h
+
+
+def _get(url, cfg):
+    req = urllib.request.Request(url, headers=_headers(cfg))
+    with urllib.request.urlopen(req, timeout=120, context=_ctx(cfg)) as r:
+        return r.read()
+
+
+def _run(fn):
+    try:
+        print(json.dumps(fn(_cfg(), _inputs())))
+    except urllib.error.HTTPError as e:
+        print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")}))
+        sys.exit(1)
+    except Exception as e:
+        print(json.dumps({"error": str(e)}))
+        sys.exit(1)
+
+
+def main(cfg, inputs):
+    url = cfg.get("feed_url")
+    if not url:
+        raise Exception("feed_url is required")
+
+    raw = _get(url, cfg)
+    root = ET.fromstring(raw)
+
+    texts = []
+    for el in root.iter():
+        if _localname(el.tag) in ("item", "entry"):
+            texts.append(_defang(" ".join(el.itertext())))
+    blob = " ".join(texts) if texts else _defang(ET.tostring(root, encoding="unicode"))
+
+    maxn = int(inputs.get("max_indicators") or 0)
+    seen = set()
+    out = []
+
+    def add(value, typ):
+        key = (typ, value)
+        if key in seen:
+            return
+        seen.add(key)
+        out.append({"value": value, "type": typ})
+
+    for m in _URL.finditer(blob):
+        add(m.group(0).rstrip(".,);]’"), "url")
+    for m in _HASH.finditer(blob):
+        add(m.group(0), "hash")
+    for m in _IP.finditer(blob):
+        v = m.group(0)
+        if _valid_ip(v):
+            add(v, "ip")
+
+    if maxn and len(out) > maxn:
+        out = out[:maxn]
+
+    return {"source": url, "count": len(out), "indicators": out}
+
+
+_run(main)

integrations/feed-rss/scripts/test_connection.py

@@ -0,0 +1,61 @@
+import json, os, sys, ssl, urllib.request, urllib.error
+import xml.etree.ElementTree as ET
+
+
+def _localname(tag):
+    return tag.split("}")[-1] if "}" in tag else tag
+
+
+def _cfg():
+    return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}"))
+
+
+def _inputs():
+    return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}"))
+
+
+def _ctx(cfg):
+    if cfg.get("insecure"):
+        c = ssl.create_default_context()
+        c.check_hostname = False
+        c.verify_mode = ssl.CERT_NONE
+        return c
+    return None
+
+
+def _headers(cfg):
+    h = {"User-Agent": "Riposte-SOAR", "Accept": "application/rss+xml, application/xml, text/xml"}
+    t = cfg.get("api_token")
+    if t:
+        h["Authorization"] = "Bearer " + str(t)
+    return h
+
+
+def _get(url, cfg):
+    req = urllib.request.Request(url, headers=_headers(cfg))
+    with urllib.request.urlopen(req, timeout=60, context=_ctx(cfg)) as r:
+        return r.read()
+
+
+def _run(fn):
+    try:
+        print(json.dumps(fn(_cfg(), _inputs())))
+    except urllib.error.HTTPError as e:
+        print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")}))
+        sys.exit(1)
+    except Exception as e:
+        print(json.dumps({"error": str(e)}))
+        sys.exit(1)
+
+
+def main(cfg, inputs):
+    url = cfg.get("feed_url")
+    if not url:
+        raise Exception("feed_url is required")
+    raw = _get(url, cfg)
+    root = ET.fromstring(raw)
+    n = sum(1 for el in root.iter() if _localname(el.tag) in ("item", "entry"))
+    return {"ok": True, "items": n}
+
+
+_run(main)