feat(feed): add URLhaus feed connector (abuse.ch malware URL feed)

integrations/feed-urlhaus/scripts/fetch_indicators.py

@@ -0,0 +1,80 @@
+import json, os, sys, ssl, io, csv, urllib.request, urllib.error
+
+BASE = "https://urlhaus.abuse.ch/downloads/"
+
+
+def _cfg():
+    return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}"))
+
+
+def _inputs():
+    return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}"))
+
+
+def _ctx(cfg):
+    if cfg.get("insecure"):
+        c = ssl.create_default_context()
+        c.check_hostname = False
+        c.verify_mode = ssl.CERT_NONE
+        return c
+    return None
+
+
+def _headers(cfg):
+    h = {"User-Agent": "Riposte-SOAR"}
+    k = cfg.get("api_key")
+    if k:
+        h["Auth-Key"] = str(k)
+    return h
+
+
+def _get(url, cfg):
+    req = urllib.request.Request(url, headers=_headers(cfg))
+    with urllib.request.urlopen(req, timeout=120, context=_ctx(cfg)) as r:
+        return r.read()
+
+
+def _run(fn):
+    try:
+        print(json.dumps(fn(_cfg(), _inputs())))
+    except urllib.error.HTTPError as e:
+        print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")}))
+        sys.exit(1)
+    except Exception as e:
+        print(json.dumps({"error": str(e)}))
+        sys.exit(1)
+
+
+def main(cfg, inputs):
+    which = str(inputs.get("list") or cfg.get("list") or "recent").lower()
+    fname = "csv_online" if which == "online" else "csv_recent"
+    raw = _get(BASE + fname + "/", cfg).decode("utf-8", "replace")
+
+    maxn = int(inputs.get("max_indicators") or 0)
+    out = []
+    for row in csv.reader(io.StringIO(raw)):
+        if not row or row[0].lstrip().startswith("#"):
+            continue
+        if row[0].strip().strip('"') == "id":  # header row
+            continue
+        # columns: id,dateadded,url,url_status,last_online,threat,tags,urlhaus_link,reporter
+        if len(row) < 3:
+            continue
+        url = row[2].strip()
+        if not url:
+            continue
+        item = {"value": url, "type": "url"}
+        if len(row) > 3:
+            item["status"] = row[3].strip()
+        if len(row) > 5:
+            item["threat"] = row[5].strip()
+        if len(row) > 6:
+            item["tags"] = row[6].strip()
+        out.append(item)
+        if maxn and len(out) >= maxn:
+            break
+
+    return {"source": "urlhaus:" + fname, "count": len(out), "indicators": out}
+
+
+_run(main)

integrations/feed-urlhaus/scripts/test_connection.py

@@ -0,0 +1,54 @@
+import json, os, sys, ssl, urllib.request, urllib.error
+
+BASE = "https://urlhaus.abuse.ch/downloads/"
+
+
+def _cfg():
+    return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}"))
+
+
+def _inputs():
+    return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}"))
+
+
+def _ctx(cfg):
+    if cfg.get("insecure"):
+        c = ssl.create_default_context()
+        c.check_hostname = False
+        c.verify_mode = ssl.CERT_NONE
+        return c
+    return None
+
+
+def _headers(cfg):
+    h = {"User-Agent": "Riposte-SOAR"}
+    k = cfg.get("api_key")
+    if k:
+        h["Auth-Key"] = str(k)
+    return h
+
+
+def _get(url, cfg):
+    req = urllib.request.Request(url, headers=_headers(cfg))
+    with urllib.request.urlopen(req, timeout=60, context=_ctx(cfg)) as r:
+        return r.read()
+
+
+def _run(fn):
+    try:
+        print(json.dumps(fn(_cfg(), _inputs())))
+    except urllib.error.HTTPError as e:
+        print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")}))
+        sys.exit(1)
+    except Exception as e:
+        print(json.dumps({"error": str(e)}))
+        sys.exit(1)
+
+
+def main(cfg, inputs):
+    raw = _get(BASE + "csv_recent/", cfg).decode("utf-8", "replace")
+    n = sum(1 for ln in raw.splitlines() if ln and not ln.lstrip().startswith("#"))
+    return {"ok": True, "sample_lines": max(0, n - 1)}
+
+
+_run(main)