diff --git a/integrations/feed-json/manifest.yaml b/integrations/feed-json/manifest.yaml new file mode 100644 index 0000000..de1bda6 --- /dev/null +++ b/integrations/feed-json/manifest.yaml @@ -0,0 +1,59 @@ +id: feed_json +name: JSON Feed +version: 1.0.0 +description: "Generic JSON threat-intel feed connector — fetch any JSON feed from a URL, dig into a configurable array path, pull a value field (and optional type field) and emit normalized IOCs (value + type) for import into the Threat Indicator Manager. One connector, many feeds. Optional bearer token; stdlib-only, no extra Python dependencies." +changelog: "1.0.0 — Initial release: fetch indicators from a JSON feed with configurable field mapping." +category: feed + +# The feed is fetched over HTTP(S) from feed_url. array_path digs into nested +# objects (dot notation) to reach the list of indicators; value_field/type_field +# select the fields on each element. +config_schema: + properties: + feed_url: + type: string + description: "URL of the JSON feed" + api_token: + type: string + description: "Optional bearer token (if the feed requires auth)" + x-soar-sensitive: true + array_path: + type: string + description: "Dot path to the array of indicators (e.g. 'data.indicators'). Empty = the top-level value" + value_field: + type: string + description: "Field holding the IOC value on each element (dot path allowed, default 'value')" + default: "value" + type_field: + type: string + description: "Optional field holding the IOC type on each element" + insecure: + type: boolean + description: "Trust any TLS certificate (not secure)" + default: false + required: + - feed_url + +commands: + - id: fetch_indicators + name: feed-json-fetch-indicators + description: "Fetch and parse the JSON feed, returning normalized indicators." + risk: read + inputs_schema: + properties: + array_path: { type: string, description: "Dot path to the array (overrides config)" } + value_field: { type: string, description: "Value field / dot path (overrides config)" } + type_field: { type: string, description: "Type field (overrides config)" } + ioc_type: { type: string, description: "auto | ip | domain | url | hash | email — fallback type when no type_field (default auto)" } + max_indicators: { type: number, description: "Max indicators to return (0 = no limit, default 0)" } + required: [] + outputs_schema: { properties: {} } + + - id: test_connection + name: feed-json-test-connection + description: "Verify the feed URL returns valid JSON (used by the Test button)." + risk: read + inputs_schema: + properties: {} + required: [] + outputs_schema: { properties: {} } diff --git a/integrations/feed-json/scripts/fetch_indicators.py b/integrations/feed-json/scripts/fetch_indicators.py new file mode 100644 index 0000000..6e4fc9e --- /dev/null +++ b/integrations/feed-json/scripts/fetch_indicators.py @@ -0,0 +1,120 @@ +import json, os, sys, re, ssl, urllib.request, urllib.error + +_IP = re.compile(r"^(?:\d{1,3}\.){3}\d{1,3}$") +_HASH = re.compile(r"^[a-fA-F0-9]{32}$|^[a-fA-F0-9]{40}$|^[a-fA-F0-9]{64}$") +_EMAIL = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") +_URL = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.\-]*://") +_DOMAIN = re.compile(r"^(?:[a-zA-Z0-9_-]+\.)+[a-zA-Z]{2,}$") + + +def detect_type(v): + v = v.strip() + if _IP.match(v): + return "ip" + if _URL.match(v): + return "url" + if _EMAIL.match(v): + return "email" + if _HASH.match(v): + return "hash" + if _DOMAIN.match(v): + return "domain" + return "unknown" + + +def _cfg(): + return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}")) + + +def _inputs(): + return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}")) + + +def _ctx(cfg): + if cfg.get("insecure"): + c = ssl.create_default_context() + c.check_hostname = False + c.verify_mode = ssl.CERT_NONE + return c + return None + + +def _headers(cfg): + h = {"User-Agent": "Riposte-SOAR", "Accept": "application/json"} + t = cfg.get("api_token") + if t: + h["Authorization"] = "Bearer " + str(t) + return h + + +def _get(url, cfg): + req = urllib.request.Request(url, headers=_headers(cfg)) + with urllib.request.urlopen(req, timeout=120, context=_ctx(cfg)) as r: + return r.read() + + +def _run(fn): + try: + print(json.dumps(fn(_cfg(), _inputs()))) + except urllib.error.HTTPError as e: + print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")})) + sys.exit(1) + except Exception as e: + print(json.dumps({"error": str(e)})) + sys.exit(1) + + +def _dig(obj, path): + if not path: + return obj + cur = obj + for part in path.split("."): + if isinstance(cur, dict): + cur = cur.get(part) + else: + return None + return cur + + +def main(cfg, inputs): + url = cfg.get("feed_url") + if not url: + raise Exception("feed_url is required") + + array_path = inputs.get("array_path") or cfg.get("array_path") or "" + value_field = inputs.get("value_field") or cfg.get("value_field") or "value" + type_field = inputs.get("type_field") or cfg.get("type_field") + ioc_type = str(inputs.get("ioc_type") or "auto").lower() + + raw = _get(url, cfg) + data = json.loads(raw) if raw else {} + + arr = _dig(data, array_path) if array_path else data + if isinstance(arr, dict): + arr = list(arr.values()) + if not isinstance(arr, list): + arr = [] + + maxn = int(inputs.get("max_indicators") or 0) + out = [] + for e in arr: + if isinstance(e, dict): + val = _dig(e, value_field) if "." in value_field else e.get(value_field) + t = e.get(type_field) if type_field else None + else: + val = e + t = None + if val in (None, ""): + continue + val = str(val).strip() + if not val: + continue + typ = str(t).lower() if t else (ioc_type if ioc_type != "auto" else detect_type(val)) + out.append({"value": val, "type": typ}) + if maxn and len(out) >= maxn: + break + + return {"source": url, "count": len(out), "indicators": out} + + +_run(main) diff --git a/integrations/feed-json/scripts/test_connection.py b/integrations/feed-json/scripts/test_connection.py new file mode 100644 index 0000000..78cc065 --- /dev/null +++ b/integrations/feed-json/scripts/test_connection.py @@ -0,0 +1,56 @@ +import json, os, sys, ssl, urllib.request, urllib.error + + +def _cfg(): + return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}")) + + +def _inputs(): + return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}")) + + +def _ctx(cfg): + if cfg.get("insecure"): + c = ssl.create_default_context() + c.check_hostname = False + c.verify_mode = ssl.CERT_NONE + return c + return None + + +def _headers(cfg): + h = {"User-Agent": "Riposte-SOAR", "Accept": "application/json"} + t = cfg.get("api_token") + if t: + h["Authorization"] = "Bearer " + str(t) + return h + + +def _get(url, cfg): + req = urllib.request.Request(url, headers=_headers(cfg)) + with urllib.request.urlopen(req, timeout=60, context=_ctx(cfg)) as r: + return r.read() + + +def _run(fn): + try: + print(json.dumps(fn(_cfg(), _inputs()))) + except urllib.error.HTTPError as e: + print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")})) + sys.exit(1) + except Exception as e: + print(json.dumps({"error": str(e)})) + sys.exit(1) + + +def main(cfg, inputs): + url = cfg.get("feed_url") + if not url: + raise Exception("feed_url is required") + raw = _get(url, cfg) + data = json.loads(raw) if raw else {} + kind = "array" if isinstance(data, list) else type(data).__name__ + return {"ok": True, "top_level": kind} + + +_run(main)