feat(feed): add generic JSON feed connector (configurable field mapping)

integrations/feed-json/manifest.yaml

@@ -0,0 +1,59 @@
+id: feed_json
+name: JSON Feed
+version: 1.0.0
+description: "Generic JSON threat-intel feed connector — fetch any JSON feed from a URL, dig into a configurable array path, pull a value field (and optional type field) and emit normalized IOCs (value + type) for import into the Threat Indicator Manager. One connector, many feeds. Optional bearer token; stdlib-only, no extra Python dependencies."
+changelog: "1.0.0 — Initial release: fetch indicators from a JSON feed with configurable field mapping."
+category: feed
+
+# The feed is fetched over HTTP(S) from feed_url. array_path digs into nested
+# objects (dot notation) to reach the list of indicators; value_field/type_field
+# select the fields on each element.
+config_schema:
+  properties:
+    feed_url:
+      type: string
+      description: "URL of the JSON feed"
+    api_token:
+      type: string
+      description: "Optional bearer token (if the feed requires auth)"
+      x-soar-sensitive: true
+    array_path:
+      type: string
+      description: "Dot path to the array of indicators (e.g. 'data.indicators'). Empty = the top-level value"
+    value_field:
+      type: string
+      description: "Field holding the IOC value on each element (dot path allowed, default 'value')"
+      default: "value"
+    type_field:
+      type: string
+      description: "Optional field holding the IOC type on each element"
+    insecure:
+      type: boolean
+      description: "Trust any TLS certificate (not secure)"
+      default: false
+  required:
+    - feed_url
+
+commands:
+  - id: fetch_indicators
+    name: feed-json-fetch-indicators
+    description: "Fetch and parse the JSON feed, returning normalized indicators."
+    risk: read
+    inputs_schema:
+      properties:
+        array_path: { type: string, description: "Dot path to the array (overrides config)" }
+        value_field: { type: string, description: "Value field / dot path (overrides config)" }
+        type_field: { type: string, description: "Type field (overrides config)" }
+        ioc_type: { type: string, description: "auto | ip | domain | url | hash | email — fallback type when no type_field (default auto)" }
+        max_indicators: { type: number, description: "Max indicators to return (0 = no limit, default 0)" }
+      required: []
+    outputs_schema: { properties: {} }
+
+  - id: test_connection
+    name: feed-json-test-connection
+    description: "Verify the feed URL returns valid JSON (used by the Test button)."
+    risk: read
+    inputs_schema:
+      properties: {}
+      required: []
+    outputs_schema: { properties: {} }

integrations/feed-json/scripts/fetch_indicators.py

@@ -0,0 +1,120 @@
+import json, os, sys, re, ssl, urllib.request, urllib.error
+
+_IP = re.compile(r"^(?:\d{1,3}\.){3}\d{1,3}$")
+_HASH = re.compile(r"^[a-fA-F0-9]{32}$|^[a-fA-F0-9]{40}$|^[a-fA-F0-9]{64}$")
+_EMAIL = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
+_URL = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.\-]*://")
+_DOMAIN = re.compile(r"^(?:[a-zA-Z0-9_-]+\.)+[a-zA-Z]{2,}$")
+
+
+def detect_type(v):
+    v = v.strip()
+    if _IP.match(v):
+        return "ip"
+    if _URL.match(v):
+        return "url"
+    if _EMAIL.match(v):
+        return "email"
+    if _HASH.match(v):
+        return "hash"
+    if _DOMAIN.match(v):
+        return "domain"
+    return "unknown"
+
+
+def _cfg():
+    return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}"))
+
+
+def _inputs():
+    return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}"))
+
+
+def _ctx(cfg):
+    if cfg.get("insecure"):
+        c = ssl.create_default_context()
+        c.check_hostname = False
+        c.verify_mode = ssl.CERT_NONE
+        return c
+    return None
+
+
+def _headers(cfg):
+    h = {"User-Agent": "Riposte-SOAR", "Accept": "application/json"}
+    t = cfg.get("api_token")
+    if t:
+        h["Authorization"] = "Bearer " + str(t)
+    return h
+
+
+def _get(url, cfg):
+    req = urllib.request.Request(url, headers=_headers(cfg))
+    with urllib.request.urlopen(req, timeout=120, context=_ctx(cfg)) as r:
+        return r.read()
+
+
+def _run(fn):
+    try:
+        print(json.dumps(fn(_cfg(), _inputs())))
+    except urllib.error.HTTPError as e:
+        print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")}))
+        sys.exit(1)
+    except Exception as e:
+        print(json.dumps({"error": str(e)}))
+        sys.exit(1)
+
+
+def _dig(obj, path):
+    if not path:
+        return obj
+    cur = obj
+    for part in path.split("."):
+        if isinstance(cur, dict):
+            cur = cur.get(part)
+        else:
+            return None
+    return cur
+
+
+def main(cfg, inputs):
+    url = cfg.get("feed_url")
+    if not url:
+        raise Exception("feed_url is required")
+
+    array_path = inputs.get("array_path") or cfg.get("array_path") or ""
+    value_field = inputs.get("value_field") or cfg.get("value_field") or "value"
+    type_field = inputs.get("type_field") or cfg.get("type_field")
+    ioc_type = str(inputs.get("ioc_type") or "auto").lower()
+
+    raw = _get(url, cfg)
+    data = json.loads(raw) if raw else {}
+
+    arr = _dig(data, array_path) if array_path else data
+    if isinstance(arr, dict):
+        arr = list(arr.values())
+    if not isinstance(arr, list):
+        arr = []
+
+    maxn = int(inputs.get("max_indicators") or 0)
+    out = []
+    for e in arr:
+        if isinstance(e, dict):
+            val = _dig(e, value_field) if "." in value_field else e.get(value_field)
+            t = e.get(type_field) if type_field else None
+        else:
+            val = e
+            t = None
+        if val in (None, ""):
+            continue
+        val = str(val).strip()
+        if not val:
+            continue
+        typ = str(t).lower() if t else (ioc_type if ioc_type != "auto" else detect_type(val))
+        out.append({"value": val, "type": typ})
+        if maxn and len(out) >= maxn:
+            break
+
+    return {"source": url, "count": len(out), "indicators": out}
+
+
+_run(main)

integrations/feed-json/scripts/test_connection.py

@@ -0,0 +1,56 @@
+import json, os, sys, ssl, urllib.request, urllib.error
+
+
+def _cfg():
+    return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}"))
+
+
+def _inputs():
+    return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}"))
+
+
+def _ctx(cfg):
+    if cfg.get("insecure"):
+        c = ssl.create_default_context()
+        c.check_hostname = False
+        c.verify_mode = ssl.CERT_NONE
+        return c
+    return None
+
+
+def _headers(cfg):
+    h = {"User-Agent": "Riposte-SOAR", "Accept": "application/json"}
+    t = cfg.get("api_token")
+    if t:
+        h["Authorization"] = "Bearer " + str(t)
+    return h
+
+
+def _get(url, cfg):
+    req = urllib.request.Request(url, headers=_headers(cfg))
+    with urllib.request.urlopen(req, timeout=60, context=_ctx(cfg)) as r:
+        return r.read()
+
+
+def _run(fn):
+    try:
+        print(json.dumps(fn(_cfg(), _inputs())))
+    except urllib.error.HTTPError as e:
+        print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")}))
+        sys.exit(1)
+    except Exception as e:
+        print(json.dumps({"error": str(e)}))
+        sys.exit(1)
+
+
+def main(cfg, inputs):
+    url = cfg.get("feed_url")
+    if not url:
+        raise Exception("feed_url is required")
+    raw = _get(url, cfg)
+    data = json.loads(raw) if raw else {}
+    kind = "array" if isinstance(data, list) else type(data).__name__
+    return {"ok": True, "top_level": kind}
+
+
+_run(main)