feat(feed): add generic JSON feed connector (configurable field mapping)
This commit is contained in:
Guillaume BOURGEOIS
@@ -0,0 +1,59 @@
|
||||
id: feed_json
|
||||
name: JSON Feed
|
||||
version: 1.0.0
|
||||
description: "Generic JSON threat-intel feed connector — fetch any JSON feed from a URL, dig into a configurable array path, pull a value field (and optional type field) and emit normalized IOCs (value + type) for import into the Threat Indicator Manager. One connector, many feeds. Optional bearer token; stdlib-only, no extra Python dependencies."
|
||||
changelog: "1.0.0 — Initial release: fetch indicators from a JSON feed with configurable field mapping."
|
||||
category: feed
|
||||
|
||||
# The feed is fetched over HTTP(S) from feed_url. array_path digs into nested
|
||||
# objects (dot notation) to reach the list of indicators; value_field/type_field
|
||||
# select the fields on each element.
|
||||
config_schema:
|
||||
properties:
|
||||
feed_url:
|
||||
type: string
|
||||
description: "URL of the JSON feed"
|
||||
api_token:
|
||||
type: string
|
||||
description: "Optional bearer token (if the feed requires auth)"
|
||||
x-soar-sensitive: true
|
||||
array_path:
|
||||
type: string
|
||||
description: "Dot path to the array of indicators (e.g. 'data.indicators'). Empty = the top-level value"
|
||||
value_field:
|
||||
type: string
|
||||
description: "Field holding the IOC value on each element (dot path allowed, default 'value')"
|
||||
default: "value"
|
||||
type_field:
|
||||
type: string
|
||||
description: "Optional field holding the IOC type on each element"
|
||||
insecure:
|
||||
type: boolean
|
||||
description: "Trust any TLS certificate (not secure)"
|
||||
default: false
|
||||
required:
|
||||
- feed_url
|
||||
|
||||
commands:
|
||||
- id: fetch_indicators
|
||||
name: feed-json-fetch-indicators
|
||||
description: "Fetch and parse the JSON feed, returning normalized indicators."
|
||||
risk: read
|
||||
inputs_schema:
|
||||
properties:
|
||||
array_path: { type: string, description: "Dot path to the array (overrides config)" }
|
||||
value_field: { type: string, description: "Value field / dot path (overrides config)" }
|
||||
type_field: { type: string, description: "Type field (overrides config)" }
|
||||
ioc_type: { type: string, description: "auto | ip | domain | url | hash | email — fallback type when no type_field (default auto)" }
|
||||
max_indicators: { type: number, description: "Max indicators to return (0 = no limit, default 0)" }
|
||||
required: []
|
||||
outputs_schema: { properties: {} }
|
||||
|
||||
- id: test_connection
|
||||
name: feed-json-test-connection
|
||||
description: "Verify the feed URL returns valid JSON (used by the Test button)."
|
||||
risk: read
|
||||
inputs_schema:
|
||||
properties: {}
|
||||
required: []
|
||||
outputs_schema: { properties: {} }
|
||||
@@ -0,0 +1,120 @@
|
||||
import json, os, sys, re, ssl, urllib.request, urllib.error
|
||||
|
||||
_IP = re.compile(r"^(?:\d{1,3}\.){3}\d{1,3}$")
|
||||
_HASH = re.compile(r"^[a-fA-F0-9]{32}$|^[a-fA-F0-9]{40}$|^[a-fA-F0-9]{64}$")
|
||||
_EMAIL = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
|
||||
_URL = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.\-]*://")
|
||||
_DOMAIN = re.compile(r"^(?:[a-zA-Z0-9_-]+\.)+[a-zA-Z]{2,}$")
|
||||
|
||||
|
||||
def detect_type(v):
|
||||
v = v.strip()
|
||||
if _IP.match(v):
|
||||
return "ip"
|
||||
if _URL.match(v):
|
||||
return "url"
|
||||
if _EMAIL.match(v):
|
||||
return "email"
|
||||
if _HASH.match(v):
|
||||
return "hash"
|
||||
if _DOMAIN.match(v):
|
||||
return "domain"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _cfg():
|
||||
return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}"))
|
||||
|
||||
|
||||
def _inputs():
|
||||
return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}"))
|
||||
|
||||
|
||||
def _ctx(cfg):
|
||||
if cfg.get("insecure"):
|
||||
c = ssl.create_default_context()
|
||||
c.check_hostname = False
|
||||
c.verify_mode = ssl.CERT_NONE
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def _headers(cfg):
|
||||
h = {"User-Agent": "Riposte-SOAR", "Accept": "application/json"}
|
||||
t = cfg.get("api_token")
|
||||
if t:
|
||||
h["Authorization"] = "Bearer " + str(t)
|
||||
return h
|
||||
|
||||
|
||||
def _get(url, cfg):
|
||||
req = urllib.request.Request(url, headers=_headers(cfg))
|
||||
with urllib.request.urlopen(req, timeout=120, context=_ctx(cfg)) as r:
|
||||
return r.read()
|
||||
|
||||
|
||||
def _run(fn):
|
||||
try:
|
||||
print(json.dumps(fn(_cfg(), _inputs())))
|
||||
except urllib.error.HTTPError as e:
|
||||
print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")}))
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": str(e)}))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _dig(obj, path):
|
||||
if not path:
|
||||
return obj
|
||||
cur = obj
|
||||
for part in path.split("."):
|
||||
if isinstance(cur, dict):
|
||||
cur = cur.get(part)
|
||||
else:
|
||||
return None
|
||||
return cur
|
||||
|
||||
|
||||
def main(cfg, inputs):
|
||||
url = cfg.get("feed_url")
|
||||
if not url:
|
||||
raise Exception("feed_url is required")
|
||||
|
||||
array_path = inputs.get("array_path") or cfg.get("array_path") or ""
|
||||
value_field = inputs.get("value_field") or cfg.get("value_field") or "value"
|
||||
type_field = inputs.get("type_field") or cfg.get("type_field")
|
||||
ioc_type = str(inputs.get("ioc_type") or "auto").lower()
|
||||
|
||||
raw = _get(url, cfg)
|
||||
data = json.loads(raw) if raw else {}
|
||||
|
||||
arr = _dig(data, array_path) if array_path else data
|
||||
if isinstance(arr, dict):
|
||||
arr = list(arr.values())
|
||||
if not isinstance(arr, list):
|
||||
arr = []
|
||||
|
||||
maxn = int(inputs.get("max_indicators") or 0)
|
||||
out = []
|
||||
for e in arr:
|
||||
if isinstance(e, dict):
|
||||
val = _dig(e, value_field) if "." in value_field else e.get(value_field)
|
||||
t = e.get(type_field) if type_field else None
|
||||
else:
|
||||
val = e
|
||||
t = None
|
||||
if val in (None, ""):
|
||||
continue
|
||||
val = str(val).strip()
|
||||
if not val:
|
||||
continue
|
||||
typ = str(t).lower() if t else (ioc_type if ioc_type != "auto" else detect_type(val))
|
||||
out.append({"value": val, "type": typ})
|
||||
if maxn and len(out) >= maxn:
|
||||
break
|
||||
|
||||
return {"source": url, "count": len(out), "indicators": out}
|
||||
|
||||
|
||||
_run(main)
|
||||
@@ -0,0 +1,56 @@
|
||||
import json, os, sys, ssl, urllib.request, urllib.error
|
||||
|
||||
|
||||
def _cfg():
|
||||
return json.loads(os.environ.get("INTEGRATION_SECRETS", "{}"))
|
||||
|
||||
|
||||
def _inputs():
|
||||
return json.loads(os.environ.get("INTEGRATION_INPUTS", "{}"))
|
||||
|
||||
|
||||
def _ctx(cfg):
|
||||
if cfg.get("insecure"):
|
||||
c = ssl.create_default_context()
|
||||
c.check_hostname = False
|
||||
c.verify_mode = ssl.CERT_NONE
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def _headers(cfg):
|
||||
h = {"User-Agent": "Riposte-SOAR", "Accept": "application/json"}
|
||||
t = cfg.get("api_token")
|
||||
if t:
|
||||
h["Authorization"] = "Bearer " + str(t)
|
||||
return h
|
||||
|
||||
|
||||
def _get(url, cfg):
|
||||
req = urllib.request.Request(url, headers=_headers(cfg))
|
||||
with urllib.request.urlopen(req, timeout=60, context=_ctx(cfg)) as r:
|
||||
return r.read()
|
||||
|
||||
|
||||
def _run(fn):
|
||||
try:
|
||||
print(json.dumps(fn(_cfg(), _inputs())))
|
||||
except urllib.error.HTTPError as e:
|
||||
print(json.dumps({"error": "HTTP " + str(e.code), "detail": e.read().decode("utf-8", "replace")}))
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": str(e)}))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def main(cfg, inputs):
|
||||
url = cfg.get("feed_url")
|
||||
if not url:
|
||||
raise Exception("feed_url is required")
|
||||
raw = _get(url, cfg)
|
||||
data = json.loads(raw) if raw else {}
|
||||
kind = "array" if isinstance(data, list) else type(data).__name__
|
||||
return {"ok": True, "top_level": kind}
|
||||
|
||||
|
||||
_run(main)
|
||||
Reference in New Issue
Block a user