From 67eaf409b98edd961469196453224b2405eddcf8 Mon Sep 17 00:00:00 2001 From: Steffen Skui Date: Wed, 24 Jun 2026 18:42:38 +0200 Subject: [PATCH] collector: emit ONE consolidated report per run (SOC_SLUG), not one per source All enabled sources merge into a single report keyed by SOC_SLUG, criticals first. One server = one report; run per server with a distinct SOC_SLUG. Avoids the 4-reports-per-host explosion when scanning several servers. README updated. --- README.md | 26 +++++++++++------ collector.py | 80 +++++++++++++++++++++++++++++++--------------------- 2 files changed, 65 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 0eb844a..e2b213e 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,16 @@ and alerts you only on *new* findings. docker run --rm \ -e SOC_URL=https://soc.ethica.no \ -e SOC_TOKEN=soc_xxxxxxxx \ + -e SOC_SLUG=my-server \ -e SOC_TARGETS=10.0.0.5,10.0.0.6 \ -e SOC_DOMAINS=soc.ethica.no,ethica.no \ -v /var/run/docker.sock:/var/run/docker.sock:ro \ git.skui.io/ethica/soc-collector:latest ``` +Each run produces **one report** (named by `SOC_SLUG`) merging all the sources below. +One server = one report; run it per server with a distinct `SOC_SLUG` for one report each. + It runs once and exits. To keep reports fresh, schedule that command — e.g. nightly: ```cron @@ -30,16 +34,19 @@ Reports are owned by, and visible per, that token's user. ## What it collects -| Report (slug) | Source | Needs | +One run = **one consolidated report** (slug = `SOC_SLUG`) merging whichever of these +sources have their inputs, criticals first: + +| Source | What it checks | Needs | |---|---|---| -| `attack-surface` | `nmap` open-port / service scan, flags risky exposed ports | `SOC_TARGETS` | -| `container-cves` | `trivy` vuln scan of every **running** image (HIGH/CRITICAL) | Docker socket | -| `tls-exposure` | cert-expiry, weak TLS, missing security headers | `SOC_DOMAINS` | -| `config-drift` | `0.0.0.0` port binds, `--privileged`, rw docker-socket mounts | Docker socket | +| surface | `nmap` open-port / service scan, flags risky exposed ports | `SOC_TARGETS` | +| cve | `trivy` of every **running** image — **fixable** HIGH/CRITICAL only, grouped by package | Docker socket | +| tls | cert-expiry, weak TLS, missing security headers | `SOC_DOMAINS` | +| config | `0.0.0.0` port binds, `--privileged`, rw docker-socket mounts | Docker socket | Each source **self-skips** when its inputs are absent — no `SOC_TARGETS` → no nmap; no -socket mount → no Trivy / config-drift. So the same image is safe to run anywhere; you -just enable what you give it. +socket mount → no Trivy / config. So the same image is safe to run anywhere; you just +enable what you give it. ## Configuration (all via env) @@ -50,7 +57,8 @@ just enable what you give it. | `SOC_TARGETS` | for nmap | — | comma/space hosts or IPs to scan | | `SOC_DOMAINS` | for tls | — | comma hostnames to TLS-probe | | `SOC_SOURCES` | | `surface,cve,tls,config` | which sources to run | -| `SOC_SLUG_PREFIX` | | — | prefix slugs (e.g. `hetzner`) so multiple sites don't collide | +| `SOC_SLUG` | | `soc-scan` | the report's slug — **set one per server** (e.g. `hetzner-prod`) | +| `SOC_TITLE` | | `Security scan — ` | the report's title | | `SOC_VISIBILITY` | | `private` | `private` or `authenticated` | | `SOC_TAGS` | | `collector` | extra report tags | @@ -60,7 +68,7 @@ just enable what you give it. containers and *inspect* their config. For extra isolation, point it at a scoped socket-proxy instead of the raw socket. - The collector is **pure-stdlib Python**; the image just adds `nmap` and `trivy`. -- Multi-site: run one container per site with a distinct `SOC_SLUG_PREFIX`. +- Multi-server: run one container per server with a distinct `SOC_SLUG` — one report each. ## License diff --git a/collector.py b/collector.py index 2cbc98d..4894c2e 100644 --- a/collector.py +++ b/collector.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 -"""soc-collector — scan an estate and push reports to SOC Center. +"""soc-collector — scan an estate and push ONE consolidated report to SOC Center. -Runs once and exits (schedule it with cron / a systemd timer / a CI job). Each -enabled source becomes one report, re-ingested under a stable slug so SOC Center -versions it, diffs each run, and alerts only on new findings. +Each run produces a single report (all enabled sources merged), keyed by SOC_SLUG — +so one server = one report. Run it per server with a distinct SOC_SLUG and you get one +report each, not one-per-source-per-server. Re-running the same slug lets SOC Center +version it, diff each run, and alert only on new findings. -Configuration is entirely via env vars — see the README. Required: SOC_URL, SOC_TOKEN. -A source self-skips when its inputs are absent (no SOC_TARGETS -> no nmap; no Docker -socket -> no Trivy / config-drift), so the same image is safe to run anywhere. +Runs once and exits (schedule with cron / a systemd timer). Configuration is entirely +via env vars — see the README. Required: SOC_URL, SOC_TOKEN. A source self-skips when +its inputs are absent (no SOC_TARGETS -> no nmap; no Docker socket -> no Trivy / config). """ import os import sys @@ -15,6 +16,8 @@ import sys import soc from sources import cve, config_drift, surface, tls +SEV_ORDER = {"crit": 0, "high": 1, "med": 2, "low": 3} + def _list(name): return [x.strip() for x in os.environ.get(name, "").replace(";", ",").split(",") if x.strip()] @@ -23,51 +26,64 @@ def _list(name): def main(): url = soc._env("SOC_URL", required=True) token = soc._env("SOC_TOKEN", required=True) - prefix = os.environ.get("SOC_SLUG_PREFIX", "").strip() visibility = os.environ.get("SOC_VISIBILITY", "private") base_tags = _list("SOC_TAGS") or ["collector"] want = set(_list("SOC_SOURCES") or ["surface", "cve", "tls", "config"]) targets, domains = _list("SOC_TARGETS"), _list("SOC_DOMAINS") + # One report per run; the slug names this server. Fall back to the old prefix var. + slug = (os.environ.get("SOC_SLUG") or os.environ.get("SOC_SLUG_PREFIX") or "soc-scan").strip() + title = os.environ.get("SOC_TITLE") or f"Security scan — {slug}" - def slug(s): - return f"{prefix}-{s}" if prefix else s - - plan = [] + runners = [] if "surface" in want: - plan.append(("surface", "Attack surface (nmap)", slug("attack-surface"), lambda: surface.run(targets))) + runners.append(("surface", lambda: surface.run(targets))) if "cve" in want: - plan.append(("cve", "Container CVEs (Trivy)", slug("container-cves"), cve.run)) + runners.append(("cve", cve.run)) if "tls" in want: - plan.append(("tls", "TLS & web exposure", slug("tls-exposure"), lambda: tls.run(domains))) + runners.append(("tls", lambda: tls.run(domains))) if "config" in want: - plan.append(("config", "Config drift (Docker)", slug("config-drift"), config_drift.run)) + runners.append(("config", config_drift.run)) - print(f"[soc-collector] target={url} sources={sorted(want)}") - rc = 0 - for key, title, rslug, fn in plan: + print(f"[soc-collector] target={url} slug={slug} sources={sorted(want)}") + findings, surface_rows, good, scope, ran = [], [], [], [], [] + for key, fn in runners: print(f"[soc-collector] source: {key}") try: data = fn() except Exception as e: print(f" ! {key} errored: {e}") - rc = 1 continue if data is None: print(f" - {key} skipped (no targets / no Docker socket)") continue - n = len(data.get("findings", [])) - status, resp = soc.post_report(url, token, rslug, title, data, - visibility=visibility, tags=base_tags + [key]) - if status in (200, 201): - d = resp.get("diff", {}) - print(f" ✓ {rslug}: {n} finding(s), {'created' if resp.get('created') else 'updated'} " - f"(+{d.get('added', 0)}/-{d.get('removed', 0)}/~{d.get('changed', 0)}) " - f"alerted={resp.get('alerted')}") - else: - print(f" ! {rslug}: HTTP {status} {resp}") - rc = 1 + ran.append(key) + n = data.get("findings", []) + findings += n + surface_rows += data.get("surface", []) + good += data.get("good", []) + if data.get("scope"): + scope.append(f"{key}: {data['scope']}") + print(f" ✓ {key}: {len(n)} finding(s)") - sys.exit(rc) + if not ran: + print("[soc-collector] no sources ran (need SOC_TARGETS / SOC_DOMAINS / a Docker socket)") + sys.exit(1) + + findings.sort(key=lambda f: SEV_ORDER.get(f.get("sev"), 9)) # criticals first + data = {"scope": " · ".join(scope), "findings": findings, + "surface": surface_rows, "good": good} + + status, resp = soc.post_report(url, token, slug, title, data, + visibility=visibility, tags=base_tags + ran) + if status in (200, 201): + d = resp.get("diff", {}) + print(f"[soc-collector] ✓ {slug}: {len(findings)} finding(s) from {ran}, " + f"{'created' if resp.get('created') else 'updated'} " + f"(+{d.get('added', 0)}/-{d.get('removed', 0)}/~{d.get('changed', 0)}) " + f"alerted={resp.get('alerted')}") + sys.exit(0) + print(f"[soc-collector] ! {slug}: HTTP {status} {resp}") + sys.exit(1) if __name__ == "__main__":