commit 7336b17f792798622c07a943af81735d8746b127 Author: Steffen Skui Date: Wed Jun 24 18:17:39 2026 +0200 soc-collector: run-anywhere scanner that pushes reports to SOC Center A single Docker image, configured entirely by env vars (SOC_URL + SOC_TOKEN + targets). Runs once and exits; schedule with cron. Four self-skipping sources: - surface nmap open-port/service scan (SOC_TARGETS) - cve Trivy HIGH/CRITICAL scan of running images (Docker socket) - tls cert-expiry / weak-TLS / missing-headers (SOC_DOMAINS) - config 0.0.0.0 binds, --privileged, rw docker-socket mounts (Docker socket) Stable slugs per source so SOC Center versions/diffs/alerts. Pure stdlib; image adds nmap + trivy. MIT, intended as a public Ethica repo. diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c69348a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.git +.gitea +__pycache__/ +*.pyc +README.md +LICENSE diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml new file mode 100644 index 0000000..45b5023 --- /dev/null +++ b/.gitea/workflows/build.yml @@ -0,0 +1,49 @@ +# Build the soc-collector image and push it to the Gitea registry (public package). +# This is a run-anywhere tool — there is NO deploy step; users `docker run` the image. +# Reuses the Ethica org CI config: var REGISTRY_USER, secret REGISTRY_TOKEN. + +name: Build and push soc-collector + +on: + push: + branches: [main] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + IMAGE: git.skui.io/ethica/soc-collector + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Log in to the Gitea registry + run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.skui.io -u "${{ vars.REGISTRY_USER }}" --password-stdin + + - name: Determine version tag + id: version + run: | + echo "version=$(date -u +%Y.%m.%d).${{ github.run_number }}" >> "$GITHUB_OUTPUT" + + - name: Build (retry transient base-image pulls) + run: | + n=0 + until docker build --pull -t "$IMAGE:${{ steps.version.outputs.version }}" -t "$IMAGE:latest" .; do + n=$((n+1)); [ "$n" -ge 3 ] && { echo "build failed after $n attempts"; exit 1; } + echo "build attempt $n failed — retrying in $((n*15))s"; sleep $((n*15)) + done + + - name: Push (retry transient registry TLS timeouts) + run: | + for ref in "$IMAGE:${{ steps.version.outputs.version }}" "$IMAGE:latest"; do + n=0 + until docker push "$ref"; do + n=$((n+1)); [ "$n" -ge 3 ] && { echo "push of $ref failed after $n attempts"; exit 1; } + echo "push attempt $n failed (transient registry TLS timeout) — retrying in $((n*15))s"; sleep $((n*15)) + done + done diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..876107d --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.pyc +.env +.venv/ +venv/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..878a305 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.12-slim + +ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 \ + DOCKER_HOST=unix:///var/run/docker.sock + +# nmap for the attack-surface scan; trivy for container CVEs (it reaches the +# Docker daemon via DOCKER_HOST, so no docker CLI is needed). The collector +# itself is pure stdlib Python — no pip deps. +RUN apt-get update \ + && apt-get install -y --no-install-recommends nmap ca-certificates wget \ + && wget -qO- https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh \ + | sh -s -- -b /usr/local/bin \ + && apt-get purge -y wget && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY . . + +ENTRYPOINT ["python", "collector.py"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7e5f3cd --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Ethica + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0eb844a --- /dev/null +++ b/README.md @@ -0,0 +1,67 @@ +# soc-collector + +A single, run-anywhere container that scans your estate and pushes the results to a +[SOC Center](https://soc.ethica.no) instance as auto-updating reports. No JSON to write, +no curl — you give it an **API endpoint** and a **token**, point it at some targets, and +run it. Same report slug every run, so SOC Center versions each scan, diffs what changed, +and alerts you only on *new* findings. + +## Run it + +```bash +docker run --rm \ + -e SOC_URL=https://soc.ethica.no \ + -e SOC_TOKEN=soc_xxxxxxxx \ + -e SOC_TARGETS=10.0.0.5,10.0.0.6 \ + -e SOC_DOMAINS=soc.ethica.no,ethica.no \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ + git.skui.io/ethica/soc-collector:latest +``` + +It runs once and exits. To keep reports fresh, schedule that command — e.g. nightly: + +```cron +0 2 * * * docker run --rm -e SOC_URL=… -e SOC_TOKEN=… -e SOC_TARGETS=… -e SOC_DOMAINS=… \ + -v /var/run/docker.sock:/var/run/docker.sock:ro git.skui.io/ethica/soc-collector:latest +``` + +The `SOC_TOKEN` is a **write-scoped** token you mint once in SOC Center (**Admin → Tokens**). +Reports are owned by, and visible per, that token's user. + +## What it collects + +| Report (slug) | Source | Needs | +|---|---|---| +| `attack-surface` | `nmap` open-port / service scan, flags risky exposed ports | `SOC_TARGETS` | +| `container-cves` | `trivy` vuln scan of every **running** image (HIGH/CRITICAL) | Docker socket | +| `tls-exposure` | cert-expiry, weak TLS, missing security headers | `SOC_DOMAINS` | +| `config-drift` | `0.0.0.0` port binds, `--privileged`, rw docker-socket mounts | Docker socket | + +Each source **self-skips** when its inputs are absent — no `SOC_TARGETS` → no nmap; no +socket mount → no Trivy / config-drift. So the same image is safe to run anywhere; you +just enable what you give it. + +## Configuration (all via env) + +| Var | Required | Default | Meaning | +|---|---|---|---| +| `SOC_URL` | ✅ | — | SOC Center base URL, e.g. `https://soc.ethica.no` | +| `SOC_TOKEN` | ✅ | — | write-scoped API token | +| `SOC_TARGETS` | for nmap | — | comma/space hosts or IPs to scan | +| `SOC_DOMAINS` | for tls | — | comma hostnames to TLS-probe | +| `SOC_SOURCES` | | `surface,cve,tls,config` | which sources to run | +| `SOC_SLUG_PREFIX` | | — | prefix slugs (e.g. `hetzner`) so multiple sites don't collide | +| `SOC_VISIBILITY` | | `private` | `private` or `authenticated` | +| `SOC_TAGS` | | `collector` | extra report tags | + +## Notes + +- **Docker socket** is mounted **read-only** (`:ro`) and is only used to *list* running + containers and *inspect* their config. For extra isolation, point it at a scoped + socket-proxy instead of the raw socket. +- The collector is **pure-stdlib Python**; the image just adds `nmap` and `trivy`. +- Multi-site: run one container per site with a distinct `SOC_SLUG_PREFIX`. + +## License + +MIT — see [LICENSE](LICENSE). diff --git a/collector.py b/collector.py new file mode 100644 index 0000000..2cbc98d --- /dev/null +++ b/collector.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""soc-collector — scan an estate and push reports to SOC Center. + +Runs once and exits (schedule it with cron / a systemd timer / a CI job). Each +enabled source becomes one report, re-ingested under a stable slug so SOC Center +versions it, diffs each run, and alerts only on new findings. + +Configuration is entirely via env vars — see the README. Required: SOC_URL, SOC_TOKEN. +A source self-skips when its inputs are absent (no SOC_TARGETS -> no nmap; no Docker +socket -> no Trivy / config-drift), so the same image is safe to run anywhere. +""" +import os +import sys + +import soc +from sources import cve, config_drift, surface, tls + + +def _list(name): + return [x.strip() for x in os.environ.get(name, "").replace(";", ",").split(",") if x.strip()] + + +def main(): + url = soc._env("SOC_URL", required=True) + token = soc._env("SOC_TOKEN", required=True) + prefix = os.environ.get("SOC_SLUG_PREFIX", "").strip() + visibility = os.environ.get("SOC_VISIBILITY", "private") + base_tags = _list("SOC_TAGS") or ["collector"] + want = set(_list("SOC_SOURCES") or ["surface", "cve", "tls", "config"]) + targets, domains = _list("SOC_TARGETS"), _list("SOC_DOMAINS") + + def slug(s): + return f"{prefix}-{s}" if prefix else s + + plan = [] + if "surface" in want: + plan.append(("surface", "Attack surface (nmap)", slug("attack-surface"), lambda: surface.run(targets))) + if "cve" in want: + plan.append(("cve", "Container CVEs (Trivy)", slug("container-cves"), cve.run)) + if "tls" in want: + plan.append(("tls", "TLS & web exposure", slug("tls-exposure"), lambda: tls.run(domains))) + if "config" in want: + plan.append(("config", "Config drift (Docker)", slug("config-drift"), config_drift.run)) + + print(f"[soc-collector] target={url} sources={sorted(want)}") + rc = 0 + for key, title, rslug, fn in plan: + print(f"[soc-collector] source: {key}") + try: + data = fn() + except Exception as e: + print(f" ! {key} errored: {e}") + rc = 1 + continue + if data is None: + print(f" - {key} skipped (no targets / no Docker socket)") + continue + n = len(data.get("findings", [])) + status, resp = soc.post_report(url, token, rslug, title, data, + visibility=visibility, tags=base_tags + [key]) + if status in (200, 201): + d = resp.get("diff", {}) + print(f" ✓ {rslug}: {n} finding(s), {'created' if resp.get('created') else 'updated'} " + f"(+{d.get('added', 0)}/-{d.get('removed', 0)}/~{d.get('changed', 0)}) " + f"alerted={resp.get('alerted')}") + else: + print(f" ! {rslug}: HTTP {status} {resp}") + rc = 1 + + sys.exit(rc) + + +if __name__ == "__main__": + main() diff --git a/soc.py b/soc.py new file mode 100644 index 0000000..d8a69a8 --- /dev/null +++ b/soc.py @@ -0,0 +1,40 @@ +"""Minimal SOC Center API client + report helpers (stdlib only).""" +import datetime +import json +import os +import sys +import urllib.error +import urllib.request + + +def _env(name, default=None, required=False): + v = os.environ.get(name, default) + if required and not v: + sys.exit(f"[soc-collector] missing required env var: {name}") + return v + + +def today(): + return datetime.datetime.utcnow().strftime("%Y-%m-%d") + + +def finding(fid, sev, title, host="", evidence="", impact=None, fix=None): + """One finding in the SOC schema. sev in crit|high|med|low.""" + return {"id": fid, "sev": sev, "title": title, "host": host, + "evidence": evidence, "impact": impact or [], "fix": fix or []} + + +def post_report(url, token, slug, title, data, visibility="private", tags=None, date=None): + """POST a report to SOC Center. Returns (http_status, json_body).""" + body = {"slug": slug, "title": title, "date": date or today(), + "visibility": visibility, "tags": tags or [], "data": data} + req = urllib.request.Request( + url.rstrip("/") + "/api/reports", data=json.dumps(body).encode(), method="POST", + headers={"Authorization": "Bearer " + token, "Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=60) as r: + return r.status, json.loads(r.read().decode() or "{}") + except urllib.error.HTTPError as e: + return e.code, {"error": e.read().decode()} + except Exception as e: # network / DNS / timeout + return 0, {"error": str(e)} diff --git a/sources/__init__.py b/sources/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sources/_docker.py b/sources/_docker.py new file mode 100644 index 0000000..fd441c1 --- /dev/null +++ b/sources/_docker.py @@ -0,0 +1,35 @@ +"""Tiny read-only Docker Engine API client over the unix socket (stdlib only). + +Used by the cve and config-drift sources. HTTP/1.0 so the daemon sends the whole +body then closes the connection (no chunked decoding needed). +""" +import json +import socket + +SOCK = "/var/run/docker.sock" + + +def available(): + try: + s = socket.socket(socket.AF_UNIX) + s.connect(SOCK) + s.close() + return True + except OSError: + return False + + +def get(path): + s = socket.socket(socket.AF_UNIX) + s.settimeout(30) + s.connect(SOCK) + s.sendall(f"GET {path} HTTP/1.0\r\nHost: docker\r\n\r\n".encode()) + buf = b"" + while True: + chunk = s.recv(65536) + if not chunk: + break + buf += chunk + s.close() + body = buf.split(b"\r\n\r\n", 1)[1] if b"\r\n\r\n" in buf else b"" + return json.loads(body or b"null") diff --git a/sources/config_drift.py b/sources/config_drift.py new file mode 100644 index 0000000..7d1adab --- /dev/null +++ b/sources/config_drift.py @@ -0,0 +1,54 @@ +"""Config-drift source: read-only Docker socket -> 0.0.0.0 binds, privileged, rw-socket mounts.""" +from soc import finding + +from . import _docker + + +def run(): + if not _docker.available(): + return None # socket not mounted -> skip + conts = _docker.get("/containers/json") or [] + + findings = [] + for c in conts: + name = (c.get("Names") or ["?"])[0].lstrip("/") + cid = c.get("Id", "")[:12] + try: + d = _docker.get(f"/containers/{cid}/json") + except Exception: + continue + hc = d.get("HostConfig", {}) or {} + + # Ports published on all interfaces (0.0.0.0 / ::) — i.e. exposed off-host. + exposed = [] + for port, binds in ((d.get("NetworkSettings", {}) or {}).get("Ports") or {}).items(): + for b in (binds or []): + if b.get("HostIp") in ("0.0.0.0", "::", ""): + exposed.append(f"{b.get('HostIp') or '0.0.0.0'}:{b.get('HostPort')}->{port}") + if exposed: + findings.append(finding( + f"drift-bind-{name}", "high", f"{name} publishes ports on all interfaces", + host=name, evidence="; ".join(exposed), + impact=["reachable from outside the host, bypassing the reverse proxy"], + fix=["bind to 127.0.0.1, or drop host ports and route via the proxy"])) + + if hc.get("Privileged"): + findings.append(finding( + f"drift-priv-{name}", "high", f"{name} runs --privileged", + host=name, evidence="Privileged=true", + impact=["a container escape becomes full host root"], + fix=["drop --privileged; grant only the specific capabilities needed"])) + + for m in (d.get("Mounts") or []): + if m.get("Source") == "/var/run/docker.sock" and m.get("RW"): + findings.append(finding( + f"drift-sock-{name}", "high", f"{name} mounts the Docker socket read-write", + host=name, evidence="/var/run/docker.sock (rw)", + impact=["the container can control the host's Docker = host root"], + fix=["mount it :ro, or put a scoped socket-proxy in front"])) + + return { + "scope": f"{len(conts)} container(s)", + "findings": findings, + "good": [] if findings else ["No 0.0.0.0 binds, privileged, or rw-socket containers"], + } diff --git a/sources/cve.py b/sources/cve.py new file mode 100644 index 0000000..58a5b19 --- /dev/null +++ b/sources/cve.py @@ -0,0 +1,54 @@ +"""Container-CVE source: Trivy-scan the images of running containers (HIGH/CRITICAL).""" +import json +import subprocess + +from soc import finding + +from . import _docker + +SEV = {"CRITICAL": "crit", "HIGH": "high", "MEDIUM": "med", "LOW": "low"} + + +def _running_images(): + conts = _docker.get("/containers/json") or [] + seen = [] + for c in conts: + img = c.get("Image", "") + if img and img not in seen: + seen.append(img) + return seen + + +def run(): + if not _docker.available(): + return None # socket not mounted -> skip + images = _running_images() + if not images: + return None + + findings = [] + for img in images: + try: + proc = subprocess.run( + ["trivy", "image", "--quiet", "--scanners", "vuln", + "--severity", "HIGH,CRITICAL", "--format", "json", img], + capture_output=True, text=True, timeout=900) + rep = json.loads(proc.stdout or "{}") + except Exception: + continue + vulns = [v for res in (rep.get("Results") or []) for v in (res.get("Vulnerabilities") or [])] + if not vulns: + continue + worst = "crit" if any(v.get("Severity") == "CRITICAL" for v in vulns) else "high" + ids = sorted({v.get("VulnerabilityID", "") for v in vulns if v.get("VulnerabilityID")}) + findings.append(finding( + f"cve-{img}", worst, f"{len(vulns)} HIGH/CRITICAL CVEs in {img}", host=img, + evidence=", ".join(ids[:10]) + (" …" if len(ids) > 10 else ""), + impact=["known-exploitable vulnerabilities in a running image"], + fix=[f"pull/rebuild a patched {img.split(':')[0]} image"])) + + return { + "scope": f"{len(images)} running image(s)", + "findings": findings, + "good": [] if findings else ["No HIGH/CRITICAL CVEs in running images"], + } diff --git a/sources/surface.py b/sources/surface.py new file mode 100644 index 0000000..3aa2c86 --- /dev/null +++ b/sources/surface.py @@ -0,0 +1,60 @@ +"""Attack-surface source: nmap the targets -> SOC surface[] + findings for risky open ports.""" +import subprocess +import xml.etree.ElementTree as ET + +from soc import finding + +# Ports / service-names that usually should NOT be world-reachable -> severity. +RISKY_PORT = { + "23": "high", "2375": "crit", "2376": "high", "3389": "high", "5432": "high", + "3306": "high", "6379": "high", "27017": "high", "9200": "high", "5984": "high", + "11211": "high", "2049": "high", "445": "high", "139": "med", "135": "med", + "5900": "med", "9000": "med", "8006": "high", "61616": "high", +} +RISKY_NAME = { + "telnet": "high", "ms-wbt-server": "high", "postgresql": "high", "mysql": "high", + "redis": "high", "mongodb": "high", "vnc": "med", "docker": "crit", "memcached": "high", + "elasticsearch": "high", "rdp": "high", +} + + +def run(targets): + if not targets: + return None + proc = subprocess.run( + ["nmap", "-Pn", "-sV", "-T4", "--open", "-oX", "-"] + targets, + capture_output=True, text=True, timeout=1800) + if not proc.stdout.strip(): + raise RuntimeError(proc.stderr.strip() or "nmap produced no output") + root = ET.fromstring(proc.stdout) + + surface, findings = [], [] + for host in root.findall("host"): + addr = host.find("address") + ip = addr.get("addr") if addr is not None else "?" + open_ports = [] + for p in host.findall("./ports/port"): + st = p.find("state") + if st is None or st.get("state") != "open": + continue + num = p.get("portid") + svc = p.find("service") + name = svc.get("name") if svc is not None else "" + open_ports.append(num) + sev = RISKY_PORT.get(num) or RISKY_NAME.get(name) + if sev: + findings.append(finding( + f"surface-{ip}-{num}", sev, + f"Exposed {name or 'service'} on port {num}", host=ip, + evidence=f"{ip}:{num} ({name or 'unknown'})", + impact=["reachable over the network"], + fix=[f"restrict port {num} to trusted networks, bind to localhost, " + "or front it with the reverse proxy"])) + surface.append({"ip": ip, "role": "", "ports": ",".join(open_ports)}) + + return { + "scope": "nmap " + " ".join(targets), + "surface": surface, + "findings": findings, + "good": [] if findings else ["No risky ports found open on the scanned targets"], + } diff --git a/sources/tls.py b/sources/tls.py new file mode 100644 index 0000000..a2e187d --- /dev/null +++ b/sources/tls.py @@ -0,0 +1,76 @@ +"""TLS & web-exposure source: cert expiry, weak TLS, missing security headers per domain.""" +import datetime +import http.client +import socket +import ssl + +from soc import finding + +SECURITY_HEADERS = ["strict-transport-security", "content-security-policy", + "x-content-type-options", "x-frame-options"] + + +def _peer(host, port=443): + ctx = ssl.create_default_context() + with socket.create_connection((host, port), timeout=15) as sock: + with ctx.wrap_socket(sock, server_hostname=host) as ss: + return ss.getpeercert(), ss.version() + + +def run(domains): + if not domains: + return None + findings, good = [], [] + + for host in domains: + host = host.strip() + if not host: + continue + try: + cert, ver = _peer(host) + except Exception as e: + findings.append(finding( + f"tls-conn-{host}", "med", f"TLS connection to {host} failed", host=host, + evidence=str(e), impact=["the site may be down or its TLS misconfigured"], + fix=["check the service and its certificate"])) + continue + + not_after = cert.get("notAfter") + try: + exp = datetime.datetime.strptime(not_after, "%b %d %H:%M:%S %Y %Z") + days = (exp - datetime.datetime.utcnow()).days + sev = "crit" if days < 7 else "high" if days < 21 else "med" if days < 40 else None + if sev: + findings.append(finding( + f"tls-exp-{host}", sev, f"Certificate for {host} expires in {days} days", + host=host, evidence=f"notAfter={not_after}", + impact=["an outage the moment it expires"], + fix=["renew it / verify auto-renewal (e.g. Traefik 'le' resolver)"])) + else: + good.append(f"{host}: certificate valid for {days} more days") + except (ValueError, TypeError): + pass + + if ver in ("TLSv1", "TLSv1.1"): + findings.append(finding( + f"tls-weak-{host}", "med", f"{host} negotiated deprecated {ver}", host=host, + evidence=ver, impact=["deprecated, downgradeable TLS"], + fix=["disable TLS < 1.2 at the edge"])) + + try: + conn = http.client.HTTPSConnection(host, timeout=15) + conn.request("HEAD", "/") + resp = conn.getresponse() + present = {k.lower() for k, _ in resp.getheaders()} + conn.close() + missing = [h for h in SECURITY_HEADERS if h not in present] + if missing: + findings.append(finding( + f"hdr-{host}", "low", f"{host} is missing {len(missing)} security header(s)", + host=host, evidence=", ".join(missing), + impact=["weaker browser-side protections"], + fix=["add at the edge/app: " + ", ".join(missing)])) + except Exception: + pass + + return {"scope": ", ".join(domains), "findings": findings, "good": good}