"""Command line: shadow-ai-report LOGFILE [LOGFILE...] -o report.html""" import argparse import os import sys from . import FREE_EDITION_NOTE, FREE_LIST_SIZE, __version__ from .parse import FORMATS, InputError from .report import Analysis, build_summary, write_csv, write_html, write_json DEFAULT_ORG = "Your organisation" DEFAULT_OUTPUT = "shadow-ai-report.html" def build_parser(): p = argparse.ArgumentParser( prog="shadow-ai-report", description="Turn DNS or web-filter logs into a one-file shadow AI report (HTML). " "Runs locally and sends nothing anywhere.", epilog="Inputs: CSV/TSV from any filter (domain, user and time columns are detected " "from the header), plain lists of domains, and Cloudflare Gateway DNS logs " "(Logpush dataset gateway_dns, JSON lines). Files may be gzip-compressed. " + FREE_EDITION_NOTE, ) p.add_argument("logfiles", nargs="+", metavar="LOGFILE", help="log file(s) to read (.gz accepted)") p.add_argument("-o", "--output", default=DEFAULT_OUTPUT, metavar="FILE", help="HTML report to write (default: %(default)s)") p.add_argument("--org", default=DEFAULT_ORG, metavar="NAME", help="organisation name shown on the report (default: %(default)s)") p.add_argument("--format", default="auto", choices=FORMATS, help="input format (default: auto-detect per file)") p.add_argument("--anonymise", "--anonymize", dest="anonymise", action="store_true", help="replace users, devices and IP addresses by stable pseudonyms (User 1, User 2, ...)") p.add_argument("--json", metavar="FILE", help="also write a machine-readable summary (JSON)") p.add_argument("--csv", metavar="FILE", help="also write the tools-in-use table (CSV)") p.add_argument("--version", action="version", version="shadow-ai-report " + __version__) return p def _fmt(n): return "{:,}".format(n) def _pct(x): if not x: return "0%" return "<0.1%" if x < 0.001 else "{:.1f}%".format(x * 100) def print_summary(s, out=None): out = out or sys.stdout t = s["totals"] w = out.write for f in s["inputs"]: cols = ", ".join("%s=%s" % (k, v) for k, v in f["columns"].items()) w("Read %s: %s [%s] - %s lines, %s used, %s skipped\n" % ( f["file"], f["format"], cols, _fmt(f["lines"]), _fmt(f["records"]), _fmt(f["skipped"]))) for n in f["notes"]: w(" note: %s\n" % n) rows = [ ("Requests analysed", _fmt(t["requests"])), ("Distinct domains", _fmt(t["distinct_domains"])), ("Lines skipped", _fmt(t["lines_skipped"])), ("AI tools detected", _fmt(t["ai_tools"])), ("AI-tool requests", "%s (%s of all requests)" % (_fmt(t["ai_requests"]), _pct(t["ai_share"]))), ] if t["ai_users"] is not None: rows.append(("Users/devices on AI", "%s of %s" % (_fmt(t["ai_users"]), _fmt(t["users_seen"])))) if t["ai_blocked"] is not None: rows.append(("AI requests blocked", _fmt(t["ai_blocked"]))) rows.append(("High risk/exposure", "%s tool(s)" % _fmt(t["high_risk_or_exposure_tools"]))) rows.append(("Flagged (high/medium)", "%s tool(s)" % _fmt(t["flagged_tools"]))) width = max(len(k) for k, _ in rows) w("\n") for k, v in rows: w(" %s %s\n" % (k.ljust(width), v)) if t["ai_tools"]: w("\n Top tools:\n") for r in s["tools"][:5]: w(" %-32s %8s requests (risk %s, sovereignty %s)\n" % ( r["name"][:32], _fmt(r["requests"]), r["risk_level"], r["data_sovereignty"])) else: w("\n No AI tools from the free list (%d tools) were found. Requests to AI tools that are\n" " not on the list count as ordinary traffic, so this does not mean nobody uses AI.\n" % FREE_LIST_SIZE) w("\n%s\n" % FREE_EDITION_NOTE) def run(args, out=None): out = out or sys.stdout a = Analysis() for path in args.logfiles: a.add_file(path, args.format) if a.total == 0: skipped = sum(f.skipped for f in a.files) raise InputError( "no usable log lines found (%s line(s) skipped). Check that the file is a DNS or " "web-filter log with a domain, host or URL column, or use --format." % _fmt(skipped)) s = build_summary(a, org=args.org, anonymise=args.anonymise) outputs = [(write_html, args.output, "Report")] if args.json: outputs.append((write_json, args.json, "JSON")) if args.csv: outputs.append((write_csv, args.csv, "CSV")) for writer, path, _ in outputs: try: writer(s, path) except OSError as e: raise InputError("cannot write %s (%s)" % (path, e.strerror or e)) print_summary(s, out) for _, path, label in outputs: out.write("%s written: %s\n" % (label, os.path.abspath(path))) return s def main(argv=None): for stream in (sys.stdout, sys.stderr): try: # never crash on a terminal that cannot show a tool name stream.reconfigure(errors="replace") except (AttributeError, ValueError): pass parser = build_parser() args = parser.parse_args(argv) try: run(args) except InputError as e: sys.stderr.write("shadow-ai-report: error: %s\n" % e) return 2 except KeyboardInterrupt: sys.stderr.write("shadow-ai-report: interrupted\n") return 130 return 0