Each client’s monthly AI report,
built from its own firewall.

1#!/usr/bin/env python3
2"""
3AI Tools Blocklist report code, Python version 1.0 (2026-09-28)
4https://www.aitoolsblocklist.com/ai-governance-for-msps.php#report-code
5
6Builds a monthly AI report for each client from the client's own firewall or DNS-filter logs. The logs stay on
7your machine: only the list of domain names is checked against the AI Tools Blocklist API (your plan's API key).
8
9  python3 atb_report.py collect --config atb-report.json            pull the latest logs (run daily)
10  python3 atb_report.py syslog  --config atb-report.json --port 5514 receive syslog (SonicWall, Cisco ASA / Firepower)
11  python3 atb_report.py import  --config atb-report.json --client "Example Dental" --file export.csv
12  python3 atb_report.py report  --config atb-report.json --month 2026-09 [--client "Example Dental"]
13
14Sources: umbrella, meraki, fortigate, paloalto, dnsfilter, cloudflare, defender, watchguard, syslog, file.
15Needs Python 3.8 or later and nothing else. Settings: see atb-report.example.json.
16"""
17import argparse, base64, calendar, csv, datetime as dt, io, json, os, re, socket, ssl, sys, time
18import urllib.error, urllib.parse, urllib.request
19import xml.etree.ElementTree as ET
20
21VERSION = '1.0'
22MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
23HOST_RE = re.compile(r'^(?=.{4,253}$)([a-z0-9_](?:[a-z0-9_-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}$')
24SOURCE_LABELS = {'umbrella': 'Cisco Umbrella', 'meraki': 'Cisco Meraki', 'fortigate': 'FortiGate', 'paloalto': 'Palo Alto Networks',
25                 'dnsfilter': 'DNSFilter', 'cloudflare': 'Cloudflare Zero Trust', 'defender': 'Microsoft Defender for Endpoint',
26                 'watchguard': 'WatchGuard', 'syslog': 'the firewall', 'file': 'the firewall'}
27SOURCES_WITH_BLOCKS = {'umbrella', 'fortigate', 'paloalto', 'dnsfilter', 'syslog', 'file'}
28
29
30def utcnow():
31    """Current UTC time; ATB_NOW (e.g. 2026-09-28T12:00:00) fixes it for tests."""
32    v = os.environ.get('ATB_NOW')
33    return dt.datetime.strptime(v, '%Y-%m-%dT%H:%M:%S') if v else dt.datetime.utcnow().replace(microsecond=0)
34
35
36class Err(Exception):
37    pass
38
39
40def log(msg):
41    print(time.strftime('%Y-%m-%d %H:%M:%S') + '  ' + msg, flush=True)
42
43
44# ------------------------------------------------------------------ helpers
45def norm_host(raw):
46    """URL, host:port or host -> lower-case host without www. and trailing dot, or '' if not a host name."""
47    s = (raw or '').strip().strip('"\'').lower()
48    s = re.sub(r'^[a-z][a-z0-9+.-]*://', '', s)
49    s = s.split('/', 1)[0].split('?', 1)[0].split('@')[-1]
50    if s.count(':') == 1:
51        s = s.split(':', 1)[0]
52    s = s.strip('.')
53    if s.startswith('www.'):
54        s = s[4:]
55    if re.match(r'^\d+\.\d+\.\d+\.\d+$', s) or not HOST_RE.match(s):
56        return ''
57    return s
58
59
60def http(method, url, headers=None, body=None, form=None, insecure=False, tries=4, raw=False):
61    headers = dict(headers or {})
62    data = None
63    if form is not None:
64        data = urllib.parse.urlencode(form).encode()
65        headers.setdefault('Content-Type', 'application/x-www-form-urlencoded')
66    elif body is not None:
67        data = json.dumps(body).encode()
68        headers.setdefault('Content-Type', 'application/json')
69    headers.setdefault('User-Agent', 'atb-report/%s (python)' % VERSION)
70    ctx = ssl._create_unverified_context() if insecure else None
71    for attempt in range(tries):
72        req = urllib.request.Request(url, data=data, headers=headers, method=method)
73        try:
74            with urllib.request.urlopen(req, timeout=120, context=ctx) as r:
75                text = r.read().decode('utf-8', 'replace')
76                if raw:
77                    return text, dict(r.headers)
78                return json.loads(text) if text.strip() else None
79        except urllib.error.HTTPError as e:
80            text = e.read().decode('utf-8', 'replace')
81            if e.code in (429, 500, 502, 503, 504) and attempt < tries - 1:
82                wait = e.headers.get('Retry-After')
83                time.sleep(min(int(wait), 60) if wait and wait.isdigit() else 3 * (attempt + 1))
84                continue
85            raise Err('%s %s: HTTP %s %s' % (method, url.split('?')[0], e.code, text[:200]))
86        except urllib.error.URLError as e:
87            if attempt < tries - 1:
88                time.sleep(3 * (attempt + 1))
89                continue
90            raise Err('%s %s: %s' % (method, url.split('?')[0], e.reason))
91    raise Err('gave up: ' + url.split('?')[0])
92
93
94def slug(name):
95    return re.sub(r'[^a-z0-9]+', '-', name.lower()).strip('-') or 'client'
96
97
98# ------------------------------------------------------------------ local store: one JSON file per client and day
99class Store:
100    """data_dir/<client-slug>/<YYYY-MM-DD>.json = {"<host>\\t<user>\\t<action>": count}; action is allowed|blocked|seen."""
101    def __init__(self, cfg):
102        self.dir = cfg.get('data_dir') or './atb-data'
103        self.buf = {}
104
105    def add(self, client, day, host, user, action, count=1):
106        host = norm_host(host)
107        if not host or count <= 0:
108            return
109        user = (user or '').strip()[:120] or '-'
110        action = action if action in ('allowed', 'blocked') else 'seen'
111        k = (slug(client), day)
112        key = host + '\t' + user.replace('\t', ' ') + '\t' + action
113        b = self.buf.setdefault(k, {})
114        b[key] = b.get(key, 0) + int(count)
115
116    def flush(self):
117        n = 0
118        for (cs, day), rows in self.buf.items():
119            d = os.path.join(self.dir, cs)
120            os.makedirs(d, exist_ok=True)
121            p = os.path.join(d, day + '.json')
122            cur = {}
123            if os.path.exists(p):
124                with open(p, encoding='utf-8') as f:
125                    cur = json.load(f)
126            for k, v in rows.items():
127                cur[k] = cur.get(k, 0) + v
128                n += v
129            with open(p + '.tmp', 'w', encoding='utf-8') as f:
130                json.dump(cur, f, sort_keys=True)
131            os.replace(p + '.tmp', p)
132        self.buf = {}
133        return n
134
135    def month(self, client, month):
136        out = {}
137        d = os.path.join(self.dir, slug(client))
138        if not os.path.isdir(d):
139            return out
140        for fn in sorted(os.listdir(d)):
141            if fn.startswith(month + '-') and fn.endswith('.json'):
142                with open(os.path.join(d, fn), encoding='utf-8') as f:
143                    out[fn[:-5]] = json.load(f)
144        return out
145
146
147def day_of(value):
148    """Many timestamp shapes -> YYYY-MM-DD (UTC)."""
149    if value is None or value == '':
150        return utcnow().strftime('%Y-%m-%d')
151    if isinstance(value, (int, float)) or (isinstance(value, str) and value.isdigit()):
152        v = float(value)
153        v = v / 1e9 if v > 1e17 else (v / 1e6 if v > 1e14 else (v / 1000 if v > 1e11 else v))
154        return dt.datetime.utcfromtimestamp(v).strftime('%Y-%m-%d')
155    v = str(value)
156    m = re.search(r'(\d{4})[-/](\d{2})[-/](\d{2})', v)
157    if m:
158        return '%s-%s-%s' % m.groups()
159    # syslog style: "Aug 15 2019 12:00:00" or "Aug 15 12:00:00" (no year: the latest such date that is not in the future)
160    m = re.search(r'\b(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\s+(\d{1,2})(?:,?\s+(\d{4}))?', v)
161    if m:
162        mo = MONTHS.index(m.group(1)) + 1
163        today = utcnow()
164        y = int(m.group(3)) if m.group(3) else (today.year if (mo, int(m.group(2))) <= (today.month, today.day) else today.year - 1)
165        return '%04d-%02d-%02d' % (y, mo, int(m.group(2)))
166    return utcnow().strftime('%Y-%m-%d')
167
168
169# ------------------------------------------------------------------ sources (API)
170def src_umbrella(c, s, since, until, store):
171    base = s.get('api_base', 'https://api.umbrella.com')
172    hdr = {'Authorization': 'Basic ' + base64.b64encode(('%s:%s' % (s['api_key'], s['api_secret'])).encode()).decode()}
173    if s.get('org_id'):
174        hdr['X-Umbrella-OrgId'] = str(s['org_id'])
175    tok = http('POST', base + '/auth/v2/token', hdr, form={'grant_type': 'client_credentials'})['access_token']
176    h = {'Authorization': 'Bearer ' + tok}
177    off, n = 0, 0
178    while True:
179        q = urllib.parse.urlencode({'from': calendar.timegm(since.timetuple()) * 1000, 'to': calendar.timegm(until.timetuple()) * 1000, 'limit': 5000, 'offset': off})  # UTC epoch ms
180        r = http('GET', base + '/reports/v2/activity/dns?' + q, h)
181        rows = (r or {}).get('data') or []
182        for x in rows:
183            ids = x.get('identities') or []
184            user = next((i.get('label') for i in ids if 'user' in str(i.get('type', '')).lower()), None) or x.get('internalip') or (ids[0].get('label') if ids else '')
185            store.add(c['name'], day_of(x.get('timestamp') or x.get('date')), x.get('domain'), user, 'blocked' if x.get('verdict') == 'blocked' else 'allowed')
186            n += 1
187        if len(rows) < 5000:
188            return n
189        off += 5000
190        time.sleep(0.25)