Each client’s monthly AI report,
built from its own firewall.

1#!/usr/bin/env node
2/*
3 * AI Tools Blocklist report code, Node.js version 1.0 (2026-09-28)
4 * https://www.aitoolsblocklist.com/ai-governance-for-msps.php#report-code
5 *
6 * Builds a monthly AI report for each client from the client's own firewall or DNS-filter logs. The logs stay on
7 * your machine: only the list of domain names is checked against the AI Tools Blocklist API (your plan's API key).
8 *
9 *   node atb-report.js collect --config atb-report.json            pull the latest logs (run daily)
10 *   node atb-report.js syslog  --config atb-report.json --port 5514 receive syslog (SonicWall, Cisco ASA / Firepower)
11 *   node atb-report.js import  --config atb-report.json --client "Example Dental" --file export.csv
12 *   node atb-report.js report  --config atb-report.json --month 2026-09 [--client "Example Dental"]
13 *
14 * Sources: umbrella, meraki, fortigate, paloalto, dnsfilter, cloudflare, defender, watchguard, syslog, file.
15 * Needs Node.js 18 or later and nothing else (no npm packages). Settings: see atb-report.example.json.
16 *
17 * This is a line-by-line port of the Python version (atb_report.py): same commands, settings file, local store
18 * and report output. Where Python and JavaScript differ (string stripping, regular expressions, JSON formatting,
19 * "truthiness", CSV quoting, URL encoding) small helpers below reproduce the Python behaviour, so both versions
20 * write the same files for the same input.
21 */
22'use strict';
23
24const fs = require('fs');
25const path = require('path');
26const http = require('http');
27const https = require('https');
28const tls = require('tls');
29const net = require('net');
30const dgram = require('dgram');
31
32const VERSION = '1.0';
33const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
34const MONTH_NAMES = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'];
35const SOURCE_LABELS = {
36  umbrella: 'Cisco Umbrella', meraki: 'Cisco Meraki', fortigate: 'FortiGate', paloalto: 'Palo Alto Networks',
37  dnsfilter: 'DNSFilter', cloudflare: 'Cloudflare Zero Trust', defender: 'Microsoft Defender for Endpoint',
38  watchguard: 'WatchGuard', syslog: 'the firewall', file: 'the firewall',
39};
40const SOURCES_WITH_BLOCKS = new Set(['umbrella', 'fortigate', 'paloalto', 'dnsfilter', 'syslog', 'file']);
41
42// ------------------------------------------------------------------ Python-compatible building blocks
43// Python's str.strip() / \s / \S / \w use Unicode rules that differ slightly from JavaScript's; these match Python.
44const WS = '\\t\\n\\x0b\\x0c\\r\\x1c-\\x20\\x85\\xa0\\u1680\\u2000-\\u200a\\u2028\\u2029\\u202f\\u205f\\u3000';
45const S = `[^${WS}]`;          // Python \S
46const SP = `[${WS}]`;          // Python \s
47const W = '[\\p{L}\\p{N}_]';   // Python \w (letters, digits, underscore in any script)
48const EOS = '(?=\\n?$)';       // Python "$" without MULTILINE: end of text, or just before a final newline
49
50const HOST_RE = new RegExp(`^(?=[^\\n]{4,253}${EOS})([a-z0-9_](?:[a-z0-9_-]{0,61}[a-z0-9])?\\.)+[a-z]{2,63}${EOS}`);
51const IPV4_RE = new RegExp(`^\\d+\\.\\d+\\.\\d+\\.\\d+${EOS}`);
52
53/** Errors the program reports as one clean line (Python: class Err). */
54class Err extends Error {}
55/** Python KeyError: a required setting or answer field is missing. The message is the key in quotes. */
56class KeyErr extends Error {}
57/** Python ValueError: bad number, bad JSON, bad date. */
58class ValueErr extends Error {}
59/** Python xml.etree.ElementTree.ParseError. */
60class XmlParseError extends Error {}
61
62/** Python truthiness: None, False, 0, '', [] and {} are false. */
63function truthy(v) {
64  if (v === null || v === undefined || v === false || v === 0 || v === '' || Number.isNaN(v)) return false;
65  if (Array.isArray(v)) return v.length > 0;
66  if (typeof v === 'object') return Object.keys(v).length > 0;
67  return true;
68}
69
70/** Python "a or b or c": the first true value, else the last one. */
71function or(...vals) {
72  for (let i = 0; i < vals.length - 1; i++) if (truthy(vals[i])) return vals[i];
73  const last = vals[vals.length - 1];
74  return last === undefined ? null : last;
75}
76
77/** Python dict.get(key, default) on a parsed JSON object. */
78function get(obj, key, dflt = null) {
79  return obj !== null && typeof obj === 'object' && !Array.isArray(obj) && Object.prototype.hasOwnProperty.call(obj, key) ? obj[key] : dflt;
80}
81
82/** Python obj[key]: raises KeyError (reported as "missing setting 'key'") if the key is not there. */
83function need(obj, key) {
84  if (obj === null || typeof obj !== 'object' || !Object.prototype.hasOwnProperty.call(obj, key)) throw new KeyErr(pyRepr(key));
85  return obj[key];
86}
87
88function pyStrip(s, chars) {
89  const set = chars === undefined ? null : new Set(Array.from(chars));
90  const isStrip = chars === undefined ? (c) => new RegExp(`^${SP}$`, 'u').test(c) : (c) => set.has(c);
91  const a = Array.from(s);
92  let i = 0, j = a.length;
93  while (i < j && isStrip(a[i])) i++;
94  while (j > i && isStrip(a[j - 1])) j--;
95  return a.slice(i, j).join('');
96}
97
98/** Python s[a:b] counts characters (code points), not UTF-16 units. */
99function pySlice(s, a, b) {
100  return Array.from(s).slice(a, b).join('');
101}
102
103/** Python int(): whole numbers only, from numbers or digit strings. */
104function pyInt(v) {
105  if (typeof v === 'boolean') return v ? 1 : 0;
106  if (typeof v === 'number') {
107    if (!Number.isFinite(v)) throw new ValueErr('cannot convert float ' + v + ' to integer');
108    return Math.trunc(v);
109  }
110  const s = pyStrip(String(v));
111  if (!/^[+-]?\d+(?:_\d+)*$/.test(s)) throw new ValueErr(`invalid literal for int() with base 10: ${pyRepr(String(v))}`);
112  return parseInt(s.replace(/_/g, ''), 10);
113}
114
115/** Python repr() of a string: single quotes unless the text contains one. */
116function pyReprStr(s) {
117  const q = s.includes("'") && !s.includes('"') ? '"' : "'";
118  let out = q;
119  for (const ch of s) {
120    const c = ch.codePointAt(0);
121    if (ch === '\\') out += '\\\\';
122    else if (ch === q) out += '\\' + q;
123    else if (ch === '\n') out += '\\n';
124    else if (ch === '\r') out += '\\r';
125    else if (ch === '\t') out += '\\t';
126    else if (c < 0x20 || c === 0x7f) out += '\\x' + c.toString(16).padStart(2, '0');
127    else out += ch;
128  }
129  return out + q;
130}
131
132/** Python repr() of JSON-like values. */
133function pyRepr(v) {
134  if (typeof v === 'string') return pyReprStr(v);
135  if (v === null || v === undefined) return 'None';
136  if (v === true) return 'True';
137  if (v === false) return 'False';
138  if (typeof v === 'number') return pyNum(v);
139  if (Array.isArray(v)) return '[' + v.map(pyRepr).join(', ') + ']';
140  return '{' + Object.keys(v).map((k) => pyReprStr(k) + ': ' + pyRepr(v[k])).join(', ') + '}';
141}
142
143/** Python str(): strings as they are, everything else as repr(). */
144function pyStr(v) {
145  return typeof v === 'string' ? v : pyRepr(v);
146}
147
148/** A number the way Python prints it (ints as digits; floats with Python's exponent rules). */
149function pyNum(v) {
150  if (Number.isNaN(v)) return 'NaN';
151  if (!Number.isFinite(v)) return v > 0 ? 'Infinity' : '-Infinity';
152  if (Number.isInteger(v)) return BigInt(v).toString();
153  const [mant, e] = v.toExponential().split('e');
154  const exp = parseInt(e, 10);
155  if (exp < -4 || exp >= 16) return mant + 'e' + (exp < 0 ? '-' : '+') + String(Math.abs(exp)).padStart(2, '0');
156  return String(v);
157}
158
159/** Compare strings by code point, like Python's sorted(). */
160function cmpStr(a, b) {
161  const x = Array.from(a), y = Array.from(b);
162  for (let i = 0; i < Math.min(x.length, y.length); i++) {
163    const d = x[i].codePointAt(0) - y[i].codePointAt(0);
164    if (d) return d < 0 ? -1 : 1;
165  }
166  return x.length - y.length;
167}
168
169/** json.dumps() with Python's defaults (ASCII-only output, ", " and ": " separators), optional indent / sort_keys. */
170function pyDumps(v, opts = {}, level = 0) {
171  const ind = opts.indent;
172  const nl = ind === undefined ? '' : '\n' + ' '.repeat(ind * (level + 1));
173  const end = ind === undefined ? '' : '\n' + ' '.repeat(ind * level);
174  const sep = ind === undefined ? ', ' : ',';
175  if (v === null || v === undefined) return 'null';
176  if (v === true) return 'true';
177  if (v === false) return 'false';
178  if (typeof v === 'number') return pyNum(v);
179  if (typeof v === 'string') return pyJsonStr(v);
180  if (Array.isArray(v)) {
181    if (!v.length) return '[]';
182    return '[' + nl + v.map((x) => pyDumps(x, opts, level + 1)).join(sep + nl) + end + ']';
183  }
184  let keys = Object.keys(v);
185  if (!keys.length) return '{}';
186  if (opts.sortKeys) keys = keys.sort(cmpStr);
187  return '{' + nl + keys.map((k) => pyJsonStr(k) + ': ' + pyDumps(v[k], opts, level + 1)).join(sep + nl) + end + '}';
188}
189
190function pyJsonStr(s) {
191  let out = '"';
192  for (let i = 0; i < s.length; i++) {        // UTF-16 units: non-ASCII becomes \uXXXX (surrogate pairs too), as in Python
193    const ch = s[i], c = s.charCodeAt(i);
194    if (ch === '"') out += '\\"';
195    else if (ch === '\\') out += '\\\\';
196    else if (ch === '\n') out += '\\n';
197    else if (ch === '\r') out += '\\r';
198    else if (ch === '\t') out += '\\t';
199    else if (ch === '\b') out += '\\b';
200    else if (ch === '\f') out += '\\f';
201    else if (c < 0x20 || c > 0x7e) out += '\\u' + c.toString(16).padStart(4, '0');
202    else out += ch;
203  }
204  return out + '"';
205}
206
207/** json.loads(): a parse failure is a ValueError, as in Python. */
208function jsonLoads(text) {
209  try {
210    return JSON.parse(text);
211  } catch (e) {
212    throw new ValueErr(e.message);
213  }
214}
215
216/** Read a UTF-8 text file the way Python's open() does: bad bytes become U+FFFD, \r\n and \r become \n. */
217function readText(p) {
218  return decodeUtf8(fs.readFileSync(p)).replace(/\r\n?/g, '\n');
219}
220
221function decodeUtf8(buf) {
222  return new TextDecoder('utf-8', { ignoreBOM: true }).decode(buf);
223}
224
225/** os.path.join (no normalisation, so paths in log lines read the same as in the Python version). */
226function pjoin(a, ...parts) {
227  let out = a;
228  for (const p of parts) out = p.startsWith('/') ? p : (out === '' || out.endsWith('/') ? out + p : out + '/' + p);
229  return out;
230}
231
232/** urllib.parse.quote_plus (used by urlencode): everything except letters, digits and _.-~ is %-encoded, space is +. */
233function quotePlus(s) {
234  return encodeURIComponent(s).replace(/[!'()*]/g, (c) => '%' + c.charCodeAt(0).toString(16).toUpperCase()).replace(/%20/g, '+');
235}
236
237/** urllib.parse.quote with its default safe='/'. */
238function quote(s) {
239  return encodeURIComponent(s).replace(/[!'()*]/g, (c) => '%' + c.charCodeAt(0).toString(16).toUpperCase()).replace(/%2F/g, '/');
240}
241
242/** urllib.parse.urlencode over [key, value] pairs (keeps the order given). */
243function urlencode(pairs) {
244  return pairs.map(([k, v]) => quotePlus(pyStr(k)) + '=' + quotePlus(pyStr(v))).join('&');
245}
246
247const sleep = (sec) => new Promise((r) => setTimeout(r, sec * 1000));
248
249// ------------------------------------------------------------------ dates: naive UTC date-times held in Date objects
250const pad = (n, w = 2) => String(n).padStart(w, '0');
251
252/** Small strftime: %Y %m %d %H %M %S %B. */
253function fmt(d, pattern) {
254  return pattern.replace(/%([YmdHMSB])/g, (_, k) => ({
255    Y: pad(d.getUTCFullYear(), 4), m: pad(d.getUTCMonth() + 1), d: pad(d.getUTCDate()),
256    H: pad(d.getUTCHours()), M: pad(d.getUTCMinutes()), S: pad(d.getUTCSeconds()), B: MONTH_NAMES[d.getUTCMonth()],
257  })[k]);
258}
259
260/** strptime(v, '%Y-%m-%dT%H:%M:%S'); a malformed value is a ValueError. */
261function parseStamp(v) {
262  const m = /^(\d{4})-(\d{1,2})-(\d{1,2})T(\d{1,2}):(\d{1,2}):(\d{1,2})$/.exec(v);
263  const bad = () => new ValueErr(`time data ${pyRepr(v)} does not match format '%Y-%m-%dT%H:%M:%S'`);
264  if (!m) throw bad();
265  const [y, mo, d, h, mi, s] = m.slice(1).map(Number);
266  const t = new Date(Date.UTC(y, mo - 1, d, h, mi, s));
267  if (t.getUTCMonth() !== mo - 1 || t.getUTCDate() !== d || h > 23 || mi > 59 || s > 61) throw bad();
268  if (y < 100) t.setUTCFullYear(y);
269  return t;
270}
271
272/** Current UTC time; ATB_NOW (e.g. 2026-09-28T12:00:00) fixes it for tests. */
273function utcnow() {
274  const v = process.env.ATB_NOW;
275  if (v) return parseStamp(v);
276  const d = new Date();
277  d.setUTCMilliseconds(0);
278  return d;
279}
280
281const addDays = (d, n) => new Date(d.getTime() + n * 86400000);
282
283
284function log(msg) {
285  const d = new Date();
286  const t = `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`;
287  process.stdout.write(t + '  ' + msg + '\n');
288}
289
290// ------------------------------------------------------------------ helpers
291/** URL, host:port or host -> lower-case host without www. and trailing dot, or '' if not a host name. */
292function normHost(raw) {
293  let s = pyStrip(pyStrip(truthy(raw) ? pyStr(raw) : ''), '"\'').toLowerCase();
294  s = s.replace(/^[a-z][a-z0-9+.-]*:\/\//, '');
295  s = s.split('/')[0].split('?')[0].split('@').pop();
296  if (s.split(':').length === 2) s = s.split(':')[0];
297  s = pyStrip(s, '.');
298  if (s.startsWith('www.')) s = s.slice(4);
299  if (IPV4_RE.test(s) || !HOST_RE.test(s)) return '';
300  return s;
301}
302
303function slug(name) {
304  return pyStrip(name.toLowerCase().replace(/[^a-z0-9]+/g, '-'), '-') || 'client';
305}
306
307// ------------------------------------------------------------------ HTTP (built-in http/https, proxy aware like urllib)
308/** Proxies from the environment, as urllib.request.getproxies() reads them (lower-case names win). */
309function envProxies() {
310  const out = {};
311  for (const [k, v] of Object.entries(process.env)) if (v && k.slice(-6).toLowerCase() === '_proxy') out[k.slice(0, -6).toLowerCase()] = v;
312  if ('REQUEST_METHOD' in process.env) delete out.http;
313  for (const [k, v] of Object.entries(process.env)) {
314    if (k.slice(-6) === '_proxy') {
315      const name = k.slice(0, -6).toLowerCase();
316      if (v) out[name] = v; else delete out[name];
317    }
318  }
319  return out;
320}
321
322/** urllib's no_proxy check (host may include :port). */
323function bypassProxy(hostport, noProxy) {
324  if (!noProxy) return false;
325  if (noProxy === '*') return true;
326  const hostonly = hostport.replace(/:\d+$/, '');
327  for (let name of noProxy.split(',')) {
328    name = pyStrip(name);
329    if (!name) continue;
330    name = name.replace(/^\.+/, '');
331    const re = new RegExp('^(.+\\.)?' + name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '$', 'i');
332    if (re.test(hostonly) || re.test(hostport)) return true;
333  }
334  return false;
335}
336
337/** One HTTP exchange, no retries or redirects. Resolves {status, headers, body}; rejects on network errors. */
338function requestOnce(method, url, headers, data, insecure) {
339  return new Promise((resolve, reject) => {
340    const u = new URL(url);
341    const isHttps = u.protocol === 'https:';
342    const proxies = envProxies();
343    let proxy = proxies[isHttps ? 'https' : 'http'];
344    if (proxy && bypassProxy(u.host, proxies.no)) proxy = null;
345    const hdrs = Object.assign({ 'Accept-Encoding': 'identity', Connection: 'close' }, headers);
346    if (data !== null) hdrs['Content-Length'] = String(data.length);
347    const port = u.port || (isHttps ? 443 : 80);
348    const bareHost = u.hostname.replace(/^\[|\]$/g, '');
349    const tlsOpts = { rejectUnauthorized: !insecure };
350    if (!net.isIP(bareHost)) tlsOpts.servername = bareHost;     // SNI only for names, as Python does
351
352    let pu = null, proxyAuth = null;
353    if (proxy) {
354      pu = new URL(/^[a-z][a-z0-9+.-]*:\/\//i.test(proxy) ? proxy : 'http://' + proxy);
355      if (pu.username) proxyAuth = 'Basic ' + Buffer.from(decodeURIComponent(pu.username) + ':' + decodeURIComponent(pu.password)).toString('base64');
356    }
357
358    const send = (mod, opts) => {
359      const req = mod.request(opts, (res) => {
360        const chunks = [];
361        res.on('data', (c) => chunks.push(c));
362        res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks) }));
363        res.on('error', reject);
364      });
365      req.setTimeout(120000, () => req.destroy(Object.assign(new Error('timed out'), { code: 'ETIMEDOUT' })));
366      req.on('error', reject);
367      if (data !== null) req.write(data);
368      req.end();
369    };
370
371    if (!proxy) {
372      send(isHttps ? https : http, Object.assign({ method, hostname: u.hostname.replace(/^\[|\]$/g, ''), port, path: u.pathname + u.search, headers: hdrs, agent: false }, isHttps ? tlsOpts : {}));
373    } else if (!isHttps) {
374      // plain HTTP through the proxy: the full URL goes in the request line
375      if (proxyAuth) hdrs['Proxy-Authorization'] = proxyAuth;
376      hdrs.Host = u.host;
377      send(pu.protocol === 'https:' ? https : http, { method, hostname: pu.hostname, port: pu.port || (pu.protocol === 'https:' ? 443 : 80), path: url, headers: hdrs, agent: false });
378    } else {
379      // HTTPS through the proxy: CONNECT tunnel, then TLS to the real host
380      const ch = { Host: `${u.hostname}:${port}` };
381      if (proxyAuth) ch['Proxy-Authorization'] = proxyAuth;
382      const creq = http.request({ method: 'CONNECT', hostname: pu.hostname, port: pu.port || 80, path: `${u.hostname}:${port}`, headers: ch, agent: false });
383      creq.setTimeout(120000, () => creq.destroy(Object.assign(new Error('timed out'), { code: 'ETIMEDOUT' })));
384      creq.on('error', reject);
385      creq.on('connect', (res, socket) => {
386        if (res.statusCode !== 200) {
387          socket.destroy();
388          return reject(Object.assign(new Error(`Tunnel connection failed: ${res.statusCode} ${res.statusMessage}`), { code: 'ETUNNEL' }));
389        }
390        send(https, Object.assign({
391          method, hostname: u.hostname, port, path: u.pathname + u.search, headers: hdrs, agent: false,
392          createConnection: () => tls.connect(Object.assign({ socket }, tlsOpts)),
393        }, tlsOpts));
394      });
395      creq.end();
396    }
397  });
398}