iconify-data.js

// First-run icon data: downloads the @iconify/json npm tarball and extracts
// json/*.json + collections.json into a writable directory. No dependencies —
// https + zlib + a minimal streaming tar parser.
const https = require('https');
const zlib = require('zlib');
const fs = require('fs');
const path = require('path');
const { Writable } = require('stream');

const REGISTRY = 'https://registry.npmjs.org/@iconify/json/latest';

function fetchJSON(url) {
  return new Promise((resolve, reject) => {
    https.get(url, { headers: { accept: 'application/json' } }, (res) => {
      if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
        return resolve(fetchJSON(res.headers.location));
      }
      if (res.statusCode !== 200) return reject(new Error(`HTTP ${res.statusCode} for ${url}`));
      let body = '';
      res.on('data', (c) => { body += c; });
      res.on('end', () => { try { resolve(JSON.parse(body)); } catch (e) { reject(e); } });
    }).on('error', reject);
  });
}

function download(url, dest, onPct) {
  return new Promise((resolve, reject) => {
    https.get(url, (res) => {
      if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
        return resolve(download(res.headers.location, dest, onPct));
      }
      if (res.statusCode !== 200) return reject(new Error(`HTTP ${res.statusCode} for ${url}`));
      const total = parseInt(res.headers['content-length'] || '0', 10);
      let got = 0;
      const out = fs.createWriteStream(dest);
      res.on('data', (c) => {
        got += c.length;
        if (total && onPct) onPct(Math.round((got / total) * 100));
      });
      res.pipe(out);
      out.on('finish', () => out.close(resolve));
      out.on('error', reject);
      res.on('error', reject);
    }).on('error', reject);
  });
}

// Minimal tar stream parser: enough for npm tarballs (regular files, pax headers, longnames).
class TarExtractor extends Writable {
  constructor(destRoot, filter, onFile) {
    super();
    this.destRoot = destRoot;
    this.filter = filter;
    this.onFile = onFile;
    this.bufs = [];
    this.buffered = 0;
    this.needData = 0;   // bytes of file data (padded) still expected
    this.fileSize = 0;   // real size of current file
    this.fileBufs = [];
    this.fileName = null; // null => skip current entry's data
    this.longName = null;
    this.done = false;
  }
  take(n) { // consume exactly n bytes from the head
    const out = Buffer.allocUnsafe(n);
    let off = 0;
    while (off < n) {
      const head = this.bufs[0];
      const want = n - off;
      if (head.length <= want) {
        head.copy(out, off);
        off += head.length;
        this.bufs.shift();
      } else {
        head.copy(out, off, 0, want);
        this.bufs[0] = head.subarray(want);
        off += want;
      }
    }
    this.buffered -= n;
    return out;
  }
  _write(chunk, _enc, cb) {
    if (!this.done) {
      this.bufs.push(chunk);
      this.buffered += chunk.length;
      try { this.process(); } catch (e) { return cb(e); }
    }
    cb();
  }
  process() {
    for (;;) {
      if (this.needData > 0) {
        if (this.buffered < this.needData) {
          // consume what we have in bulk
          while (this.bufs.length && this.needData >= this.bufs[0].length) {
            const c = this.bufs.shift();
            this.buffered -= c.length;
            this.needData -= c.length;
            if (this.fileName) {
              const real = Math.min(c.length, this.fileSize);
              if (real > 0) { this.fileBufs.push(c.subarray(0, real)); this.fileSize -= real; }
            }
          }
          return;
        }
        const c = this.take(this.needData);
        if (this.fileName) {
          this.fileBufs.push(c.subarray(0, this.fileSize));
          this.finishFile();
        }
        this.needData = 0;
      }
      if (this.buffered < 512) return;
      const h = this.take(512);
      if (h.every((b) => b === 0)) { this.done = true; return; }
      let name = h.toString('utf8', 0, 100).replace(/\0.*$/, '');
      const prefix = h.toString('utf8', 345, 500).replace(/\0.*$/, '');
      if (prefix) name = `${prefix}/${name}`;
      if (this.longName) { name = this.longName; this.longName = null; }
      const size = parseInt(h.toString('ascii', 124, 136).replace(/[^0-7]/g, ''), 8) || 0;
      const type = String.fromCharCode(h[156]);
      const padded = Math.ceil(size / 512) * 512;
      if (type === 'L') { // GNU longname: data is the real name of the NEXT entry
        this.needData = padded;
        this.fileSize = size;
        this.fileBufs = [];
        this.fileName = '\0longname';
        continue;
      }
      const isFile = type === '0' || type === '\0' || type === '';
      const clean = name.replace(/^package\//, '');
      if (isFile && this.filter(clean)) {
        this.fileName = clean;
        this.fileSize = size;
        this.fileBufs = [];
      } else {
        this.fileName = null;
        this.fileSize = 0;
      }
      this.needData = padded;
    }
  }
  finishFile() {
    const data = Buffer.concat(this.fileBufs);
    this.fileBufs = [];
    if (this.fileName === '\0longname') {
      this.longName = data.toString('utf8').replace(/\0.*$/, '');
    } else {
      const dest = path.join(this.destRoot, this.fileName);
      fs.mkdirSync(path.dirname(dest), { recursive: true });
      fs.writeFileSync(dest, data);
      if (this.onFile) this.onFile(this.fileName);
    }
    this.fileName = null;
  }
}

function extractTarGz(tgzPath, destRoot, filter, onFile) {
  return new Promise((resolve, reject) => {
    const extractor = new TarExtractor(destRoot, filter, onFile);
    fs.createReadStream(tgzPath)
      .on('error', reject)
      .pipe(zlib.createGunzip())
      .on('error', reject)
      .pipe(extractor)
      .on('error', reject)
      .on('finish', resolve);
  });
}

function hasIconData(root) {
  return fs.existsSync(path.join(root, 'collections.json')) &&
    fs.existsSync(path.join(root, 'json'));
}

// Ensure @iconify/json content exists under `root`. Reports progress via onProgress({phase, pct, detail}).
async function ensureIconifyData(root, onProgress) {
  if (hasIconData(root)) return false;
  const report = onProgress || (() => {});
  report({ phase: 'metadata', pct: 0 });
  const meta = await fetchJSON(REGISTRY);
  const tmp = path.join(root, '_download.tgz');
  fs.mkdirSync(root, { recursive: true });
  report({ phase: 'download', pct: 0, detail: meta.version });
  await download(meta.dist.tarball, tmp, (pct) => report({ phase: 'download', pct, detail: meta.version }));
  report({ phase: 'extract', pct: 0 });
  let files = 0;
  await extractTarGz(tmp, root,
    (name) => name === 'collections.json' || (name.startsWith('json/') && name.endsWith('.json')),
    () => { files++; if (files % 20 === 0) report({ phase: 'extract', pct: Math.min(99, Math.round(files / 2.45)) }); });
  fs.rmSync(tmp, { force: true });
  if (!hasIconData(root)) throw new Error('Extraction finished but icon data is incomplete');
  fs.writeFileSync(path.join(root, 'version.txt'), meta.version);
  report({ phase: 'extract', pct: 100 });
  return true;
}

module.exports = { ensureIconifyData, hasIconData };