fix: Windows install 404s, skill list duplication, Python frontend false positives

packages/autoskills/installer.ts

@@ -22,7 +22,10 @@ import { log, write, dim, green, cyan, red, HIDE_CURSOR, SHOW_CURSOR, SPINNER }
 // ── Registry ─────────────────────────────────────────────────
 
 const DEFAULT_REGISTRY_RAW_BASE_URL_PREFIX = "https://raw.githubusercontent.com/midudev/autoskills";
-const GITHUB_TOKEN = process.env.GITHUB_TOKEN || process.env.GH_TOKEN || "";
+
+function getGithubToken(): string {
+  return process.env.GITHUB_TOKEN || process.env.GH_TOKEN || "";
+}
 
 export interface RegistryEntry {
   source: string;
@@ -112,8 +115,33 @@ export function _setRegistryDir(dir: string | null): void {
 
 // ── Integrity ────────────────────────────────────────────────
 
-function sha256File(path: string): string {
-  return createHash("sha256").update(readFileSync(path)).digest("hex");
+/** Paths that should never be rewritten when normalizing line endings. */
+function isBinarySkillPath(rel: string): boolean {
+  return /\.(png|jpe?g|gif|webp|ico|pdf|zip|gz|tgz|woff2?|ttf|otf|eot|mp3|mp4|wasm|bin)$/i.test(
+    rel,
+  );
+}
+
+/**
+ * Registry manifests hash LF-only content. Windows checkouts with
+ * `core.autocrlf=true` rewrite text to CRLF and break SHA-256 verification.
+ * Normalize CRLF→LF for text files before hashing/copying so local installs work.
+ */
+function readRegistryFileBytes(absPath: string, rel: string): Buffer {
+  const buf = readFileSync(absPath);
+  if (isBinarySkillPath(rel) || !buf.includes(0x0d)) return buf;
+  // Strip CR that forms CRLF pairs; leave lone CR alone (rare binary-ish content).
+  const out: number[] = [];
+  for (let i = 0; i < buf.length; i++) {
+    const b = buf[i];
+    if (b === 0x0d && buf[i + 1] === 0x0a) continue;
+    out.push(b);
+  }
+  return Buffer.from(out);
+}
+
+function sha256File(path: string, rel: string = ""): string {
+  return createHash("sha256").update(readRegistryFileBytes(path, rel)).digest("hex");
 }
 
 export function verifyRegistryEntry(
@@ -135,7 +163,7 @@ export function verifyRegistryEntry(
     if (!expected) {
       return { ok: false, reason: `no recorded hash for ${normalizedRel}` };
     }
-    const actual = sha256File(abs);
+    const actual = sha256File(abs, normalizedRel);
     if (actual !== expected) {
       return { ok: false, reason: `hash mismatch for ${normalizedRel}` };
     }
@@ -259,8 +287,13 @@ function encodeRawPath(skillName: string, rel: string): string {
 function githubDownloadHeaders(url: string): HeadersInit {
   const headers: Record<string, string> = { "User-Agent": "autoskills" };
   const host = new URL(url).hostname;
-  if (GITHUB_TOKEN && /(^|\.)githubusercontent\.com$/i.test(host)) {
-    headers.Authorization = `Bearer ${GITHUB_TOKEN}`;
+  const token = getGithubToken();
+  // Only attach tokens to api.github.com. Sending an invalid/expired
+  // GITHUB_TOKEN/GH_TOKEN Bearer to raw.githubusercontent.com makes GitHub
+  // return 404 for public files (common on Windows when a stale User env
+  // token shadows anonymous access). See midudev/autoskills#124.
+  if (token && /(^|\.)github\.com$/i.test(host) && !/githubusercontent\.com$/i.test(host)) {
+    headers.Authorization = `Bearer ${token}`;
   }
   return headers;
 }
@@ -371,7 +404,14 @@ function copyRegistryEntryFromLocal(
   }
 
   rmSync(destDir, { recursive: true, force: true });
-  copyDir(join(registryDir, skillName), destDir);
+  // Copy with LF-normalized text so installed skills match manifest hashes on Windows.
+  for (const rel of entry.files) {
+    const normalizedRel = normalizeRegistryRelPath(rel);
+    const src = join(registryDir, skillName, ...normalizedRel.split("/"));
+    const dest = join(destDir, ...normalizedRel.split("/"));
+    mkdirSync(dirname(dest), { recursive: true });
+    writeFileSync(dest, readRegistryFileBytes(src, normalizedRel));
+  }
   opts.onTrace?.(`copied from local registry: ${join(registryDir, skillName)}`);
   return true;
 }

packages/autoskills/lib.ts

@@ -557,29 +557,71 @@ export interface DetectResult {
   combos: ComboSkill[];
 }
 
+/**
+ * Non-JS backends often ship HTML/CSS/templates (docs, email, admin assets)
+ * that must not classify the whole repo as a web frontend (#48).
+ * Package-based frontend detection is always trusted.
+ */
+const FILE_FRONTEND_SUPPRESS_TECH_IDS: ReadonlySet<string> = new Set([
+  "python",
+  "django",
+  "fastapi",
+  "flask",
+  "celery",
+  "sqlalchemy",
+  "pytest",
+  "pandas",
+  "numpy",
+  "scikit-learn",
+  "java",
+  "springboot",
+  "kotlin",
+  "kotlin-multiplatform",
+  "go",
+  "rust",
+  "dotnet",
+  "csharp",
+  "aspnet-core",
+  "blazor",
+  "php",
+  "laravel",
+  "symfony",
+  "wordpress",
+  "ruby",
+  "rails",
+]);
+
+function trustFileBasedFrontend(detected: Technology[]): boolean {
+  return !detected.some((t) => FILE_FRONTEND_SUPPRESS_TECH_IDS.has(t.id));
+}
+
 export function detectTechnologies(projectDir: string): DetectResult {
   const pkg = readPackageJson(projectDir);
   const denoJson = readDenoJson(projectDir);
   const root = detectTechnologiesInDir(projectDir, { pkg, denoJson });
   const seenIds = new Map<string, Technology>(root.detected.map((t) => [t.id, t]));
-  let isFrontend = root.isFrontendByPackages || root.isFrontendByFiles;
+  let isFrontendByPackages = root.isFrontendByPackages;
+  let isFrontendByFiles = root.isFrontendByFiles;
 
   const workspaceDirs = resolveWorkspaces(projectDir, { pkg, denoJson });
   for (const wsDir of workspaceDirs) {
-    const ws = detectTechnologiesInDir(wsDir, { skipFrontendFiles: isFrontend });
+    const ws = detectTechnologiesInDir(wsDir, {
+      skipFrontendFiles: isFrontendByPackages || isFrontendByFiles,
+    });
 
     for (const tech of ws.detected) {
       if (!seenIds.has(tech.id)) {
         seenIds.set(tech.id, tech);
       }
     }
 
-    if (ws.isFrontendByPackages || ws.isFrontendByFiles) {
-      isFrontend = true;
-    }
+    if (ws.isFrontendByPackages) isFrontendByPackages = true;
+    if (ws.isFrontendByFiles) isFrontendByFiles = true;
   }
 
   const detected = [...seenIds.values()];
+  const isFrontend =
+    isFrontendByPackages || (isFrontendByFiles && trustFileBasedFrontend(detected));
   const detectedIds = detected.map((t) => t.id);
   const combos = detectCombos(detectedIds);
 

packages/autoskills/tests/detect.test.ts

@@ -381,6 +381,29 @@ describe("detectTechnologies", () => {
     strictEqual(isFrontend, false);
   });
 
+  it("does not mark Python backends as frontend when HTML assets exist (#48)", () => {
+    writeFile(tmp.path, "requirements.txt", "fastapi==0.100.0\npydantic==2.0.0\n");
+    writeFile(tmp.path, "app/main.py", "from fastapi import FastAPI\napp = FastAPI()\n");
+    writeFile(tmp.path, "docs/index.html", "<html><body>API docs</body></html>");
+    writeFile(tmp.path, "static/style.css", "body { margin: 0 }");
+    const { detected, isFrontend } = detectTechnologies(tmp.path);
+    ok(detected.some((t) => t.id === "python" || t.id === "fastapi"));
+    strictEqual(isFrontend, false);
+  });
+
+  it("still marks pure HTML sites as frontend without backend signals", () => {
+    writeFile(tmp.path, "public/index.html", "<html></html>");
+    const { isFrontend } = detectTechnologies(tmp.path);
+    strictEqual(isFrontend, true);
+  });
+
+  it("still marks frontend when packages say so even with Python present", () => {
+    writeFile(tmp.path, "requirements.txt", "fastapi==0.100.0\n");
+    writePackageJson(tmp.path, { dependencies: { react: "^19.0.0" } });
+    const { isFrontend } = detectTechnologies(tmp.path);
+    strictEqual(isFrontend, true);
+  });
+
   it("detects combos when multiple technologies match", () => {
     writePackageJson(tmp.path, { dependencies: { expo: "^52.0.0", tailwindcss: "^4.0.0" } });
     const { combos } = detectTechnologies(tmp.path);

packages/autoskills/tests/installer.test.ts

@@ -424,6 +424,67 @@ describe("installSkill", () => {
     }
   });
 
+  it("does not send Authorization headers to raw.githubusercontent.com (#124)", async () => {
+    const regDir = join(tmp.path, "registry");
+    const projectDir = join(tmp.path, "project");
+    mkdirSync(projectDir, { recursive: true });
+    buildRegistry(regDir, [
+      { name: "raw-auth-skill", source: "owner/repo", files: { "SKILL.md": "# raw" } },
+    ]);
+    _setRegistryDir(regDir);
+
+    const prevCacheDir = process.env.AUTOSKILLS_CACHE_DIR;
+    const prevToken = process.env.GITHUB_TOKEN;
+    process.env.AUTOSKILLS_CACHE_DIR = join(tmp.path, "raw-auth-cache");
+    process.env.GITHUB_TOKEN = "github_pat_invalid_should_not_be_sent";
+    const seenAuth: Array<string | null> = [];
+    try {
+      const result = await installSkill("owner/repo/raw-auth-skill", [], {
+        projectDir,
+        registryDir: join(tmp.path, "manifest-only"),
+        fetchImpl: (async (url: string | URL | Request, init?: RequestInit) => {
+          const href = typeof url === "string" || url instanceof URL ? String(url) : url.url;
+          ok(href.includes("raw.githubusercontent.com"));
+          const headers = new Headers(init?.headers);
+          seenAuth.push(headers.get("authorization"));
+          return fetchFromRegistry(regDir)(url);
+        }) as typeof fetch,
+      });
+      ok(result.success, result.output);
+      ok(seenAuth.length > 0);
+      ok(seenAuth.every((value) => value === null));
+    } finally {
+      if (prevCacheDir === undefined) delete process.env.AUTOSKILLS_CACHE_DIR;
+      else process.env.AUTOSKILLS_CACHE_DIR = prevCacheDir;
+      if (prevToken === undefined) delete process.env.GITHUB_TOKEN;
+      else process.env.GITHUB_TOKEN = prevToken;
+    }
+  });
+
+  it("accepts local registry text files that were checked out with CRLF on Windows", async () => {
+    const regDir = join(tmp.path, "registry");
+    const projectDir = join(tmp.path, "project");
+    mkdirSync(projectDir, { recursive: true });
+    // Manifest hashes LF content; on-disk file is CRLF (Windows autocrlf).
+    const lf = "---\nname: crlf-skill\n---\n# hello\n";
+    buildRegistry(regDir, [{ name: "crlf-skill", source: "owner/repo", files: { "SKILL.md": lf } }]);
+    writeFileSync(join(regDir, "crlf-skill", "SKILL.md"), lf.replace(/\n/g, "\r\n"));
+    _setRegistryDir(regDir);
+
+    const result = await installSkill("owner/repo/crlf-skill", [], {
+      projectDir,
+      registryDir: regDir,
+      fetchImpl: (async () => {
+        throw new Error("unexpected fetch — should use local registry");
+      }) as typeof fetch,
+    });
+    ok(result.success, result.output);
+    equal(
+      readFileSync(join(projectDir, ".agents", "skills", "crlf-skill", "SKILL.md"), "utf-8"),
+      lf,
+    );
+  });
+
   it("downloads from the raw GitHub registry by default", async () => {
     const regDir = join(tmp.path, "registry");
     const projectDir = join(tmp.path, "project");

packages/autoskills/ui.ts

@@ -126,16 +126,15 @@ export function multiSelect<T>(
     }
 
     const showGroups = groupCount > 1;
-    const visibleGroupCount = showGroups ? groupCount : 0;
-    const separatorCount = showGroups ? groupCount - 1 : 0;
 
-    function renderedLineCount(): number {
-      return items.length + visibleGroupCount + separatorCount + 1;
-    }
+    // Track exact newline count so re-renders clear the previous frame.
+    // Under-counting leaves stale rows (duplicate list items) in Warp/Windows
+    // terminals when navigating with arrows (#126).
+    let linesDrawn = 0;
 
     function clearRendered(): void {
-      if (rendered) {
-        write(`\x1b[${renderedLineCount()}A\r\x1b[J`);
+      if (rendered && linesDrawn > 0) {
+        write(`\x1b[${linesDrawn}A\r\x1b[J`);
       }
     }
 
@@ -149,29 +148,36 @@ export function multiSelect<T>(
       const count = selected.filter(Boolean).length;
       let lastGroup: string | null = null;
       let isFirstGroup = true;
+      let newlines = 0;
+
+      const writeln = (line: string): void => {
+        write(line + "\n");
+        newlines += 1;
+      };
 
       for (let i = 0; i < items.length; i++) {
         if (showGroups && groupFn) {
           const group = groupFn(items[i]);
           if (group !== lastGroup) {
-            if (!isFirstGroup) write("\n");
+            if (!isFirstGroup) writeln("");
             isFirstGroup = false;
             lastGroup = group;
-            write(`   ${bold(yellow(group))}\n`);
+            writeln(`   ${bold(yellow(group))}`);
           }
         }
         const pointer = i === cursor ? cyan("❯") : " ";
         const check = selected[i] ? green("◼") : dim("◻");
         const label = labelFn(items[i], i);
         const hint = hintFn ? hintFn(items[i], i) : "";
-        write(`     ${pointer} ${check} ${label}${hint ? "  " + dim(hint) : ""}\n`);
+        writeln(`     ${pointer} ${check} ${label}${hint ? "  " + dim(hint) : ""}`);
       }
-      write("\n");
+      writeln("");
       const shortcutHints = shortcuts
         .map((s) => white(bold(`[${s.key}]`)) + dim(` ${s.label}`))
         .join(dim(" · "));
       const shortcutPart = shortcuts.length > 0 ? shortcutHints + dim(" · ") : "";
-      write(
+      // Footer ends with a newline so the next clear moves by a stable line count.
+      writeln(
         dim("   ") +
           white(bold("[↑↓]")) +
           dim(" move · ") +
@@ -183,6 +189,7 @@ export function multiSelect<T>(
           white(bold("[enter]")) +
           dim(` confirm (${count}/${items.length})`),
       );
+      linesDrawn = newlines;
     }
 
     write(HIDE_CURSOR);
@@ -198,14 +205,27 @@ export function multiSelect<T>(
     function onData(data: string): void {
       if (settled) return;
 
-      if (data.startsWith("\x1b")) {
-        processKey(data);
-        return;
-      }
-
-      for (const ch of data.replace(/\r\n/g, "\r")) {
-        if (settled) return;
-        processKey(ch);
+      // Parse CSI / single keys so batched arrow sequences from Warp/etc.
+      // each move the cursor once instead of being ignored as a long string.
+      let i = 0;
+      const s = data.replace(/\r\n/g, "\r");
+      while (i < s.length && !settled) {
+        if (s[i] === "\x1b") {
+          if (s[i + 1] === "[") {
+            let j = i + 2;
+            while (j < s.length && /[0-9;]/.test(s[j])) j++;
+            if (j < s.length) {
+              processKey(s.slice(i, j + 1));
+              i = j + 1;
+              continue;
+            }
+          }
+          // Bare ESC or incomplete sequence — skip one char
+          i += 1;
+          continue;
+        }
+        processKey(s[i]);
+        i += 1;
       }
     }