diff --git a/.github/workflows/build-desktop-app.yml b/.github/workflows/build-desktop-app.yml new file mode 100644 index 0000000000..537049bbc8 --- /dev/null +++ b/.github/workflows/build-desktop-app.yml @@ -0,0 +1,443 @@ +name: Nightly desktop app build + +permissions: + contents: write + +on: + push: + branches: + - main + schedule: + # 04:00 UTC daily — offset from the 00:00 UTC daily-ci-checks run so + # they don't compete for runners. Produces fresh downloadable bundles + # off latest main each night. + - cron: '0 4 * * *' + # Lets maintainers kick off a build on demand (handy before tagging a release). + workflow_dispatch: + inputs: + update_track: + description: "Desktop update track to build" + type: choice + options: + - nightly + - stable + default: nightly + app_version: + description: "Stable semver to publish, required when update_track=stable" + type: string + default: "" + publish_updates: + description: "Publish Velopack feed assets when running on main" + type: boolean + default: true + # Run on PRs that touch the bundling plumbing so reviewers get downloadable + # test builds without waiting for a nightly. Scoped to avoid wasting CI + # on unrelated PRs. + pull_request: + paths: + - 'lively.app/**' + - '.github/workflows/build-desktop-app.yml' + - 'install.sh' + - 'lively.installer/packages-config.json' + - 'flatn/**' + +concurrency: + group: build-desktop-app-${{ github.ref_name }} + cancel-in-progress: true + +env: + VELOPACK_VERSION: 0.0.1444-gc245055 + +jobs: + metadata: + name: Desktop update metadata + runs-on: ubuntu-latest + outputs: + version: ${{ steps.meta.outputs.version }} + track: ${{ steps.meta.outputs.track }} + release_tag: ${{ steps.meta.outputs.release_tag }} + release_title: ${{ steps.meta.outputs.release_title }} + update_url: ${{ steps.meta.outputs.update_url }} + publish_updates: ${{ steps.meta.outputs.publish_updates }} + steps: + - name: Compute update metadata + id: meta + env: + EVENT_NAME: ${{ github.event_name }} + REF_NAME: ${{ github.ref_name }} + REPOSITORY: ${{ github.repository }} + RUN_NUMBER: ${{ github.run_number }} + RUN_ATTEMPT: ${{ github.run_attempt }} + INPUT_TRACK: ${{ inputs.update_track }} + INPUT_VERSION: ${{ inputs.app_version }} + INPUT_PUBLISH_UPDATES: ${{ inputs.publish_updates }} + run: | + set -euo pipefail + + track="${INPUT_TRACK:-nightly}" + if [ "$EVENT_NAME" != "workflow_dispatch" ]; then + track="nightly" + fi + if [ "$track" != "nightly" ] && [ "$track" != "stable" ]; then + echo "Unsupported update track: $track" >&2 + exit 1 + fi + + if [ "$track" = "stable" ]; then + version="$INPUT_VERSION" + if [ -z "$version" ]; then + echo "workflow_dispatch input app_version is required for stable builds" >&2 + exit 1 + fi + else + version="0.1.0-nightly.${RUN_NUMBER}.${RUN_ATTEMPT}" + fi + + if ! [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then + echo "Velopack requires a semver2 version, got: $version" >&2 + exit 1 + fi + + release_tag="lively-next-$track" + release_title="lively.next $track" + update_url="https://github.com/$REPOSITORY/releases/download/$release_tag" + publish_updates=false + if [ "$REF_NAME" = "main" ] && [ "$EVENT_NAME" != "pull_request" ]; then + publish_updates=true + fi + if [ "$EVENT_NAME" = "workflow_dispatch" ] && [ "$INPUT_PUBLISH_UPDATES" = "false" ]; then + publish_updates=false + fi + + { + echo "version=$version" + echo "track=$track" + echo "release_tag=$release_tag" + echo "release_title=$release_title" + echo "update_url=$update_url" + echo "publish_updates=$publish_updates" + } >> "$GITHUB_OUTPUT" + + build: + name: Build ${{ matrix.bundle }} + needs: metadata + strategy: + # One platform failing shouldn't abort the others — each bundle is + # independent, and we want whatever OSes build to still publish. + fail-fast: false + matrix: + include: + - os: ubuntu-latest + bundle: lively.next-linux-x64 + target_platform: linux + target_arch: x64 + archive_ext: tar.gz + - os: ubuntu-latest + # Cross-compile Windows from Linux. build.mjs just downloads + # the Windows NW.js + Node.js binaries and stages them; no + # native Windows runner involved (install.sh doesn't cleanly + # run on Windows due to Git Bash path mangling). + bundle: lively.next-win-x64 + target_platform: win + target_arch: x64 + archive_ext: zip + - os: macos-latest + bundle: lively.next-osx-arm64 + target_platform: osx + target_arch: arm64 + archive_ext: tar.gz + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version: '24' + + - name: Setup Python + # node-gyp@9 (pulled in by leveldown etc.) imports + # distutils.version, which was removed in Python 3.12. macOS + # runners ship Python 3.14 by default, so pin an older version + # that still has distutils. Ubuntu's default is currently fine + # but the pin keeps it future-proof. + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-wasip1 + + - name: Restore lively.next dep cache + id: cache-lively-deps + uses: actions/cache/restore@v3 + env: + cache-name: lively-deps + with: + path: | + lively.next-node_modules/ + .puppeteer-browser-cache/ + key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('lively*/package.json', 'flatn/package.json', 'mocha-es6/package.json') }} + + - name: Install lively.next + uses: nick-fields/retry@v3 + with: + timeout_minutes: 20 + max_attempts: 3 + retry_on: error + shell: bash + # --no-desktop skips the NW.js SDK fetch inside install.sh. + # The build step below fetches its own Normal-flavor NW.js for + # the matrix-target platform directly into dist/. + command: chmod +x ./install.sh && ./install.sh --no-desktop + + - name: Save lively.next dep cache + if: ${{ steps.cache-lively-deps.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v3 + env: + cache-name: lively-deps + with: + path: | + lively.next-node_modules/ + .puppeteer-browser-cache/ + key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('lively*/package.json', 'flatn/package.json', 'mocha-es6/package.json') }} + + - name: Install icon-generation tools + # librsvg2-bin → rsvg-convert (SVG rasterizer) + # imagemagick → `magick`/`convert` for .ico + # .icns is built by scripts/build-icns.mjs — pure Node, no system dep + # needed (icnsutils / libicns-utils was removed from Ubuntu 24.04). + run: | + if [ "${{ runner.os }}" = "Linux" ]; then + sudo apt-get update + sudo apt-get install -y librsvg2-bin imagemagick + else + brew install librsvg imagemagick + fi + + - name: Generate app icons + run: bash ./lively.app/scripts/generate-icons.sh + + - name: Build freezer pages + # Build landing-page and loading-screen through the unified freezer + # pipeline so the desktop bundle uses the SWC freezer transform path. + run: env CI=true npm --silent --prefix lively.freezer run build + + - name: Pre-build library snapshot + # Ship the tar.gz that dav.js would otherwise regenerate on every + # server launch (~3-5s on a fresh machine). Start-server.cjs sets + # LIVELY_PREBUILT_LIBRARY_SNAPSHOT and dav.js reads the file into + # memStore directly. + run: node ./lively.server/scripts/build-library-snapshot.cjs + + - name: Build desktop bundle + env: + PACK: '1' + # SDK flavor ships Chromium DevTools. Costs ~20% more bundle + # size but makes every "blank screen, no clue why" bug + # inspectable via Cmd+Opt+I / right-click → Inspect. Switch + # back to FLAVOR=normal once the app is stable. + FLAVOR: sdk + # Stamp the bundle with the commit so user bug reports can + # tell us which version they tested without guessing. For PR + # builds use the head SHA, not GitHub's synthetic merge SHA. + LIVELY_APP_BUILD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + LIVELY_APP_VERSION: ${{ needs.metadata.outputs.version }} + LIVELY_APP_UPDATE_CHANNEL: ${{ needs.metadata.outputs.track }}-${{ matrix.target_platform }}-${{ matrix.target_arch }} + LIVELY_APP_UPDATE_URL: ${{ needs.metadata.outputs.update_url }} + run: node ./lively.app/scripts/build.mjs --platform=${{ matrix.target_platform }} --arch=${{ matrix.target_arch }} + + - name: Setup .NET for Velopack + # vpk pack is platform-native. Windows packaging is handled by the + # package-windows-velopack job from the cross-built raw Windows bundle. + if: ${{ matrix.target_platform != 'win' }} + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Install Velopack CLI + if: ${{ matrix.target_platform != 'win' }} + run: | + dotnet tool install --global vpk --version "$VELOPACK_VERSION" + echo "$HOME/.dotnet/tools" >> "$GITHUB_PATH" + + - name: Build Velopack package + if: ${{ matrix.target_platform != 'win' }} + env: + LIVELY_APP_VERSION: ${{ needs.metadata.outputs.version }} + LIVELY_APP_UPDATE_CHANNEL: ${{ needs.metadata.outputs.track }}-${{ matrix.target_platform }}-${{ matrix.target_arch }} + run: node ./lively.app/scripts/build-velopack.mjs --platform=${{ matrix.target_platform }} --arch=${{ matrix.target_arch }} + + - name: Smoke Velopack update check + if: ${{ matrix.target_platform == 'osx' }} + run: | + node ./lively.app/scripts/smoke-velopack-updates.mjs \ + --artifactDir=dist/velopack/${{ matrix.bundle }} \ + --packagesDir="$RUNNER_TEMP/lively-next-velopack-packages" + + - name: Upload bundle artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.bundle }} + path: dist/${{ matrix.bundle }}.${{ matrix.archive_ext }} + # Free-tier retention; bump or cut per project policy. + retention-days: 30 + if-no-files-found: error + + - name: Upload Velopack artifact + if: ${{ matrix.target_platform != 'win' }} + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.bundle }}-velopack + path: dist/velopack/${{ matrix.bundle }}/** + retention-days: 30 + if-no-files-found: error + + package-windows-velopack: + name: Package lively.next-win-x64 Velopack + needs: + - metadata + - build + runs-on: windows-latest + defaults: + run: + shell: pwsh + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version: '24' + + - name: Setup .NET for Velopack + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Install Velopack CLI + run: | + dotnet tool install --global vpk --version "$env:VELOPACK_VERSION" + "$env:USERPROFILE\.dotnet\tools" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + - name: Download raw Windows bundle + uses: actions/download-artifact@v4 + with: + name: lively.next-win-x64 + path: dist + + - name: Extract raw Windows bundle + run: | + tar -xf dist/lively.next-win-x64.zip -C dist + if (-not (Test-Path dist/lively.next-win-x64/lively.next.exe)) { + throw "Extracted Windows bundle is missing lively.next.exe" + } + + - name: Build Windows Velopack package + env: + LIVELY_APP_VERSION: ${{ needs.metadata.outputs.version }} + LIVELY_APP_UPDATE_CHANNEL: ${{ needs.metadata.outputs.track }}-win-x64 + run: | + node ./lively.app/scripts/build-velopack.mjs ` + --platform=win ` + --arch=x64 ` + --bundleDir="$PWD/dist/lively.next-win-x64" + + - name: Upload Windows Velopack artifact + uses: actions/upload-artifact@v4 + with: + name: lively.next-win-x64-velopack + path: dist/velopack/lively.next-win-x64/** + retention-days: 30 + if-no-files-found: error + + publish-velopack-feed: + name: Publish Velopack update feed + needs: + - metadata + - build + - package-windows-velopack + if: ${{ needs.metadata.outputs.publish_updates == 'true' }} + runs-on: ubuntu-latest + steps: + - name: Download Velopack artifacts + uses: actions/download-artifact@v4 + with: + pattern: lively.next-*-velopack + path: dist/velopack-release + merge-multiple: true + + - name: Publish rolling update release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ needs.metadata.outputs.release_tag }} + RELEASE_TITLE: ${{ needs.metadata.outputs.release_title }} + UPDATE_TRACK: ${{ needs.metadata.outputs.track }} + APP_VERSION: ${{ needs.metadata.outputs.version }} + UPDATE_URL: ${{ needs.metadata.outputs.update_url }} + GITHUB_SHA: ${{ github.sha }} + GITHUB_REPOSITORY: ${{ github.repository }} + run: | + set -euo pipefail + + mapfile -d '' assets < <(find dist/velopack-release -type f -print0 | sort -z) + if [ "${#assets[@]}" -eq 0 ]; then + echo "No Velopack assets found to publish" >&2 + exit 1 + fi + + notes=$(printf 'Rolling %s desktop update feed.\n\nVersion: %s\nCommit: %s\nFeed URL: %s\n' \ + "$UPDATE_TRACK" "$APP_VERSION" "$GITHUB_SHA" "$UPDATE_URL") + release_flags=() + if [ "$UPDATE_TRACK" = "nightly" ]; then + release_flags=(--prerelease --latest=false) + else + release_flags=(--latest) + fi + + if gh release view "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + gh release edit "$RELEASE_TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --title "$RELEASE_TITLE" \ + --notes "$notes" \ + "${release_flags[@]}" + else + gh release create "$RELEASE_TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --target "$GITHUB_SHA" \ + --title "$RELEASE_TITLE" \ + --notes "$notes" \ + "${release_flags[@]}" + fi + + gh release upload "$RELEASE_TAG" "${assets[@]}" \ + --repo "$GITHUB_REPOSITORY" \ + --clobber + + - name: Verify published feed files + env: + UPDATE_TRACK: ${{ needs.metadata.outputs.track }} + UPDATE_URL: ${{ needs.metadata.outputs.update_url }} + run: | + set -euo pipefail + for channel in "$UPDATE_TRACK-linux-x64" "$UPDATE_TRACK-osx-arm64" "$UPDATE_TRACK-win-x64"; do + for file in "releases.$channel.json" "assets.$channel.json" "RELEASES-$channel"; do + curl --retry 3 --retry-delay 2 --fail --silent --show-error --location \ + --output /dev/null "$UPDATE_URL/$file" + done + done diff --git a/.gitignore b/.gitignore index 0e9774a6ae..ed98d81532 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ lively.next-node_modules/ dev-deps/ +dist/ +lively.server/.library-snapshot.tar.gz esm_cache/ .livelydbs/ .module_cache/ diff --git a/flatn/build.js b/flatn/build.js index 829b732aa9..ebb202743d 100644 --- a/flatn/build.js +++ b/flatn/build.js @@ -1,5 +1,6 @@ /* global System,process,__dirname */ import { join as j } from 'path'; +import { fileURLToPath } from 'url'; import fs from 'fs'; import { tmpdir } from './util.js'; import { execSync } from 'child_process'; @@ -8,7 +9,7 @@ import { x, npmFallbackEnv } from './util.js'; const dir = typeof __dirname !== 'undefined' ? __dirname - : System.decanonicalize('flatn/').replace('file://', ''); + : fileURLToPath(System.decanonicalize('flatn/')); const helperBinDir = j(dir, 'bin'); let _npmEnv; @@ -69,7 +70,7 @@ function linkBins (packageSpecs, linkState = {}, verbose = false) { let linkLocation = j(tmpdir(), 'npm-helper-bin-dir'); if (!fs.existsSync(linkLocation)) fs.mkdirSync(linkLocation); packageSpecs.forEach(({ bin, location }) => { - if (location.startsWith('file://')) { location = location.replace(/^file:\/\//, ''); } + if (location.startsWith('file://')) { location = fileURLToPath(location); } if (!bin) return; if (linkState[location]) return; for (let linkName in bin) { diff --git a/flatn/flatn-cjs.js b/flatn/flatn-cjs.js index f068aeea18..b4a85f7c2a 100644 --- a/flatn/flatn-cjs.js +++ b/flatn/flatn-cjs.js @@ -14009,6 +14009,13 @@ var resourceExtension$3 = { /* global process */ +function fileURLToPathname (url) { + const parsed = new URL(url); + let decodedPath = decodeURIComponent(parsed.pathname); + if (parsed.host) decodedPath = `//${parsed.host}${decodedPath}`; + return decodedPath.replace(/^\/([a-z]:\/)/i, '$1').replace(/\\/g, '/'); +} + function exists (path, cb) { return fs.access(path, fs.constants.F_OK, (err) => { if (err) { @@ -14040,7 +14047,7 @@ class NodeJSFileResource extends Resource { get isNodeJSFileResource () { return true; } path () { - return this.url.replace('file://', ''); + return fileURLToPathname(this.url); } async stat () { @@ -14181,7 +14188,7 @@ class NodeJSWindowsFileResource extends NodeJSFileResource { } path () { - return this.url.replace('file:///', ''); + return fileURLToPathname(this.url); } isRoot () { @@ -15498,7 +15505,7 @@ function buildStages (packageSpec, packageMap, dependencyFields) { const dir = typeof __dirname !== 'undefined' ? __dirname - : System.decanonicalize('flatn/').replace('file://', ''); + : url.fileURLToPath(System.decanonicalize('flatn/')); const helperBinDir = path.join(dir, 'bin'); let _npmEnv; @@ -15559,7 +15566,7 @@ function linkBins (packageSpecs, linkState = {}, verbose = false) { let linkLocation = path.join(tmpdir(), 'npm-helper-bin-dir'); if (!fs.existsSync(linkLocation)) fs.mkdirSync(linkLocation); packageSpecs.forEach(({ bin, location }) => { - if (location.startsWith('file://')) { location = location.replace(/^file:\/\//, ''); } + if (location.startsWith('file://')) { location = url.fileURLToPath(location); } if (!bin) return; if (linkState[location]) return; for (let linkName in bin) { @@ -15748,10 +15755,22 @@ function resolveImportMapping(name, mapping, context) { return mapping; } +function equivalentImportMapScopesFor (url) { + if (!url || typeof url !== 'string') return []; + const urls = [url]; + if (url.startsWith('https://ga.jspm.io/')) { + urls.push(url.replace('https://ga.jspm.io/', 'esm://ga.jspm.io/')); + } else if (url.startsWith('esm://ga.jspm.io/')) { + urls.push(url.replace('esm://ga.jspm.io/', 'https://ga.jspm.io/')); + } + return urls; +} + function resolveViaImportMap (id, importMap, importer) { let scope, remapped = importMap.imports?.[id] || null; + const importers = equivalentImportMapScopesFor(importer); if (scope = Object.entries(importMap.scopes || {}) - .filter(([k]) => importer.startsWith(k)) + .filter(([k]) => importers.some(ea => ea.startsWith(k))) .sort((a, b) => a[0].length - b[0].length) .map(([_, scope]) => scope) .reduce((a, b) => ({ ...a, ...b }), false)) { @@ -15785,7 +15804,7 @@ function ensurePathFormat (dirOrArray) { // This ensures that... if (Array.isArray(dirOrArray)) return dirOrArray.map(ensurePathFormat); if (dirOrArray.isResource) return dirOrArray.path(); - if (dirOrArray.startsWith('file://')) dirOrArray = dirOrArray.replace('file://', ''); + if (dirOrArray.startsWith('file://')) dirOrArray = url.fileURLToPath(dirOrArray); return dirOrArray; } @@ -15806,9 +15825,9 @@ function ensurePackageMap (packageCollectionDirs, individualPackageDirs, devPack function packageDirsFromEnv () { let env = process.env; return { - packageCollectionDirs: [...new Set((env.FLATN_PACKAGE_COLLECTION_DIRS || '').split(':').filter(Boolean))], - individualPackageDirs: [...new Set((env.FLATN_PACKAGE_DIRS || '').split(':').filter(Boolean))], - devPackageDirs: [...new Set((env.FLATN_DEV_PACKAGE_DIRS || '').split(':').filter(Boolean))] + packageCollectionDirs: [...new Set((env.FLATN_PACKAGE_COLLECTION_DIRS || '').split(path.delimiter).filter(Boolean))], + individualPackageDirs: [...new Set((env.FLATN_PACKAGE_DIRS || '').split(path.delimiter).filter(Boolean))], + devPackageDirs: [...new Set((env.FLATN_DEV_PACKAGE_DIRS || '').split(path.delimiter).filter(Boolean))] }; } @@ -15816,9 +15835,9 @@ function setPackageDirsOfEnv (packageCollectionDirs, individualPackageDirs, devP packageCollectionDirs = ensurePathFormat(packageCollectionDirs); individualPackageDirs = ensurePathFormat(individualPackageDirs); devPackageDirs = ensurePathFormat(devPackageDirs); - process.env.FLATN_PACKAGE_COLLECTION_DIRS = packageCollectionDirs.join(':'); - process.env.FLATN_PACKAGE_DIRS = individualPackageDirs.join(':'); - process.env.FLATN_DEV_PACKAGE_DIRS = devPackageDirs.join(':'); + process.env.FLATN_PACKAGE_COLLECTION_DIRS = packageCollectionDirs.join(path.delimiter); + process.env.FLATN_PACKAGE_DIRS = individualPackageDirs.join(path.delimiter); + process.env.FLATN_DEV_PACKAGE_DIRS = devPackageDirs.join(path.delimiter); } async function buildPackage ( diff --git a/flatn/helpers.mjs b/flatn/helpers.mjs index 37e61956d6..88846c200f 100644 --- a/flatn/helpers.mjs +++ b/flatn/helpers.mjs @@ -43,10 +43,22 @@ export function resolveImportMapping(name, mapping, context) { return mapping; } +function equivalentImportMapScopesFor (url) { + if (!url || typeof url !== 'string') return []; + const urls = [url]; + if (url.startsWith('https://ga.jspm.io/')) { + urls.push(url.replace('https://ga.jspm.io/', 'esm://ga.jspm.io/')); + } else if (url.startsWith('esm://ga.jspm.io/')) { + urls.push(url.replace('esm://ga.jspm.io/', 'https://ga.jspm.io/')); + } + return urls; +} + export function resolveViaImportMap (id, importMap, importer) { let scope, remapped = importMap.imports?.[id] || null; + const importers = equivalentImportMapScopesFor(importer); if (scope = Object.entries(importMap.scopes || {}) - .filter(([k]) => importer.startsWith(k)) + .filter(([k]) => importers.some(ea => ea.startsWith(k))) .sort((a, b) => a[0].length - b[0].length) .map(([_, scope]) => scope) .reduce((a, b) => ({ ...a, ...b }), false)) { @@ -59,4 +71,4 @@ export function resolveViaImportMap (id, importMap, importer) { if (remapped) { return remapped; } -} \ No newline at end of file +} diff --git a/flatn/index.js b/flatn/index.js index 22e3d5dd49..d1c5ec0836 100644 --- a/flatn/index.js +++ b/flatn/index.js @@ -1,9 +1,10 @@ /* global process, global */ -import { dirname, join as j } from 'path'; +import { delimiter, dirname, join as j } from 'path'; import fs from 'fs'; import { inspect } from 'util'; import semver from 'semver'; import node_fetch from 'node-fetch'; +import { fileURLToPath } from 'url'; export { default as parseArgs } from 'minimist'; import { packageDownload } from './download.js'; @@ -29,7 +30,7 @@ function ensurePathFormat (dirOrArray) { // This ensures that... if (Array.isArray(dirOrArray)) return dirOrArray.map(ensurePathFormat); if (dirOrArray.isResource) return dirOrArray.path(); - if (dirOrArray.startsWith('file://')) dirOrArray = dirOrArray.replace('file://', ''); + if (dirOrArray.startsWith('file://')) dirOrArray = fileURLToPath(dirOrArray); return dirOrArray; } @@ -50,9 +51,9 @@ function ensurePackageMap (packageCollectionDirs, individualPackageDirs, devPack function packageDirsFromEnv () { let env = process.env; return { - packageCollectionDirs: [...new Set((env.FLATN_PACKAGE_COLLECTION_DIRS || '').split(':').filter(Boolean))], - individualPackageDirs: [...new Set((env.FLATN_PACKAGE_DIRS || '').split(':').filter(Boolean))], - devPackageDirs: [...new Set((env.FLATN_DEV_PACKAGE_DIRS || '').split(':').filter(Boolean))] + packageCollectionDirs: [...new Set((env.FLATN_PACKAGE_COLLECTION_DIRS || '').split(delimiter).filter(Boolean))], + individualPackageDirs: [...new Set((env.FLATN_PACKAGE_DIRS || '').split(delimiter).filter(Boolean))], + devPackageDirs: [...new Set((env.FLATN_DEV_PACKAGE_DIRS || '').split(delimiter).filter(Boolean))] }; } @@ -60,9 +61,9 @@ function setPackageDirsOfEnv (packageCollectionDirs, individualPackageDirs, devP packageCollectionDirs = ensurePathFormat(packageCollectionDirs); individualPackageDirs = ensurePathFormat(individualPackageDirs); devPackageDirs = ensurePathFormat(devPackageDirs); - process.env.FLATN_PACKAGE_COLLECTION_DIRS = packageCollectionDirs.join(':'); - process.env.FLATN_PACKAGE_DIRS = individualPackageDirs.join(':'); - process.env.FLATN_DEV_PACKAGE_DIRS = devPackageDirs.join(':'); + process.env.FLATN_PACKAGE_COLLECTION_DIRS = packageCollectionDirs.join(delimiter); + process.env.FLATN_PACKAGE_DIRS = individualPackageDirs.join(delimiter); + process.env.FLATN_DEV_PACKAGE_DIRS = devPackageDirs.join(delimiter); } async function buildPackage ( diff --git a/flatn/module-resolver.js b/flatn/module-resolver.js index b27690dbce..2d12dd4dc6 100644 --- a/flatn/module-resolver.js +++ b/flatn/module-resolver.js @@ -7,6 +7,30 @@ process.execPath = process.argv[0] = path.join(__dirname, 'bin/node'); const moduleUrlToConfig = new Map(); +function equivalentModuleUrls (url) { + if (!url || typeof url !== 'string') return []; + const urls = [url]; + if (url.startsWith('https://ga.jspm.io/')) { + urls.push(url.replace('https://ga.jspm.io/', 'esm://ga.jspm.io/')); + } else if (url.startsWith('esm://ga.jspm.io/')) { + urls.push(url.replace('esm://ga.jspm.io/', 'https://ga.jspm.io/')); + } + return urls; +} + +function configForModuleUrl (modulePath) { + for (const url of equivalentModuleUrls(modulePath)) { + if (moduleUrlToConfig.has(url)) return moduleUrlToConfig.get(url); + } +} + +function rememberConfigForModuleUrl (modulePath, config) { + if (!config) return; + for (const url of equivalentModuleUrls(modulePath)) { + if (!moduleUrlToConfig.has(url)) moduleUrlToConfig.set(url, config); + } +} + /** * Handles the proper base name resolution of @ prefixed package names or * SystemJS specific import mappings. @@ -64,9 +88,8 @@ function traverseUntilPkgDir (modulePath, cb) { */ function findPackageConfig (modulePath) { let configs = []; - if (moduleUrlToConfig.has(modulePath)) { - return moduleUrlToConfig.get(modulePath); - } + const cachedConfig = configForModuleUrl(modulePath); + if (cachedConfig) return cachedConfig; traverseUntilPkgDir(modulePath, (dir) => { let config; if (fs.existsSync(path.join(dir, 'package.json'))) { @@ -186,7 +209,7 @@ function flatnResolve (request, parentId = '', context = 'node') { if (basename === '@empty') return '@empty'; if (basename && basename.match(/^(https|esm)?\:\/\//)) { - if (config && !moduleUrlToConfig.has(basename)) moduleUrlToConfig.set(basename, config); + rememberConfigForModuleUrl(basename, config); return basename; } if (resolved) { diff --git a/flatn/resolver.mjs b/flatn/resolver.mjs index 4ba1d66202..59ea8db18e 100644 --- a/flatn/resolver.mjs +++ b/flatn/resolver.mjs @@ -1,5 +1,6 @@ /*global process, URL */ import path from 'path'; +import { fileURLToPath, pathToFileURL } from 'url'; import { flatnResolve } from './module-resolver.js'; process.execPath = process.argv[0] = path.join(import.meta.url, 'bin/node'); @@ -12,8 +13,8 @@ export async function resolve(request, parent, originalResolve) { result = await originalResolve(request, parent, originalResolve); return result; } catch (err) { - if (result = flatnResolve(request, new URL(parent.parentURL).pathname, 'node-import')) { - return { url: 'file://' + result }; + if (result = flatnResolve(request, fileURLToPath(parent.parentURL), 'node-import')) { + return { url: pathToFileURL(result).href }; } throw err; } diff --git a/install.sh b/install.sh index 4688b96e62..9a4266dfad 100755 --- a/install.sh +++ b/install.sh @@ -105,11 +105,11 @@ fi export NODE_OPTIONS="--no-warnings --experimental-modules --loader $lv_next_dir/flatn/resolver.mjs"; section "Installing packages" -node lively.installer/install-with-node.js $PWD \ +node lively.installer/install-with-node.js $PWD || exit 1 section "Building class runtime" step "Compiling lively.classes runtime..." -env CI=true npm --silent --prefix $lv_next_dir/lively.classes/ run build +env CI=true npm --silent --prefix $lv_next_dir/lively.classes/ run build || exit 1 step "Class runtime built" if [ "$1" = "--freezer-only" ]; @@ -129,13 +129,24 @@ step "SWC plugin built" section "Building freezer bundles" if [ -z "${CI}" ]; then step "Building unified bundle (landing page + loading screen)..." - env CI=true npm --silent --prefix $lv_next_dir/lively.freezer/ run build-unified + env CI=true npm --silent --prefix $lv_next_dir/lively.freezer/ run build-unified || exit 1 else step "Building loading screen..." - env CI=true npm --silent --prefix $lv_next_dir/lively.freezer/ run build-loading-screen + env CI=true npm --silent --prefix $lv_next_dir/lively.freezer/ run build-loading-screen || exit 1 +fi + +if [ -d "$lv_next_dir/lively.app" ] && [ "$1" != "--no-desktop" ]; then + section "Setting up lively.app desktop binary" + # The `nw` npm package's postinstall can't decompress through flatn's flat + # layout, so we download the NW.js SDK directly. + if bash "$lv_next_dir/lively.app/setup.sh"; then + step "NW.js SDK ready (launch the desktop app with: bash lively.app/start.sh)" + else + warn "lively.app setup failed — the web server still works, but the desktop app won't launch" + fi fi echo "" echo "Done! Start the server with ./start-server.sh" -echo "Then visit http://localhost:9011" +echo "Or launch the desktop app with ./lively.app/start.sh" echo "" diff --git a/lively.app/.gitignore b/lively.app/.gitignore new file mode 100644 index 0000000000..041daa405f --- /dev/null +++ b/lively.app/.gitignore @@ -0,0 +1,5 @@ +boot.log +dist/ +assets/icon.icns +assets/icon.ico +assets/icon.png diff --git a/lively.app/VELOPACK_SPIKE.md b/lively.app/VELOPACK_SPIKE.md new file mode 100644 index 0000000000..8b91524013 --- /dev/null +++ b/lively.app/VELOPACK_SPIKE.md @@ -0,0 +1,226 @@ +# Velopack Spike + +## Current Verdict + +Velopack looks viable for the NW.js desktop bundle, but the integration should +be staged. The JavaScript SDK can load in an NW.js node-main process, and the +packaging model matches what we need: versioned installers/update packages, +static update feeds, GitHub release uploads, and delta updates. + +The main blockers are not SDK compatibility. They are release layout, version +policy, mutable app data, and CI packaging. + +## What Was Validated + +- Official Velopack docs describe it as a cross-platform installer/update + framework for Windows, macOS, and Linux. +- The JavaScript SDK is distributed as the `velopack` npm package. Current npm + metadata showed `0.0.1589-ga2c5a97`. +- The initial implementation pins `velopack@0.0.1444-gc245055`, because that is + the newest version found in both npm and the NuGet `vpk` CLI feed during the + spike. The newer npm-only package would leave CI without a matching CLI. +- The SDK contains native `.node` addons for Linux, macOS, and Windows. +- A plain Node smoke test can `require("velopack")` and see both + `VelopackApp` and `UpdateManager`. +- An NW.js smoke test with the local NW.js SDK `0.110.1` can also load the + native SDK from the NW node-main context. +- Constructing `UpdateManager` outside a Velopack-installed app fails with the + expected "application is not properly installed" error. That means the next + validation step must use a real Velopack package/install layout. +- Linux packaging was validated locally after installing .NET 8.0.420 and + `vpk 0.0.1444-gc245055`. `vpk pack` accepts the current Linux bundle with + `--mainExe nw` and produces an AppImage plus release-feed files. +- After adding the per-user runtime root, the direct bundled server smoke + reaches `Server ready` and stays running until the test timeout reaps it. +- The Linux AppImage smoke was validated with + `APPIMAGE_EXTRACT_AND_RUN=1`, because this host does not provide FUSE for + normal AppImage mounting. The packaged app reaches `Server ready`, loads the + dashboard, and connects the renderer through L2L. + +Useful upstream references: + +- https://docs.velopack.io/ +- https://docs.velopack.io/getting-started/javascript +- https://docs.velopack.io/packaging/overview +- https://docs.velopack.io/reference/js/Class.UpdateManager +- https://docs.velopack.io/reference/cli/content/vpk-linux + +## Current Bundle Shape + +The existing GitHub workflow builds raw archives: + +- Linux: `dist/lively.next-linux-x64.tar.gz` +- Windows: `dist/lively.next-win-x64.zip` +- macOS: `dist/lively.next-osx-arm64.tar.gz` + +The current generated bundle sizes are large: + +- Linux x64: about 2.1 GB locally +- macOS arm64: about 1.9 GB locally + +Current app entry points: + +- Linux: `launch.sh` executes `nw "$BUNDLE_DIR"` +- macOS: `lively.next.app` +- Windows: `lively.next.exe` + +Velopack needs a stable `--mainExe` value for each packaged platform. The +Linux case needs the most care because Velopack produces an AppImage and +expects the entry executable name, not a shell launcher path. + +## Packaging Requirements + +Velopack packaging is a second step after the normal app build: + +```bash +vpk pack \ + --packId next.lively.app \ + --packVersion 0.1.0-nightly.123 \ + --packDir dist/lively.next-- \ + --mainExe \ + --packTitle lively.next \ + --outputDir dist/velopack/- +``` + +The real command should be validated per platform. Based on the current layout, +the likely candidates are: + +- Linux: needs validation; probably `nw` if we make NW run the bundled package + directly, or a small native/Node launcher if Velopack cannot use `launch.sh`. +- macOS: likely `lively.next.app`. +- Windows: `lively.next.exe`. + +The implementation currently wires these defaults: + +- Linux: `--packDir dist/lively.next-linux-x64 --mainExe nw` +- macOS: `--packDir dist/lively.next-osx-arm64/lively.next.app --mainExe nwjs` +- Windows: not packaged on CI yet, because `vpk pack` is platform-native and + the current Windows bundle is cross-built on Linux. + +The validated Linux test output was: + +- `next.lively.app-nightly-linux-x64.AppImage` +- `next.lively.app-0.1.0-test.38-nightly-linux-x64-full.nupkg` +- `releases.nightly-linux-x64.json` +- `assets.nightly-linux-x64.json` +- `RELEASES-nightly-linux-x64` + +Velopack versions must be semver2. Four-part versions are not supported. For +nightly builds, use a deterministic semver such as: + +```text +0.1.0-nightly.+ +``` + +The package id should stay stable. `next.lively.app` matches the current macOS +bundle identifier. + +## Runtime Architecture + +Velopack startup code should run as early as possible in the main process: + +```js +const { VelopackApp } = require("velopack"); +VelopackApp.build().run(); +``` + +For this app, the practical insertion point is `lively.app/desktop/start-server.cjs` +because it runs as NW.js `node-main` before the Lively server process is +started. That keeps the native Velopack addon out of browser/world code and +lets update hooks exit quickly before UI/server startup. + +Update checks should be exposed through the desktop native bridge, not the old +in-world git version checker: + +- background menu item: `Check for Updates...` +- optional startup check with no blocking UI +- progress reporting while downloading +- restart/apply confirmation once the update is ready +- a clear "not installed by Velopack" state for local dev/raw archive builds + +The API shape can mirror Velopack's JS guide: + +- `getVersion()` +- `checkForUpdates()` +- `downloadUpdate(updateInfo, onProgress)` +- `applyUpdate(updateInfo)` + +## CI Integration Plan + +1. Keep the current raw archives while Velopack is introduced. +2. Add .NET setup to `.github/workflows/build-desktop-app.yml`. +3. Install or run the Velopack CLI at the same version as the JS package. +4. After `node lively.app/scripts/build.mjs`, run `vpk pack` for the matrix + platform. +5. Upload Velopack output as a separate artifact first. +6. Once validated, publish Velopack release assets and the + `releases..json` feed to GitHub Releases or static hosting. + +## Data Layout Risk + +Mutable app data must stay out of the bundle directory. Velopack updates replace +the installed app payload, and Linux AppImage payloads are read-only at runtime. + +The implementation now creates a per-user `runtime-root` for bundled desktop +launches. The server runs with that directory as its `System.baseURL` and DAV +root. Immutable packages are symlinked from the installed app payload, while +mutable top-level directories live directly in the runtime root. + +Candidates for per-user writable state: + +- macOS: `~/Library/Application Support/lively.next` +- Windows: `%APPDATA%/lively.next` +- Linux: `${XDG_DATA_HOME:-~/.local/share}/lively.next` + +The current bundle contains directories such as: + +- `app/esm_cache` +- `app/local_projects` +- `app/custom-npm-modules` +- `app/snapshots` + +Those are redirected to the runtime root in bundled mode. `lively.morphic` also +uses a package overlay: most package files are symlinked to the installed app, +but `lively.morphic/objectdb` is kept writable in the runtime root. + +Linux validation output after the runtime-root and AppImage smoke fixes: + +- `next.lively.app-nightly-linux-x64.AppImage` at about 647 MB +- `next.lively.app-0.1.0-test.38-nightly-linux-x64-full.nupkg` at about 645 MB +- `releases.nightly-linux-x64.json` +- `assets.nightly-linux-x64.json` +- `RELEASES-nightly-linux-x64` + +Smoke validation: + +- direct bundled server entry: `Server ready, loading lively...`, then timeout + exit `124` after the server stayed up +- AppImage with `APPIMAGE_EXTRACT_AND_RUN=1`: `Server ready, loading lively...` + followed by an L2L renderer connection, then timeout exit `124` + +## Recommended Phases + +1. Move mutable desktop-app state into per-user data directories, while keeping + local development behavior unchanged. +2. Add Velopack packaging in CI for one platform and keep raw archives. +3. Add startup hook handling in `desktop/start-server.cjs`. +4. Add a native update-check API and menu item that can report "not a Velopack + install" cleanly. +5. Validate download/apply/restart from a local/static update feed. +6. Expand to all platforms, signing, notarization, and published GitHub release + feeds. + +## Open Questions + +- What should the final Linux distribution policy be now that `--mainExe nw` + works for the AppImage? +- Should Linux be distributed only as Velopack AppImage, or should the raw + `.tar.gz` remain a supported portable build? +- Which channel names should we use: `nightly`, `stable`, platform-specific + defaults, or both channel plus platform? +- Do we want update checks to be manual-only initially, or a quiet startup + check with no automatic install? +- How do Velopack deltas behave with a 2 GB NW.js/lively.next payload in CI and + over GitHub Releases? +- When do we introduce macOS signing/notarization and Windows Authenticode + signing? diff --git a/lively.app/assets/icon.svg b/lively.app/assets/icon.svg new file mode 100644 index 0000000000..359a050355 --- /dev/null +++ b/lively.app/assets/icon.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + diff --git a/lively.app/desktop/background-menu.js b/lively.app/desktop/background-menu.js new file mode 100644 index 0000000000..71109ce0d7 --- /dev/null +++ b/lively.app/desktop/background-menu.js @@ -0,0 +1,383 @@ +// Persistent NW.js background-page menu. +// +// NW.js explicitly warns that menus created in navigable pages stop working +// after navigation/reload. This script lives in `bg-script`, so the menu and +// its callbacks survive the boot.html -> dashboard transition. + +(function () { + 'use strict'; + + const fs = require('fs'); + const os = require('os'); + const path = require('path'); + const { createUpdateService } = require('./updates.cjs'); + + function uniquePaths (paths) { + return Array.from(new Set(paths.filter(Boolean).map(p => path.resolve(p)))); + } + + function candidateRootsFrom (base) { + if (!base) return []; + const roots = []; + let dir = path.resolve(base); + try { + if (fs.existsSync(dir) && !fs.statSync(dir).isDirectory()) dir = path.dirname(dir); + } catch (_) {} + + for (let i = 0; i < 8; i++) { + roots.push( + dir, + path.join(dir, 'app'), + path.join(dir, 'app.nw', 'app'), + path.join(dir, 'Resources', 'app.nw', 'app'), + path.join(dir, 'Contents', 'Resources', 'app.nw', 'app') + ); + const parent = path.dirname(dir); + if (parent === dir) break; + dir = parent; + } + return roots; + } + + function findRootDir (logFn) { + const appStartPath = typeof nw !== 'undefined' && nw.App && nw.App.startPath; + const candidates = uniquePaths([ + ...candidateRootsFrom(__dirname), + ...candidateRootsFrom(process.cwd && process.cwd()), + ...candidateRootsFrom(process.execPath), + ...candidateRootsFrom(process.argv && process.argv[0]), + ...candidateRootsFrom(appStartPath) + ]); + for (const c of candidates) { + if (fs.existsSync(path.join(c, 'lively.installer/packages-config.json'))) return c; + } + if (logFn) { + logFn( + 'rootDir unavailable; __dirname=' + __dirname + + ', cwd=' + (process.cwd && process.cwd()) + + ', execPath=' + process.execPath + + ', appStartPath=' + (appStartPath || '') + + ', checked=' + candidates.slice(0, 20).join(', ') + ); + } + return null; + } + + function desktopDataDir () { + if (process.platform === 'darwin') { + return path.join(os.homedir(), 'Library', 'Application Support', 'lively.next'); + } + if (process.platform === 'win32') { + return path.join(process.env.APPDATA || path.join(os.homedir(), 'AppData', 'Roaming'), 'lively.next'); + } + return path.join(process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share'), 'lively.next'); + } + + const fallbackLogFile = path.join(desktopDataDir(), 'boot.log'); + let logFile = fallbackLogFile; + function log (msg) { + try { + fs.mkdirSync(path.dirname(logFile), { recursive: true }); + fs.appendFileSync(logFile, '[' + new Date().toISOString() + '] menu: ' + msg + '\n'); + } catch (_) {} + } + + const rootDir = findRootDir(log); + const bundled = !rootDir || !__dirname.startsWith(rootDir + path.sep); + logFile = bundled + ? fallbackLogFile + : path.join(rootDir, 'lively.app', 'boot.log'); + log('background menu rootDir=' + (rootDir || '(unavailable)') + ', bundled=' + bundled); + + const updateService = rootDir + ? createUpdateService({ rootDir, desktopDir: __dirname, log }) + : null; + + function withMainWindow (fn, onMissing) { + nw.Window.getAll(function (wins) { + wins = wins || []; + const win = wins.find(isMainWindow) || wins[0]; + if (!win) { + log('menu action skipped: no window'); + if (onMissing) onMissing(); + return; + } + log('menu action using window: ' + windowLocation(win)); + fn(win); + }); + } + + function windowLocation (win) { + try { + return String(win.window && win.window.location && win.window.location.href || ''); + } catch (_) { + return ''; + } + } + + function isMainWindow (win) { + try { + const w = win.window; + if (!w) return false; + if (w.__LIVELY_DESKTOP_APP__ || w.livelyDesktop || w.livelyBoot) return true; + const href = windowLocation(win); + return /\/boot\.html$/.test(href) || /^https?:\/\/(127\.0\.0\.1|localhost):\d+\//.test(href); + } catch (_) { + return false; + } + } + + function pageHelper (win) { + try { + return win.window && win.window.livelyDesktop; + } catch (err) { + log('page helper access failed: ' + err.message); + return null; + } + } + + function showMessage (win, title, message) { + try { + const helper = pageHelper(win); + if (helper && typeof helper.showDesktopMessage === 'function') { + helper.showDesktopMessage(title, message); + return; + } + win.window.alert([title, message].filter(Boolean).join('\n\n')); + } catch (err) { + log('message display failed: ' + (err.stack || err)); + } + } + + function confirmAction (win, title, message) { + try { + const helper = pageHelper(win); + if (helper && typeof helper.confirmDesktopAction === 'function') { + return helper.confirmDesktopAction(title, message); + } + return win.window.confirm([title, message].filter(Boolean).join('\n\n')); + } catch (err) { + log('confirmation display failed: ' + (err.stack || err)); + return false; + } + } + + function navigateToDashboard () { + log('Dashboard clicked'); + withMainWindow(function (win) { + try { + const helper = pageHelper(win); + if (helper && typeof helper.navigateToDashboard === 'function') { + helper.navigateToDashboard(); + return; + } + + const href = String(win.window.location && win.window.location.href || ''); + if (/^https?:\/\//.test(href)) { + win.window.location.href = new URL('/dashboard/', href).toString(); + return; + } + + log('dashboard navigation unavailable from href=' + href); + } catch (err) { + log('dashboard click failed: ' + (err.stack || err)); + } + }); + } + + function showDevTools () { + log('Open Dev Tools clicked'); + withMainWindow(function (win) { + try { + const helper = pageHelper(win); + if (helper && typeof helper.showDevTools === 'function') { + try { + if (helper.showDevTools()) return; + } catch (err) { + log('page helper DevTools failed: ' + (err.stack || err)); + } + log('page helper could not open DevTools, falling back to background window'); + } + + if (typeof win.showDevTools !== 'function') { + log('devtools unavailable; NW.js SDK flavor is required'); + return; + } + win.showDevTools(); + } catch (err) { + log('devtools click failed: ' + (err.stack || err)); + } + }); + } + + let updateBusy = false; + let updateMenuItem = null; + + function setUpdateMenuState (label, enabled) { + if (!updateMenuItem) return; + updateMenuItem.label = label; + updateMenuItem.enabled = enabled; + } + + function versionLabel (result) { + return result && (result.version || + result.buildInfo && result.buildInfo.version || + result.buildInfo && result.buildInfo.sha) || + 'unknown'; + } + + function failureDetails (result) { + return [result.message, result.error].filter(Boolean).join('\n\n'); + } + + function closeForUpdate (win) { + try { + win.close(); + } catch (err) { + log('window close for update failed: ' + (err.stack || err)); + } + setTimeout(function () { + try { + if (nw.App && typeof nw.App.quit === 'function') nw.App.quit(); + } catch (err) { + log('app quit for update failed: ' + (err.stack || err)); + } + }, 2500); + } + + function checkForUpdates () { + log('Check for Updates clicked'); + if (updateBusy) return; + if (!updateService) { + log('update check skipped: rootDir unavailable'); + return; + } + + updateBusy = true; + setUpdateMenuState('Checking for Updates...', false); + + withMainWindow(async function (win) { + try { + log('update check started: source=' + updateService.source + ', channel=' + updateService.channel); + const checked = await updateService.checkForUpdates(); + log('update check completed: state=' + checked.state + ', ok=' + checked.ok); + if (!checked.ok) { + showMessage(win, 'lively.next updates', failureDetails(checked)); + return; + } + + if (!checked.updateInfo) { + showMessage( + win, + 'lively.next is up to date', + 'Current version: ' + versionLabel(checked) + ); + return; + } + + const targetVersion = checked.targetVersion || 'latest'; + const download = confirmAction( + win, + 'Update available', + 'Version ' + targetVersion + ' is available. Download it now?' + ); + if (!download) return; + + let lastProgress = -1; + setUpdateMenuState('Downloading Update...', false); + const downloaded = await updateService.downloadUpdate(checked.updateInfo, function (percent) { + const rounded = Math.max(0, Math.min(100, Math.floor(percent))); + if (rounded === lastProgress || rounded < lastProgress + 5 && rounded !== 100) return; + lastProgress = rounded; + log('update download progress: ' + rounded + '%'); + setUpdateMenuState('Downloading Update ' + rounded + '%', false); + }); + + if (!downloaded.ok) { + showMessage(win, 'Update download failed', failureDetails(downloaded)); + return; + } + + const restart = confirmAction( + win, + 'Update ready', + 'Restart lively.next now to apply version ' + targetVersion + '?' + ); + if (!restart) { + showMessage(win, 'Update ready', 'The update will be applied when you restart lively.next.'); + return; + } + + const applying = updateService.applyUpdate(checked.updateInfo, { restart: true }); + if (!applying.ok) { + showMessage(win, 'Update apply failed', failureDetails(applying)); + return; + } + + log('Velopack updater launched, closing app for update'); + closeForUpdate(win); + } catch (err) { + log('update check failed: ' + (err.stack || err)); + showMessage(win, 'Update check failed', err && err.message || String(err)); + } finally { + updateBusy = false; + setUpdateMenuState('Check for Updates...', true); + } + }, function () { + updateBusy = false; + setUpdateMenuState('Check for Updates...', true); + }); + } + + const menu = new nw.Menu({ type: 'menubar' }); + if (process.platform === 'darwin') { + menu.createMacBuiltin('lively.next', { hideEdit: false }); + } + + const goMenu = new nw.Menu(); + const mod = process.platform === 'darwin' ? 'cmd' : 'ctrl'; + goMenu.append(new nw.MenuItem({ + label: 'Dashboard', + key: 'd', + modifiers: mod + '+shift', + click: navigateToDashboard + })); + goMenu.append(new nw.MenuItem({ type: 'separator' })); + goMenu.append(new nw.MenuItem({ + label: 'Open Dev Tools', + key: 'i', + modifiers: mod + '+alt', + click: showDevTools + })); + goMenu.append(new nw.MenuItem({ type: 'separator' })); + updateMenuItem = new nw.MenuItem({ + label: 'Check for Updates...', + click: checkForUpdates + }); + goMenu.append(updateMenuItem); + menu.append(new nw.MenuItem({ label: 'Go', submenu: goMenu })); + + function attachMenu (win, reason) { + try { + win.menu = menu; + log('native menu attached (' + reason + ')'); + } catch (err) { + log('menu attach failed (' + reason + '): ' + (err.stack || err)); + } + } + + function attachMenuWhenWindowExists () { + nw.Window.getAll(function (wins) { + const win = wins && wins[0]; + if (!win) { + setTimeout(attachMenuWhenWindowExists, 250); + return; + } + + win.on('loaded', function () { attachMenu(win, 'loaded'); }); + attachMenu(win, 'initial'); + }); + } + + attachMenuWhenWindowExists(); +})(); diff --git a/lively.app/desktop/boot.html b/lively.app/desktop/boot.html new file mode 100644 index 0000000000..0cddfe6c6e --- /dev/null +++ b/lively.app/desktop/boot.html @@ -0,0 +1,135 @@ + + + + + lively.next + + + + + + + + + + + + + +
+ + + +
+
Starting server…
+
+ + + + diff --git a/lively.app/desktop/inject.js b/lively.app/desktop/inject.js new file mode 100644 index 0000000000..31b561bbdc --- /dev/null +++ b/lively.app/desktop/inject.js @@ -0,0 +1,77 @@ +// Injected into every page loaded in the NW.js window via inject_js_end. +// +// Exposes page-side helpers that the persistent background-page menu can call, +// and keeps a couple of keyboard shortcuts as a fallback on macOS. + +(function () { + 'use strict'; + + // Marker so node-main can verify inject_js_end is actually running. + window.__LIVELY_INJECT_LOADED__ = Date.now(); + window.__LIVELY_DESKTOP_APP__ = true; + + function resolveDashboardUrl () { + if ((window.location.protocol === 'http:' || window.location.protocol === 'https:') && + window.location.origin && window.location.origin !== 'null') { + return window.location.origin + '/dashboard/'; + } + + const boot = window.livelyBoot; + if (boot && typeof boot.dashboardUrl === 'string' && boot.dashboardUrl) { + return boot.dashboardUrl; + } + + return ''; + } + + function navigateToDashboard () { + const url = resolveDashboardUrl(); + if (url) window.location.href = url; + } + + function showDevTools () { + try { + if (!window.nw || !window.nw.Window) return false; + window.nw.Window.get().showDevTools(); + return true; + } catch (_) { + return false; + } + } + + function showDesktopMessage (title, message) { + window.alert([title, message].filter(Boolean).join('\n\n')); + } + + function confirmDesktopAction (title, message) { + return window.confirm([title, message].filter(Boolean).join('\n\n')); + } + + window.livelyDesktop = { + navigateToDashboard: navigateToDashboard, + showDevTools: showDevTools, + showDesktopMessage: showDesktopMessage, + confirmDesktopAction: confirmDesktopAction + }; + + // Keyboard shortcut: Cmd/Ctrl + Shift + D → Dashboard. + // Works from any page, regardless of menu/window focus — a reliable + // fallback if the native menu hotkey fails to register. + window.addEventListener('keydown', function (e) { + const mod = e.metaKey || e.ctrlKey; + if (!mod || !e.shiftKey) return; + if (e.key === 'D' || e.key === 'd') { + e.preventDefault(); + navigateToDashboard(); + } + }, true); + + window.addEventListener('keydown', function (e) { + const mod = e.metaKey || e.ctrlKey; + if (!mod || !e.altKey) return; + if (e.key === 'I' || e.key === 'i') { + e.preventDefault(); + showDevTools(); + } + }, true); +})(); diff --git a/lively.app/desktop/server-config.js b/lively.app/desktop/server-config.js new file mode 100644 index 0000000000..260202640c --- /dev/null +++ b/lively.app/desktop/server-config.js @@ -0,0 +1,27 @@ +// NW.js desktop app configuration. +// Excludes plugins that require puppeteer (test-runner, headless) +// since NW.js provides its own Chromium instance. + +var config = { + server: { + authServerURL: "https://auth.lively-next.org", + port: 9011, + hostname: "127.0.0.1", + plugins: [ + "lively.server/plugins/cors.js", + "lively.server/plugins/dav.js", + "lively.server/plugins/eval.js", + "lively.server/plugins/l2l.js", + "lively.server/plugins/lib-lookup.js", + "lively.server/plugins/proxy.js", + "lively.server/plugins/remote-shell.js", + "lively.server/plugins/socketio.js", + "lively.server/plugins/world-loading.js", + "lively.server/plugins/file-upload.js", + "lively.server/plugins/objectdb.js", + "lively.server/plugins/subserver.js" + ] + } +} + +export default config; diff --git a/lively.app/desktop/start-server.cjs b/lively.app/desktop/start-server.cjs new file mode 100644 index 0000000000..fd736b07b4 --- /dev/null +++ b/lively.app/desktop/start-server.cjs @@ -0,0 +1,545 @@ +// NW.js node-main script +// Runs in Node context BEFORE any window opens. +// Boots lively.server, then navigates the window to it. +// +// Works in two modes: +// - Dev mode: lively.app/ inside the monorepo at /lively.app/ +// - Bundled mode: standalone distribution where lively source lives at +// /app/ next to the NW.js binary. The server runs from a +// per-user runtime root so caches/projects/uploads stay outside the app. +// +// ESM resolver hooks (module.register, registerHooks, NODE_OPTIONS) all crash +// NW.js's Blink renderer. So the server runs in a managed child process where +// --experimental-loader works normally. From the user's perspective it's +// invisible — launch the app, lively starts, close the window, everything stops. + +const path = require('path'); +const fs = require('fs'); +const net = require('net'); +const os = require('os'); +const { spawn, execSync } = require('child_process'); +const { pathToFileURL } = require('url'); +const { runVelopackStartup } = require('./updates.cjs'); + +// --------------------------------------------------------------------------- +// 0. Detect mode: dev (monorepo) vs bundled (standalone distribution) +// --------------------------------------------------------------------------- +// Marker: lively.installer/packages-config.json always present at the repo +// (or bundled app) root. + +function findRootDir () { + const candidates = [ + path.resolve(__dirname, '..', '..'), // dev: lively.app/desktop/ → monorepo + path.resolve(__dirname, '..', 'app'), // bundled: desktop/ → ../app/ + path.resolve(__dirname, '..') // fallback: desktop/ → bundle root + ]; + for (const c of candidates) { + if (fs.existsSync(path.join(c, 'lively.installer/packages-config.json'))) return c; + } + throw new Error('Could not locate lively.next root directory from ' + __dirname); +} + +function desktopDataDir () { + if (process.platform === 'darwin') { + return path.join(os.homedir(), 'Library', 'Application Support', 'lively.next'); + } + if (process.platform === 'win32') { + return path.join(process.env.APPDATA || path.join(os.homedir(), 'AppData', 'Roaming'), 'lively.next'); + } + return path.join(process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share'), 'lively.next'); +} + +function desktopCacheDir () { + if (process.platform === 'darwin') { + return path.join(os.homedir(), 'Library', 'Caches', 'lively.next'); + } + if (process.platform === 'win32') { + return path.join(process.env.LOCALAPPDATA || path.join(os.homedir(), 'AppData', 'Local'), 'lively.next', 'Cache'); + } + return path.join(process.env.XDG_CACHE_HOME || path.join(os.homedir(), '.cache'), 'lively.next'); +} + +const sourceRootDir = findRootDir(); +// In dev mode this script lives inside the monorepo, so __dirname is under rootDir. +// In bundled mode this script lives next to the NW.js binary (at /desktop/) +// and rootDir is at /app/ — __dirname is NOT under rootDir. +const bundled = !__dirname.startsWith(sourceRootDir + path.sep); +const appPayloadRoot = bundled ? path.resolve(sourceRootDir, '..') : sourceRootDir; + +// --------------------------------------------------------------------------- +// 1. Logging +// --------------------------------------------------------------------------- +// Dev mode: log to lively.app/boot.log (alongside source). +// Bundled mode: log to ~/.local/share/lively.next/boot.log (user-writable). + +const logFile = bundled + ? path.join(desktopDataDir(), 'boot.log') + : path.join(sourceRootDir, 'lively.app', 'boot.log'); +fs.mkdirSync(path.dirname(logFile), { recursive: true }); +fs.writeFileSync(logFile, ''); +function log (msg) { + fs.appendFileSync(logFile, '[' + new Date().toISOString() + '] ' + msg + '\n'); +} +// Stamp bundle build info so the log identifies the exact commit +let buildInfo = '(no build-info.json)'; +try { + const p = path.join(__dirname, 'build-info.json'); + if (fs.existsSync(p)) buildInfo = fs.readFileSync(p, 'utf8').replace(/\s+/g, ' ').trim(); +} catch (_) {} +log('node-main starting, mode=' + (bundled ? 'bundled' : 'dev') + ', sourceRootDir=' + sourceRootDir); +log('build: ' + buildInfo); + +// Velopack must see its install/update hook arguments before the app starts +// expensive UI/server work. In raw/dev builds this simply reports unavailable. +runVelopackStartup({ rootDir: sourceRootDir, desktopDir: __dirname, log }); + +function ensureSymlink (target, link, type, logFn) { + if (!fs.existsSync(target)) return; + + let shouldCreate = true; + try { + const stat = fs.lstatSync(link); + if (stat.isSymbolicLink()) { + let pointsToTarget = false; + try { + pointsToTarget = fs.realpathSync(link) === fs.realpathSync(target); + } catch (_) { + // AppImage mounts live below /tmp/.mount_*. Between launches those + // mount points disappear, leaving stale runtime-root symlinks behind. + // Broken symlinks must be removed before we can recreate them. + } + if (pointsToTarget) shouldCreate = false; + else fs.rmSync(link, { recursive: true, force: true }); + } else { + shouldCreate = false; + logFn('runtime-root path exists and is not a symlink, keeping it: ' + link); + } + } catch (_) {} + + if (!shouldCreate) return; + fs.mkdirSync(path.dirname(link), { recursive: true }); + fs.symlinkSync(target, link, process.platform === 'win32' && type === 'dir' ? 'junction' : type); +} + +function ensureDirectoryOverlay (sourceDir, targetDir, mutableNames, logFn) { + try { + if (fs.lstatSync(targetDir).isSymbolicLink()) { + fs.rmSync(targetDir, { recursive: true, force: true }); + } + } catch (_) {} + fs.mkdirSync(targetDir, { recursive: true }); + for (const entry of fs.readdirSync(sourceDir, { withFileTypes: true })) { + if (mutableNames.includes(entry.name)) continue; + ensureSymlink( + path.join(sourceDir, entry.name), + path.join(targetDir, entry.name), + entry.isDirectory() ? 'dir' : 'file', + logFn); + } +} + +function copyFileWithMode (source, target) { + const sourceStat = fs.statSync(source); + try { + if (fs.lstatSync(target).isDirectory()) fs.rmSync(target, { recursive: true, force: true }); + } catch (_) {} + fs.copyFileSync(source, target); + fs.chmodSync(target, sourceStat.mode & 0o777); +} + +function ensureDirectoryCopy (sourceDir, targetDir, logFn) { + try { + if (fs.lstatSync(targetDir).isSymbolicLink()) { + fs.rmSync(targetDir, { recursive: true, force: true }); + } + } catch (_) {} + + fs.mkdirSync(targetDir, { recursive: true }); + for (const entry of fs.readdirSync(sourceDir, { withFileTypes: true })) { + const source = path.join(sourceDir, entry.name); + const target = path.join(targetDir, entry.name); + + try { + if (entry.isDirectory()) { + ensureDirectoryCopy(source, target, logFn); + } else if (entry.isSymbolicLink()) { + let shouldCreate = true; + try { + if (fs.lstatSync(target).isSymbolicLink() && fs.readlinkSync(target) === fs.readlinkSync(source)) { + shouldCreate = false; + } else { + fs.rmSync(target, { recursive: true, force: true }); + } + } catch (_) {} + if (shouldCreate) fs.symlinkSync(fs.readlinkSync(source), target); + } else if (entry.isFile()) { + try { + if (fs.lstatSync(target).isSymbolicLink()) fs.rmSync(target, { force: true }); + } catch (_) {} + copyFileWithMode(source, target); + } + } catch (err) { + logFn('failed preparing runtime copy ' + target + ': ' + (err.stack || err)); + } + } +} + +function prepareDesktopRuntimeRoot (sourceRoot, dataDir, logFn) { + const runtimeRoot = path.join(dataDir, 'runtime-root'); + fs.mkdirSync(runtimeRoot, { recursive: true }); + fs.writeFileSync(path.join(runtimeRoot, '.lively-desktop-runtime-root'), + JSON.stringify({ sourceRoot, updatedAt: new Date().toISOString() }, null, 2)); + + for (const d of ['esm_cache', 'snapshots', 'local_projects', 'custom-npm-modules', 'uploads', 'users']) { + fs.mkdirSync(path.join(runtimeRoot, d), { recursive: true }); + } + + const topLevelDirs = fs.readdirSync(sourceRoot, { withFileTypes: true }) + .filter(ea => + ea.isDirectory() && + (ea.name.startsWith('lively.') || + ea.name === 'lively-system-interface' || + ea.name === 'flatn' || + ea.name === 'mocha-es6' || + ea.name === 'scripts' || + ea.name === 'assets' || + ea.name === 'documents' || + ea.name === 'doc-style' || + ea.name === 'lively.next-node_modules')) + .map(ea => ea.name); + + for (const name of topLevelDirs) { + if (['esm_cache', 'snapshots', 'local_projects', 'custom-npm-modules'].includes(name)) continue; + if (name === 'lively.morphic') { + ensureDirectoryOverlay( + path.join(sourceRoot, name), + path.join(runtimeRoot, name), + ['objectdb'], + logFn); + for (const d of ['morphicdb', 'morphicdb/snapshots', 'morphicdb-commits', 'morphicdb-version-graph']) { + fs.mkdirSync(path.join(runtimeRoot, name, 'objectdb', d), { recursive: true }); + } + continue; + } + if (name === 'lively.server') { + ensureDirectoryOverlay( + path.join(sourceRoot, name), + path.join(runtimeRoot, name), + ['.module_cache'], + logFn); + fs.mkdirSync(path.join(runtimeRoot, name, '.module_cache'), { recursive: true }); + continue; + } + if (name === 'lively.shell') { + ensureDirectoryOverlay( + path.join(sourceRoot, name), + path.join(runtimeRoot, name), + ['bin'], + logFn); + ensureDirectoryCopy( + path.join(sourceRoot, name, 'bin'), + path.join(runtimeRoot, name, 'bin'), + logFn); + continue; + } + ensureSymlink(path.join(sourceRoot, name), path.join(runtimeRoot, name), 'dir', logFn); + } + + for (const name of ['config.js', 'localconfig.js', 'conf.json', 'chrome.json', 'favicon.ico', 'README.md', 'LICENSE']) { + ensureSymlink(path.join(sourceRoot, name), path.join(runtimeRoot, name), 'file', logFn); + } + + logFn('desktop runtime root ready: ' + runtimeRoot); + return runtimeRoot; +} + +const rootDir = bundled + ? prepareDesktopRuntimeRoot(sourceRootDir, desktopDataDir(), log) + : sourceRootDir; +if (rootDir !== sourceRootDir) log('server rootDir=' + rootDir); + +// --------------------------------------------------------------------------- +// 2. Locate the desktop/ directory (always next to this script) +// --------------------------------------------------------------------------- + +const desktopDir = __dirname; + +// --------------------------------------------------------------------------- +// 3. Locate a node binary +// --------------------------------------------------------------------------- +// Bundled mode: look in the packaged Node.js directory. +// Dev mode: first PATH entry that isn't flatn/bin/node. + +function findNodeBinary () { + const nodeName = process.platform === 'win32' ? 'node.exe' : 'node'; + const bundleCandidates = [ + path.resolve(__dirname, '..', 'node', 'bin', nodeName), + path.resolve(__dirname, '..', 'node', nodeName) + ]; + const bundleNode = bundleCandidates.find(candidate => fs.existsSync(candidate)); + if (bundleNode) return bundleNode; + + try { + const lookup = process.platform === 'win32' ? 'where node' : 'which -a node'; + const found = execSync(lookup, { encoding: 'utf8' }) + .split('\n').map(p => p.trim()) + .find(p => p && !p.replace(/\\/g, '/').includes('/flatn/')); + if (found) return found; + } catch (_) {} + throw new Error('No node binary found (checked bundle and PATH)'); +} + +function bundledGitPathEntries () { + const gitRoot = path.join(appPayloadRoot, 'tools', 'git'); + if (process.platform !== 'win32' || !fs.existsSync(gitRoot)) return []; + return [ + path.join(gitRoot, 'cmd'), + path.join(gitRoot, 'bin'), + path.join(gitRoot, 'usr', 'bin'), + path.join(gitRoot, 'mingw64', 'bin') + ].filter(d => fs.existsSync(d)); +} + +function findBundledWindowsBash () { + if (process.platform !== 'win32') return null; + const gitRoot = path.join(appPayloadRoot, 'tools', 'git'); + const candidates = [ + path.join(gitRoot, 'bin', 'bash.exe'), + path.join(gitRoot, 'usr', 'bin', 'bash.exe'), + path.join(gitRoot, 'usr', 'bin', 'sh.exe') + ]; + return candidates.find(candidate => fs.existsSync(candidate)) || null; +} + +// --------------------------------------------------------------------------- +// 4. Helpers +// --------------------------------------------------------------------------- + +function findFreePort (start) { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.listen(start, '127.0.0.1', () => { + const port = server.address().port; + server.close(() => resolve(port)); + }); + server.on('error', () => findFreePort(start + 1).then(resolve, reject)); + }); +} + +function waitForServer (port, timeout = 120000) { + const start = Date.now(); + return new Promise((resolve, reject) => { + (function attempt () { + if (Date.now() - start > timeout) return reject(new Error('Server start timed out')); + const sock = net.connect(port, '127.0.0.1'); + sock.on('connect', () => { sock.destroy(); resolve(); }); + sock.on('error', () => setTimeout(attempt, 500)); + })(); + }); +} + +function livelyBoot () { + try { + const w = nw.Window.get().window; + return w && w.livelyBoot; // undefined until boot.html's script runs + } catch (_) { return null; } +} + +function asImportSpecifier (filePath) { + return process.platform === 'win32' ? pathToFileURL(filePath).href : filePath; +} + +function pathEnvValue (env) { + const key = Object.keys(env || {}).find(key => + process.platform === 'win32' ? key.toLowerCase() === 'path' : key === 'PATH'); + return key ? env[key] : ''; +} + +function withPathEnv (env, value) { + const result = { ...env }; + if (process.platform === 'win32') { + for (const key of Object.keys(result)) { + if (key.toLowerCase() === 'path') delete result[key]; + } + result.Path = value; + } else { + result.PATH = value; + } + return result; +} + +function emitStatus (msg) { + log(msg); + const b = livelyBoot(); + if (b && b.status) b.status(msg); +} + +function emitError (msg) { + log('ERROR: ' + msg); + const b = livelyBoot(); + if (b && b.error) b.error(msg); +} + +// --------------------------------------------------------------------------- +// 5. Flatn env setup +// --------------------------------------------------------------------------- +// In dev mode start.sh sources scripts/lively-next-env.sh before launching. +// In bundled mode there's no launcher script — we set the env vars here. + +function setupFlatnEnv () { + if (process.env.FLATN_DEV_PACKAGE_DIRS) return; // already set by launcher + const collectionRoots = Array.from(new Set([rootDir, sourceRootDir].filter(Boolean))); + const pkgs = JSON.parse(fs.readFileSync( + path.join(sourceRootDir, 'lively.installer/packages-config.json'), 'utf8')); + const devDirs = pkgs + .map(p => path.join(rootDir, p.name)) + .filter(d => fs.existsSync(d)); + const localProjects = path.join(rootDir, 'local_projects'); + if (fs.existsSync(localProjects)) { + for (const d of fs.readdirSync(localProjects, { withFileTypes: true })) { + if (d.isDirectory()) devDirs.push(path.join(localProjects, d.name)); + } + } + const collectionDirs = collectionRoots.flatMap(root => [ + path.join(root, 'lively.next-node_modules'), + path.join(root, 'custom-npm-modules') + ]).filter(d => fs.existsSync(d)); + process.env.FLATN_PACKAGE_COLLECTION_DIRS = Array.from(new Set(collectionDirs)).join(path.delimiter); + process.env.FLATN_DEV_PACKAGE_DIRS = Array.from(new Set(devDirs)).join(path.delimiter); + process.env.FLATN_PACKAGE_DIRS = ''; + process.env.lv_next_dir = rootDir; +} + +// --------------------------------------------------------------------------- +// 6. Boot +// --------------------------------------------------------------------------- + +(async () => { + setupFlatnEnv(); + + // Runtime directories the server's library-snapshot step expects. Excluded + // from the bundle since they're populated at runtime; create empty ones + // on first launch. + for (const d of ['esm_cache', 'snapshots', 'local_projects', 'custom-npm-modules']) { + fs.mkdirSync(path.join(rootDir, d), { recursive: true }); + } + + emitStatus('Finding free port...'); + const port = await findFreePort(9011); + + let configFile = path.join(desktopDir, 'server-config.js'); + if (!fs.existsSync(configFile)) configFile = path.join(rootDir, 'config.js'); + if (!fs.existsSync(configFile)) configFile = path.join(rootDir, 'lively.installer/assets/config.js'); + + const nodeBin = findNodeBinary(); + const bundledGitDirs = bundledGitPathEntries(); + const bundledWindowsBash = findBundledWindowsBash(); + const commandPath = [ + path.join(sourceRootDir, 'flatn', 'bin'), + ...bundledGitDirs, + path.dirname(nodeBin), + pathEnvValue(process.env) + ].filter(Boolean).join(path.delimiter); + log('Using node: ' + nodeBin); + if (bundledGitDirs.length) log('Using bundled Git for Windows: ' + path.join(appPayloadRoot, 'tools', 'git')); + else if (process.platform === 'win32') log('Bundled Git for Windows not found; falling back to PATH'); + if (bundledWindowsBash) log('Using Windows shell: ' + bundledWindowsBash); + + // Per-user cache directory for V8 bytecode + (pre-built) snapshot mtime stamp + const userCacheDir = desktopCacheDir(); + const v8CacheDir = path.join(userCacheDir, 'v8'); + const moduleTranslationCacheDir = path.join(userCacheDir, 'module-translation-cache'); + fs.mkdirSync(v8CacheDir, { recursive: true }); + fs.mkdirSync(moduleTranslationCacheDir, { recursive: true }); + + // If the bundle ships a pre-built library snapshot, point dav.js at it so + // the server skips the tar+gzip step on every startup. + const prebuiltSnapshot = bundled + ? path.join(sourceRootDir, 'lively.server', '.library-snapshot.tar.gz') + : ''; + + emitStatus('Starting lively.server on 127.0.0.1:' + port + '...'); + + const childEnv = withPathEnv({ + ...process.env, + ENTR_SUPPORT: '0', + NODE_OPTIONS: '', + LIVELY_DESKTOP_APP: '1', + LIVELY_APP_PARENT_PID: String(process.pid), + ...(process.platform === 'win32' ? { + HOME: process.env.HOME || process.env.USERPROFILE || os.homedir(), + LIVELY_WINDOWS_BASH: bundledWindowsBash || '' + } : {}), + LIVELY_MODULE_TRANSLATION_CACHE_DIR: moduleTranslationCacheDir, + // Node 22+ caches V8 bytecode to this dir — makes launches after + // the first much faster (20-40% typically). + NODE_COMPILE_CACHE: v8CacheDir, + // Use the pre-built library snapshot if the bundle shipped one. + ...(prebuiltSnapshot && fs.existsSync(prebuiltSnapshot) + ? { LIVELY_PREBUILT_LIBRARY_SNAPSHOT: prebuiltSnapshot } + : {}) + }, commandPath); + + const child = spawn(nodeBin, [ + '--no-warnings', + '--dns-result-order', 'ipv4first', + // Parent-death watchdog (first so other preloads failing can't orphan us) + '-r', path.join(desktopDir, 'watchdog.cjs'), + // Flatn CJS resolver hook + '-r', path.join(sourceRootDir, 'flatn/resolver.cjs'), + // Flatn ESM resolver hook + '--experimental-loader', asImportSpecifier(path.join(sourceRootDir, 'flatn/resolver.mjs')), + path.join(sourceRootDir, 'lively.server/bin/start-server.js'), + '--root-directory', rootDir, + '--config', configFile, + '--port', String(port), + '--hostname', '127.0.0.1' + ], { + cwd: path.join(rootDir, 'lively.server'), + env: childEnv, + stdio: ['ignore', 'pipe', 'pipe'] + }); + + child.stdout.on('data', d => log('server: ' + d.toString().trimEnd())); + child.stderr.on('data', d => log('server err: ' + d.toString().trimEnd())); + child.on('error', err => { + emitError('Server process failed to start: ' + (err.stack || err)); + }); + child.on('exit', (code, signal) => { + log('Server process exited (code=' + code + ', signal=' + signal + ')'); + if (code !== 0) emitError('Server crashed (exit code ' + code + ', signal ' + signal + ')'); + }); + + emitStatus('Waiting for server...'); + await waitForServer(port); + + emitStatus('Server ready, loading lively...'); + + const dashboardUrl = 'http://127.0.0.1:' + port + '/dashboard/'; + if (typeof nw === 'undefined') { + log('NW.js global not available; server is ready for direct smoke mode.'); + return; + } + + const win = nw.Window.get(); + + const b = livelyBoot(); + if (b && b.setDashboardUrl) b.setDashboardUrl(dashboardUrl); + if (b && b.navigate) b.navigate(dashboardUrl); + else { + // boot.html's script hasn't run yet — fall back and hope the direct + // assignment works on this platform. Shouldn't happen in practice + // since server boot takes many seconds by which point boot.html is + // long loaded, but be defensive. + log('livelyBoot helper missing, using direct location.href assignment'); + win.window.location.href = dashboardUrl; + } + + win.on('close', function () { + log('Window closing, killing server...'); + child.kill('SIGTERM'); + setTimeout(() => this.close(true), 2000); + }); +})().catch(err => { + emitError('Boot failed: ' + (err.stack || err)); +}); diff --git a/lively.app/desktop/updates.cjs b/lively.app/desktop/updates.cjs new file mode 100644 index 0000000000..886894f712 --- /dev/null +++ b/lively.app/desktop/updates.cjs @@ -0,0 +1,816 @@ +// Velopack integration for the NW.js desktop app. +// +// This module intentionally stays CJS and desktop-only. The Velopack JS SDK is +// native-backed, so browser/world code should only talk to it through the native +// desktop menu/bridge. + +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const Module = require('module'); +const { spawn } = require('child_process'); + +const VELOPACK_PACKAGE = 'velopack'; +const NEON_LOAD_PACKAGE = '@neon-rs/load'; + +let velopackModule = null; +let velopackLoadError = null; +let flatnResolverInstalled = false; + +function noop () {} + +function readJson (file) { + try { + return JSON.parse(fs.readFileSync(file, 'utf8')); + } catch (_) { + return null; + } +} + +function packageDirName (name) { + return name.replace(/\//g, '__SLASH__'); +} + +function findFlatnPackageDir (rootDir, name) { + const parent = path.join(rootDir, 'lively.next-node_modules', packageDirName(name)); + if (!fs.existsSync(parent)) return null; + + const versions = fs.readdirSync(parent, { withFileTypes: true }) + .filter(ea => ea.isDirectory()) + .map(ea => ea.name) + .sort((a, b) => a.localeCompare(b, undefined, { numeric: true })); + + return versions.length ? path.join(parent, versions[versions.length - 1]) : null; +} + +function packageMain (dir) { + const pkg = readJson(path.join(dir, 'package.json')); + return path.join(dir, pkg && pkg.main ? pkg.main : 'index.js'); +} + +function installFlatnResolver (rootDir) { + if (flatnResolverInstalled) return; + + const velopackDir = findFlatnPackageDir(rootDir, VELOPACK_PACKAGE); + const neonLoadDir = findFlatnPackageDir(rootDir, NEON_LOAD_PACKAGE); + if (!velopackDir || !neonLoadDir) return; + + const velopackMain = packageMain(velopackDir); + const neonLoadMain = packageMain(neonLoadDir); + const resolveFilename = Module._resolveFilename; + + Module._resolveFilename = function livelyDesktopVelopackResolve (request, parent, isMain, options) { + if (request === VELOPACK_PACKAGE) return velopackMain; + if (request === NEON_LOAD_PACKAGE) return neonLoadMain; + return resolveFilename.call(this, request, parent, isMain, options); + }; + + flatnResolverInstalled = true; +} + +function loadVelopack (rootDir, log = noop) { + if (velopackModule) return { ok: true, module: velopackModule }; + if (velopackLoadError) return { ok: false, error: velopackLoadError }; + if (nwWindowsSdkUnavailable()) { + velopackLoadError = new Error('Velopack native SDK is not loaded in-process under NW.js on Windows. Use a bundled Node helper process for Velopack operations.'); + return { ok: false, error: velopackLoadError }; + } + + try { + // Packaged builds use flatn instead of a conventional node_modules tree. + // Install these aliases before the first require; unresolved package lookup + // can be surprisingly expensive in NW.js. + log('Velopack SDK installing flatn resolver'); + installFlatnResolver(rootDir); + log('Velopack SDK requiring package'); + velopackModule = require(VELOPACK_PACKAGE); + log('Velopack SDK package required'); + return { ok: true, module: velopackModule }; + } catch (err) { + velopackLoadError = err; + return { ok: false, error: err }; + } +} + +function readBuildInfo (desktopDir) { + const info = readJson(path.join(desktopDir, 'build-info.json')); + return info || {}; +} + +function updateSourceFrom (buildInfo) { + return process.env.LIVELY_APP_UPDATE_URL || buildInfo.updateUrl || ''; +} + +function updateChannelFrom (buildInfo) { + return process.env.LIVELY_APP_UPDATE_CHANNEL || buildInfo.updateChannel || ''; +} + +function serializableError (err) { + const msg = err && (err.stack || err.message || String(err)); + return msg || 'Unknown Velopack error'; +} + +function errorMessage (err) { + const msg = err && (err.message || String(err)); + return msg || 'Unknown Velopack error'; +} + +function nwWindowsSdkUnavailable () { + return process.platform === 'win32' && typeof nw !== 'undefined'; +} + +function classifyError (err) { + const message = err && (err.message || String(err)) || ''; + if (/not properly installed|Could not locate|auto-locate app manifest/i.test(message)) return 'not-installed'; + if (/Cannot find module|no precompiled module|unsupported|not loaded in-process|helper process/i.test(message)) return 'sdk-unavailable'; + return 'error'; +} + +function assetVersion (asset) { + return asset && asset.Version || ''; +} + +function updateVersion (updateInfo) { + return updateInfo && updateInfo.TargetFullRelease && assetVersion(updateInfo.TargetFullRelease) || ''; +} + +function createManager (Velopack, source, channel, locator) { + const options = channel ? { ExplicitChannel: channel } : undefined; + return new Velopack.UpdateManager(source, options, locator || undefined); +} + +function pathIsInside (candidate, parent) { + const rel = path.relative(parent, candidate); + return rel === '' || (!!rel && !rel.startsWith('..') && !path.isAbsolute(rel)); +} + +function ancestorAppBundles (start) { + if (!start) return []; + + let dir = start; + try { + const stat = fs.existsSync(dir) && fs.statSync(dir); + if (stat && stat.isFile()) dir = path.dirname(dir); + } catch (_) {} + + const bundles = []; + while (dir && dir !== path.dirname(dir)) { + if (/\.app$/i.test(dir)) bundles.push(dir); + dir = path.dirname(dir); + } + + return bundles; +} + +function uniquePaths (paths) { + const seen = new Set(); + return paths.filter(file => { + if (!file || seen.has(file)) return false; + seen.add(file); + return true; + }); +} + +function appBundleForRootDir (rootDir) { + const candidates = uniquePaths([ + ...ancestorAppBundles(rootDir), + ...ancestorAppBundles(process.execPath), + ...ancestorAppBundles(process.argv && process.argv[0]) + ]); + const resolvedRoot = rootDir && path.resolve(rootDir); + + if (resolvedRoot) { + const rootBundle = candidates.find(bundle => { + const appNwDir = path.join(bundle, 'Contents', 'Resources', 'app.nw'); + return pathIsInside(resolvedRoot, appNwDir); + }); + if (rootBundle) return rootBundle; + } + + const bundleWithAppNw = candidates.find(bundle => { + return fs.existsSync(path.join(bundle, 'Contents', 'Resources', 'app.nw')); + }); + if (bundleWithAppNw) return bundleWithAppNw; + + return candidates[candidates.length - 1] || null; +} + +function manifestId (manifestPath) { + let source = ''; + try { + source = fs.readFileSync(manifestPath, 'utf8'); + } catch (_) { + return ''; + } + + const match = source.match(/([^<]+)<\/id>/i); + return match && match[1] || ''; +} + +function defaultMacPackagesDir (appId) { + return path.join(os.homedir(), 'Library', 'Caches', 'velopack', appId, 'packages'); +} + +function windowsAppContentDir (rootDir, desktopDir) { + const candidates = uniquePaths([ + rootDir && path.resolve(rootDir, '..'), + desktopDir && path.resolve(desktopDir, '..'), + rootDir, + desktopDir + ]); + return candidates.find(dir => { + return dir && + fs.existsSync(path.join(dir, 'desktop')) && + fs.existsSync(path.join(dir, 'app')); + }) || candidates[0] || null; +} + +function firstExistingPath (paths) { + return paths.find(file => file && fs.existsSync(file)) || paths.find(Boolean) || null; +} + +function createWindowsVelopackLocator (rootDir, options = {}) { + if (process.platform !== 'win32') return null; + + const currentDir = windowsAppContentDir(rootDir, options.desktopDir); + if (!currentDir) return null; + + const parentDir = path.dirname(currentDir); + const updateExePath = firstExistingPath([ + path.join(parentDir, 'Update.exe'), + path.join(currentDir, 'Update.exe') + ]); + const manifestPath = firstExistingPath([ + path.join(currentDir, 'sq.version'), + path.join(parentDir, 'sq.version') + ]); + const isPortable = + fs.existsSync(path.join(currentDir, 'Update.exe')) || + path.basename(currentDir).toLowerCase() !== 'current'; + + return { + RootAppDir: currentDir, + UpdateExePath: updateExePath, + PackagesDir: options.packagesDir || path.join(isPortable ? currentDir : parentDir, 'packages'), + ManifestPath: manifestPath, + CurrentBinaryDir: currentDir, + IsPortable: isPortable + }; +} + +function createVelopackLocator (rootDir, options = {}) { + if (process.platform === 'win32') return createWindowsVelopackLocator(rootDir, options); + if (process.platform !== 'darwin') return null; + + const bundleDir = appBundleForRootDir(rootDir); + if (!bundleDir) return null; + + const macosDir = path.join(bundleDir, 'Contents', 'MacOS'); + const updateExePath = path.join(macosDir, 'UpdateMac'); + const manifestPath = path.join(macosDir, 'sq.version'); + const appId = manifestId(manifestPath) || 'next.lively.app'; + + return { + RootAppDir: bundleDir, + UpdateExePath: updateExePath, + PackagesDir: options.packagesDir || defaultMacPackagesDir(appId), + ManifestPath: manifestPath, + CurrentBinaryDir: macosDir, + IsPortable: true + }; +} + +function macInstallProblem (rootDir, locator = createVelopackLocator(rootDir)) { + if (process.platform !== 'darwin' || !locator) return null; + + const required = [ + locator.UpdateExePath, + locator.ManifestPath + ]; + const missing = required.filter(file => !fs.existsSync(file)); + if (!missing.length) return null; + + return { + bundleDir: locator.RootAppDir, + missing, + message: [ + 'This app is missing Velopack updater files, so it cannot self-update.', + '', + 'Expected files:', + ...missing.map(file => '- ' + file), + '', + 'Install the macOS Setup.pkg from the lively.next-osx-arm64-velopack artifact. If this app was copied from the raw NW.js bundle, or an older /Applications/lively.next.app was kept during installation, Velopack cannot manage it.' + ].join('\n') + }; +} + +function updateCheckTimeoutMs () { + const timeout = Number(process.env.LIVELY_APP_UPDATE_CHECK_TIMEOUT_MS || 30000); + return Number.isFinite(timeout) && timeout > 0 ? timeout : 30000; +} + +function helperCommandTimeoutMs (command) { + if (command === 'check') return updateCheckTimeoutMs() + 10000; + if (command === 'status') return 10000; + return 0; +} + +function withTimeout (promise, timeout, label) { + let timer = null; + return Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(label + ' timed out after ' + timeout + 'ms')), timeout); + }) + ]).finally(() => { + if (timer) clearTimeout(timer); + }); +} + +function findBundledNodeBinary (rootDir, desktopDir) { + const nodeName = process.platform === 'win32' ? 'node.exe' : 'node'; + const candidates = uniquePaths([ + rootDir && path.resolve(rootDir, '..', 'node', nodeName), + rootDir && path.resolve(rootDir, '..', 'node', 'bin', nodeName), + desktopDir && path.resolve(desktopDir, '..', 'node', nodeName), + desktopDir && path.resolve(desktopDir, '..', 'node', 'bin', nodeName) + ]); + return candidates.find(file => fs.existsSync(file)) || null; +} + +function windowsHelperConfig (rootDir, desktopDir) { + if (process.platform !== 'win32' || typeof nw === 'undefined') return null; + const node = findBundledNodeBinary(rootDir, desktopDir); + const helper = desktopDir && path.join(desktopDir, 'velopack-helper.cjs'); + if (!node || !helper || !fs.existsSync(helper)) return null; + return { node, helper }; +} + +function helperUnavailable (source, channel, buildInfo, message, extra = {}) { + return { + ok: false, + state: 'helper-unavailable', + message, + source, + channel, + buildInfo, + ...extra + }; +} + +function invokeWindowsHelper ({ command, rootDir, desktopDir, payload = {}, log = noop, progress = noop }) { + const config = windowsHelperConfig(rootDir, desktopDir); + const buildInfo = readBuildInfo(desktopDir); + const source = updateSourceFrom(buildInfo); + const channel = updateChannelFrom(buildInfo); + if (!config) { + return Promise.resolve(helperUnavailable( + source, + channel, + buildInfo, + 'The Windows Velopack helper is not available in this build.' + )); + } + + return new Promise(resolve => { + const child = spawn(config.node, [config.helper], { + cwd: path.dirname(desktopDir), + env: process.env, + stdio: ['pipe', 'pipe', 'pipe'], + windowsHide: true + }); + let settled = false; + let result = null; + let stderr = ''; + let stdoutBuffer = ''; + let timeout = null; + + function finish (value) { + if (settled) return; + settled = true; + if (timeout) clearTimeout(timeout); + resolve(value); + } + + function parseLine (line) { + if (!line.trim()) return; + let event; + try { + event = JSON.parse(line); + } catch (_) { + log('Velopack helper stdout: ' + line); + return; + } + + if (event.type === 'progress') { + progress(Number(event.percent) || 0); + } else if (event.type === 'log') { + log('Velopack helper: ' + event.message); + } else if (event.type === 'result') { + result = event.result; + } else if (event.type === 'error') { + result = { + ok: false, + state: 'error', + message: 'Velopack helper failed.', + error: event.error + }; + } + } + + function parseStdout (chunk) { + stdoutBuffer += chunk; + const lines = stdoutBuffer.split(/\r?\n/); + stdoutBuffer = lines.pop() || ''; + lines.forEach(parseLine); + } + + child.on('error', err => { + finish(helperUnavailable( + source, + channel, + buildInfo, + 'The Windows Velopack helper could not be started.', + { error: errorMessage(err) } + )); + }); + child.stdout.on('data', data => parseStdout(String(data))); + child.stderr.on('data', data => { stderr += String(data); }); + child.on('close', code => { + if (stdoutBuffer) parseLine(stdoutBuffer); + if (result) return finish(result); + finish(helperUnavailable( + source, + channel, + buildInfo, + 'The Windows Velopack helper exited without a result.', + { error: stderr.trim() || 'exit code ' + code } + )); + }); + + const timeoutMs = helperCommandTimeoutMs(command); + if (timeoutMs > 0) { + timeout = setTimeout(() => { + try { child.kill(); } catch (_) {} + finish(helperUnavailable( + source, + channel, + buildInfo, + 'The Windows Velopack helper timed out.', + { error: command + ' timed out after ' + timeoutMs + 'ms' } + )); + }, timeoutMs); + } + + child.stdin.end(JSON.stringify({ + command, + rootDir, + desktopDir, + ...payload + })); + }); +} + +function startWindowsApplyHelper ({ rootDir, desktopDir, updateInfo, options = {}, log = noop }) { + const config = windowsHelperConfig(rootDir, desktopDir); + const buildInfo = readBuildInfo(desktopDir); + const source = updateSourceFrom(buildInfo); + const channel = updateChannelFrom(buildInfo); + if (!config) { + return helperUnavailable( + source, + channel, + buildInfo, + 'The Windows Velopack helper is not available in this build.' + ); + } + + const payloadFile = path.join( + os.tmpdir(), + 'lively-next-velopack-apply-' + process.pid + '-' + Date.now() + '.json' + ); + fs.writeFileSync(payloadFile, JSON.stringify({ + command: 'apply', + rootDir, + desktopDir, + updateInfo, + options, + parentPid: process.pid + })); + + try { + const child = spawn(config.node, [config.helper, '--payload', payloadFile], { + cwd: path.dirname(desktopDir), + env: process.env, + detached: true, + stdio: 'ignore', + windowsHide: true + }); + child.unref(); + log('Velopack helper apply process started: pid=' + child.pid); + return { + ok: true, + state: 'applying', + source, + channel, + buildInfo, + targetVersion: updateVersion(updateInfo) || assetVersion(updateInfo), + helperPid: child.pid + }; + } catch (err) { + try { fs.rmSync(payloadFile, { force: true }); } catch (_) {} + return helperUnavailable( + source, + channel, + buildInfo, + 'The Windows Velopack helper could not be started.', + { error: errorMessage(err) } + ); + } +} + +function createWindowsHelperUpdateService ({ rootDir, desktopDir, log = noop }) { + const buildInfo = readBuildInfo(desktopDir); + const source = updateSourceFrom(buildInfo); + const channel = updateChannelFrom(buildInfo); + log('Velopack update service initialized: source=' + (source || '(none)') + ', channel=' + (channel || '(none)') + ', helper=windows-node'); + + function unavailable (state, message, extra = {}) { + return { + ok: false, + state, + message, + source, + channel, + buildInfo, + ...extra + }; + } + + function status () { + const config = windowsHelperConfig(rootDir, desktopDir); + if (!source) { + return unavailable('unconfigured', 'No Velopack update feed is configured for this build.'); + } + if (!config) { + return helperUnavailable( + source, + channel, + buildInfo, + 'The Windows Velopack helper is not available in this build.' + ); + } + return { + ok: true, + state: 'helper-ready', + source, + channel, + buildInfo + }; + } + + async function checkForUpdates () { + if (!source) return unavailable('unconfigured', 'No Velopack update feed is configured for this build.'); + return invokeWindowsHelper({ command: 'check', rootDir, desktopDir, log }); + } + + async function downloadUpdate (updateInfo, progress) { + if (!source) return unavailable('unconfigured', 'No Velopack update feed is configured for this build.'); + return invokeWindowsHelper({ + command: 'download', + rootDir, + desktopDir, + payload: { updateInfo }, + log, + progress + }); + } + + function applyUpdate (updateInfoOrAsset, options = {}) { + if (!source) return unavailable('unconfigured', 'No Velopack update feed is configured for this build.'); + if (!updateInfoOrAsset) return unavailable('no-pending-update', 'There is no downloaded update to apply.'); + return startWindowsApplyHelper({ + rootDir, + desktopDir, + updateInfo: updateInfoOrAsset, + options, + log + }); + } + + return { + buildInfo, + source, + channel, + status, + checkForUpdates, + downloadUpdate, + applyUpdate + }; +} + +function runVelopackStartup ({ rootDir, desktopDir, log = noop }) { + log('Velopack startup hook loading SDK'); + const loaded = loadVelopack(rootDir, log); + log('Velopack startup hook SDK ' + (loaded.ok ? 'loaded' : 'unavailable')); + const buildInfo = readBuildInfo(desktopDir); + const locator = createVelopackLocator(rootDir, { desktopDir }); + + if (!loaded.ok) { + log('Velopack SDK unavailable: ' + serializableError(loaded.error)); + return { ok: false, state: 'sdk-unavailable', error: serializableError(loaded.error), buildInfo }; + } + + try { + log('Velopack startup hook running'); + const app = loaded.module.VelopackApp.build() + .setLogger((level, msg) => log('Velopack ' + level + ': ' + msg)); + if (locator) app.setLocator(locator); + app.run(); + log('Velopack startup hook completed'); + return { ok: true, state: 'ready', buildInfo }; + } catch (err) { + log('Velopack startup hook failed: ' + serializableError(err)); + return { ok: false, state: classifyError(err), error: serializableError(err), buildInfo }; + } +} + +function createUpdateService ({ rootDir, desktopDir, log = noop, locator: locatorOverride = null, packagesDir = null }) { + const helperConfig = windowsHelperConfig(rootDir, desktopDir); + if (helperConfig) return createWindowsHelperUpdateService({ rootDir, desktopDir, log }); + + const buildInfo = readBuildInfo(desktopDir); + const source = updateSourceFrom(buildInfo); + const channel = updateChannelFrom(buildInfo); + const locator = locatorOverride || createVelopackLocator(rootDir, { desktopDir, packagesDir }); + log('Velopack update service initialized: source=' + (source || '(none)') + ', channel=' + (channel || '(none)')); + + function unavailable (state, message, extra = {}) { + return { + ok: false, + state, + message, + source, + channel, + buildInfo, + ...extra + }; + } + + function managerResult () { + if (!source) { + return unavailable( + 'unconfigured', + 'No Velopack update feed is configured for this build.' + ); + } + + const loaded = loadVelopack(rootDir); + if (!loaded.ok) { + return unavailable( + 'sdk-unavailable', + 'Velopack is not available in this build.', + { error: errorMessage(loaded.error) } + ); + } + + const installProblem = macInstallProblem(rootDir, locator); + if (installProblem) { + return unavailable( + 'not-installed', + installProblem.message, + { missingFiles: installProblem.missing, bundleDir: installProblem.bundleDir } + ); + } + + try { + const manager = createManager(loaded.module, source, channel, locator); + return { ok: true, manager, source, channel, buildInfo }; + } catch (err) { + return unavailable( + classifyError(err), + classifyError(err) === 'not-installed' + ? 'This app was not installed by Velopack, so it cannot self-update.' + : 'Velopack could not be initialized.', + { error: errorMessage(err) } + ); + } + } + + function status () { + const result = managerResult(); + if (!result.ok) return result; + + try { + return { + ok: true, + state: 'ready', + source, + channel, + buildInfo, + appId: result.manager.getAppId(), + version: result.manager.getCurrentVersion(), + portable: result.manager.isPortable(), + pendingRestart: result.manager.getUpdatePendingRestart() + }; + } catch (err) { + return unavailable(classifyError(err), 'Velopack status is unavailable.', { + error: serializableError(err) + }); + } + } + + async function checkForUpdates () { + const result = managerResult(); + if (!result.ok) return result; + + try { + log('Velopack checking for updates from ' + source + (channel ? ' channel=' + channel : '')); + const updateInfo = await withTimeout( + result.manager.checkForUpdatesAsync(), + updateCheckTimeoutMs(), + 'Velopack update check' + ); + return { + ok: true, + state: updateInfo ? 'update-available' : 'up-to-date', + source, + channel, + buildInfo, + version: result.manager.getCurrentVersion(), + appId: result.manager.getAppId(), + updateInfo, + targetVersion: updateVersion(updateInfo) + }; + } catch (err) { + return unavailable(classifyError(err), 'Update check failed.', { + error: serializableError(err) + }); + } + } + + async function downloadUpdate (updateInfo, progress) { + const result = managerResult(); + if (!result.ok) return result; + + try { + await result.manager.downloadUpdateAsync(updateInfo, progress || noop); + return { + ok: true, + state: 'downloaded', + source, + channel, + buildInfo, + updateInfo, + targetVersion: updateVersion(updateInfo) + }; + } catch (err) { + return unavailable(classifyError(err), 'Update download failed.', { + error: serializableError(err) + }); + } + } + + function applyUpdate (updateInfoOrAsset, { silent = false, restart = true, restartArgs = [] } = {}) { + const result = managerResult(); + if (!result.ok) return result; + + try { + const update = updateInfoOrAsset || result.manager.getUpdatePendingRestart(); + if (!update) { + return unavailable('no-pending-update', 'There is no downloaded update to apply.'); + } + result.manager.waitExitThenApplyUpdate(update, silent, restart, restartArgs); + return { + ok: true, + state: 'applying', + source, + channel, + buildInfo, + targetVersion: updateVersion(update) || assetVersion(update) + }; + } catch (err) { + return unavailable(classifyError(err), 'Update apply failed.', { + error: serializableError(err) + }); + } + } + + return { + buildInfo, + source, + channel, + status, + checkForUpdates, + downloadUpdate, + applyUpdate + }; +} + +module.exports = { + createUpdateService, + runVelopackStartup, + loadVelopack, + appBundleForRootDir, + createVelopackLocator +}; diff --git a/lively.app/desktop/velopack-helper.cjs b/lively.app/desktop/velopack-helper.cjs new file mode 100644 index 0000000000..c48e519cb7 --- /dev/null +++ b/lively.app/desktop/velopack-helper.cjs @@ -0,0 +1,135 @@ +// Windows Velopack helper for the NW.js desktop app. +// +// Velopack's native Node module can hang inside NW.js on Windows. This helper +// is launched with the bundled plain Node.js binary and performs update +// operations on behalf of the NW background menu. + +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { createUpdateService } = require('./updates.cjs'); + +function desktopDataDir () { + if (process.platform === 'win32') { + return path.join(process.env.APPDATA || path.join(os.homedir(), 'AppData', 'Roaming'), 'lively.next'); + } + if (process.platform === 'darwin') { + return path.join(os.homedir(), 'Library', 'Application Support', 'lively.next'); + } + return path.join(process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share'), 'lively.next'); +} + +const logFile = path.join(desktopDataDir(), 'boot.log'); + +function emit (event) { + try { + process.stdout.write(JSON.stringify(event) + '\n'); + } catch (_) {} +} + +function log (msg) { + try { + fs.mkdirSync(path.dirname(logFile), { recursive: true }); + fs.appendFileSync(logFile, '[' + new Date().toISOString() + '] velopack-helper: ' + msg + '\n'); + } catch (_) {} + emit({ type: 'log', message: msg }); +} + +function serializableError (err) { + const msg = err && (err.stack || err.message || String(err)); + return msg || 'Unknown Velopack helper error'; +} + +function readStdin () { + return new Promise((resolve, reject) => { + let input = ''; + process.stdin.setEncoding('utf8'); + process.stdin.on('data', chunk => { input += chunk; }); + process.stdin.on('end', () => resolve(input)); + process.stdin.on('error', reject); + }); +} + +function payloadFileFromArgs () { + const idx = process.argv.indexOf('--payload'); + return idx >= 0 ? process.argv[idx + 1] : null; +} + +async function readPayload () { + const payloadFile = payloadFileFromArgs(); + if (payloadFile) { + try { + return JSON.parse(fs.readFileSync(payloadFile, 'utf8')); + } finally { + try { fs.rmSync(payloadFile, { force: true }); } catch (_) {} + } + } + const input = await readStdin(); + return JSON.parse(input || '{}'); +} + +function processIsAlive (pid) { + if (!pid || pid === process.pid) return false; + try { + process.kill(pid, 0); + return true; + } catch (err) { + return err && err.code === 'EPERM'; + } +} + +function delay (ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function waitForParentExit (parentPid, timeoutMs = 120000) { + if (!parentPid) return; + const deadline = Date.now() + timeoutMs; + log('waiting for parent process to exit: pid=' + parentPid); + while (processIsAlive(parentPid) && Date.now() < deadline) { + await delay(500); + } + if (processIsAlive(parentPid)) { + log('parent process still alive after ' + timeoutMs + 'ms; launching updater anyway'); + } else { + log('parent process exited; launching updater'); + } +} + +async function run () { + const payload = await readPayload(); + const command = payload.command; + const rootDir = payload.rootDir; + const desktopDir = payload.desktopDir || __dirname; + const service = createUpdateService({ rootDir, desktopDir, log }); + let result; + + if (command === 'status') { + result = service.status(); + } else if (command === 'check') { + result = await service.checkForUpdates(); + } else if (command === 'download') { + result = await service.downloadUpdate(payload.updateInfo, percent => { + emit({ type: 'progress', percent }); + }); + } else if (command === 'apply') { + await waitForParentExit(payload.parentPid, payload.parentExitTimeoutMs || 120000); + result = service.applyUpdate(payload.updateInfo, payload.options || {}); + } else { + result = { + ok: false, + state: 'bad-command', + message: 'Unknown Velopack helper command: ' + command + }; + } + + emit({ type: 'result', result }); + process.exit(result && result.ok ? 0 : 1); +} + +run().catch(err => { + const error = serializableError(err); + log('failed: ' + error); + emit({ type: 'error', error }); + process.exit(1); +}); diff --git a/lively.app/desktop/watchdog.cjs b/lively.app/desktop/watchdog.cjs new file mode 100644 index 0000000000..fc17d62a25 --- /dev/null +++ b/lively.app/desktop/watchdog.cjs @@ -0,0 +1,18 @@ +// Parent-death watchdog for the server child process. +// If the NW.js parent dies (crash, SIGKILL, force-quit), exit this process too. +// Preloaded via `node -r watchdog.cjs`. + +const parentPid = Number(process.env.LIVELY_APP_PARENT_PID); +if (!parentPid) return; + +// On Linux, prctl(PR_SET_PDEATHSIG, SIGTERM) is the kernel-level solution, +// but it requires a native addon. Fall back to a polling watchdog (cross-platform). +setInterval(() => { + try { + // Signal 0 = existence check, doesn't actually send a signal. + process.kill(parentPid, 0); + } catch (_) { + // Parent no longer exists — shut down. + process.exit(0); + } +}, 1000).unref(); diff --git a/lively.app/index.js b/lively.app/index.js new file mode 100644 index 0000000000..68fb8d24c7 --- /dev/null +++ b/lively.app/index.js @@ -0,0 +1,16 @@ +// lively.app — programmatic API +// Utilities for NW.js integration, shared between node-main and browser contexts. + +export function isNWjs () { + try { + return typeof nw !== 'undefined' || + (typeof process !== 'undefined' && !!process.versions?.['node-webkit']); + } catch (_) { + return false; + } +} + +export function nwjsFlavor () { + if (!isNWjs()) return null; + return process.versions['nw-flavor'] || 'unknown'; // "sdk" or "normal" +} diff --git a/lively.app/package.json b/lively.app/package.json new file mode 100644 index 0000000000..11bcb27620 --- /dev/null +++ b/lively.app/package.json @@ -0,0 +1,38 @@ +{ + "name": "lively.app", + "version": "0.1.0", + "description": "lively.next as a standalone NW.js desktop application", + "type": "module", + "main": "desktop/boot.html", + "systemjs": { + "main": "index.js" + }, + "bg-script": "desktop/background-menu.js", + "node-main": "desktop/start-server.cjs", + "inject_js_end": "desktop/inject.js", + "single-instance": true, + "window": { + "title": "lively.next", + "width": 1440, + "height": 900, + "min_width": 800, + "min_height": 600, + "position": "center", + "show": true, + "frame": true, + "resizable": true + }, + "chromium-args": "--disable-raf-throttling --remote-debugging-port=9222", + "exports": { + ".": "./index.js" + }, + "dependencies": { + "lively.server": "*", + "lively.installer": "*", + "velopack": "0.0.1444-gc245055" + }, + "scripts": { + "start": "bash start.sh", + "setup": "bash setup.sh" + } +} diff --git a/lively.app/scripts/build-icns.mjs b/lively.app/scripts/build-icns.mjs new file mode 100644 index 0000000000..918e2b8dbd --- /dev/null +++ b/lively.app/scripts/build-icns.mjs @@ -0,0 +1,46 @@ +#!/usr/bin/env node +// Pack a set of square PNGs into an Apple .icns file. Pure Node, no +// dependencies — libicns was dropped from Ubuntu 24.04 so we can't rely +// on png2icns. The .icns format is just a 4-byte magic + 4-byte big-endian +// total size, then a sequence of chunks of the form: +// <4-byte type code><4-byte big-endian size (header + data)> +// Type codes map to icon sizes: +// 16→icp4 32→icp5 64→icp6 128→ic07 256→ic08 512→ic09 1024→ic10 +// +// Usage: node build-icns.mjs /16.png [32.png ...] +// (Filenames only matter to infer size — must be numbered .png.) + +import fs from 'node:fs'; +import path from 'node:path'; + +const SIZE_TO_TYPE = { + 16: 'icp4', 32: 'icp5', 64: 'icp6', + 128: 'ic07', 256: 'ic08', 512: 'ic09', 1024: 'ic10' +}; + +const [, , outFile, ...inputs] = process.argv; +if (!outFile || inputs.length === 0) { + console.error('usage: build-icns.mjs out.icns size1.png size2.png ...'); + process.exit(2); +} + +const chunks = []; +let dataSize = 0; +for (const pngPath of inputs) { + const size = Number(path.basename(pngPath, '.png')); + const type = SIZE_TO_TYPE[size]; + if (!type) { console.warn(`skipping ${pngPath} — size ${size}px has no icns type`); continue; } + const png = fs.readFileSync(pngPath); + const header = Buffer.alloc(8); + header.write(type, 0, 4, 'ascii'); + header.writeUInt32BE(png.length + 8, 4); + chunks.push(Buffer.concat([header, png])); + dataSize += png.length + 8; +} + +const top = Buffer.alloc(8); +top.write('icns', 0, 4, 'ascii'); +top.writeUInt32BE(dataSize + 8, 4); + +fs.writeFileSync(outFile, Buffer.concat([top, ...chunks])); +console.log(`wrote ${outFile} (${chunks.length} sizes, ${dataSize + 8} bytes)`); diff --git a/lively.app/scripts/build-velopack.mjs b/lively.app/scripts/build-velopack.mjs new file mode 100644 index 0000000000..aab07e596f --- /dev/null +++ b/lively.app/scripts/build-velopack.mjs @@ -0,0 +1,251 @@ +#!/usr/bin/env node +// Package an already-built lively.app desktop bundle with Velopack. +// +// This is intentionally a thin wrapper around `vpk pack`: build.mjs owns the +// NW.js/Node/Lively bundle layout, while this script owns version/channel/main +// executable selection for Velopack. + +import fs from 'node:fs'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const APP_DIR = path.resolve(__dirname, '..'); +const ROOT_DIR = path.resolve(APP_DIR, '..'); +const DIST_DIR = path.join(ROOT_DIR, 'dist'); + +const PLATFORM_KV = { linux: 'linux', darwin: 'osx', win32: 'win' }; +const ARCH_KV = { x64: 'x64', arm64: 'arm64', ia32: 'ia32' }; + +function parseArgs () { + const args = {}; + for (const a of process.argv.slice(2)) { + const m = a.match(/^--([^=]+)=(.*)$/); + if (m) args[m[1]] = m[2]; + } + return args; +} + +const args = parseArgs(); +const targetPlatform = args.platform || PLATFORM_KV[process.platform]; +const targetArch = args.arch || ARCH_KV[process.arch]; +const bundleName = args.bundle || `lively.next-${targetPlatform}-${targetArch}`; +const bundleDir = path.resolve(args.bundleDir || path.join(DIST_DIR, bundleName)); +const outputDir = path.resolve(args.outputDir || path.join(DIST_DIR, 'velopack', bundleName)); +const version = args.version || process.env.LIVELY_APP_VERSION || process.env.VPK_PACK_VERSION; +const channel = args.channel || process.env.LIVELY_APP_UPDATE_CHANNEL || process.env.VPK_CHANNEL || `nightly-${targetPlatform}-${targetArch}`; +const runtime = args.runtime || `${targetPlatform}-${targetArch}`; +const packId = args.packId || process.env.VPK_PACK_ID || 'next.lively.app'; +const packTitle = args.packTitle || process.env.VPK_PACK_TITLE || 'lively.next'; +const packAuthors = args.packAuthors || process.env.VPK_PACK_AUTHORS || 'Lively Kernel'; +const delta = args.delta || process.env.VPK_DELTA || 'BestSpeed'; +const vpk = process.env.VPK || 'vpk'; + +function die (msg) { + console.error('ERROR: ' + msg); + process.exit(1); +} + +function semverish (v) { + return /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/.test(v); +} + +function hostCanPackTarget () { + return PLATFORM_KV[process.platform] === targetPlatform; +} + +function mainExeFor () { + if (targetPlatform === 'linux') return 'nw'; + if (targetPlatform === 'win') return 'lively.next.exe'; + if (targetPlatform === 'osx') return 'nwjs'; + die(`Unsupported target platform: ${targetPlatform}`); +} + +function packDirFor () { + if (targetPlatform !== 'osx') return bundleDir; + const appBundle = path.join(bundleDir, 'lively.next.app'); + return fs.existsSync(appBundle) ? appBundle : bundleDir; +} + +function realpath (p) { + return (fs.realpathSync.native || fs.realpathSync)(p); +} + +function pathIsInside (candidate, parent) { + const rel = path.relative(parent, candidate); + return rel === '' || (!!rel && !rel.startsWith('..') && !path.isAbsolute(rel)); +} + +function materializeExternalSymlink (linkPath, packRoot) { + const linkTarget = fs.readlinkSync(linkPath); + const absoluteTarget = path.resolve(path.dirname(linkPath), linkTarget); + let resolvedTarget; + + try { + resolvedTarget = realpath(absoluteTarget); + } catch (_) { + return 'dangling'; + } + + if (pathIsInside(resolvedTarget, packRoot)) return false; + + const targetStat = fs.statSync(resolvedTarget); + const tmpPath = `${linkPath}.velopack-tmp-${process.pid}`; + fs.rmSync(tmpPath, { recursive: true, force: true }); + + if (targetStat.isDirectory()) { + fs.cpSync(resolvedTarget, tmpPath, { recursive: true, dereference: true }); + fs.chmodSync(tmpPath, targetStat.mode); + } else if (targetStat.isFile()) { + fs.copyFileSync(resolvedTarget, tmpPath); + fs.chmodSync(tmpPath, targetStat.mode); + } else { + die(`Velopack cannot package external symlink ${linkPath} -> ${linkTarget}: target is neither a file nor a directory.`); + } + + fs.rmSync(linkPath, { recursive: true, force: true }); + fs.renameSync(tmpPath, linkPath); + console.log(` materialized external symlink: ${path.relative(packRoot, linkPath)} -> ${resolvedTarget}`); + return 'materialized'; +} + +function sanitizeSymlinksForVelopack (packDir) { + const packRoot = realpath(packDir); + let materialized = 0; + let dangling = 0; + + function walk (dir) { + for (const ent of fs.readdirSync(dir, { withFileTypes: true })) { + const entry = path.join(dir, ent.name); + if (ent.isSymbolicLink()) { + const result = materializeExternalSymlink(entry, packRoot); + if (result === 'materialized') materialized++; + else if (result === 'dangling') dangling++; + } else if (ent.isDirectory()) { + walk(entry); + } + } + } + + walk(packDir); + if (materialized) { + console.log(`Materialized ${materialized} external symlink${materialized === 1 ? '' : 's'} before Velopack packaging.`); + } + if (dangling) { + console.log(`Left ${dangling} dangling symlink${dangling === 1 ? '' : 's'} unchanged before Velopack packaging.`); + } +} + +function iconFor () { + const candidates = targetPlatform === 'win' + ? [path.join(APP_DIR, 'assets', 'icon.ico')] + : targetPlatform === 'osx' + ? [path.join(APP_DIR, 'assets', 'icon.icns')] + : [path.join(APP_DIR, 'assets', 'icon.png')]; + return candidates.find(p => fs.existsSync(p)); +} + +function listZipEntries (zipFile) { + return execFileSync('unzip', ['-Z1', zipFile], { + cwd: ROOT_DIR, + encoding: 'utf8', + maxBuffer: 128 * 1024 * 1024 + }).split(/\r?\n/).filter(Boolean); +} + +function listPkgPayloadEntries (pkgFile) { + return execFileSync('pkgutil', ['--payload-files', pkgFile], { + cwd: ROOT_DIR, + encoding: 'utf8', + maxBuffer: 128 * 1024 * 1024 + }).split(/\r?\n/).filter(Boolean); +} + +function assertEntriesContain (label, entries, required) { + const missing = required.filter(({ name, test }) => !entries.some(test)); + if (!missing.length) return; + + die(`${label} is missing Velopack runtime file${missing.length === 1 ? '' : 's'}: ${missing.map(ea => ea.name).join(', ')}`); +} + +function escapeRegExp (string) { + return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function validateMacVelopackOutput () { + const outputs = fs.readdirSync(outputDir).map(name => path.join(outputDir, name)); + const portableZip = outputs.find(file => /-Portable\.zip$/.test(path.basename(file))); + const setupPkg = outputs.find(file => /-Setup\.pkg$/.test(path.basename(file))); + const rootAppEntry = escapeRegExp(`${packTitle}.app`); + const required = [ + { + name: 'Contents/MacOS/UpdateMac', + test: entry => new RegExp(`(^|/)${rootAppEntry}/Contents/MacOS/UpdateMac$`).test(entry) + }, + { + name: 'Contents/MacOS/sq.version', + test: entry => new RegExp(`(^|/)${rootAppEntry}/Contents/MacOS/sq\\.version$`).test(entry) + }, + { + name: 'Contents/Resources/sq.version', + test: entry => new RegExp(`(^|/)${rootAppEntry}/Contents/Resources/sq\\.version$`).test(entry) + } + ]; + + if (!portableZip) die(`Velopack did not produce a macOS portable zip in ${outputDir}`); + if (!setupPkg) die(`Velopack did not produce a macOS setup pkg in ${outputDir}`); + + assertEntriesContain(path.basename(portableZip), listZipEntries(portableZip), required); + assertEntriesContain(path.basename(setupPkg), listPkgPayloadEntries(setupPkg), required); + console.log('Validated macOS Velopack updater files in portable zip and setup pkg.'); +} + +function validateVelopackOutput () { + if (targetPlatform === 'osx') validateMacVelopackOutput(); +} + +if (!targetPlatform || !targetArch) die(`Unsupported platform/arch: ${process.platform}/${process.arch}`); +if (!version) die('No version specified. Pass --version or set LIVELY_APP_VERSION.'); +if (!semverish(version)) die(`Velopack requires a semver2 version, got: ${version}`); + +if (!hostCanPackTarget()) { + console.log(`Skipping Velopack package for ${targetPlatform}-${targetArch}; vpk pack is platform-native and this runner is ${process.platform}.`); + process.exit(0); +} + +if (!fs.existsSync(bundleDir)) die(`Bundle does not exist: ${bundleDir}`); + +fs.mkdirSync(outputDir, { recursive: true }); + +const packDir = packDirFor(); +const icon = iconFor(); +sanitizeSymlinksForVelopack(packDir); +const cmd = [ + '--yes', + '--skip-updates', + 'pack', + '--packId', packId, + '--packVersion', version, + '--packDir', packDir, + '--mainExe', mainExeFor(), + '--packTitle', packTitle, + '--packAuthors', packAuthors, + '--channel', channel, + '--runtime', runtime, + '--delta', delta, + '--outputDir', outputDir +]; + +if (icon) cmd.push('--icon', icon); +if (targetPlatform === 'osx') cmd.push('--bundleId', packId); + +console.log(`Packing ${bundleName} with Velopack`); +console.log(` version: ${version}`); +console.log(` channel: ${channel}`); +console.log(` runtime: ${runtime}`); +console.log(` packDir: ${packDir}`); +console.log(` output: ${outputDir}`); + +execFileSync(vpk, cmd, { cwd: ROOT_DIR, stdio: 'inherit' }); +validateVelopackOutput(); diff --git a/lively.app/scripts/build.mjs b/lively.app/scripts/build.mjs new file mode 100644 index 0000000000..85e97e452c --- /dev/null +++ b/lively.app/scripts/build.mjs @@ -0,0 +1,726 @@ +#!/usr/bin/env node +// Cross-platform bundle builder for lively.app. +// Runs on Linux, macOS, Windows — no bash / rsync dependency. +// +// Produces dist/lively.next--/ — a self-contained +// distribution that launches by double-clicking its native entrypoint. +// +// Usage: +// node lively.app/scripts/build.mjs # build for the current host +// node lively.app/scripts/build.mjs --platform=osx --arch=arm64 +// PACK=1 node lively.app/scripts/build.mjs # also produce tar.gz (linux/osx) or zip (win) +// LOCALES="en-US fr de" node lively.app/scripts/build.mjs # keep additional Chromium locales +// FLAVOR=sdk node lively.app/scripts/build.mjs # SDK build (has DevTools) + +import fs from 'node:fs'; +import path from 'node:path'; +import os from 'node:os'; +import https from 'node:https'; +import crypto from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const APP_DIR = path.resolve(__dirname, '..'); +const ROOT_DIR = path.resolve(APP_DIR, '..'); +const DIST_DIR = path.join(ROOT_DIR, 'dist'); + +const NW_VERSION = process.env.LIVELY_NW_VERSION || '0.111.1'; +const NW_DOWNLOAD_BASE = process.env.LIVELY_NW_DOWNLOAD_BASE || 'https://dl.nwjs.io/live-build/v0.111.1-04292210-39517e80d'; +const NW_DOWNLOAD_BASE_KEY = crypto.createHash('sha1').update(NW_DOWNLOAD_BASE).digest('hex').slice(0, 8); +const NODE_VERSION = '25.6.1'; +const GIT_FOR_WINDOWS_VERSION = process.env.LIVELY_GIT_FOR_WINDOWS_VERSION || '2.54.0'; +const GIT_FOR_WINDOWS_RELEASE = process.env.LIVELY_GIT_FOR_WINDOWS_RELEASE || `v${GIT_FOR_WINDOWS_VERSION}.windows.1`; +const APP_VERSION = process.env.LIVELY_APP_VERSION || '0.1.0'; +const APP_UPDATE_CHANNEL = process.env.LIVELY_APP_UPDATE_CHANNEL || ''; +const APP_UPDATE_URL = process.env.LIVELY_APP_UPDATE_URL || ''; + +// --------------------------------------------------------------------------- +// Platform detection + overrides +// --------------------------------------------------------------------------- + +function parseArgs () { + const args = {}; + for (const a of process.argv.slice(2)) { + const m = a.match(/^--([^=]+)=(.*)$/); + if (m) args[m[1]] = m[2]; + } + return args; +} + +const args = parseArgs(); + +const NW_PLATFORM_KV = { linux: 'linux', darwin: 'osx', win32: 'win' }; +const NODE_PLATFORM_KV = { linux: 'linux', darwin: 'darwin', win32: 'win' }; +const ARCH_KV = { x64: 'x64', arm64: 'arm64', ia32: 'ia32' }; + +const HOST_NW_PLATFORM = NW_PLATFORM_KV[process.platform]; +const HOST_NODE_PLATFORM = NODE_PLATFORM_KV[process.platform]; +const HOST_ARCH = ARCH_KV[process.arch]; + +const TARGET_NW_PLATFORM = args.platform || HOST_NW_PLATFORM; +const TARGET_ARCH = args.arch || HOST_ARCH; +const TARGET_NODE_PLATFORM = TARGET_NW_PLATFORM === 'osx' ? 'darwin' : TARGET_NW_PLATFORM; + +// Normal NW.js flavor for distribution; SDK when DevTools are needed +const FLAVOR = process.env.FLAVOR || 'normal'; +const LOCALES = (process.env.LOCALES || 'en-US').split(/\s+/).filter(Boolean); +const PACK = process.env.PACK === '1'; + +if (!TARGET_NW_PLATFORM || !TARGET_ARCH) { + die(`Unsupported host platform/arch: ${process.platform}/${process.arch}`); +} + +const BUNDLE_NAME = `lively.next-${TARGET_NW_PLATFORM}-${TARGET_ARCH}`; +const BUNDLE = path.join(DIST_DIR, BUNDLE_NAME); + +// NW.js tarball naming differs by flavor: +// Normal flavor: nwjs-vX.Y.Z-- +// SDK flavor: nwjs-sdk-vX.Y.Z-- +const NW_DIR_NAME = FLAVOR === 'normal' + ? `nwjs-v${NW_VERSION}-${TARGET_NW_PLATFORM}-${TARGET_ARCH}` + : `nwjs-${FLAVOR}-v${NW_VERSION}-${TARGET_NW_PLATFORM}-${TARGET_ARCH}`; +const NW_EXT = TARGET_NW_PLATFORM === 'linux' ? 'tar.gz' : 'zip'; + +// Node.js tarball naming: +const NODE_DIR_NAME = `node-v${NODE_VERSION}-${TARGET_NODE_PLATFORM}-${TARGET_ARCH}`; +const NODE_EXT = TARGET_NODE_PLATFORM === 'win' ? 'zip' : 'tar.xz'; + +const GIT_FOR_WINDOWS = { + x64: { + name: `Git-${GIT_FOR_WINDOWS_VERSION}-64-bit.tar.bz2`, + sha256: process.env.LIVELY_GIT_FOR_WINDOWS_SHA256 || 'e1819cee60d09793dde322cdb1170e03663c41cd9265cf45246219fc5e6aeecd' + }, + arm64: { + name: `Git-${GIT_FOR_WINDOWS_VERSION}-arm64.tar.bz2`, + sha256: process.env.LIVELY_GIT_FOR_WINDOWS_SHA256 || 'ce10b24c74ac9c724ab81e2ee30d06e7ee693977a552b8da4e434e909a641847' + } +}; + +// --------------------------------------------------------------------------- +// Pretty logging +// --------------------------------------------------------------------------- + +const ORANGE = '\x1b[1;38;5;208m'; +const NC = '\x1b[0m'; +const section = msg => console.log(`\n${ORANGE}── ${msg} ──${NC}`); +const step = msg => console.log(` ${msg}`); +const die = msg => { console.error(`ERROR: ${msg}`); process.exit(1); }; + +// --------------------------------------------------------------------------- +// Downloads (streamed, follow redirects) +// --------------------------------------------------------------------------- + +function download (url, dest) { + return new Promise((resolve, reject) => { + function attempt (u, redirects = 0) { + if (redirects > 5) return reject(new Error('Too many redirects: ' + u)); + https.get(u, res => { + if ([301, 302, 303, 307, 308].includes(res.statusCode)) { + res.resume(); + return attempt(new URL(res.headers.location, u).toString(), redirects + 1); + } + if (res.statusCode !== 200) { + return reject(new Error(`HTTP ${res.statusCode} for ${u}`)); + } + const total = Number(res.headers['content-length']) || 0; + let got = 0, lastPct = -1; + const out = fs.createWriteStream(dest); + res.on('data', chunk => { + got += chunk.length; + if (total) { + const pct = Math.floor(got / total * 100); + if (pct !== lastPct) { + process.stdout.write(`\r ${pct}% `); + lastPct = pct; + } + } + }); + res.pipe(out); + out.on('finish', () => { out.close(); process.stdout.write('\n'); resolve(); }); + out.on('error', reject); + }).on('error', reject); + } + attempt(url); + }); +} + +// --------------------------------------------------------------------------- +// Archive extraction +// --------------------------------------------------------------------------- +// `tar` ships with all modern Linux/macOS/Windows (Windows since 1803 has +// bsdtar which handles .zip, .tar.gz, .tar.xz). `unzip` is common on Unix. + +function extract (archive, destDir) { + fs.mkdirSync(destDir, { recursive: true }); + if (archive.endsWith('.tar.gz') || archive.endsWith('.tgz')) { + execFileSync('tar', ['xzf', archive, '-C', destDir], { stdio: 'inherit' }); + } else if (archive.endsWith('.tar.bz2') || archive.endsWith('.tbz2')) { + execFileSync('tar', ['xjf', archive, '-C', destDir], { stdio: 'inherit' }); + } else if (archive.endsWith('.tar.xz')) { + execFileSync('tar', ['xJf', archive, '-C', destDir], { stdio: 'inherit' }); + } else if (archive.endsWith('.zip')) { + // Windows + macOS ship bsdtar that reads .zip; on Linux we prefer unzip + if (process.platform === 'linux') { + execFileSync('unzip', ['-q', '-o', archive, '-d', destDir], { stdio: 'inherit' }); + } else { + execFileSync('tar', ['-xf', archive, '-C', destDir], { stdio: 'inherit' }); + } + } else { + die('Unknown archive format: ' + archive); + } +} + +// --------------------------------------------------------------------------- +// Caching wrapper: download once per version into dist/.cache/ +// --------------------------------------------------------------------------- + +async function fetchAndExtract (url, extractTo, flagFile) { + if (fs.existsSync(flagFile)) return; + const cacheDir = path.join(DIST_DIR, '.cache'); + fs.mkdirSync(cacheDir, { recursive: true }); + const archivePath = path.join(cacheDir, path.basename(url)); + if (!fs.existsSync(archivePath)) { + step(`Downloading ${path.basename(url)}...`); + await download(url, archivePath); + } else { + step(`(cached) ${path.basename(url)}`); + } + step(`Extracting to ${path.relative(ROOT_DIR, extractTo)}...`); + extract(archivePath, extractTo); + fs.writeFileSync(flagFile, 'ok'); +} + +async function fetchAndExtractVerified (url, extractTo, flagFile, sha256) { + if (fs.existsSync(flagFile)) return; + const cacheDir = path.join(DIST_DIR, '.cache'); + fs.mkdirSync(cacheDir, { recursive: true }); + const archivePath = path.join(cacheDir, path.basename(url)); + if (!fs.existsSync(archivePath)) { + step(`Downloading ${path.basename(url)}...`); + await download(url, archivePath); + } else { + step(`(cached) ${path.basename(url)}`); + } + if (sha256) { + const actual = crypto.createHash('sha256').update(fs.readFileSync(archivePath)).digest('hex'); + if (actual !== sha256) die(`Checksum mismatch for ${path.basename(url)}: expected ${sha256}, got ${actual}`); + } + step(`Extracting to ${path.relative(ROOT_DIR, extractTo)}...`); + extract(archivePath, extractTo); + fs.writeFileSync(flagFile, 'ok'); +} + +// --------------------------------------------------------------------------- +// Recursive copy with exclude patterns (rsync replacement, pure Node) +// --------------------------------------------------------------------------- + +// Patterns follow a simple subset of rsync/gitignore syntax: +// '/foo/' — anchored: only matches at the source root +// 'foo/' — anywhere: matches any dir called foo/ at any depth +// '**/bar/' — anywhere (explicit form) +// 'foo/*.md' — anywhere: matches *.md inside any foo/ +// Each pattern is compiled to a RegExp over the POSIX-slash path relative +// to the copy source root. + +function compilePattern (pat) { + // Detect if the pattern matches a directory + const isDir = pat.endsWith('/'); + const body = isDir ? pat.slice(0, -1) : pat; + const anchored = body.startsWith('/'); + const parts = (anchored ? body.slice(1) : body).split('/'); + + // Build a regex piece for each segment + const segs = parts.map(seg => { + if (seg === '**') return '(?:.+/)?'; + // escape + translate glob wildcards + return seg + .replace(/[.+^${}()|[\]\\]/g, '\\$&') + .replace(/\*/g, '[^/]*') + .replace(/\?/g, '[^/]'); + }); + + let rx = anchored ? '^' : '(?:^|/)'; + rx += segs.join('/'); + rx += isDir ? '(?:/|$)' : '$'; + return new RegExp(rx); +} + +function makeFilter (patterns) { + const regexes = patterns.map(compilePattern); + return (relPath) => { + const posix = relPath.split(path.sep).join('/'); + for (const r of regexes) { + if (r.test(posix)) return false; // excluded + // also test for trailing slash (directory semantics) + if (r.test(posix + '/')) return false; + } + return true; + }; +} + +function copyMonorepo (src, dst, excludeFilter) { + function walk (currentSrc, currentDst) { + const ents = fs.readdirSync(currentSrc, { withFileTypes: true }); + for (const ent of ents) { + const s = path.join(currentSrc, ent.name); + const d = path.join(currentDst, ent.name); + const rel = path.relative(src, s); + if (!excludeFilter(rel)) continue; + if (ent.isSymbolicLink()) { + const link = fs.readlinkSync(s); + try { fs.symlinkSync(link, d); } catch (_) {} + } else if (ent.isDirectory()) { + fs.mkdirSync(d, { recursive: true }); + walk(s, d); + } else if (ent.isFile()) { + fs.copyFileSync(s, d); + } + } + } + fs.mkdirSync(dst, { recursive: true }); + walk(src, dst); +} + +// --------------------------------------------------------------------------- +// rm -rf +// --------------------------------------------------------------------------- + +function rmrf (p) { + fs.rmSync(p, { recursive: true, force: true }); +} + +function dirSize (dir) { + let total = 0; + function walk (d) { + for (const ent of fs.readdirSync(d, { withFileTypes: true })) { + const p = path.join(d, ent.name); + if (ent.isDirectory()) walk(p); + else if (ent.isFile()) { + try { total += fs.statSync(p).size; } catch (_) {} + } + } + } + walk(dir); + return total; +} + +function humanSize (bytes) { + const u = ['B', 'KB', 'MB', 'GB', 'TB']; + let i = 0; + while (bytes >= 1024 && i < u.length - 1) { bytes /= 1024; i++; } + return `${bytes.toFixed(1)} ${u[i]}`; +} + +// --------------------------------------------------------------------------- +// Platform-specific launcher/layout +// --------------------------------------------------------------------------- + +function finalizeLinux () { + // launch.sh — terminal-friendly entry point + fs.writeFileSync(path.join(BUNDLE, 'launch.sh'), +`#!/bin/bash +BUNDLE_DIR="$(cd "$(dirname "$0")" && pwd)" +exec "$BUNDLE_DIR/nw" "$BUNDLE_DIR" +`, { mode: 0o755 }); + + // freedesktop .desktop entry — double-click target + fs.writeFileSync(path.join(BUNDLE, 'lively-next.desktop'), +`[Desktop Entry] +Type=Application +Version=1.0 +Name=lively.next +Comment=Live, interactive development environment +Exec=%k/../launch.sh +Icon=%k/../icon.png +Terminal=false +Categories=Development;IDE; +StartupWMClass=lively.next +`, { mode: 0o755 }); + + // Bundle-root icon for the .desktop file's Icon= field + const pngIcon = path.join(APP_DIR, 'assets', 'icon.png'); + if (fs.existsSync(pngIcon)) fs.copyFileSync(pngIcon, path.join(BUNDLE, 'icon.png')); +} + +function finalizeMacOS () { + // Turn the bundle into a single self-contained .app: all our files live + // inside nwjs.app/Contents/Resources/app.nw/ (where NW.js expects the app + // payload), then rename nwjs.app → lively.next.app so one .app is the + // entire distribution. + // + // Before: + // / + // nwjs.app/ ← just the NW.js runtime + // credits.html, ... ← junk from the tarball + // package.json ← our NW.js manifest (wrong place!) + // boot.html + // desktop/ + // app/ + // node/ + // + // After: + // / + // lively.next.app/ + // Contents/ + // MacOS/nwjs ← runtime + // Resources/ + // app.nw/ ← our app payload + // package.json + // boot.html + // desktop/ + // app/ + // node/ + // ... + // Info.plist ← patched metadata + + const nwjsApp = path.join(BUNDLE, 'nwjs.app'); + const appNw = path.join(nwjsApp, 'Contents', 'Resources', 'app.nw'); + fs.mkdirSync(appNw, { recursive: true }); + + // Move our payload into Contents/Resources/app.nw/ + for (const f of ['package.json', 'boot.html', 'desktop', 'app', 'node']) { + const src = path.join(BUNDLE, f); + if (fs.existsSync(src)) fs.renameSync(src, path.join(appNw, f)); + } + + // Strip the loose NW.js tarball junk so the bundle root is just the .app + for (const f of fs.readdirSync(BUNDLE)) { + if (f === 'nwjs.app') continue; + rmrf(path.join(BUNDLE, f)); + } + + // App icon: copy lively.app/assets/icon.icns → Contents/Resources/app.icns + // (matches the CFBundleIconFile value "app" we set below). + const icnsSrc = path.join(APP_DIR, 'assets', 'icon.icns'); + if (fs.existsSync(icnsSrc)) { + fs.copyFileSync(icnsSrc, path.join(nwjsApp, 'Contents', 'Resources', 'app.icns')); + // Strip the stock nwjs icon so macOS doesn't fall back to it if + // Info.plist resolution hiccups. + const stock = path.join(nwjsApp, 'Contents', 'Resources', 'nw.icns'); + if (fs.existsSync(stock)) rmrf(stock); + } + + // Patch Info.plist so macOS treats this as our app (not a generic NW.js + // instance that would share state / keychain / crash reports) and + // picks up our icon. + const plist = path.join(nwjsApp, 'Contents', 'Info.plist'); + if (fs.existsSync(plist)) { + let xml = fs.readFileSync(plist, 'utf8'); + xml = xml.replace( + /CFBundleIdentifier<\/key>\s*[^<]*<\/string>/, + 'CFBundleIdentifiernext.lively.app'); + xml = xml.replace( + /CFBundleName<\/key>\s*[^<]*<\/string>/, + 'CFBundleNamelively.next'); + xml = xml.replace( + /CFBundleDisplayName<\/key>\s*[^<]*<\/string>/, + 'CFBundleDisplayNamelively.next'); + xml = xml.replace( + /CFBundleIconFile<\/key>\s*[^<]*<\/string>/, + 'CFBundleIconFileapp'); + fs.writeFileSync(plist, xml); + } + + // Final rename: nwjs.app → lively.next.app + fs.renameSync(nwjsApp, path.join(BUNDLE, 'lively.next.app')); + + // First-run help: an unsigned .app downloaded from the internet is + // quarantined by Gatekeeper ("is damaged and can't be opened"). Ship a + // tiny README next to the .app explaining the workaround until we set + // up code-signing + notarization. See the "macOS code-signing" tracking + // issue in the repo. + fs.writeFileSync(path.join(BUNDLE, 'README-macOS.txt'), +`lively.next for macOS — first-run notice +========================================== + +On first launch macOS may complain that "lively.next is damaged and +can't be opened". This is standard macOS Gatekeeper behavior for +apps downloaded from the internet that aren't code-signed with an +Apple Developer ID. The app is fine — it just needs the quarantine +attribute stripped. + +One-shot fix (Terminal, from this folder): + + xattr -cr lively.next.app + open lively.next.app + +Alternative — GUI: + + 1. Double-click lively.next.app (fails with the "damaged" dialog). + 2. Open System Settings → Privacy & Security. + 3. Scroll to the Security section — an "Open anyway" button + appears for lively.next. Click it. + 4. Confirm in the follow-up dialog. + +Once launched successfully once, subsequent double-clicks work normally. + +We intentionally do not ship an executable "fix" helper next to the app: +macOS applies the same downloaded-file trust checks to helper scripts/apps +inside the archive, so they get blocked for the same reason. + +This will go away once the project sets up Apple Developer code- +signing + notarization for its CI builds. +`); +} + +function finalizeWindows () { + // Rename nw.exe -> lively.next.exe so Windows shell shows the right name. + const fromExe = path.join(BUNDLE, 'nw.exe'); + const toExe = path.join(BUNDLE, 'lively.next.exe'); + if (fs.existsSync(fromExe)) fs.renameSync(fromExe, toExe); + + // launch.bat — optional double-click launcher (users can also just click + // lively.next.exe directly). + fs.writeFileSync(path.join(BUNDLE, 'launch.bat'), +`@echo off +"%~dp0lively.next.exe" "%~dp0." +`); +} + +async function stageGitForWindows () { + if (TARGET_NW_PLATFORM !== 'win') return; + + const spec = GIT_FOR_WINDOWS[TARGET_ARCH]; + if (!spec) die(`No Git for Windows bundle configured for ${TARGET_ARCH}`); + + section(`Fetching Git for Windows v${GIT_FOR_WINDOWS_VERSION} for ${TARGET_ARCH}`); + const gitCache = path.join(DIST_DIR, '.cache', 'git-for-windows', `${GIT_FOR_WINDOWS_VERSION}-${TARGET_ARCH}`); + await fetchAndExtractVerified( + `https://github.com/git-for-windows/git/releases/download/${GIT_FOR_WINDOWS_RELEASE}/${spec.name}`, + gitCache, + path.join(gitCache, '.extracted'), + spec.sha256); + + step('Copying Git for Windows into bundle/tools/git...'); + const gitDst = path.join(BUNDLE, 'tools', 'git'); + rmrf(gitDst); + copyMonorepo(gitCache, gitDst, makeFilter([ + '/.extracted', + '/dev/', + '/etc/mtab', + '/mingw64/share/doc/', + '/mingw64/share/man/', + '/usr/share/doc/', + '/usr/share/man/', + '/usr/share/info/', + '/usr/share/vim/', + '/usr/share/git-gui/', + '/usr/share/gitk/', + '/usr/share/gitweb/' + ])); +} + +// --------------------------------------------------------------------------- +// Build steps +// --------------------------------------------------------------------------- + +async function main () { + section(`Target: ${TARGET_NW_PLATFORM}-${TARGET_ARCH} (flavor: ${FLAVOR})`); + step(`Bundle: ${BUNDLE}`); + + // Fresh bundle + rmrf(BUNDLE); + fs.mkdirSync(BUNDLE, { recursive: true }); + + // ----------------------------------------------------------------------- + // 1. NW.js runtime + // ----------------------------------------------------------------------- + section(`Fetching NW.js ${FLAVOR} v${NW_VERSION} for ${TARGET_NW_PLATFORM}-${TARGET_ARCH}`); + const nwCache = path.join(DIST_DIR, '.cache', 'nw', `${NW_VERSION}-${FLAVOR}-${TARGET_NW_PLATFORM}-${TARGET_ARCH}-${NW_DOWNLOAD_BASE_KEY}`); + await fetchAndExtract( + `${NW_DOWNLOAD_BASE}/${NW_DIR_NAME}.${NW_EXT}`, + nwCache, + path.join(nwCache, '.extracted')); + + // Copy NW.js runtime contents into the bundle root + const nwExtractedRoot = path.join(nwCache, NW_DIR_NAME); + step('Copying NW.js runtime into bundle...'); + copyMonorepo(nwExtractedRoot, BUNDLE, () => true); + + // Strip Chromium locales — keep only what we asked for + const localesDir = path.join(BUNDLE, 'locales'); + if (fs.existsSync(localesDir)) { + step(`Stripping locales (keeping: ${LOCALES.join(', ')})`); + for (const f of fs.readdirSync(localesDir)) { + const keep = LOCALES.some(l => f === `${l}.pak` || f === `${l}.pak.info`); + if (!keep) rmrf(path.join(localesDir, f)); + } + } + + // ----------------------------------------------------------------------- + // 2. Standalone Node.js (for the server subprocess) + // ----------------------------------------------------------------------- + section(`Fetching Node.js v${NODE_VERSION} for ${TARGET_NODE_PLATFORM}-${TARGET_ARCH}`); + const nodeCache = path.join(DIST_DIR, '.cache', 'node', `${NODE_VERSION}-${TARGET_NODE_PLATFORM}-${TARGET_ARCH}`); + await fetchAndExtract( + `https://nodejs.org/dist/v${NODE_VERSION}/${NODE_DIR_NAME}.${NODE_EXT}`, + nodeCache, + path.join(nodeCache, '.extracted')); + + step('Copying Node.js binary into bundle...'); + const nodeBinSrc = TARGET_NODE_PLATFORM === 'win' + ? path.join(nodeCache, NODE_DIR_NAME, 'node.exe') + : path.join(nodeCache, NODE_DIR_NAME, 'bin', 'node'); + const nodeBinDst = TARGET_NODE_PLATFORM === 'win' + ? path.join(BUNDLE, 'node', 'node.exe') + : path.join(BUNDLE, 'node', 'bin', 'node'); + fs.mkdirSync(path.dirname(nodeBinDst), { recursive: true }); + fs.copyFileSync(nodeBinSrc, nodeBinDst); + if (TARGET_NODE_PLATFORM !== 'win') fs.chmodSync(nodeBinDst, 0o755); + + await stageGitForWindows(); + + // ----------------------------------------------------------------------- + // 3. App manifest + desktop/ scripts + boot.html + // ----------------------------------------------------------------------- + section('Copying app manifest + node-main scripts'); + const manifest = JSON.parse(fs.readFileSync(path.join(APP_DIR, 'package.json'), 'utf8')); + delete manifest.dependencies; + delete manifest.exports; + delete manifest.scripts; + manifest.version = APP_VERSION; + manifest.main = 'boot.html'; + manifest['bg-script'] = 'desktop/background-menu.js'; + manifest['node-main'] = 'desktop/start-server.cjs'; + fs.writeFileSync(path.join(BUNDLE, 'package.json'), JSON.stringify(manifest, null, 2)); + + fs.copyFileSync(path.join(APP_DIR, 'desktop', 'boot.html'), path.join(BUNDLE, 'boot.html')); + fs.mkdirSync(path.join(BUNDLE, 'desktop'), { recursive: true }); + for (const f of ['background-menu.js', 'start-server.cjs', 'watchdog.cjs', 'server-config.js', 'inject.js', 'updates.cjs', 'velopack-helper.cjs']) { + fs.copyFileSync(path.join(APP_DIR, 'desktop', f), path.join(BUNDLE, 'desktop', f)); + } + // Stamp the build SHA so boot.log identifies the exact commit, no more + // "which bundle am I running?" confusion across CI reruns. + const buildSha = process.env.LIVELY_APP_BUILD_SHA || '(local)'; + fs.writeFileSync(path.join(BUNDLE, 'desktop', 'build-info.json'), + JSON.stringify({ + sha: buildSha, + builtAt: new Date().toISOString(), + version: APP_VERSION, + updateChannel: APP_UPDATE_CHANNEL, + updateUrl: APP_UPDATE_URL, + nwVersion: NW_VERSION, + nwDownloadBase: NW_DOWNLOAD_BASE, + ...(TARGET_NW_PLATFORM === 'win' ? { gitForWindowsVersion: GIT_FOR_WINDOWS_VERSION } : {}), + platform: TARGET_NW_PLATFORM, + arch: TARGET_ARCH + }, null, 2)); + + // ----------------------------------------------------------------------- + // 4. Monorepo content → bundle/app/ + // ----------------------------------------------------------------------- + section('Copying lively.next source + node_modules into bundle/app/'); + + const crossPlatformNativeExcludes = (() => { + // Strip native bindings from other platforms to save space. + const patterns = []; + const KEEP = { + 'linux-x64': ['linux-x64'], + 'linux-arm64': ['linux-arm64'], + 'osx-x64': ['darwin-x64'], + 'osx-arm64': ['darwin-arm64'], + 'win-x64': ['win32-x64', 'win-x64'], + }; + const keep = KEEP[`${TARGET_NW_PLATFORM}-${TARGET_ARCH}`] || []; + const ALL = ['darwin-arm64', 'darwin-x64', 'linux-x64', 'linux-arm64', 'linux-x64-gnu', 'linux-x64-musl', 'linux-arm64-gnu', 'linux-arm64-musl', 'win32-x64', 'win32-x64-msvc']; + for (const tag of ALL) { + if (keep.some(k => tag.includes(k))) continue; + patterns.push(`lively.next-node_modules/@swc__SLASH__core-${tag}/`); + patterns.push(`lively.next-node_modules/@rollup__SLASH__rollup-${tag}/`); + } + return patterns; + })(); + + const excludes = [ + // Anchored (source-root-relative) — avoid stripping legit nested dirs + // like systemjs/0.21.6/dist/ (that IS the package source). + '/.git/', + '/.claude/', + '/.github/', + '/dist/', + '/esm_cache/', + '/tmp/', + '/.module_cache/', + '/local_projects/', + '/node_modules/', + '/lively.freezer/swc-plugin/target/', + '/lively.freezer/swc-plugin/src/', + '/lively.freezer/swc-plugin/Cargo.toml', + '/lively.freezer/swc-plugin/Cargo.lock', + '/lively.next-node_modules/nw/', + '/lively.next-node_modules/puppeteer/', + '/lively.next-node_modules/puppeteer-core/', + '/lively.next-node_modules/@puppeteer/', + '/lively.headless/chrome-data-dir/', + '/lively.app/dist/', + '/lively.app/boot.log', + // Anywhere + '**/.cachedImportMap.json', + // In-dep cruft + 'lively.next-node_modules/**/test/', + 'lively.next-node_modules/**/tests/', + 'lively.next-node_modules/**/__tests__/', + 'lively.next-node_modules/**/example/', + 'lively.next-node_modules/**/examples/', + 'lively.next-node_modules/**/docs/', + 'lively.next-node_modules/**/*.md', + 'lively.next-node_modules/**/*.markdown', + 'lively.next-node_modules/**/*.map', + 'lively.next-node_modules/**/.bin/', + 'lively.next-node_modules/**/CHANGELOG*', + ...crossPlatformNativeExcludes, + ]; + + step('Copying monorepo (this may take a minute)...'); + copyMonorepo(ROOT_DIR, path.join(BUNDLE, 'app'), makeFilter(excludes)); + + // ----------------------------------------------------------------------- + // 5. Platform-specific launchers / layout + // ----------------------------------------------------------------------- + section('Creating launchers'); + if (TARGET_NW_PLATFORM === 'linux') finalizeLinux(); + else if (TARGET_NW_PLATFORM === 'osx') finalizeMacOS(); + else if (TARGET_NW_PLATFORM === 'win') finalizeWindows(); + + // ----------------------------------------------------------------------- + // 6. Report + optional pack + // ----------------------------------------------------------------------- + section('Bundle complete'); + step(`Location: ${BUNDLE}`); + step(`Size: ${humanSize(dirSize(BUNDLE))}`); + step(''); + if (TARGET_NW_PLATFORM === 'win') { + step(`Run: double-click ${BUNDLE_NAME}\\lively.next.exe`); + } else if (TARGET_NW_PLATFORM === 'osx') { + step(`Run: double-click ${BUNDLE_NAME}/lively.next.app`); + } else { + step(`Run: double-click ${BUNDLE_NAME}/lively-next.desktop (file manager)`); + step(` or ${BUNDLE}/launch.sh (terminal)`); + } + + if (PACK) { + step(''); + if (TARGET_NW_PLATFORM === 'win') { + const zipPath = path.join(DIST_DIR, `${BUNDLE_NAME}.zip`); + step(`Packing ${path.basename(zipPath)}...`); + // Host-dependent: Windows bsdtar and macOS bsdtar both recognize + // `-a` for format-from-extension; GNU tar (Linux) does not and + // needs us to call `zip` directly instead. + if (process.platform === 'linux') { + // zip is preinstalled on ubuntu-latest runners. + execFileSync('zip', ['-r', '-q', zipPath, BUNDLE_NAME], { cwd: DIST_DIR, stdio: 'inherit' }); + } else { + execFileSync('tar', ['-a', '-c', '-f', zipPath, '-C', DIST_DIR, BUNDLE_NAME], { stdio: 'inherit' }); + } + step(`Archive: ${zipPath}`); + } else { + const tgzPath = path.join(DIST_DIR, `${BUNDLE_NAME}.tar.gz`); + step(`Packing ${path.basename(tgzPath)}...`); + execFileSync('tar', ['czf', tgzPath, '-C', DIST_DIR, BUNDLE_NAME], { stdio: 'inherit' }); + step(`Archive: ${tgzPath}`); + } + } +} + +main().catch(err => { console.error(err); process.exit(1); }); diff --git a/lively.app/scripts/generate-icons.sh b/lively.app/scripts/generate-icons.sh new file mode 100755 index 0000000000..269bf18360 --- /dev/null +++ b/lively.app/scripts/generate-icons.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# Rasterize lively.app/assets/icon.svg into platform-specific icon bundles: +# lively.app/assets/icon.icns — macOS .app Contents/Resources/ +# lively.app/assets/icon.ico — Windows .exe embedded icon +# lively.app/assets/icon.png — Linux / window title bar +# +# Hosts: +# Linux → rsvg-convert (librsvg2-bin) + png2icns (libicns-utils) + +# ImageMagick `convert` (imagemagick) +# macOS → rsvg-convert (brew install librsvg) + iconutil (native) + +# ImageMagick (brew install imagemagick) +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +APP_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +ASSETS="$APP_DIR/assets" +SVG="$ASSETS/icon.svg" +TMP="$ASSETS/.tmp-icons" + +if [ ! -f "$SVG" ]; then + echo "Source icon not found: $SVG" + exit 1 +fi + +rm -rf "$TMP" +mkdir -p "$TMP" + +SIZES="16 32 64 128 256 512 1024" + +# ---- SVG → PNG (multiple sizes) ----------------------------------------- +if command -v rsvg-convert >/dev/null; then + for s in $SIZES; do + rsvg-convert -w $s -h $s "$SVG" -o "$TMP/${s}.png" + done +else + echo "rsvg-convert not found — install librsvg2-bin (Linux) or librsvg (brew)" + exit 1 +fi + +# ---- PNGs → macOS .icns ------------------------------------------------- +# Prefer native iconutil on macOS for correctness (it handles Retina +# suffixes etc). Fall back to our pure-Node builder everywhere else. +if [ "$(uname -s)" = "Darwin" ] && command -v iconutil >/dev/null; then + IS="$TMP/icon.iconset" + mkdir -p "$IS" + cp "$TMP/16.png" "$IS/icon_16x16.png" + cp "$TMP/32.png" "$IS/icon_16x16@2x.png" + cp "$TMP/32.png" "$IS/icon_32x32.png" + cp "$TMP/64.png" "$IS/icon_32x32@2x.png" + cp "$TMP/128.png" "$IS/icon_128x128.png" + cp "$TMP/256.png" "$IS/icon_128x128@2x.png" + cp "$TMP/256.png" "$IS/icon_256x256.png" + cp "$TMP/512.png" "$IS/icon_256x256@2x.png" + cp "$TMP/512.png" "$IS/icon_512x512.png" + cp "$TMP/1024.png" "$IS/icon_512x512@2x.png" + iconutil -c icns "$IS" -o "$ASSETS/icon.icns" + echo "Generated icon.icns via iconutil" +else + node "$SCRIPT_DIR/build-icns.mjs" "$ASSETS/icon.icns" \ + "$TMP/16.png" "$TMP/32.png" "$TMP/64.png" \ + "$TMP/128.png" "$TMP/256.png" "$TMP/512.png" "$TMP/1024.png" +fi + +# ---- PNGs → Windows .ico ------------------------------------------------ +if command -v magick >/dev/null; then + magick "$TMP/16.png" "$TMP/32.png" "$TMP/64.png" \ + "$TMP/128.png" "$TMP/256.png" "$ASSETS/icon.ico" + echo "Generated icon.ico via magick (ImageMagick 7)" +elif command -v convert >/dev/null; then + convert "$TMP/16.png" "$TMP/32.png" "$TMP/64.png" \ + "$TMP/128.png" "$TMP/256.png" "$ASSETS/icon.ico" + echo "Generated icon.ico via convert (ImageMagick 6)" +else + echo "No .ico generator found (ImageMagick not installed)" +fi + +# ---- PNG (Linux) -------------------------------------------------------- +cp "$TMP/512.png" "$ASSETS/icon.png" + +rm -rf "$TMP" +echo "" +echo "Generated:" +ls -lh "$ASSETS/icon."{icns,ico,png} 2>/dev/null || true diff --git a/lively.app/scripts/smoke-velopack-updates.mjs b/lively.app/scripts/smoke-velopack-updates.mjs new file mode 100644 index 0000000000..6f00d21aa3 --- /dev/null +++ b/lively.app/scripts/smoke-velopack-updates.mjs @@ -0,0 +1,217 @@ +#!/usr/bin/env node +// Non-interactive Velopack update smoke test for the macOS NW.js bundle. +// +// This extracts the portable Velopack artifact when needed, loads the same +// desktop update service used by the native menu, and exercises status() plus +// checkForUpdates() against a local or remote feed. + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { createRequire } from 'node:module'; + +function parseArgs () { + const args = {}; + const raw = process.argv.slice(2); + for (let i = 0; i < raw.length; i++) { + const arg = raw[i]; + if (arg === '--help' || arg === '-h') args.help = true; + else if (arg === '--json') args.json = true; + else if (arg === '--keep') args.keep = true; + else { + const eq = arg.match(/^--([^=]+)=(.*)$/); + if (eq) args[eq[1]] = eq[2]; + else if (arg.startsWith('--')) args[arg.slice(2)] = raw[++i]; + } + } + return args; +} + +function usage () { + console.log(`Usage: + node lively.app/scripts/smoke-velopack-updates.mjs --artifactDir=dist/velopack/lively.next-osx-arm64 + node lively.app/scripts/smoke-velopack-updates.mjs --app=/Applications/lively.next.app --updateUrl=/path/to/velopack/feed + +Options: + --artifactDir Directory containing the Velopack Portable.zip and feed files. + --portableZip Portable.zip to extract instead of discovering it in artifactDir. + --app Installed or extracted lively.next.app to test directly. + --updateUrl Velopack update feed URL/path. Defaults to artifactDir. + --channel Override LIVELY_APP_UPDATE_CHANNEL for the smoke run. + --packagesDir Temporary Velopack packages directory for this smoke run. + --json Print machine-readable JSON. + --keep Keep the temporary extracted portable app.`); +} + +function die (msg, details = null) { + console.error('ERROR: ' + msg); + if (details) console.error(details); + process.exit(1); +} + +function requiredArg (args) { + if (args.app || args.artifactDir || args.portableZip) return; + die('Pass --artifactDir, --portableZip, or --app. Use --help for examples.'); +} + +function firstExistingFile (files) { + return files.find(file => file && fs.existsSync(file) && fs.statSync(file).isFile()); +} + +function portableZipFor (artifactDir) { + if (!artifactDir) return null; + const entries = fs.readdirSync(artifactDir).map(name => path.join(artifactDir, name)); + return firstExistingFile(entries.filter(file => /-Portable\.zip$/.test(path.basename(file)))); +} + +function extractPortableZip (zipFile, keep) { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'lively-velopack-smoke-')); + execFileSync('unzip', ['-q', zipFile, '-d', tmpDir], { stdio: 'inherit' }); + + return { + tmpDir, + cleanup () { + if (!keep) fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }; +} + +function findAppBundle (dir) { + const queue = [dir]; + while (queue.length) { + const current = queue.shift(); + for (const ent of fs.readdirSync(current, { withFileTypes: true })) { + const file = path.join(current, ent.name); + if (ent.isDirectory() && ent.name.endsWith('.app')) { + const appNw = path.join(file, 'Contents', 'Resources', 'app.nw'); + if (fs.existsSync(appNw)) return file; + } + if (ent.isDirectory() && !ent.name.endsWith('.app')) queue.push(file); + } + } + + return null; +} + +function appLayout (appBundle) { + const appNwDir = path.join(appBundle, 'Contents', 'Resources', 'app.nw'); + return { + appBundle, + appNwDir, + rootDir: path.join(appNwDir, 'app'), + desktopDir: path.join(appNwDir, 'desktop'), + updatesPath: path.join(appNwDir, 'desktop', 'updates.cjs') + }; +} + +function assertPath (label, file) { + if (!fs.existsSync(file)) die(`${label} does not exist: ${file}`); +} + +function configureEnvironment (args, artifactDir) { + const updateUrl = args.updateUrl || artifactDir; + if (!updateUrl) die('No update feed configured. Pass --updateUrl or --artifactDir.'); + + process.env.LIVELY_APP_UPDATE_URL = /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(updateUrl) + ? updateUrl + : path.resolve(updateUrl); + if (args.channel) process.env.LIVELY_APP_UPDATE_CHANNEL = args.channel; + if (!process.env.LIVELY_APP_UPDATE_CHECK_TIMEOUT_MS) { + process.env.LIVELY_APP_UPDATE_CHECK_TIMEOUT_MS = '30000'; + } +} + +function loadUpdatesModule (updatesPath) { + const require = createRequire(import.meta.url); + return require(updatesPath); +} + +async function main () { + const args = parseArgs(); + if (args.help) { + usage(); + return; + } + if (process.platform !== 'darwin') { + die('The Velopack update smoke test must run on macOS because the SDK loads the macOS native binding.'); + } + + requiredArg(args); + + let extracted = null; + let appBundle = args.app && path.resolve(args.app); + const artifactDir = args.artifactDir && path.resolve(args.artifactDir); + + if (!appBundle) { + const portableZip = args.portableZip || portableZipFor(artifactDir); + if (!portableZip) die(`No Velopack Portable.zip found in ${artifactDir || '(no artifactDir)'}`); + const zipFile = path.resolve(portableZip); + assertPath('Portable zip', zipFile); + extracted = extractPortableZip(zipFile, args.keep); + appBundle = findAppBundle(extracted.tmpDir); + if (!appBundle) die(`No .app bundle with Contents/Resources/app.nw found in ${zipFile}`); + } + + const layout = appLayout(appBundle); + assertPath('App bundle', layout.appBundle); + assertPath('Desktop update service', layout.updatesPath); + configureEnvironment(args, artifactDir); + + const updates = loadUpdatesModule(layout.updatesPath); + const packagesDir = args.packagesDir && path.resolve(args.packagesDir); + const locator = updates.createVelopackLocator(layout.rootDir, { packagesDir }); + if (!locator) die(`Could not construct Velopack locator for ${layout.rootDir}`); + + assertPath('Velopack UpdateMac helper', locator.UpdateExePath); + assertPath('Velopack manifest', locator.ManifestPath); + fs.mkdirSync(locator.PackagesDir, { recursive: true }); + + const log = []; + const service = updates.createUpdateService({ + rootDir: layout.rootDir, + desktopDir: layout.desktopDir, + locator, + log: msg => log.push(msg) + }); + + const status = service.status(); + if (!status.ok) { + die(status.message || 'Velopack status failed.', status.error || JSON.stringify(status, null, 2)); + } + + const checked = await service.checkForUpdates(); + if (!checked.ok) { + die(checked.message || 'Velopack update check failed.', checked.error || JSON.stringify(checked, null, 2)); + } + + const summary = { + ok: true, + appBundle: layout.appBundle, + source: checked.source, + channel: checked.channel, + appId: status.appId, + version: status.version, + state: checked.state, + targetVersion: checked.targetVersion || null, + locator, + log + }; + + if (args.json) console.log(JSON.stringify(summary, null, 2)); + else { + console.log('Velopack update smoke passed'); + console.log(' app: ' + summary.appBundle); + console.log(' appId: ' + summary.appId); + console.log(' version: ' + summary.version); + console.log(' source: ' + summary.source); + console.log(' channel: ' + (summary.channel || '(default)')); + console.log(' state: ' + summary.state + (summary.targetVersion ? ` (${summary.targetVersion})` : '')); + } + + if (extracted) extracted.cleanup(); +} + +main().catch(err => { + die(err && (err.stack || err.message) || String(err)); +}); diff --git a/lively.app/setup.sh b/lively.app/setup.sh new file mode 100755 index 0000000000..f60668f3f3 --- /dev/null +++ b/lively.app/setup.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Downloads the NW.js SDK binary into the flatn package directory. +# Run this after `flatn install` if the nw postinstall failed +# (common because nw's JS decompression deps don't resolve in flatn layout). +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +NW_VERSION="${LIVELY_NW_VERSION:-0.111.1}" +FLAVOR="${LIVELY_NW_FLAVOR:-sdk}" +NW_DOWNLOAD_BASE="${LIVELY_NW_DOWNLOAD_BASE:-https://dl.nwjs.io/live-build/v0.111.1-04292210-39517e80d}" +PLATFORM="$(uname -s | tr '[:upper:]' '[:lower:]')" +ARCH="$(uname -m)" + +# Map to NW.js naming +case "$PLATFORM" in + darwin) PLATFORM="osx" ;; + linux) PLATFORM="linux" ;; + *) echo "Unsupported platform: $PLATFORM"; exit 1 ;; +esac + +case "$ARCH" in + x86_64) ARCH="x64" ;; + aarch64|arm64) ARCH="arm64" ;; + *) echo "Unsupported arch: $ARCH"; exit 1 ;; +esac + +NW_PKG_DIR="$ROOT_DIR/lively.next-node_modules/nw/${NW_VERSION}-${FLAVOR}" +NW_DIR_NAME="nwjs-${FLAVOR}-v${NW_VERSION}-${PLATFORM}-${ARCH}" +EXT="tar.gz" +[ "$PLATFORM" = "osx" ] && EXT="zip" +ARCHIVE="${NW_DIR_NAME}.${EXT}" +URL="${NW_DOWNLOAD_BASE}/${ARCHIVE}" +MARKER_FILE="$NW_PKG_DIR/.download-url" + +if [ -x "$NW_PKG_DIR/$NW_DIR_NAME/nw" ] || [ -x "$NW_PKG_DIR/$NW_DIR_NAME/nwjs.app/Contents/MacOS/nwjs" ]; then + if [ -f "$MARKER_FILE" ] && [ "$(cat "$MARKER_FILE")" = "$URL" ]; then + echo "NW.js binary already present at $NW_PKG_DIR/$NW_DIR_NAME" + exit 0 + fi + echo "NW.js binary already present but download source changed; refreshing..." + rm -rf "$NW_PKG_DIR/$NW_DIR_NAME" +fi + +echo "Downloading NW.js SDK v${NW_VERSION} for ${PLATFORM}-${ARCH}..." +curl -L --progress-bar -o "/tmp/${ARCHIVE}" "$URL" + +echo "Extracting to $NW_PKG_DIR..." +mkdir -p "$NW_PKG_DIR" +if [ "$EXT" = "tar.gz" ]; then + tar xzf "/tmp/${ARCHIVE}" -C "$NW_PKG_DIR" +else + unzip -q -o "/tmp/${ARCHIVE}" -d "$NW_PKG_DIR" +fi + +rm "/tmp/${ARCHIVE}" +printf '%s\n' "$URL" > "$MARKER_FILE" +echo "NW.js SDK v${NW_VERSION} ready at $NW_PKG_DIR/$NW_DIR_NAME" diff --git a/lively.app/start.sh b/lively.app/start.sh new file mode 100755 index 0000000000..3ff93042dd --- /dev/null +++ b/lively.app/start.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Launch lively.next as an NW.js desktop app. +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +NW_VERSION="${LIVELY_NW_VERSION:-0.111.1}" +FLAVOR="${LIVELY_NW_FLAVOR:-sdk}" + +PLATFORM="$(uname -s | tr '[:upper:]' '[:lower:]')" +ARCH="$(uname -m)" +case "$PLATFORM" in darwin) PLATFORM="osx" ;; esac +case "$ARCH" in x86_64) ARCH="x64" ;; aarch64|arm64) ARCH="arm64" ;; esac + +NW_DIR_NAME="nwjs-${FLAVOR}-v${NW_VERSION}-${PLATFORM}-${ARCH}" +NW_PKG_DIR="$ROOT_DIR/lively.next-node_modules/nw/${NW_VERSION}-${FLAVOR}" + +if [ "$PLATFORM" = "osx" ]; then + NW_BIN="$NW_PKG_DIR/$NW_DIR_NAME/nwjs.app/Contents/MacOS/nwjs" +else + NW_BIN="$NW_PKG_DIR/$NW_DIR_NAME/nw" +fi + +if [ ! -x "$NW_BIN" ]; then + echo "NW.js binary not found. Run: cd lively.app && bash setup.sh" + exit 1 +fi + +# Set up flatn environment (FLATN_* vars, PATH). +# NODE_OPTIONS must be unset — ESM loader hooks crash NW.js's Blink renderer. +# The server child process handles its own --experimental-loader flag. +. "$ROOT_DIR/scripts/lively-next-env.sh" +lively_next_env "$ROOT_DIR" +unset NODE_OPTIONS + +exec "$NW_BIN" "$SCRIPT_DIR" "$@" diff --git a/lively.ast/lib/acorn-extension.js b/lively.ast/lib/acorn-extension.js index a6059643cb..687189b42a 100644 --- a/lively.ast/lib/acorn-extension.js +++ b/lively.ast/lib/acorn-extension.js @@ -4,9 +4,9 @@ import Decorators from './acorn-decorators.mjs'; import _ClassFields from 'acorn-class-fields'; import _StaticClassFeatures from 'acorn-static-class-features'; import _PrivateMethods from 'acorn-private-methods'; -import * as acornDefault from 'acorn'; -import * as walk from 'acorn-walk'; -import * as loose from 'acorn-loose'; +import * as _acornDefault from 'acorn'; +import * as _walk from 'acorn-walk'; +import * as _loose from 'acorn-loose'; // If we are running in node, load the modules natively. // The reason is, that in node.js we can not load the esm @@ -22,19 +22,46 @@ const isNode = typeof System !== 'undefined' : false; let ClassFields, StaticClassFeatures, PrivateMethods; +function acornPlugin (imported, moduleName) { + let plugin = imported; + while (plugin && typeof plugin === 'object' && 'default' in plugin) { + plugin = plugin.default; + } + return typeof plugin === 'function' + ? plugin + : System._nodeRequire(moduleName); +} + +function acornNamespace (imported, expectedProperty, requireName) { + let namespace = imported; + while (namespace && typeof namespace === 'object' && typeof namespace[expectedProperty] !== 'function' && 'default' in namespace) { + namespace = namespace.default; + } + if (typeof namespace?.[expectedProperty] !== 'function' && requireName && globalThis.System?._nodeRequire) { + namespace = globalThis.System._nodeRequire(requireName); + while (namespace && typeof namespace === 'object' && typeof namespace[expectedProperty] !== 'function' && 'default' in namespace) { + namespace = namespace.default; + } + } + return namespace; +} + if (isNode) { // we need to utilize the native require here to bypass the source transform of the class // we can not use the native import, since that is asynchronous. // top level import is causing class instrumentation - ClassFields = _ClassFields || System._nodeRequire('acorn-class-fields'); - StaticClassFeatures = _StaticClassFeatures || System._nodeRequire('acorn-static-class-features'); - PrivateMethods = _PrivateMethods || System._nodeRequire('acorn-private-methods'); + ClassFields = acornPlugin(_ClassFields, 'acorn-class-fields'); + StaticClassFeatures = acornPlugin(_StaticClassFeatures, 'acorn-static-class-features'); + PrivateMethods = acornPlugin(_PrivateMethods, 'acorn-private-methods'); } else { ClassFields = _ClassFields; StaticClassFeatures = _StaticClassFeatures; PrivateMethods = _PrivateMethods; } +const acornDefault = acornNamespace(_acornDefault, 'Parser', 'acorn'); +const walk = acornNamespace(_walk, 'make', 'acorn-walk'); +const loose = acornNamespace(_loose, 'parse', 'acorn-loose'); const custom = {}; custom.forEachNode = forEachNode; diff --git a/lively.ast/lib/query.js b/lively.ast/lib/query.js index 473162596c..c0a1337b99 100644 --- a/lively.ast/lib/query.js +++ b/lively.ast/lib/query.js @@ -9,7 +9,17 @@ import stringify from './stringify.js'; // Importing ASTQ in SystemJS 0.21 on node.js fails is it was not loaded natively before. // This causes issues with setups where we can not possible load astq, such as the install bundle. // To make these scripts work, we backtrack to import via native require instead. -let ASTQ = _ASTQ || System._nodeRequire('astq'); +function nativeConstructor (imported, moduleName) { + let constructor = imported; + while (constructor && typeof constructor === 'object' && 'default' in constructor) { + constructor = constructor.default; + } + return typeof constructor === 'function' + ? constructor + : System._nodeRequire(moduleName); +} + +let ASTQ = nativeConstructor(_ASTQ, 'astq'); // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- diff --git a/lively.ast/lib/stringify.js b/lively.ast/lib/stringify.js index 6e7e01949c..f0ccaf5ab3 100644 --- a/lively.ast/lib/stringify.js +++ b/lively.ast/lib/stringify.js @@ -1,7 +1,52 @@ /* global require,global */ import { obj } from 'lively.lang'; -import estraverse from '@javascript-obfuscator/estraverse'; -import esutils from 'esutils'; +import { + Syntax as _estraverseSyntax, + attachComments as _estraverseAttachComments +} from '@javascript-obfuscator/estraverse'; +import { code as _esutilsCode } from 'esutils'; + +const isNode = typeof System !== 'undefined' + ? System.get('@system-env').node + : false; + +function loadedSystemModule (requireName, expectedProperty) { + const System = globalThis.System; + if (!System) return null; + const encodedName = requireName.replace('/', '__SLASH__'); + const candidates = Object.keys(System.loads || {}).filter(key => + key.includes(requireName) || key.includes(encodedName)); + for (const key of candidates) { + const value = System.loads[key]?.exports || System.get?.(key, false); + if (value?.[expectedProperty] != null) return value; + } + return null; +} + +function cjsDefault (module, expectedProperty, requireName) { + let value = module; + while (value && typeof value === 'object' && value[expectedProperty] == null && 'default' in value) { + value = value.default; + } + if (value?.[expectedProperty] == null && requireName) { + value = loadedSystemModule(requireName, expectedProperty) || value; + } + if (isNode && value?.[expectedProperty] == null && requireName && globalThis.System?._nodeRequire) { + value = globalThis.System._nodeRequire(requireName); + while (value && typeof value === 'object' && value[expectedProperty] == null && 'default' in value) { + value = value.default; + } + } + return value; +} + +const _estraverse = { + Syntax: _estraverseSyntax, + attachComments: _estraverseAttachComments +}; +const _esutils = { code: _esutilsCode }; +const estraverse = cjsDefault(_estraverse, 'Syntax', '@javascript-obfuscator/estraverse'); +const esutils = cjsDefault(_esutils, 'code', 'esutils'); // FORKED ESCODEGEN function ESCODEGEN () { diff --git a/lively.freezer/package.json b/lively.freezer/package.json index 5c875657df..3277d05fff 100644 --- a/lively.freezer/package.json +++ b/lively.freezer/package.json @@ -12,8 +12,6 @@ "@babel/plugin-transform-runtime": "^7.12.1", "babel-standalone": "^6.21.1-0", "uglify-es": "^3.3.9", - "google-closure-compiler-linux": "^20200927.0.0", - "google-closure-compiler-osx": "^20200927.0.0", "wasm-brotli": "2.0.2", "@rollup/wasm-node": "4.27.3", "@rollup/browser": "4.27.3", diff --git a/lively.freezer/src/bundler.js b/lively.freezer/src/bundler.js index 8463a39693..b53a2810b2 100644 --- a/lively.freezer/src/bundler.js +++ b/lively.freezer/src/bundler.js @@ -63,7 +63,7 @@ const CLASS_INSTRUMENTATION_MODULES = [ 'https://jspm.dev/npm:rollup@2.28.2' // this contains a bunch of class definitions which right now screws up the closure compiler ]; -const ESM_CDNS = [/esm:\/\/([^\/]*)\//]; +const ESM_CDNS = [/esm:\/\/([^\/]*)\//, /https:\/\/ga\.jspm\.io\//]; // fixme: Why is a blacklist nessecary if there is a whitelist? const CLASS_INSTRUMENTATION_MODULES_EXCLUSION = ['lively.lang']; @@ -200,6 +200,17 @@ function resolutionId (id, importer) { else return importer + ' -> ' + id; } +function equivalentCdnModuleIds (id) { + if (!id || typeof id !== 'string') return []; + const ids = [id]; + if (id.startsWith('https://ga.jspm.io/')) { + ids.push(id.replace('https://ga.jspm.io/', 'esm://ga.jspm.io/')); + } else if (id.startsWith('esm://ga.jspm.io/')) { + ids.push(id.replace('esm://ga.jspm.io/', 'https://ga.jspm.io/')); + } + return ids; +} + /** * For a given id, returns wether or not it is imported * from one of the supported ESM CDNs. @@ -307,6 +318,44 @@ export default class LivelyRollup { // fixme: how to configure "system-node" } + modulePackageFor (moduleId) { + return this.resolver.resolvePackage(moduleId, this.getResolutionContext()) || + equivalentCdnModuleIds(moduleId).map(id => this.moduleToPkg.get(id)).find(Boolean); + } + + rememberModulePackage (moduleId, pkg) { + if (!pkg) return; + equivalentCdnModuleIds(moduleId).forEach(id => this.moduleToPkg.set(id, pkg)); + } + + defaultImportMapForCdnImports () { + if (this._defaultCdnImportMap !== undefined) return this._defaultCdnImportMap; + const knownPackage = [...new Set(this.moduleToPkg.values())].find(pkg => pkg?.systemjs?.importMap); + if (knownPackage) return this._defaultCdnImportMap = knownPackage.systemjs.importMap; + try { + const freezerPackage = this.resolver.resolvePackage( + this.resolver.normalizeFileName('lively.freezer/index.js'), + this.getResolutionContext()); + return this._defaultCdnImportMap = freezerPackage?.systemjs?.importMap || null; + } catch (err) { + return this._defaultCdnImportMap = null; + } + } + + resolveBareLoadId (id) { + if (!id || id === ROOT_ID || id.startsWith('__rootModule__:') || + id.startsWith('.') || id.startsWith('/') || id.startsWith('\0') || id.includes(':')) return null; + const importMap = this.defaultImportMapForCdnImports(); + const remapped = importMap && resolveViaImportMap(id, importMap, 'esm://ga.jspm.io/'); + if (remapped && remapped !== id) return remapped; + try { + const fallbackImporter = this.resolver.normalizeFileName('lively.freezer/index.js'); + return this.resolver.resolveModuleId(id, fallbackImporter, this.getResolutionContext()); + } catch (err) { + return null; + } + } + /** * Dispatches the responsibility of resolving relative imports to the loaded SystemJS. * @param { string } moduleId - The module from where the import happens. @@ -339,7 +388,7 @@ export default class LivelyRollup { if (modId === '@empty.js') return {}; const parsedGlobals = parsedSource.scope?.globals && Object.keys(parsedSource.scope?.globals) || GlobalInjector.getGlobals(null, parsedSource); let version, name; - const pkg = this.resolver.resolvePackage(modId, this.getResolutionContext()) || this.moduleToPkg.get(modId); + const pkg = this.modulePackageFor(modId); if (pkg) { name = pkg.name; version = pkg.version; @@ -429,7 +478,7 @@ export default class LivelyRollup { resolvedImports = {}; } let version, name; - const pkg = this.resolver.resolvePackage(modId, this.getResolutionContext()) || this.moduleToPkg.get(modId); + const pkg = this.modulePackageFor(modId); if (pkg) { name = pkg.name; version = pkg.version; @@ -877,9 +926,12 @@ export default class LivelyRollup { } } - const importingPackage = this.resolver.resolvePackage(importer, this.getResolutionContext()) || this.moduleToPkg.get(importer); + const importingPackage = this.modulePackageFor(importer); // honor the systemjs options within the package config - const { map: mapping, importMap } = importingPackage?.systemjs || {}; + let { map: mapping, importMap } = importingPackage?.systemjs || {}; + if (!importMap && this.wasFetchedFromEsmCdn(importer)) { + importMap = this.defaultImportMapForCdnImports(); + } if (mapping) { let remapped = mapping[id]; if (remapped) { @@ -891,11 +943,12 @@ export default class LivelyRollup { } if (importMap) { + this._defaultCdnImportMap = importMap; let remapped = resolveViaImportMap(id, importMap, importer) if (remapped) id = remapped; } - this.moduleToPkg.set(id, importingPackage); + this.rememberModulePackage(id, importingPackage); let absolutePath; @@ -971,6 +1024,9 @@ export default class LivelyRollup { return ''; } + const resolvedBareId = this.resolveBareLoadId(id); + if (resolvedBareId && resolvedBareId !== id) return this.perform_load(resolvedBareId); + if (id === ROOT_ID) { const res = await this.getRootModule(); return res; diff --git a/lively.freezer/src/resolvers/node.cjs b/lively.freezer/src/resolvers/node.cjs index 5a761be94f..5a98d9658e 100644 --- a/lively.freezer/src/resolvers/node.cjs +++ b/lively.freezer/src/resolvers/node.cjs @@ -44,6 +44,7 @@ async function availableFonts(fontCSSFile) { function isCdnImport(url) { return url.includes('jspm.dev') || + url.includes('ga.jspm.io') || url.includes('esm://'); } diff --git a/lively.freezer/src/util/bootstrap.js b/lively.freezer/src/util/bootstrap.js index 7c13d0d983..771e3e2b90 100644 --- a/lively.freezer/src/util/bootstrap.js +++ b/lively.freezer/src/util/bootstrap.js @@ -227,7 +227,6 @@ function bootstrapLivelySystem (progress, fastLoad = query.fastLoad !== false || initBaseURL = initBaseURL.slice(0, -1); } const packageCached = await resource(baseURL).join('package-registry.json').readJson(); - await loadViaScript(resource(baseURL).join('/lively.next-node_modules/@babel/standalone/babel.js').url); const migratedMeta = { ...oldSystem.meta }; const System = lively.modules.getSystem('bootstrapped', { baseURL, meta: migratedMeta }); // the meta of the not yet loaded modules needs to be transformed into register $world.env.uninstallSystemChangeHandlers(); diff --git a/lively.freezer/tools/build.unified.mjs b/lively.freezer/tools/build.unified.mjs index 02fbed4f39..a3a793772b 100644 --- a/lively.freezer/tools/build.unified.mjs +++ b/lively.freezer/tools/build.unified.mjs @@ -2,6 +2,7 @@ import { rollup } from '@rollup/wasm-node'; import jsonPlugin from '@rollup/plugin-json'; import util from 'node:util'; +import fs from 'node:fs/promises'; import { lively } from 'lively.freezer/src/plugins/rollup'; import resolver from 'lively.freezer/src/resolvers/node.cjs'; @@ -67,6 +68,9 @@ try { console.log(' Writing outputs...'); + await fs.rm('landing-page', { recursive: true, force: true }); + await fs.rm('loading-screen', { recursive: true, force: true }); + // Write landing-page output await build.write({ format: 'system', @@ -96,8 +100,6 @@ try { console.log(' Loading screen written to loading-screen/'); // Post-process: Copy the correct index.html for each directory - const fs = await import('fs/promises'); - try { await fs.copyFile('landing-page/index-landing-page.html', 'landing-page/index.html'); } catch (err) { diff --git a/lively.ide/service-worker.js b/lively.ide/service-worker.js index 4f2787ecdd..27b6da063e 100644 --- a/lively.ide/service-worker.js +++ b/lively.ide/service-worker.js @@ -42,7 +42,6 @@ function initWorker () { const w = worker.create({ workerId: '@lively-worker', scriptsToLoad: [ - 'lively.next-node_modules/babel-standalone/babel.js', 'lively.next-node_modules/systemjs/dist/system.src.js', 'lively.modules/dist/lively.modules.js', 'lively.ide/jsdom.worker.js' diff --git a/lively.ide/world.js b/lively.ide/world.js index 9d51d50f92..aa7b680326 100644 --- a/lively.ide/world.js +++ b/lively.ide/world.js @@ -45,6 +45,10 @@ import { currentUsername, isUserLoggedIn, currentUser } from 'lively.user'; import { subscribe } from 'lively.notifications/index.js'; import { supportedImageFormats } from './assets.js'; +function isLivelyDesktopApp () { + return !!globalThis.__LIVELY_DESKTOP_APP__; +} + export class LivelyWorld extends World { static get properties () { return { @@ -448,17 +452,20 @@ export class LivelyWorld extends World { } async initializeStudioUI () { - const { LivelyVersionChecker } = await System.import('lively.ide/studio/version-checker.cp.js'); - const versionChecker = part(LivelyVersionChecker); - versionChecker.name = 'lively version checker'; - versionChecker.openInWorld(); - versionChecker.relayout(); - try { - await promise.timeout(1000, versionChecker.checkVersion()); - } catch (err) { - // if the version checking takes too long due to slow - // networking or being offline entirely, we proceed - // without waiting longer than 1000ms. + let versionChecker; + if (!isLivelyDesktopApp()) { + const { LivelyVersionChecker } = await System.import('lively.ide/studio/version-checker.cp.js'); + versionChecker = part(LivelyVersionChecker); + versionChecker.name = 'lively version checker'; + versionChecker.openInWorld(); + versionChecker.relayout(); + try { + await promise.timeout(1000, versionChecker.checkVersion()); + } catch (err) { + // if the version checking takes too long due to slow + // networking or being offline entirely, we proceed + // without waiting longer than 1000ms. + } } const { WorldZoomIndicator } = await System.import('lively.ide/studio/zoom-indicator.cp.js'); const zoomIndicator = part(WorldZoomIndicator); @@ -468,7 +475,7 @@ export class LivelyWorld extends World { const { Flap } = await System.import('lively.ide/studio/flap.cp.js'); $world.sceneGraphFlap = part(Flap, { viewModel: { target: 'scene graph', action: toggleSidebar, openingRoutine: openSidebarFlapInWorld, relayoutRoutine: relayoutSidebarFlapInWorld } }).openInWorld(); - connect($world.sceneGraphFlap, 'position', versionChecker, 'relayout'); + if (versionChecker) connect($world.sceneGraphFlap, 'position', versionChecker, 'relayout'); $world.propertiesPanelFlap = part(Flap, { viewModel: { target: 'properties panel', action: toggleSidebar, openingRoutine: openSidebarFlapInWorld, relayoutRoutine: relayoutSidebarFlapInWorld } }).openInWorld(); connect($world.propertiesPanelFlap, 'position', zoomIndicator, 'relayout'); this._uiInitialized = true; diff --git a/lively.installer/install.js b/lively.installer/install.js index 6598ea77d3..0949ce22d6 100644 --- a/lively.installer/install.js +++ b/lively.installer/install.js @@ -1,10 +1,14 @@ /*global process,System,global*/ -import { buildPackageMap, installDependenciesOfPackage, buildPackage, resetPackageMap } from "flatn"; +import { createRequire } from 'node:module'; +import { delimiter } from 'node:path'; +import { pathToFileURL } from 'node:url'; import { exec } from "./shell-exec.js"; import { Package } from "./package.js"; import { resource } from 'lively.resources'; import { promise, string } from 'lively.lang'; +const require = createRequire(import.meta.url); +const { buildPackageMap, installDependenciesOfPackage, buildPackage, resetPackageMap } = require('flatn'); var modules, join, getPackageSpec, readPackageSpec; // ── Logging helpers ── @@ -159,11 +163,11 @@ export async function install(baseDir, dependenciesDir, verbose) { if (!env.PATH.includes(flatnBinDir)) { env.PATH = flatnBinDir + ":" + env.PATH; } - if (env.FLATN_DEV_PACKAGE_DIRS !== packageMap.devPackageDirs.join(":")) { - env.FLATN_DEV_PACKAGE_DIRS = packageMap.devPackageDirs.join(":"); + if (env.FLATN_DEV_PACKAGE_DIRS !== packageMap.devPackageDirs.join(delimiter)) { + env.FLATN_DEV_PACKAGE_DIRS = packageMap.devPackageDirs.join(delimiter); } - if (env.FLATN_PACKAGE_COLLECTION_DIRS !== packageMap.packageCollectionDirs.join(":")) { - env.FLATN_PACKAGE_COLLECTION_DIRS = packageMap.packageCollectionDirs.join(":"); + if (env.FLATN_PACKAGE_COLLECTION_DIRS !== packageMap.packageCollectionDirs.join(delimiter)) { + env.FLATN_PACKAGE_COLLECTION_DIRS = packageMap.packageCollectionDirs.join(delimiter); } } @@ -389,14 +393,15 @@ export async function setupSystem(baseURL) { let livelySystem = modules.getSystem("lively", {baseURL, _nodeRequire: System._nodeRequire }); modules.changeSystem(livelySystem, true); var registry = livelySystem["__lively.modules__packageRegistry"] = new modules.PackageRegistry(livelySystem); - registry.packageBaseDirs = process.env.FLATN_PACKAGE_COLLECTION_DIRS.split(":").map(ea => resource(`file://${ea}`)); - registry.devPackageDirs = process.env.FLATN_DEV_PACKAGE_DIRS.split(":").map(ea => resource(`file://${ea}`)); - registry.individualPackageDirs = process.env.FLATN_PACKAGE_DIRS.split(":").map(ea => ea.length > 0 ? resource(`file://${ea}`) : false).filter(Boolean); + const flatnEnvPaths = name => (process.env[name] || "").split(delimiter).filter(Boolean).map(ea => resource(pathToFileURL(ea).href)); + registry.packageBaseDirs = flatnEnvPaths("FLATN_PACKAGE_COLLECTION_DIRS"); + registry.devPackageDirs = flatnEnvPaths("FLATN_DEV_PACKAGE_DIRS"); + registry.individualPackageDirs = flatnEnvPaths("FLATN_PACKAGE_DIRS"); await registry.update(); resetPackageMap(); const { setupBabelTranspiler } = await import('lively.source-transform/babel/plugin.js'); - await setupBabelTranspiler(livelySystem); + setupBabelTranspiler(livelySystem); return livelySystem; } diff --git a/lively.installer/packages-config.json b/lively.installer/packages-config.json index f8fd15fac7..953c2b8e01 100644 --- a/lively.installer/packages-config.json +++ b/lively.installer/packages-config.json @@ -131,5 +131,9 @@ { "name": "lively.user", "repoURL": "https://github.com/LivelyKernel/lively.next" + }, + { + "name": "lively.app", + "repoURL": "https://github.com/LivelyKernel/lively.next" } ] diff --git a/lively.modules/src/cache.js b/lively.modules/src/cache.js index 15276b18a6..724511939d 100644 --- a/lively.modules/src/cache.js +++ b/lively.modules/src/cache.js @@ -16,24 +16,41 @@ export class ModuleTranslationCache { } let nodejsCacheDirURL = null; + +function normalizeModuleId (moduleId) { + return String(moduleId) + .replace(/^file:\/\//, '') + .replace(/^\/([a-z]:[\\/])/i, '$1') + .replace(/\\/g, '/') + .replace(/^\/+/, ''); +} + +function moduleIdToCachePath (moduleId) { + return normalizeModuleId(moduleId) + .replace(/:/g, '%3A'); +} + function prepareNodejsCaching () { const fs = System._nodeRequire('fs'); const path = System._nodeRequire('path'); + const { pathToFileURL } = System._nodeRequire('url'); const isWindows = process.platform === 'win32'; - const nodejsCacheDir = + const configuredCacheDir = process.env.LIVELY_MODULE_TRANSLATION_CACHE_DIR; + const nodejsCacheRoot = !isWindows && process.cwd() === '/' ? path.join(process.env.HOME, '.lively.next') : process.cwd(); - nodejsCacheDirURL = isWindows - ? `file:///${nodejsCacheDir.replace(/\\/g, '/')}` - : `file://${nodejsCacheDir}`; - if (!fs.existsSync(nodejsCacheDir)) fs.mkdirSync(nodejsCacheDir); + const nodejsCacheDir = configuredCacheDir + ? path.resolve(configuredCacheDir) + : path.join(nodejsCacheRoot, '.module_cache'); + nodejsCacheDirURL = pathToFileURL(nodejsCacheDir).href; + if (!fs.existsSync(nodejsCacheDir)) fs.mkdirSync(nodejsCacheDir, { recursive: true }); } export class NodeModuleTranslationCache extends ModuleTranslationCache { get moduleCacheDir () { if (!nodejsCacheDirURL) prepareNodejsCaching(); - return resource(`${nodejsCacheDirURL}/.module_cache/`); + return resource(nodejsCacheDirURL.endsWith('/') ? nodejsCacheDirURL : `${nodejsCacheDirURL}/`); } async ensurePath (path) { @@ -69,10 +86,10 @@ export class NodeModuleTranslationCache extends ModuleTranslationCache { async fetchStoredModuleSource (moduleId) { if (moduleId.endsWith('package.json')) return null; - moduleId = moduleId.replace('file://', ''); - const fname = this.getFileName(moduleId); - const fpath = moduleId.replace(fname, ''); - const r = this.moduleCacheDir.join(moduleId); + const cachePath = moduleIdToCachePath(moduleId); + const fname = this.getFileName(cachePath); + const fpath = cachePath.replace(fname, ''); + const r = this.moduleCacheDir.join(cachePath); if (!await r.exists()) return null; try { const { birthtime: timestamp } = await r.stat(); @@ -83,18 +100,18 @@ export class NodeModuleTranslationCache extends ModuleTranslationCache { return { source, timestamp, hash, sourceMap, exports }; } catch (e) { // Stale or corrupt cache entry — delete it and return null so the module is re-transformed - await this.deleteCachedData(moduleId); + await this.deleteCachedData(cachePath); return null; } } async cacheModuleSource (moduleId, hash, source, exports = [], sourceMap = {}) { if (moduleId.endsWith('package.json')) return; - moduleId = moduleId.replace('file://', ''); - const fname = this.getFileName(moduleId); - const fpath = moduleId.replace(fname, ''); + const cachePath = moduleIdToCachePath(moduleId); + const fname = this.getFileName(cachePath); + const fpath = cachePath.replace(fname, ''); await this.ensurePath(fpath); - await this.moduleCacheDir.join(moduleId).write(source); + await this.moduleCacheDir.join(cachePath).write(source); await this.moduleCacheDir.join(fpath).join('.hash_' + fname).write(hash); await this.moduleCacheDir.join(fpath).join('.source_map_' + fname).writeJson(sourceMap); await this.moduleCacheDir.join(fpath).join('.exports_' + fname).writeJson(exports.map(({ @@ -103,10 +120,13 @@ export class NodeModuleTranslationCache extends ModuleTranslationCache { } async deleteCachedData (moduleId) { - moduleId = moduleId.replace('file://', ''); - const r = this.moduleCacheDir.join(moduleId); + const r = this.moduleCacheDir.join(moduleIdToCachePath(moduleId)); if (!await r.exists()) return false; - await r.remove(); + try { + await r.remove(); + } catch (err) { + if (err.code !== 'ENOENT') throw err; + } return true; } } diff --git a/lively.modules/src/instrumentation.js b/lively.modules/src/instrumentation.js index 8f4cba225f..e638a29a79 100644 --- a/lively.modules/src/instrumentation.js +++ b/lively.modules/src/instrumentation.js @@ -189,7 +189,7 @@ function addNodejsWrapperSource (System, load, markedPackageName) { const m = getCachedNodejsModule(System, load, markedPackageName); if (m) { load.metadata.format = 'esm'; - load.source = `var exports = System._nodeRequire('${m.id}'); export default exports;\n` + + load.source = `var exports = System._nodeRequire(${JSON.stringify(m.id)}); export default exports;\n` + `export var __useDefault = exports;\n` + properties.allOwnPropertiesOrFunctions(m.exports).map(k => isValidIdentifier(k) diff --git a/lively.modules/src/nodejs.js b/lively.modules/src/nodejs.js index 637e1a7617..296ce16cdf 100644 --- a/lively.modules/src/nodejs.js +++ b/lively.modules/src/nodejs.js @@ -1,10 +1,17 @@ import { resource } from 'lively.resources'; +function fileURLToPathname (url) { + const parsed = new URL(url); + let decodedPath = decodeURIComponent(parsed.pathname); + if (parsed.host) decodedPath = `//${parsed.host}${decodedPath}`; + return decodedPath.replace(/^\/([a-z]:\/)/i, '$1').replace(/\\/g, '/'); +} + function ensureParent (currentModule, name, parent) { if (parent) return parent; let { id, System } = currentModule; - let idForNode = id.startsWith('file://') ? id.replace('file://', '') : id; + let idForNode = id.startsWith('file://') ? fileURLToPathname(id) : id; let module = System._nodeRequire('module'); parent = module.Module._cache[id]; @@ -16,7 +23,7 @@ function ensureParent (currentModule, name, parent) { } function relative (module, name) { - return resource(module.id).parent().join(name).url.replace('file://', ''); + return fileURLToPathname(resource(module.id).parent().join(name).url); } export function _require (currentModule, name, parent) { diff --git a/lively.modules/src/packages/package-registry.js b/lively.modules/src/packages/package-registry.js index 1631dfe200..57a3cef747 100644 --- a/lively.modules/src/packages/package-registry.js +++ b/lively.modules/src/packages/package-registry.js @@ -1,11 +1,59 @@ -import semver from 'semver'; +import { + sort as _semverSort, + gt as _semverGt, + compare as _semverCompare, + validRange as _semverValidRange, + satisfies as _semverSatisfies +} from 'semver'; import { arr, obj, promise } from 'lively.lang'; import { Package } from './package.js'; import { resource } from 'lively.resources'; import { isURL } from '../url-helpers.js'; import { classHolder } from '../cycle-breaker.js'; +function loadedSystemModule (requireName, expectedProperty) { + const System = globalThis.System; + if (!System) return null; + const encodedName = requireName.replace('/', '__SLASH__'); + const candidates = Object.keys(System.loads || {}).filter(key => + key.includes(requireName) || key.includes(encodedName)); + for (const key of candidates) { + const value = System.loads[key]?.exports || System.get?.(key, false); + if (typeof value?.[expectedProperty] === 'function') return value; + } + return null; +} + +function cjsDefault (module, expectedProperty, requireName) { + let value = module; + while (value && typeof value === 'object' && typeof value[expectedProperty] !== 'function' && 'default' in value) { + value = value.default; + } + if (typeof value?.[expectedProperty] !== 'function' && requireName) { + value = loadedSystemModule(requireName, expectedProperty) || value; + } + if (typeof value?.[expectedProperty] !== 'function' && requireName && globalThis.System?._nodeRequire) { + value = globalThis.System._nodeRequire(requireName); + while (value && typeof value === 'object' && typeof value[expectedProperty] !== 'function' && 'default' in value) { + value = value.default; + } + } + return value; +} + +const _semver = { + sort: _semverSort, + gt: _semverGt, + compare: _semverCompare, + validRange: _semverValidRange, + satisfies: _semverSatisfies +}; +const semver = cjsDefault(_semver, 'validRange', 'semver'); const urlStartRe = /^[a-z\.-_\+]+:/i; +function sortVersions (versions) { + return semver.sort(versions, true); +} + function isAbsolute (path) { return ( path.startsWith('/') || @@ -434,13 +482,13 @@ export class PackageRegistry { _updateLatestPackages (name) { let { packageMap } = this; if (name && packageMap[name]) { - packageMap[name].latest = arr.last(semver.sort( - Object.keys(packageMap[name].versions), true)); + packageMap[name].latest = arr.last(sortVersions( + Object.keys(packageMap[name].versions))); return; } for (let eaName in packageMap) { - packageMap[eaName].latest = arr.last(semver.sort( - Object.keys(packageMap[eaName].versions), true)); + packageMap[eaName].latest = arr.last(sortVersions( + Object.keys(packageMap[eaName].versions))); } } diff --git a/lively.modules/src/system.js b/lively.modules/src/system.js index 66282592a8..834b327092 100644 --- a/lively.modules/src/system.js +++ b/lively.modules/src/system.js @@ -385,6 +385,10 @@ const cjsJsExtRe = /(.*\.cjs)\.js$/i; const doubleSlashRe = /.\/{2,}/g; const nodeModRe = /\@node.*/; +function normalizeUrlPathSegments (name) { + return typeof name === 'string' && /^[^:]+:\/\//.test(name) ? urlResolve(name) : name; +} + function preNormalize (System, name, parent) { // console.log(`> [preNormalize] ${name}`); @@ -395,6 +399,7 @@ function preNormalize (System, name, parent) { // that... // name = name.replace(/([^:])\/\/+/g, "$1\/"); name = name.replace(doubleSlashRe, (match) => match[0] === ':' ? match : match[0] + '/'); + name = normalizeUrlPathSegments(name); // systemjs' decanonicalize has by default not the fancy // '{node: "events", "~node": "@empty"}' mapping but we need it @@ -475,6 +480,8 @@ function preNormalize (System, name, parent) { function postNormalize (System, normalizeResult, isSync) { // console.log(`> [postNormalize] ${normalizeResult}`); + normalizeResult = normalizeUrlPathSegments(normalizeResult); + // lookup package main const base = normalizeResult.replace('/index.js', '') @@ -499,7 +506,7 @@ function postNormalize (System, normalizeResult, isSync) { let main = System.CONFIG.packages[base].main; if (main) { let withMain = base.replace(trailingSlashRe, '') + '/' + main.replace(dotSlashStartRe, ''); - return withMain; + return normalizeUrlPathSegments(withMain); } } } @@ -519,6 +526,7 @@ async function checkExistence (url, System) { } async function finalizeNormalization (System, name, normalized) { + normalized = normalizeUrlPathSegments(normalized); const isNodePath = normalized.startsWith('file:'); // SystemJS may append ".js" to non-JS extensions during normalization. // Normalize those cases before trying JS index fallbacks. @@ -554,7 +562,7 @@ async function finalizeNormalization (System, name, normalized) { } } - return normalized; + return normalizeUrlPathSegments(normalized); } async function normalizeHook (proceed, name, parent, parentAddress) { diff --git a/lively.project/project.js b/lively.project/project.js index a03f6703ec..cd22d431d4 100644 --- a/lively.project/project.js +++ b/lively.project/project.js @@ -33,13 +33,45 @@ import * as semver from 'semver'; export const repositoryOwnerAndNameRegex = /\.com\/(.+)\/(.*)/; const fontCSSWarningString = `/*\nDO NOT CHANGE THE CONTENTS OF THIS FILE! Its content is managed automatically by lively.next. It will automatically be loaded/bundled together with this project!\n*/\n\n`; + +function resolveShellDirectory (basePath, ...parts) { + const normalizedPath = [String(basePath).replace(/\\/g, '/'), ...parts].join('/'); + const root = normalizedPath.match(/^[a-z]:/i)?.[0] || (normalizedPath.startsWith('/') ? '/' : ''); + const rest = root ? normalizedPath.slice(root.length) : normalizedPath; + const resolvedParts = []; + + for (const part of rest.split('/')) { + if (!part || part === '.') continue; + if (part === '..') resolvedParts.pop(); + else resolvedParts.push(part); + } + + return (root === '/' ? '/' : root ? `${root}/` : '') + resolvedParts.join('/') + '/'; +} + +async function addPackageAtIfMissing (registry, packageDir, preferredLocation) { + const existing = registry.findPackageWithURL(packageDir); + if (existing) return existing; + return registry.addPackageAt(packageDir, preferredLocation); +} + +async function addProjectPackageOnServerIfMissing (fullName) { + await evalOnServer(`(async () => { + const registry = System.get("@lively-env").packageRegistry; + const projectURL = System.baseURL + "local_projects/" + ${JSON.stringify(fullName)}; + if (!registry.findPackageWithURL(projectURL)) await registry.addPackageAt(projectURL); + return true; + })()`); +} + export class Project { static get systemInterface () { return localInterface.coreInterface; } static async ensureGitResource (fullName) { - return resource('git/' + await defaultDirectory()).join('..').join('local_projects').join(fullName).withRelativePartsResolved().asDirectory(); + const projectDirectory = resolveShellDirectory(await defaultDirectory(), '..', 'local_projects', fullName); + return resource('git/' + projectDirectory).asDirectory(); } static fetchInfoPreflight (fullName) { @@ -233,8 +265,8 @@ export class Project { } const projectDir = await Project.projectDirectory(`${projectRepoOwner}--${projectName}`); - await evalOnServer(`System.get("@lively-env").packageRegistry.addPackageAt(System.baseURL + 'local_projects/${projectRepoOwner}--${projectName}')`); - await System.get('@lively-env').packageRegistry.addPackageAt(projectDir); + await addProjectPackageOnServerIfMissing(`${projectRepoOwner}--${projectName}`); + await addPackageAtIfMissing(System.get('@lively-env').packageRegistry, projectDir); li.remove(); return `${projectRepoOwner}--${projectName}`; @@ -251,7 +283,7 @@ export class Project { static async deleteProject (name, repoOwner, token, deleteRemote) { const baseURL = (await Project.systemInterface.getConfig()).baseURL; const projectsDir = lively.FreezerRuntime ? resource(baseURL).join(`../local_projects/${repoOwner}--${name}`).withRelativePartsResolved().asDirectory() : (await Project.projectDirectory(`${repoOwner}--${name}`)); - const gitResource = await resource('git/' + projectsDir.url); + const gitResource = await Project.ensureGitResource(`${repoOwner}--${name}`); try { await evalOnServer(` @@ -621,8 +653,8 @@ export class Project { } await this.saveConfigData(); - await evalOnServer(`System.get("@lively-env").packageRegistry.addPackageAt(System.baseURL + 'local_projects/${this.fullName}')`); - await System.get('@lively-env').packageRegistry.addPackageAt(projectDir); + await addProjectPackageOnServerIfMissing(this.fullName); + await addPackageAtIfMissing(System.get('@lively-env').packageRegistry, projectDir); const pkg = await loadPackage(system, { name: this.fullName, @@ -635,6 +667,7 @@ export class Project { }); this.package = pkg; } catch (error) { + console.error('Error creating project files', error); throw Error('Error creating project files', { cause: error }); } await Project.installCSSForProject(this.url, true, this.fullName, this); @@ -643,6 +676,7 @@ export class Project { await this.gitResource.createAndAddRemoteToGitRepository(currentUserToken(), this.name, gitHubUser, this.config.description, gitHubUser !== currentUsername(), priv); await this.regeneratePipelines(); } catch (e) { + console.error('Error setting up project remote', e); throw Error('Error setting up remote', { cause: e }); } } @@ -789,8 +823,8 @@ export class Project { if (cmd.exitCode !== 0) throw Error('Error cloning uninstalled dependency project.'); // Refresh the cache of available projects and their version. const projectDir = await Project.projectDirectory(`${depRepoOwner}--${depName}`); - await evalOnServer(`System.get("@lively-env").packageRegistry.addPackageAt(System.baseURL + 'local_projects/${depRepoOwner}--${depName}')`); - await PackageRegistry.ofSystem(System).addPackageAt(projectDir); + await addProjectPackageOnServerIfMissing(`${depRepoOwner}--${depName}`); + await addPackageAtIfMissing(PackageRegistry.ofSystem(System), projectDir); availableProjects = await Project.listAvailableProjects(); } // Add all transitive dependencies from the current dependency to the list. diff --git a/lively.project/prompts.cp.js b/lively.project/prompts.cp.js index af1b91b642..65695fb7ab 100644 --- a/lively.project/prompts.cp.js +++ b/lively.project/prompts.cp.js @@ -264,6 +264,7 @@ class ProjectCreationPromptModel extends AbstractPromptModel { li.remove(); super.resolve(createdProject); } catch (err) { + console.error('Error initializing project or remote', err); this.enableButtons(); li?.remove(); this.view.setStatusMessage('There was an error initializing the project or its remote.', StatusMessageError); @@ -425,6 +426,7 @@ class RepoCreationPromptModel extends AbstractPromptModel { $world.setStatusMessage('Project uploaded!', StatusMessageConfirm); super.resolve(true); } catch (err) { + console.error('Error creating repository', err); this.enableButtons(); li?.remove(); this.view.setStatusMessage('There was an error creating the repository.', StatusMessageError); diff --git a/lively.resources/src/fs-resource.js b/lively.resources/src/fs-resource.js index b134eeeb8c..dc50dd6975 100644 --- a/lively.resources/src/fs-resource.js +++ b/lively.resources/src/fs-resource.js @@ -4,6 +4,13 @@ import { applyExclude, windowsURLPrefixRe, windowsRootPathRe, ensurePlatformInde import { createWriteStream, createReadStream, readFile, writeFile, stat, mkdir, rmdir, unlink, readdir, lstat, rename, constants, access } from 'fs'; +function fileURLToPathname (url) { + const parsed = new URL(url); + let decodedPath = decodeURIComponent(parsed.pathname); + if (parsed.host) decodedPath = `//${parsed.host}${decodedPath}`; + return decodedPath.replace(/^\/([a-z]:\/)/i, '$1').replace(/\\/g, '/'); +} + function exists (path, cb) { return access(path, constants.F_OK, (err) => { if (err) { @@ -35,7 +42,7 @@ export class NodeJSFileResource extends Resource { get isNodeJSFileResource () { return true; } path () { - return this.url.replace('file://', ''); + return fileURLToPathname(this.url); } async stat () { @@ -178,7 +185,7 @@ export class NodeJSWindowsFileResource extends NodeJSFileResource { } path () { - return this.url.replace('file:///', ''); + return fileURLToPathname(this.url); } isRoot () { diff --git a/lively.server/bin/start-server.js b/lively.server/bin/start-server.js index 902dd0a0e9..0f52d3c8e3 100755 --- a/lively.server/bin/start-server.js +++ b/lively.server/bin/start-server.js @@ -1,11 +1,13 @@ #!/bin/sh ':' //; exec "$(command -v nodejs || command -v node)" "$0" "$@" -import System from 'systemjs'; -import parseArgs from 'minimist'; +import { createRequire } from 'module'; import url from 'url' -global.System = System; +const require = createRequire(import.meta.url); +const System = require('systemjs'); +const parseArgs = require('minimist'); +global.System = System; const isMain = import.meta.url === url.pathToFileURL(process.argv[1]).href; const defaultRootDirectory = process.cwd(); @@ -21,4 +23,3 @@ if (isMain) { args["root-directory"] || defaultRootDirectory); }); } - diff --git a/lively.server/index.js b/lively.server/index.js index 6d2274ca22..65a46b5900 100644 --- a/lively.server/index.js +++ b/lively.server/index.js @@ -5,10 +5,14 @@ import { resource } from 'lively.resources'; import { obj } from 'lively.lang'; import "socket.io"; import util from 'node:util'; -import winston from "winston"; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { pathToFileURL } from 'node:url'; import { setupSystem } from "lively.installer"; import { Generator } from "@jspm/generator"; +const require = createRequire(import.meta.url); +const winston = require('winston'); const defaultServerDir = process.cwd(); var livelySystem; var config = { @@ -21,7 +25,7 @@ var config = { export default async function start(hostname, port, configFile, rootDirectory, serverDir) { config.rootDirectory = rootDirectory || process.cwd(); - config.serverDir = serverDir || defaultServerDir; + config.serverDir = directoryURL(serverDir || defaultServerDir); setupLogger(); var step = 1; console.log(`[lively.server] system base directory: ${rootDirectory}`); @@ -68,7 +72,7 @@ export default async function start(hostname, port, configFile, rootDirectory, s // 2. this loads and starts the server .then(() => livelySystem.set('@jspm_generator', livelySystem.newModule({ default: Generator }))) .then(() => console.log(`[lively.server] ${step++}. starting server`)) - .then(() => livelySystem.import(config.serverDir + "/server.js")) + .then(() => livelySystem.import(serverModuleURL(config.serverDir))) .then(serverMod => startServer(serverMod, config)) .then(server => { console.log(`[lively.server] ${step++}. server sucessfully started`); @@ -81,6 +85,20 @@ export default async function start(hostname, port, configFile, rootDirectory, s }); }; +function directoryURL(dir) { + if (/^[a-z][a-z0-9+.-]*:\/\//i.test(dir)) { + return resource(dir.endsWith('/') ? dir : `${dir}/`).url.replace(/\/$/, ''); + } + return pathToFileURL(path.resolve(dir)).href; +} + +function serverModuleURL(serverDir) { + if (/^[a-z][a-z0-9+.-]*:\/\//i.test(serverDir)) { + return resource(serverDir.endsWith('/') ? serverDir : `${serverDir}/`).join('server.js').url; + } + return pathToFileURL(path.join(serverDir, 'server.js')).href; +} + async function silenceDuring(filter, promise) { let {stdout, stderr} = process, {write: stdoutWrite} = stdout, @@ -98,7 +116,7 @@ function formatArgs(args){ } function setupLogger() { - let logger = new winston.createLogger(); + let logger = winston.createLogger(); const myFormat = winston.format.printf(({ level, message, label, timestamp }) => { return `${timestamp} ${level}: ${message}`; }); diff --git a/lively.server/plugins/dav.js b/lively.server/plugins/dav.js index b8fcb9fabc..1a0446e2f6 100644 --- a/lively.server/plugins/dav.js +++ b/lively.server/plugins/dav.js @@ -6,6 +6,8 @@ const tar = System._nodeRequire('tar-fs'); import stream from 'stream'; import util from 'util'; import zlib from 'zlib'; +import fs from 'node:fs'; +import { fileURLToPath } from 'node:url'; const COMPRESSABLE_URLS = [ 'components_cache' @@ -16,6 +18,28 @@ const compression = 'gzip'; // can be either 'gzip' or 'brotli' where 'brotli' t // FIXME... let DavHandler; let FsTree; const jsDavPlugins = {}; + +function patchJsDAVUtilForWindowsPaths (Util) { + if (!Util || Util.__livelyWindowsPathPatch) return; + + const originalSplitPath = Util.splitPath; + Util.rtrim = function (str, charlist) { + charlist = !charlist + ? ' \\s\u00A0' + : (charlist + '').replace(/([\\[\]().?/*{}+$^:-])/g, '\\$1'); + const re = new RegExp('[' + charlist + ']+$', 'g'); + return (str + '').replace(re, ''); + }; + + if (process.platform === 'win32') { + Util.splitPath = function (inputPath) { + return originalSplitPath.call(this, String(inputPath).replace(/\\/g, '/')); + }; + } + + Util.__livelyWindowsPathPatch = true; +} + (function loadJsDAV () { // jsDAV shows unimportant console logs while loading, hide those... const log = console.log; @@ -34,6 +58,7 @@ const jsDavPlugins = {}; DavHandler = System._nodeRequire('jsDAV/lib/DAV/handler'); FsTree = System._nodeRequire('jsDAV/lib/DAV/backends/fs/tree'); jsDavPlugins.browser = System._nodeRequire('jsDAV/lib/DAV/plugins/browser.js'); + patchJsDAVUtilForWindowsPaths(System._nodeRequire('jsDAV/lib/shared/util')); } catch (err) { console.error('cannot load jsdav:', err); } finally { console.log = log; } })(); @@ -119,6 +144,22 @@ export default class LivelyDAVPlugin { this.fileHashes[file.url.replace(System.baseURL, '/')] = string.hashCode(await file.read()); } console.log('[lively.server] finished file hash map'); + + // Skip the tar+gzip step when a pre-built snapshot is available + // (bundled distributions set LIVELY_PREBUILT_LIBRARY_SNAPSHOT so the + // server doesn't re-generate the blob on every launch). + const prebuiltSnapshot = process.env.LIVELY_PREBUILT_LIBRARY_SNAPSHOT; + if (prebuiltSnapshot && fs.existsSync(prebuiltSnapshot)) { + try { + console.log('[lively.server] loading pre-built library snapshot: ' + prebuiltSnapshot); + memStore.compressedLibrary = fs.readFileSync(prebuiltSnapshot); + console.log('[lively.server] pre-built library snapshot loaded'); + return; + } catch (err) { + console.warn('[lively.server] failed to load pre-built snapshot, regenerating:', err.message); + } + } + console.log('[lively.server] creating library snapshot...'); await this.compressLibraryCode(); console.log('[lively.server] finished library snapshot'); @@ -147,7 +188,7 @@ export default class LivelyDAVPlugin { 'lively.freezer/swc-plugin/target', 'lively.modules/dist' ]; - tar.pack(System.baseURL.replace('file://', ''), { + tar.pack(fileURLToPath(System.baseURL), { ignore (name) { if (excludedDirs.find(path => name.includes(path))) return true; else return false; diff --git a/lively.server/plugins/l2l.js b/lively.server/plugins/l2l.js index a7d82b6658..bec60b8b70 100644 --- a/lively.server/plugins/l2l.js +++ b/lively.server/plugins/l2l.js @@ -34,6 +34,8 @@ export default class Lively2LivelyPlugin { .then( async () => { livelyServer.debug && console.log(`[lively.server] started ${this.l2lTracker}`); + if (process.env.LIVELY_DESKTOP_APP === '1') return; + const client = new L2LClient.ensure({ url: `http://${hostname}:${port}/lively-socket.io`, namespace: "l2l", @@ -64,4 +66,4 @@ export default class Lively2LivelyPlugin { console.error(`Error closing l2l tracker ${this.l2lTracker}: ${e.stack} (${this})`); } finally { this.l2lTracker = null; } } -} \ No newline at end of file +} diff --git a/lively.server/plugins/lib-lookup.js b/lively.server/plugins/lib-lookup.js index 76b08c8150..40c8c397ef 100644 --- a/lively.server/plugins/lib-lookup.js +++ b/lively.server/plugins/lib-lookup.js @@ -26,6 +26,7 @@ function isUnresolvableOnCDN ([name, version]) { * "Package @ not found on ..." */ function extractFailingPackage (errMsg) { + if (!/Unable to fetch|Package\s+@?[^\s@]+@\S+\s+not found/i.test(errMsg)) return null; const cdnMatch = errMsg.match(/ga\.jspm\.io\/npm:(@?[^@]+)@([^/]+)\//); if (cdnMatch) return { name: cdnMatch[1], version: cdnMatch[2] }; const pkgMatch = errMsg.match(/Package\s+(@?[^\s@]+)@(\S+)/); @@ -140,7 +141,11 @@ async function installDeps (generator, deps, failed, resolutions, inputMap) { if (!depNames.includes(failedDep)) delete failed[failedDep]; } const toUninstall = arr.withoutAll(Object.keys(generator.map.imports), deps.map(d => d[0])); - await generator.uninstall(toUninstall); + try { + await generator.uninstall(toUninstall); + } catch (err) { + console.warn('\x1b[33m [!] Import map: cleanup failed: ' + (err.message || err) + '\x1b[0m'); + } return generator; } diff --git a/lively.server/scripts/build-library-snapshot.cjs b/lively.server/scripts/build-library-snapshot.cjs new file mode 100644 index 0000000000..eb19ce55fd --- /dev/null +++ b/lively.server/scripts/build-library-snapshot.cjs @@ -0,0 +1,101 @@ +#!/usr/bin/env node +// Pre-builds the library snapshot that LivelyDAVPlugin.compressLibraryCode +// normally regenerates on every server startup (tar + gzip of ~28 lively +// package directories, several seconds of work). Running this once at +// build time and shipping the resulting .library-snapshot.tar.gz inside +// the bundle cuts server startup time significantly. +// +// Keep the cachedDirs / excludedDirs lists in sync with dav.js manually +// for now — extracting them into a shared module would require either +// inlining them via a build step or converting this script to ESM with +// flatn resolution. + +const fs = require('node:fs'); +const path = require('node:path'); +const zlib = require('node:zlib'); + +const rootDir = path.resolve(__dirname, '..', '..'); +const outFile = path.join(__dirname, '..', '.library-snapshot.tar.gz'); + +// Self-sufficient: set FLATN env vars + install the CJS resolver hook, +// so this script runs standalone without requiring the caller to source +// scripts/lively-next-env.sh first. +if (!process.env.FLATN_DEV_PACKAGE_DIRS) { + const pkgs = JSON.parse(fs.readFileSync( + path.join(rootDir, 'lively.installer/packages-config.json'), 'utf8')); + const devDirs = pkgs + .map(p => path.join(rootDir, p.name)) + .filter(d => fs.existsSync(d)); + process.env.FLATN_PACKAGE_COLLECTION_DIRS = [ + path.join(rootDir, 'lively.next-node_modules'), + path.join(rootDir, 'custom-npm-modules') + ].join(':'); + process.env.FLATN_DEV_PACKAGE_DIRS = devDirs.join(':'); + process.env.FLATN_PACKAGE_DIRS = ''; +} + +// Load flatn's CJS hook to resolve packages in its flat-layout node_modules. +// Preserve process.execPath — flatn/resolver.cjs overrides it with its wrapper +// script path, which breaks things that rely on execPath downstream. +const _savedExecPath = process.execPath; +const _savedArgv0 = process.argv[0]; +require(path.join(rootDir, 'flatn', 'resolver.cjs')); +process.execPath = _savedExecPath; +process.argv[0] = _savedArgv0; + +const tar = require('tar-fs'); + +// Must match LivelyDAVPlugin.compressLibraryCode in lively.server/plugins/dav.js +const cachedDirs = [ + 'esm_cache', + 'lively.morphic', 'lively.lang', 'lively.bindings', 'lively.ast', + 'lively.source-transform', 'lively.classes', 'lively.vm', 'lively.resources', + 'lively.storage', 'lively.notifications', 'lively.modules', + 'lively-system-interface', 'lively.installer', 'lively.serializer2', + 'lively.graphics', 'lively.keyboard', 'lively.changesets', 'lively.2lively', + 'lively.git', 'lively.traits', 'lively.components', 'lively.ide', + 'lively.headless', 'lively.freezer', 'lively.collab', 'lively.project', + 'lively.user' +]; + +const excludedDirs = [ + 'lively.morphic/objectdb', + 'lively.morphic/assets', + 'lively.morphic/web', + 'lively.ast/dist', + 'lively.classes/build', + 'lively.ide/jsdom.worker.js', + 'lively.headless/chrome-data-dir', + 'lively.freezer/landing-page', + 'lively.freezer/loading-screen', + 'lively.freezer/.swc', + 'lively.freezer/swc-plugin/target', + 'lively.modules/dist' +]; + +// Only include cachedDirs that actually exist on disk (esm_cache is created +// lazily by the server, so may be missing in CI). +const presentDirs = cachedDirs.filter(d => fs.existsSync(path.join(rootDir, d))); +console.log(`[build-library-snapshot] packing ${presentDirs.length}/${cachedDirs.length} dirs from ${rootDir}`); +if (presentDirs.length < cachedDirs.length) { + const missing = cachedDirs.filter(d => !presentDirs.includes(d)); + console.log(`[build-library-snapshot] missing (skipped): ${missing.join(', ')}`); +} + +fs.mkdirSync(path.dirname(outFile), { recursive: true }); + +tar.pack(rootDir, { + ignore (name) { + return excludedDirs.some(p => name.includes(p)); + }, + entries: presentDirs +}).pipe(zlib.Gzip()).pipe(fs.createWriteStream(outFile)) + .on('finish', () => { + const size = fs.statSync(outFile).size; + const mb = (size / 1024 / 1024).toFixed(1); + console.log(`[build-library-snapshot] wrote ${outFile} (${mb} MB)`); + }) + .on('error', err => { + console.error('[build-library-snapshot] failed:', err); + process.exit(1); + }); diff --git a/lively.shell/bin/askpass.sh b/lively.shell/bin/askpass.sh index ecca4dfb42..dbba74f79b 100755 --- a/lively.shell/bin/askpass.sh +++ b/lively.shell/bin/askpass.sh @@ -1,6 +1,11 @@ #!/usr/bin/env bash PASSWORD_QUERY=$1 -DIR=${WORKSPACE_LK-"$0/../.."} -RESOLVER=$(node -e "console.log(require.resolve('flatn/resolver.mjs'))") -node --no-warnings --experimental-loader $RESOLVER --dns-result-order ipv4first $DIR/bin/askpass.js $PASSWORD_QUERY +DIR="$(cd "$(dirname "$0")" && pwd)" +if [ -z "$WORKSPACE_LK" ]; then + export WORKSPACE_LK="$(cd "$DIR/.." && pwd)" +fi +ROOT_DIR="$(cd "$WORKSPACE_LK/.." && pwd)" +RESOLVER="$ROOT_DIR/flatn/resolver.mjs" + +node --no-warnings --experimental-loader "$RESOLVER" --dns-result-order ipv4first "$WORKSPACE_LK/bin/askpass.js" "$PASSWORD_QUERY" diff --git a/lively.shell/bin/lively-as-editor.sh b/lively.shell/bin/lively-as-editor.sh index c75376a6f6..b1aecfe00e 100755 --- a/lively.shell/bin/lively-as-editor.sh +++ b/lively.shell/bin/lively-as-editor.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash FILE=$1 -DIR=`dirname $0` +DIR="$(cd "$(dirname "$0")" && pwd)" UNAME=$(uname | tr '[:upper:]' '[:lower:]') [ $UNAME = "darwin" ] && IS_DARWIN=1 [ $UNAME = "linux" ] && IS_LINUX=1 if [ -z "$WORKSPACE_LK" ]; then - export WORKSPACE_LK=$(dirname $DIR) + export WORKSPACE_LK="$(cd "$DIR/.." && pwd)" fi @@ -19,5 +19,7 @@ if [ "${FILE:0:1}" != "/" ]; then fi fi -RESOLVER=$(node -e "console.log(require.resolve('flatn/resolver.mjs'))") -node --no-warnings --experimental-loader $RESOLVER --dns-result-order ipv4first $DIR/lively-as-editor.js "$FILE" +ROOT_DIR="$(cd "$WORKSPACE_LK/.." && pwd)" +RESOLVER="$ROOT_DIR/flatn/resolver.mjs" + +node --no-warnings --experimental-loader "$RESOLVER" --dns-result-order ipv4first "$DIR/lively-as-editor.js" "$FILE" diff --git a/lively.shell/bin/lively.profile b/lively.shell/bin/lively.profile index 1485c32c07..866a21dd34 100644 --- a/lively.shell/bin/lively.profile +++ b/lively.shell/bin/lively.profile @@ -2,17 +2,13 @@ eval $( node -e 'let pathParts = process.env.PATH.split(":"); let found = pathParts.findIndex(ea => ea.endsWith("flatn/bin")); if (found > 0) { console.log("export PATH=" + [...pathParts.splice(found, 1), ...pathParts].join(":").replace(/([ ])/g, "\\$1"));}' ) -function normalize_path { - echo $(builtin cd "$1"; pwd); -} - function cd { - DIR=$(normalize_path "$1") + builtin cd "${1:-$HOME}" || return $? + DIR=$(pwd) send-to-lively.sh \ changeWorkingDirectory \ - $DIR \ - $LIVELY_COMMAND_OWNER > /dev/null; - builtin cd "${DIR}" + "$DIR" \ + "$LIVELY_COMMAND_OWNER" > /dev/null 2>&1 || true; } function em { @@ -42,4 +38,3 @@ function grep_in_lively { builtin cd "$LIVELY"; find_in_lively "$FILE_MATCH" -print0 | xargs -0 grep -nH $1 } - diff --git a/lively.shell/bin/send-to-lively.sh b/lively.shell/bin/send-to-lively.sh index 891c158178..60c78ef06a 100755 --- a/lively.shell/bin/send-to-lively.sh +++ b/lively.shell/bin/send-to-lively.sh @@ -1,14 +1,15 @@ #!/usr/bin/env bash -DIR=`dirname $0` +DIR="$(cd "$(dirname "$0")" && pwd)" UNAME=$(uname | tr '[:upper:]' '[:lower:]') [ $UNAME = "darwin" ] && IS_DARWIN=1 [ $UNAME = "linux" ] && IS_LINUX=1 if [ -z "$WORKSPACE_LK" ]; then - export WORKSPACE_LK=$(dirname $DIR) + export WORKSPACE_LK="$(cd "$DIR/.." && pwd)" fi -RESOLVER=$(node -e "console.log(require.resolve('flatn/resolver.mjs'))") +ROOT_DIR="$(cd "$WORKSPACE_LK/.." && pwd)" +RESOLVER="$ROOT_DIR/flatn/resolver.mjs" -node --no-warnings --experimental-loader $RESOLVER --dns-result-order ipv4first $WORKSPACE_LK/bin/send-to-lively.js $@ +node --no-warnings --experimental-loader "$RESOLVER" --dns-result-order ipv4first "$WORKSPACE_LK/bin/send-to-lively.js" "$@" diff --git a/lively.shell/git-client-resource.js b/lively.shell/git-client-resource.js index c0b6ccdeb1..80ee49bea8 100644 --- a/lively.shell/git-client-resource.js +++ b/lively.shell/git-client-resource.js @@ -6,7 +6,8 @@ import L2LClient from 'lively.2lively/client.js'; export default class GitShellResource extends ShellClientResource { constructor (url) { - url = url.replace('git\/', ''); + url = url.replace(/^git\//, ''); + if (url.match(/^\/[a-z]:\//i)) url = url.slice(1); super(url); this.options.cwd = this.url; if (!this.options.l2lClient) { @@ -133,28 +134,40 @@ export default class GitShellResource extends ShellClientResource { } async createAndAddRemoteToGitRepository (token, repoName, repoUser, repoDescription, orgScope, priv) { - const repoCreationCommand = orgScope - ? `curl -L \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${token}"\ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/orgs/${repoUser}/repos \ - -d '{"name":"${repoName}","description":"${repoDescription}", "private":${!!priv}}'` - : `curl -L \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${token}" \ - https://api.github.com/user/repos \ - -d '{"name":"${repoName}", "description": "${repoDescription}", "private":${!!priv}}'`; - let cmd = this.runCommand(repoCreationCommand); - await cmd.whenDone(); - if (cmd.exitCode !== 0) throw Error('Error executing curl call to create GitHub repository'); - this.addRemoteURLWithToken(token, repoUser, repoName); + const url = orgScope + ? `https://api.github.com/orgs/${repoUser}/repos` + : 'https://api.github.com/user/repos'; + const response = await fetch(url, { + method: 'POST', + headers: { + accept: 'application/vnd.github+json', + authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28' + }, + body: JSON.stringify({ + name: repoName, + description: repoDescription || '', + private: !!priv + }) + }); + + if (!response.ok) { + let details = ''; + try { + const body = await response.json(); + details = body.message || JSON.stringify(body); + } catch (err) { + details = await response.text().catch(() => ''); + } + throw Error(`Error creating GitHub repository (${response.status} ${response.statusText})${details ? `: ${details}` : ''}`); + } + + await this.addRemoteURLWithToken(token, repoUser, repoName); } async addRemoteURLWithToken (token, repoUser, repoName) { - const addingRemoteCommand = `git remote add origin https://${token}@github.com/${repoUser}/${repoName}`; + const remoteURL = `https://${token}@github.com/${repoUser}/${repoName}`; + const addingRemoteCommand = `git remote get-url origin >/dev/null 2>&1 && git remote set-url origin ${remoteURL} || git remote add origin ${remoteURL}`; const cmd = this.runCommand(addingRemoteCommand); await cmd.whenDone(); if (cmd.exitCode !== 0) throw Error('Error adding the remote to local repository'); diff --git a/lively.shell/server-command.js b/lively.shell/server-command.js index 731735d621..f43408205d 100644 --- a/lively.shell/server-command.js +++ b/lively.shell/server-command.js @@ -6,9 +6,11 @@ import { signal } from 'lively.bindings'; import { format } from 'util'; import fs from 'fs'; import path from 'path'; +import os from 'os'; import proc from 'child_process'; +import { fileURLToPath } from 'url'; -let spawn, exec, isWindows, defaultEnv; +let spawn, exec, isWindows, defaultEnv, windowsBash; let askPassScript = ''; let editorScript = ''; let doKill; @@ -23,6 +25,46 @@ let binDir = typeof System !== 'undefined' // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- +function pathEnvValue (env) { + const key = Object.keys(env || {}).find(key => + isWindows ? key.toLowerCase() === 'path' : key === 'PATH'); + return key ? env[key] : ''; +} + +function setPathEnvValue (env, value) { + if (isWindows) { + for (const key of Object.keys(env)) { + if (key.toLowerCase() === 'path') delete env[key]; + } + env.Path = value; + } else { + env.PATH = value; + } + return env; +} + +function normalizedEnv (env) { + const result = {}; + for (const key of Object.keys(env || {})) { + if (isWindows && key.toLowerCase() === 'path') continue; + if (env[key] !== undefined) result[key] = env[key]; + } + setPathEnvValue(result, pathEnvValue(env)); + return result; +} + +function normalizeCwd (cwd) { + let normalized = String(cwd || process.cwd()); + if (!isWindows) return normalized; + + if (normalized.startsWith('file:')) { + try { normalized = fileURLToPath(normalized); } catch (_) {} + } + normalized = normalized.replace(/^git\//, ''); + if (normalized.match(/^\/[a-z]:[\\/]/i)) normalized = normalized.slice(1); + return normalized; +} + /* * determine the kill command at startup. if pkill is available then use that * to do a full process tree kill @@ -101,8 +143,10 @@ export default class ServerCommand extends CommandInterface { } ({ command, cwd, stdin } = options); - let env = Object.assign(Object.create(defaultEnv), this.envForCommand(options)); - cwd = cwd || process.cwd(); + const commandEnv = this.envForCommand(options); + let env = { ...defaultEnv, ...commandEnv }; + if (isWindows) setPathEnvValue(env, pathEnvValue(commandEnv) || pathEnvValue(defaultEnv)); + cwd = normalizeCwd(cwd); command = Array.isArray(command) ? command.join(' ') : String(command); @@ -112,22 +156,26 @@ export default class ServerCommand extends CommandInterface { /\$[a-zA-Z0-9_]+/g, match => env[match.slice(1, match.length)] || match); - if (!isWindows) command = `source ${binDir}/lively.profile; ${command}`; - - ([command, args] = (isWindows - ? ['cmd', ['/C', command]] - : ['/bin/bash', ['-c', command]])); + if (isWindows) { + ([command, args] = windowsBash + ? [windowsBash, ['-lc', command]] + : ['cmd', ['/C', command]]); + } else { + command = `source ${binDir}/lively.profile; ${command}`; + ([command, args] = ['/bin/bash', ['-c', command]]); + } + const commandLine = [command].concat(args).join(' '); let proc = spawn(command, args, { env, cwd, stdio: 'pipe', detached: true }); - if (this.debug) console.log('Running command: "%s" (%s)', [command].concat(args).join(' '), proc.pid); + if (this.debug) console.log('Running command: "%s" (%s)', commandLine, proc.pid); if (stdin) { this.debug && console.log('setting stdin to: %s', stdin); proc.stdin.end(stdin); } - this.attachTo(proc, options); + this.attachTo(proc, { ...options, commandLine, cwd, envPath: pathEnvValue(env) }); return this; } @@ -170,11 +218,17 @@ export default class ServerCommand extends CommandInterface { }); proc.on('error', (err) => { - this.debug && console.log('shell command errored ' + err); + const message = [ + err.stack || String(err), + options.commandLine ? `Command: ${options.commandLine}` : null, + options.cwd ? `Cwd: ${options.cwd}` : null, + options.envPath ? `Path: ${options.envPath}` : null + ].filter(Boolean).join('\n'); + this.debug && console.log('shell command errored ' + message); arr.remove(this.constructor.commands, this); - this._stderr += err.stack; - this.emit('error', err.stack); - signal(this, 'error', err.stack); + this._stderr += message; + this.emit('error', message); + signal(this, 'error', message); this.exitCode = 1; }); } @@ -272,19 +326,21 @@ try { spawn = proc.spawn; exec = proc.exec; - isWindows = process.platform !== 'linux' && - process.platform !== 'darwin' && - process.platform.include('win'); - - defaultEnv = Object.assign( - Object.create(process.env), { - SHELL: '/bin/bash', - PAGER: 'ul | cat -s', - MANPAGER: 'ul | cat -s', - TERM: 'xterm', - PATH: binDir + path.delimiter + process.env.PATH, - LIVELY: LIVELY - }); + isWindows = process.platform === 'win32'; + windowsBash = isWindows && process.env.LIVELY_WINDOWS_BASH && fs.existsSync(process.env.LIVELY_WINDOWS_BASH) + ? process.env.LIVELY_WINDOWS_BASH + : ''; + + defaultEnv = { + ...normalizedEnv(process.env), + SHELL: windowsBash || '/bin/bash', + PAGER: 'ul | cat -s', + MANPAGER: 'ul | cat -s', + TERM: 'xterm', + LIVELY: LIVELY, + ...(isWindows ? { HOME: process.env.HOME || process.env.USERPROFILE || os.homedir() } : {}) + }; + setPathEnvValue(defaultEnv, [binDir, pathEnvValue(defaultEnv)].filter(Boolean).join(path.delimiter)); /* * ASKPASS support for tunneling sudo / ssh / git password requests back to the @@ -308,5 +364,5 @@ try { exec('which pkill', function (code) { if (!code) doKill = pkillKill; }); })(); } catch (err) { - + console.error('[lively.shell] failed to initialize server command support:', err && (err.stack || err)); } diff --git a/lively.source-transform/babel/helpers.js b/lively.source-transform/babel/helpers.js index 0cb1279e76..d1fe1b9cb5 100644 --- a/lively.source-transform/babel/helpers.js +++ b/lively.source-transform/babel/helpers.js @@ -1,6 +1,41 @@ -import t from '@babel/types'; +import _t from '@babel/types'; import { helpers } from 'lively.ast/lib/query.js'; +function cjsDefault (module, expectedProperty) { + let value = module; + const hasExpected = candidate => { + if (!expectedProperty) return true; + if (typeof candidate?.[expectedProperty] === 'function') return true; + const canonicalName = expectedProperty[0].toLowerCase() + expectedProperty.slice(1); + return typeof candidate?.[canonicalName] === 'function'; + }; + while (value && typeof value === 'object' && !hasExpected(value) && 'default' in value) { + value = value.default; + } + return value; +} + +function babelTypes (types) { + const target = types && (typeof types === 'object' || typeof types === 'function') ? types : {}; + return new Proxy(target, { + get (target, property, receiver) { + const value = Reflect.get(target, property, receiver); + const fallback = globalThis.babel?.types || globalThis.babel?.default?.types; + if (typeof property === 'string' && /^[A-Z]/.test(property) && typeof value !== 'function') { + const canonicalName = property[0].toLowerCase() + property.slice(1); + const canonical = Reflect.get(target, canonicalName, receiver); + if (typeof canonical === 'function') return canonical; + if (typeof fallback?.[property] === 'function') return fallback[property]; + if (typeof fallback?.[canonicalName] === 'function') return fallback[canonicalName]; + } + if (typeof value !== 'function' && typeof fallback?.[property] === 'function') return fallback[property]; + return value; + } + }); +} + +const t = babelTypes(cjsDefault(_t, 'Identifier')); + export function getAncestryPath (path) { return path.getAncestry().map(m => m.inList ? [m.key, m.listKey] : m.key).flat().slice(0, -1).reverse(); } diff --git a/lively.source-transform/babel/plugin.js b/lively.source-transform/babel/plugin.js index 790ae33c28..551a58dbc6 100644 --- a/lively.source-transform/babel/plugin.js +++ b/lively.source-transform/babel/plugin.js @@ -1,8 +1,8 @@ /* global require,global */ -import t from '@babel/types'; -import babel from '@babel/core'; -import systemjsTransform from '@babel/plugin-transform-modules-systemjs'; -import dynamicImport from '@babel/plugin-proposal-dynamic-import'; +import _t from '@babel/types'; +import _babel from '@babel/core'; +import _systemjsTransform from '@babel/plugin-transform-modules-systemjs'; +import _dynamicImport from '@babel/plugin-proposal-dynamic-import'; import { arr, Path } from 'lively.lang'; import { query } from 'lively.ast'; import { topLevelFuncDecls } from 'lively.ast/lib/visitors.js'; @@ -10,6 +10,51 @@ import { classToFunctionTransformBabel } from 'lively.classes'; import { getGlobal } from 'lively.vm/lib/util.js'; import { declarationWrapperCall, annotationSym, assignExpr, varDeclOrAssignment, transformPattern, generateUniqueName, varDeclAndImportCall, importCallStmt, shouldDeclBeCaptured, importCall, exportCallStmt, exportFromImport, additionalIgnoredDecls, additionalIgnoredRefs } from './helpers.js'; +function cjsDefault (module, expectedProperty, requireName) { + let value = module; + const hasExpected = candidate => { + if (typeof candidate === 'function') return true; + if (!expectedProperty) return true; + if (candidate?.[expectedProperty] != null) return true; + const canonicalName = expectedProperty[0].toLowerCase() + expectedProperty.slice(1); + return candidate?.[canonicalName] != null; + }; + while (value && typeof value === 'object' && !hasExpected(value) && 'default' in value) { + value = value.default; + } + if (!hasExpected(value) && requireName && globalThis.System?._nodeRequire) { + value = globalThis.System._nodeRequire(requireName); + while (value && typeof value === 'object' && !hasExpected(value) && 'default' in value) { + value = value.default; + } + } + return value; +} + +function babelTypes (types, fallbackTypes) { + const target = types && (typeof types === 'object' || typeof types === 'function') ? types : {}; + return new Proxy(target, { + get (target, property, receiver) { + const value = Reflect.get(target, property, receiver); + const fallback = globalThis.babel?.types || globalThis.babel?.default?.types || fallbackTypes; + if (typeof property === 'string' && /^[A-Z]/.test(property) && typeof value !== 'function') { + const canonicalName = property[0].toLowerCase() + property.slice(1); + const canonical = Reflect.get(target, canonicalName, receiver); + if (typeof canonical === 'function') return canonical; + if (typeof fallback?.[property] === 'function') return fallback[property]; + if (typeof fallback?.[canonicalName] === 'function') return fallback[canonicalName]; + } + if (typeof value !== 'function' && typeof fallback?.[property] === 'function') return fallback[property]; + return value; + } + }); +} + +const babel = cjsDefault(_babel, 'parse'); +const t = babelTypes(cjsDefault(_t, 'Identifier'), babel?.types); +const systemjsTransform = cjsDefault(_systemjsTransform, 'visitor', '@babel/plugin-transform-modules-systemjs'); +const dynamicImport = cjsDefault(_dynamicImport, 'visitor', '@babel/plugin-proposal-dynamic-import'); + export function babel_parse (source) { return babel.parse(source).program.body; } @@ -1236,15 +1281,16 @@ function rewriteToRegisterModuleToCaptureSetters (path, state, options) { const { _renamedExports: renamedExports = {} } = state.opts.module || {}; - const registerCall = path.get('body.0.expression'); - const printAst = () => babel.transformFromAstSync(path.node).code.slice(0, 300); - if (registerCall.node.callee.object.name !== 'System') { - throw new Error(`rewriteToRegisterModuleToCaptureSetters: input doesn't seem to be a System.register call: ${printAst()}...`); - } - if (registerCall.node.callee.property.name !== 'register') { + const registerStmt = path.get('body').find(stmt => { + const expr = stmt.get('expression').node; + return (expr?.callee?.object?.name === 'System' || expr?.callee?.object?.name === 'SystemJS') && + expr?.callee?.property?.name === 'register'; + }); + if (!registerStmt) { throw new Error(`rewriteToRegisterModuleToCaptureSetters: input doesn't seem to be a System.register call: ${printAst()}...`); } + const registerCall = registerStmt.get('expression'); const registerBody = registerCall.get('arguments.1.body'); const registerReturn = arr.last(registerBody.get('body')); @@ -1395,7 +1441,7 @@ class BabelTranspiler { let opts = System.babelOptions; let needsBabel = (opts.plugins && opts.plugins.length) || (opts.presets && opts.presets.length); return needsBabel - ? System.global.babel.transform(source, opts).code + ? babel.transform(source, opts).code : source; } @@ -1403,6 +1449,7 @@ class BabelTranspiler { let System = this.System; let opts = Object.assign({}, System.babelOptions); opts.sourceFileName = options.module?.id; + opts.sourceType = 'module'; opts.plugins = opts.plugins ? opts.plugins.slice() : [ diff --git a/lively.user/user-flap.cp.js b/lively.user/user-flap.cp.js index a1ad049d6a..87a6cca6b3 100644 --- a/lively.user/user-flap.cp.js +++ b/lively.user/user-flap.cp.js @@ -17,6 +17,20 @@ import { Spinner } from 'lively.components/loading-indicator.cp.js'; const livelyAuthGithubAppId = 'd523a69022b9ef6be515'; +function parseGithubAuthResponse (response) { + const trimmed = String(response || '').trim(); + if (!trimmed) return {}; + try { + return JSON.parse(trimmed); + } catch (err) { + return Object.fromEntries(new URLSearchParams(trimmed)); + } +} + +function githubAuthErrorMessage (response) { + return response.error_description || response.error || null; +} + const CompactConfirmPrompt = component(ConfirmPrompt, { master: DarkPrompt, layout: new TilingLayout({ @@ -154,26 +168,29 @@ export class UserFlapModel extends ViewModel { $world.setStatusMessage('Login is not possible while in offline mode'); return; } - let cmdString = `curl -X POST -F 'client_id=${livelyAuthGithubAppId}' -F 'scope=user,repo,delete_repo,workflow' https://github.com/login/device/code`; - const { stdout: resOne } = await runCommand(cmdString).whenDone(); - if (resOne === '') { + let cmdString = `curl -sS -X POST -H 'Accept: application/json' -F 'client_id=${livelyAuthGithubAppId}' -F 'scope=user,repo,delete_repo,workflow' https://github.com/login/device/code`; + const codeCmd = await runCommand(cmdString).whenDone(); + const resOne = parseGithubAuthResponse(codeCmd.stdout); + if (codeCmd.exitCode !== 0 || !codeCmd.stdout) { + console.error('[github-login] device code request failed:', codeCmd.stderr || codeCmd.stdout); // eslint-disable-line no-console $world.setStatusMessage('You seem to be offline.', StatusMessageError); return; } - if (resOne === 'NOT FOUND') { - $world.setStatusMessage('An unexpected error occured. Please contact the lively.next team.', StatusMessageError); + const initialError = githubAuthErrorMessage(resOne); + if (initialError) { + console.error('[github-login] device code request rejected:', resOne); // eslint-disable-line no-console + $world.setStatusMessage(`GitHub login failed: ${initialError}`, StatusMessageError); return; } - const deviceCodeMatch = resOne.match(new RegExp('device_code=(.*)&e')); - const userCodeMatch = resOne.match(new RegExp('user_code=(.*)&')); - if (!deviceCodeMatch || !userCodeMatch) { - $world.setStatusMessage('An unexpected error occured. Please contact the lively.next team.', StatusMessageError); + const deviceCode = resOne.device_code; + const userCode = resOne.user_code; + if (!deviceCode || !userCode) { + console.error('[github-login] unexpected device code response:', codeCmd.stdout); // eslint-disable-line no-console + $world.setStatusMessage('GitHub login failed: unexpected device code response.', StatusMessageError); return; } - const deviceCode = deviceCodeMatch[1]; - const userCode = userCodeMatch[1]; // GitHub sends us an Interval (in s) that we need to wait between polling for login status, otherwise we get timeouted - const interval = resOne.match(/interval=(\d*)&/)[1]; + const interval = Number(resOne.interval || 5); this.toggleLoadingAnimation(); let confirm; window.open('https://github.com/login/device', 'Github Authentification', 'width=500,height=600,top=100,left=100'); @@ -187,8 +204,10 @@ export class UserFlapModel extends ViewModel { }).then(conf => { confirm = conf; }); - cmdString = `curl -X POST -F 'client_id=${livelyAuthGithubAppId}' -F 'device_code=${deviceCode}' -F 'grant_type=urn:ietf:params:oauth:grant-type:device_code' https://github.com/login/oauth/access_token`; + cmdString = `curl -sS -X POST -H 'Accept: application/json' -F 'client_id=${livelyAuthGithubAppId}' -F 'device_code=${deviceCode}' -F 'grant_type=urn:ietf:params:oauth:grant-type:device_code' https://github.com/login/oauth/access_token`; let curlCmd; + let tokenResponse; + let lastLoginError; let loginSuccessful = false; for (let i = 0; i < 20; i++) { let elapsedTimeWaitingForGitHub = await timeToRun(waitFor(interval * 1000, () => confirm !== undefined, false)); @@ -204,24 +223,33 @@ export class UserFlapModel extends ViewModel { return; } curlCmd = await runCommand(cmdString).whenDone(); - if (curlCmd.exitCode === 0 && !curlCmd.stdout.includes('error')) { + tokenResponse = parseGithubAuthResponse(curlCmd.stdout); + if (curlCmd.exitCode === 0 && tokenResponse.access_token) { loginSuccessful = true; $world.get('github login prompt')?.remove(); break; } + lastLoginError = githubAuthErrorMessage(tokenResponse) || curlCmd.stderr; + if (tokenResponse.error === 'authorization_pending') continue; + if (tokenResponse.error === 'slow_down') { + await delay(interval * 1000); + continue; + } + if (lastLoginError) break; } if (!loginSuccessful) { this.toggleLoadingAnimation(); - $world.setStatusMessage('Login failed.', StatusMessageError); + if (lastLoginError) console.error('[github-login] access token request failed:', lastLoginError); // eslint-disable-line no-console + $world.setStatusMessage(`Login failed${lastLoginError ? `: ${lastLoginError}` : '.'}`, StatusMessageError); return; } - const { stdout: resTwo } = curlCmd; - const userToken = resTwo.match(new RegExp('access_token=(.*)&s'))[1]; + const userToken = tokenResponse.access_token; if (!userToken) { this.toggleLoadingAnimation(); - $world.setStatusMessage('An unexpected error occured. Please contact the lively.next team.', StatusMessageError); + console.error('[github-login] access token missing in response.'); // eslint-disable-line no-console + $world.setStatusMessage('GitHub login failed: access token missing in response.', StatusMessageError); return; } this.blockLogoutAttempt = true; @@ -317,6 +345,7 @@ export class UserFlapModel extends ViewModel { leftUserLabel.textString = userData.login; rightUserLabel.textAndAttributes = Icon.textAttribute('right-from-bracket'); } catch (err) { + console.error('[github-login] failed to show user data:', err); // eslint-disable-line no-console $world.setStatusMessage('An unexpected error occured. Please contact the lively.next team.', StatusMessageError); } } diff --git a/start-server.sh b/start-server.sh index 024014b4e0..d53502ad67 100755 --- a/start-server.sh +++ b/start-server.sh @@ -11,13 +11,18 @@ lv_next_dir=$PWD . $lv_next_dir/scripts/lively-next-env.sh lively_next_env $lv_next_dir +config_file="$lv_next_dir/config.js" +if [ ! -f "$config_file" ]; then + config_file="$lv_next_dir/lively.installer/assets/config.js" +fi + cd lively.server; options="--no-warnings --dns-result-order ipv4first \ --experimental-loader $lv_next_dir/flatn/resolver.mjs \ bin/start-server.js \ --root-directory $lv_next_dir \ - --config $lv_next_dir/config.js" + --config $config_file" if [ "$1" = "--debug" ]; then options="--inspect $options"