diff --git a/.codex/hooks.json b/.codex/hooks.json new file mode 100644 index 00000000..779977d5 --- /dev/null +++ b/.codex/hooks.json @@ -0,0 +1,15 @@ +{ + "hooks": { + "PostToolUse": [ + { + "matcher": "Write|Edit|MultiEdit", + "hooks": [ + { + "type": "command", + "command": "sh \"$CLAUDE_PROJECT_DIR/scripts/check-conventions.sh\" --hook" + } + ] + } + ] + } +} diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..29556a17 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,391 @@ +# AGENTS.md + +Guidance for Codex (and human contributors) working in this repository. + +## What ArozOS is + +ArozOS is a self-hosted, web-based cloud desktop / NAS operating system written +in Go. It runs as a single binary on everything from a Raspberry Pi to a desktop +server. The Go module lives in [`src/`](src/) (module path `imuslab.com/arozos`); +the repository root holds docs, the installer and release tooling. + +License: **GPLv3** (see [`LICENSE`](LICENSE)). + +## What AGI is + +In this codebase **AGI** stands for **ArOZ Online JavaScript Gateway Interface** +— *not* "Artificial General Intelligence". It is the server-side JavaScript +runtime that powers ArozOS web apps: module scripts with a `.agi` (or `.js`) +extension are executed inside a sandboxed [Otto](https://github.com/robertkrimen/otto) +JavaScript VM, **one fresh VM per request/script**, with permission-checked +access to ArozOS functions (file system, database, sharing, IoT, image/zip/ +ffmpeg helpers, SQLite, WebSockets, an LLM/`aimodel` chat library, and more). + +- **Where it lives:** [`src/mod/agi/`](src/mod/agi/) (`agi*.go`); the runtime + version is the `AgiVersion` constant in [`src/mod/agi/agi.go`](src/mod/agi/agi.go). + The gateway is constructed in [`src/agi.go`](src/agi.go) (`AGIInit`, run from + [`src/startup.go`](src/startup.go)) via `agi.NewGateway(agi.AgiSysInfo{…})`. +- **Built-in globals & functions:** core globals (`USERNAME`, `USERICON`, + `HTTP_RESP`, `LOADED_MODULES`, …) and functions (`sendResp`, `sendJSONResp`, + `registerModule`, `requirelib`, `includes`, `execd`, the `…DBItem` DB helpers, + …) are injected per VM by `injectStandardLibs` / `injectUserFunctions` + ([`src/mod/agi/agi.user.go`](src/mod/agi/agi.user.go)). +- **Loadable libraries** — pulled in on demand with `requirelib("name")`; + registered in `LoadAllFunctionalModules` + ([`src/mod/agi/moduleManager.go`](src/mod/agi/moduleManager.go)): `filelib`, + `imagelib`, `http`, `share`, `iot`, `appdata`, `sysinfo`, `ziplib` (incl. 7z), + `sqlite`, `aimodel`, and `ffmpeg` (only when ffmpeg is on the host), plus + `websocket` and `scheduler` which are injected only in an HTTP request context. +- **Execution entry points:** + - **`init.agi`** — a web app's startup/registration script, scanned at boot + from `./web/*/init.agi` (`InitiateAllWebAppModules`) and run with **system + scope only** (no user functions); used to `registerModule(…)`. + - **Front-end calls** — `/system/ajgi/interface` (logged-in users) and + `/api/ajgi/interface` (token auth; this is the `-rpt` callback subservices + receive). Both run scripts scoped to the invoking user's permissions. + - **Serverless / external endpoints**, plus nightly tasks and user-approved + scheduled (cron) tasks. +- **Full API reference:** [`src/mod/agi/README.md`](src/mod/agi/README.md). When + you change AGI functions or signatures, also update the in-app help data file + [`src/web/Terminal/docs/api.json`](src/web/Terminal/docs/api.json) to match + (one object per library section; the README's maintainer note explains how). + +## What a WebApp is + +A **WebApp** (a.k.a. a *module*) is a user-facing application that shows up on +the ArozOS desktop. Each one is a folder under [`src/web/`](src/web/) — e.g. +`Photo/`, `Music/`, `NotepadA/`, `Calendar/`, `Code Studio/` — holding the +front-end assets (HTML/JS/CSS) plus an optional `init.agi` and any backend +`.agi` scripts it calls. + +- **Registration:** a web app announces itself from its `init.agi` by calling + `registerModule(JSON.stringify(moduleLaunchInfo))`. The launch-info object + maps field-for-field to the `ModuleInfo` struct in + [`src/mod/modules/module.go`](src/mod/modules/module.go): `Name`, `Desc`, + `Group`, `IconPath`, `Version`, `StartDir`, `SupportFW`/`LaunchFWDir`, + `SupportEmb`/`LaunchEmb`, `InitFWSize`, `InitEmbSize`, `SupportedExt`. See + [`src/web/Photo/init.agi`](src/web/Photo/init.agi) for a minimal example. +- **Launch modes:** full page (`StartDir`), **floatWindow** (`SupportFW` + + `LaunchFWDir`, sized by `InitFWSize`), and **embedded** (`SupportEmb` + + `LaunchEmb`, sized by `InitEmbSize`) — embedded is what loads when a file is + opened *with* the module. `SupportedExt` registers the file-type associations + that let the module become a default opener. +- **Go-side handling:** `ModuleHandler` + ([`src/mod/modules/module.go`](src/mod/modules/module.go)) holds the + loaded-module list; `RegisterModuleFromAGI` is the hook `init.agi` drives, and + module visibility is filtered per user by `GetModuleListJSONForUser` (users + only see modules they have permission for). +- **Scope reminder:** `init.agi` runs with **system scope** (registration / + system functions only — don't call user or file functions there); backend + `.agi` scripts invoked from the front end run with the **invoking user's** + scope. Subservices (below) also register through `ModuleInfo`, so they appear + on the desktop just like WebApps. + +## What a SubService is + +A **SubService** lets ArozOS launch an **independent binary web server** (written +in any language) as a child process and **reverse-proxy it under the main +server**, so it appears as a normal ArozOS module. Reach for it when a feature +needs a real native binary rather than a sandboxed AGI script — heavy compute, +an existing Go/Rust/… server, or third-party software such as Syncthing. + +- **Where it lives:** package [`src/mod/subservice/`](src/mod/subservice/) + (`SubService`, `SubServiceRouter`); wiring + boot-time scan in + [`src/subservice.go`](src/subservice.go) (`SubserviceInit`, run from + [`src/startup.go`](src/startup.go)). Disable it all with the + `-disable_subservice` flag; reverse-proxy ports start at `subserviceBasePort` + (`12810`, [`src/flags.go`](src/flags.go)). +- **Folder convention:** drop the binary in `./subservice//`, named for + its platform — `.exe` (Windows) or `__` (e.g. + `demo_linux_amd64`); on Linux an apt-installed binary on `PATH` is preferred. +- **Lifecycle:** ArozOS probes the binary with ` -info` (it must print its + `ModuleInfo` as JSON — or ship a `moduleInfo.json` instead), then launches it + as ` -port -rpt http://localhost:/api/ajgi/interface` + (the `-rpt` URL is the AGI callback so the service can call ArozOS APIs back). + The reverse-proxy URL prefix is the directory part of `StartDir` and must not + collide with a reserved path (`web`, `system`, `ws`, …, + [`src/subservice.go`](src/subservice.go)). A failed proxy is auto-restarted. +- **Auth:** proxied requests are permission-checked (per-module access) and get + `aouser`, `aotoken` and `X-Forwarded-Host` headers injected; routing happens + in the authenticated branch of [`src/main.router.go`](src/main.router.go). +- **Marker files** (next to the binary): `.disabled` (skip at boot; toggle in + the admin UI), `.noproxy` (run but don't proxy — compatibility mode), + `.startscript` (launch `start.sh`/`start.bat` instead of the binary), + `.intport` (pass the port without a leading `:`), `moduleInfo.json` (static + info, skips the `-info` probe). +- **Admin endpoints:** `/system/subservice/{list,kill,start}` (admin only). + Full guide: the "Subservice Logics and Configuration" section of + [`src/README.md`](src/README.md). + +## What subservices are + +A **subservice** is a *separate* program — usually a small Go web server, but it +can be any binary — that ArozOS launches as a child process and stitches into the +desktop through an authenticated reverse proxy. Subservices are how you extend +ArozOS in the language/runtime of your choice, or wrap an existing third-party web +app (e.g. Syncthing), *without* touching the core binary. Contrast with AGI, which +runs JavaScript *inside* the core: a subservice runs *outside* it as its own OS +process and only talks back through the gateway. + +A complete, buildable example is the "demo" service at +[aroz-online/ArozOS-Subservice-Example](https://github.com/aroz-online/ArozOS-Subservice-Example). +The canonical reference is the **"Subservice Logics and Configuration"** section of +[`src/README.md`](src/README.md). + +Key points: + +- **Where it lives in code:** the launcher, reverse proxy and lifecycle logic are + in [`src/mod/subservice/`](src/mod/subservice/); the wiring (scan directory, + admin endpoints, graceful shutdown) is in + [`src/subservice.go`](src/subservice.go). Disable the whole subsystem with the + `-disable_subservice` flag. +- **Where services live on disk:** one folder per service under + `./subservice//` at the ArozOS root. The executable must be named after + the folder with a platform suffix — `__` (e.g. + `demo_linux_amd64`) or `.exe` on Windows. (On Linux, a system-installed + binary found via `which ` is used if present.) +- **Startup handshake:** the core first reads the module's metadata — from a + `moduleInfo.json` in the folder, or by running ` -info` and parsing the + JSON it prints — then relaunches the binary as a long-running web server with + `-port :` (the next free port from base `12810`) and + `-rpt "http://localhost:/api/ajgi/interface"` (the AGI gateway the + subservice calls back into for filesystem/user access). +- **Routing & desktop integration:** the reverse-proxy endpoint is the *directory* + of `StartDir`, so `StartDir: "demo/home.html"` proxies `/demo/*` to the service. + The metadata is registered as a normal module, so the service appears on the + desktop like a built-in app, gated by per-module permission. The endpoint must + not collide with reserved paths (`web`, `system`, `SystemAO`, `img`, `ws`, …). + If the proxied process stops responding, the core kills and restarts it. +- **Control files** (empty marker files dropped in the service folder): + `.disabled` (skip at boot — an admin can re-enable it in System Settings), + `.noproxy` (compatibility mode: just run the binary, no port/proxy injection), + `.startscript` (run `start.sh`/`start.bat` instead of the binary, e.g. to wrap + Syncthing), `.intport` (pass the port as `12810` instead of `:12810`). +- **Admin control at runtime:** the endpoints + `/system/subservice/{list,kill,start}` and the UI in + [`src/web/SystemAO/modules/subservices.html`](src/web/SystemAO/modules/subservices.html) + let an admin start and stop services without restarting ArozOS. + +Minimal example — a `./subservice/demo/` folder with a binary and its metadata: + +``` +subservice/demo/ +├── demo_linux_amd64 # binary, named __ +├── demo.exe # a Windows build (optional, one per target) +└── moduleInfo.json # metadata — OR print the same JSON on `-info` +``` + +```json +{ + "Name": "Demo Subservice", + "Desc": "A simple subservice showing how subservices work in ArozOS", + "Group": "Development", + "IconPath": "demo/icon.png", + "Version": "0.0.1", + "StartDir": "demo/home.html", + "SupportFW": true, + "LaunchFWDir": "demo/home.html", + "SupportEmb": true, + "LaunchEmb": "demo/embedded.html", + "InitFWSize": [720, 480], + "InitEmbSize": [720, 480], + "SupportedExt": [".txt", ".md"] +} +``` + +```go +// The binary answers -info (and exits), then serves its web UI on -port. +func main() { + info := flag.Bool("info", false, "Print module info as JSON and exit") + port := flag.String("port", ":8000", "Listen address assigned by ArozOS") + flag.String("rpt", "", "ArozOS AGI gateway endpoint for callbacks") + flag.Parse() + + if *info { + // Same JSON as moduleInfo.json above; StartDir's dir ("demo") is the proxy endpoint. + fmt.Println(`{"Name":"Demo Subservice","Group":"Development","StartDir":"demo/home.html","Version":"0.0.1"}`) + return + } + + http.Handle("/demo/", http.StripPrefix("/demo/", http.FileServer(http.Dir("./web")))) + http.ListenAndServe(*port, nil) // ArozOS reverse-proxies /demo/* here +} +``` + +### Webapp vs. subservice + +Both a **webapp** and a **subservice** register the same `ModuleInfo` and, once +loaded, look identical on the desktop. The difference is *what runs the code* and +*where it lives*: + +| | Webapp | Subservice | +|---|---|---| +| **What it is** | Static front-end (HTML/CSS/JS) plus optional server-side AGI scripts | A standalone compiled binary (any language) | +| **Lives in** | `src/web//`, served by the core's static file server | `./subservice//`, run as its own executable | +| **Process model** | No process of its own — backend logic runs as JavaScript *inside* the core's Otto VM (one fresh VM per request) | Its own OS process on its own port, reached through a reverse proxy | +| **How it registers** | An `init.agi` startup script calls `registerModule(...)` from inside the VM | The core reads `-info` / `moduleInfo.json` when it launches the binary | +| **Talks to the host via** | AGI globals/libraries in-VM (`requirelib("filelib")`, …) | HTTP calls back to the `-rpt` AGI gateway endpoint | +| **Reach for it when** | A standard ArozOS app whose logic fits the AGI/JS sandbox | You need native code, heavy/long-running work, a non-Go runtime, or to wrap an existing third-party server | + +In short: a **webapp** is front-end assets + JavaScript executed *inside* ArozOS +through AGI, while a **subservice** is an *external* program ArozOS launches, +supervises and reverse-proxies. Use a webapp by default; reach for a subservice +when the work doesn't fit the in-core JavaScript sandbox. + +## Build, run and test + +All Go commands run from `src/`: + +```bash +cd src +go mod tidy +go build # produces ./arozos +./arozos -port 8080 # run (sudo only needed for hardware/WiFi features) + +go test ./... # run the test suite +go vet ./... # static checks +gofmt -l . # list unformatted files (should be empty) +make binary # cross-compile every supported OS/arch (see Makefile) +``` + +## Mandatory contribution rules + +These five rules are enforced on **new and changed code** by a Codex +`PostToolUse` hook (during editing) and by CI (on every pull request). Both call +[`scripts/check-conventions.sh`](scripts/check-conventions.sh). Existing legacy +code is grandfathered — CI only inspects the lines a change adds — but do not add +new violations, and prefer fixing nearby ones when you touch them. + +### 1. Use the managed logger, never the standard `log` package + +New code must send log output through the system logger so it lands in the +managed, rotated system log instead of bare stdout. + +```go +import "imuslab.com/arozos/mod/info/logger" + +// Good — title, message, and the originating error (nil if none): +logger.PrintAndLog("ModuleName", "could not open config", err) + +// Bad — bypasses the system log: +log.Println("could not open config", err) // and log.Printf/Fatal/Panic +``` + +The package-level `logger.PrintAndLog` delegates to the system-wide logger wired +up in [`src/main.go`](src/main.go); you do not need your own `*logger.Logger` +instance. The only file allowed to wrap the standard `log` package is the logger +implementation itself ([`src/mod/info/logger/`](src/mod/info/logger/)). + +*Enforced as an ERROR* (blocks CI) on added `log.Print*` / `log.Fatal*` / +`log.Panic*` calls. + +### 2. New functions ship with tests + +Every package under `src/mod/` is expected to carry a `*_test.go` file, and new +functions must come with table-driven Go tests (see +[`src/mod/info/logger/logger_test.go`](src/mod/info/logger/logger_test.go) for +the house style: `t.TempDir()`, `t.Fatalf`/`t.Errorf`, one `Test…` per behaviour). + +```bash +cd src && go test ./mod/yourpackage/ # must pass before you push +``` + +*Enforced* by the CI `go test ./...` gate; the convention checker additionally +*warns* when a touched `mod/` package has no test file at all. + +### 3. Dependencies must be MIT / commercial-use-OK + +Any module added to [`src/go.mod`](src/go.mod) must be licensed **MIT, BSD-2/3, +Apache-2.0, MPL-2.0, or ISC** — permissive, GPL-compatible, and fine for +commercial redistribution. **Do not add GPL/AGPL/LGPL, source-available +(BSL/SSPL), or unknown-licensed modules.** When unsure, state the dependency's +license in your summary so it can be reviewed before merge. + +```bash +go install github.com/google/go-licenses@latest # optional audit helper +go-licenses report imuslab.com/arozos +``` + +*Enforced* by a CI reminder whenever `go.mod`/`go.sum` changes — verify the +license of each new dependency before merging. + +### 4. New endpoints get the right security control + +Register authenticated endpoints through the permission router, not raw +`http.HandleFunc`, so they inherit login, per-module permission and (optionally) +admin/LAN/CSRF checks: + +```go +import prout "imuslab.com/arozos/mod/prouter" + +router := prout.NewModuleRouter(prout.RouterOption{ + ModuleName: "System Setting", + AdminOnly: true, // gate admin-only actions + UserHandler: userHandler, + DeniedHandler: func(w http.ResponseWriter, r *http.Request) { + utils.SendErrorResponse(w, "Permission Denied") + }, +}) +router.HandleFunc("/system/yourmodule/action", yourHandler) +``` + +Use raw `http.HandleFunc` **only** for deliberately public endpoints (e.g. the +`/public/...` registration pages) and treat all request input as untrusted — +validate parameters with `mod/utils` helpers and never interpolate user input +into shell commands or file paths. See +[`src/main.router.go`](src/main.router.go) and +[`src/register.go`](src/register.go) for the patterns. + +*Enforced* by a CI/hook *warning* on every added raw `http.HandleFunc`, prompting +a deliberate "authenticated vs. intentionally public" decision. + +### 5. Stay portable — no system dependencies, cross-platform safe + +ArozOS ships as one self-contained binary that must build and run across the +targets in the [`Makefile`](src/Makefile) (Linux amd64/386/arm/arm64/mipsle/ +riscv64, macOS, Windows). Therefore: + +- **No hardcoded OS paths.** Build paths with `filepath.Join`, and resolve + locations via `os.TempDir()`, `os.UserHomeDir()`, or paths relative to the + binary — never literal `"/usr/..."`, `"/etc/..."` or `"C:\\..."`. +- **No shelling out to platform tools** in shared code. Avoid making features + depend on external binaries. When a platform-specific call (`exec.Command`, + `syscall`) is unavoidable, isolate it in a build-tagged file — `foo_linux.go`, + `foo_windows.go`, `foo_darwin.go`, or behind a `//go:build` constraint — and + provide a fallback for other platforms. See + [`src/mod/network/wifi/`](src/mod/network/wifi/) for the pattern. +- Cross-compile to sanity-check: `cd src && GOOS=windows GOARCH=amd64 go build ./...`. + +*Enforced as an ERROR* on added hardcoded OS path literals, and as a *warning* +when `exec.Command`/`syscall` appears in a non-build-tagged file. + +## How enforcement works + +| Mechanism | When it runs | What it does | +|-----------|--------------|--------------| +| `PostToolUse` hook ([`.Codex/settings.json`](.Codex/settings.json)) | After Codex edits a Go file | Runs the checker on that file and feeds any finding back so Codex self-corrects | +| GitHub Actions ([`.github/workflows/ci.yml`](.github/workflows/ci.yml)) | On every push / PR | `gofmt`, `go build`, `go test ./...`, and the diff-scoped convention checker (blocking); plus module-wide `go vet` (advisory — never fails CI on grandfathered legacy code) | +| [`scripts/check-conventions.sh`](scripts/check-conventions.sh) | Manually or from the above | Single source of truth for the rules above | + +Run it yourself before pushing: + +```bash +sh scripts/check-conventions.sh src/path/to/file.go # check specific files +sh scripts/check-conventions.sh --diff origin/master # check everything you changed +``` + +**Escape hatch:** in the rare, justified case where a line must keep a raw +`log`/path literal, append the marker `arozos-lint-ignore` to that line with a +short comment explaining why. Use it sparingly — it is reviewed. + +## Repository layout cheatsheet + +- [`src/`](src/) — Go module root; `main*.go` boot the server, `*.go` are feature handlers. +- [`src/mod/`](src/mod/) — self-contained library packages (each with its own tests). +- [`src/mod/info/logger/`](src/mod/info/logger/) — the system logger (rule 1). +- [`src/mod/prouter/`](src/mod/prouter/) — permission/auth router (rule 4). +- [`src/mod/agi/`](src/mod/agi/) — the AGI JavaScript gateway runtime (see "What AGI is"); API reference in [`src/mod/agi/README.md`](src/mod/agi/README.md). +- [`src/mod/modules/`](src/mod/modules/) — module registry and the `ModuleInfo` struct shared by WebApps and SubServices (see "What a WebApp is"). +- [`src/mod/subservice/`](src/mod/subservice/) — reverse-proxied binary subservices (see "What a SubService is"); wired up in [`src/subservice.go`](src/subservice.go). +- [`src/web/`](src/web/) — front-end assets and WebApps (one folder per module; see "What a WebApp is"). +- [`src/system/`](src/system/) — runtime data and config (not shipped in release). diff --git a/src/file_system.go b/src/file_system.go index bfdcfe31..a12c47fd 100644 --- a/src/file_system.go +++ b/src/file_system.go @@ -47,6 +47,7 @@ import ( var ( thumbRenderHandler *metadata.RenderHandler shareEntryTable *shareEntry.ShareEntryTable + uploadLinkTable *shareEntry.UploadLinkTable shareManager *share.Manager wsConnectionStore sync.Map ) @@ -189,12 +190,15 @@ func FileSystemInit() { */ //Create a share manager to handle user file sharae shareEntryTable = shareEntry.NewShareEntryTable(sysdb) + uploadLinkTable = shareEntry.NewUploadLinkTable(sysdb) shareManager = share.NewShareManager(share.Options{ AuthAgent: authAgent, ShareEntryTable: shareEntryTable, + UploadLinkTable: uploadLinkTable, UserHandler: userHandler, HostName: *host_name, TmpFolder: *tmp_directory, + MaxUploadSize: max_upload_size, }) //Share related functions @@ -203,6 +207,10 @@ func FileSystemInit() { router.HandleFunc("/system/file_system/share/edit", shareManager.HandleEditShare) router.HandleFunc("/system/file_system/share/checkShared", shareManager.HandleShareCheck) router.HandleFunc("/system/file_system/share/list", shareManager.HandleListAllShares) + router.HandleFunc("/system/file_system/share/upload/new", shareManager.HandleCreateUploadLink) + router.HandleFunc("/system/file_system/share/upload/edit", shareManager.HandleEditUploadLink) + router.HandleFunc("/system/file_system/share/upload/delete", shareManager.HandleDeleteUploadLink) + router.HandleFunc("/system/file_system/share/upload/list", shareManager.HandleListUploadLinks) //Handle the main share function //Share function is now routed by the main router diff --git a/src/mod/share/share.go b/src/mod/share/share.go index 48ec8497..d24eff06 100644 --- a/src/mod/share/share.go +++ b/src/mod/share/share.go @@ -48,8 +48,10 @@ type Options struct { AuthAgent *auth.AuthAgent UserHandler *user.UserHandler ShareEntryTable *shareEntry.ShareEntryTable + UploadLinkTable *shareEntry.UploadLinkTable HostName string TmpFolder string + MaxUploadSize int64 } // ZipJob tracks the state of an async zip operation @@ -66,15 +68,18 @@ type ZipJob struct { } type Manager struct { - options Options - zipJobs sync.Map // map[string]*ZipJob + options Options + zipJobs sync.Map // map[string]*ZipJob + uploadNameMu sync.Mutex + uploadReservedNames map[string]bool } // Create a new Share Manager func NewShareManager(options Options) *Manager { //Return a new manager object return &Manager{ - options: options, + options: options, + uploadReservedNames: map[string]bool{}, } } @@ -282,6 +287,10 @@ func (s *Manager) HandleShareAccess(w http.ResponseWriter, r *http.Request) { { cleanParts := strings.Split(strings.TrimPrefix(filepath.ToSlash(filepath.Clean(r.URL.Path)), "/"), "/") if len(cleanParts) >= 3 { + if cleanParts[1] == "upload" { + s.HandleUploadLinkAccess(w, r, cleanParts) + return + } switch cleanParts[1] { case "zip-status": s.handleZipStatus(w, r, cleanParts[2]) @@ -291,6 +300,10 @@ func (s *Manager) HandleShareAccess(w http.ResponseWriter, r *http.Request) { return } } + if len(cleanParts) >= 2 && cleanParts[1] == "upload" { + s.HandleUploadLinkAccess(w, r, cleanParts) + return + } } //New download method variables diff --git a/src/mod/share/shareEntry/shareEntry_test.go b/src/mod/share/shareEntry/shareEntry_test.go index 0b977453..0eff9dc3 100644 --- a/src/mod/share/shareEntry/shareEntry_test.go +++ b/src/mod/share/shareEntry/shareEntry_test.go @@ -18,6 +18,7 @@ func openTempDB(t *testing.T) *db.Database { if err != nil { t.Fatalf("failed to open test database: %v", err) } + t.Cleanup(database.Close) return database } @@ -324,6 +325,7 @@ func TestNewShareEntryTable_LoadsExistingEntries(t *testing.T) { if err != nil { t.Fatalf("failed to open database: %v", err) } + t.Cleanup(database.Close) // Write a share entry into the DB before creating the table database.NewTable("share") diff --git a/src/mod/share/shareEntry/uploadLink.go b/src/mod/share/shareEntry/uploadLink.go new file mode 100644 index 00000000..85a9b1d3 --- /dev/null +++ b/src/mod/share/shareEntry/uploadLink.go @@ -0,0 +1,273 @@ +package shareEntry + +import ( + "encoding/json" + "errors" + "path/filepath" + "sync" + + uuid "github.com/satori/go.uuid" + "imuslab.com/arozos/mod/database" + "imuslab.com/arozos/mod/filesystem" +) + +const UploadLinkTableName = "share-upload" + +type UploadLinkTable struct { + UrlToUploadMap *sync.Map + Database *database.Database + + mu sync.Mutex + pendingFileCounts map[string]int64 + pendingBytes map[string]int64 + pendingOwnerBytes map[string]int64 +} + +type UploadLinkOption struct { + UUID string + PathHash string + TargetVirtualPath string + TargetRealPath string + Owner string + CreatedUnix int64 + ExpiresUnix int64 + MaxFileCount int64 + MaxFileSize int64 + MaxTotalSize int64 + UploadedFileCount int64 + UploadedBytes int64 + Disabled bool +} + +func NewUploadLinkTable(db *database.Database) *UploadLinkTable { + db.NewTable(UploadLinkTableName) + + UrlToUploadMap := sync.Map{} + entries, _ := db.ListTable(UploadLinkTableName) + for _, keypairs := range entries { + uploadObject := new(UploadLinkOption) + json.Unmarshal(keypairs[1], &uploadObject) + if uploadObject != nil && uploadObject.UUID != "" { + UrlToUploadMap.Store(uploadObject.UUID, uploadObject) + } + } + + return &UploadLinkTable{ + UrlToUploadMap: &UrlToUploadMap, + Database: db, + pendingFileCounts: map[string]int64{}, + pendingBytes: map[string]int64{}, + pendingOwnerBytes: map[string]int64{}, + } +} + +func (s *UploadLinkTable) CreateNewUploadLink(srcFsh *filesystem.FileSystemHandler, vpath string, username string, createdUnix int64, expiresUnix int64, maxFileCount int64, maxFileSize int64, maxTotalSize int64) (*UploadLinkOption, error) { + rpath, err := srcFsh.FileSystemAbstraction.VirtualPathToRealPath(vpath, username) + if err != nil { + return nil, errors.New("Unable to translate path given") + } + + rpath = filepath.ToSlash(filepath.Clean(rpath)) + if !srcFsh.FileSystemAbstraction.FileExists(rpath) { + return nil, errors.New("Unable to find the folder on disk") + } + if !srcFsh.FileSystemAbstraction.IsDir(rpath) { + return nil, errors.New("Upload link target must be a folder") + } + + sharePathHash, err := GetPathHash(srcFsh, vpath, username) + if err != nil { + return nil, err + } + + uploadUUID := uuid.NewV4().String() + uploadOption := UploadLinkOption{ + UUID: uploadUUID, + PathHash: sharePathHash, + TargetVirtualPath: vpath, + TargetRealPath: rpath, + Owner: username, + CreatedUnix: createdUnix, + ExpiresUnix: expiresUnix, + MaxFileCount: maxFileCount, + MaxFileSize: maxFileSize, + MaxTotalSize: maxTotalSize, + UploadedFileCount: 0, + UploadedBytes: 0, + Disabled: false, + } + + s.UrlToUploadMap.Store(uploadUUID, &uploadOption) + err = s.Database.Write(UploadLinkTableName, uploadUUID, uploadOption) + if err != nil { + s.UrlToUploadMap.Delete(uploadUUID) + return nil, err + } + + return &uploadOption, nil +} + +func (s *UploadLinkTable) GetUploadLinkFromUUID(uuid string) *UploadLinkOption { + if val, ok := s.UrlToUploadMap.Load(uuid); ok { + return val.(*UploadLinkOption) + } + return nil +} + +func (s *UploadLinkTable) ListUploadLinksByPathHash(pathHash string) []*UploadLinkOption { + results := []*UploadLinkOption{} + s.UrlToUploadMap.Range(func(_, v interface{}) bool { + thisUploadOption := v.(*UploadLinkOption) + if thisUploadOption.PathHash == pathHash { + results = append(results, thisUploadOption) + } + return true + }) + return results +} + +func (s *UploadLinkTable) ListUploadLinksByOwner(owner string) []*UploadLinkOption { + results := []*UploadLinkOption{} + s.UrlToUploadMap.Range(func(_, v interface{}) bool { + thisUploadOption := v.(*UploadLinkOption) + if thisUploadOption.Owner == owner { + results = append(results, thisUploadOption) + } + return true + }) + return results +} + +func (s *UploadLinkTable) DeleteUploadLinkByUUID(uuid string) error { + val, ok := s.UrlToUploadMap.Load(uuid) + if !ok { + return errors.New("Upload link with given uuid not exists") + } + link := val.(*UploadLinkOption) + s.UrlToUploadMap.Delete(uuid) + s.mu.Lock() + if pendingBytes := s.pendingBytes[uuid]; pendingBytes > 0 { + if s.pendingOwnerBytes[link.Owner] <= pendingBytes { + delete(s.pendingOwnerBytes, link.Owner) + } else { + s.pendingOwnerBytes[link.Owner] -= pendingBytes + } + } + delete(s.pendingFileCounts, uuid) + delete(s.pendingBytes, uuid) + s.mu.Unlock() + return s.Database.Delete(UploadLinkTableName, uuid) +} + +func (s *UploadLinkTable) UpdateUploadLink(updated *UploadLinkOption) error { + if updated == nil || updated.UUID == "" { + return errors.New("Invalid upload link") + } + if updated.MaxFileCount > 0 && updated.MaxFileCount < updated.UploadedFileCount { + return errors.New("Max file count is below current uploaded file count") + } + if updated.MaxTotalSize > 0 && updated.MaxTotalSize < updated.UploadedBytes { + return errors.New("Max total size is below current uploaded size") + } + + s.UrlToUploadMap.Store(updated.UUID, updated) + return s.Database.Write(UploadLinkTableName, updated.UUID, updated) +} + +func (s *UploadLinkOption) IsActive(nowUnix int64) bool { + if s == nil || s.Disabled { + return false + } + if s.ExpiresUnix > 0 && nowUnix > s.ExpiresUnix { + return false + } + if s.MaxFileCount > 0 && s.UploadedFileCount >= s.MaxFileCount { + return false + } + if s.MaxTotalSize > 0 && s.UploadedBytes >= s.MaxTotalSize { + return false + } + return true +} + +func (s *UploadLinkTable) ReserveUpload(uuid string, size int64, nowUnix int64, maxUploadSize int64, ownerRemainingQuota int64) error { + if size <= 0 { + return errors.New("Invalid upload size") + } + + s.mu.Lock() + defer s.mu.Unlock() + + link := s.GetUploadLinkFromUUID(uuid) + if link == nil { + return errors.New("Upload link not exists") + } + if !link.IsActive(nowUnix) { + return errors.New("Upload link expired or disabled") + } + if maxUploadSize > 0 && size > maxUploadSize { + return errors.New("File size too large") + } + if link.MaxFileSize > 0 && size > link.MaxFileSize { + return errors.New("File size exceeds upload link limit") + } + if link.MaxFileCount > 0 && link.UploadedFileCount+s.pendingFileCounts[uuid]+1 > link.MaxFileCount { + return errors.New("Upload link file count limit reached") + } + if link.MaxTotalSize > 0 && link.UploadedBytes+s.pendingBytes[uuid]+size > link.MaxTotalSize { + return errors.New("Upload link total size limit reached") + } + if ownerRemainingQuota >= 0 && s.pendingOwnerBytes[link.Owner]+size > ownerRemainingQuota { + return errors.New("User Storage Quota Exceeded") + } + + s.pendingFileCounts[uuid]++ + s.pendingBytes[uuid] += size + s.pendingOwnerBytes[link.Owner] += size + return nil +} + +func (s *UploadLinkTable) CommitUpload(uuid string, size int64) error { + s.mu.Lock() + defer s.mu.Unlock() + + link := s.GetUploadLinkFromUUID(uuid) + if link == nil { + return errors.New("Upload link not exists") + } + + s.releaseUploadLocked(link.Owner, uuid, size) + link.UploadedFileCount++ + link.UploadedBytes += size + return s.Database.Write(UploadLinkTableName, uuid, link) +} + +func (s *UploadLinkTable) ReleaseUpload(uuid string, size int64) { + s.mu.Lock() + defer s.mu.Unlock() + + link := s.GetUploadLinkFromUUID(uuid) + if link == nil { + return + } + s.releaseUploadLocked(link.Owner, uuid, size) +} + +func (s *UploadLinkTable) releaseUploadLocked(owner string, uuid string, size int64) { + if s.pendingFileCounts[uuid] > 0 { + s.pendingFileCounts[uuid]-- + } + if s.pendingBytes[uuid] <= size { + delete(s.pendingBytes, uuid) + } else { + s.pendingBytes[uuid] -= size + } + if s.pendingOwnerBytes[owner] <= size { + delete(s.pendingOwnerBytes, owner) + } else { + s.pendingOwnerBytes[owner] -= size + } + if s.pendingFileCounts[uuid] == 0 { + delete(s.pendingFileCounts, uuid) + } +} diff --git a/src/mod/share/shareEntry/uploadLink_test.go b/src/mod/share/shareEntry/uploadLink_test.go new file mode 100644 index 00000000..eee3fa32 --- /dev/null +++ b/src/mod/share/shareEntry/uploadLink_test.go @@ -0,0 +1,280 @@ +package shareEntry + +import ( + "os" + "path/filepath" + "sync" + "testing" + "time" +) + +func newTestUploadLinkTable(t *testing.T) *UploadLinkTable { + t.Helper() + return NewUploadLinkTable(openTempDB(t)) +} + +func createUploadTarget(t *testing.T, owner string) (*UploadLinkTable, string) { + t.Helper() + fsh, root := newTestFSH(t) + targetDir := filepath.Join(root, "users", owner, "uploads") + if err := os.MkdirAll(targetDir, 0755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + + table := newTestUploadLinkTable(t) + now := time.Now().Unix() + link, err := table.CreateNewUploadLink(fsh, "testfsh:/uploads", owner, now, now+3600, 10, 1024, 4096) + if err != nil { + t.Fatalf("CreateNewUploadLink: %v", err) + } + return table, link.PathHash +} + +func TestNewUploadLinkTableLoadsExistingEntries(t *testing.T) { + database := openTempDB(t) + firstTable := NewUploadLinkTable(database) + fsh, root := newTestFSH(t) + userDir := filepath.Join(root, "users", "alice", "uploads") + if err := os.MkdirAll(userDir, 0755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + + now := time.Now().Unix() + link, err := firstTable.CreateNewUploadLink(fsh, "testfsh:/uploads", "alice", now, now+3600, 5, 100, 500) + if err != nil { + t.Fatalf("CreateNewUploadLink: %v", err) + } + + reloaded := NewUploadLinkTable(database) + got := reloaded.GetUploadLinkFromUUID(link.UUID) + if got == nil { + t.Fatal("expected persisted upload link to be loaded") + } + if got.TargetVirtualPath != "testfsh:/uploads" || got.Owner != "alice" { + t.Errorf("unexpected loaded link: %+v", got) + } +} + +func TestListUploadLinksByPathHashAllowsMultipleLinks(t *testing.T) { + fsh, root := newTestFSH(t) + userDir := filepath.Join(root, "users", "alice", "uploads") + if err := os.MkdirAll(userDir, 0755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + + table := newTestUploadLinkTable(t) + now := time.Now().Unix() + first, err := table.CreateNewUploadLink(fsh, "testfsh:/uploads", "alice", now, now+3600, 5, 100, 500) + if err != nil { + t.Fatalf("first CreateNewUploadLink: %v", err) + } + second, err := table.CreateNewUploadLink(fsh, "testfsh:/uploads", "alice", now, now+7200, 10, 200, 1000) + if err != nil { + t.Fatalf("second CreateNewUploadLink: %v", err) + } + + links := table.ListUploadLinksByPathHash(first.PathHash) + if len(links) != 2 { + t.Fatalf("expected two links for same path hash, got %d", len(links)) + } + seen := map[string]bool{} + for _, link := range links { + seen[link.UUID] = true + } + if !seen[first.UUID] || !seen[second.UUID] { + t.Errorf("missing one of the expected links: %+v", seen) + } +} + +func TestUploadLinkOptionIsActive(t *testing.T) { + now := time.Now().Unix() + tests := []struct { + name string + link UploadLinkOption + want bool + }{ + { + name: "active", + link: UploadLinkOption{ExpiresUnix: now + 10, MaxFileCount: 2, MaxTotalSize: 100, UploadedFileCount: 1, UploadedBytes: 20}, + want: true, + }, + { + name: "disabled", + link: UploadLinkOption{ExpiresUnix: now + 10, Disabled: true}, + want: false, + }, + { + name: "expired", + link: UploadLinkOption{ExpiresUnix: now - 1}, + want: false, + }, + { + name: "file count exhausted", + link: UploadLinkOption{ExpiresUnix: now + 10, MaxFileCount: 2, UploadedFileCount: 2}, + want: false, + }, + { + name: "total size exhausted", + link: UploadLinkOption{ExpiresUnix: now + 10, MaxTotalSize: 100, UploadedBytes: 100}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.link.IsActive(now); got != tt.want { + t.Errorf("IsActive() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestUpdateAndDeleteUploadLink(t *testing.T) { + table, pathHash := createUploadTarget(t, "alice") + links := table.ListUploadLinksByPathHash(pathHash) + if len(links) != 1 { + t.Fatalf("expected one test link, got %d", len(links)) + } + link := *links[0] + link.Disabled = true + + if err := table.UpdateUploadLink(&link); err != nil { + t.Fatalf("UpdateUploadLink: %v", err) + } + if table.GetUploadLinkFromUUID(link.UUID).IsActive(time.Now().Unix()) { + t.Error("disabled upload link should not be active") + } + + if err := table.DeleteUploadLinkByUUID(link.UUID); err != nil { + t.Fatalf("DeleteUploadLinkByUUID: %v", err) + } + if got := table.GetUploadLinkFromUUID(link.UUID); got != nil { + t.Errorf("expected deleted link lookup to return nil, got %+v", got) + } +} + +func TestReserveUploadLimits(t *testing.T) { + now := time.Now().Unix() + tests := []struct { + name string + link UploadLinkOption + size int64 + maxUploadSize int64 + ownerRemainingQuota int64 + wantErr bool + }{ + { + name: "accepted", + link: UploadLinkOption{UUID: "ok", Owner: "alice", ExpiresUnix: now + 10, MaxFileCount: 2, MaxFileSize: 50, MaxTotalSize: 100}, + size: 40, maxUploadSize: 50, ownerRemainingQuota: 100, + }, + { + name: "global size exceeded", + link: UploadLinkOption{UUID: "global", Owner: "alice", ExpiresUnix: now + 10, MaxFileCount: 2, MaxFileSize: 100, MaxTotalSize: 100}, + size: 60, maxUploadSize: 50, ownerRemainingQuota: 100, wantErr: true, + }, + { + name: "link file size exceeded", + link: UploadLinkOption{UUID: "file", Owner: "alice", ExpiresUnix: now + 10, MaxFileCount: 2, MaxFileSize: 50, MaxTotalSize: 100}, + size: 60, maxUploadSize: 100, ownerRemainingQuota: 100, wantErr: true, + }, + { + name: "owner quota exceeded", + link: UploadLinkOption{UUID: "quota", Owner: "alice", ExpiresUnix: now + 10, MaxFileCount: 2, MaxFileSize: 100, MaxTotalSize: 100}, + size: 60, maxUploadSize: 100, ownerRemainingQuota: 50, wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + table := newTestUploadLinkTable(t) + link := tt.link + table.UrlToUploadMap.Store(link.UUID, &link) + err := table.ReserveUpload(link.UUID, tt.size, now, tt.maxUploadSize, tt.ownerRemainingQuota) + if (err != nil) != tt.wantErr { + t.Fatalf("ReserveUpload() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestConcurrentReserveUploadAccounting(t *testing.T) { + table := newTestUploadLinkTable(t) + now := time.Now().Unix() + link := &UploadLinkOption{ + UUID: "concurrent", + Owner: "alice", + ExpiresUnix: now + 10, + MaxFileCount: 2, + MaxFileSize: 50, + MaxTotalSize: 80, + } + table.UrlToUploadMap.Store(link.UUID, link) + + var wg sync.WaitGroup + var mu sync.Mutex + successes := 0 + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if err := table.ReserveUpload(link.UUID, 40, now, 50, 80); err == nil { + mu.Lock() + successes++ + mu.Unlock() + } + }() + } + wg.Wait() + + if successes != 2 { + t.Fatalf("expected exactly 2 reservations, got %d", successes) + } + if got := table.pendingFileCounts[link.UUID]; got != 2 { + t.Errorf("pendingFileCounts = %d, want 2", got) + } + if got := table.pendingBytes[link.UUID]; got != 80 { + t.Errorf("pendingBytes = %d, want 80", got) + } + if got := table.pendingOwnerBytes[link.Owner]; got != 80 { + t.Errorf("pendingOwnerBytes = %d, want 80", got) + } + + if err := table.CommitUpload(link.UUID, 40); err != nil { + t.Fatalf("CommitUpload: %v", err) + } + table.ReleaseUpload(link.UUID, 40) + if got := table.pendingFileCounts[link.UUID]; got != 0 { + t.Errorf("pendingFileCounts after commit/release = %d, want 0", got) + } + if got := table.pendingOwnerBytes[link.Owner]; got != 0 { + t.Errorf("pendingOwnerBytes after commit/release = %d, want 0", got) + } +} + +func TestDeleteUploadLinkReleasesPendingOwnerBytes(t *testing.T) { + table := newTestUploadLinkTable(t) + now := time.Now().Unix() + link := &UploadLinkOption{ + UUID: "delete-pending", + Owner: "alice", + ExpiresUnix: now + 10, + MaxFileCount: 2, + MaxFileSize: 100, + MaxTotalSize: 100, + } + table.UrlToUploadMap.Store(link.UUID, link) + if err := table.Database.Write(UploadLinkTableName, link.UUID, link); err != nil { + t.Fatalf("database write: %v", err) + } + if err := table.ReserveUpload(link.UUID, 40, now, 100, 100); err != nil { + t.Fatalf("ReserveUpload: %v", err) + } + + if err := table.DeleteUploadLinkByUUID(link.UUID); err != nil { + t.Fatalf("DeleteUploadLinkByUUID: %v", err) + } + if got := table.pendingOwnerBytes[link.Owner]; got != 0 { + t.Errorf("pendingOwnerBytes after delete = %d, want 0", got) + } +} diff --git a/src/mod/share/upload.go b/src/mod/share/upload.go new file mode 100644 index 00000000..8420d12e --- /dev/null +++ b/src/mod/share/upload.go @@ -0,0 +1,952 @@ +package share + +import ( + "encoding/hex" + "encoding/json" + "errors" + "hash/crc32" + "html" + "io" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/gorilla/websocket" + uuid "github.com/satori/go.uuid" + + filesystem "imuslab.com/arozos/mod/filesystem" + "imuslab.com/arozos/mod/filesystem/arozfs" + "imuslab.com/arozos/mod/share/shareEntry" + "imuslab.com/arozos/mod/user" + "imuslab.com/arozos/mod/utils" +) + +const ( + defaultUploadLinkTTLSeconds int64 = 24 * 60 * 60 + maxUploadLinkTTLSeconds int64 = 365 * 24 * 60 * 60 + defaultUploadLinkMaxFileCount int64 = 10 + defaultUploadLinkMaxFileSize int64 = 100 << 20 + defaultUploadLinkMaxTotalSize int64 = 1 << 30 + publicPostUploadCutoff int64 = 25 << 20 + publicUploadWSMaxFrameSize int64 = 1 << 20 +) + +type uploadLinkResponse struct { + UUID string + TargetVirtualPath string + Owner string + CreatedUnix int64 + ExpiresUnix int64 + MaxFileCount int64 + MaxFileSize int64 + MaxTotalSize int64 + UploadedFileCount int64 + UploadedBytes int64 + Disabled bool + RemainingFileCount int64 + RemainingBytes int64 + URL string +} + +type uploadContext struct { + Link *shareEntry.UploadLinkOption + Owner *user.User + TargetFsh *filesystem.FileSystemHandler + TargetDir string + DestPath string + UploadSize int64 + releaseName func() +} + +func (s *Manager) HandleCreateUploadLink(w http.ResponseWriter, r *http.Request) { + table, err := s.getUploadLinkTable() + if err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + + userinfo, err := s.options.UserHandler.GetUserInfoFromRequest(w, r) + if err != nil { + utils.SendErrorResponse(w, "User not logged in") + return + } + + if err := r.ParseForm(); err != nil { + utils.SendErrorResponse(w, "Invalid upload link settings") + return + } + vpath := r.Form.Get("path") + targetFsh, _, err := s.validateUploadLinkTarget(userinfo, vpath) + if err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + + now := time.Now().Unix() + ttlSeconds, err := parseUploadLinkTTLSeconds(r.Form.Get("ttl"), defaultUploadLinkTTLSeconds) + if err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + maxFileCount := parseInt64FormDefault(r, "maxFileCount", defaultUploadLinkMaxFileCount) + maxFileSize := parseInt64FormDefault(r, "maxFileSize", defaultUploadLinkMaxFileSize) + maxTotalSize := parseInt64FormDefault(r, "maxTotalSize", defaultUploadLinkMaxTotalSize) + maxFileCount, maxFileSize, maxTotalSize, err = s.normalizeUploadLinkLimits(userinfo, maxFileCount, maxFileSize, maxTotalSize) + if err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + + link, err := table.CreateNewUploadLink(targetFsh, vpath, userinfo.Username, now, now+ttlSeconds, maxFileCount, maxFileSize, maxTotalSize) + if err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + + js, _ := json.Marshal(s.toUploadLinkResponse(link, r)) + utils.SendJSONResponse(w, string(js)) +} + +func (s *Manager) HandleEditUploadLink(w http.ResponseWriter, r *http.Request) { + table, err := s.getUploadLinkTable() + if err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + + userinfo, err := s.options.UserHandler.GetUserInfoFromRequest(w, r) + if err != nil { + utils.SendErrorResponse(w, "User not logged in") + return + } + + if err := r.ParseForm(); err != nil { + utils.SendErrorResponse(w, "Invalid upload link settings") + return + } + linkUUID := r.Form.Get("uuid") + link := table.GetUploadLinkFromUUID(linkUUID) + if link == nil { + utils.SendErrorResponse(w, "Upload link UUID not exists") + return + } + if !s.canModifyUploadLink(userinfo, link) { + utils.SendErrorResponse(w, "Permission Denied") + return + } + + owner, err := s.options.UserHandler.GetUserInfoFromUsername(link.Owner) + if err != nil { + utils.SendErrorResponse(w, "Upload link owner not exists") + return + } + + maxFileCount := parseInt64FormDefault(r, "maxFileCount", link.MaxFileCount) + maxFileSize := parseInt64FormDefault(r, "maxFileSize", link.MaxFileSize) + maxTotalSize := parseInt64FormDefault(r, "maxTotalSize", link.MaxTotalSize) + maxFileCount, maxFileSize, maxTotalSize, err = s.normalizeUploadLinkLimits(owner, maxFileCount, maxFileSize, maxTotalSize) + if err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + + updated := *link + updated.MaxFileCount = maxFileCount + updated.MaxFileSize = maxFileSize + updated.MaxTotalSize = maxTotalSize + if ttl := r.Form.Get("ttl"); ttl != "" { + ttlSeconds, err := parseUploadLinkTTLSeconds(ttl, 0) + if err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + updated.ExpiresUnix = time.Now().Unix() + ttlSeconds + } + if disabled := r.Form.Get("disabled"); disabled != "" { + updated.Disabled = disabled == "true" || disabled == "1" + } + + if err := table.UpdateUploadLink(&updated); err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + + js, _ := json.Marshal(s.toUploadLinkResponse(&updated, r)) + utils.SendJSONResponse(w, string(js)) +} + +func (s *Manager) HandleDeleteUploadLink(w http.ResponseWriter, r *http.Request) { + table, err := s.getUploadLinkTable() + if err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + + userinfo, err := s.options.UserHandler.GetUserInfoFromRequest(w, r) + if err != nil { + utils.SendErrorResponse(w, "User not logged in") + return + } + + if err := r.ParseForm(); err != nil { + utils.SendErrorResponse(w, "Invalid upload link settings") + return + } + linkUUID := r.Form.Get("uuid") + link := table.GetUploadLinkFromUUID(linkUUID) + if link == nil { + utils.SendErrorResponse(w, "Upload link UUID not exists") + return + } + if !s.canModifyUploadLink(userinfo, link) { + utils.SendErrorResponse(w, "Permission Denied") + return + } + + if err := table.DeleteUploadLinkByUUID(linkUUID); err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + utils.SendOK(w) +} + +func (s *Manager) HandleListUploadLinks(w http.ResponseWriter, r *http.Request) { + table, err := s.getUploadLinkTable() + if err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + + userinfo, err := s.options.UserHandler.GetUserInfoFromRequest(w, r) + if err != nil { + utils.SendErrorResponse(w, "User not logged in") + return + } + + results := []*shareEntry.UploadLinkOption{} + vpath := r.URL.Query().Get("path") + if vpath != "" { + fsh := userinfo.GetRootFSHFromVpathInUserScope(vpath) + if fsh == nil { + utils.SendErrorResponse(w, "Invalid path given") + return + } + pathHash, err := shareEntry.GetPathHash(fsh, vpath, userinfo.Username) + if err != nil { + utils.SendErrorResponse(w, "Unable to get upload links from given path") + return + } + results = table.ListUploadLinksByPathHash(pathHash) + } else { + results = table.ListUploadLinksByOwner(userinfo.Username) + } + + reduced := []uploadLinkResponse{} + for _, link := range results { + if s.canModifyUploadLink(userinfo, link) { + reduced = append(reduced, s.toUploadLinkResponse(link, r)) + } + } + + js, _ := json.Marshal(reduced) + utils.SendJSONResponse(w, string(js)) +} + +func (s *Manager) HandleUploadLinkAccess(w http.ResponseWriter, r *http.Request, cleanParts []string) { + if len(cleanParts) < 3 { + http.NotFound(w, r) + return + } + + if len(cleanParts) == 3 { + s.HandlePublicUploadLinkPage(w, r, cleanParts[2]) + return + } + + switch cleanParts[2] { + case "post": + if len(cleanParts) < 4 { + http.NotFound(w, r) + return + } + s.HandlePublicUploadLinkPost(w, r, cleanParts[3]) + case "ws": + if len(cleanParts) < 4 { + http.NotFound(w, r) + return + } + s.HandlePublicUploadLinkWebSocket(w, r, cleanParts[3]) + default: + http.NotFound(w, r) + } +} + +func (s *Manager) HandlePublicUploadLinkPage(w http.ResponseWriter, r *http.Request, linkUUID string) { + table, err := s.getUploadLinkTable() + if err != nil { + http.NotFound(w, r) + return + } + link := table.GetUploadLinkFromUUID(linkUUID) + if link == nil || !link.IsActive(time.Now().Unix()) { + ServePermissionDeniedPage(w) + return + } + + content, err := utils.Templateload("./system/share/uploadPage.html", map[string]string{ + "hostname": html.EscapeString(s.options.HostName), + "target": html.EscapeString(arozfs.Base(link.TargetVirtualPath)), + "uuid": html.EscapeString(link.UUID), + "uploadurl": "/share/upload/post/" + link.UUID, + "wsurl": "/share/upload/ws/" + link.UUID, + "maxfilesize": strconv.FormatInt(link.MaxFileSize, 10), + "remainingfiles": strconv.FormatInt(remainingUploadFileCount(link), 10), + "remainingbytes": strconv.FormatInt(remainingUploadBytes(link), 10), + "postcutoff": strconv.FormatInt(publicPostUploadCutoff, 10), + "expires": strconv.FormatInt(link.ExpiresUnix, 10), + "uploadedfiles": strconv.FormatInt(link.UploadedFileCount, 10), + "uploadedbytes": strconv.FormatInt(link.UploadedBytes, 10), + "maxfilecount": strconv.FormatInt(link.MaxFileCount, 10), + "maxtotalsize": strconv.FormatInt(link.MaxTotalSize, 10), + "server_timestamp": strconv.FormatInt(time.Now().Unix(), 10), + }) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("500 - Internal Server Error")) + return + } + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Write([]byte(content)) +} + +func (s *Manager) HandlePublicUploadLinkPost(w http.ResponseWriter, r *http.Request, linkUUID string) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + maxBodySize, err := s.maxPublicUploadBodySize(linkUUID) + if err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + if maxBodySize > 0 { + r.Body = http.MaxBytesReader(w, r.Body, maxBodySize+(1<<20)) + } + + if err := r.ParseMultipartForm(32 << 20); err != nil { + utils.SendErrorResponse(w, "File too large") + return + } + defer r.MultipartForm.RemoveAll() + + file, handler, err := r.FormFile("file") + if err != nil { + utils.SendErrorResponse(w, "Unable to parse file from upload") + return + } + defer file.Close() + + ctx, err := s.prepareUploadLinkUpload(linkUUID, handler.Filename, handler.Size) + if err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + defer ctx.releaseName() + + if err := ctx.TargetFsh.FileSystemAbstraction.WriteStream(ctx.DestPath, file, 0775); err != nil { + s.options.UploadLinkTable.ReleaseUpload(ctx.Link.UUID, ctx.UploadSize) + utils.SendErrorResponse(w, "Write upload to destination disk failed") + return + } + + if err := s.completeUploadLinkUpload(ctx, ctx.UploadSize); err != nil { + utils.SendErrorResponse(w, err.Error()) + return + } + + js, _ := json.Marshal(map[string]string{ + "status": "OK", + "filename": arozfs.Base(ctx.DestPath), + }) + utils.SendJSONResponse(w, string(js)) +} + +func (s *Manager) HandlePublicUploadLinkWebSocket(w http.ResponseWriter, r *http.Request, linkUUID string) { + filename, err := utils.GetPara(r, "filename") + if filename == "" || err != nil { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("400 - Invalid filename given")) + return + } + sizeStr, err := utils.GetPara(r, "size") + if sizeStr == "" || err != nil { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("400 - Invalid size given")) + return + } + uploadSize, err := strconv.ParseInt(sizeStr, 10, 64) + if err != nil || uploadSize <= 0 { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("400 - Invalid size given")) + return + } + + ctx, err := s.prepareUploadLinkUpload(linkUUID, filename, uploadSize) + if err != nil { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte("403 - " + err.Error())) + return + } + defer ctx.releaseName() + + uploadFolder := filepath.Join(s.options.TmpFolder, "uploads", uuid.NewV4().String()) + if err := os.MkdirAll(uploadFolder, 0700); err != nil { + s.options.UploadLinkTable.ReleaseUpload(ctx.Link.UUID, ctx.UploadSize) + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("500 - Unable to create upload buffer")) + return + } + defer os.RemoveAll(uploadFolder) + + var upgrader = websocket.Upgrader{} + upgrader.CheckOrigin = func(r *http.Request) bool { return true } + c, err := upgrader.Upgrade(w, r, nil) + if err != nil { + s.options.UploadLinkTable.ReleaseUpload(ctx.Link.UUID, ctx.UploadSize) + return + } + defer c.Close() + c.SetReadLimit(publicUploadWSMaxFrameSize) + + if err := s.receiveUploadLinkChunks(c, uploadFolder, uploadSize); err != nil { + s.options.UploadLinkTable.ReleaseUpload(ctx.Link.UUID, ctx.UploadSize) + c.WriteMessage(1, []byte(`{"error":"`+escapeJSONError(err.Error())+`"}`)) + return + } + + if err := mergeUploadLinkChunks(uploadFolder, ctx.DestPath, ctx.TargetFsh); err != nil { + s.options.UploadLinkTable.ReleaseUpload(ctx.Link.UUID, ctx.UploadSize) + c.WriteMessage(1, []byte(`{"error":"Failed to write upload to destination disk"}`)) + return + } + + if err := s.completeUploadLinkUpload(ctx, uploadSize); err != nil { + c.WriteMessage(1, []byte(`{"error":"`+escapeJSONError(err.Error())+`"}`)) + return + } + + c.WriteMessage(1, []byte(`{"status":"OK","filename":"`+escapeJSONError(arozfs.Base(ctx.DestPath))+`"}`)) + c.WriteControl(8, []byte{}, time.Now().Add(time.Second)) +} + +func (s *Manager) receiveUploadLinkChunks(c *websocket.Conn, uploadFolder string, expectedSize int64) error { + blockCounter := 0 + chunkName := []string{} + totalFileSize := int64(0) + fileCRC32Hasher := crc32.NewIEEE() + + var pendingChunkIndex int + var pendingChunkChecksum string + expectingBinary := false + lastChunkArrivalTime := time.Now().Unix() + + ticker := time.NewTicker(60 * time.Second) + defer ticker.Stop() + done := make(chan bool) + defer close(done) + go func() { + for { + select { + case <-done: + return + case <-ticker.C: + if time.Now().Unix()-lastChunkArrivalTime > 300 { + c.WriteControl(8, []byte{}, time.Now().Add(time.Second)) + c.Close() + return + } + } + } + }() + + for { + mt, message, err := c.ReadMessage() + if err != nil { + return errors.New("Upload terminated by client") + } + + if mt == 1 { + textMsg := strings.TrimSpace(string(message)) + if !expectingBinary { + var doneSignal struct { + Done bool `json:"done"` + TotalChunks int `json:"totalChunks"` + FileChecksum string `json:"fileChecksum"` + } + if jsonErr := json.Unmarshal([]byte(textMsg), &doneSignal); jsonErr == nil && doneSignal.Done { + if doneSignal.FileChecksum != "" { + computedSum := fileCRC32Hasher.Sum32() + computedSumBytes := []byte{byte(computedSum >> 24), byte(computedSum >> 16), byte(computedSum >> 8), byte(computedSum)} + computedHex := hex.EncodeToString(computedSumBytes) + if doneSignal.FileChecksum != computedHex { + return errors.New("File integrity check failed") + } + } + break + } + + var meta struct { + Index int `json:"index"` + Checksum string `json:"checksum"` + } + if jsonErr := json.Unmarshal([]byte(textMsg), &meta); jsonErr != nil { + return errors.New("Invalid chunk metadata received") + } + pendingChunkIndex = meta.Index + pendingChunkChecksum = meta.Checksum + expectingBinary = true + } + } else if mt == 2 { + if !expectingBinary { + return errors.New("Received chunk without metadata") + } + if int64(len(message)) > publicUploadWSMaxFrameSize { + return errors.New("Upload chunk too large") + } + expectingBinary = false + + chunkSum := crc32.ChecksumIEEE(message) + chunkSumBytes := []byte{byte(chunkSum >> 24), byte(chunkSum >> 16), byte(chunkSum >> 8), byte(chunkSum)} + chunkHex := hex.EncodeToString(chunkSumBytes) + if pendingChunkChecksum != "" && pendingChunkChecksum != chunkHex { + retryMsg, _ := json.Marshal(map[string]int{"retryChunk": pendingChunkIndex}) + c.WriteMessage(1, retryMsg) + continue + } + + chunkFilepath := filepath.Join(uploadFolder, "upld_"+strconv.Itoa(pendingChunkIndex)) + if pendingChunkIndex == blockCounter { + chunkName = append(chunkName, chunkFilepath) + blockCounter++ + } + + if err := os.WriteFile(chunkFilepath, message, 0700); err != nil { + return errors.New("Write file chunk to disk failed") + } + + fileCRC32Hasher.Write(message) + lastChunkArrivalTime = time.Now().Unix() + totalFileSize += int64(len(message)) + if totalFileSize > expectedSize { + return errors.New("File size exceeds declared upload size") + } + c.WriteMessage(1, []byte("next")) + } + } + + if totalFileSize != expectedSize { + return errors.New("File size does not match declared upload size") + } + + manifest, _ := json.Marshal(chunkName) + return os.WriteFile(filepath.Join(uploadFolder, "manifest.json"), manifest, 0600) +} + +func mergeUploadLinkChunks(uploadFolder string, destPath string, targetFsh *filesystem.FileSystemHandler) error { + manifestBytes, err := os.ReadFile(filepath.Join(uploadFolder, "manifest.json")) + if err != nil { + return err + } + chunkName := []string{} + if err := json.Unmarshal(manifestBytes, &chunkName); err != nil { + return err + } + + targetFs := targetFsh.FileSystemAbstraction + if targetFsh.RequireBuffer { + mergeFileLocation := filepath.Join(uploadFolder, "merged") + out, err := os.OpenFile(mergeFileLocation, os.O_CREATE|os.O_WRONLY, 0755) + if err != nil { + return err + } + for _, filesrc := range chunkName { + srcChunkReader, err := os.Open(filesrc) + if err != nil { + out.Close() + return err + } + if _, err := io.Copy(out, srcChunkReader); err != nil { + srcChunkReader.Close() + out.Close() + return err + } + srcChunkReader.Close() + } + out.Close() + + f, err := os.Open(mergeFileLocation) + if err != nil { + return err + } + defer f.Close() + return targetFs.WriteStream(destPath, f, 0775) + } + + out, err := targetFs.OpenFile(destPath, os.O_CREATE|os.O_WRONLY, 0755) + if err != nil { + return err + } + defer out.Close() + for _, filesrc := range chunkName { + srcChunkReader, err := os.Open(filesrc) + if err != nil { + return err + } + if _, err := io.Copy(out, srcChunkReader); err != nil { + srcChunkReader.Close() + return err + } + srcChunkReader.Close() + } + return nil +} + +func (s *Manager) prepareUploadLinkUpload(linkUUID string, filename string, uploadSize int64) (*uploadContext, error) { + table, err := s.getUploadLinkTable() + if err != nil { + return nil, err + } + link := table.GetUploadLinkFromUUID(linkUUID) + if link == nil { + return nil, errors.New("Upload link not exists") + } + if !link.IsActive(time.Now().Unix()) { + return nil, errors.New("Upload link expired or disabled") + } + + owner, err := s.options.UserHandler.GetUserInfoFromUsername(link.Owner) + if err != nil { + return nil, errors.New("Upload link owner not exists") + } + targetFsh, realUploadPath, err := s.validateUploadLinkTarget(owner, link.TargetVirtualPath) + if err != nil { + return nil, err + } + + filename = strings.TrimSpace(filename) + if filename == "" || filename == "." || arozfs.Base(filename) != filename || !utils.FilenameIsWebSafe(filename) { + return nil, errors.New("Invalid filename given") + } + + ownerRemainingQuota := getOwnerRemainingQuota(owner) + if err := table.ReserveUpload(link.UUID, uploadSize, time.Now().Unix(), s.options.MaxUploadSize, ownerRemainingQuota); err != nil { + return nil, err + } + + destPath, releaseName, err := s.reserveAnonymousUploadDestination(targetFsh.FileSystemAbstraction, realUploadPath, filename) + if err != nil { + table.ReleaseUpload(link.UUID, uploadSize) + return nil, err + } + + return &uploadContext{ + Link: link, + Owner: owner, + TargetFsh: targetFsh, + TargetDir: realUploadPath, + DestPath: destPath, + UploadSize: uploadSize, + releaseName: releaseName, + }, nil +} + +func (s *Manager) completeUploadLinkUpload(ctx *uploadContext, uploadSize int64) error { + if ctx == nil || ctx.Link == nil { + return errors.New("Invalid upload context") + } + if _, _, err := s.validateUploadLinkTarget(ctx.Owner, ctx.Link.TargetVirtualPath); err != nil { + s.options.UploadLinkTable.ReleaseUpload(ctx.Link.UUID, ctx.UploadSize) + return err + } + if err := s.options.UploadLinkTable.CommitUpload(ctx.Link.UUID, uploadSize); err != nil { + return err + } + if ctx.TargetFsh.Hierarchy == "user" { + ctx.Owner.StorageQuota.AllocateSpace(uploadSize) + } + return nil +} + +func (s *Manager) validateUploadLinkTarget(userinfo *user.User, vpath string) (*filesystem.FileSystemHandler, string, error) { + if userinfo == nil { + return nil, "", errors.New("User not logged in") + } + if strings.TrimSpace(vpath) == "" { + return nil, "", errors.New("Invalid path given") + } + if !userinfo.CanWrite(vpath) { + return nil, "", errors.New("Access Denied") + } + fsh := userinfo.GetRootFSHFromVpathInUserScope(vpath) + if fsh == nil { + return nil, "", errors.New("Invalid path given") + } + if fsh.ReadOnly { + return nil, "", errors.New("The upload target is Read Only.") + } + realPath, err := fsh.FileSystemAbstraction.VirtualPathToRealPath(vpath, userinfo.Username) + if err != nil { + return nil, "", errors.New("Upload target is invalid or permission denied.") + } + if !fsh.FileSystemAbstraction.FileExists(realPath) { + return nil, "", errors.New("Folder not exists") + } + if !fsh.FileSystemAbstraction.IsDir(realPath) { + return nil, "", errors.New("Upload link target must be a folder") + } + return fsh, realPath, nil +} + +func (s *Manager) reserveAnonymousUploadDestination(targetFs filesystem.FileSystemAbstraction, realUploadPath string, filename string) (string, func(), error) { + stem := strings.TrimSuffix(filename, filepath.Ext(filename)) + ext := filepath.Ext(filename) + timestamp := time.Now().Format("20060102-150405") + + s.uploadNameMu.Lock() + defer s.uploadNameMu.Unlock() + + for i := 0; i <= 1024; i++ { + candidate := filename + if i == 0 { + originalPath := filepath.Join(realUploadPath, candidate) + originalKey := filepath.ToSlash(filepath.Clean(originalPath)) + if !targetFs.FileExists(originalPath) && !s.uploadReservedNames[originalKey] { + s.uploadReservedNames[originalKey] = true + return originalPath, func() { + s.uploadNameMu.Lock() + delete(s.uploadReservedNames, originalKey) + s.uploadNameMu.Unlock() + }, nil + } + candidate = stem + "_" + timestamp + ext + } else { + candidate = stem + "_" + timestamp + "_" + strconv.Itoa(i) + ext + } + targetPath := filepath.Join(realUploadPath, candidate) + key := filepath.ToSlash(filepath.Clean(targetPath)) + if !targetFs.FileExists(targetPath) && !s.uploadReservedNames[key] { + s.uploadReservedNames[key] = true + return targetPath, func() { + s.uploadNameMu.Lock() + delete(s.uploadReservedNames, key) + s.uploadNameMu.Unlock() + }, nil + } + } + return "", func() {}, errors.New("Too many files with identical names") +} + +func (s *Manager) maxPublicUploadBodySize(linkUUID string) (int64, error) { + table, err := s.getUploadLinkTable() + if err != nil { + return 0, err + } + link := table.GetUploadLinkFromUUID(linkUUID) + if link == nil || !link.IsActive(time.Now().Unix()) { + return 0, errors.New("Upload link expired or disabled") + } + owner, err := s.options.UserHandler.GetUserInfoFromUsername(link.Owner) + if err != nil { + return 0, errors.New("Upload link owner not exists") + } + remainingQuota := getOwnerRemainingQuota(owner) + remainingLinkBytes := remainingUploadBytes(link) + maxBodySize := minPositiveInt64(link.MaxFileSize, remainingLinkBytes) + if s.options.MaxUploadSize > 0 { + maxBodySize = minPositiveInt64(maxBodySize, s.options.MaxUploadSize) + } + if remainingQuota >= 0 { + maxBodySize = minPositiveInt64(maxBodySize, remainingQuota) + } + if maxBodySize <= 0 { + return 0, errors.New("Upload link has no remaining quota") + } + return maxBodySize, nil +} + +func (s *Manager) getUploadLinkTable() (*shareEntry.UploadLinkTable, error) { + if s.options.UploadLinkTable == nil { + return nil, errors.New("Upload link manager not initialized") + } + return s.options.UploadLinkTable, nil +} + +func (s *Manager) canModifyUploadLink(userinfo *user.User, link *shareEntry.UploadLinkOption) bool { + if userinfo == nil || link == nil { + return false + } + if userinfo.IsAdmin() || userinfo.Username == link.Owner { + return true + } + return false +} + +func (s *Manager) normalizeUploadLinkLimits(owner *user.User, maxFileCount int64, maxFileSize int64, maxTotalSize int64) (int64, int64, int64, error) { + if maxFileCount <= 0 { + maxFileCount = defaultUploadLinkMaxFileCount + } + if maxFileSize <= 0 { + maxFileSize = defaultUploadLinkMaxFileSize + } + if maxTotalSize <= 0 { + maxTotalSize = defaultUploadLinkMaxTotalSize + } + + if s.options.MaxUploadSize > 0 && maxFileSize > s.options.MaxUploadSize { + maxFileSize = s.options.MaxUploadSize + } + + remainingQuota := getOwnerRemainingQuota(owner) + if remainingQuota == 0 { + return 0, 0, 0, errors.New("User Storage Quota Exceeded") + } + if remainingQuota > 0 { + if maxTotalSize > remainingQuota { + maxTotalSize = remainingQuota + } + if maxFileSize > remainingQuota { + maxFileSize = remainingQuota + } + } + if maxTotalSize > 0 && maxFileSize > maxTotalSize { + maxFileSize = maxTotalSize + } + if maxFileSize <= 0 || maxTotalSize <= 0 { + return 0, 0, 0, errors.New("User Storage Quota Exceeded") + } + + return maxFileCount, maxFileSize, maxTotalSize, nil +} + +func (s *Manager) toUploadLinkResponse(link *shareEntry.UploadLinkOption, r *http.Request) uploadLinkResponse { + return uploadLinkResponse{ + UUID: link.UUID, + TargetVirtualPath: link.TargetVirtualPath, + Owner: link.Owner, + CreatedUnix: link.CreatedUnix, + ExpiresUnix: link.ExpiresUnix, + MaxFileCount: link.MaxFileCount, + MaxFileSize: link.MaxFileSize, + MaxTotalSize: link.MaxTotalSize, + UploadedFileCount: link.UploadedFileCount, + UploadedBytes: link.UploadedBytes, + Disabled: link.Disabled, + RemainingFileCount: remainingUploadFileCount(link), + RemainingBytes: remainingUploadBytes(link), + URL: getRequestBaseURL(r) + "/share/upload/" + link.UUID, + } +} + +func getOwnerRemainingQuota(owner *user.User) int64 { + if owner == nil || owner.StorageQuota == nil { + return 0 + } + if owner.StorageQuota.TotalStorageQuota == -1 { + return -1 + } + remaining := owner.StorageQuota.TotalStorageQuota - owner.StorageQuota.UsedStorageQuota + if remaining < 0 { + return 0 + } + return remaining +} + +func remainingUploadFileCount(link *shareEntry.UploadLinkOption) int64 { + if link.MaxFileCount <= 0 { + return -1 + } + remaining := link.MaxFileCount - link.UploadedFileCount + if remaining < 0 { + return 0 + } + return remaining +} + +func remainingUploadBytes(link *shareEntry.UploadLinkOption) int64 { + if link.MaxTotalSize <= 0 { + return -1 + } + remaining := link.MaxTotalSize - link.UploadedBytes + if remaining < 0 { + return 0 + } + return remaining +} + +func parseInt64FormDefault(r *http.Request, key string, fallback int64) int64 { + raw := strings.TrimSpace(r.Form.Get(key)) + if raw == "" { + return fallback + } + value, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return fallback + } + return value +} + +func parseUploadLinkTTLSeconds(raw string, fallback int64) (int64, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return fallback, nil + } + value, err := strconv.ParseInt(raw, 10, 64) + if err != nil || value <= 0 { + return 0, errors.New("Invalid link ttl") + } + if value > maxUploadLinkTTLSeconds { + return 0, errors.New("Link ttl exceeds maximum allowed duration") + } + now := time.Now().Unix() + const maxInt64 = int64(1<<63 - 1) + if value > maxInt64-now { + return 0, errors.New("Invalid link ttl") + } + return value, nil +} + +func minPositiveInt64(values ...int64) int64 { + result := int64(0) + for _, value := range values { + if value <= 0 { + continue + } + if result == 0 || value < result { + result = value + } + } + return result +} + +func getRequestBaseURL(r *http.Request) string { + scheme := "http" + if r.TLS != nil { + scheme = "https" + } + if forwardedProto := r.Header.Get("X-Forwarded-Proto"); forwardedProto != "" { + scheme = strings.Split(forwardedProto, ",")[0] + } + return scheme + "://" + r.Host +} + +func escapeJSONError(message string) string { + encoded, _ := json.Marshal(message) + trimmed := strings.TrimPrefix(string(encoded), `"`) + trimmed = strings.TrimSuffix(trimmed, `"`) + return trimmed +} diff --git a/src/mod/share/upload_handler_test.go b/src/mod/share/upload_handler_test.go new file mode 100644 index 00000000..f749043a --- /dev/null +++ b/src/mod/share/upload_handler_test.go @@ -0,0 +1,255 @@ +package share + +import ( + "bytes" + "encoding/json" + "mime/multipart" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "imuslab.com/arozos/mod/auth" + db "imuslab.com/arozos/mod/database" + "imuslab.com/arozos/mod/filesystem" + "imuslab.com/arozos/mod/filesystem/abstractions/localfs" + "imuslab.com/arozos/mod/permission" + "imuslab.com/arozos/mod/share/shareEntry" + "imuslab.com/arozos/mod/storage" + "imuslab.com/arozos/mod/user" +) + +type uploadHandlerFixture struct { + manager *Manager + table *shareEntry.UploadLinkTable + root string + user string +} + +func newUploadHandlerFixture(t *testing.T, username string, quota int64) *uploadHandlerFixture { + t.Helper() + tmpDir := t.TempDir() + origDir, err := os.Getwd() + if err != nil { + t.Fatalf("Getwd: %v", err) + } + if err := os.Chdir(tmpDir); err != nil { + t.Fatalf("Chdir: %v", err) + } + + database, err := db.NewDatabase(filepath.Join(tmpDir, "system.db"), false) + if err != nil { + t.Fatalf("NewDatabase: %v", err) + } + authAgent := auth.NewAuthenticationAgent("testsession", []byte("supersecretkey1234567890"), database, false, nil) + t.Cleanup(func() { + authAgent.Close() + database.Close() + os.Chdir(origDir) + }) + + ph, err := permission.NewPermissionHandler(database) + if err != nil { + t.Fatalf("NewPermissionHandler: %v", err) + } + groupName := "uploadtest_" + username + ph.NewPermissionGroup(groupName, false, quota, []string{"File Manager"}, "Desktop") + if err := authAgent.CreateUserAccount(username, "password", []string{groupName}); err != nil { + t.Fatalf("CreateUserAccount: %v", err) + } + + root := filepath.Join(tmpDir, "storage") + fsa := localfs.NewLocalFileSystemAbstraction("testfsh", root, "user", false) + fsh := &filesystem.FileSystemHandler{ + UUID: "testfsh", + Name: "test", + Path: root, + Hierarchy: "user", + FileSystemAbstraction: fsa, + } + if err := os.MkdirAll(filepath.Join(root, "users", username, "uploads"), 0755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + sp, err := storage.NewStoragePool([]*filesystem.FileSystemHandler{fsh}, "system") + if err != nil { + t.Fatalf("NewStoragePool: %v", err) + } + + shareTable := shareEntry.NewShareEntryTable(database) + userHandler, err := user.NewUserHandler(database, authAgent, ph, sp, &shareTable) + if err != nil { + t.Fatalf("NewUserHandler: %v", err) + } + uploadTable := shareEntry.NewUploadLinkTable(database) + manager := NewShareManager(Options{ + UserHandler: userHandler, + UploadLinkTable: uploadTable, + TmpFolder: filepath.Join(tmpDir, "tmp"), + MaxUploadSize: 1024, + }) + + return &uploadHandlerFixture{ + manager: manager, + table: uploadTable, + root: root, + user: username, + } +} + +func (f *uploadHandlerFixture) newLink(t *testing.T, expiresUnix int64, maxFileCount int64, maxFileSize int64, maxTotalSize int64) *shareEntry.UploadLinkOption { + t.Helper() + userInfo, err := f.manager.options.UserHandler.GetUserInfoFromUsername(f.user) + if err != nil { + t.Fatalf("GetUserInfoFromUsername: %v", err) + } + fsh := userInfo.GetRootFSHFromVpathInUserScope("testfsh:/uploads") + if fsh == nil { + t.Fatal("test filesystem handler not found") + } + link, err := f.table.CreateNewUploadLink(fsh, "testfsh:/uploads", f.user, time.Now().Unix(), expiresUnix, maxFileCount, maxFileSize, maxTotalSize) + if err != nil { + t.Fatalf("CreateNewUploadLink: %v", err) + } + return link +} + +func newMultipartUploadRequest(t *testing.T, linkUUID string, filename string, content []byte) *http.Request { + t.Helper() + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + part, err := writer.CreateFormFile("file", filename) + if err != nil { + t.Fatalf("CreateFormFile: %v", err) + } + if _, err := part.Write(content); err != nil { + t.Fatalf("part.Write: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("writer.Close: %v", err) + } + + req := httptest.NewRequest(http.MethodPost, "/share/upload/post/"+linkUUID, body) + req.Header.Set("Content-Type", writer.FormDataContentType()) + return req +} + +func TestHandlePublicUploadLinkPostSuccess(t *testing.T) { + fixture := newUploadHandlerFixture(t, "uploadsuccess", -1) + link := fixture.newLink(t, time.Now().Unix()+3600, 2, 100, 200) + req := newMultipartUploadRequest(t, link.UUID, "hello.txt", []byte("hello")) + rr := httptest.NewRecorder() + + fixture.manager.HandlePublicUploadLinkPost(rr, req, link.UUID) + + if strings.Contains(rr.Body.String(), `"error"`) { + t.Fatalf("unexpected upload error: %s", rr.Body.String()) + } + var response map[string]string + if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil { + t.Fatalf("json.Unmarshal: %v", err) + } + if response["status"] != "OK" || response["filename"] != "hello.txt" { + t.Fatalf("unexpected response: %+v", response) + } + if !fixture.table.GetUploadLinkFromUUID(link.UUID).IsActive(time.Now().Unix()) { + t.Fatal("link should remain active after one upload") + } + if !localfs.NewLocalFileSystemAbstraction("testfsh", fixture.root, "user", false).FileExists(filepath.Join(fixture.root, "users", fixture.user, "uploads", "hello.txt")) { + t.Fatal("uploaded file was not written to target folder") + } +} + +func TestHandlePublicUploadLinkPostRejectsInvalidUploads(t *testing.T) { + tests := []struct { + name string + quota int64 + filename string + content []byte + mutateLink func(*shareEntry.UploadLinkOption) + }{ + { + name: "expired link", + quota: -1, + filename: "file.txt", + content: []byte("hello"), + mutateLink: func(link *shareEntry.UploadLinkOption) { + link.ExpiresUnix = time.Now().Unix() - 1 + }, + }, + { + name: "revoked link", + quota: -1, + filename: "file.txt", + content: []byte("hello"), + mutateLink: func(link *shareEntry.UploadLinkOption) { + link.Disabled = true + }, + }, + { + name: "file size limit", + quota: -1, + filename: "file.txt", + content: []byte("toolarge"), + mutateLink: func(link *shareEntry.UploadLinkOption) { + link.MaxFileSize = 3 + }, + }, + { + name: "total size limit", + quota: -1, + filename: "file.txt", + content: []byte("toolarge"), + mutateLink: func(link *shareEntry.UploadLinkOption) { + link.MaxTotalSize = 3 + }, + }, + { + name: "file count limit", + quota: -1, + filename: "file.txt", + content: []byte("hello"), + mutateLink: func(link *shareEntry.UploadLinkOption) { + link.MaxFileCount = 1 + link.UploadedFileCount = 1 + }, + }, + { + name: "owner quota limit", + quota: 3, + filename: "file.txt", + content: []byte("hello"), + }, + { + name: "unsafe filename", + quota: -1, + filename: "bad:name.txt", + content: []byte("hello"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + username := strings.ReplaceAll(tt.name, " ", "") + fixture := newUploadHandlerFixture(t, username, tt.quota) + link := fixture.newLink(t, time.Now().Unix()+3600, 2, 100, 200) + if tt.mutateLink != nil { + updated := *link + tt.mutateLink(&updated) + if err := fixture.table.UpdateUploadLink(&updated); err != nil { + t.Fatalf("UpdateUploadLink: %v", err) + } + link = &updated + } + + req := newMultipartUploadRequest(t, link.UUID, tt.filename, tt.content) + rr := httptest.NewRecorder() + fixture.manager.HandlePublicUploadLinkPost(rr, req, link.UUID) + if !strings.Contains(rr.Body.String(), `"error"`) { + t.Fatalf("expected error response, got %s", rr.Body.String()) + } + }) + } +} diff --git a/src/mod/share/upload_test.go b/src/mod/share/upload_test.go new file mode 100644 index 00000000..b90a0852 --- /dev/null +++ b/src/mod/share/upload_test.go @@ -0,0 +1,189 @@ +package share + +import ( + "os" + "path/filepath" + "regexp" + "testing" + + "imuslab.com/arozos/mod/filesystem/abstractions/localfs" + "imuslab.com/arozos/mod/quota" + "imuslab.com/arozos/mod/user" +) + +func TestNormalizeUploadLinkLimits(t *testing.T) { + tests := []struct { + name string + maxUploadSize int64 + quotaTotal int64 + quotaUsed int64 + inputFileCount int64 + inputFileSize int64 + inputTotalSize int64 + wantFileCount int64 + wantFileSize int64 + wantTotalSize int64 + wantErr bool + }{ + { + name: "clamps to remaining quota and global max upload size", + maxUploadSize: 75, + quotaTotal: 100, + quotaUsed: 50, + inputFileCount: 20, + inputFileSize: 500, + inputTotalSize: 500, + wantFileCount: 20, + wantFileSize: 50, + wantTotalSize: 50, + }, + { + name: "unlimited quota still clamps to global max upload size", + maxUploadSize: 75, + quotaTotal: -1, + inputFileCount: 20, + inputFileSize: 500, + inputTotalSize: 500, + wantFileCount: 20, + wantFileSize: 75, + wantTotalSize: 500, + }, + { + name: "defaults invalid limits", + maxUploadSize: 0, + quotaTotal: -1, + inputFileCount: 0, + inputFileSize: 0, + inputTotalSize: 0, + wantFileCount: defaultUploadLinkMaxFileCount, + wantFileSize: defaultUploadLinkMaxFileSize, + wantTotalSize: defaultUploadLinkMaxTotalSize, + }, + { + name: "rejects exhausted quota", + maxUploadSize: 100, + quotaTotal: 100, + quotaUsed: 100, + inputFileCount: 10, + inputFileSize: 10, + inputTotalSize: 10, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + manager := &Manager{options: Options{MaxUploadSize: tt.maxUploadSize}} + owner := &user.User{StorageQuota: "a.QuotaHandler{ + TotalStorageQuota: tt.quotaTotal, + UsedStorageQuota: tt.quotaUsed, + }} + gotCount, gotFileSize, gotTotalSize, err := manager.normalizeUploadLinkLimits(owner, tt.inputFileCount, tt.inputFileSize, tt.inputTotalSize) + if (err != nil) != tt.wantErr { + t.Fatalf("normalizeUploadLinkLimits() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + if gotCount != tt.wantFileCount || gotFileSize != tt.wantFileSize || gotTotalSize != tt.wantTotalSize { + t.Errorf("normalizeUploadLinkLimits() = (%d, %d, %d), want (%d, %d, %d)", + gotCount, gotFileSize, gotTotalSize, tt.wantFileCount, tt.wantFileSize, tt.wantTotalSize) + } + }) + } +} + +func TestParseUploadLinkTTLSeconds(t *testing.T) { + tests := []struct { + name string + raw string + fallback int64 + want int64 + wantErr bool + }{ + { + name: "default empty value", + raw: "", + fallback: defaultUploadLinkTTLSeconds, + want: defaultUploadLinkTTLSeconds, + }, + { + name: "valid ttl", + raw: "3600", + want: 3600, + }, + { + name: "zero rejected", + raw: "0", + wantErr: true, + }, + { + name: "too large rejected", + raw: "31536001", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseUploadLinkTTLSeconds(tt.raw, tt.fallback) + if (err != nil) != tt.wantErr { + t.Fatalf("parseUploadLinkTTLSeconds() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + if got != tt.want { + t.Errorf("parseUploadLinkTTLSeconds() = %d, want %d", got, tt.want) + } + }) + } +} + +func TestReserveAnonymousUploadDestinationGeneratesTimestampDuplicateName(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "report.txt"), []byte("existing"), 0644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + manager := &Manager{uploadReservedNames: map[string]bool{}} + targetFs := localfs.NewLocalFileSystemAbstraction("test", root, "user", false) + destPath, releaseName, err := manager.reserveAnonymousUploadDestination(targetFs, root, "report.txt") + if err != nil { + t.Fatalf("reserveAnonymousUploadDestination: %v", err) + } + defer releaseName() + + base := filepath.Base(destPath) + matched, err := regexp.MatchString(`^report_[0-9]{8}-[0-9]{6}(_[0-9]+)?\.txt$`, base) + if err != nil { + t.Fatalf("regexp.MatchString: %v", err) + } + if !matched { + t.Fatalf("expected timestamp duplicate filename, got %q", base) + } + if base == "report.txt" { + t.Fatal("duplicate upload destination must not reuse the original filename") + } +} + +func TestReserveAnonymousUploadDestinationRejectsReservedCollision(t *testing.T) { + root := t.TempDir() + manager := &Manager{uploadReservedNames: map[string]bool{}} + targetFs := localfs.NewLocalFileSystemAbstraction("test", root, "user", false) + + first, releaseFirst, err := manager.reserveAnonymousUploadDestination(targetFs, root, "new.txt") + if err != nil { + t.Fatalf("first reserveAnonymousUploadDestination: %v", err) + } + defer releaseFirst() + second, releaseSecond, err := manager.reserveAnonymousUploadDestination(targetFs, root, "new.txt") + if err != nil { + t.Fatalf("second reserveAnonymousUploadDestination: %v", err) + } + defer releaseSecond() + + if first == second { + t.Fatalf("expected second reservation to choose a different path, got %q", second) + } +} diff --git a/src/system/share/uploadPage.html b/src/system/share/uploadPage.html new file mode 100644 index 00000000..512e6919 --- /dev/null +++ b/src/system/share/uploadPage.html @@ -0,0 +1,371 @@ + + + + + + Upload to {{target}} - {{hostname}} + + + + + +
+
{{hostname}}
+
{{target}}
+
+
+
+

Upload files

+

This link can only add files to the shared folder. Existing files will not be overwritten.

+
+
+
+
+
+
+
+

Drop files here or select files from this device.

+ + +
+
+
+
+
+ + + + diff --git a/src/web/SystemAO/file_system/file_explorer.html b/src/web/SystemAO/file_system/file_explorer.html index e5f7a588..173f156e 100644 --- a/src/web/SystemAO/file_system/file_explorer.html +++ b/src/web/SystemAO/file_system/file_explorer.html @@ -53,6 +53,7 @@ +



+

@@ -378,6 +380,27 @@

+ + +