diff --git a/PRODUCT.md b/PRODUCT.md new file mode 100644 index 0000000000..d0a287be49 --- /dev/null +++ b/PRODUCT.md @@ -0,0 +1,33 @@ +# Product + +## Register + +product + +## Users + +Tlon Messenger group members, group administrators, and Tlonbots collaborating inside an existing group. People may arrive to quickly retrieve a file, organize an active project, or review what another member or agent changed. Tlonbots participate through their own ship identities and permissions rather than inheriting their human owner's authority. + +## Product Purpose + +Tlon Messenger is a group communication and collaboration product built on Urbit. Buckets extends a group with durable, group-owned file storage: a shared Drive or Dropbox that members and authorized Tlonbots can browse and change. Success means a member can understand where a file lives, who or what changed it, and what actions are available without needing to understand object storage, while files remain with the group when an uploader leaves. + +## Brand Personality + +Familiar, calm, communal. The interface should feel like a native part of Tlon Messenger and earn trust through predictable file-management patterns, concise language, and visible authorship. Agents are ordinary collaborators with a subtle bot marker, not a separate novelty layer. + +## Anti-references + +Do not resemble an S3 administration console, a second social gallery or feed, or a heavyweight enterprise document-management suite. Avoid storage jargon, dashboard decoration, card-grid sprawl, agent theater, and controls that imply per-file capabilities the backend cannot yet enforce. + +## Design Principles + +1. Make group ownership unmistakable: files belong to the shared space, not their uploader. +2. Use earned familiarity: folders, breadcrumbs, list and grid views, selection, and direct manipulation should behave like trusted file tools. +3. Treat humans and agents as accountable peers: authorize and attribute every action to the acting ship. +4. Reveal complexity progressively: keep retrieval and upload immediate while placing metadata, activity, and administration one layer deeper. +5. Preserve continuity with Messenger: reuse its channel shell, identity treatment, media previews, upload feedback, and responsive navigation. + +## Accessibility & Inclusion + +Target WCAG 2.2 AA. All file operations must work with keyboard and screen readers, have visible focus, use color-independent state cues, maintain touch targets of at least 44 by 44 points on mobile, respect reduced motion, and handle long user-generated names without hiding essential actions. diff --git a/apps/tlon-web/vite.config.mts b/apps/tlon-web/vite.config.mts index 8c0c6e1f6c..b004a65095 100644 --- a/apps/tlon-web/vite.config.mts +++ b/apps/tlon-web/vite.config.mts @@ -257,6 +257,17 @@ export default ({ mode }: { mode: string }) => { // `platform`" at boot. define: { 'process.env.EXPO_OS': JSON.stringify('web'), + // Nothing else reaches the browser through process.env -- envPrefix only + // exposes VITE_* via import.meta.env -- so without this the storage + // broker override is inert on web and uploads silently go to production + // while the host pushes read grants somewhere else. + // + // Both spellings, for the same reason SHIP_URL takes both above: only + // VITE_* names survive loadEnv into an .env file, while the shell can + // set the bare name. + 'process.env.TLON_MEMEX_URL': JSON.stringify( + process.env.TLON_MEMEX_URL ?? process.env.VITE_TLON_MEMEX_URL ?? '' + ), }, base: base(mode), server: { diff --git a/desk/app/buckets.hoon b/desk/app/buckets.hoon new file mode 100644 index 0000000000..d5830483c8 --- /dev/null +++ b/desk/app/buckets.hoon @@ -0,0 +1,2507 @@ +:: buckets: group-owned shared file manifest and upload coordinator +:: +:: This agent is a third-party %groups channel host. It stores metadata and +:: upload lifecycle only; file bytes move directly between clients and object +:: storage. Group admins may request creation, but the group host remains the +:: authoritative Bucket and storage owner. +:: +:: Every client action carries a request-id and gets exactly one terminal +:: response. Bearer tokens for the storage broker are minted here and +:: returned only to the requester — they never appear in a broadcast. +:: +/- b=buckets, gv=groups-ver +/+ default-agent, dbug, verb, server +/= buckets-json /lib/buckets/json +|% ++$ card card:agent:gall ++$ current-state state:b +:: +upload-window: how long a pending upload session stays usable. +:: +:: Must stay comfortably longer than Memex's own window, which is at most +:: BUCKETS_PUT_URL_SECONDS (capped at 900) plus BUCKETS_COMPLETION_GRACE_SECONDS +:: (default 600). If this expires first, a completion Memex still considers +:: live arrives to find no session. +:: +++ upload-window ~h1 +:: +object-window: lifetime of a delete capability. +:: +++ object-window ~m10 +:: +read-window: lifetime of a bucket-read token, and +token-margin the slack +:: we re-mint within so a client never finds an expired one. +:: +:: Long on purpose. Expiry is a backstop, not the revocation mechanism: the +:: broker re-checks the live group on every exchange today, and once it holds +:: a pushed token the host revokes explicitly when access changes. What the +:: expiry buys is failing closed if a host dies or a revoke is ever missed. +:: +++ read-window ~d1 +++ token-margin ~h1 +:: +request-timeout: how long a subscriber waits for the host's answer +:: before reporting failure to its client. +:: +++ request-timeout ~m2 +:: +max-object-size: mirrors Memex's BUCKETS_MAX_OBJECT_BYTES default, so an +:: oversized upload is refused before any state is committed. +:: +++ max-object-size 5.368.709.120 +:: +request-grace: how long a settled request record is kept so a slow +:: poller can still read its result. +:: +++ request-grace ~m5 +:: +default-broker-base: where the storage broker lives, absent a poke +:: saying otherwise. The live value is .broker-base in state. +:: +:: Must stay in step with BUCKETS_BROKER_URL in +:: packages/shared/src/store/storage/bucketsBroker.ts — clients and hosts +:: talk to the same service, from opposite directions, and a host pointed at +:: one while its clients upload to the other is broken in both. +:: +++ default-broker-base 'https://memex.tlon.network/v2/buckets' +:: +push-retry: how soon to mint again after the broker refused a token. +:: +:: A token the broker never accepted is never stored, so there is nothing to +:: fall back on; this is how long before we try again unprompted. +:: +++ push-retry ~m1 +:: +groups-retry: how soon to ask %groups for its updates again after it +:: refuses. Longer than a push retry because the usual cause is %groups not +:: being up yet, which resolves on its own. +:: +++ groups-retry ~m5 +-- +=| current-state +=* state - +%- agent:dbug +%^ verb | %warn +^- agent:gall +=< + |_ =bowl:gall + +* this . + def ~(. (default-agent this %|) bowl) + cor ~(. +> [bowl ~ ~]) + ++ on-init + ^- (quip card _this) + =^ cards state abet:init:cor + [cards this] + ++ on-save !>(state) + ++ on-load + |= old=vase + ^- (quip card _this) + =^ cards state abet:(load:cor old) + [cards this] + ++ on-poke + |= [=mark =vase] + ^- (quip card _this) + =^ cards state abet:(poke:cor mark vase) + [cards this] + ++ on-watch + |= =path + ^- (quip card _this) + =^ cards state abet:(watch:cor `(pole knot)`path) + [cards this] + ++ on-peek + |= =path + ^- (unit (unit cage)) + ?> =(src our):bowl + (peek:cor `(pole knot)`path) + ++ on-agent + |= [=wire =sign:agent:gall] + ^- (quip card _this) + =^ cards state abet:(agent:cor `(pole knot)`wire sign) + [cards this] + ++ on-arvo + |= [=wire =sign-arvo] + ^- (quip card _this) + =^ cards state abet:(arvo:cor `(pole knot)`wire sign-arvo) + [cards this] + ++ on-leave on-leave:def + ++ on-fail + |= [=term =tang] + ^- (quip card _this) + %- (slog 'buckets: on-fail' >term< tang) + [~ this] + -- +:: +|_ [=bowl:gall cards=(list card) reply=(unit response-body:b)] +++ cor . +++ abet [(flop cards) state] +++ emit |=(=card cor(cards [card cards])) +++ emil |=(caz=(list card) cor(cards (welp (flop caz) cards))) +++ give |=(=gift:agent:gall (emit %give gift)) +:: +answer: record the terminal body for the action being applied. Arms that +:: mint a token or refuse call this; +settle turns it into the response. +:: +++ answer |=(body=response-body:b cor(reply `body)) +:: +:: +watch-groups: subscribe to the group updates revocation depends on. +:: +:: Named rather than inlined because four places need it -- init, load, a +:: kick, and the retry after a refusal -- and the one that was written out +:: separately is the one that forgot to. +:: +++ watch-groups + ^+ cor + (emit [%pass /groups %agent [our.bowl %groups] %watch /v1/groups]) +:: +++ init + ^+ cor + =. broker-base default-broker-base + =. cor (emit [%pass /eyre %arvo %e %connect [~ /buckets] %buckets]) + watch-groups +:: +:: +load: %buckets has never run on a live ship, so there is nothing to +:: migrate from yet. When that changes, add a +state-N-to-N+1 arm per +:: version and chain them with =? — never migrate straight to current. +:: +++ load + |= old=vase + ^+ cor + =/ loaded=versioned-state:b !<(versioned-state:b old) + ?> ?=(%0 -.loaded) + =. state loaded + =? cor !(~(has by wex.bowl) [/groups our.bowl %groups]) + watch-groups + :: binding an already-bound route is refused harmlessly; +arvo logs it. + (emit [%pass /eyre %arvo %e %connect [~ /buckets] %buckets]) +:: +++ poke + |= [=mark =vase] + ^+ cor + ?+ mark ~|(bad-buckets-mark+mark !!) + %handle-http-request + (serve-http !<([eyre-id=@ta =inbound-request:eyre] vase)) + :: + %buckets-action-1 + ?> =(src.bowl our.bowl) + =+ cmd=!<(command:b vase) + :: The same one-answer contract the HTTP surface keeps: a caller that + :: lost our answer retries with the id it already used, and running the + :: action again would duplicate a folder or a session and answer twice. + :: A settled request replays its result; one still in flight is left to + :: finish, since its answer goes to the same subscription either way. + ?^ seen=(~(get by requests) request-id.cmd) + ?~ result.u.seen cor + (respond request-id.cmd ~[/v1/requests] u.result.u.seen) + =. cor (track-request request-id.cmd ~) + (dispatch-local request-id.cmd act.cmd) + :: + %buckets-command-1 + =+ cmd=!<(command:b vase) + (dispatch-remote request-id.cmd act.cmd) + :: + %buckets-broker-command-1 + ?> =(src.bowl our.bowl) + (apply-broker-command !<(broker-command:b vase)) + :: + :: Operator knob rather than client surface, so it rides %noun instead of + :: earning a mark. A ship has no environment to read, and pointing a host + :: at a test broker has to be possible on the build we ship rather than on + :: a patched desk. ~ restores the default. + :: + :: :buckets &noun [%set-broker-base `'https://memex.test.tlon.systems/v2/buckets'] + :: + %noun + ?> =(src.bowl our.bowl) + =+ !<([%set-broker-base base=(unit @t)] vase) + (set-broker-base base) + :: + %group-channel-join + ?> =(src.bowl our.bowl) + =+ join=!<(channel-join:b vase) + ?> =(%buckets kind.nest.join) + =/ =flag:b [host.nest.join name.nest.join] + ?: =(our.bowl ship.flag) cor + (start-sub flag group.join) + :: + %group-channel-leave + ?> =(src.bowl our.bowl) + =+ leave=!<(channel-leave:b vase) + ?> =(%buckets kind.nest.leave) + =/ =flag:b [host.nest.leave name.nest.leave] + ?: =(our.bowl ship.flag) cor + (stop-sub flag) + == +:: +:: +serve-http: route one Eyre request. +:: +:: Only two shapes exist: POST /buckets/~/v1 submits an action and is held +:: open until its terminal answer, and GET /buckets/~/v1/... reads. The +:: action's answer is the HTTP response, so a client needs no correlation +:: machinery of its own. +:: +++ serve-http + |= [eyre-id=@ta =inbound-request:eyre] + ^+ cor + =/ =request-line:server + (parse-request-line:server url.request.inbound-request) + =* site site.request-line + =/ method=@tas method.request.inbound-request + ?. (request-authorized inbound-request) + (http-error eyre-id 401 'unauthorized') + ?: =(site ~[%buckets %~.~ %v1]) + ?. =(%'POST' method) + (http-error eyre-id 405 'method not allowed') + (handle-post eyre-id inbound-request) + ?. ?=([%buckets %~.~ %v1 *] site) + (http-error eyre-id 404 'not found') + ?. =(%'GET' method) + (http-error eyre-id 405 'method not allowed') + :: A @uv request id carries dots, and apat mistakes its trailing dot-group + :: for a file extension and splits it off -- so glue it back on before the + :: id is parsed, or most ids resolve to a different request and 404. + :: %notes' surface has the same wrinkle and does the same thing. + :: Reattach by flopping rather than with snip/rear: those are wet gates, + :: and handing one a list already narrowed to non-empty breaks its own + :: recursive call on the tail. + =/ raw=(list @t) t.t.t.site + =/ pax=(list @t) + ?~ ext.request-line raw + =/ back=(list @t) (flop raw) + ?~ back raw + %- flop + [(rap 3 i.back '.' u.ext.request-line ~) t.back] + (handle-read eyre-id pax) +:: +:: +handle-post: parse an action, hold the request open, and dispatch. +:: +:: Parsing is defensive — a malformed body is a client error, not a crash. +:: requestId is optional: a caller that cannot produce a valid @uv gets one +:: minted here, and never needs to know it, because the answer comes back +:: on this same request. +:: +++ handle-post + |= [eyre-id=@ta =inbound-request:eyre] + ^+ cor + :: a cookie or api-key is the host's own capability, so the actor for + :: anything submitted over HTTP is us. + =. src.bowl our.bowl + ?~ body.request.inbound-request + (http-error eyre-id 400 'missing body') + ?~ jon=(de:json:html q.u.body.request.inbound-request) + (http-error eyre-id 400 'invalid json') + ?. ?=([%o *] u.jon) + (http-error eyre-id 400 'body must be a json object') + ?~ act-j=(~(get by p.u.jon) 'action') + (http-error eyre-id 400 'missing `action` field') + =/ parsed=(each action:b tang) + (mule |.((action:dejs:buckets-json u.act-j))) + ?: ?=(%| -.parsed) + (http-error eyre-id 400 'malformed action') + =/ rid=request-id:b + =/ rj=(unit json) (~(get by p.u.jon) 'requestId') + ?. ?&(?=(^ rj) ?=([%s *] u.rj)) + `@uv`eny.bowl + =/ got=(each @uv tang) (mule |.((slav %uv p.u.rj))) + ?:(?=(%& -.got) p.got `@uv`eny.bowl) + :: A retry of a dropped POST arrives with the id it already used, so the + :: action must not run twice: a %create-folder or %begin-upload would + :: duplicate state, and overwriting the record would strand the first + :: connection with no answer coming. + ?^ seen=(~(get by requests) rid) + :: Already settled: answer from the record rather than running again. + ?^ result.u.seen (give-response eyre-id [rid u.result.u.seen]) + :: Still in flight: hold this connection open for the one answer instead + :: of starting a second attempt. The earlier connection is dropped, which + :: is why the caller retried, and only one can be answered anyway. + cor(requests (~(put by requests) rid u.seen(http-id `eyre-id))) + =. cor (track-request rid `eyre-id) + (dispatch-local rid p.parsed) +:: +:: +handle-read: GET surface — the local snapshot list, one bucket, or the +:: state of a submitted request. +:: +++ handle-read + |= [eyre-id=@ta pax=(list @t)] + ^+ cor + ?+ pax (http-error eyre-id 404 'not found') + [%buckets ~] + %^ give-http eyre-id 200 + :- 'application/json' + (en:json:html (summaries:enjs:buckets-json local-summaries)) + :: + [%buckets %full ~] + %^ give-http eyre-id 200 + :- 'application/json' + (en:json:html (snapshots:enjs:buckets-json local-snapshots)) + :: + [%buckets @ @ ~] + ?~ who=(slaw %p i.t.pax) + (http-error eyre-id 400 'malformed host') + =/ =flag:b [u.who `@tas`i.t.t.pax] + ?~ sp=(~(get by spaces) flag) + (http-error eyre-id 404 'no such bucket') + ?~ state.u.sp + (http-error eyre-id 404 'no such bucket') + %^ give-http eyre-id 200 + :- 'application/json' + %- en:json:html + (response:enjs:buckets-json [%snapshot flag u.state.u.sp]) + :: + [%request @ ~] + =/ got=(each @uv tang) (mule |.((slav %uv i.t.pax))) + ?: ?=(%| -.got) + (http-error eyre-id 400 'malformed request id') + ?~ req=(~(get by requests) p.got) + (http-error eyre-id 404 'no such request') + ?~ result.u.req + (give-response eyre-id [p.got [%pending ~]]) + (give-response eyre-id [p.got u.result.u.req]) + == +:: +:: +request-authorized: Eyre validated a session cookie. Bots would want an +:: X-Api-Key path as well — see %notes for that pattern — but nothing needs +:: it yet, so there is no key to manage. +:: +++ request-authorized + |= req=inbound-request:eyre + ^- ? + authenticated.req +:: +++ give-http + |= [eyre-id=@ta code=@ud ct=@t body=@t] + ^+ cor + =/ data=octs (as-octs:mimes:html body) + %- emil + :~ [%give %fact [/http-response/[eyre-id]]~ %http-response-header !>(`response-header:http`[code ~[['content-type' ct]]])] + [%give %fact [/http-response/[eyre-id]]~ %http-response-data !>(`data)] + [%give %kick [/http-response/[eyre-id]]~ ~] + == +:: +++ http-error + |= [eyre-id=@ta code=@ud message=@t] + ^+ cor + (give-http eyre-id code 'text/plain' message) +:: +++ give-response + |= [eyre-id=@ta res=req-response:b] + ^+ cor + %^ give-http eyre-id 200 + ['application/json' (en:json:html (req-response:enjs:buckets-json res))] +:: +:: +track-request: start tracking a request, sweeping settled ones as we go. +:: The map only grows here, so sweeping on insert bounds it without a timer. +:: +++ track-request + |= [rid=request-id:b http-id=(unit @ta)] + ^+ cor + =. requests (sweep-requests now.bowl) + =. requests (~(put by requests) rid [rid http-id ~ ~]) + cor +:: +++ sweep-requests + |= now=@da + ^- requests:b + %- malt + %+ skip ~(tap by requests) + |= [rid=request-id:b req=incoming-request:b] + ?~ final-at.req | + (gth now (add u.final-at.req request-grace)) +:: +:: +dispatch-local: a client on our own ship submitted an action. Apply it +:: if we host the bucket, otherwise forward it to the host and wait. +:: +++ dispatch-local + |= [rid=request-id:b act=a-buckets:b] + ^+ cor + =/ paths=(list path) ~[/v1/requests] + ?- -.act + %create + ?. ?=(%duke (clan:title ship.group.act)) + (deny rid paths %invalid-input 'only a planet may host a bucket') + ?. (group-is-admin-for-create group.act our.bowl) + (deny rid paths %not-authorized 'only a group admin may create a bucket') + ?. =(ship.group.act our.bowl) + (forward rid act ship.group.act) + =. cor + (create-bucket name.act title.act group.act readers.act writers.act our.bowl) + (settle rid paths) + :: + %bucket + ?~ sp=(~(get by spaces) flag.act) + (deny rid paths %not-found 'no such bucket') + ?. =(%pub net.u.sp) + (forward rid act ship.flag.act) + =/ st=bucket-state:b (need-state flag.act) + ?. (action-authorized st flag.act our.bowl a-bucket.act) + (deny rid paths %not-authorized 'not authorized for this bucket') + =. cor (apply-bucket flag.act a-bucket.act our.bowl rid) + (settle rid paths) + == +:: +:: +dispatch-remote: a subscriber forwarded a command to us as host. The +:: actor is src.bowl; the answer goes back on that ship's request path. +:: +++ dispatch-remote + |= [rid=request-id:b act=a-buckets:b] + ^+ cor + =/ paths=(list path) ~[(host-req-path src.bowl rid)] + ?- -.act + %create + ?> =(ship.group.act our.bowl) + ?. ?=(%duke (clan:title ship.group.act)) + (deny rid paths %invalid-input 'only a planet may host a bucket') + ?. (group-is-admin-for-create group.act src.bowl) + (deny rid paths %not-authorized 'only a group admin may create a bucket') + =. cor + (create-bucket name.act title.act group.act readers.act writers.act src.bowl) + (settle rid paths) + :: + %bucket + ?> =(ship.flag.act our.bowl) + ?~ sp=(~(get by spaces) flag.act) + (deny rid paths %not-found 'no such bucket') + =/ st=bucket-state:b (need-state flag.act) + ?. (action-authorized st flag.act src.bowl a-bucket.act) + (deny rid paths %not-authorized 'not authorized for this bucket') + =. cor (apply-bucket flag.act a-bucket.act src.bowl rid) + (settle rid paths) + == +:: +:: +forward: hand a command to the authoritative host, subscribe for its +:: answer, and arm a timeout so a silent host can't strand the client. +:: +++ forward + |= [rid=request-id:b act=a-buckets:b host=ship] + ^+ cor + =/ until=@da (add now.bowl request-timeout) + :: Only a read-token request gets a bucket recorded: it is the only answer + :: that is filed anywhere, and the only failure that should touch a token. + =/ token-for=(unit flag:b) + ?. ?=(%bucket -.act) ~ + ?. ?=(%issue-bucket-read -.a-bucket.act) ~ + `flag.act + =. pending (~(put by pending) rid [host until token-for]) + :: Watch first, then poke. Cards are delivered in order, so a host that + :: answers in the same event it is poked would publish the terminal fact + :: before we are listening, and the request would sit until its timeout. + =. cor + %- emit + :* %pass (req-watch-wire host rid) %agent [host %buckets] + %watch (host-req-path our.bowl rid) + == + =. cor + %- emit + :* %pass (req-poke-wire host rid) %agent [host %buckets] + %poke buckets-command-1+!>(`command:b`[rid act]) + == + =. cor + (emit [%pass (req-wake-wire host rid) %arvo %b %wait until]) + (respond rid ~[/v1/requests] [%pending ~]) +:: +:: +settle: emit the terminal response for a request, defaulting to %ok when +:: the applied action produced nothing to hand back. +:: +++ settle + |= [rid=request-id:b paths=(list path)] + ^+ cor + =/ body=response-body:b ?~(reply [%ok ~] u.reply) + =. cor cor(reply ~) + (respond rid paths body) +:: +++ deny + |= [rid=request-id:b paths=(list path) type=action-error:b msg=@t] + ^+ cor + (respond rid paths [%error type msg]) +:: +:: +respond: emit an answer on the subscription paths, and complete a held +:: HTTP request if this one came in over Eyre. +:: +:: %pending is not terminal — a held request keeps waiting for the host's +:: real answer, which arrives on the same rid. +:: +++ respond + |= [rid=request-id:b paths=(list path) body=response-body:b] + ^+ cor + =/ res=req-response:b [rid body] + =. cor (give [%fact paths buckets-req-response-1+!>(res)]) + ?: ?=(%pending -.body) cor + ?~ req=(~(get by requests) rid) cor + =. requests + %+ ~(put by requests) rid + u.req(result `body, final-at `now.bowl, http-id ~) + ?~ http-id.u.req cor + (give-response u.http-id.u.req res) +:: +++ host-req-path + |= [who=ship rid=request-id:b] + ^- path + /v1/request/(scot %p who)/(scot %uv rid) +:: +:: +answer-paths: where the answer to one requester's action goes. +:: +:: The same choice +dispatch-local and +dispatch-remote make, pulled out for +:: arms that finish a request in a later event than the one that took it. +:: +++ answer-paths + |= [who=ship rid=request-id:b] + ^- (list path) + ?: =(who our.bowl) ~[/v1/requests] + ~[(host-req-path who rid)] +:: +++ req-poke-wire + |= [host=ship rid=request-id:b] + ^- wire + /buckets/req/(scot %p host)/(scot %uv rid)/poke +:: +++ req-watch-wire + |= [host=ship rid=request-id:b] + ^- wire + /buckets/req/(scot %p host)/(scot %uv rid)/watch +:: +++ req-wake-wire + |= [host=ship rid=request-id:b] + ^- wire + /buckets/req/(scot %p host)/(scot %uv rid)/wake +:: +:: +request-live: is a forwarded request still waiting on the host? +:: +:: Tracked in state rather than read off wex.bowl, because Gall has already +:: dropped the subscription by the time a %kick reaches us — the one case +:: where we most need to know the request was still outstanding. +:: +++ request-live + |= rid=request-id:b + ^- ? + (~(has by pending) rid) +:: +:: +close-request: retire a settled request, dropping its subscription and +:: cancelling the timeout at the instant it was armed for. +:: +:: +abandon-request: end a forwarded request that will never be answered. +:: +:: There are four ways one dies -- the host kicks the stream, refuses the +:: watch, nacks the poke, or never answers at all -- and they lose the same +:: thing, so they end the same way. A renewal in particular has already had +:: its refresh fire, and nothing else will rearm it, so the local scry would +:: go on serving a token past its expiry. Handling these separately is how +:: one of the four came to be missing it. +:: +++ abandon-request + |= [host=ship rid=request-id:b why=@t] + ^+ cor + ?. (request-live rid) cor + =/ token-for=(unit flag:b) + ?~(got=(~(get by pending) rid) ~ token-for.u.got) + =. cor (close-request host rid) + =? cor ?=(^ token-for) (retry-read-token u.token-for) + (deny rid ~[/v1/requests] %unknown why) +:: +++ close-request + |= [host=ship rid=request-id:b] + ^+ cor + =/ got=(unit [host=ship until=@da token-for=(unit flag:b)]) + (~(get by pending) rid) + =. pending (~(del by pending) rid) + =. cor + %- emit + :* %pass (req-watch-wire host rid) %agent [host %buckets] + %leave ~ + == + ?~ got cor + (emit [%pass (req-wake-wire host rid) %arvo %b %rest until.u.got]) +:: +++ need-space + |= =flag:b + ^- space:b + (~(got by spaces) flag) +:: +++ need-state + |= =flag:b + ^- bucket-state:b + =/ sp=space:b (need-space flag) + ?> ?=(^ state.sp) + u.state.sp +:: +++ put-state + |= [=flag:b st=bucket-state:b] + ^+ cor + =/ sp=space:b (need-space flag) + =. spaces (~(put by spaces) flag [net.sp `st `group.st]) + cor +:: +++ create-bucket + |= [name=@tas title=@t group=flag:b readers=(set @tas) writers=(set @tas) actor=ship] + ^+ cor + ?> =(ship.group our.bowl) + =/ =flag:b [our.bowl name] + :: A create naming a bucket we already have is either the same create + :: arriving twice -- a retry, a forward that raced its own answer -- or a + :: real collision on the name. The first re-registers, which is what makes + :: create idempotent; the second is bad input from a client, and crashing + :: the event on ?> made it a nack or a timeout rather than the typed error + :: the contract promises. .actor is deliberately not compared: an + :: otherwise identical create from a different admin is still the same + :: create, and demanding the original creator only turned a retry into a + :: crash. + ?: (~(has by spaces) flag) + =/ taken=_cor (answer [%error %invalid-input 'that bucket name is taken']) + =/ sp=space:b (~(got by spaces) flag) + ?. =(%pub net.sp) taken + ?~ state.sp taken + =/ st=bucket-state:b u.state.sp + ?. ?& =(group group.st) + =(title title.bucket.st) + =(writers writers.st) + == + taken + =. cor (register-bucket flag st readers) + (give [%fact ~[/v1] buckets-response-1+!>(`response:b`[%snapshot flag st])]) + =/ id=@ud +(next-id) + =. next-id id + =/ buc=bucket:b [id title actor now.bowl actor now.bowl] + =/ st=bucket-state:b [buc group writers ~ 0] + =. spaces (~(put by spaces) flag [%pub `st `group]) + =. cor (register-bucket flag st readers) + (give [%fact ~[/v1] buckets-response-1+!>(`response:b`[%snapshot flag st])]) +:: +:: +register-bucket: hand the channel to %groups. The reader roles come from +:: the action rather than from state, because %groups keeps them from here +:: on and this agent has no copy to drift from. +:: +++ register-bucket + |= [=flag:b st=bucket-state:b readers=(set @tas)] + ^+ cor + =/ channel=group-channel:b + [[title.bucket.st '' '' ''] now.bowl %default readers |] + =/ add=group-create:b + [%group group.st %channel [%buckets flag] %add channel] + %- emit + :* %pass /buckets/(scot %p ship.flag)/[name.flag]/create + %agent [our.bowl %groups] + %poke group-action-4+!>(add) + == +:: +:: +apply-bucket: run one verb against a bucket we host. The flag and the +:: actor come from the envelope rather than from the payload. +:: +++ apply-bucket + |= [=flag:b act=a-bucket:b actor=ship rid=request-id:b] + ^+ cor + ?- -.act + %delete (delete-bucket flag actor) + %set-title (set-title flag title.act actor) + %set-writers (set-writers flag writers.act actor) + %create-folder (create-folder flag parent.act name.act actor) + %begin-upload (begin-upload flag parent.act name.act mime.act size.act checksum.act actor) + %cancel-upload (cancel-upload flag session.act reason.act actor) + %issue-bucket-read (issue-read-token flag actor `rid) + %issue-delete (issue-delete-capability flag id.act actor) + %entry (apply-entry flag id.act a-entry.act actor) + == +:: +++ apply-entry + |= [=flag:b id=@ud act=a-entry:b actor=ship] + ^+ cor + ?- -.act + %rename (rename-entry flag id name.act actor) + %move (move-entry flag id parent.act actor) + %delete (delete-entry flag id recursive.act actor) + == +:: +++ delete-bucket + |= [=flag:b actor=ship] + ^+ cor + =/ st=bucket-state:b (need-state flag) + =/ del=group-channel-del:b + [%group group.st %channel [%buckets flag] %del ~] + =. cor + %- emit + :* %pass /buckets/(scot %p ship.flag)/[name.flag]/delete + %agent [our.bowl %groups] + %poke group-action-4+!>(del) + == + =/ res=response:b + [%update flag +(revision.st) [%delete ~]] + =. cor (give [%fact ~[/v1 (updates-path flag)] buckets-response-1+!>(res)]) + =. sessions (drop-bucket-sessions flag) + =. cor (drop-read-token flag) + =. spaces (~(del by spaces) flag) + cor +:: +++ set-title + |= [=flag:b title=@t actor=ship] + ^+ cor + =/ st=bucket-state:b (need-state flag) + =. bucket.st + bucket.st(title title, updated-by actor, updated-at now.bowl) + (commit-update flag st [%meta bucket.st] actor) +:: +++ set-writers + |= [=flag:b writers=(set @tas) actor=ship] + ^+ cor + =/ st=bucket-state:b (need-state flag) + =. writers.st writers + (commit-update flag st [%writers writers] actor) +:: +++ create-folder + |= [=flag:b parent=(unit @ud) name=@t actor=ship] + ^+ cor + =/ st=bucket-state:b (need-state flag) + ?. (valid-parent st parent) + (answer [%error %not-found 'no such parent folder']) + =/ id=@ud +(next-id) + =. next-id id + =/ ent=entry:b + [id parent name actor now.bowl actor now.bowl [%folder ~]] + =. entries.st (~(put by entries.st) id ent) + (commit-update flag st [%entry id [%create ent]] actor) +:: +:: +begin-upload: reserve an entry id and object key, open a host-private +:: session, and hand the session id back to the uploader as its broker +:: token. The entry is not published until the object lands, so nothing is +:: broadcast here and the token never leaves this response. +:: +++ begin-upload + |= $: =flag:b + parent=(unit @ud) + name=@t + mime=@t + size=@ud + checksum=(unit @t) + actor=ship + == + ^+ cor + =/ st=bucket-state:b (need-state flag) + ?. (valid-parent st parent) + (answer [%error %not-found 'no such parent folder']) + ?: =(0 size) + (answer [%error %invalid-input 'file size must be greater than zero']) + ?: (gth size max-object-size) + (answer [%error %invalid-input 'file exceeds the maximum object size']) + ?. (valid-mime mime) + (answer [%error %invalid-input 'missing or malformed content type']) + =. cor prune-broker-authority + =/ id=@ud +(next-id) + =. next-id id + =/ sid=@uv `@uv`eny.bowl + =/ fil=file:b [mime size checksum (scot %uv sid) %pending] + =/ ent=entry:b + [id parent name actor now.bowl actor now.bowl [%file fil]] + =/ expiry=@da (add now.bowl upload-window) + =/ ses=upload-session:b + [sid flag ent actor now.bowl expiry %pending ~ ~] + =. sessions (~(put by sessions) sid ses) + (answer [%grant [(scot %uv sid) id expiry]]) +:: +:: +publish-upload: move a completed session's entry into the manifest and +:: broadcast it. The session is retained as %complete so a repeated +:: completion is a no-op rather than a second entry. +:: +++ publish-upload + |= [ses=upload-session:b actor=ship] + ^+ cor + =/ st=bucket-state:b (need-state flag.ses) + =/ ent=entry:b entry.ses + =/ fil=file:b (entry-file ent) + =. fil fil(status %ready) + =. ent ent(updated-by actor, updated-at now.bowl, kind [%file fil]) + =. sessions (~(put by sessions) id.ses ses(status %complete, entry ent)) + =. entries.st (~(put by entries.st) id.ent ent) + (commit-update flag.ses st [%entry id.ent [%create ent]] actor) +:: +:: +cancel-upload: the uploader is withdrawing from a session it opened. +:: +:: Withdrawing is all it can report. Whether the bytes reached storage is the +:: broker's to say, and the client asks that question and can lose the +:: answer -- so this does not settle the upload, it only stops a new upload +:: URL being issued against the session. A completion that arrives afterwards +:: is still honoured, because the broker knows something we do not. +:: +++ cancel-upload + |= [=flag:b sid=@uv reason=@t actor=ship] + ^+ cor + ?~ got=(~(get by sessions) sid) + (answer [%error %not-found 'no such upload session']) + =/ ses=upload-session:b u.got + ?. =(flag flag.ses) + (answer [%error %not-found 'no such upload session']) + ?. =(%pending status.ses) + (answer [%error %invalid-input 'upload session is not pending']) + ?. =(requested-by.ses actor) + (answer [%error %not-authorized 'not the uploader']) + :: Nothing was published, so there is nothing to broadcast. + =. sessions (~(put by sessions) sid ses(status %cancelled, error `reason)) + cor +:: +:: +held-read-token: a live token we have already minted for this reader. +:: +:: Keyed by reader as well as bucket: every reader gets its own token, so +:: that one reader can be revoked without disturbing the rest. Looking this +:: up by bucket alone would hand a remote reader whichever token came first, +:: and make per-reader revocation impossible. +:: +++ held-read-token + |= [=flag:b actor=ship] + ^- (unit read-token:b) + ?~ got=(~(get by readers) [flag actor]) ~ + =/ sync=reader-sync:b u.got + :: Only a grant the broker has confirmed is worth handing out; anything + :: still owed would 403 on first use. + ?. ?=(%settled (reader-status sync)) ~ + ?. ?=(%granted -.desired.sync) ~ + ?. (gth expires-at.desired.sync (add now.bowl token-margin)) ~ + `[token.desired.sync expires-at.desired.sync] +:: +:: +issue-read-token: mint a reader's bucket-read token, reusing the one it +:: already holds while that has useful life left. +:: +:: One token covers every ready object in the bucket. What that saves is the +:: host: the broker answers a read from its own table, so opening a file +:: costs nothing here and keeps working while this ship is down. The client +:: still exchanges the token at the broker once per file it opens, because a +:: signed URL is per-object, short-lived, and only the broker can sign one. +:: +:: A fresh mint answers %pending: the token has to reach the broker before +:: it is worth anything, +:: and a client that is told otherwise would hold one that 403s. +:: +++ issue-read-token + |= [=flag:b actor=ship rid=(unit request-id:b)] + ^+ cor + =. cor prune-broker-authority + ?^ held=(held-read-token flag actor) + (answer [%token u.held]) + =/ token=@t (scot %uv `@uv`eny.bowl) + =/ expiry=@da (add now.bowl read-window) + :: Record the grant whether or not we can send it right now. Without the + :: %genuine secret +sync-cards emits nothing, but the pair is owed and the + :: retry timer sends it once the secret appears -- where bailing out here + :: left a renewal dead for good, its refresh already fired and nothing to + :: rearm it. + :: + :: Bound to a leg before the test on purpose: ?~ on a bare arm refines + :: along the wing's axis, an arm has none, and the ~ case mints as vain. + =/ secret=(unit @t) genuine-secret + =/ st=bucket-state:b (need-state flag) + :: Only claim the request as this pair's waiter if we are going to send. + :: With no secret it is answered below, in this event, and a record still + :: naming it would answer it again when the retry is finally confirmed. + =/ waiter=(unit request-id:b) ?~(secret ~ rid) + =. cor + %- sync-reader + [flag actor (scot %ud id.bucket.st) [%granted token expiry] expiry waiter] + ?^ secret (answer [%pending ~]) + :: A client should not be left holding a request we cannot act on yet, so + :: it is told; the timer path has no one waiting and just retries. + %- (slog leaf+"buckets: no %genuine secret, reader sync deferred" ~) + ?~ rid cor + (answer [%error %unknown 'this ship cannot reach storage yet']) +:: +:: +issue-delete-capability: mint a short-lived delete grant for one ready +:: file. Deletes stay per-object — they are destructive and unrecoverable. +:: +++ issue-delete-capability + |= [=flag:b id=@ud actor=ship] + ^+ cor + =. cor prune-broker-authority + =/ st=bucket-state:b (need-state flag) + ?~ got=(~(get by entries.st) id) + (answer [%error %not-found 'no such entry']) + =/ ent=entry:b u.got + ?. ?=(%file -.kind.ent) + (answer [%error %invalid-input 'entry is a folder']) + =/ fil=file:b +.kind.ent + ?. =(%ready status.fil) + (answer [%error %invalid-input 'file is not ready']) + =/ token=@t (scot %uv `@uv`eny.bowl) + =/ expiry=@da (add now.bowl object-window) + =. object-capabilities + (~(put by object-capabilities) token [%delete flag `id actor expiry]) + (answer [%grant [token id expiry]]) +:: +:: +arm-token-refresh: re-mint before the current token lapses, so a local +:: client never has to wait on one. +:: +++ arm-token-refresh + |= [=flag:b expiry=@da] + ^+ cor + %- emit + :* %pass (token-wire flag) %arvo %b + %wait (sub expiry token-margin) + == +:: +:: +disarm-token-refresh: cancel the refresh armed for the token we hold. +:: +:: Behn keys a timer by its instant, so this has to name the one +:: +arm-token-refresh set, which is derived from the held token's expiry. +:: Dropping a token and taking another before that instant otherwise leaves +:: the old wake live: it fires alongside the replacement's, and the two +:: renewals supersede each other at the host for as long as the bucket lives. +:: +++ disarm-token-refresh + |= =flag:b + ^+ cor + ?~ tok=(~(get by read-tokens) flag) cor + %- emit + :* %pass (token-wire flag) %arvo %b + %rest (sub expires-at.u.tok token-margin) + == +:: +:: +recover-local-reader: our own renewal has stopped making progress. +:: +:: A renewal has no waiting request, so nothing else reports its failure and +:: nothing else restarts it. The token we still hold was minted against a +:: revision the broker never took, or has run out its life while we retried, +:: and the local scry will keep serving it until something intervenes. Drop +:: it so a reader asks again, and come back for a fresh mint. +:: +++ recover-local-reader + |= =flag:b + ^+ cor + :: Holding a token is the whole reason to act: it is the stale thing being + :: served, and dropping it is what makes a reader ask again. With none + :: there is nothing to recover and the next request mints on its own -- + :: and saying so here is what keeps this idempotent, because the sweep + :: sees the same lapsed record on every pass until it is pruned, and an + :: unguarded +retry-read-token would leave a timer behind each time. + ?~ (~(get by read-tokens) flag) cor + =. cor (disarm-token-refresh flag) + =. read-tokens (~(del by read-tokens) flag) + (retry-read-token flag) +:: +++ token-wire + |= =flag:b + ^- wire + /buckets/token/(scot %p ship.flag)/[name.flag] +:: +:: +retry-read-token: come back to a mint the broker refused. +:: +++ retry-read-token + |= =flag:b + ^+ cor + %- emit + :* %pass (token-wire flag) %arvo %b + %wait (add now.bowl push-retry) + == +:: +:: +set-broker-base: point this ship's syncs at a different broker. +:: +:: Refuses anything but an https origin. The credential +sync-cards sends is +:: a bearer header, so a base naming a plaintext or unexpected host does not +:: fail closed -- it discloses the secret to whoever was named. One trailing +:: slash is trimmed rather than refused, since every use appends its own path +:: and a doubled slash would 404 against a broker that is otherwise right. +:: +++ set-broker-base + |= base=(unit @t) + ^+ cor + ?~ base + %- (slog leaf+"buckets: broker base reset to the default" ~) + :: Going back is a move between brokers like any other: the default has + :: heard nothing we said while we were pointed elsewhere. + (rebase-readers default-broker-base) + =/ txt=tape (trip u.base) + :: Indexed rather than +rear/+snip on purpose: testing with ?= narrows the + :: tape, and those wet gates do not survive being handed a narrowed list. + =. txt + ?: =(~ txt) txt + =/ last=@ud (dec (lent txt)) + ?.(=('/' (snag last txt)) txt (scag last txt)) + ?. =("https://" (scag 8 txt)) + %- (slog leaf+"buckets: refusing a broker base that is not https" ~) + cor + %- (slog leaf+"buckets: broker base is now {txt}" ~) + (rebase-readers (crip txt)) +:: +:: +rebase-readers: point every live grant at the broker we just moved to. +:: +:: A broker holds only what it has been told. Swapping the address alone +:: leaves every record reading as synced, so +owed skips them and the new +:: broker learns nothing until each grant renews -- a day of reads failing +:: against a broker that has never heard of them. Marking them owed again +:: re-sends the state we already decided; revisions carry over, and a broker +:: with no record of a pair accepts any revision above zero. +:: +:: What this does not do is retire the grants the old broker still holds. +:: Doing so means keeping the old address and revoking against it, which is +:: a second broker's worth of bookkeeping for an operator action; their own +:: expiry is the backstop, which is the same guarantee a missed revoke has. +:: +++ rebase-readers + |= base=@t + ^+ cor + ?: =(base broker-base) cor + =. broker-base base + =. readers + %- malt + %+ turn ~(tap by readers) + |= [key=reader-key:b sync=reader-sync:b] + ^- [reader-key:b reader-sync:b] + :: Nothing to re-send for a pair whose token could not be used anyway. + ?: ?=(%lapsed (reader-status sync)) [key sync] + :: A new revision rather than the same one resent, because a request to + :: the broker we just left may still be in flight and its wire carries + :: the revision. Reusing it would let that broker's late 2xx confirm + :: state the new broker has never been told, and +owed would then stop + :: retrying it -- clients failing against the new broker until renewal. + [key sync(revision +(revision.sync), synced 0, failed |)] + retry-readers +:: +:: +genuine-secret: this ship's shared secret with the broker. +:: +:: %genuine mints it and serves it back over its own Eyre binding, which is +:: how the broker checks a request really came from us. Absent until %genuine +:: has initialised, which is a real state on a fresh ship rather than a bug, +:: so this answers a unit instead of crashing the event. +:: +++ genuine-secret + ^- (unit @t) + ?. .^(? %gu /(scot %p our.bowl)/genuine/(scot %da now.bowl)/$) ~ + =/ jon=json + .^(json %gx /(scot %p our.bowl)/genuine/(scot %da now.bowl)/secret/json) + ?. ?=([%s *] jon) ~ + `p.jon +:: +:: +answer-waiter: give a reader record's held request its one terminal +:: answer, and stop holding it. +:: +:: `awaiting` is a promise that exactly one answer is still owed on this +:: pair. Every transition that resolves or abandons that promise goes +:: through here, so the clearing and the answering cannot drift apart -- +:: doing them separately is how a request came to be answered twice. +:: +++ answer-waiter + |= [key=reader-key:b body=response-body:b] + ^+ cor + ?~ got=(~(get by readers) key) cor + ?~ awaiting.u.got cor + =/ rid=request-id:b u.awaiting.u.got + =. readers (~(put by readers) key u.got(awaiting ~)) + (respond rid (answer-paths reader.key rid) body) +:: +:: +sync-reader: record what a reader's access should be, and tell the +:: broker. Grant, rotation and revoke are all this one operation. +:: +:: The revision is what makes delivery order stop mattering: the broker keeps +:: only the highest it has seen, so a delayed or duplicated request loses to +:: the truth rather than overwriting it. That is why a revoke can be sent +:: while a grant is still in flight, and why a retry of that grant is +:: harmless when it lands afterwards. +:: +++ sync-reader + |= $: =flag:b + reader=ship + bucket-id=@t + desired=reader-state:b + expires=@da + awaiting=(unit request-id:b) + == + ^+ cor + =/ key=reader-key:b [flag reader] + =/ prior=(unit reader-sync:b) (~(get by readers) key) + =/ revision=@ud ?~(prior 1 +(revision.u.prior)) + =/ synced=@ud ?~(prior 0 synced.u.prior) + :: A client still waiting on the grant this supersedes will never be + :: answered by it -- the broker will keep the newer state -- so tell it + :: now rather than leaving it to time out. + =/ stale=(unit request-id:b) ?~(prior ~ awaiting.u.prior) + =? cor !=(awaiting stale) + %+ answer-waiter key + [%error %not-authorized 'access changed while the token was being issued'] + =. readers + (~(put by readers) key [revision bucket-id desired expires synced | awaiting]) + =. cor (emil (sync-cards ~[[key revision bucket-id desired]])) + :: Unconditionally: one timer walks the whole owed set, and +arm-reader-retry + :: is what keeps repeated arming from meaning repeated timers. + arm-reader-retry +:: +:: +reader-status: the one place a record's state is decided. +:: +:: Expiry dominates everything else: past it the token the record names can no +:: longer be used, so there is nothing left to owe, serve or retry whatever +:: the revisions say. A refusal settles it next -- the broker will answer the +:: same way again -- then being level with the broker, and anything else is +:: still owed. +:: +++ reader-status + |= sync=reader-sync:b + ^- reader-status:b + ?: (lte expires.sync now.bowl) %lapsed + ?: failed.sync %refused + ?: (gte synced.sync revision.sync) %settled + %owed +:: +:: +owed: pairs the broker has not caught up with. +:: +++ owed + ^- (list [key=reader-key:b revision=@ud bucket-id=@t desired=reader-state:b]) + %+ murn ~(tap by readers) + |= [key=reader-key:b sync=reader-sync:b] + ^- (unit [reader-key:b @ud @t reader-state:b]) + ?. ?=(%owed (reader-status sync)) ~ + `[key revision.sync bucket-id.sync desired.sync] +:: +:: +sync-cards: one request per pair. The credential goes in a header: a +:: query string lands in access logs, and so would a bearer token in a path. +:: +++ sync-cards + |= $: wants=(list [key=reader-key:b revision=@ud bucket-id=@t desired=reader-state:b]) + == + ^- (list card) + ?~ wants ~ + =/ secret=(unit @t) genuine-secret + ?~ secret + %- (slog leaf+"buckets: no %genuine secret, cannot sync readers" ~) + ~ + %+ turn wants + |= [key=reader-key:b revision=@ud bucket-id=@t desired=reader-state:b] + ^- card + :: Everything the request needs is on the record. Rebuilding it from live + :: bucket state would make a revoke undeliverable exactly when it matters + :: most -- the bucket has been deleted and its objects still exist. + =/ common=(list [@t json]) + :~ ['bucketHost' s+(ship-text ship.flag.key)] + ['bucketName' s+(scot %tas name.flag.key)] + ['bucketId' s+bucket-id] + ['actorShip' s+(ship-text reader.key)] + ['revision' (numb:enjs:format revision)] + == + :: A revoke sends both fields as null rather than omitting them: the broker + :: reads the pair (token, expiresAtMillis) to tell a revoke from a grant, + :: and an explicit null says so without depending on how absent keys decode. + =/ body=@t + %- en:json:html + %- pairs:enjs:format + ?- -.desired + %revoked + :* ['token' ~] + ['expiresAtMillis' ~] + common + == + :: + %granted + :* ['token' s+token.desired] + :- 'expiresAtMillis' + (numb:enjs:format (mul 1.000 (unt:chrono:userlib expires-at.desired))) + common + == + == + =/ url=@t + (rap 3 broker-base '/tokens/' (ship-text our.bowl) ~) + =/ =request:http + :* %'PUT' url + :~ ['content-type' 'application/json'] + ['x-landscape-token' u.secret] + == + `[(met 3 body) body] + == + :* %pass (reader-wire key revision) %arvo %i + %request request *outbound-config:iris + == +:: +:: +broker-revision: the revision the broker says it holds, if it said. +:: +:: It only matters when it is ahead of ours; a body we cannot parse simply +:: tells us nothing, which is not an error. +:: +++ broker-body + |= res=client-response:iris + ^- (unit (map @t json)) + ?. ?=(%finished -.res) ~ + ?~ full-file.res ~ + ?~ jon=(de:json:html q.data.u.full-file.res) ~ + ?. ?=([%o *] u.jon) ~ + `p.u.jon +:: +:: +broker-applied: whether the broker took the write, as it reported it. +:: +:: The receipt says so outright, and inferring it from revisions instead gets +:: the equal case wrong: a reader whose record was pruned at its expiry opens +:: again at revision 1 while the broker still retains 1, which it answers 200 +:: and does not apply. A body we cannot read tells us nothing, and the +:: revision comparison remains the fallback. +:: +++ broker-applied + |= res=client-response:iris + ^- (unit ?) + ?~ body=(broker-body res) ~ + ?~ got=(~(get by u.body) 'applied') ~ + ?. ?=([%b *] u.got) ~ + `p.u.got +:: +++ broker-revision + |= res=client-response:iris + ^- (unit @ud) + ?~ body=(broker-body res) ~ + ?~ got=(~(get by u.body) 'currentRevision') ~ + ?. ?=([%n *] u.got) ~ + `(rash p.u.got dem) +:: +:: +broker-retryable: whether the broker says another attempt could work. +:: +:: It marks a validation failure retryable:false and a service failure +:: retryable:true. Absent, we assume it is worth another go -- a transport +:: failure carries no body at all, and those are exactly the retryable ones. +:: +++ broker-retryable + |= res=client-response:iris + ^- ? + ?~ body=(broker-body res) & + ?~ got=(~(get by u.body) 'retryable') & + ?. ?=([%b *] u.got) & + p.u.got +:: +++ reader-wire + |= [key=reader-key:b revision=@ud] + ^- wire + %+ weld + /buckets/reader/(scot %p ship.flag.key)/[name.flag.key] + /(scot %p reader.key)/(scot %ud revision) +:: +:: +reader-retry-at: the instant the retry timer may next fire. +:: +:: Snapped to a fixed grid instead of now-plus-an-interval, so every arming +:: that happens close together names the same instant. That is what lets +:: +arm-reader-retry cancel before it sets. +:: +++ reader-retry-at + ^- @da + (add (sub now.bowl (mod now.bowl push-retry)) (mul 2 push-retry)) +:: +:: +arm-reader-retry: make sure a sweep of what is owed is coming. +:: +:: Cancel before setting, because guarding on "is anything owed right now" +:: does not bound this: a sync that confirms in milliseconds leaves the +:: timer armed and the next change finds nothing owed and arms another. +:: Harmless while they wake to no work -- but if an outage begins first, +:: every one of them re-sends the whole owed set and rearms, and the +:: duplication becomes permanent. Two grid slots can be live at once, which +:: is the most this can drift to. +:: +++ arm-reader-retry + ^+ cor + =/ at=@da reader-retry-at + =. cor (emit [%pass /buckets/reader-retry %arvo %b %rest at]) + (emit [%pass /buckets/reader-retry %arvo %b %wait at]) +:: +:: +retry-readers: re-send everything still owed. +:: +:: Blind retries are safe here — a stale one loses to the revision the broker +:: already holds — so this needs no bookkeeping beyond what is owed. +:: +++ retry-readers + ^+ cor + :: A pair drops out of +owed at its expiry whether or not it ever landed, + :: and this arm then returns without rearming, so nothing else revisits it. + :: What that strands depends on whose reader it is: ours has an expired + :: token still installed and a renewal loop that has quietly stopped, while + :: a subscriber's has a request waiting for an answer that is no longer + :: coming. Neither has anyone else to notice. + =. cor + %+ roll ~(tap by readers) + |= [[key=reader-key:b sync=reader-sync:b] acc=_cor] + ?. ?=(%lapsed (reader-status sync)) acc + ?: =(reader.key our.bowl) (recover-local-reader:acc flag.key) + %+ answer-waiter:acc key + [%error %unknown 'storage did not take this grant before it lapsed'] + =/ wants owed + ?~ wants cor + =. cor (emil (sync-cards wants)) + arm-reader-retry +:: +:: +confirm-reader: the broker has caught up to `revision` for this pair. +:: +:: If it reports a higher revision than we sent, our counter is behind its -- +:: state loss on our side, or a message from an earlier incarnation. Adopt +:: its number and re-send, so our desired state wins rather than being +:: silently discarded as stale forever. +:: +:: +fail-reader: the broker refused this revision as invalid. +:: +++ fail-reader + |= [key=reader-key:b sent=@ud] + ^+ cor + ?~ got=(~(get by readers) key) cor + =/ sync=reader-sync:b u.got + :: Only the revision we sent; a newer one may still be in flight. + ?. =(sent revision.sync) cor + =. readers (~(put by readers) key sync(failed &)) + :: Nothing is owed for a failed revision, so for our own reader this is + :: where the renewal loop would otherwise stop for good. + =? cor =(reader.key our.bowl) (recover-local-reader flag.key) + %+ answer-waiter key + [%error %unknown 'storage refused this access change'] +:: +++ confirm-reader + |= [key=reader-key:b sent=@ud theirs=(unit @ud) applied=(unit ?)] + ^+ cor + ?~ got=(~(get by readers) key) cor + =/ sync=reader-sync:b u.got + :: Whether this ack tells us anything we did not already know. A repeat + :: delivery must not re-install or re-arm anything. + =/ advanced=? (gth sent synced.sync) + =? sync advanced sync(synced sent) + :: The broker did not take this write, so what we asked for is not what it + :: holds however the numbers compare. Adopt its revision and re-send above + :: it, or our desired state is discarded as stale from here on. + :: + :: Its own report is the authority, not the comparison: a reader whose + :: record was pruned at its expiry opens again at revision 1 while the + :: broker still retains 1, and a strictly-greater test reads that as + :: agreement -- the client is then handed a token the broker never stored. + :: Where it says nothing, being behind is the only case we can detect. + =/ stale=? + ?^ applied !u.applied + ?&(?=(^ theirs) (gth u.theirs revision.sync)) + ?: ?&(?=(^ theirs) stale) + :: Above what it kept, so the resend cannot tie with it again. + =. sync sync(revision +(u.theirs), synced u.theirs) + =. readers (~(put by readers) key sync) + %- (slog leaf+"buckets: broker was ahead of us, resending" ~) + (emil (sync-cards ~[[key revision.sync bucket-id.sync desired.sync]])) + =. readers (~(put by readers) key sync) + ?. advanced cor + :: Only once the broker is level with what we last decided -- an ack for a + :: superseded revision says nothing about the state we now want. + ?. ?=(%settled (reader-status sync)) cor + ?. ?=(%granted -.desired.sync) cor + =/ tok=read-token:b [token.desired.sync expires-at.desired.sync] + :: Installing is independent of anyone waiting: a renewal fired by the + :: refresh timer has no request behind it, and skipping it here left the + :: local scry serving the previous token until it lapsed and then forever. + =? read-tokens =(reader.key our.bowl) + (~(put by read-tokens) flag.key tok) + =? cor =(reader.key our.bowl) + (arm-token-refresh flag.key expires-at.desired.sync) + (answer-waiter key [%token tok]) +:: +:: +granted-readers: pairs `test` accepts that currently hold a grant. +:: +++ granted-readers + |= test=$-([reader-key:b reader-sync:b] ?) + ^- (list reader-key:b) + %+ murn ~(tap by readers) + |= [key=reader-key:b sync=reader-sync:b] + ^- (unit reader-key:b) + ?. ?=(%granted -.desired.sync) ~ + ?.((test key sync) ~ `key) +:: +:: +revoke-readers: move each pair to revoked at a higher revision. +:: +:: There is nothing to undo locally beyond the desired state itself: the +:: grant only ever lived here and at the broker, and a revoked record is +:: what stops +held-read-token and the read verdict honouring it. +:: +++ revoke-readers + |= keys=(list reader-key:b) + ^+ cor + %+ roll keys + |= [key=reader-key:b acc=_cor] + :: Carry the bucket id and expiry off the record being replaced, so the + :: revoke stays deliverable after the bucket itself is gone. + ?~ got=(~(get by readers.acc) key) acc + %- sync-reader:acc + [flag.key reader.key bucket-id.u.got [%revoked ~] expires.u.got ~] +:: +++ url-encode + |= txt=@t + ^- @t + (crip (en-urlt:html (trip txt))) +:: +:: +keep-read-token: store a token the host issued us, and arm its refresh. +:: +++ keep-read-token + |= [=flag:b tok=read-token:b] + ^+ cor + ?~ sp=(~(get by spaces) flag) cor + ?. =(%sub net.u.sp) cor + =. read-tokens (~(put by read-tokens) flag tok) + (arm-token-refresh flag expires-at.tok) +:: +:: +renew-read-token: keep this ship's token current without a client asking. +:: +:: Hosting a bucket means minting for ourselves; subscribing means asking the +:: host, over the same forwarding path a client action uses. +:: +++ renew-read-token + |= =flag:b + ^+ cor + ?~ sp=(~(get by spaces) flag) cor + ?: =(%pub net.u.sp) + =/ st=bucket-state:b (need-state flag) + ?. (group-can-read group.st flag our.bowl) + (drop-read-token flag) + (issue-read-token flag our.bowl ~) + (forward `@uv`eny.bowl [%bucket flag [%issue-bucket-read ~]] ship.flag) +:: +:: +drop-read-token: forget a bucket's token and revoke the capability behind +:: it. Called when we lose the bucket, and when a subscriber loses access. +:: +++ drop-read-token + |= =flag:b + ^+ cor + =. cor (disarm-token-refresh flag) + =. read-tokens (~(del by read-tokens) flag) + :: On a subscriber this finds nothing: only a host mints, so only a host + :: has anything to revoke. + %- revoke-readers + %- granted-readers + |=([key=reader-key:b sync=reader-sync:b] =(flag flag.key)) +:: +:: +prune-broker-authority: drop expired capabilities, expired pending +:: sessions, and any reservation whose session is gone. +:: +++ prune-broker-authority + ^+ cor + =. object-capabilities + %- malt + %+ skim ~(tap by object-capabilities) + |= [token=@t aut=object-capability:b] + (gth expires-at.aut now.bowl) + :: A settled session is kept only so a re-delivered completion can no-op + :: and its uploader can read the reason it failed. Both are short-lived, + :: and keeping them forever meant routine upload failures grew persisted + :: state without bound -- their entries were never published, so no + :: delete-entry could reach them either. + =. sessions + %- malt + %+ skim ~(tap by sessions) + |= [sid=@uv ses=upload-session:b] + ?: =(%pending status.ses) + (gth expires-at.ses now.bowl) + (gth (add expires-at.ses request-grace) now.bowl) + =. reservations + %- malt + %+ skim ~(tap by reservations) + |= [reservation=@t sid=@uv] + (~(has by sessions) sid) + =. read-tokens + %- malt + %+ skim ~(tap by read-tokens) + |= [=flag:b tok=read-token:b] + (gth expires-at.tok now.bowl) + :: One rule for both desired states, because past `expires` they say the + :: same nothing: the token the record names can no longer be used, so a + :: grant is worthless and a revoke is moot. Judging only the revoked ones + :: left every reader that ever held a token on record for good. + :: + :: Anything still owed stays, whatever its age, or the broker never hears + :: about it; so does anything with a request still waiting on it. Dropping + :: a settled pair is safe because the broker keeps the higher revision: a + :: later grant re-opens at 1, loses as a stale write, and +confirm-reader + :: adopts the number it is told and re-sends. + =. readers + %- malt + %+ skim ~(tap by readers) + |= [key=reader-key:b sync=reader-sync:b] + :: A record with a request still waiting on it stays until that request + :: is answered, whatever else is true of it. + ?. ?=(~ awaiting.sync) & + ?- (reader-status sync) + :: Still work to do, or still the answer to a read. + %owed & + %settled & + :: The broker refused this revision and will refuse it again, but the + :: record still stands until its own expiry says otherwise. + %refused & + :: Nothing left to owe, serve or retry. + %lapsed | + == + cor +:: +++ drop-bucket-sessions + |= =flag:b + ^- (map @uv upload-session:b) + %- malt + %+ skip ~(tap by sessions) + |= [sid=@uv ses=upload-session:b] + =(flag flag.ses) +:: +:: +session-token: resolve the opaque string Memex presents back to the +:: session that minted it. +:: +++ session-token + |= token=@t + ^- (unit upload-session:b) + ?~ sid=(slaw %uv token) ~ + (~(get by sessions) u.sid) +:: +++ apply-broker-command + |= cmd=broker-command:b + ^+ cor + ?- -.cmd + %authorize-upload + (authorize-broker-upload capability.cmd broker-reservation-id.cmd) + :: + %complete-upload + (complete-broker-upload broker-receipt.cmd) + == +:: +++ authorize-broker-upload + |= [token=@t reservation=@t] + ^+ cor + ?~ got=(session-token token) cor + =/ ses=upload-session:b u.got + ?. =(%pending status.ses) cor + ?. (gth expires-at.ses now.bowl) cor + ?~ sp=(~(get by spaces) flag.ses) cor + ?~ st-unit=state.u.sp cor + =/ st=bucket-state:b u.st-unit + ?. (group-can-write group.st flag.ses writers.st requested-by.ses) cor + ?^ accepted=reservation.ses cor + ?^ occupied=(~(get by reservations) reservation) cor + =. sessions (~(put by sessions) id.ses ses(reservation `reservation)) + =. reservations (~(put by reservations) reservation id.ses) + cor +:: +:: +complete-broker-upload: the broker says the bytes landed, so publish. +:: +:: Every refusal here says why. A completion the host drops changes nothing +:: and answers nobody -- the broker has already taken payment for the object +:: -- so a silent return is the one outcome that leaves no way to find out +:: what happened. A re-delivered completion is the exception: the broker +:: retries, and the second one is expected rather than wrong. +:: +++ complete-broker-upload + |= receipt=broker-receipt:b + ^+ cor + ?~ sid=(~(get by reservations) broker-reservation-id.receipt) + (drop-completion %no-such-reservation) + ?~ got=(~(get by sessions) u.sid) (drop-completion %no-such-session) + =/ ses=upload-session:b u.got + :: Already published: the broker retried, which is fine. + ?: =(%complete status.ses) cor + :: A cancelled session still publishes. The uploader withdrew because its + :: own completion call failed, which does not mean the broker did not take + :: the bytes -- and if it did, refusing here leaves the object stored and + :: paid for with nothing in the manifest pointing at it. Every other check + :: below still has to pass, including the receipt matching this entry. + ?. ?=(?(%pending %cancelled) status.ses) + (drop-completion %session-not-pending) + ?. (gth expires-at.ses now.bowl) (drop-completion %session-expired) + ?~ sp=(~(get by spaces) flag.ses) (drop-completion %no-such-bucket) + ?~ st-unit=state.u.sp (drop-completion %bucket-state-missing) + =/ st=bucket-state:b u.st-unit + ?. (group-can-write group.st flag.ses writers.st requested-by.ses) + (drop-completion %not-a-writer) + =/ fil=file:b (entry-file entry.ses) + ?. ?& =(object-id.receipt object-key.fil) + ?| =(host.receipt (ship-text our.bowl)) + =(host.receipt (scot %p our.bowl)) + == + =(bucket-id.receipt (scot %ud id.bucket.st)) + =(size.receipt size.fil) + =(mime-type.receipt mime.fil) + == + (drop-completion %receipt-mismatch) + (publish-upload ses requested-by.ses) +:: +++ drop-completion + |= why=@tas + ^+ cor + %- (slog leaf+"buckets: dropped an upload completion, {}" ~) + cor +:: +++ rename-entry + |= [=flag:b id=@ud name=@t actor=ship] + ^+ cor + =/ st=bucket-state:b (need-state flag) + ?~ got=(~(get by entries.st) id) + (answer [%error %not-found 'no such entry']) + =/ ent=entry:b u.got + =. ent ent(name name, updated-by actor, updated-at now.bowl) + =. entries.st (~(put by entries.st) id ent) + (commit-update flag st [%entry id [%update ent]] actor) +:: +++ move-entry + |= [=flag:b id=@ud parent=(unit @ud) actor=ship] + ^+ cor + =/ st=bucket-state:b (need-state flag) + ?. (valid-parent st parent) + (answer [%error %not-found 'no such parent folder']) + ?~ got=(~(get by entries.st) id) + (answer [%error %not-found 'no such entry']) + =/ ent=entry:b u.got + ?: ?&(?=(^ parent) =(u.parent id)) + (answer [%error %invalid-input 'an entry cannot contain itself']) + ?: ?& ?=(%folder -.kind.ent) + ?=(^ parent) + (descendant st id u.parent) + == + (answer [%error %invalid-input 'a folder cannot move inside itself']) + =. ent ent(parent parent, updated-by actor, updated-at now.bowl) + =. entries.st (~(put by entries.st) id ent) + (commit-update flag st [%entry id [%update ent]] actor) +:: +++ delete-entry + |= [=flag:b id=@ud recursive=? actor=ship] + ^+ cor + =/ st=bucket-state:b (need-state flag) + ?. (~(has by entries.st) id) + (answer [%error %not-found 'no such entry']) + =/ ids=(set @ud) (descendants st id) + ?. ?|(recursive =(1 ~(wyt in ids))) + (answer [%error %invalid-input 'folder is not empty']) + =. entries.st + %- ~(rep in ids) + |= [key=@ud acc=_entries.st] + (~(del by acc) key) + =. sessions + %- malt + %+ skip ~(tap by sessions) + |= [key=@uv ses=upload-session:b] + ?. =(flag flag.ses) | + :: An in-flight upload's entry is deliberately absent from entries.st, + :: so it is never among the descendants -- but its parent can be. Drop + :: those too: otherwise its completion still authorizes and publishes an + :: entry under a folder that no longer exists, which nothing can reach. + ?| (~(has in ids) id.entry.ses) + ?& ?=(^ parent.entry.ses) + (~(has in ids) u.parent.entry.ses) + == + == + (commit-update flag st [%entries-deleted ~(tap in ids)] actor) +:: +:: +commit-update: bump the revision, stamp attribution on the bucket, and +:: broadcast. The actor is passed in rather than read from src.bowl, which on +:: a broker callback is us rather than the uploader. +:: +++ commit-update + |= [=flag:b st=bucket-state:b upd=u-bucket:b actor=ship] + ^+ cor + =. revision.st +(revision.st) + =. bucket.st + bucket.st(updated-by actor, updated-at now.bowl) + =. cor (put-state flag st) + =/ res=response:b [%update flag revision.st upd] + (give [%fact ~[/v1 (updates-path flag)] buckets-response-1+!>(res)]) +:: +++ valid-parent + |= [st=bucket-state:b parent=(unit @ud)] + ^- ? + ?~ parent & + ?~ ent=(~(get by entries.st) u.parent) | + =(%folder -.kind.u.ent) +:: +:: +valid-mime: a content type must be present and look like type/subtype. +:: Memex refuses anything else, so refuse it here before committing state. +:: +++ valid-mime + |= mime=@t + ^- ? + =/ txt=tape (trip mime) + ?~ txt | + ?~ cut=(find "/" txt) | + &(!=(0 u.cut) !=(+(u.cut) (lent txt))) +:: +++ entry-file + |= ent=entry:b + ^- file:b + ?- -.kind.ent + %folder ~|(%entry-is-a-folder !!) + %file +.kind.ent + == +:: +++ descendant + |= [st=bucket-state:b ancestor=@ud candidate=@ud] + ^- ? + =/ cur=(unit @ud) `candidate + |- + ?~ cur | + ?: =(u.cur ancestor) & + ?~ ent=(~(get by entries.st) u.cur) | + $(cur parent.u.ent) +:: +++ descendants + |= [st=bucket-state:b root=@ud] + ^- (set @ud) + ?> (~(has by entries.st) root) + =/ acc=(set @ud) (silt ~[root]) + =/ queue=(list @ud) ~[root] + |- + ?~ queue acc + =/ kids=(list @ud) + %+ murn ~(tap by entries.st) + |= [id=@ud ent=entry:b] + ?~ parent.ent ~ + ?: =(u.parent.ent i.queue) `id ~ + %= $ + queue (weld t.queue kids) + acc (~(gas in acc) kids) + == +:: +:: +group-exists: does %groups still hold this group? +:: +:: Every permission read below has to ask this first. %groups' scry dispatch +:: answers no-such-path for a group it does not have, and a scry that +:: resolves to nothing crashes the event rather than returning ~ -- so +:: without this guard, deleting a group takes down the very pass that would +:: revoke its buckets' tokens, at exactly the moment it is needed. +:: +:: Answering | is safe rather than merely convenient: a bucket is only ever +:: hosted by its group's host, so a group we host is always local and +:: "missing" means deleted, not not-yet-synced. +:: +++ group-exists + |= group=flag:b + ^- ? + =/ pax=path + /(scot %p our.bowl)/groups/(scot %da now.bowl)/groups/(scot %p ship.group)/[name.group] + .^(? %gu pax) +:: +++ group-can-read + |= [group=flag:b =flag:b who=ship] + ^- ? + ?: =(who ship.flag) & + ?. (group-exists group) | + =/ pax=path + /(scot %p our.bowl)/groups/(scot %da now.bowl)/v2/groups/(scot %p ship.group)/[name.group]/channels/can-read/noun + =/ test=$-([ship nest:b] ?) .^($-([ship nest:b] ?) %gx pax) + (test who [%buckets ship.flag name.flag]) +:: +++ group-is-admin-for-create + |= [group=flag:b who=ship] + ^- ? + ?: =(who ship.group) & + ?. (group-exists group) | + =/ pax=path + /(scot %p our.bowl)/groups/(scot %da now.bowl)/v2/groups/(scot %p ship.group)/[name.group]/seats/(scot %p who)/is-admin/noun + .^(? %gx pax) +:: +++ group-permissions + |= [group=flag:b =flag:b who=ship] + ^- (unit [admin=? roles=(set @tas)]) + ?: =(who ship.flag) `[& ~] + ?. (group-exists group) ~ + =/ pax=path + /(scot %p our.bowl)/groups/(scot %da now.bowl)/v2/groups/(scot %p ship.group)/[name.group]/channels/buckets/(scot %p ship.flag)/[name.flag]/can-write/(scot %p who)/noun + .^((unit [admin=? roles=(set @tas)]) %gx pax) +:: +++ group-is-admin + |= [group=flag:b =flag:b who=ship] + ^- ? + =/ permissions=(unit [admin=? roles=(set @tas)]) + (group-permissions group flag who) + ?~ permissions | + admin.u.permissions +:: +++ group-can-write + |= [group=flag:b =flag:b writers=(set @tas) who=ship] + ^- ? + ?. (group-can-read group flag who) | + =/ permissions=(unit [admin=? roles=(set @tas)]) + (group-permissions group flag who) + ?~ permissions | + ?| admin.u.permissions + =(~ writers) + !=(~ (~(int in writers) roles.u.permissions)) + == +:: +:: +action-authorized: may `who` run this verb on this bucket? Admin verbs +:: gate on the group's admin set, writes on the bucket's writer roles, and a +:: read grant only needs read access. +:: +++ action-authorized + |= [st=bucket-state:b =flag:b who=ship act=a-bucket:b] + ^- ? + ?- -.act + %delete (group-is-admin group.st flag who) + %set-title (group-is-admin group.st flag who) + %set-writers (group-is-admin group.st flag who) + %issue-bucket-read (group-can-read group.st flag who) + %create-folder (group-can-write group.st flag writers.st who) + %begin-upload (group-can-write group.st flag writers.st who) + %cancel-upload (group-can-write group.st flag writers.st who) + %issue-delete (group-can-write group.st flag writers.st who) + %entry (group-can-write group.st flag writers.st who) + == +:: +++ ship-text + |= who=ship + ^- @t + (crip (slag 1 (trip (scot %p who)))) +:: +++ broker-simple-verdict + |= result=@t + ^- json + (pairs:enjs:format ~[['result' s+result]]) +:: +:: +refuse: answer the broker with a denial, and say locally which condition +:: produced it. +:: +:: The wire vocabulary is closed: Pioneer parses only authorized, denied and +:: expired, and fails outright on anything else -- see +outcomeFrom in +:: pkg/runtime/pioneer/lib/Pioneer/Buckets.hs. So a dozen conditions have to +:: share one value, and splitting them properly means changing the broker +:: protocol, not just this agent. +:: +:: What they do not have to share is silence. Memex sees a 403 and maps it to +:: non-retryable; without this the ship keeps no record of whether that was a +:: real permission failure or its own state being missing, which is the +:: difference between a bug and a correct refusal. +:: +++ refuse + |= why=@tas + ^- json + %- (slog leaf+"buckets: refused a broker request, {}" ~) + (broker-simple-verdict 'denied') +:: +++ broker-upload-verdict + |= [token=@t reservation=@t] + ^- json + ?~ got=(session-token token) (refuse %no-such-session) + =/ ses=upload-session:b u.got + ?. (gth expires-at.ses now.bowl) + (broker-simple-verdict 'expired') + ?. =(%pending status.ses) (refuse %session-not-pending) + :: Echo the reservation bound on first exchange, ignoring the one Memex + :: proposed. Memex mints a fresh id per grant call, so a client retrying + :: after a lost response arrives with a new one; denying that would make + :: the upload unrecoverable, and the Pioneer contract requires the echo. + ?~ accepted=reservation.ses (refuse %reservation-unbound) + ?~ sp=(~(get by spaces) flag.ses) (refuse %no-such-bucket) + ?~ st-unit=state.u.sp (refuse %bucket-state-missing) + =/ st=bucket-state:b u.st-unit + ?. (group-can-write group.st flag.ses writers.st requested-by.ses) + (refuse %not-a-writer) + =/ fil=file:b (entry-file entry.ses) + =/ checksum-json=json + ?~ checksum.fil ~ + %- pairs:enjs:format + :~ ['algorithm' s+'crc32c'] + ['value' s+u.checksum.fil] + == + =/ upload=json + %- pairs:enjs:format + :~ ['bucketName' s+(scot %tas name.flag.ses)] + ['bucketId' s+(scot %ud id.bucket.st)] + ['sessionId' s+(scot %uv id.ses)] + ['objectId' s+object-key.fil] + ['actorShip' s+(ship-text requested-by.ses)] + ['size' (numb:enjs:format size.fil)] + ['mimeType' s+mime.fil] + ['checksum' checksum-json] + ['expiresAtMillis' (numb:enjs:format (mul 1.000 (unt:chrono:userlib expires-at.ses)))] + ['brokerReservationId' s+u.accepted] + == + %- pairs:enjs:format + :~ ['result' s+'authorized'] + ['upload' upload] + == +:: +:: +broker-object-verdict: answer Memex about one object. +:: +:: A read capability covers the bucket, so the object is resolved by its key +:: rather than named by the capability; a delete capability names its entry. +:: Either way access is re-checked against the live group here. +:: +:: The %read arm stays even though a broker holding a pushed token answers +:: reads from its own table without asking. The broker keeps a Pioneer +:: fallback for hosts that do not push yet, and it fires whenever it has no +:: row -- so this is what answers when a token is live here but absent +:: there. It cannot resurrect a revoked one: revocation deletes the local +:: capability too, so this arm refuses it as well. Deletes always ask. +:: +++ broker-object-verdict + |= [kind=object-kind:b token=@t object=@t] + ^- json + :: A read token lives in the desired state we sync, a delete token in the + :: per-object capabilities. Resolving both to the same shape here keeps the + :: checks below common; a revoked reader resolves to nothing, so the arm + :: refuses it exactly as it refuses an unknown token. + =/ resolved=(unit object-capability:b) + ?: =(%delete kind) (~(get by object-capabilities) token) + %- ~(rep by readers) + |= [[key=reader-key:b sync=reader-sync:b] acc=(unit object-capability:b)] + ?^ acc acc + ?. ?=(%granted -.desired.sync) ~ + ?. =(token token.desired.sync) ~ + `[%read flag.key ~ reader.key expires-at.desired.sync] + ?~ resolved (refuse %no-such-capability) + =/ aut=object-capability:b u.resolved + ?. =(kind kind.aut) (refuse %capability-wrong-kind) + ?. (gth expires-at.aut now.bowl) + (broker-simple-verdict 'expired') + ?~ sp=(~(get by spaces) flag.aut) (refuse %no-such-bucket) + ?~ st-unit=state.u.sp (refuse %bucket-state-missing) + =/ st=bucket-state:b u.st-unit + ?. ?: =(%read kind.aut) + (group-can-read group.st flag.aut actor.aut) + (group-can-write group.st flag.aut writers.st actor.aut) + (refuse %not-permitted) + =/ found=(unit entry:b) + ?^ entry-id.aut + (~(get by entries.st) u.entry-id.aut) + :: bucket-scoped: find the entry this object key belongs to + %- ~(rep by entries.st) + |= [[id=@ud ent=entry:b] acc=(unit entry:b)] + ?^ acc acc + ?. ?=(%file -.kind.ent) ~ + ?.(=(object object-key.file.kind.ent) ~ `ent) + ?~ found (refuse %no-such-entry) + =/ ent=entry:b u.found + ?. ?=(%file -.kind.ent) (refuse %entry-is-a-folder) + =/ fil=file:b +.kind.ent + ?. =(%ready status.fil) (refuse %file-not-ready) + ?. =(object object-key.fil) (refuse %object-key-mismatch) + =/ payload=json + ?: =(%read kind.aut) + %- pairs:enjs:format + :~ ['bucketId' s+(scot %ud id.bucket.st)] + ['objectId' s+object-key.fil] + ['displayFilename' s+name.ent] + == + %- pairs:enjs:format + :~ ['bucketId' s+(scot %ud id.bucket.st)] + ['objectId' s+object-key.fil] + == + =/ key=@t ?:(=(%read kind.aut) 'read' 'delete') + %- pairs:enjs:format + :~ ['result' s+'authorized'] + [key payload] + == +:: +:: +broker-complete-verdict: expiry reports as %expired rather than %denied. +:: Memex maps 403 to non-retryable and 410 to expired, so collapsing a lapsed +:: window into a denial turns a recoverable upload into a dead one. +:: +++ broker-complete-verdict + |= reservation=@t + ^- json + ?~ sid=(~(get by reservations) reservation) + (refuse %no-such-reservation) + ?~ got=(~(get by sessions) u.sid) (refuse %no-such-session) + =/ ses=upload-session:b u.got + ?: =(%complete status.ses) (broker-simple-verdict 'completed') + ?. (gth expires-at.ses now.bowl) + (broker-simple-verdict 'expired') + (refuse %session-not-pending) +:: +++ updates-path + |= =flag:b + ^- path + /v1/buckets/(scot %p ship.flag)/[name.flag]/updates +:: +++ sub-wire + |= =flag:b + ^- wire + /buckets/sub/(scot %p ship.flag)/[name.flag] +:: +++ start-sub + |= [=flag:b group=flag:b] + ^+ cor + ?: (~(has by spaces) flag) cor + =. spaces (~(put by spaces) flag [%sub ~ `group]) + %- emit + [%pass (sub-wire flag) %agent [ship.flag %buckets] %watch (updates-path flag)] +:: +++ stop-sub + |= =flag:b + ^+ cor + ?~ sp=(~(get by spaces) flag) cor + ?. =(%sub net.u.sp) cor + =. cor (drop-read-token flag) + =. cor (emil (drop (report-active flag u.sp |))) + :: Local clients watch our /v1, not the host's, so leaving the host says + :: nothing to them. Without this a still-mounted client keeps showing the + :: manifest of a replica this ship no longer has -- it refreshes on mount, + :: on an operation, or on a revision gap, and none of those arrive on + :: their own. + =/ rev=@ud ?~(state.u.sp 0 +(revision.u.state.u.sp)) + =/ res=response:b [%update flag rev [%delete ~]] + =. cor (give [%fact ~[/v1 (updates-path flag)] buckets-response-1+!>(res)]) + =. spaces (~(del by spaces) flag) + %- emit + [%pass (sub-wire flag) %agent [ship.flag %buckets] %leave ~] +:: +:: +resub: re-establish a dropped subscription without discarding the +:: replica. A kick is not a revocation — the host kicks deliberately when +:: access is pulled, but Gall also kicks on restart and transient failure. +:: +++ resub + |= =flag:b + ^+ cor + ?~ sp=(~(get by spaces) flag) cor + ?. =(%sub net.u.sp) cor + %- emit + [%pass (sub-wire flag) %agent [ship.flag %buckets] %watch (updates-path flag)] +:: +++ report-active + |= [=flag:b sp=space:b joined=?] + ^- (unit card) + =/ grp=(unit flag:b) + ?~ state.sp pending-group.sp + `group.u.state.sp + ?~ grp ~ + =/ nes=nest:b [%buckets ship.flag name.flag] + :- ~ + :* %pass /report-active %agent [our.bowl %groups] + %poke group-channel-active+!>([u.grp nes joined]) + == +:: +++ watch + |= =(pole knot) + ^+ cor + ?+ pole ~|(bad-buckets-watch+pole !!) + [%http-response *] cor + :: + [%v1 ~] + ?> =(src.bowl our.bowl) + =/ facts=(list card) + %+ murn local-snapshots + |= snap=snapshot:b + `[%give %fact ~ buckets-response-1+!>(`response:b`[%snapshot flag.snap bucket-state.snap])] + (emil facts) + :: + :: Terminal responses for actions submitted by clients on this ship. All + :: local clients share one path; tokens in a %grant are scoped to this + :: ship's user, who is the only subscriber. + :: + [%v1 %requests ~] + ?> =(src.bowl our.bowl) + cor + :: + :: A subscriber attaches here while waiting for our answer to one of its + :: forwarded commands. Only the requester named in the path may listen. + :: + [%v1 %request who=@ rid=@ ~] + ?> =(src.bowl (slav %p who.pole)) + cor + :: + [%v1 %buckets host=@ name=@ %updates ~] + =/ =flag:b [(slav %p host.pole) `@tas`name.pole] + ?> =(ship.flag our.bowl) + =/ st=bucket-state:b (need-state flag) + ?> (group-can-read group.st flag src.bowl) + (give [%fact ~ buckets-response-1+!>(`response:b`[%snapshot flag st])]) + == +:: +++ peek + |= =(pole knot) + ^- (unit (unit cage)) + ?+ pole ~ + [%x %v1 %buckets full=?(~ [%full ~])] + ?^ full.pole + ``buckets-snapshots-1+!>(local-snapshots) + ``buckets-summaries-1+!>(local-summaries) + :: + [%x %v1 %buckets host=@ name=@ ~] + =/ =flag:b [(slav %p host.pole) `@tas`name.pole] + ?~ sp=(~(get by spaces) flag) ~ + ?~ state.u.sp ~ + ``buckets-response-1+!>(`response:b`[%snapshot flag u.state.u.sp]) + :: + [%x %v1 %broker %upload cap=@ reservation=@ ~] + ``json+!>((broker-upload-verdict cap.pole reservation.pole)) + :: + [%x %v1 %broker %read cap=@ object=@ ~] + ``json+!>((broker-object-verdict %read cap.pole object.pole)) + :: + [%x %v1 %broker %delete cap=@ object=@ ~] + ``json+!>((broker-object-verdict %delete cap.pole object.pole)) + :: + [%x %v1 %broker %complete reservation=@ ~] + ``json+!>((broker-complete-verdict reservation.pole)) + :: + :: So an operator can confirm which broker a live host is pointed at + :: without reading its state. + :: + [%x %v1 %broker %base ~] + ``json+!>(`json`s+broker-base) + :: + :: Versioned mark rather than %noun: the client reads this over Eyre as + :: JSON, and %noun grows only to mime, so a bare noun answers 500 here. + :: + [%x %v1 %buckets host=@ name=@ %read-token ~] + =/ =flag:b [(slav %p host.pole) `@tas`name.pole] + ?~ tok=(~(get by read-tokens) flag) ~ + ``buckets-read-token-1+!>(`read-token:b`u.tok) + :: + :: +ready: a constant, because the answer existing is the whole signal -- + :: a client asking whether this ship can host Buckets needs to know the + :: desk is installed, nothing more. It reads /v1/buckets for this before, + :: which serialises every bucket's entire manifest, entries and all, and + :: so grew slower the more anyone stored. + :: + [%x %v1 %ready ~] + ``json+!>(`json`b+&) + :: + [%u %joined host=@ name=@ ~] + =/ =flag:b [(slav %p host.pole) `@tas`name.pole] + ``loob+!>((~(has by spaces) flag)) + == +:: +++ local-snapshots + ^- (list snapshot:b) + %+ murn ~(tap by spaces) + |= [=flag:b sp=space:b] + ?~ state.sp ~ + `[flag u.state.sp] +:: +:: +local-summaries: the same buckets without their entries. Drops the one +:: unbounded field, so asking which buckets exist costs the same whether +:: they hold nothing or everything. +:: +++ local-summaries + ^- (list summary:b) + %+ murn ~(tap by spaces) + |= [=flag:b sp=space:b] + ?~ state.sp ~ + =/ st=bucket-state:b u.state.sp + `[flag bucket.st group.st writers.st revision.st] +:: +++ agent + |= [=(pole knot) =sign:agent:gall] + ^+ cor + ?+ pole cor + [%groups ~] + ?+ -.sign cor + %kick watch-groups + %fact + (take-groups !<(r-groups:v9:gv q.cage.sign)) + :: + :: A refusal loses the subscription exactly as a kick does, and losing it + :: is not survivable: these facts are the only thing that calls + :: +recheck-host-subs, which is the only thing that revokes. Without them + :: a reader who loses access keeps a working token until it expires, + :: silently, for as long as the ship runs. Logging it was not a recovery. + %watch-ack + ?~ p.sign cor + %- (slog leaf+"buckets: groups watch refused, retrying" u.p.sign) + (emit [%pass /groups/retry %arvo %b %wait (add now.bowl groups-retry)]) + == + :: + + :: + [%buckets %sub host=@ name=@ ~] + =/ =flag:b [(slav %p host.pole) `@tas`name.pole] + ?~ sp=(~(get by spaces) flag) cor + ?. =(%sub net.u.sp) cor + ?+ -.sign cor + %fact + ?. =(%buckets-response-1 p.cage.sign) cor + (apply-response !<(response:b q.cage.sign)) + :: + :: A kick is not a revocation, so re-watch rather than dropping the + :: replica. The host's nack below is what tells us access is gone. + :: + :: The token does go, though: a host that revoked our access kicks us in + :: the same breath, and if access is restored before the re-watch is + :: acknowledged we never reach the nack. Keeping it would leave the local + :: scry answering with a token the broker has already dropped, and the + :: client would never ask for another. + %kick + =. cor (drop-read-token flag) + (resub flag) + :: + %watch-ack + ?~ p.sign cor + (stop-sub flag) + == + :: + [%buckets %req host=@ rid=@ %watch ~] + =/ host=ship (slav %p host.pole) + =/ rid=request-id:b (slav %uv rid.pole) + ?+ -.sign cor + %fact + ?. =(%buckets-req-response-1 p.cage.sign) cor + ?. (request-live rid) cor + =/ res=req-response:b !<(req-response:b q.cage.sign) + :: %pending is the host saying it is still working -- minting a read + :: token, say, while its push to the broker is in flight. Closing here + :: would drop the request and cancel its timeout, and the real answer + :: would arrive to find nothing waiting. Our client was already told + :: pending when we forwarded, so there is nothing to pass on. + ?: ?=(%pending -.body.res) cor + :: A token answer is ours to keep, but only under the bucket the + :: request named: tokens are bucket-scoped, so filing one under a + :: sibling bucket on the same host would leave both wrong. Read it + :: before closing, which is what drops the record. + =/ token-for=(unit flag:b) + ?~(got=(~(get by pending) rid) ~ token-for.u.got) + =. cor (close-request host rid) + =? cor &(?=(%token -.body.res) ?=(^ token-for)) + (keep-read-token u.token-for read-token.body.res) + :: A refused token request leaves no refresh armed, and our old token + :: may already have lapsed -- the client would go on reading it either + :: way. Come back to it, unless the answer was that we may not read + :: this bucket at all, in which case stop serving what we hold. + =? cor &(?=(%error -.body.res) ?=(^ token-for)) + ?: =(%not-authorized type.body.res) + (drop-read-token u.token-for) + (retry-read-token u.token-for) + (respond rid ~[/v1/requests] body.res) + :: + :: A dropped request stream is the same loss as a timeout: if it was a + :: renewal, its refresh has already fired and nothing else will rearm it, + :: so the local scry would keep serving a token past its expiry. + %kick + (abandon-request host rid 'host closed the request stream') + :: + %watch-ack + ?~ p.sign cor + (abandon-request host rid 'host refused the request stream') + == + :: + :: The poke-ack only reports delivery. A nack means the host crashed on + :: the command, so answer now rather than waiting for the timeout. + [%buckets %req host=@ rid=@ %poke ~] + =/ host=ship (slav %p host.pole) + =/ rid=request-id:b (slav %uv rid.pole) + ?+ -.sign cor + %poke-ack + ?~ p.sign cor + %- (slog leaf+"buckets: host command failed" u.p.sign) + (abandon-request host rid 'host rejected the command') + == + :: + [%buckets @ @ ?(%create %delete) ~] + ?+ -.sign cor + %poke-ack + ?~ p.sign cor + ((slog leaf+"buckets: group channel registration failed" u.p.sign) cor) + == + :: + [%report-active ~] + ?+ -.sign cor + %poke-ack + ?~ p.sign cor + ((slog leaf+"buckets: active-channel report failed" u.p.sign) cor) + == + == +:: +++ arvo + |= [=(pole knot) =sign-arvo] + ^+ cor + ?+ pole cor + [%eyre ~] + ?. ?=([%eyre %bound *] sign-arvo) cor + ?: accepted.sign-arvo cor + %- (slog leaf+"buckets: eyre bind rejected" ~) + cor + :: + [%buckets %token host=@ name=@ ~] + ?. ?=([%behn %wake *] sign-arvo) cor + =/ =flag:b [(slav %p host.pole) `@tas`name.pole] + ?~ sp=(~(get by spaces) flag) cor + :: Refreshing is the same request a client makes; when we host the bucket + :: Gall loops the poke back to us and it is served locally. + (renew-read-token flag) + :: + [%buckets %reader host=@ name=@ who=@ rev=@ ~] + ?. ?=([%iris %http-response *] sign-arvo) cor + =* res client-response.sign-arvo + ?: ?=(%progress -.res) cor + =/ key=reader-key:b + :- [(slav %p host.pole) `@tas`name.pole] + (slav %p who.pole) + =/ sent=@ud (slav %ud rev.pole) + :: A cancelled request is a refusal, not silence. Nothing is undone: the + :: desired state stands and the retry timer will send it again. + ?: ?=(%cancel -.res) + %- (slog leaf+"buckets: reader sync was cancelled" ~) + cor + =/ code=@ud status-code.response-header.res + =/ theirs=(unit @ud) (broker-revision res) + :: A stale write is not a failure -- it answers 200 with the revision it + :: kept, and adopting that is how we catch up. + ?: &((gte code 200) (lth code 300)) + (confirm-reader key sent theirs (broker-applied res)) + :: Only a success may confirm. An error body carries no revision under + :: the broker's contract, and adopting one from a rejection would install + :: a grant it just refused. Falling behind is recovered on the success + :: path instead: a stale write answers 200 with the revision the broker + :: kept, which +confirm-reader adopts. + ?: (broker-retryable res) + %- (slog leaf+"buckets: reader sync failed, status {}, retrying" ~) + cor + :: Refused as invalid rather than stale. Another attempt gets the same + :: answer, so stop owing it and tell anyone waiting. + %- (slog leaf+"buckets: reader sync rejected, status {}" ~) + (fail-reader key sent) + :: + [%groups %retry ~] + ?. ?=([%behn %wake *] sign-arvo) cor + :: Ask again whether or not we still lack it; a subscription we already + :: hold answers with a %watch-ack we ignore. + watch-groups + :: + [%buckets %reader-retry ~] + ?. ?=([%behn %wake *] sign-arvo) cor + retry-readers + :: + [%buckets %req host=@ rid=@ %wake ~] + ?. ?=([%behn %wake *] sign-arvo) cor + =/ host=ship (slav %p host.pole) + =/ rid=request-id:b (slav %uv rid.pole) + ?. (request-live rid) cor + :: A timed-out refresh leaves no timer armed and our old token in place, + :: which the local scry keeps answering with even once it lapses. Come + :: back to it rather than waiting on something that was never set. + =/ token-for=(unit flag:b) + ?~(got=(~(get by pending) rid) ~ token-for.u.got) + =. pending (~(del by pending) rid) + =? cor ?=(^ token-for) (retry-read-token u.token-for) + =. cor + %- emit + :* %pass (req-watch-wire host rid) %agent [host %buckets] + %leave ~ + == + (deny rid ~[/v1/requests] %unknown 'the host did not answer in time') + == +:: +++ apply-response + |= res=response:b + ^+ cor + ?- -.res + %snapshot + =/ sp=space:b (need-space flag.res) + ?> =(%sub net.sp) + =. sp sp(state `bucket-state.res, pending-group `group.bucket-state.res) + =. spaces (~(put by spaces) flag.res sp) + =. cor (emil (drop (report-active flag.res sp &))) + (give [%fact ~[/v1] buckets-response-1+!>(res)]) + :: + %update + =/ sp=space:b (need-space flag.res) + ?> =(%sub net.sp) + ?~ state.sp cor + =/ st=bucket-state:b u.state.sp + :: Ignore duplicates and re-establish the subscription on a gap. The + :: replacement watch begins with a full snapshot, so later deltas cannot + :: be applied to a stale replica. + ?: (lte revision.res revision.st) cor + ?. =(revision.res +(revision.st)) + (resub flag.res) + ?: =(%delete -.u-bucket.res) + =. cor (give [%fact ~[/v1] buckets-response-1+!>(res)]) + =. cor (emil (drop (report-active flag.res sp |))) + :: Drop the token with the bucket, as +stop-sub does. Left behind, the + :: local scry keeps answering with it, so a bucket recreated under the + :: same flag is read with a token the host has already revoked and the + :: client never asks for a new one. + =. cor (drop-read-token flag.res) + =. spaces (~(del by spaces) flag.res) + cor + =. st (apply-update st u-bucket.res) + =. revision.st revision.res + =. spaces (~(put by spaces) flag.res [net.sp `st `group.st]) + (give [%fact ~[/v1] buckets-response-1+!>(res)]) + == +:: +++ apply-update + |= [st=bucket-state:b upd=u-bucket:b] + ^- bucket-state:b + ?- -.upd + %create st(bucket bucket.upd) + %delete st + %meta st(bucket bucket.upd) + %writers st(writers writers.upd) + :: + %entry + st(entries (~(put by entries.st) id.upd entry.u-entry.upd)) + :: + %entries-deleted + =. entries.st + %- ~(rep in (silt ids.upd)) + |= [key=@ud acc=_entries.st] + (~(del by acc) key) + st + == +:: +:: +recheck-host-subs: read permissions may have shifted in `changed`, so +:: re-run can-read for subscribers of buckets bound to that group and kick +:: any who lost access. Scoped to the one group — a fact about some other +:: group is not a reason to scry for every subscriber we have. Grants are +:: handled by %groups' auto-join, so this only revokes. +:: +:: +take-groups: react to a change in a group we host buckets for. +:: +:: Follows %channels-server, which owns its channels' writer roles the way we +:: own ours: react to the specific change when it arrives, and reconcile the +:: whole set whenever the group arrives whole. The pair matters because +:: neither half is sufficient -- a fact we miss is repaired by the sweep, and +:: the sweep only happens when a group is re-read. +:: +++ take-groups + |= =r-groups:v9:gv + ^+ cor + =* r-group r-group.r-groups + =. cor (recheck-host-subs flag.r-groups) + ?+ r-group cor + :: A role that no longer exists must stop granting writes. Role ids are + :: minted from the role's title, so deleting one and making another by + :: the same name reuses the id -- and a stale id left in .writers would + :: hand write and delete on every bucket that named it to whoever joins + :: the new role. + [%role * %del ~] + (strip-writers flag.r-groups roles.r-group) + :: + :: The group arrived whole, so anything we hold that it does not have + :: is stale however we came to miss it. + [%create *] + %+ strip-writers flag.r-groups + %- ~(dif in (held-writers flag.r-groups)) + ~(key by roles.group.r-group) + == +:: +:: +held-writers: every role id our buckets in this group grant writes to. +:: +++ held-writers + |= group=flag:b + ^- (set @tas) + %+ roll ~(val by spaces) + |= [sp=space:b acc=(set @tas)] + ?. =(%pub net.sp) acc + ?~ state.sp acc + ?. =(group group.u.state.sp) acc + (~(uni in acc) writers.u.state.sp) +:: +:: +strip-writers: drop role ids from every bucket in this group that names +:: them, and tell subscribers the permissions changed. +:: +++ strip-writers + |= [group=flag:b roles=(set @tas)] + ^+ cor + ?: =(~ roles) cor + %+ roll ~(tap by spaces) + |= [[=flag:b sp=space:b] acc=_cor] + ?. =(%pub net.sp) acc + ?~ state.sp acc + =/ st=bucket-state:b u.state.sp + ?. =(group group.st) acc + =/ kept=(set @tas) (~(dif in writers.st) roles) + ?: =(kept writers.st) acc + %- (slog leaf+"buckets: dropping deleted roles from {} writers" ~) + (commit-update:acc flag st(writers kept) [%writers kept] our.bowl) +:: +++ recheck-host-subs + |= changed=flag:b + ^+ cor + :: Buckets we host that belong to the group that changed, mapped to it. + :: Everything below is scoped to these: a fact about one group is no + :: reason to re-scry permissions for buckets bound to another. + =/ affected=(map flag:b flag:b) + %- malt + %+ murn ~(tap by spaces) + |= [=flag:b sp=space:b] + ^- (unit [flag:b flag:b]) + ?. =(%pub net.sp) ~ + ?~ state.sp ~ + ?. =(changed group.u.state.sp) ~ + `[flag group.u.state.sp] + ?: =(~ affected) cor + =/ kicks=(list card) + %+ murn ~(val by sup.bowl) + |= [who=ship pax=path] + ^- (unit card) + ?. ?=([%v1 %buckets @ @ %updates ~] pax) ~ + =/ =flag:b [(slav %p i.t.t.pax) `@tas`i.t.t.t.pax] + ?~ group=(~(get by affected) flag) ~ + ?: (group-can-read u.group flag who) ~ + `[%give %kick ~[pax] `who] + :: Revocation is driven by the tokens, not by the subscriptions. A reader + :: that took a token and then left or dropped its subscription has no + :: entry in sup.bowl, so it produces no kick -- and because the broker + :: serves a pushed token without asking us again, that token would keep + :: working for every ready file until it lapsed. Asking the group about + :: each token's own reader and bucket catches those, and stops a reader + :: losing one bucket from having its tokens for other buckets revoked. + =. cor + %- revoke-readers + %- granted-readers + |= [key=reader-key:b sync=reader-sync:b] + ?~ group=(~(get by affected) flag.key) | + !(group-can-read u.group flag.key reader.key) + (emil kicks) +-- diff --git a/desk/app/groups.hoon b/desk/app/groups.hoon index 87db36a34d..efd4500a36 100644 --- a/desk/app/groups.hoon +++ b/desk/app/groups.hoon @@ -4652,8 +4652,8 @@ [%can-write ship=@ ~] =+ ship=(slav %p ship.rest.pole) ^- (unit (unit rail)) - ?~ seat=(~(get by seats.group) ship) [~ ~] - ?: (go-is-banned ship) [~ ~] + ?~ seat=(~(get by seats.group) ship) ``unsafe+noun+!>(~) + ?: (go-is-banned ship) ``unsafe+noun+!>(~) =- ``unsafe+noun+!>(-) %- some :- admin=(go-is-admin ship) diff --git a/desk/desk.bill b/desk/desk.bill index fcb0315302..8f2d3acd35 100644 --- a/desk/desk.bill +++ b/desk/desk.bill @@ -20,4 +20,5 @@ %metagrab %presence %steward + %buckets == diff --git a/desk/lib/buckets/json.hoon b/desk/lib/buckets/json.hoon new file mode 100644 index 0000000000..58c8656049 --- /dev/null +++ b/desk/lib/buckets/json.hoon @@ -0,0 +1,319 @@ +:: JSON codecs for the Buckets client surface +:: +/- b=buckets +|% +:: +++ enjs + =, enjs:format + |% + ++ flag + |= f=flag:b + ^- json + (pairs ~[['host' s+(scot %p ship.f)] ['name' s+name.f]]) + :: + ++ bucket + |= buc=bucket:b + ^- json + %- pairs + :~ ['id' (numb id.buc)] + ['title' s+title.buc] + ['createdBy' s+(scot %p created-by.buc)] + ['createdAt' (numb (unt:chrono:userlib created-at.buc))] + ['updatedBy' s+(scot %p updated-by.buc)] + ['updatedAt' (numb (unt:chrono:userlib updated-at.buc))] + == + :: + ++ file + |= fil=file:b + ^- json + %- pairs + :~ ['mime' s+mime.fil] + ['size' (numb size.fil)] + ['checksum' ?~(checksum.fil ~ s+u.checksum.fil)] + ['objectKey' s+object-key.fil] + ['status' s+(scot %tas status.fil)] + == + :: + ++ entry + |= ent=entry:b + ^- json + =/ common=(list [cord json]) + :~ ['id' (numb id.ent)] + ['parentId' ?~(parent.ent ~ (numb u.parent.ent))] + ['name' s+name.ent] + ['createdBy' s+(scot %p created-by.ent)] + ['createdAt' (numb (unt:chrono:userlib created-at.ent))] + ['updatedBy' s+(scot %p updated-by.ent)] + ['updatedAt' (numb (unt:chrono:userlib updated-at.ent))] + == + ?- -.kind.ent + %folder (pairs (snoc common ['kind' s+'folder'])) + %file + %- pairs + %+ snoc (snoc common ['kind' s+'file']) + ['file' (file +.kind.ent)] + == + :: + ++ bucket-state + |= st=bucket-state:b + ^- json + =/ ents=(list json) + %+ turn ~(val by entries.st) + entry + =/ writer-roles=(list json) + %+ turn ~(tap in writers.st) + |=(role=@tas s+(scot %tas role)) + %- pairs + :~ ['bucket' (bucket bucket.st)] + ['group' (flag group.st)] + ['writers' [%a writer-roles]] + ['entries' [%a ents]] + ['revision' (numb revision.st)] + == + :: + ++ update + |= upd=u-bucket:b + ^- json + ?- -.upd + %create + (pairs ~[['type' s+'bucket-created'] ['bucket' (bucket bucket.upd)]]) + :: + %delete + (pairs ~[['type' s+'bucket-deleted']]) + :: + %meta + (pairs ~[['type' s+'bucket-updated'] ['bucket' (bucket bucket.upd)]]) + :: + %writers + =/ roles=(list json) + %+ turn ~(tap in writers.upd) + |=(role=@tas s+(scot %tas role)) + (pairs ~[['type' s+'writers-updated'] ['writers' [%a roles]]]) + :: + %entry + =/ typ=@t + ?- -.u-entry.upd + %create 'entry-created' + %update 'entry-updated' + == + %- pairs + :~ ['type' s+typ] + ['id' (numb id.upd)] + ['entry' (entry entry.u-entry.upd)] + == + :: + %entries-deleted + =/ ids=(list json) (turn ids.upd numb) + (pairs ~[['type' s+'entries-deleted'] ['ids' [%a ids]]]) + == + :: + ++ grant + |= gra=grant:b + ^- json + %- pairs + :~ ['token' s+token.gra] + ['entryId' (numb entry-id.gra)] + ['expiresAt' s+(scot %da expires-at.gra)] + == + :: + ++ read-token + |= tok=read-token:b + ^- json + %- pairs + :~ ['token' s+token.tok] + ['expiresAt' s+(scot %da expires-at.tok)] + == + :: + ++ req-response + |= res=req-response:b + ^- json + =/ bod=json + ?- -.body.res + %ok (frond 'ok' ~) + %pending (frond 'pending' ~) + %grant (frond 'grant' (grant grant.body.res)) + %token (frond 'token' (read-token read-token.body.res)) + :: + %error + %- frond + :- 'error' + %- pairs + :~ ['type' s+(scot %tas type.body.res)] + ['message' s+message.body.res] + == + == + %- pairs + :~ ['requestId' s+(scot %uv request-id.res)] + ['body' bod] + == + :: + ++ response + |= res=response:b + ^- json + ?- -.res + %snapshot + %- pairs + :~ ['type' s+'snapshot'] + ['flag' (flag flag.res)] + ['state' (bucket-state bucket-state.res)] + == + :: + %update + %- pairs + :~ ['type' s+'update'] + ['flag' (flag flag.res)] + ['revision' (numb revision.res)] + ['update' (update u-bucket.res)] + == + == + :: + ++ snapshots + |= snaps=(list snapshot:b) + ^- json + :- %a + %+ turn snaps + |= snap=snapshot:b + (pairs ~[['flag' (flag flag.snap)] ['state' (bucket-state bucket-state.snap)]]) + :: + :: +summaries: the same shape as +snapshots minus 'entries', so a client + :: reading either finds the fields it shares where it expects them. + :: + ++ summaries + |= sums=(list summary:b) + ^- json + :- %a + %+ turn sums + |= sum=summary:b + =/ writer-roles=(list json) + %+ turn ~(tap in writers.sum) + |=(role=@tas s+(scot %tas role)) + %- pairs + :~ ['flag' (flag flag.sum)] + :- 'state' + %- pairs + :~ ['bucket' (bucket bucket.sum)] + ['group' (flag group.sum)] + ['writers' [%a writer-roles]] + ['revision' (numb revision.sum)] + == + == + -- +:: +++ dejs + =, dejs:format + |% + ++ get + |= [key=@t jon=json] + ^- json + ?> ?=([%o *] jon) + =/ val=(unit json) (~(get by p.jon) key) + ?> ?=(^ val) + u.val + :: + ++ maybe + |= [key=@t jon=json decoder=$-(json @t)] + ^- (unit @t) + ?> ?=([%o *] jon) + =/ val=(unit json) (~(get by p.jon) key) + ?~ val ~ + ((mu decoder) u.val) + :: + ++ maybe-ud + |= [key=@t jon=json] + ^- (unit @ud) + ?> ?=([%o *] jon) + =/ val=(unit json) (~(get by p.jon) key) + ?~ val ~ + ((mu ni) u.val) + :: + ++ knot + |= txt=@t + ^- @tas + =/ parsed=(unit @tas) (slaw %tas txt) + ?~ parsed ~|(invalid-buckets-knot+txt !!) + u.parsed + :: + ++ flag + |= jon=json + ^- flag:b + =/ raw + %. jon + (ot ~[['host' (su ;~(pfix sig fed:ag))] ['name' so]]) + [-.raw (knot +.raw)] + :: + ++ roles + |= jon=json + ^- (set @tas) + ((as (cu |=(t=@t `@tas``@`t) so)) jon) + :: + :: +command: a request-id'd action. The client mints the id so it can + :: correlate the terminal response it gets back on /v1/requests. + :: + ++ command + |= jon=json + ^- command:b + ?> ?=([%o *] jon) + :- ((se %uv) (get 'requestId' jon)) + (action jon) + :: + :: +action: the client's JSON stays flat — one "type" plus its fields — and + :: is folded into the nested $a-buckets envelope here rather than making + :: every caller construct it. + :: + ++ action + |= jon=json + ^- action:b + ?> ?=([%o *] jon) + =/ typ=@t (so (get 'type' jon)) + ?: =(%'create' typ) + :* %create + (knot (so (get 'name' jon))) + (so (get 'title' jon)) + (flag (get 'group' jon)) + (roles (get 'readers' jon)) + (roles (get 'writers' jon)) + == + :+ %bucket (flag (get 'flag' jon)) + ?+ typ ~|(unknown-buckets-action+typ !!) + %'delete-bucket' [%delete ~] + %'set-title' [%set-title (so (get 'title' jon))] + %'set-writers' [%set-writers (roles (get 'writers' jon))] + :: + %'create-folder' + :* %create-folder + (maybe-ud 'parentId' jon) + (so (get 'name' jon)) + == + :: + %'begin-upload' + :* %begin-upload + (maybe-ud 'parentId' jon) + (so (get 'name' jon)) + (so (get 'mime' jon)) + (ni (get 'size' jon)) + (maybe 'checksum' jon so) + == + :: + %'cancel-upload' + :* %cancel-upload + ((se %uv) (get 'sessionId' jon)) + (so (get 'reason' jon)) + == + :: + %'issue-bucket-read' [%issue-bucket-read ~] + %'issue-delete' [%issue-delete (ni (get 'id' jon))] + :: + %'rename-entry' + :+ %entry (ni (get 'id' jon)) + [%rename (so (get 'name' jon))] + :: + %'move-entry' + :+ %entry (ni (get 'id' jon)) + [%move (maybe-ud 'parentId' jon)] + :: + %'delete-entry' + :+ %entry (ni (get 'id' jon)) + [%delete (bo (get 'recursive' jon))] + == + -- +-- diff --git a/desk/mar/buckets/action-1.hoon b/desk/mar/buckets/action-1.hoon new file mode 100644 index 0000000000..b8651b8688 --- /dev/null +++ b/desk/mar/buckets/action-1.hoon @@ -0,0 +1,16 @@ +:: buckets-action-1: local client actions, carrying a request id +:: +/- b=buckets +/= buckets-json /lib/buckets/json +|_ =command:b +++ grad %noun +++ grab + |% + ++ noun command:b + ++ json command:dejs:buckets-json + -- +++ grow + |% + ++ noun command + -- +-- diff --git a/desk/mar/buckets/broker-command-1.hoon b/desk/mar/buckets/broker-command-1.hoon new file mode 100644 index 0000000000..66fb2c3497 --- /dev/null +++ b/desk/mar/buckets/broker-command-1.hoon @@ -0,0 +1,14 @@ +:: buckets-broker-command-1: noun-only Pioneer broker callbacks +:: +/- b=buckets +|_ =broker-command:b +++ grad %noun +++ grab + |% + ++ noun broker-command:b + -- +++ grow + |% + ++ noun broker-command + -- +-- diff --git a/desk/mar/buckets/command-1.hoon b/desk/mar/buckets/command-1.hoon new file mode 100644 index 0000000000..bb40e72fa5 --- /dev/null +++ b/desk/mar/buckets/command-1.hoon @@ -0,0 +1,14 @@ +:: buckets-command-1: noun-only cross-ship commands +:: +/- b=buckets +|_ =command:b +++ grad %noun +++ grab + |% + ++ noun command:b + -- +++ grow + |% + ++ noun command + -- +-- diff --git a/desk/mar/buckets/read-token-1.hoon b/desk/mar/buckets/read-token-1.hoon new file mode 100644 index 0000000000..cd78246b57 --- /dev/null +++ b/desk/mar/buckets/read-token-1.hoon @@ -0,0 +1,16 @@ +:: buckets-read-token-1: the bucket-read token this ship holds +:: +/- b=buckets +/= buckets-json /lib/buckets/json +|_ tok=read-token:b +++ grad %noun +++ grab + |% + ++ noun read-token:b + -- +++ grow + |% + ++ noun tok + ++ json (read-token:enjs:buckets-json tok) + -- +-- diff --git a/desk/mar/buckets/req-response-1.hoon b/desk/mar/buckets/req-response-1.hoon new file mode 100644 index 0000000000..dcd629d1e8 --- /dev/null +++ b/desk/mar/buckets/req-response-1.hoon @@ -0,0 +1,16 @@ +:: buckets-req-response-1: terminal answer to one client action +:: +/- b=buckets +/= buckets-json /lib/buckets/json +|_ res=req-response:b +++ grad %noun +++ grab + |% + ++ noun req-response:b + -- +++ grow + |% + ++ noun res + ++ json (req-response:enjs:buckets-json res) + -- +-- diff --git a/desk/mar/buckets/response-1.hoon b/desk/mar/buckets/response-1.hoon new file mode 100644 index 0000000000..a60b9b0dc3 --- /dev/null +++ b/desk/mar/buckets/response-1.hoon @@ -0,0 +1,16 @@ +:: buckets-response-1: snapshots and manifest updates +:: +/- b=buckets +/= buckets-json /lib/buckets/json +|_ =response:b +++ grad %noun +++ grab + |% + ++ noun response:b + -- +++ grow + |% + ++ noun response + ++ json (response:enjs:buckets-json response) + -- +-- diff --git a/desk/mar/buckets/snapshots-1.hoon b/desk/mar/buckets/snapshots-1.hoon new file mode 100644 index 0000000000..0003e54be3 --- /dev/null +++ b/desk/mar/buckets/snapshots-1.hoon @@ -0,0 +1,16 @@ +:: buckets-snapshots-1: complete local manifest replica +:: +/- b=buckets +/= buckets-json /lib/buckets/json +|_ snaps=(list snapshot:b) +++ grad %noun +++ grab + |% + ++ noun (list snapshot:b) + -- +++ grow + |% + ++ noun snaps + ++ json (snapshots:enjs:buckets-json snaps) + -- +-- diff --git a/desk/mar/buckets/summaries-1.hoon b/desk/mar/buckets/summaries-1.hoon new file mode 100644 index 0000000000..e78ce50e95 --- /dev/null +++ b/desk/mar/buckets/summaries-1.hoon @@ -0,0 +1,19 @@ +:: buckets-summaries-1: the local buckets without their contents +:: +:: What listing and routing need. buckets-snapshots-1 carries every entry of +:: every bucket, which is unbounded and is only wanted when opening one. +:: +/- b=buckets +/= buckets-json /lib/buckets/json +|_ sums=(list summary:b) +++ grad %noun +++ grab + |% + ++ noun (list summary:b) + -- +++ grow + |% + ++ noun sums + ++ json (summaries:enjs:buckets-json sums) + -- +-- diff --git a/desk/sur/buckets.hoon b/desk/sur/buckets.hoon new file mode 100644 index 0000000000..811f0372ae --- /dev/null +++ b/desk/sur/buckets.hoon @@ -0,0 +1,456 @@ +:: buckets: shared group file-space protocol +:: +:: Gall owns only the logical manifest and upload lifecycle. File bytes and +:: signed object-store grants must never be stored in this state or sent over +:: Ames. +:: +|% +:: $flag: global bucket identity — its host ship plus a slug. +:: ++$ flag [=ship name=@tas] +:: $nest: channel identity shared with %groups. For a bucket the kind is +:: always %buckets and [host name] is the bucket's flag. +:: ++$ nest [kind=@tas host=@p name=@tas] +:: +:: Channel-host messages used by %groups for third-party channel kinds. +:: ++$ channel-join [=nest group=flag] +:: $channel-leave: %groups tells us a member left, or lost access. +:: ++$ channel-leave [=nest] +:: +:: Minimal %groups channel-registration payloads. These intentionally mirror +:: the protocol shapes consumed by %group-action-4. +:: ++$ group-channel + $: meta=[title=@t description=@t image=@t cover=@t] + created=@da + section=@tas + readers=(set @tas) + join=? + == ++$ group-create + $: %group + =flag + %channel + =nest + %add + channel=group-channel + == ++$ group-channel-del + $: %group + =flag + %channel + =nest + %del + ~ + == +:: +:: $bucket: one shared file space, owned by its group host. +:: ++$ bucket + $: id=@ud + title=@t + created-by=ship + created-at=@da + updated-by=ship + updated-at=@da + == +:: +:: $file: object-store metadata for a leaf entry. +:: +:: .object-key is host-generated and unrelated to .name on the entry, so +:: rename and move stay metadata-only. +:: ++$ upload-status ?(%pending %ready %failed) ++$ file + $: mime=@t + size=@ud + checksum=(unit @t) + object-key=@t + status=upload-status + == +:: $entry-kind: a tree node is either a folder or a file. +:: ++$ entry-kind + $% [%folder ~] + [%file =file] + == +:: +:: $entry: a folder or file in a bucket's tree. +:: ++$ entry + $: id=@ud + parent=(unit @ud) + name=@t + created-by=ship + created-at=@da + updated-by=ship + updated-at=@da + kind=entry-kind + == +:: +:: $upload-session: host-private record of one in-flight upload. +:: +:: .id doubles as the opaque broker token the uploader presents to Memex. +:: It is minted from bowl entropy and returned only to .requested-by, never +:: broadcast — so it is safe to use as a bearer secret. +:: .entry is the not-yet-published entry: it joins the bucket's manifest +:: only once the object lands, so an in-flight upload is invisible to +:: everyone but its uploader. +:: .reservation is the broker reservation id bound on first exchange. +:: +:: .cancelled is the uploader withdrawing, which is all it can report: only +:: the broker knows whether the bytes landed, so a cancelled session still +:: accepts a completion. It stops a new upload URL being issued against it. +:: ++$ session-status ?(%pending %complete %cancelled) ++$ upload-session + $: id=@uv + =flag + =entry + requested-by=ship + created-at=@da + expires-at=@da + status=session-status + reservation=(unit @t) + error=(unit @t) + == +:: +:: $object-capability: host-private grant to read or delete objects. +:: +:: Minted per request and returned only to .actor. Uploads do not appear +:: here — their token is the upload session id. +:: +:: Reads are scoped to a whole bucket, not one object. Read access is uniform +:: across a bucket — group-can-read answers for the channel, not the file — so +:: a per-object grant would be exactly as precise while costing the reader a +:: round trip per file. Deletes stay per-object because they are destructive +:: and a mistake is not recoverable. +:: ++$ object-kind ?(%read %delete) +:: .entry-id is ~ for a read, which covers the whole bucket, and set for a +:: delete, which names one object. +:: ++$ object-capability + $: kind=object-kind + =flag + entry-id=(unit @ud) + actor=ship + expires-at=@da + == +:: +:: $reader-state: what one reader's access to one bucket should look like at +:: the broker. Granted carries the bearer token; revoked carries nothing +:: usable, because there is nothing the reader should be able to present. +:: ++$ reader-state + $% [%granted token=@t expires-at=@da] + [%revoked ~] + == +:: $reader-sync: one (bucket, reader) pair as desired state rather than as a +:: sequence of pushes and revokes. +:: +:: .revision is strictly increasing per pair and bumped on every access +:: change -- grant, rotation, revoke alike. The broker keeps only the highest +:: it has seen, so a delayed or duplicated request is harmless and delivery +:: order stops mattering: the message says what should be true, not what to +:: do. .synced is the highest revision the broker has confirmed, so anything +:: above it is still owed. +:: +:: .awaiting is the client request still holding open for this pair's grant +:: to land at the broker. It rides here rather than on the wire so a retry +:: does not lose it. +:: +:: .bucket-id and .expires make the record self-sufficient. A revoke issued +:: as a bucket is deleted still has to be deliverable afterwards, so the +:: request cannot be rebuilt from live bucket state that is already gone. +:: .expires is when this pair's authority stops mattering either way -- the +:: granted token's expiry, carried onto the revoke that replaces it -- which +:: is also when a confirmed revoked record can be dropped, since past it a +:: lost revoke is moot. +:: +:: .failed marks a revision the broker refused as invalid rather than stale. +:: Retrying that is pointless -- it is a bug on this side, not a race -- so +:: it stops being owed until the next access change supersedes it. +:: ++$ reader-sync + $: revision=@ud + bucket-id=@t + desired=reader-state + expires=@da + synced=@ud + failed=? + awaiting=(unit request-id) + == +:: $reader-key: a bucket and one of its readers. +:: ++$ reader-key [=flag reader=ship] +:: $reader-status: what a reader record still asks of us. +:: +:: Derived from .revision, .synced, .failed and .expires together, which +:: every caller used to do for itself from whichever fields it happened to +:: care about. They disagreed, and each disagreement was a bug: records the +:: broker had refused were owed by nobody and prunable by nobody, so they +:: accumulated for good. Switching on this with ?- makes a missed state a +:: compile error instead of something review has to catch. +:: ++$ reader-status ?(%owed %settled %refused %lapsed) +:: +:: $read-token: the bucket-read capability this ship currently holds. +:: +:: Every ship keeps its own, refreshed on a timer, so a local client always +:: has one to hand without a network round trip — and because the token names +:: its actor, the broker still sees who is reading. +:: ++$ read-token + $: token=@t + expires-at=@da + == +:: +:: Readability is %groups' business alone, so no reader set is kept here: the +:: roles named at creation are handed to %groups with the channel and belong +:: to it from then on, and +group-can-read asks it. Bucket writers are a +:: separate subset of group roles that %groups does not model, so those do +:: live here; an empty set means every readable member may write, matching +:: the convention used by %channels. +:: ++$ bucket-state + $: =bucket + group=flag + writers=(set @tas) + entries=(map @ud entry) + revision=@ud + == +:: +:: Actions (client -> agent) +:: +:: $a-buckets: what a client on our own ship asks for. The outer tag carries +:: identity; the inner unions carry only the verb. +:: +:: Tokens are never supplied by the caller — the host mints them and hands +:: them back in a $response-body. +:: +:: .readers on %create is passed straight to %groups as the new channel's +:: reader roles and is not retained; change it there, not here. +:: ++$ a-buckets + $% [%create name=@tas title=@t group=flag readers=(set @tas) writers=(set @tas)] + [%bucket =flag =a-bucket] + == +:: +:: $a-bucket: actions on one bucket. The flag lives on the outer envelope. +:: ++$ a-bucket + $% [%delete ~] + [%set-title title=@t] + [%set-writers writers=(set @tas)] + [%create-folder parent=(unit @ud) name=@t] + [%begin-upload parent=(unit @ud) name=@t mime=@t size=@ud checksum=(unit @t)] + [%cancel-upload session=@uv reason=@t] + [%issue-bucket-read ~] + [%issue-delete id=@ud] + [%entry id=@ud =a-entry] + == +:: +:: $a-entry: actions on one entry. The id lives on the outer envelope. +:: ++$ a-entry + $% [%rename name=@t] + [%move parent=(unit @ud)] + [%delete recursive=?] + == +:: ++$ request-id @uv +:: +:: $command: a request-id'd action, carried by both %buckets-action-1 (a +:: local client) and %buckets-command-1 (a subscriber forwarding to the +:: host). +:: +:: There is deliberately no separate c-* family: every verb a local client +:: can send is also one a peer may forward, so an a-/c- split would be two +:: identical unions. The trust boundary is the poke handler's gate — +:: ?> =(src.bowl our.bowl) for actions, a permission check for commands. +:: Add the split when the first local-only or peer-only verb appears. +:: ++$ command [=request-id act=a-buckets] +:: +:: $grant: a host-minted bearer token returned to the requester alone. +:: +:: .token is the opaque string presented to Memex — for an upload it is the +:: session id, for a read or delete it is a freshly minted capability. +:: ++$ grant + $: token=@t + entry-id=@ud + expires-at=@da + == +:: +:: $action-error: enumerated failure modes returned to the requester. +:: ++$ action-error + $? %not-authorized + %not-found + %invalid-input + %unknown + == +:: +:: $response-body: terminal answer to one client action. +:: +:: %pending is emitted by a subscriber once it has forwarded the command to +:: the host and is waiting; the host's real answer replaces it. +:: ++$ response-body + $% [%ok ~] + [%grant =grant] + [%token =read-token] + [%pending ~] + [%error type=action-error message=@t] + == +:: +:: $req-response: a $response-body addressed to one in-flight request. +:: ++$ req-response [=request-id body=response-body] +:: +:: $incoming-request: an action we are tracking to its terminal answer. +:: +:: .http-id is set while an Eyre POST is held open waiting for that answer; +:: clearing it after delivery stops a late update re-answering a closed +:: request. .final-at is stamped once .result is terminal, so cleanup can +:: evict the record after a grace window. +:: ++$ incoming-request + $: =request-id + http-id=(unit @ta) + result=(unit response-body) + final-at=(unit @da) + == ++$ requests (map request-id incoming-request) +:: +:: Opaque, short-lived bearer capabilities exchanged by Memex through +:: Pioneer's local spider threads. They are host-only authority state and +:: are never included in Bucket snapshots or Ames updates. +:: ++$ broker-receipt + $: broker-reservation-id=@t + object-id=@t + host=@t + bucket-id=@t + size=@ud + mime-type=@t + == +:: $broker-command: what a Pioneer thread relays in from Memex. The +:: capability is opaque on the Memex side, so this shape is fixed by the +:: cross-repo contract — do not change it without changing ylem. +:: ++$ broker-command + $% [%authorize-upload capability=@t broker-reservation-id=@t] + [%complete-upload =broker-receipt] + == +:: +:: Updates (host -> subscribers) +:: +:: $u-bucket: a canonical manifest change on one bucket, broadcast to every +:: subscriber. Updates are fat: an arm carries the whole post-change entity, +:: so a replica overwrites rather than merging. +:: +:: Upload lifecycle arms are deliberately absent. A pending upload lives on +:: its session, not in the manifest, so it produces no update at all — the +:: file arrives as [%entry id %create] when the object lands, and a failed +:: upload is reported to its uploader in a $response-body instead. +:: ++$ u-bucket + $% [%create =bucket] + [%delete ~] + [%meta =bucket] + [%writers writers=(set @tas)] + [%entry id=@ud =u-entry] + [%entries-deleted ids=(list @ud)] + == +:: +:: $u-entry: a change to one entry. The id lives on the outer envelope, and +:: attribution rides on the entity's own .updated-by / .updated-at. +:: ++$ u-entry + $% [%create =entry] + [%update =entry] + == +:: +:: Responses (agent -> local client subscribers) +:: +:: $r-buckets: facts on /v1 and on a bucket's update path. No actor field — +:: who changed what is recorded on $bucket and $entry. +:: ++$ r-buckets + $% [%snapshot =flag =bucket-state] + [%update =flag revision=@ud =u-bucket] + == +:: ++$ snapshot [=flag =bucket-state] +:: $summary: a bucket without its contents. +:: +:: Every use but opening one -- listing them, routing to them, reading a +:: channel's writer roles -- needs the metadata and none of the entries, and +:: a bucket's entries are unbounded. Answering those from $snapshot made the +:: cost of asking about buckets scale with everything stored in them. +:: ++$ summary + $: =flag + =bucket + group=flag + writers=(set @tas) + revision=@ud + == +:: +:: Type aliases used by the mark files. +:: ++$ action a-buckets ++$ update u-bucket ++$ response r-buckets +:: ++$ net ?(%pub %sub) +:: $space: one bucket as this ship sees it — whether we host it or replicate +:: it, the replica itself, and the group we expect it to belong to before the +:: first snapshot arrives. +:: ++$ space [=net state=(unit bucket-state) pending-group=(unit flag)] +:: +:: Persisted state. %buckets has never run on a live ship, so there is +:: nothing to migrate from — this is version 0. A future change adds +:: $state-1 plus a +state-0-to-1 arm chained from +on-load; never mutate a +:: numbered mold in place. +:: ++$ state-0 + $: %0 + :: .broker-base is where this ship's storage broker lives. Ship + :: configuration rather than bucket data: a ship has no environment to + :: read, so pointing a host at a test broker is a poke, and the default + :: +init installs is production. + broker-base=@t + spaces=(map flag space) + next-id=@ud + sessions=(map @uv upload-session) + object-capabilities=(map @t object-capability) + read-tokens=(map flag read-token) + :: .readers is the desired access state we owe the broker, per bucket + :: and reader. It replaces separate push and revoke effects: both are + :: the same idempotent sync of one pair at one revision. + readers=(map reader-key reader-sync) + reservations=(map @t @uv) + :: .token-for is set only when the forwarded action was a request for + :: a read token, and names the bucket it is for. It answers two + :: questions at once: which bucket to file a %token answer under -- + :: guessing from the host is wrong once two buckets share one -- and + :: whether this request's failure has anything to do with tokens at + :: all, so that a refused folder rename does not discard one. + pending=(map request-id [host=ship until=@da token-for=(unit flag)]) + requests=requests + == +:: $versioned-state: every persisted shape +on-load may be handed. +:: ++$ versioned-state $%(state-0) +:: $state: the current persisted shape. +:: ++$ state state-0 +-- diff --git a/desk/ted/pioneer/buckets/authorize-delete.hoon b/desk/ted/pioneer/buckets/authorize-delete.hoon new file mode 100644 index 0000000000..c1325c78f4 --- /dev/null +++ b/desk/ted/pioneer/buckets/authorize-delete.hoon @@ -0,0 +1,21 @@ +:: Exchange an opaque delete capability for an object-bound verdict. +:: +/- spider +/+ *strandio +/= buckets-json /lib/buckets/json +=, strand=strand:spider +=, dejs:format +^- thread:spider +|= arg=vase +=/ m (strand ,vase) +^- form:m +=+ !<(arg=(unit json) arg) +?> ?=(^ arg) +=* jon u.arg +=/ capability=@t + (so (get:dejs:buckets-json 'capability' jon)) +=/ object=@t + (so (get:dejs:buckets-json 'objectId' jon)) +;< verdict=json bind:m + (scry json /gx/buckets/v1/broker/delete/[capability]/[object]/json) +(pure:m !>(verdict)) diff --git a/desk/ted/pioneer/buckets/authorize-read.hoon b/desk/ted/pioneer/buckets/authorize-read.hoon new file mode 100644 index 0000000000..c24575553a --- /dev/null +++ b/desk/ted/pioneer/buckets/authorize-read.hoon @@ -0,0 +1,21 @@ +:: Exchange an opaque read capability for an object-bound verdict. +:: +/- spider +/+ *strandio +/= buckets-json /lib/buckets/json +=, strand=strand:spider +=, dejs:format +^- thread:spider +|= arg=vase +=/ m (strand ,vase) +^- form:m +=+ !<(arg=(unit json) arg) +?> ?=(^ arg) +=* jon u.arg +=/ capability=@t + (so (get:dejs:buckets-json 'capability' jon)) +=/ object=@t + (so (get:dejs:buckets-json 'objectId' jon)) +;< verdict=json bind:m + (scry json /gx/buckets/v1/broker/read/[capability]/[object]/json) +(pure:m !>(verdict)) diff --git a/desk/ted/pioneer/buckets/authorize-upload.hoon b/desk/ted/pioneer/buckets/authorize-upload.hoon new file mode 100644 index 0000000000..fcfd2f9432 --- /dev/null +++ b/desk/ted/pioneer/buckets/authorize-upload.hoon @@ -0,0 +1,25 @@ +:: Exchange an opaque upload capability for authoritative Bucket metadata. +:: +/- spider, b=buckets +/+ *strandio +/= buckets-json /lib/buckets/json +=, strand=strand:spider +=, dejs:format +^- thread:spider +|= arg=vase +=/ m (strand ,vase) +^- form:m +;< our=@p bind:m get-our +=+ !<(arg=(unit json) arg) +?> ?=(^ arg) +=* jon u.arg +=/ capability=@t + (so (get:dejs:buckets-json 'capability' jon)) +=/ reservation=@t + (so (get:dejs:buckets-json 'brokerReservationId' jon)) +;< ~ bind:m + %+ poke [our %buckets] + buckets-broker-command-1+!>(`broker-command:b`[%authorize-upload capability reservation]) +;< verdict=json bind:m + (scry json /gx/buckets/v1/broker/upload/[capability]/[reservation]/json) +(pure:m !>(verdict)) diff --git a/desk/ted/pioneer/buckets/complete-upload.hoon b/desk/ted/pioneer/buckets/complete-upload.hoon new file mode 100644 index 0000000000..9c20956788 --- /dev/null +++ b/desk/ted/pioneer/buckets/complete-upload.hoon @@ -0,0 +1,37 @@ +:: Commit Memex's verified object receipt to the Bucket manifest. +:: +/- spider, b=buckets +/+ *strandio +/= buckets-json /lib/buckets/json +=, strand=strand:spider +=, dejs:format +^- thread:spider +|= arg=vase +=/ m (strand ,vase) +^- form:m +;< our=@p bind:m get-our +=+ !<(arg=(unit json) arg) +?> ?=(^ arg) +=* jon u.arg +=/ reservation=@t + (so (get:dejs:buckets-json 'brokerReservationId' jon)) +=/ receipt=json + (get:dejs:buckets-json 'receipt' jon) +=/ object=@t + (so (get:dejs:buckets-json 'objectId' receipt)) +=/ host=@t + (so (get:dejs:buckets-json 'host' receipt)) +=/ bucket=@t + (so (get:dejs:buckets-json 'bucketId' receipt)) +=/ size=@ud + (ni (get:dejs:buckets-json 'size' receipt)) +=/ mime=@t + (so (get:dejs:buckets-json 'mimeType' receipt)) +=/ broker-receipt=broker-receipt:b + [reservation object host bucket size mime] +;< ~ bind:m + %+ poke [our %buckets] + buckets-broker-command-1+!>(`broker-command:b`[%complete-upload broker-receipt]) +;< verdict=json bind:m + (scry json /gx/buckets/v1/broker/complete/[reservation]/json) +(pure:m !>(verdict)) diff --git a/desk/tests/app/buckets.hoon b/desk/tests/app/buckets.hoon new file mode 100644 index 0000000000..5344d39e61 --- /dev/null +++ b/desk/tests/app/buckets.hoon @@ -0,0 +1,2077 @@ +:: behavior tests for the %buckets Gall agent +:: +/- bu=buckets, gv=groups-ver +/+ *test-agent +/= buckets-agent /app/buckets +/= buckets-json /lib/buckets/json +|% +++ dap %buckets +:: +++ flag + ^- flag:bu + [~sampel-palnet %project-files] +:: +++ group + ^- flag:bu + [~sampel-palnet %test-group] +:: +:: The bowl pins eny, so a session id is predictable: +begin-upload mints it +:: as `@uv`eny.bowl, which is also the token handed to the broker. +:: +++ seed 0v1234 +++ seed-token ^-(@t (scot %uv seed)) +:: +++ setup + =/ m (mare ,~) + ^- form:m + ;< ~ bind:m (jab-bowl |=(b=bowl b(our ~sampel-palnet, src ~sampel-palnet))) + ;< * bind:m (do-init dap buckets-agent) + ;< ~ bind:m + (jab-bowl |=(b=bowl b(now ~2026.1.1, eny seed))) + (pure:m ~) +:: +++ setup-as + |= who=ship + =/ m (mare ,~) + ^- form:m + ;< ~ bind:m (jab-bowl |=(b=bowl b(our who, src who))) + ;< * bind:m (do-init dap buckets-agent) + ;< ~ bind:m + (jab-bowl |=(b=bowl b(now ~2026.1.1, eny seed))) + (pure:m ~) +:: +:: +ask: submit a local client action under a request id. +:: +++ ask + |= [rid=@uv act=action:bu] + =/ m (mare ,(list card)) + ^- form:m + (do-poke %buckets-action-1 !>(`command:bu`[rid act])) +:: +++ create + =/ m (mare ,~) + ^- form:m + ;< * bind:m + (ask 0v0 [%create %project-files 'Project Files' group ~ ~]) + (pure:m ~) +:: +++ file-of + |= ent=entry:bu + ^- file:bu + ?- -.kind.ent + %folder ~|(%expected-file !!) + %file +.kind.ent + == +:: +++ state-for + |= [st=state-0:bu =flag:bu] + ^- bucket-state:bu + =/ sp=space:bu (~(got by spaces.st) flag) + (need state.sp) +:: +++ only-session + |= st=state-0:bu + ^- upload-session:bu + =/ sessions=(list upload-session:bu) ~(val by sessions.st) + ?~(sessions !! i.sessions) +:: +:: +grant-fact: the single card a settled local action produces. +:: +++ grant-fact + |= [rid=@uv body=response-body:bu] + (ex-fact ~[/v1/requests] %buckets-req-response-1 !>(`req-response:bu`[rid body])) +:: +:: +genuine-scries: %buckets authenticates itself to the broker with the +:: secret %genuine holds, so anything that mints or revokes a token reads it. +:: +++ group-exists-path + |= pax=path + ^- ? + ?=([%gu @ %groups @ %groups @ @ ~] pax) +:: +++ genuine-scries + |= pax=path + ^- (unit vase) + ?: ?=([%gu @ %genuine *] pax) `!>(&) + ?: (group-exists-path pax) `!>(&) + ?: ?=([%gx @ %genuine @ %secret %json ~] pax) + `!>(`json`[%s '0wsecret']) + ~ +:: +:: +only-iris: the single outbound HTTP request in a card list. +:: +++ only-iris + |= caz=(list card) + ^- [=wire =request:http] + =/ found=(list [wire request:http]) + %+ murn caz + |= =card + ^- (unit [wire request:http]) + ?. ?=([%pass * %arvo %i %request * *] card) ~ + `[p.card request.q.card] + ?~ found ~|(%no-iris-card !!) + i.found +:: +:: +ex-iris: an outbound HTTP request on the given wire. The request itself +:: is checked field by field rather than matched whole, so the assertion +:: does not depend on JSON key order. +:: +++ ex-iris + |= =wire + |= car=card + ^- tang + ?. ?=([%pass * %arvo %i %request * *] car) + ~[leaf+"expected an outbound http request, got:" >car<] + ?: =(wire p.car) ~ + ~[leaf+"iris request on the wrong wire" >wire< >p.car<] +:: +:: +reader-wire-for: the wire one reader-sync rides. The revision is what +:: the broker answers about, so it is what the wire carries. +:: +++ reader-wire-for + |= [reader=@p revision=@ud] + ^- wire + %+ weld + /buckets/reader/~sampel-palnet/project-files + /(scot %p reader)/(scot %ud revision) +:: +:: +granted-count: pairs currently holding a grant, whatever their sync state. +:: +++ granted-count + |= st=state-0:bu + ^- @ud + %- lent + %+ skim ~(val by readers.st) + |=(sync=reader-sync:bu ?=(%granted -.desired.sync)) +:: +:: +owed-count: pairs the broker has not caught up with. +:: +++ owed-count + |= st=state-0:bu + ^- @ud + %- lent + %+ skim ~(val by readers.st) + :: mirrors +owed: a revision the broker rejected as invalid is not owed + |=(sync=reader-sync:bu &(!failed.sync (gth revision.sync synced.sync))) +:: +++ iris-ok + ^- sign-arvo + [%iris %http-response [%finished [200 ~] ~]] +:: +++ iris-status + |= code=@ud + ^- sign-arvo + [%iris %http-response [%finished [code ~] ~]] +:: +:: +iris-revision: an answer carrying the revision the broker says it holds. +:: +++ iris-revision + |= [code=@ud revision=@ud] + ^- sign-arvo + =/ body=@t + (en:json:html (pairs:enjs:format ~[['currentRevision' (numb:enjs:format revision)]])) + :+ %iris %http-response + :+ %finished [code ~] + `['application/json' [(met 3 body) body]] +:: +:: +iris-receipt: the broker's full answer to a sync, which says outright +:: whether it took the write rather than leaving it to be inferred. +:: +++ iris-receipt + |= [revision=@ud applied=?] + ^- sign-arvo + =/ body=@t + %- en:json:html + %- pairs:enjs:format + :~ ['currentRevision' (numb:enjs:format revision)] + ['applied' b+applied] + == + :+ %iris %http-response + :+ %finished [200 ~] + `['application/json' [(met 3 body) body]] +:: +:: +iris-refusal: the broker's shape for a failure it has classified. +:: +++ iris-refusal + |= [code=@ud retryable=?] + ^- sign-arvo + =/ body=@t + %- en:json:html + %- pairs:enjs:format + :~ ['code' s+'malformed_input'] + ['message' s+'nope'] + ['retryable' b+retryable] + == + :+ %iris %http-response + :+ %finished [code ~] + `['application/json' [(met 3 body) body]] +:: +:: +sync-for: the pair's sync record, for asserting revisions directly. +:: +++ sync-for + |= [st=state-0:bu reader=@p] + ^- reader-sync:bu + (~(got by readers.st) [flag reader]) +:: +:: +group-changed: a well-formed %groups fact about `group`. +:: +:: The agent decodes these strictly, as %channels-server does, so a stub +:: noun here would pass a test the live path could not survive. %meta is the +:: least interesting variant that still says "this group changed". +:: +++ group-changed + ^- r-groups:v9:gv + [group [%meta ['' '' '' '']]] +:: +:: +role-deleted: the same, saying a role is gone. +:: +++ role-deleted + |= roles=(set @tas) + ^- r-groups:v9:gv + [group [%role roles [%del ~]]] +:: +:: +answers: the terminal responses in a card list. Not every fact is one -- +:: /lib/verb emits its own -- so this matches on the mark rather than on +:: %give %fact, which quietly counted logging as an answer. +:: +++ answers + |= caz=(list card) + ^- (list card) + %+ skim caz + |=(=card ?=([%give %fact * %buckets-req-response-1 *] card)) +:: +:: +secretless-scries: the group is there, but %genuine has not initialised. +:: A real state on a fresh ship, and the one place a mint is answered in the +:: same event that records what is owed. +:: +++ secretless-scries + |= pax=path + ^- (unit vase) + ?: ?=([%gu @ %genuine *] pax) `!>(|) + ?: (group-exists-path pax) `!>(&) + ~ +:: +:: +reader-scries: %genuine plus a group that grants read access. +:: +++ reader-scries + |= pax=path + ^- (unit vase) + ?: ?=([%gx @ %groups @ %v2 %groups @ @ %channels %can-read %noun ~] pax) + `!>(|=([who=ship =nest:bu] &)) + (genuine-scries pax) +:: +:: +revoked-scries: the same, after read access has been pulled. +:: +++ revoked-scries + |= pax=path + ^- (unit vase) + ?: ?=([%gx @ %groups @ %v2 %groups @ @ %channels %can-read %noun ~] pax) + `!>(|=([who=ship =nest:bu] |)) + (genuine-scries pax) +:: +:: +missing-group-scries: %groups no longer holds the group, which is what a +:: deletion looks like. The permission gates must not be reached at all here +:: -- if they were, the mock would block and the example would fail. +:: +++ missing-group-scries + |= pax=path + ^- (unit vase) + ?: (group-exists-path pax) `!>(|) + (genuine-scries pax) +:: +++ deny-group-scries + |= pax=path + ^- (unit vase) + ?: (group-exists-path pax) `!>(&) + ?. ?=([%gx @ %groups @ %v2 %groups @ @ %channels %can-read %noun ~] pax) ~ + `!>(|=([who=ship =nest:bu] |)) +:: +++ group-permission-scries + |= pax=path + ^- (unit vase) + ?: (group-exists-path pax) `!>(&) + ?: ?=([%gx @ %groups @ %v2 %groups @ @ %channels %can-read %noun ~] pax) + `!>(|=([who=ship =nest:bu] &)) + ?. ?=([%gx @ %groups @ %v2 %groups @ @ %channels %buckets @ @ %can-write @ %noun ~] pax) ~ + `!>(`(unit [admin=? roles=(set @tas)])`[~ [admin=| roles=(silt ~[%editor])]]) +:: +++ missing-group-permission-scries + |= pax=path + ^- (unit vase) + ?: (group-exists-path pax) `!>(&) + ?: ?=([%gx @ %groups @ %v2 %groups @ @ %channels %can-read %noun ~] pax) + `!>(|=([who=ship =nest:bu] &)) + ?. ?=([%gx @ %groups @ %v2 %groups @ @ %channels %buckets @ @ %can-write @ %noun ~] pax) ~ + `!>(`(unit [admin=? roles=(set @tas)])`~) +:: +++ allow-admin-create-scries + |= pax=path + ^- (unit vase) + ?: (group-exists-path pax) `!>(&) + ?. ?=([%gx @ %groups @ %v2 %groups @ @ %seats @ %is-admin %noun ~] pax) ~ + `!>(&) +:: +++ deny-admin-create-scries + |= pax=path + ^- (unit vase) + ?: (group-exists-path pax) `!>(&) + ?. ?=([%gx @ %groups @ %v2 %groups @ @ %seats @ %is-admin %noun ~] pax) ~ + `!>(|) +:: +:: A pending upload is answered privately and announced to nobody. The token +:: goes to the requester alone, and the manifest gains nothing until the +:: object lands — so no subscriber can learn the session id. +:: +++ test-begin-upload-grants-token-privately + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< caz=(list card) b + (ask 0v7 [%bucket flag [%begin-upload ~ 'private.pdf' 'application/pdf' 42 ~]]) + ;< ~ b + %+ ex-cards caz + :~ %+ grant-fact 0v7 + [%grant seed-token 2 (add ~2026.1.1 ~h1)] + == + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + =/ bs=bucket-state:bu (state-for st flag) + =/ ses=upload-session:bu (only-session st) + %+ ex-equal + !> :* ~(wyt by entries.bs) + revision.bs + [flag.ses status.ses requested-by.ses] + [id.entry.ses name.entry.ses] + == + !> :* 0 + 0 + [flag %pending ~sampel-palnet] + [2 'private.pdf'] + == +:: +:: Input Memex would refuse is refused here first, so a bad request never +:: allocates an entry id or a session. +:: +++ test-begin-upload-rejects-bad-input + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< caz=(list card) b + (ask 0v8 [%bucket flag [%begin-upload ~ 'empty.pdf' 'application/pdf' 0 ~]]) + ;< ~ b + %+ ex-cards caz + :~ %+ grant-fact 0v8 + [%error %invalid-input 'file size must be greater than zero'] + == + ;< caz2=(list card) b + (ask 0v9 [%bucket flag [%begin-upload ~ 'bad.pdf' 'pdf' 42 ~]]) + ;< ~ b + %+ ex-cards caz2 + :~ %+ grant-fact 0v9 + [%error %invalid-input 'missing or malformed content type'] + == + ;< caz3=(list card) b + (ask 0v10 [%bucket flag [%begin-upload `99 'orphan.pdf' 'application/pdf' 42 ~]]) + ;< ~ b + %+ ex-cards caz3 + :~ %+ grant-fact 0v10 + [%error %not-found 'no such parent folder'] + == + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + (ex-equal !>([~(wyt by sessions.st) next-id.st]) !>([0 1])) +:: +:: A pending upload is absent from the manifest until the object lands, then +:: appears in one revision. Driven through the broker, which is now the only +:: completion path. +:: +++ test-upload-invisible-until-ready + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + =/ rid=@t '00000000-0000-0000-0000-00000000000a' + ;< ~ b setup + ;< ~ b create + ;< * b (ask 0v1 [%bucket flag [%create-folder ~ 'Launch']]) + ;< * b (ask 0v2 [%bucket flag [%begin-upload `2 'meadow.png' 'image/png' 2.048 ~]]) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + =/ bs=bucket-state:bu (state-for st flag) + =/ ses=upload-session:bu (only-session st) + :: the folder is published, the pending file is not + ;< ~ b (ex-equal !>([~(wyt by entries.bs) revision.bs]) !>([1 1])) + ;< * b + (do-poke %buckets-broker-command-1 !>(`broker-command:bu`[%authorize-upload seed-token rid])) + =/ fil=file:bu (file-of entry.ses) + =/ receipt=broker-receipt:bu + [rid object-key.fil 'sampel-palnet' (scot %ud id.bucket.bs) 2.048 'image/png'] + ;< * b + (do-poke %buckets-broker-command-1 !>(`broker-command:bu`[%complete-upload receipt])) + ;< sv2=vase b get-save + =/ st2=state-0:bu !<(state-0:bu sv2) + =/ bs2=bucket-state:bu (state-for st2 flag) + =/ ent=entry:bu (~(got by entries.bs2) id.entry.ses) + =/ fil2=file:bu (file-of ent) + =/ ses2=upload-session:bu (~(got by sessions.st2) id.ses) + %+ ex-equal + !>([~(wyt by entries.bs2) revision.bs2 status.fil2 status.ses2 parent.ent]) + !>([2 2 %ready %complete `2]) +:: +:: The session id is the broker token. Memex's reservation binds once, and +:: the file only becomes visible after a verified receipt. +:: +:: Cancelling says the uploader gave up, not that the bytes did not land -- +:: it cannot know that. Its own completion call is what failed, and losing +:: that answer says nothing about whether the broker took the object. So a +:: completion arriving afterwards still publishes; refusing it would leave +:: the object stored and paid for with nothing in the manifest for it. +:: +++ test-a-cancelled-session-still-publishes-a-completion + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + =/ rid=@t '00000000-0000-0000-0000-000000000009' + ;< ~ b setup + ;< ~ b create + ;< * b (ask 0v1 [%bucket flag [%begin-upload ~ 'private.pdf' 'application/pdf' 42 ~]]) + ;< * b + (do-poke %buckets-broker-command-1 !>(`broker-command:bu`[%authorize-upload seed-token rid])) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + =/ ses=upload-session:bu (only-session st) + =/ bs=bucket-state:bu (state-for st flag) + =/ fil=file:bu (file-of entry.ses) + :: the uploader's completion call fails, so it withdraws + ;< * b (ask 0v2 [%bucket flag [%cancel-upload id.ses 'connection lost']]) + ;< mid=vase b get-save + =/ cancelled=upload-session:bu (only-session !<(state-0:bu mid)) + ;< ~ b (ex-equal !>(status.cancelled) !>(%cancelled)) + :: but the broker did take the bytes, and says so + =/ receipt=broker-receipt:bu + [rid object-key.fil 'sampel-palnet' (scot %ud id.bucket.bs) 42 'application/pdf'] + ;< * b + (do-poke %buckets-broker-command-1 !>(`broker-command:bu`[%complete-upload receipt])) + ;< after=vase b get-save + =/ st2=state-0:bu !<(state-0:bu after) + =/ bs2=bucket-state:bu (state-for st2 flag) + =/ published=upload-session:bu (only-session st2) + :: the entry is in the manifest and the session settled as complete + %+ ex-equal + !>([~(wyt by entries.bs2) status.published]) + !>([1 %complete]) +:: +++ test-broker-upload-lifecycle + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + =/ rid=@t '00000000-0000-0000-0000-000000000001' + ;< ~ b setup + ;< ~ b create + ;< * b (ask 0v1 [%bucket flag [%begin-upload ~ 'private.pdf' 'application/pdf' 42 ~]]) + ;< * b + (do-poke %buckets-broker-command-1 !>(`broker-command:bu`[%authorize-upload seed-token rid])) + ;< upload-cage=cage b + (got-peek /x/v1/broker/upload/[seed-token]/[rid]) + =/ upload-result=@t + (so:dejs:format (get:dejs:buckets-json 'result' !<(json q.upload-cage))) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + =/ ses=upload-session:bu (only-session st) + =/ bs=bucket-state:bu (state-for st flag) + =/ fil=file:bu (file-of entry.ses) + ;< ~ b (ex-equal !>(reservation.ses) !>(`rid)) + =/ receipt=broker-receipt:bu + [rid object-key.fil 'sampel-palnet' (scot %ud id.bucket.bs) 42 'application/pdf'] + ;< * b + (do-poke %buckets-broker-command-1 !>(`broker-command:bu`[%complete-upload receipt])) + :: Completion retries are idempotent — no second entry, no second revision. + ;< * b + (do-poke %buckets-broker-command-1 !>(`broker-command:bu`[%complete-upload receipt])) + ;< sv2=vase b get-save + =/ st2=state-0:bu !<(state-0:bu sv2) + =/ bs2=bucket-state:bu (state-for st2 flag) + =/ ent2=entry:bu (~(got by entries.bs2) id.entry.ses) + =/ fil2=file:bu (file-of ent2) + ;< complete-cage=cage b (got-peek /x/v1/broker/complete/[rid]) + =/ complete-result=@t + (so:dejs:format (get:dejs:buckets-json 'result' !<(json q.complete-cage))) + %+ ex-equal + !> :* upload-result + complete-result + status.fil2 + [~(wyt by entries.bs2) revision.bs2] + == + !>([%'authorized' %'completed' %ready [1 1]]) +:: +:: One read token covers the whole bucket: it authorizes any ready object in +:: it, and nothing outside it. Deletes stay bound to a single object. +:: +++ test-read-token-covers-the-bucket + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + =/ rid=@t '00000000-0000-0000-0000-000000000002' + ;< ~ b setup + ;< ~ b create + ;< * b (ask 0v1 [%bucket flag [%begin-upload ~ 'private.pdf' 'application/pdf' 42 ~]]) + ;< * b + (do-poke %buckets-broker-command-1 !>(`broker-command:bu`[%authorize-upload seed-token rid])) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + =/ bs=bucket-state:bu (state-for st flag) + =/ ses=upload-session:bu (only-session st) + =/ fil=file:bu (file-of entry.ses) + =/ receipt=broker-receipt:bu + [rid object-key.fil 'sampel-palnet' (scot %ud id.bucket.bs) 42 'application/pdf'] + ;< * b + (do-poke %buckets-broker-command-1 !>(`broker-command:bu`[%complete-upload receipt])) + :: A fresh eny so the read token differs from the upload session id. + ;< ~ b (jab-bowl |=(bol=bowl bol(eny 0v5678))) + ;< ~ b (set-scry-gate genuine-scries) + ;< read-caz=(list card) b (ask 0v2 [%bucket flag [%issue-bucket-read ~]]) + =/ read-token=@t (scot %uv 0v5678) + =/ expiry=@da (add ~2026.1.1 ~d1) + :: A mint is not a token yet: it goes to the broker first, and the client + :: is told %pending until the broker has it. + ;< ~ b + %+ ex-cards read-caz + :~ (ex-iris (reader-wire-for ~sampel-palnet 1)) + :: One timer drives everything still owed. It is cancelled before it + :: is set, and lands on a fixed grid rather than now-plus-a-minute, + :: so arming it twice cannot leave two timers behind. + %- ex-arvo + [/buckets/reader-retry [%b %rest (add ~2026.1.1 ~m2)]] + %- ex-arvo + [/buckets/reader-retry [%b %wait (add ~2026.1.1 ~m2)]] + (grant-fact 0v2 [%pending ~]) + == + =/ push=[=wire =request:http] (only-iris read-caz) + =/ body=json + ?~ body.request.push ~ + (need (de:json:html q.u.body.request.push)) + =/ field=$-(@t @t) + |=(key=@t (so:dejs:format (get:dejs:buckets-json key body))) + ;< ~ b + %+ ex-equal + !> :* method.request.push + url.request.push + (field 'token') + (field 'bucketHost') + (field 'bucketName') + (field 'actorShip') + :: the credential goes in a header, never the URL + (~(got by (malt header-list.request.push)) 'x-landscape-token') + == + !> :* %'PUT' + 'https://memex.tlon.network/v2/buckets/tokens/sampel-palnet' + read-token + 'sampel-palnet' + 'project-files' + 'sampel-palnet' + '0wsecret' + == + :: Only once the broker has accepted it does the token become real: the + :: refresh is armed, then the request settles with the token. + ;< confirm-caz=(list card) b (do-arvo wire.push iris-ok) + ;< ~ b + %+ ex-cards confirm-caz + :~ %- ex-arvo + :* /buckets/token/~sampel-palnet/project-files + [%b %wait (sub expiry ~h1)] + == + (grant-fact 0v2 [%token read-token expiry]) + == + :: The client reads the held token over Eyre as JSON, so the peek has to + :: answer a mark that grows to json -- %noun does not. + ;< held-cage=cage b + (got-peek /x/v1/buckets/~sampel-palnet/project-files/read-token) + ;< ~ b + %+ ex-equal + !> :- p.held-cage + (read-token:enjs:buckets-json !<(read-token:bu q.held-cage)) + !> :- %buckets-read-token-1 + %- pairs:enjs:format + :~ ['token' s+read-token] + ['expiresAt' s+(scot %da expiry)] + == + ;< ok-cage=cage b + (got-peek /x/v1/broker/read/[read-token]/[object-key.fil]) + ;< bad-cage=cage b + (got-peek /x/v1/broker/read/[read-token]/wrong-object) + =/ ok-json=json !<(json q.ok-cage) + =/ ok-result=@t + (so:dejs:format (get:dejs:buckets-json 'result' ok-json)) + =/ payload=json (get:dejs:buckets-json 'read' ok-json) + =/ ok-name=@t + (so:dejs:format (get:dejs:buckets-json 'displayFilename' payload)) + =/ bad-result=@t + (so:dejs:format (get:dejs:buckets-json 'result' !<(json q.bad-cage))) + :: the token names no object, so the bucket's own file is authorized while + :: a key that belongs to no entry is refused + (ex-equal !>([ok-result ok-name bad-result]) !>([%'authorized' 'private.pdf' %'denied'])) +:: +:: A refused sync is owed, not lost: the desired state stands and the retry +:: timer sends it again. What must not happen is handing the token out before +:: the broker has it, so nothing is served while the pair is still owed. +:: +++ test-refused-sync-is-owed-not-served + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< ~ b (set-scry-gate reader-scries) + ;< mint-caz=(list card) b (ask 0v2 [%bucket flag [%issue-bucket-read ~]]) + =/ push=[=wire =request:http] (only-iris mint-caz) + ;< * b (do-arvo wire.push (iris-status 404)) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + :: a 404 is a refusal like any other now + :: a second reader, so the mint is fresh rather than the one just kept, + :: against a broker that refuses it outright + ;< ~ b (jab-bowl |=(bol=bowl bol(eny 0v9999))) + ;< again-caz=(list card) b + %- (do-as ~bus) + %+ do-poke %buckets-command-1 + !>(`command:bu`[0v3 [%bucket flag [%issue-bucket-read ~]]]) + =/ retry=[=wire =request:http] (only-iris again-caz) + ;< * b (do-arvo wire.retry (iris-status 500)) + ;< sv2=vase b get-save + =/ st2=state-0:bu !<(state-0:bu sv2) + :: nothing served either way, and both pairs still owed + %+ ex-equal + !> :* (~(has by read-tokens.st) flag) + (owed-count st) + (owed-count st2) + == + !>([%.n 1 2]) +:: +:: Deleting a folder has to take the uploads underneath it. Their entries are +:: deliberately absent from the manifest until the object lands, so they are +:: never among the folder's descendants -- and a session left behind would +:: later publish a file parented to a folder that no longer exists. +:: +++ test-deleting-a-folder-drops-uploads-under-it + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< * b (ask 0v1 [%bucket flag [%create-folder ~ 'Launch']]) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + =/ bs=bucket-state:bu (state-for st flag) + =/ folder=@ud + =/ ents ~(tap by entries.bs) + ?~(ents !! p.i.ents) + :: an upload lands inside that folder, and one at the root + ;< * b + (ask 0v2 [%bucket flag [%begin-upload `folder 'inside.pdf' 'application/pdf' 9 ~]]) + ;< ~ b (jab-bowl |=(bol=bowl bol(eny 0v4321))) + ;< * b + (ask 0v3 [%bucket flag [%begin-upload ~ 'outside.pdf' 'application/pdf' 9 ~]]) + ;< sv2=vase b get-save + =/ st2=state-0:bu !<(state-0:bu sv2) + =/ before=@ud ~(wyt by sessions.st2) + :: deleting the folder recursively takes the upload inside it, not the other + ;< * b (ask 0v5 [%bucket flag [%entry folder [%delete &]]]) + ;< sv3=vase b get-save + =/ st3=state-0:bu !<(state-0:bu sv3) + =/ survivors=(list @t) + %+ turn ~(val by sessions.st3) + |=(ses=upload-session:bu name.entry.ses) + (ex-equal !>([before survivors]) !>([2 ~['outside.pdf']])) +:: +:: A revoke issued while a grant is still in flight supersedes it, because +:: it carries a higher revision — the broker keeps the newer state whichever +:: order the two arrive in. The client waiting on that grant is told so +:: rather than left to time out, and the stale ack changes nothing when it +:: finally lands. +:: +++ test-revoke-supersedes-an-in-flight-grant + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< ~ b (set-scry-gate reader-scries) + ;< mint-caz=(list card) b + %- (do-as ~bus) + %+ do-poke %buckets-command-1 + !>(`command:bu`[0v9 [%bucket flag [%issue-bucket-read ~]]]) + =/ grant=[=wire =request:http] (only-iris mint-caz) + :: access is pulled while that grant is still in flight + ;< ~ b (set-scry-gate revoked-scries) + ;< revoke-caz=(list card) b + %^ do-agent + /groups + [~sampel-palnet %groups] + [%fact %group-response-1 !>(group-changed)] + ;< ~ b + %+ ex-cards (skim revoke-caz |=(car=card ?=([%give %fact *] car))) + :~ %+ ex-fact ~[/v1/request/~bus/0v9] + :- %buckets-req-response-1 + !> ^- req-response:bu + :- 0v9 + [%error %not-authorized 'access changed while the token was being issued'] + == + :: the stale grant ack arrives afterwards and resurrects nothing + ;< * b (do-arvo wire.grant iris-ok) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + (ex-equal !>((granted-count st)) !>(0)) +:: +:: A retry of a dropped POST reuses its request id, and must not run the +:: action a second time -- a create-folder or begin-upload would duplicate +:: state, and the answer would go to a connection that is already gone. +:: +++ test-reused-request-id-does-not-run-twice + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + =/ body=@t + '{"requestId":"0v30","action":{"type":"create-folder","flag":{"host":"~sampel-palnet","name":"project-files"},"name":"Launch"}}' + ;< * b (http-post & body) + ;< sv=vase b get-save + =/ before=@ud ~(wyt by entries:(state-for !<(state-0:bu sv) flag)) + :: the same id again, once the first has settled + ;< again=(list card) b (http-post & body) + ;< sv2=vase b get-save + =/ after=@ud ~(wyt by entries:(state-for !<(state-0:bu sv2) flag)) + :: one folder, and the retry still gets an answer + =/ answered=? ?=(^ (skim again |=(car=card ?=([%give %fact *] car)))) + (ex-equal !>([before after answered]) !>([1 1 %.y])) +:: +:: The broker classifies its own failures. A validation refusal will answer +:: the same way next time, so it stops being owed; a service failure is worth +:: another go and stays owed for the retry timer. +:: +++ test-refusal-is-retried-only-when-the-broker-says-so + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< ~ b (set-scry-gate reader-scries) + :: a temporary failure stays owed + ;< caz=(list card) b (ask 0v40 [%bucket flag [%issue-bucket-read ~]]) + =/ first=[=wire =request:http] (only-iris caz) + ;< * b (do-arvo wire.first (iris-refusal 503 &)) + ;< sv=vase b get-save + =/ owed-after-503=@ud (owed-count !<(state-0:bu sv)) + :: a validation refusal does not + ;< ~ b (jab-bowl |=(bol=bowl bol(eny 0v4141))) + ;< caz2=(list card) b + %- (do-as ~bus) + %+ do-poke %buckets-command-1 + !>(`command:bu`[0v41 [%bucket flag [%issue-bucket-read ~]]]) + =/ second=[=wire =request:http] (only-iris caz2) + ;< denied=(list card) b (do-arvo wire.second (iris-refusal 400 |)) + ;< sv2=vase b get-save + =/ st2=state-0:bu !<(state-0:bu sv2) + :: the rejected pair is no longer owed, and its asker was told + =/ answered=? + ?=(^ (skim denied |=(car=card ?=([%give %fact *] car)))) + %+ ex-equal + !>([owed-after-503 (owed-count st2) answered]) + !>([1 1 %.y]) +:: +:: Re-granting after a revoke has to outrank it, or the broker keeps the +:: revoked state and the reader never gets back in. +:: +++ test-regrant-after-revoke-outranks-it + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< ~ b (set-scry-gate reader-scries) + ;< * b + %- (do-as ~bus) + %+ do-poke %buckets-command-1 + !>(`command:bu`[0v20 [%bucket flag [%issue-bucket-read ~]]]) + :: access is pulled, then restored + ;< ~ b (set-scry-gate revoked-scries) + ;< * b + %^ do-agent + /groups + [~sampel-palnet %groups] + [%fact %group-response-1 !>(group-changed)] + ;< ~ b (set-scry-gate reader-scries) + ;< ~ b (jab-bowl |=(bol=bowl bol(eny 0v7777))) + ;< * b + %- (do-as ~bus) + %+ do-poke %buckets-command-1 + !>(`command:bu`[0v21 [%bucket flag [%issue-bucket-read ~]]]) + ;< sv=vase b get-save + =/ sync=reader-sync:bu (sync-for !<(state-0:bu sv) ~bus) + :: grant, revoke, grant -- each strictly above the last + %+ ex-equal + !>([revision.sync -.desired.sync]) + !>([3 %granted]) +:: +:: The same answer delivered twice settles the pair once. Nothing about the +:: second delivery should re-answer a client or move the revision. +:: +++ test-duplicate-delivery-is-harmless + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< ~ b (set-scry-gate genuine-scries) + ;< caz=(list card) b (ask 0v22 [%bucket flag [%issue-bucket-read ~]]) + =/ push=[=wire =request:http] (only-iris caz) + ;< first=(list card) b (do-arvo wire.push iris-ok) + ;< again=(list card) b (do-arvo wire.push iris-ok) + ;< sv=vase b get-save + =/ sync=reader-sync:bu (sync-for !<(state-0:bu sv) ~sampel-palnet) + :: the second delivery answers nobody and changes nothing + ;< ~ b (ex-cards again ~) + %+ ex-equal + !>([revision.sync synced.sync]) + !>([1 1]) +:: +:: If the broker reports a revision above ours, our counter is behind it -- +:: state loss on our side, or an earlier incarnation. Adopt its number and +:: re-send, or everything we say from here on is discarded as stale. +:: +:: It arrives on the success path, because a stale write is not an error to +:: the broker: it keeps the higher revision and answers 200 with the number +:: it kept. That is the only route by which we catch up. +:: +++ test-adopts-a-higher-revision-from-the-broker + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< ~ b (set-scry-gate genuine-scries) + ;< caz=(list card) b (ask 0v23 [%bucket flag [%issue-bucket-read ~]]) + =/ push=[=wire =request:http] (only-iris caz) + :: the write was stale; the broker names what it holds instead + ;< resent=(list card) b (do-arvo wire.push (iris-revision 200 41)) + =/ retry=[=wire =request:http] (only-iris resent) + ;< sv=vase b get-save + =/ sync=reader-sync:bu (sync-for !<(state-0:bu sv) ~sampel-palnet) + :: we jump past it and say the same thing again at a revision that wins + %+ ex-equal + !>([revision.sync wire.retry]) + !>([42 (reader-wire-for ~sampel-palnet 42)]) +:: +:: A rejection is a rejection whatever else it says. Reading a revision out +:: of a non-2xx body and treating it as agreement would install a grant the +:: broker just refused, and hand the client a token it will not honour. +:: +++ test-a-rejection-carrying-a-revision-is-still-a-rejection + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< ~ b (set-scry-gate genuine-scries) + ;< caz=(list card) b (ask 0v23 [%bucket flag [%issue-bucket-read ~]]) + =/ push=[=wire =request:http] (only-iris caz) + ;< refused=(list card) b (do-arvo wire.push (iris-revision 400 1)) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + =/ sync=reader-sync:bu (sync-for st ~sampel-palnet) + :: nothing was confirmed, nothing is servable, nobody was told otherwise + %+ ex-equal + !>([synced.sync ~(wyt by read-tokens.st) (lent (answers refused))]) + !>([0 0 0]) +:: +:: Deleting a group must revoke its buckets' tokens, not crash trying. The +:: permission scry answers no-such-path once the group is gone, which makes .^ +:: crash rather than return -- so the pass that revokes has to check the group +:: is still there before asking about it. +:: +++ test-deleted-group-revokes-rather-than-crashing + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< ~ b (set-scry-gate reader-scries) + ;< mint-caz=(list card) b + %- (do-as ~bus) + %+ do-poke %buckets-command-1 + !>(`command:bu`[0v12 [%bucket flag [%issue-bucket-read ~]]]) + =/ push=[=wire =request:http] (only-iris mint-caz) + ;< * b (do-arvo wire.push iris-ok) + ;< ~ b + %- jab-bowl + |= bol=bowl + %= bol + sup + %- malt + :~ :- ~[/reader] + [~bus /v1/buckets/~sampel-palnet/project-files/updates] + == + == + :: the group is deleted, so the permission gates are unreachable + ;< ~ b (set-scry-gate missing-group-scries) + ;< caz=(list card) b + %^ do-agent + /groups + [~sampel-palnet %groups] + [%fact %group-response-1 !>(group-changed)] + =/ revoke=[=wire =request:http] (only-iris caz) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + =/ kicked=(list ship) + %+ murn caz + |=(car=card ?.(?=([%give %kick * ^] car) ~ ship.p.car)) + :: the reader is kicked, and its access synced as revoked + %+ ex-equal + !> :* (granted-count st) + method.request.revoke + kicked + == + !>([0 %'PUT' ~[~bus]]) +:: +:: A revoke the broker did not confirm is not a revoke. Dropping the token +:: locally does not stop it working -- the broker honours a pushed token +:: without asking us again -- so a transient failure has to be retried rather +:: than logged and forgotten. +:: +++ test-failed-revocation-is-retried + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< ~ b (set-scry-gate reader-scries) + ;< mint-caz=(list card) b + %- (do-as ~bus) + %+ do-poke %buckets-command-1 + !>(`command:bu`[0v11 [%bucket flag [%issue-bucket-read ~]]]) + =/ push=[=wire =request:http] (only-iris mint-caz) + ;< * b (do-arvo wire.push iris-ok) + :: access is pulled, so the token is revoked at the broker + ;< ~ b (set-scry-gate revoked-scries) + ;< caz=(list card) b + %^ do-agent + /groups + [~sampel-palnet %groups] + [%fact %group-response-1 !>(group-changed)] + =/ revoke=[=wire =request:http] (only-iris caz) + :: the broker fails it, so the pair stays owed and a retry is armed + ;< * b (do-arvo wire.revoke (iris-status 503)) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + ;< ~ b (ex-equal !>((owed-count st)) !>(1)) + :: the retry timer re-sends the same revision, and a 2xx settles it + ;< retried=(list card) b + (do-arvo /buckets/reader-retry [%behn %wake ~]) + =/ again=[=wire =request:http] (only-iris retried) + ;< * b (do-arvo wire.again iris-ok) + ;< sv2=vase b get-save + =/ st2=state-0:bu !<(state-0:bu sv2) + %+ ex-equal + !> :* wire.again + method.request.again + (owed-count st2) + == + !>([wire.revoke %'PUT' 0]) +:: +:: A reader that took a token and then dropped its subscription still has to +:: be revoked. It produces no kick, and the broker honours a pushed token +:: without asking us again, so deriving revocations from the subscription +:: list alone would leave a bearer token working until it lapsed. +:: +++ test-revokes-a-token-held-by-an-unsubscribed-reader + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< ~ b (set-scry-gate reader-scries) + :: ~bus takes a token, and never appears in sup.bowl + ;< mint-caz=(list card) b + %- (do-as ~bus) + %+ do-poke %buckets-command-1 + !>(`command:bu`[0v8 [%bucket flag [%issue-bucket-read ~]]]) + =/ push=[=wire =request:http] (only-iris mint-caz) + ;< * b (do-arvo wire.push iris-ok) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + :: then it loses read access + ;< ~ b (set-scry-gate revoked-scries) + ;< caz=(list card) b + %^ do-agent + /groups + [~sampel-palnet %groups] + [%fact %group-response-1 !>(group-changed)] + =/ revoke=[=wire =request:http] (only-iris caz) + ;< sv2=vase b get-save + =/ st2=state-0:bu !<(state-0:bu sv2) + =/ kicks=(list card) + %+ skim caz + |=(car=card ?=([%give %kick *] car)) + :: no kick, because it was not subscribed -- but the token is gone locally + :: and the broker is told to drop it + %+ ex-equal + !> :* (granted-count st) + (granted-count st2) + method.request.revoke + (lent kicks) + == + !>([1 0 %'PUT' 0]) +:: +:: A reader who loses group access stops reading immediately: the host kicks +:: the subscription and tells the broker to drop that reader's token, rather +:: than leaving it live until it lapses. +:: +++ test-losing-read-access-revokes-the-token + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< ~ b (set-scry-gate reader-scries) + :: a remote reader asks the host for a token, and the broker accepts it + ;< mint-caz=(list card) b + %- (do-as ~bus) + %+ do-poke %buckets-command-1 + !>(`command:bu`[0v7 [%bucket flag [%issue-bucket-read ~]]]) + =/ push=[=wire =request:http] (only-iris mint-caz) + ;< * b (do-arvo wire.push iris-ok) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + =/ minted=@ud (granted-count st) + :: the reader is subscribed, and then loses read access + ;< ~ b + %- jab-bowl + |= bol=bowl + %= bol + sup + %- malt + :~ :- ~[/reader] + [~bus /v1/buckets/~sampel-palnet/project-files/updates] + == + == + ;< ~ b (set-scry-gate revoked-scries) + ;< kick-caz=(list card) b + %^ do-agent + /groups + [~sampel-palnet %groups] + [%fact %group-response-1 !>(group-changed)] + =/ revoke=[=wire =request:http] (only-iris kick-caz) + ;< sv2=vase b get-save + =/ st2=state-0:bu !<(state-0:bu sv2) + =/ kicked=(list ship) + %+ murn kick-caz + |= car=card + ?.(?=([%give %kick * ^] car) ~ ship.p.car) + %+ ex-equal + !> :* minted + method.request.revoke + url.request.revoke + (granted-count st2) + kicked + == + !> :* 1 + %'PUT' + 'https://memex.tlon.network/v2/buckets/tokens/sampel-palnet' + 0 + ~[~bus] + == +:: +:: Expired grants and pending sessions are swept the next time authority is +:: touched, and their reservation bindings go with them. +:: +++ test-expired-authority-is-pruned + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + =/ rid=@t '00000000-0000-0000-0000-000000000003' + ;< ~ b setup + ;< ~ b create + ;< * b (ask 0v1 [%bucket flag [%begin-upload ~ 'private.pdf' 'application/pdf' 42 ~]]) + ;< * b + (do-poke %buckets-broker-command-1 !>(`broker-command:bu`[%authorize-upload seed-token rid])) + ;< ~ b (jab-bowl |=(bol=bowl bol(now ~2026.1.2, eny 0v9999))) + ;< * b (ask 0v2 [%bucket flag [%begin-upload ~ 'later.pdf' 'application/pdf' 7 ~]]) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + %+ ex-equal + !> :* (~(has by sessions.st) seed) + (~(has by reservations.st) rid) + ~(wyt by sessions.st) + == + !>([%.n %.n 1]) +:: +:: A subscriber forwards to the host, subscribes for the answer, arms a +:: timeout, and tells its own client the request is in flight. +:: +++ test-subscriber-forwards-and-reports-pending + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + =/ act=action:bu [%bucket flag [%create-folder ~ 'Launch']] + ;< ~ b (setup-as ~bus) + ;< * b + (do-poke %group-channel-join !>(`channel-join:bu`[[%buckets ~sampel-palnet %project-files] group])) + ;< caz=(list card) b (ask 0v4 act) + :: The watch precedes the poke, so a host that answers in the same event it + :: is poked cannot publish its fact before we are listening. + %+ ex-cards caz + :~ %- ex-task + :* /buckets/req/~sampel-palnet/0v4/watch + [~sampel-palnet %buckets] + [%watch /v1/request/~bus/0v4] + == + %- ex-poke + :* /buckets/req/~sampel-palnet/0v4/poke + [~sampel-palnet %buckets] + %buckets-command-1 + !>(`command:bu`[0v4 act]) + == + %- ex-arvo + :* /buckets/req/~sampel-palnet/0v4/wake + [%b %wait (add ~2026.1.1 ~m2)] + == + (grant-fact 0v4 [%pending ~]) + == +:: +:: A host's %pending is not an answer. A forwarded request stays open across +:: it, so the real answer that follows still settles the client -- this is the +:: path a cold-start read on a remote bucket takes, where the host has to +:: reach the broker before it can hand back a token. +:: +++ test-forwarded-request-survives-a-pending-answer + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + =/ act=action:bu [%bucket flag [%issue-bucket-read ~]] + ;< ~ b (setup-as ~bus) + ;< * b + (do-poke %group-channel-join !>(`channel-join:bu`[[%buckets ~sampel-palnet %project-files] group])) + ;< * b (ask 0v4 act) + :: the host says it is still working; nothing is settled and the request + :: must stay open + ;< waiting=(list card) b + %^ do-agent + /buckets/req/~sampel-palnet/0v4/watch + [~sampel-palnet %buckets] + :+ %fact %buckets-req-response-1 + !>(`req-response:bu`[0v4 [%pending ~]]) + ;< ~ b (ex-cards waiting ~) + :: then the real answer lands, and it settles + =/ tok=read-token:bu ['0v1.2345' (add ~2026.1.1 ~d1)] + ;< answered=(list card) b + %^ do-agent + /buckets/req/~sampel-palnet/0v4/watch + [~sampel-palnet %buckets] + :+ %fact %buckets-req-response-1 + !>(`req-response:bu`[0v4 [%token tok]]) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + ;< ~ b + %+ ex-equal + !>((~(get by read-tokens.st) flag)) + !>(`tok) + %+ ex-cards answered + :~ %- ex-task + :* /buckets/req/~sampel-palnet/0v4/watch + [~sampel-palnet %buckets] + [%leave ~] + == + %- ex-arvo + :* /buckets/req/~sampel-palnet/0v4/wake + [%b %rest (add ~2026.1.1 ~m2)] + == + %- ex-arvo + :* /buckets/token/~sampel-palnet/project-files + [%b %wait (sub expires-at.tok ~h1)] + == + (grant-fact 0v4 [%token tok]) + == +:: +:: A refused action that has nothing to do with tokens must leave ours alone. +:: The subscriber only learns which request was a token request from what it +:: recorded when forwarding -- without that, a denied folder rename discards a +:: perfectly good read token, and with it the reader's offline reads. +:: +++ test-unrelated-refusal-keeps-the-read-token + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b (setup-as ~bus) + ;< * b + (do-poke %group-channel-join !>(`channel-join:bu`[[%buckets ~sampel-palnet %project-files] group])) + :: hold a token, obtained the normal way + ;< * b (ask 0v4 [%bucket flag [%issue-bucket-read ~]]) + =/ tok=read-token:bu ['0v1.2345' (add ~2026.1.1 ~d1)] + ;< * b + %^ do-agent + /buckets/req/~sampel-palnet/0v4/watch + [~sampel-palnet %buckets] + :+ %fact %buckets-req-response-1 + !>(`req-response:bu`[0v4 [%token tok]]) + :: now a folder rename is refused + ;< * b (ask 0v5 [%bucket flag [%entry 1 [%rename 'nope']]]) + ;< caz=(list card) b + %^ do-agent + /buckets/req/~sampel-palnet/0v5/watch + [~sampel-palnet %buckets] + :+ %fact %buckets-req-response-1 + !>(`req-response:bu`[0v5 [%error %not-authorized 'not authorized for this bucket']]) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + :: the token is untouched, and no token timer was armed on its behalf + =/ token-timers=(list card) + %+ skim caz + |=(car=card ?=([%pass [%buckets %token *] %arvo *] car)) + %+ ex-equal + !>([(~(get by read-tokens.st) flag) (lent token-timers)]) + !>([`tok 0]) +:: +:: A token request that times out has to come back on its own. Nothing else +:: will: the request is gone, no refresh is armed, and the local scry keeps +:: answering with the token we already hold until it lapses. +:: +:: The same loss, reached the other three ways. A forwarded request can die +:: by timeout, by kick, by a refused watch, or by a nacked poke; all four +:: leave a renewal with its refresh already fired and nothing to rearm it. +:: They are one arm now because the poke path was missing this. +:: +++ test-every-way-a-request-dies-rearms-the-renewal + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + =/ n (mare ,(list card)) + =/ o (mare ,@ud) + =/ host=gill:gall [~sampel-palnet %buckets] + =/ req=path /buckets/req/~sampel-palnet/0v6 + :: a renewal in flight, ended by `end`; answers how many refreshes it rearmed + =/ rearms + |= end=form:n + =* b bind:o + ^- form:o + ;< ~ b (setup-as ~bus) + ;< * b + (do-poke %group-channel-join !>(`channel-join:bu`[[%buckets ~sampel-palnet %project-files] group])) + ;< * b (ask 0v6 [%bucket flag [%issue-bucket-read ~]]) + ;< caz=(list card) b end + %- pure:o + (lent (skim caz |=(car=card ?=([%pass [%buckets %token *] %arvo *] car)))) + ;< timeout=@ud b (rearms (do-arvo (snoc req %wake) [%behn %wake ~])) + ;< kicked=@ud b (rearms (do-agent (snoc req %watch) host [%kick ~])) + ;< refused=@ud b + (rearms (do-agent (snoc req %watch) host [%watch-ack `~[leaf+"no"]])) + ;< nacked=@ud b + (rearms (do-agent (snoc req %poke) host [%poke-ack `~[leaf+"no"]])) + %+ ex-equal + !>([timeout kicked refused nacked]) + !>([1 1 1 1]) +:: +:: Without the %genuine secret a mint cannot be sent, so the request is +:: answered in the same event that records what is owed. The record must not +:: also name it as the one waiting: when the retry finally lands, confirming +:: it would send a second terminal answer for a request already finished. +:: +++ test-a-mint-answered-now-is-not-also-answered-later + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< ~ b (set-scry-gate secretless-scries) + ;< refused=(list card) b (ask 0v7 [%bucket flag [%issue-bucket-read ~]]) + :: told now, and nothing went out + ;< ~ b + %+ ex-cards refused + :~ %- ex-arvo + [/buckets/reader-retry [%b %rest (add ~2026.1.1 ~m2)]] + %- ex-arvo + [/buckets/reader-retry [%b %wait (add ~2026.1.1 ~m2)]] + %+ ex-fact ~[/v1/requests] + :- %buckets-req-response-1 + !> ^- req-response:bu + [0v7 %error %unknown 'this ship cannot reach storage yet'] + == + :: the secret appears and the retry timer sends what was owed + ;< ~ b (set-scry-gate genuine-scries) + ;< sent=(list card) b (do-arvo /buckets/reader-retry [%behn %wake ~]) + =/ push=[=wire =request:http] (only-iris sent) + ;< landed=(list card) b (do-arvo wire.push iris-ok) + :: the grant is installed, but nobody is answered a second time + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + %+ ex-equal + !>([(granted-count st) (lent (answers landed))]) + !>([1 0]) +:: +:: A settled record past its expiry says nothing either way: the token it +:: names has lapsed, so a grant is worthless and a revoke is moot. Judging +:: only the revoked ones kept a row for every reader that ever read. +:: +++ test-an-expired-grant-is-pruned-like-an-expired-revoke + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< ~ b (set-scry-gate genuine-scries) + ;< caz=(list card) b (ask 0v8 [%bucket flag [%issue-bucket-read ~]]) + =/ push=[=wire =request:http] (only-iris caz) + ;< * b (do-arvo wire.push iris-ok) + ;< sv=vase b get-save + =/ before=@ud ~(wyt by readers:!<(state-0:bu sv)) + :: read-window later, the grant it confirmed has lapsed + ;< ~ b (jab-bowl |=(bol=bowl bol(now (add ~2026.1.1 ~d1), eny 0v7777))) + ;< * b (ask 0v9 [%bucket flag [%issue-bucket-read ~]]) + ;< after=vase b get-save + =/ st=state-0:bu !<(state-0:bu after) + :: the lapsed row is gone; what replaced it is a fresh mint at revision 1 + =/ sync=reader-sync:bu (sync-for st ~sampel-palnet) + %+ ex-equal + !>([before ~(wyt by readers.st) revision.sync synced.sync]) + !>([1 1 1 0]) +:: +:: Both entry points keep the one-answer contract. The poke path used to +:: dispatch straight through, so a Gall caller retrying a lost answer ran the +:: action a second time and was answered twice. +:: +++ test-a-repeated-poke-request-id-does-not-run-twice + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< * b (ask 0v11 [%bucket flag [%create-folder ~ 'notes']]) + ;< sv=vase b get-save + =/ first=@ud ~(wyt by entries:(state-for !<(state-0:bu sv) flag)) + :: the same id again, as a caller that lost our answer would send it + ;< again=(list card) b (ask 0v11 [%bucket flag [%create-folder ~ 'notes']]) + ;< after=vase b get-save + =/ second=@ud ~(wyt by entries:(state-for !<(state-0:bu after) flag)) + :: no second folder, and the stored result is replayed rather than remade + %+ ex-equal + !>([first second (lent (answers again))]) + !>([1 1 1]) +:: +:: The %groups subscription is the only thing that calls +recheck-host-subs, +:: which is the only thing that revokes. A refusal loses it exactly as a kick +:: does, so it has to be recovered the same way -- otherwise a reader who +:: loses access keeps a working token for as long as the ship runs, silently. +:: +++ test-a-refused-groups-watch-is-retried + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + :: %groups refuses the subscription + ;< refused=(list card) b + %^ do-agent + /groups + [~sampel-palnet %groups] + [%watch-ack `~[leaf+"no"]] + ;< ~ b + %+ ex-cards refused + :~ (ex-arvo /groups/retry [%b %wait (add ~2026.1.1 ~m5)]) + == + :: and the wake asks again + ;< again=(list card) b (do-arvo /groups/retry [%behn %wake ~]) + %+ ex-cards again + :~ %- ex-card + :* %pass /groups %agent + [~sampel-palnet %groups] %watch /v1/groups + == + == +:: +:: A broker that retains the revision we just sent has not taken our write, +:: however the numbers compare. This is reachable precisely because pruning +:: a lapsed record resets our counter while the broker's row persists: the +:: next grant opens at 1 against a retained 1, and reading that as agreement +:: hands the client a token the broker never stored. +:: +++ test-an-equal-retained-revision-is-a-collision-not-agreement + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< ~ b (set-scry-gate genuine-scries) + ;< caz=(list card) b (ask 0v6 [%bucket flag [%issue-bucket-read ~]]) + =/ push=[=wire =request:http] (only-iris caz) + :: the broker answers 200, names the revision we sent, and says it did not + :: take it -- the shape a stale write has after our record was pruned + ;< resent=(list card) b (do-arvo wire.push (iris-receipt 1 |)) + =/ retry=[=wire =request:http] (only-iris resent) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + =/ sync=reader-sync:bu (sync-for st ~sampel-palnet) + :: we go above what it kept rather than settling, and nothing is servable + :: until it says it took one + %+ ex-equal + !> :* revision.sync + wire.retry + ~(wyt by read-tokens.st) + == + !> :* 2 + (reader-wire-for ~sampel-palnet 2) + 0 + == +:: +:: An applied write is still an applied write; the collision check must not +:: make ordinary success resend forever. +:: +++ test-an-applied-write-settles + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< ~ b (set-scry-gate genuine-scries) + ;< caz=(list card) b (ask 0v6 [%bucket flag [%issue-bucket-read ~]]) + =/ push=[=wire =request:http] (only-iris caz) + ;< settled=(list card) b (do-arvo wire.push (iris-receipt 1 &)) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + =/ sync=reader-sync:bu (sync-for st ~sampel-palnet) + %+ ex-equal + !>([revision.sync synced.sync (lent (skim settled |=(c=card ?=([%pass * %arvo %i *] c))))]) + !>([1 1 0]) +:: +:: A role that no longer exists must stop granting writes. Role ids come from +:: the role's title, so deleting a role and making another by the same name +:: reuses the id -- and a stale id left in .writers hands write and delete on +:: every bucket that named it to whoever joins the new role. +:: +++ test-a-deleted-role-stops-granting-writes + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< * b (ask 0v1 [%bucket flag [%set-writers (silt `(list @tas)`~[%editors %moderators])]]) + ;< sv=vase b get-save + =/ before=(set @tas) writers:(state-for !<(state-0:bu sv) flag) + :: %groups says the moderators role is gone + ;< * b + %^ do-agent + /groups + [~sampel-palnet %groups] + [%fact %group-response-1 !>((role-deleted (silt `(list @tas)`~[%moderators])))] + ;< after=vase b get-save + =/ kept=(set @tas) writers:(state-for !<(state-0:bu after) flag) + %+ ex-equal + !>([before kept]) + !>([(silt `(list @tas)`~[%editors %moderators]) (silt `(list @tas)`~[%editors])]) +:: +:: The incremental drop only fires for a fact we actually see. A group that +:: arrives whole is the repair for one we did not: anything we still hold +:: that its roles do not have is stale however we came to miss it. +:: +++ test-a-whole-group-reconciles-stale-writers + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< * b (ask 0v1 [%bucket flag [%set-writers (silt `(list @tas)`~[%editors %ghosts])]]) + :: the group arrives whole, and knows only about editors + ;< * b + %^ do-agent + /groups + [~sampel-palnet %groups] + =/ whole=group:v9:gv + =/ bare=group:v9:gv *group:v9:gv + bare(roles (malt ~[[%editors *role:v9:gv]])) + [%fact %group-response-1 !>(`r-groups:v9:gv`[group %create whole])] + ;< after=vase b get-save + %+ ex-equal + !>(writers:(state-for !<(state-0:bu after) flag)) + !>((silt `(list @tas)`~[%editors])) +:: +:: A record that lapsed while still owed used to be stranded: +owed skipped +:: it for being expired, and pruning kept it for never having settled, so it +:: belonged to nobody and stayed for good. A revoke is where that arises -- +:: it has no request waiting on it, so nothing else was ever going to retire +:: it. Expiry now decides a record's state on its own, which is what makes +:: the case unreachable rather than merely fixed. +:: +++ test-a-revoke-that-lapsed-while-owed-is-not-stranded + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< ~ b (set-scry-gate reader-scries) + ;< mint-caz=(list card) b + %- (do-as ~bus) + %+ do-poke %buckets-command-1 + !>(`command:bu`[0v7 [%bucket flag [%issue-bucket-read ~]]]) + =/ push=[=wire =request:http] (only-iris mint-caz) + :: confirmed, so no request is left waiting on this pair + ;< * b (do-arvo wire.push iris-ok) + ;< ~ b + %- jab-bowl + |= bol=bowl + %= bol + sup + %- malt + :~ :- ~[/reader] + [~bus /v1/buckets/~sampel-palnet/project-files/updates] + == + == + :: access is pulled, which revokes with nobody waiting on the result + ;< ~ b (set-scry-gate revoked-scries) + ;< revoke-caz=(list card) b + %^ do-agent + /groups + [~sampel-palnet %groups] + [%fact %group-response-1 !>(group-changed)] + =/ revoke=[=wire =request:http] (only-iris revoke-caz) + :: the broker keeps failing in a way worth retrying, and never takes it + ;< * b (do-arvo wire.revoke (iris-refusal 503 &)) + ;< sv=vase b get-save + =/ mid=reader-sync:bu (sync-for !<(state-0:bu sv) ~bus) + :: owed, unanswerable, and nobody waiting: the shape that used to persist + ;< ~ b + %+ ex-equal + !>([revision.mid synced.mid awaiting.mid]) + !>([2 1 ~]) + :: the grant's own expiry passes with the revoke still owed + ;< ~ b (jab-bowl |=(bol=bowl bol(now (add ~2026.1.1 ~d1), eny 0v4444))) + ;< ~ b (set-scry-gate reader-scries) + ;< * b (ask 0v5 [%bucket flag [%issue-bucket-read ~]]) + ;< after=vase b get-save + =/ st=state-0:bu !<(state-0:bu after) + :: the stranded revoke is gone; only our own fresh mint remains + %+ ex-equal + !>([~(wyt by readers.st) (~(has by readers.st) [flag ~bus])]) + !>([1 %.n]) +:: +:: The client mints its own request id so a lost answer stays addressable. +:: The agent parses it with (slav %uv) and falls back to one of its own for +:: anything that does not parse, so a shape mismatch would show up as nothing +:: at all -- the id simply not being the one the client kept. This is a real +:: id from the client's generator. +:: +++ test-a-client-minted-request-id-is-the-one-used + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + =/ rid=@t '0v6unph.s88pj.iv67p.6ch2o.kfdam' + ;< ~ b setup + ;< ~ b create + =/ body=@t + %- en:json:html + %- pairs:enjs:format + :~ ['requestId' s+rid] + :- 'action' + %- pairs:enjs:format + :~ ['type' s+'set-title'] + ['flag' (flag:enjs:buckets-json flag)] + ['title' s+'Renamed'] + == + == + ;< caz=(list card) b (http-post & body) + :: the answer comes back under the id we sent, not one the agent chose + =/ answered=(list @uv) + %+ murn caz + |= =card + ^- (unit @uv) + ?. ?=([%give %fact *] card) ~ + ?. =(%buckets-req-response-1 p.cage.p.card) ~ + `request-id:!<(req-response:bu q.cage.p.card) + %+ ex-equal + !>(answered) + !>(~[(slav %uv rid)]) +:: +:: Listing buckets must not carry their contents. Entries are unbounded, and +:: everything that lists -- channel sync, routing -- wants the metadata only. +:: The manifest is still reachable, at /full, the way %channels does it. +:: +++ test-listing-buckets-leaves-their-contents-behind + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< * b (ask 0v1 [%bucket flag [%create-folder ~ 'notes']]) + ;< brief=cage b (got-peek /x/v1/buckets) + ;< whole=cage b (got-peek /x/v1/buckets/full) + =/ sums=(list summary:bu) !<((list summary:bu) q.brief) + =/ snaps=(list snapshot:bu) !<((list snapshot:bu) q.whole) + :: same buckets either way, and the metadata a lister needs is on both + %+ ex-equal + !> :* p.brief + p.whole + (lent sums) + (lent snaps) + flag:(snag 0 sums) + writers:(snag 0 sums) + :: the folder is in the manifest, and only there + ~(wyt by entries:bucket-state:(snag 0 snaps)) + == + !> :* %buckets-summaries-1 + %buckets-snapshots-1 + 1 + 1 + flag + writers:bucket-state:(snag 0 snaps) + 1 + == +:: +:: Whether %buckets is here at all has to be answerable without reading what +:: it holds. /v1/buckets was serving that question, and it renders every +:: bucket's whole manifest, so a yes/no got slower the more anyone stored. +:: +++ test-readiness-is-a-constant-not-the-manifest + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + :: a bucket with an entry in it, so a manifest read would differ from a + :: constant one + ;< * b (ask 0v1 [%bucket flag [%create-folder ~ 'notes']]) + ;< =cage b (got-peek /x/v1/ready) + ;< ~ b (ex-equal !>(p.cage) !>(%json)) + %+ ex-equal + q.cage + !>(`json`[%b &]) +:: +:: A ship has no environment to read, so the broker it syncs to is a poke. +:: The guard matters more than the knob: the credential in a sync is a bearer +:: header, so a base naming an unexpected or plaintext host does not fail +:: closed, it hands the secret over. +:: +++ test-the-broker-base-is-settable-but-only-over-https + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + =/ test=@t 'https://memex.test.tlon.systems/v2/buckets' + ;< ~ b setup + ;< ~ b create + ;< ~ b (set-scry-gate genuine-scries) + :: a plaintext base is refused, and the default stands + ;< * b (do-poke %noun !>([%set-broker-base `'http://evil.example/v2/buckets'])) + ;< before=vase b get-save + :: so is one that is not a url at all + ;< * b (do-poke %noun !>([%set-broker-base `'memex.test.tlon.systems'])) + ;< still=vase b get-save + :: an https base lands, with its trailing slash trimmed + ;< * b (do-poke %noun !>([%set-broker-base `(cat 3 test '/')])) + ;< caz=(list card) b (ask 0v3 [%bucket flag [%issue-bucket-read ~]]) + =/ push=[=wire =request:http] (only-iris caz) + ;< ~ b + %+ ex-equal + !> :* broker-base:!<(state-0:bu before) + broker-base:!<(state-0:bu still) + url.request.push + == + !> :* 'https://memex.tlon.network/v2/buckets' + 'https://memex.tlon.network/v2/buckets' + (cat 3 test '/tokens/sampel-palnet') + == + :: and ~ puts it back + ;< * b (do-poke %noun !>([%set-broker-base `(unit @t)`~])) + ;< sv=vase b get-save + %+ ex-equal + !>(broker-base:!<(state-0:bu sv)) + !>('https://memex.tlon.network/v2/buckets') +:: +:: Read tokens are bucket-scoped, so a token answer has to be filed under the +:: bucket its request named. Two buckets on one host is where guessing from +:: the host goes wrong: whichever the map happened to yield first would take +:: the other's token, leaving one wrong and one empty. +:: +++ test-remote-token-is-filed-under-its-own-bucket + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + =/ other=flag:bu [~sampel-palnet %archive] + ;< ~ b (setup-as ~bus) + ;< * b + (do-poke %group-channel-join !>(`channel-join:bu`[[%buckets ~sampel-palnet %project-files] group])) + ;< * b + (do-poke %group-channel-join !>(`channel-join:bu`[[%buckets ~sampel-palnet %archive] group])) + :: ask for a token on the second bucket + ;< * b (ask 0v7 [%bucket other [%issue-bucket-read ~]]) + =/ tok=read-token:bu ['0v9.8765' (add ~2026.1.1 ~d1)] + ;< * b + %^ do-agent + /buckets/req/~sampel-palnet/0v7/watch + [~sampel-palnet %buckets] + :+ %fact %buckets-req-response-1 + !>(`req-response:bu`[0v7 [%token tok]]) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + :: it lands on the bucket that asked, and the sibling still holds none + %+ ex-equal + !> :- (~(get by read-tokens.st) other) + (~(get by read-tokens.st) flag) + !>([`tok ~]) +:: +:: A kick is not a revocation. The replica survives and the subscription is +:: re-established; only a nack from the host drops the bucket. +:: +++ test-kick-resubscribes + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b (setup-as ~bus) + ;< * b + (do-poke %group-channel-join !>(`channel-join:bu`[[%buckets ~sampel-palnet %project-files] group])) + ;< caz=(list card) b + %^ do-agent + /buckets/sub/~sampel-palnet/project-files + [~sampel-palnet %buckets] + [%kick ~] + ;< ~ b + %+ ex-cards caz + :~ %- ex-task + :* /buckets/sub/~sampel-palnet/project-files + [~sampel-palnet %buckets] + [%watch /v1/buckets/~sampel-palnet/project-files/updates] + == + == + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + (ex-equal !>((~(has by spaces.st) flag)) !>(%.y)) +:: +:: A member without a writer role is refused with a typed error rather than +:: a bare crash, so the requester learns why. +:: +++ test-remote-write-denied-without-permission + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< ~ b (set-scry-gate missing-group-permission-scries) + ;< caz=(list card) b + %- (do-as ~bus) + %+ do-poke %buckets-command-1 + !>(`command:bu`[0v5 [%bucket flag [%create-folder ~ 'Launch']]]) + %+ ex-cards caz + :~ %- ex-fact + :* ~[/v1/request/~bus/0v5] + %buckets-req-response-1 + !>(`req-response:bu`[0v5 [%error %not-authorized 'not authorized for this bucket']]) + == + == +:: +:: A member holding a configured writer role is accepted. +:: +++ test-remote-write-allowed-with-writer-role + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< * b + (ask 0v0 [%create %project-files 'Project Files' group ~ (silt ~[%editor])]) + ;< ~ b (set-scry-gate group-permission-scries) + ;< * b + %- (do-as ~bus) + %+ do-poke %buckets-command-1 + !>(`command:bu`[0v6 [%bucket flag [%create-folder ~ 'Launch']]]) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + =/ bs=bucket-state:bu (state-for st flag) + (ex-equal !>([~(wyt by entries.bs) revision.bs]) !>([1 1])) +:: +:: A non-host admin forwards creation to the group host rather than +:: allocating storage on its own ship. +:: +++ test-non-host-admin-forwards-create + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + =/ act=action:bu + [%create %project-files 'Project Files' group ~ ~] + ;< ~ b (setup-as ~bus) + ;< ~ b (set-scry-gate allow-admin-create-scries) + ;< caz=(list card) b (ask 0v1 act) + %+ ex-cards caz + :~ %- ex-task + :* /buckets/req/~sampel-palnet/0v1/watch + [~sampel-palnet %buckets] + [%watch /v1/request/~bus/0v1] + == + %- ex-poke + :* /buckets/req/~sampel-palnet/0v1/poke + [~sampel-palnet %buckets] + %buckets-command-1 + !>(`command:bu`[0v1 act]) + == + %- ex-arvo + :* /buckets/req/~sampel-palnet/0v1/wake + [%b %wait (add ~2026.1.1 ~m2)] + == + (grant-fact 0v1 [%pending ~]) + == +:: +:: A Moon cannot own Bucket storage, even when it hosts the group and the +:: caller is an admin. +:: +++ test-moon-host-cannot-create + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + =/ moon=ship ~pinser-botter-sampel-palnet + =/ moon-group=flag:bu [moon %test-group] + ;< ~ b (setup-as moon) + ;< ~ b (set-scry-gate allow-admin-create-scries) + ;< caz=(list card) b + (ask 0v1 [%create %project-files 'Project Files' moon-group ~ ~]) + %+ ex-cards caz + :~ %+ grant-fact 0v1 + [%error %invalid-input 'only a planet may host a bucket'] + == +:: +:: The authoritative group host accepts a live remote admin, owns the Bucket, +:: and records the initiating admin as its creator. +:: +++ test-remote-admin-creates-on-group-host + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b (set-scry-gate allow-admin-create-scries) + ;< * b + %- (do-as ~bus) + %+ do-poke %buckets-command-1 + !>(`command:bu`[0v1 [%create %project-files 'Project Files' group ~ ~]]) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + =/ bs=bucket-state:bu (state-for st flag) + %+ ex-equal + !>([created-by.bucket.bs updated-by.bucket.bs group.bs]) + !>([~bus ~bus group]) +:: +:: A create that collides on the name is bad client input, not a broken +:: invariant. It used to crash the event, which reaches the caller as a nack +:: or a timeout rather than as the error the contract promises. An otherwise +:: identical create from a second admin is still the same create. +:: +++ test-a-conflicting-create-is-an-error-not-a-crash + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< ~ b (set-scry-gate allow-admin-create-scries) + :: Same name, different title -- a real collision. + ;< caz=(list card) b + (ask 0v1 [%create %project-files 'Something Else' group ~ ~]) + ;< ~ b + %+ ex-cards caz + :~ %+ grant-fact 0v1 + [%error %invalid-input 'that bucket name is taken'] + == + :: Identical, but from another admin. .actor is not part of the comparison, + :: so this is the idempotent path and re-registers rather than erroring. + ;< caz2=(list card) b + %- (do-as ~bus) + %+ do-poke %buckets-command-1 + !>(`command:bu`[0v2 [%create %project-files 'Project Files' group ~ ~]]) + :: A re-register republishes the snapshot; an error would not. + ;< ~ b + %+ ex-cards + %+ skim caz2 + |=(=card ?=([%give %fact * %buckets-response-1 *] card)) + :~ (ex-fact-paths ~[/v1]) + == + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + =/ bs=bucket-state:bu (state-for st flag) + :: One Bucket, and the collision did not overwrite its title. + %+ ex-equal + !>([~(wyt by spaces.st) next-id.st title.bucket.bs]) + !>([1 1 'Project Files']) +:: +:: Local clients watch our /v1, not the host's, so dropping a replica has to +:: say so here or a mounted client keeps rendering a manifest this ship no +:: longer holds. +:: +++ test-dropping-a-replica-tells-local-watchers + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + =/ nes=nest:bu [%buckets ~sampel-palnet %project-files] + ;< ~ b (setup-as ~bus) + ;< * b (do-poke %group-channel-join !>(`channel-join:bu`[nes group])) + ;< caz=(list card) b + (do-poke %group-channel-leave !>(`channel-leave:bu`[nes])) + %+ ex-cards + %+ skim caz + |=(=card ?=([%give %fact * %buckets-response-1 *] card)) + :~ %^ ex-fact + ~[/v1 /v1/buckets/~sampel-palnet/project-files/updates] + %buckets-response-1 + !>(`response:bu`[%update flag 0 [%delete ~]]) + == +:: +:: Gall retries reuse the caller-selected random name. An identical retry +:: must not allocate a second Bucket. +:: +++ test-create-retry-is-idempotent + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + =/ cmd=command:bu + [0v1 [%create %project-files 'Project Files' group ~ ~]] + ;< ~ b setup + ;< ~ b (set-scry-gate allow-admin-create-scries) + ;< * b ((do-as ~bus) (do-poke %buckets-command-1 !>(cmd))) + ;< * b ((do-as ~bus) (do-poke %buckets-command-1 !>(cmd))) + ;< sv=vase b get-save + =/ st=state-0:bu !<(state-0:bu sv) + (ex-equal !>([next-id.st ~(wyt by spaces.st)]) !>([1 1])) +:: +:: +on-load must round-trip the persisted state unchanged. A cast failure +:: here silently reverts the whole |commit, so it is worth pinning even with +:: nothing to migrate from. +:: +++ test-load-round-trips-state + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< ~ b create + ;< * b (ask 0v1 [%bucket flag [%create-folder ~ 'Launch']]) + ;< before=vase b get-save + ;< * b (do-load buckets-agent `before) + ;< after=vase b get-save + (ex-equal after before) +:: +http-post: submit an action over the Eyre surface. +:: +++ http-post + |= [authed=? body=@t] + =/ m (mare ,(list card)) + ^- form:m + %+ do-poke %handle-http-request + !> ^- [@ta inbound-request:eyre] + :- 'eyre-0' + :* authenticated=authed + secure=& + address=[%ipv4 .0.0.0.0] + :* method=%'POST' + url='/buckets/~/v1' + header-list=~ + body=`(as-octs:mimes:html body) + == + == +:: +++ http-header + |= [code=@ud ct=@t] + %^ ex-fact + [/http-response/eyre-0]~ + %http-response-header + !>(`response-header:http`[code ~[['content-type' ct]]]) +:: +:: An action submitted over HTTP is answered on that same request, so a +:: client needs no correlation of its own. A refusal comes back as a typed +:: error with a 200, not as a crash. +:: +++ test-http-post-answers-inline + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + =/ body=@t + '{"requestId":"0v5","action":{"type":"create","name":"smoke","title":"Smoke","group":{"host":"~pinser-botter-sampel-palnet","name":"demo"},"readers":[],"writers":[]}}' + ;< ~ b setup + ;< caz=(list card) b (http-post & body) + :: the answer goes to the local subscription first, then closes out the + :: held request — same body, two deliveries. + %+ ex-cards caz + :~ %+ grant-fact 0v5 + [%error %invalid-input 'only a planet may host a bucket'] + (http-header 200 'application/json') + (ex-fact-paths [/http-response/eyre-0]~) + (ex-card [%give %kick [/http-response/eyre-0]~ ~]) + == +:: +:: Eyre validates the session; an unauthenticated request never reaches +:: the action layer. +:: +++ test-http-requires-authentication + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< caz=(list card) b (http-post | '{"action":{"type":"create"}}') + %+ ex-cards caz + :~ (http-header 401 'text/plain') + (ex-fact-paths [/http-response/eyre-0]~) + (ex-card [%give %kick [/http-response/eyre-0]~ ~]) + == +:: +:: A body that is not a recognizable action is a client error, not a crash. +:: +++ test-http-rejects-malformed-action + %- eval-mare + =/ m (mare ,~) + =* b bind:m + ^- form:m + ;< ~ b setup + ;< caz=(list card) b (http-post & '{"action":{"type":"no-such-verb"}}') + %+ ex-cards caz + :~ (http-header 400 'text/plain') + (ex-fact-paths [/http-response/eyre-0]~) + (ex-card [%give %kick [/http-response/eyre-0]~ ~]) + == +-- diff --git a/desk/tests/app/groups.hoon b/desk/tests/app/groups.hoon index 1939e486bb..3d7efa0665 100644 --- a/desk/tests/app/groups.hoon +++ b/desk/tests/app/groups.hoon @@ -260,6 +260,34 @@ [%fact group-log+!>(init-log)] (pure:m caz) :: +:: +test-can-write-resolves-for-a-non-member: a ship holding no seat must +:: make this scry resolve to ~, not to no-such-path. +:: +:: Both readers of it use .^ and then ?~ on a unit -- lib/channel-utils +:: +can-write, and %buckets +group-can-write. A scry that resolves to nothing +:: does not hand them ~, it crashes the event. So answering [~ ~] here meant +:: a stranger or an ex-member writing to any channel took the event down +:: instead of being denied. +:: +++ test-can-write-resolves-for-a-non-member + %- eval-mare + =/ m (mare ,~) + ^- form:m + ;< * bind:m do-groups-init + ;< * bind:m do-join-group + :: the group's only seat is ~zod, so ~fun holds none + ;< stranger=cage bind:m + (got-peek /x/v2/groups/~zod/my-test-group/channels/chat/~zod/general/can-write/~fun) + ;< seated=cage bind:m + (got-peek /x/v2/groups/~zod/my-test-group/channels/chat/~zod/general/can-write/~zod) + =/ perms + |= caz=cage + ^- (unit [admin=? roles=(set role-id:v7:gv)]) + !<((unit [admin=? roles=(set role-id:v7:gv)]) q.caz) + =/ absent (perms stranger) + =/ present (perms seated) + (ex-equal !>([absent ?~(present | admin.u.present)]) !>([~ &])) +:: ++ test-join-group %- eval-mare =/ m (mare ,~) diff --git a/docs/backend/desk/app/buckets.md b/docs/backend/desk/app/buckets.md new file mode 100644 index 0000000000..100d0f33f1 --- /dev/null +++ b/docs/backend/desk/app/buckets.md @@ -0,0 +1,232 @@ +# `%buckets` + +`%buckets` is a third-party `%groups` channel host for group-owned file spaces. It owns the logical folder tree, file metadata, authorization, and upload lifecycle. It does not store or transport file bytes. + +The channel nest is `buckets/~host/name`. The Bucket host is authoritative, while member ships keep subscribed replicas and expose those replicas to their local clients. + +## Authority and storage boundary + +- `%groups` remains the authority for membership and channel visibility. +- The Bucket host re-checks the affiliated group's live `can-read` gate for subscriptions, for every read token it mints, and again before installing one the broker has accepted. +- Mutations require either group-admin authority or a group role in the Bucket's writer set. An empty writer set means every readable group member may write. Writer roles are independent from the reader roles stored by `%groups`. +- Title changes, writer-role changes, and Bucket deletion require a live group-admin check. +- Every permission read is a `.^` scry into `%groups`, which answers no-such-path for a group it does not hold -- and an unresolvable scry crashes the event. So each one first asks `+group-exists`, and treats a missing group as no access rather than dying. A bucket is only ever hosted by its group's host, so missing means deleted, not unsynced. +- The acting principal is always `src.bowl`; commands cannot claim a different ship. +- Gall persists only the manifest, bounded upload-session metadata, and opaque short-lived broker tokens. Object bytes, storage credentials, and signed URLs must not travel over Ames or enter Gall state. +- Object keys are host-generated and independent of display names, so rename and move operations only change manifest metadata. +- Reader roles are `%groups`' alone. They are handed to it with the channel at creation and are not retained here; `+group-can-read` asks it. Writer roles live here because `%groups` does not model them. + +Hosted clients use the private Ylem/Memex broker path described below. There is no uploader-owned storage fallback and no client-generated capability: every bearer token is minted by the host. + +## State + +The protocol and persisted molds are in `desk/sur/buckets.hoon`. + +Each Bucket contains: + +- Bucket metadata and affiliated group flag +- File/folder entries keyed by stable numeric ID +- The Bucket's writer-role set +- A monotonically increasing manifest revision + +Files transition from `%pending` to `%ready` or `%failed`. Upload sessions independently transition from `%pending` to `%complete` or `%failed`. An uploading file's entry is not in the manifest at all until its object lands -- it lives in the session -- so an in-flight upload is invisible to everyone but its uploader, and a folder deletion has to match sessions by their entry's parent as well as by id. + +Alongside the buckets, the agent keeps host-private maps that never appear in snapshots or Ames updates: upload sessions, minted object capabilities, this ship's own read tokens, revocations the broker has not yet confirmed, broker-reservation bindings, and the in-flight and recently-settled client requests. + +State is a single `%0`. It has never shipped, so there are no migrations; a mold change means adding `%1` and a `+state-0-to-1` arm rather than editing `%0` in place. + +## Actions + +Local clients poke `%buckets-action-1`, which accepts JSON and noun input. Every action carries a `request-id` and gets exactly one terminal answer; bearer tokens are returned only to the requester and never appear in a broadcast. + +The outer type carries identity, the inner one carries only the verb: `%create` stands alone, and everything else is `[%bucket flag a-bucket]`. + +| Action | Purpose | +| -------------------- | ----------------------------------------------------------------------------------------------------- | +| `%create` | Create a host-owned Bucket and register the channel with `%groups`, passing it the named reader roles | +| `%delete` | Delete the Bucket and remove its `%groups` channel registration | +| `%set-title` | Update the Bucket's authoritative title; group metadata is updated separately through `%groups` | +| `%set-writers` | Replace the group-role writer set | +| `%create-folder` | Add a folder beneath an existing folder or the root | +| `%begin-upload` | Validate the request, reserve an entry id and object key, and answer with a host-minted upload token | +| `%cancel-upload` | Withdraw from the requester's own pending upload; the broker still decides whether the bytes landed | +| `%issue-bucket-read` | Answer with a read token covering every ready object in the Bucket, for the requesting ship | +| `%issue-delete` | Answer with a short-lived delete token bound to one ready file | +| `%entry` | `%rename`, `%move` (cycle-checked), or `%delete` (recursive for a folder tree) one entry | + +`.readers` on `%create` is passed straight to `%groups` and is not retained, so creating over an existing Bucket is not a way to change its ACL. + +Answers are a `$response-body`: `%ok`, `%grant` for a per-object token, `%token` for a bucket read token, `%error` with a typed reason, or `%pending`. `%pending` is not terminal -- it means the host is still working, and the real answer arrives on the same request id. + +Subscriber agents forward actions to the host with the noun-only `%buckets-command-1` mark, subscribing to the answer path _before_ poking so a host that answers in the same event cannot publish before the requester is listening. The host derives the actor from the Gall bowl and re-authorizes the command. + +The group host always creates and hosts the Bucket, but any current group admin may initiate creation. A non-host admin's local `%buckets` agent first checks its group replica, then forwards the request to the group host. The host checks the actor against its authoritative `%groups` state before allocating the Bucket or registering its channel. Consequently the channel nest, object storage, quota, and eventual billing all remain attached to the group host rather than the initiating admin. Repeating an identical create request is idempotent and re-attempts channel registration; a conflicting request for an existing Bucket name is rejected. + +The Gall delete action removes the manifest and `%groups` registration, but storage-wide object cleanup is not atomic yet. The client intentionally withholds Bucket deletion until Memex exposes a host-authorized bulk cleanup operation. + +## HTTP surface + +Eyre is bound at `/buckets`. A session cookie is the host's own capability, so anything submitted over HTTP acts as the host itself; an unauthenticated request gets 401, as in `%notes`. + +| Route | Purpose | +| -------------------------------------- | --------------------------------------------------------------------- | +| `POST /buckets/~/v1` | Submit one action, held open until its terminal answer | +| `GET /buckets/~/v1/buckets` | All locally available snapshots | +| `GET /buckets/~/v1/buckets//` | One local snapshot | +| `GET /buckets/~/v1/request/` | The state of a submitted request, for a client that lost its response | + +Because the POST is held open across the terminal answer, a client needs no correlation machinery: `requestId` is optional and one is minted if absent. The poll route exists to recover an answer after a dropped connection, within the grace period a settled request is retained for. A `@uv` id carries dots, and the request-line parser mistakes its trailing dot-group for a file extension, so that segment is reassembled before parsing. + +## Scries and subscriptions + +| Path | Mark | Description | +| ---------------------------------------- | ------------------------- | ----------------------------------------------------- | +| `/x/v1/buckets` | `%buckets-snapshots-1` | All locally available Bucket snapshots | +| `/x/v1/buckets//` | `%buckets-response-1` | One local Bucket snapshot | +| `/x/v1/buckets///read-token` | `%buckets-read-token-1` | The read token this ship currently holds, if any | +| `/u/joined//` | `%loob` | Whether the local agent has the Bucket | +| `/v1` | `%buckets-response-1` | Initial snapshots and all local replica updates | +| `/v1/requests` | `%buckets-req-response-1` | Answers to actions submitted by this ship's clients | +| `/v1/buckets///updates` | `%buckets-response-1` | Host-authorized snapshot followed by manifest updates | + +Every scry answers a mark that grows to `json`, because clients read them over Eyre; `%noun` does not, and a peek returning it answers 500. + +Direct scries are self-only. Remote consumers subscribe to the host's update path, where the host can apply the live group authorization check. Subscriber agents report joined/left state to local `%groups` using `%group-channel-active`. + +Bucket snapshots are replica observations, not command acknowledgements: an action's answer is its `$response-body`, not the appearance of anything in a snapshot. An in-flight upload never appears in a snapshot at all. Bucket channels must never be sent through `%activity` post/thread scries because they do not contain posts or threads. + +## Uploads + +1. The client sends `%begin-upload`. The host validates size, content type and parent, reserves an entry id and object key, opens a host-private session, and answers _that requester_ with a token minted from its own entropy. Nothing is broadcast and nothing enters the manifest. +2. The client presents that token and the Bucket host to Memex `POST /v2/buckets/uploads/grant`. +3. Memex asks the host's Pioneer sidecar to run `%pioneer-buckets-authorize-upload`. The host binds Memex's reservation id on first exchange and returns authoritative Bucket metadata. Pioneer stamps the host from its own ship identity. +4. Memex returns a short-lived PUT grant bound to the host, Bucket, object id, caller, size, content type, and expiry. +5. The client uploads bytes directly to object storage, with progress, cancel and retry. +6. The client calls Memex's completion endpoint. Memex HEAD-verifies the object and sends its receipt through `%pioneer-buckets-complete-upload`. +7. The host validates the reservation, object id, host, Bucket id, size and MIME type, then publishes the entry and emits a revisioned update. + +Completion is idempotent, and so is re-exchanging the upload token: the host echoes the reservation it bound first and ignores the one Memex proposes, because Memex mints a fresh id per grant call and denying the second would make a retried upload unrecoverable. Every refusal on this path names its condition in the log -- the wire vocabulary is only `authorized` / `denied` / `expired`, so a dozen conditions share one value and the log is the only way to tell them apart. + +Broker unavailability fails the upload cleanly and marks the session failed. Settled sessions are retained briefly, for idempotent re-completion and so the uploader can read the reason, then swept. + +## Reads + +Read access is uniform across a Bucket, so a reader gets one token for the whole Bucket rather than one per file, and the host pushes it to the broker instead of being asked per object. + +1. A reader's own ship holds its token; `getBucketReadToken` is a local scry with no network hop. A cold start asks for one with `%issue-bucket-read`. +2. The host checks live group access, mints a per-reader token, and `PUT`s it to Memex `/v2/buckets/tokens/`, authenticated with the ship's `%genuine` secret. The mint lives only on the wire until the broker accepts it, so a restart mid-flight drops it cleanly rather than stranding it. +3. On acceptance the host rechecks access -- it may have been pulled while the broker was answering -- and only then stores and answers `%token`. Otherwise it revokes what the broker just took and refuses the requester. +4. The reader exchanges the token at Memex for a signed URL. Memex answers from its own table without asking the ship, so reads cost no round trip to the host and survive it being offline. +5. The host re-mints on a timer before the token lapses, so a client never waits on one. + +### Desired state, not push and revoke + +Access is synced as **versioned desired state**, one record per (Bucket, reader), rather than as a push effect and a revoke effect. Grant, rotation and revoke are the same operation: `PUT /v2/buckets/tokens/` carrying the Bucket, the reader, a strictly increasing `revision`, and either a `granted` state with its token and expiry or a `revoked` state with nothing usable. + +This is what makes delivery order stop mattering. The broker keeps only the highest revision it has seen, so a message says what should be true rather than what to do — and a delayed, duplicated or retried request loses to the truth instead of overwriting it. A revoke can be issued while a grant is still in flight; whichever arrives second, the revoke wins. + +Consequences worth knowing: + +- **A record's state is derived in one place.** `$reader-status` answers `%owed`, `%settled`, `%refused` or `%lapsed` from the revisions, the failure flag and the expiry together, and every caller switches on that rather than comparing fields itself. Eight sites used to decide it independently and disagreed — records the broker had refused were owed by nobody and prunable by nobody, so they accumulated for good, and the same was true of a revoke that lapsed while still being retried. Expiry dominates the other states, which is what makes both cases unreachable rather than merely fixed. +- **Retries are blind and safe, but not endless.** Anything whose revision is above what the broker has confirmed is still owed, and one timer re-sends all of it. That timer lands on a fixed grid and is cancelled before it is set, so arming it repeatedly cannot leave a fleet of timers each re-sending the whole set. A stale write is the protocol working, not a failure — it answers 200 with `applied: false` and the revision the broker kept. The broker classifies its own failures: `retryable: true` stays owed, while `retryable: false` is a validation refusal that would answer the same way next time, so it stops being owed until the next access change supersedes it. Nothing is owed past its own expiry either, since by then a grant is worthless and a revoke is moot. +- **The broker says whether it took the write; we do not infer it.** The receipt's `applied` decides, and a `false` resends above the `currentRevision` it reports whatever the numbers look like. Comparing revisions instead gets the equal case wrong, and that case is reachable: pruning a lapsed record resets our counter while the broker's row persists, so the next grant opens at 1 against a retained 1 and a strictly-greater test reads that as agreement. Anywhere state here mirrors state the broker keeps, the two copies have independent lifetimes — assume they match and this is the shape of what goes wrong. +- **Only a success confirms, and a success carries `currentRevision`.** If that number is ahead of ours — state loss here, or an earlier incarnation of this agent — we adopt it and re-send, rather than being discarded as stale from then on. Falling behind always surfaces on the success path, because a stale write is a 200 rather than an error; a rejection is a rejection whatever its body says, and reading a revision out of one would install a grant the broker just refused. `GET /v2/buckets/tokens//revision` exists for explicit recovery, but the common path needs no separate lookup. +- **Nothing is served until the broker confirms it.** A grant is handed to a client only once its revision is synced, so a client never holds a token that 403s. +- **A superseded grant answers its waiter.** A client blocked on a grant that a revoke overtook is told so, rather than left to time out. A record names at most one waiting request, and every transition that resolves or abandons one goes through a single arm, so the clearing and the answering cannot drift apart. +- **A settled record past its expiry is dropped, whichever state it is in.** By then the token it names has lapsed, so a grant is worthless and a revoke is moot — the same reason neither is owed any more. Re-granting later opens at revision 1, loses to the broker as a stale write, and adopts the number it is told. Anything still owed, or still holding a waiter, stays. +- **Writer roles are ours to keep current.** `%groups` owns readability and repairs it itself, but its channel-registration payload has no writers field, so a Bucket holds its own set — which means a role deleted in `%groups` would otherwise go on granting writes here. Role ids are minted from the role's title, so deleting a role and making another by the same name reuses the id, and a stale entry becomes a live grant for whoever joins the new role. + +`%channels-server` owns its channels' writers the same way and settles the shape: react to a `%role %del` when it arrives, and reconcile the whole set against the group's roles whenever the group arrives whole. Neither half suffices alone — a fact we miss is repaired by the sweep, and the sweep only happens on a full read. + +The `%groups` subscription is what drives all of this: its facts are the only caller of the recheck that issues revocations, so a refused watch is retried rather than logged — losing it silently would leave a reader who lost access holding a working token until it expired. Revocation is still driven by the stored records rather than the subscription list, so a reader that took a token and then unsubscribed is covered, and losing one Bucket does not revoke tokens for others. Expiry remains a backstop for a host that dies, not the mechanism. + +The credential goes in `X-Landscape-Token`. Neither it nor a bearer read token ever appears in a path or query string, where it would land in access logs. + +**Pointing a ship at a different broker.** The broker is one service seen from two directions — the host pushes grants to it, clients upload and read through it — so both halves have to move together or the pair is broken. The host side is a poke, because a ship has no environment to read: + +``` +:buckets &noun [%set-broker-base `'https://memex.test.tlon.systems/v2/buckets'] +:buckets &noun [%set-broker-base `(unit @t)`~] :: back to the default +``` + +Changing it re-sends every live grant to the new broker, because a broker holds only what it has been told; grants the old one still holds are left to lapse at their own expiry rather than being revoked, which is the same guarantee a missed revoke has. Only `https://` bases are accepted: the credential rides in a header, so a base naming an unexpected or plaintext host does not fail closed, it discloses the secret. `/x/v1/broker/base` reads back what a live host is using. The client half is the `TLON_MEMEX_URL` environment variable, set either way round: + +```sh +# shell, unprefixed — one run +cd apps/tlon-web +SHIP_URL=https://hatrel-disnut.test.tlon.systems \ +TLON_MEMEX_URL=https://memex.test.tlon.systems \ +pnpm dev + +# or apps/tlon-web/.env, VITE_-prefixed — persists +VITE_TLON_MEMEX_URL=https://memex.test.tlon.systems +``` + +Both spellings exist for the same reason `SHIP_URL` takes both: only `VITE_*` names survive `loadEnv` out of an `.env` file, while the shell can set the bare name. It is read when the dev server or build starts, not per request, so a change means a restart. + +It reaches the bundle only because `apps/tlon-web/vite.config.mts` substitutes it in its `define` block — `envPrefix` exposes `VITE_*` through `import.meta.env`, and nothing else arrives via `process.env`. The hosted web build has no override compiled in, so testing the upload half means running the client locally against the hosted ship. Mobile would need the same wiring through Expo before the override works there. + +**Memex ships before this agent does.** A mint the broker refuses is never stored, whatever the refusal — including a 404 from a broker without the endpoint — because a token the broker does not hold would 403 on first use. There is deliberately no compatibility path for the reverse order. + +**Reads are scoped to what the caller needs.** `/v1/buckets` lists buckets without their entries, `/v1/buckets/full` includes them, and `/v1/buckets//` reads one — the split `%channels` uses for posts, for the same reason. Entries are the only unbounded field, and everything that lists rather than opens wants the metadata alone. `/v1/ready` answers whether the agent is here at all; asking `/v1/buckets` that question made a yes/no scale with everything stored. + +Deletes stay per-object, because they are destructive: `%issue-delete` binds a short-lived token to one ready file, exchanged through `%pioneer-buckets-authorize-delete` before the manifest entry is removed. Recursive client deletion commits each file's manifest removal immediately after Memex confirms its object deletion, and treats already-deleted as idempotent success. A host-authorized server-side bulk delete remains the durable atomic implementation. + +## Pioneer thread contract + +All four threads live under `desk/ted/pioneer/buckets/` and are invoked by Pioneer as `%pioneer-buckets-*`. They accept `(unit json)` and return JSON. + +| Thread | Input | Successful result | +| ----------------------------------- | ----------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `%pioneer-buckets-authorize-upload` | `{capability, brokerReservationId}` | `{result: "authorized", upload: {bucketName, bucketId, sessionId, objectId, actorShip, size, mimeType, checksum, expiresAtMillis, brokerReservationId}}` | +| `%pioneer-buckets-complete-upload` | `{brokerReservationId, receipt}` | `{result: "completed"}` | +| `%pioneer-buckets-authorize-read` | `{capability, objectId}` | `{result: "authorized", read: {bucketId, objectId, displayFilename}}` | +| `%pioneer-buckets-authorize-delete` | `{capability, objectId}` | `{result: "authorized", delete: {bucketId, objectId}}` | + +Authorization failures return `{result: "denied"}` and expired tokens return `{result: "expired"}`. Pioneer parses nothing else and fails on an unrecognized result, so adding a category is a broker protocol change rather than an agent one. `expiresAtMillis` is Unix time in milliseconds. + +`%pioneer-buckets-authorize-read` is retained because the broker keeps a Pioneer fallback for hosts that do not push yet — ships update on their own schedule, so there is no moment when every host has started pushing. It fires whenever the broker has no row for a capability. It cannot resurrect a revoked token: revocation deletes the local capability as well, so this answers a refusal too. + +## Tests + +`desk/tests/app/buckets.hoon` covers: + +**Uploads and the manifest** + +- A complete upload lifecycle, from `%begin-upload` through a verified broker receipt +- An in-flight upload staying invisible until its object lands +- Reservation binding, verified completion, and idempotent retries +- Rejection of bad input -- nonexistent parent, zero or oversized file, malformed content type +- Deleting a folder taking the in-flight uploads under it + +**Read tokens** + +- One token covering every ready object in its Bucket, and nothing outside it +- The two-phase mint: `%pending`, the outbound `PUT`, then the token and its refresh timer +- The held token's peek answering a mark that grows to `json` +- A remote token filed under the Bucket that asked for it, not a sibling on the same host +- Access rechecked before installing a token the broker has accepted +- A refused push storing nothing, whether the broker answers 404 or 503 + +**Revocation** + +- A reader losing group access being kicked and its token revoked +- A reader that never subscribed, or unsubscribed, still being revoked +- A revocation the broker refused being retried until it confirms +- A deleted group revoking rather than crashing on its permissions +- Lazy cleanup of expired capabilities and their reservation bindings + +**Requests and permissions** + +- A forwarded request surviving the host's `%pending` and settling on the real answer +- The HTTP surface: authentication, malformed bodies, and an action answered inline +- Rejection of a remote write when the live `%groups` gate denies access, and acceptance with a writer role +- Group-hosted creation by an authorized remote admin; rejection for a non-admin; idempotent retries +- A Moon being refused Bucket storage even when it hosts the group +- State round-tripping through `+on-load`, and a kick resubscribing rather than dropping the replica + +`desk/tests/app/groups.hoon` additionally covers the `can-write` peek resolving to `~` for a ship holding no seat, which both this agent and `lib/channel-utils` read with `.^`. + +Run the targeted fakezod test with the repository's normal `%test` thread against `/tests/app/buckets/hoon`. + +Marks live under `desk/mar/buckets/`, as every other agent's do, and threads under `desk/ted/pioneer/buckets/`. Neither nesting changes a name: Clay resolves a term through `+get-fit`, whose `+segments` tries every `-`-to-`/` split — `%buckets-action-1` finds `mar/buckets/action-1`, and `%pioneer-buckets-authorize-upload` finds `ted/pioneer/buckets/authorize-upload`. An earlier version of this document asserted the opposite, which is why these files sat flat. diff --git a/packages/api/src/__tests__/apiUtils.test.ts b/packages/api/src/__tests__/apiUtils.test.ts index a4c8bc84c5..a00910cb3a 100644 --- a/packages/api/src/__tests__/apiUtils.test.ts +++ b/packages/api/src/__tests__/apiUtils.test.ts @@ -2,7 +2,10 @@ import { describe, expect, test } from 'vitest'; import { getBotUserIdForUser, + getChannelIdType, isBotUserIdForUser, + isChannelId, + isGroupChannelId, isMoonOfUser, } from '../client/apiUtils'; @@ -52,3 +55,13 @@ describe('isMoonOfUser', () => { ).toBe(false); }); }); + +describe('bucket channel ids', () => { + const channelId = 'buckets/~zod/project-files'; + + test('treats Buckets as group channels', () => { + expect(isGroupChannelId(channelId)).toBe(true); + expect(isChannelId(channelId)).toBe(true); + expect(getChannelIdType(channelId)).toBe('channel'); + }); +}); diff --git a/packages/api/src/__tests__/bucketsApi.test.ts b/packages/api/src/__tests__/bucketsApi.test.ts new file mode 100644 index 0000000000..4926d61b94 --- /dev/null +++ b/packages/api/src/__tests__/bucketsApi.test.ts @@ -0,0 +1,225 @@ +import { beforeEach, expect, test, vi } from 'vitest'; + +import { + BucketsActionFailed, + getBucket, + getBucketReadToken, + getBuckets, + mintRequestId, + requestBucketReadToken, + requestBucketsGrant, + sendBucketsAction, + subscribeToBuckets, +} from '../client/bucketsApi'; +import { + BadResponseError, + requestJson, + scry, + subscribe, + unsubscribe, +} from '../client/urbit'; +import type { BucketsFlag, BucketsSnapshot } from '../urbit/buckets'; + +vi.mock('../client/urbit', () => ({ + // A real class, not a stub: getBucket separates a missing bucket from a + // failed read with instanceof, and a mocked-away constructor would make + // every failure look like a missing bucket. + BadResponseError: class BadResponseError extends Error { + constructor( + public status: number, + message?: string + ) { + super(message); + this.name = 'BadResponseError'; + } + }, + requestJson: vi.fn(), + scry: vi.fn(), + subscribe: vi.fn(), + unsubscribe: vi.fn(), +})); + +const flag: BucketsFlag = { host: '~zod', name: 'files' }; + +const snapshot = { + flag, + state: { + bucket: { + id: 1, + title: 'Files', + createdBy: '~zod', + createdAt: 0, + updatedBy: '~zod', + updatedAt: 0, + }, + group: { host: '~zod', name: 'group' }, + writers: ['admin'], + entries: [], + revision: 1, + }, +} satisfies BucketsSnapshot; + +beforeEach(() => { + vi.mocked(requestJson).mockReset(); + vi.mocked(scry).mockReset(); + vi.mocked(subscribe).mockReset(); + vi.mocked(unsubscribe).mockReset(); +}); + +test('getBuckets returns the local snapshots', async () => { + vi.mocked(scry).mockResolvedValueOnce([snapshot]); + + await expect(getBuckets()).resolves.toEqual([snapshot]); +}); + +test('getBucket reads one bucket and unwraps its snapshot', async () => { + vi.mocked(scry).mockResolvedValueOnce({ + type: 'snapshot', + flag, + state: snapshot.state, + }); + + await expect(getBucket(flag)).resolves.toEqual(snapshot); + expect(scry).toHaveBeenCalledWith({ + app: 'buckets', + path: '/v1/buckets/~zod/files', + }); +}); + +// The agent drops a bucket from this read under the same conditions it drops +// one from /v1/buckets, so a 404 means what an absence from that list meant. +test('getBucket reports a bucket this ship does not have as null', async () => { + vi.mocked(scry).mockRejectedValueOnce( + new BadResponseError(404, 'no such path') + ); + + await expect(getBucket(flag)).resolves.toBeNull(); +}); + +// Anything else is a failed read. Reporting it as a missing bucket would clear +// one that is really there the first time a connection drops. +test('getBucket raises a read failure rather than calling it missing', async () => { + vi.mocked(scry).mockRejectedValueOnce(new BadResponseError(500, 'boom')); + + await expect(getBucket(flag)).rejects.toThrow(BadResponseError); +}); + +test('sendBucketsAction submits over the v1 endpoint and returns the answer', async () => { + vi.mocked(requestJson).mockResolvedValueOnce({ + requestId: '0v5', + body: { ok: null }, + }); + + await expect( + sendBucketsAction({ type: 'delete-bucket', flag }, '0v1abcd.efghi') + ).resolves.toEqual({ ok: null }); + + // The agent answers 401 for an expired cookie, which requestJson does not + // reauth on by default — so the option is part of the contract, not noise. + // The request id is ours, so a lost answer stays addressable. + expect(requestJson).toHaveBeenCalledWith( + '/buckets/~/v1', + 'POST', + { action: { type: 'delete-bucket', flag }, requestId: '0v1abcd.efghi' }, + { reauthStatuses: [401, 403] } + ); +}); + +// The agent parses this with (slav %uv) and silently substitutes its own id +// for anything that does not parse, so a malformed one fails by doing nothing +// visible. /tests/app/buckets checks the Hoon side accepts this shape. +test('mintRequestId produces a canonical @uv', () => { + for (let i = 0; i < 200; i += 1) { + const id = mintRequestId(); + expect(id).toMatch(/^0v[1-9a-v][0-9a-v]{4}(\.[0-9a-v]{5})*$/); + } + expect(new Set(Array.from({ length: 50 }, mintRequestId)).size).toBe(50); +}); + +test('sendBucketsAction raises a typed refusal', async () => { + vi.mocked(requestJson).mockResolvedValueOnce({ + requestId: '0v5', + body: { error: { type: 'not-authorized', message: 'nope' } }, + }); + + await expect( + sendBucketsAction({ type: 'delete-bucket', flag }) + ).rejects.toMatchObject({ + name: 'BucketsActionFailed', + type: 'not-authorized', + message: 'nope', + }); +}); + +test('requestBucketsGrant returns the minted token', async () => { + vi.mocked(requestJson).mockResolvedValueOnce({ + requestId: '0v5', + body: { + grant: { token: '0vabc', entryId: 12, expiresAt: '~2026.1.1' }, + }, + }); + + await expect( + requestBucketsGrant({ type: 'issue-delete', flag, id: 12 }) + ).resolves.toEqual({ token: '0vabc', entryId: 12, expiresAt: '~2026.1.1' }); +}); + +test('requestBucketsGrant rejects an answer that carries no grant', async () => { + vi.mocked(requestJson).mockResolvedValueOnce({ + requestId: '0v5', + body: { ok: null }, + }); + + await expect( + requestBucketsGrant({ type: 'issue-delete', flag, id: 12 }) + ).rejects.toThrow(/did not return a grant/); +}); + +test('getBucketReadToken reads the token our own ship holds', async () => { + vi.mocked(scry).mockResolvedValueOnce({ + token: '0vread', + expiresAt: '~2026.1.1', + }); + + await expect(getBucketReadToken(flag)).resolves.toEqual({ + token: '0vread', + expiresAt: '~2026.1.1', + }); + expect(scry).toHaveBeenCalledWith({ + app: 'buckets', + path: '/v1/buckets/~zod/files/read-token', + }); +}); + +test('getBucketReadToken yields null before the first refresh lands', async () => { + vi.mocked(scry).mockRejectedValueOnce(new Error('404')); + + await expect(getBucketReadToken(flag)).resolves.toBeNull(); +}); + +test('requestBucketReadToken mints one on a cold start', async () => { + vi.mocked(requestJson).mockResolvedValueOnce({ + requestId: '0v5', + body: { token: { token: '0vread', expiresAt: '~2026.1.1' } }, + }); + + await expect(requestBucketReadToken(flag)).resolves.toEqual({ + token: '0vread', + expiresAt: '~2026.1.1', + }); +}); + +test('subscribeToBuckets unsubscribes the replacement id after a reset', async () => { + vi.mocked(subscribe).mockImplementationOnce( + async (_endpoint, _onUpdate, options) => { + options?.onSubscriptionId?.(7); + options?.onSubscriptionId?.(19); + return 7; + } + ); + + const stop = await subscribeToBuckets(vi.fn()); + await stop(); + + expect(unsubscribe).toHaveBeenCalledWith(19); +}); diff --git a/packages/api/src/__tests__/thirdPartyChannel.test.ts b/packages/api/src/__tests__/thirdPartyChannel.test.ts index 63c1224644..0e55b04f8b 100644 --- a/packages/api/src/__tests__/thirdPartyChannel.test.ts +++ b/packages/api/src/__tests__/thirdPartyChannel.test.ts @@ -1,6 +1,11 @@ import { describe, expect, test } from 'vitest'; -import { getThirdPartyChannelAgent, isThirdPartyChannel } from '../urbit/utils'; +import { + getChannelKindFromType, + getChannelType, + getThirdPartyChannelAgent, + isThirdPartyChannel, +} from '../urbit/utils'; describe('third party channels', () => { test('%channels-backed kinds have no third-party agent', () => { @@ -13,6 +18,13 @@ describe('third party channels', () => { test('non-%channels kinds report their backing agent', () => { expect(getThirdPartyChannelAgent('notes/~zod/book')).toBe('notes'); expect(isThirdPartyChannel('notes/~zod/book')).toBe(true); + expect(getThirdPartyChannelAgent('buckets/~zod/files')).toBe('buckets'); + expect(isThirdPartyChannel('buckets/~zod/files')).toBe(true); + }); + + test('Bucket nests map to the first-class client channel type', () => { + expect(getChannelType('buckets/~zod/files')).toBe('buckets'); + expect(getChannelKindFromType('buckets')).toBe('buckets'); }); test('non-nest ids (DMs, clubs) are not third-party channels', () => { diff --git a/packages/api/src/client/activityApi.test.ts b/packages/api/src/client/activityApi.test.ts index b4d4011423..aa3b962a7e 100644 --- a/packages/api/src/client/activityApi.test.ts +++ b/packages/api/src/client/activityApi.test.ts @@ -1,14 +1,15 @@ import { beforeEach, describe, expect, test, vi } from 'vitest'; import type * as ub from '../urbit'; -import { subscribeToActivity } from './activityApi'; -import { subscribe } from './urbit'; +import { getThreadUnreadsByChannel, subscribeToActivity } from './activityApi'; +import { scry, subscribe } from './urbit'; vi.mock('./urbit', async () => { const actual = await vi.importActual('./urbit'); return { ...actual, + scry: vi.fn(), subscribe: vi.fn(), }; }); @@ -38,9 +39,22 @@ function mockActivityAdd({ } beforeEach(() => { + vi.mocked(scry).mockReset(); vi.mocked(subscribe).mockReset(); }); +test('does not request activity thread unreads for Buckets', async () => { + await expect( + getThreadUnreadsByChannel({ + id: 'buckets/~zod/project-files', + type: 'buckets', + groupId: '~zod/test-group', + }) + ).resolves.toBeNull(); + + expect(scry).not.toHaveBeenCalled(); +}); + describe('subscribeToActivity invite conversion', () => { test('converts DM invites for opt-in subscribers', async () => { mockActivityAdd({ diff --git a/packages/api/src/client/activityApi.ts b/packages/api/src/client/activityApi.ts index a255ba4bca..a1b347aebe 100644 --- a/packages/api/src/client/activityApi.ts +++ b/packages/api/src/client/activityApi.ts @@ -38,6 +38,12 @@ export async function getGroupAndChannelUnreads() { export async function getThreadUnreadsByChannel( channel: db.Channel ): Promise { + // Buckets are group channels for navigation and permissions, but they do + // not contain posts or threads. Keep them out of the chat-shaped activity + // scries even if a caller reaches this API without the screen-level guard. + if (channel.type === 'buckets') { + return null; + } if (channel.type === 'notes') { // notes channels track per-note unreads instead of thread unreads; the // chat-shaped threads scry below doesn't exist for them. Before the diff --git a/packages/api/src/client/apiUtils.ts b/packages/api/src/client/apiUtils.ts index d71593a8b1..9b74f5caae 100644 --- a/packages/api/src/client/apiUtils.ts +++ b/packages/api/src/client/apiUtils.ts @@ -124,7 +124,8 @@ export function isGroupChannelId(channelId: string) { channelId.startsWith('chat') || channelId.startsWith('diary') || channelId.startsWith('heap') || - channelId.startsWith('notes') + channelId.startsWith('notes') || + channelId.startsWith('buckets') ); } diff --git a/packages/api/src/client/bucketsApi.ts b/packages/api/src/client/bucketsApi.ts new file mode 100644 index 0000000000..d4c1c6e737 --- /dev/null +++ b/packages/api/src/client/bucketsApi.ts @@ -0,0 +1,286 @@ +import type { + BucketsAction, + BucketsActionError, + BucketsFlag, + BucketsGrant, + BucketsReadToken, + BucketsRequestResponse, + BucketsResponse, + BucketsResponseBody, + BucketsSnapshot, + BucketsSummary, +} from '../urbit/buckets'; +import { + BadResponseError, + requestJson, + scry, + subscribe, + unsubscribe, +} from './urbit'; + +const BUCKETS_APP = 'buckets'; +const BUCKETS_V1_PATH = '/buckets/~/v1'; +// The agent answers an unauthenticated request with 401, as %notes does, but +// requestJson only reauths on 403 by default. Without both, an expired Eyre +// cookie fails every Bucket action outright instead of refreshing once. +export const BUCKETS_AUTH_FAILURE_STATUSES: readonly number[] = [401, 403]; + +/** + * A typed refusal from %buckets. + * + * The agent answers an action it won't perform with a reason rather than + * crashing, so callers can tell "you can't do that" from "the request never + * arrived". + */ +export class BucketsActionFailed extends Error { + constructor( + public readonly type: BucketsActionError, + message: string + ) { + super(message); + this.name = 'BucketsActionFailed'; + } +} + +export function bucketsFlagKey(flag: BucketsFlag) { + return `${flag.host}/${flag.name}`; +} + +export function formatBucketsChannelId(flag: BucketsFlag) { + return `buckets/${flag.host}/${flag.name}`; +} + +export function parseBucketsChannelId(channelId: string): BucketsFlag | null { + const [kind, host, name, ...rest] = channelId.split('/'); + if (kind !== 'buckets' || !host || !name || rest.length > 0) return null; + return { host, name }; +} + +/** + * Every bucket on this ship, without their contents. + * + * Entries are unbounded and nothing that lists buckets wants them, so this + * costs the same whether they hold nothing or everything. Use getBucket for + * one bucket's contents, or getBucketsFull for all of them at once. + */ +export async function getBuckets() { + return scry({ app: BUCKETS_APP, path: '/v1/buckets' }); +} + +export async function getBucketsFull() { + return scry({ + app: BUCKETS_APP, + path: '/v1/buckets/full', + }); +} + +/** + * The current state of one bucket, or null if this ship does not have it. + * + * The agent drops a bucket from both this and /v1/buckets under exactly the + * same conditions -- unknown flag, or subscribed but not yet synced -- so a + * 404 here means what an absence from that list meant. Any other failure is + * a failure and is raised, rather than being reported as a missing bucket + * and blanking one that is really there. + */ +export async function getBucket( + flag: BucketsFlag +): Promise { + try { + const response = await scry({ + app: BUCKETS_APP, + path: `/v1/buckets/${flag.host}/${flag.name}`, + }); + return response.type === 'snapshot' + ? { flag: response.flag, state: response.state } + : null; + } catch (e) { + if (e instanceof BadResponseError && e.status === 404) return null; + throw e; + } +} + +/** + * Whether %buckets is installed and running on this ship. + * + * A constant-size read. Asking /v1/buckets for this instead serialises every + * bucket's whole manifest -- entries, names, sizes, checksums -- to answer a + * yes/no, and so got slower the more anyone stored. + */ +export async function getBucketsReady() { + return scry({ app: BUCKETS_APP, path: '/v1/ready' }); +} + +/** + * Submit an action and wait for its terminal answer. + * + * The agent holds the request open until it has a real answer — including + * across the network when the bucket lives on another ship — so there is no + * correlation to do here. A `pending` body is never terminal and never + * reaches us. + */ +const UV_DIGITS = '0123456789abcdefghijklmnopqrstuv'; + +/** + * A request id of our own choosing. + * + * The agent keys both its deduplication and its /request/ lookup on this, + * and mints one itself when we leave it out -- which leaves the caller holding + * an id it never saw, so a lost response cannot be polled for and cannot be + * safely retried. A non-idempotent action is exactly where that matters: a + * dropped %begin-upload answer strands a session, and retrying opens another. + * + * Written in @uv's canonical shape -- 0v, then dot-separated groups of five + * base-32 digits -- because the agent parses it with (slav %uv) and quietly + * falls back to its own id for anything that does not parse. + */ +export function mintRequestId(): string { + const bytes = new Uint8Array(25); + if (globalThis.crypto?.getRandomValues) { + globalThis.crypto.getRandomValues(bytes); + } else { + for (let i = 0; i < bytes.length; i += 1) { + bytes[i] = Math.floor(Math.random() * 256); + } + } + const digits = Array.from(bytes, (byte) => UV_DIGITS[byte & 31]); + // A leading zero is not a shape +scot would ever produce, so do not send one. + if (digits[0] === '0') digits[0] = '1'; + const groups: string[] = []; + for (let i = 0; i < digits.length; i += 5) { + groups.push(digits.slice(i, i + 5).join('')); + } + return `0v${groups.join('.')}`; +} + +/** + * Submit an action and wait for its terminal answer, reporting the id it was + * submitted under. + * + * The id is what makes a lost answer recoverable -- /request/ reads the + * result, and resubmitting under it is answered from the record rather than + * run again -- so a caller that means to recover has to be able to learn it + * before the answer arrives, not from the answer. + */ +export async function submitBucketsAction( + action: BucketsAction, + requestId: string = mintRequestId() +): Promise<{ requestId: string; body: BucketsResponseBody }> { + return { requestId, body: await sendBucketsAction(action, requestId) }; +} + +export async function sendBucketsAction( + action: BucketsAction, + requestId: string = mintRequestId() +): Promise { + const res = await requestJson( + BUCKETS_V1_PATH, + 'POST', + { action, requestId }, + { reauthStatuses: BUCKETS_AUTH_FAILURE_STATUSES } + ); + const body = res?.body; + if (!body) { + throw new Error('%buckets response missing body'); + } + if ('error' in body) { + throw new BucketsActionFailed(body.error.type, body.error.message); + } + return body; +} + +/** + * Submit an action that mints a per-object bearer token, and return it. + * + * Uploads and deletes answer with a grant, because both name one object. + * Reads do not: one token covers the whole bucket, so they answer with a + * token instead -- see getBucketReadToken. + */ +export async function requestBucketsGrant( + action: BucketsAction, + requestId?: string +): Promise { + const body = await sendBucketsAction(action, requestId); + if (!('grant' in body)) { + throw new Error(`%buckets ${action.type} did not return a grant`); + } + return body.grant; +} + +/** + * The bucket read token our own ship currently holds. + * + * The ship keeps this fresh on a timer, so this is a local read with no + * network hop — and no call to the bucket's host. Null means we hold none + * yet, which a cold start resolves by asking for one; the catch also covers + * a genuine failure, and the caller treats both the same way. + */ +export async function getBucketReadToken( + flag: BucketsFlag +): Promise { + return scry({ + app: BUCKETS_APP, + path: `/v1/buckets/${flag.host}/${flag.name}/read-token`, + }).catch(() => null); +} + +/** + * Ask the ship to mint a read token now rather than waiting for its timer. + * + * Only needed on a cold start, when nothing has asked for this bucket yet. + */ +// In-flight mints, shared by everything in this context rather than held per +// caller. The host tracks one waiting request per bucket and reader and denies +// the one a later grant supersedes, so two callers minting at once cost one of +// them an "access changed" failure on a bucket they may read perfectly well. +// +// This does not reach across tabs, which have their own module scope. Two tabs +// opening the same cold bucket at the same instant can still collide; making +// that impossible needs the host to hold more than one waiter, not more +// bookkeeping here. +const inFlightMints = new Map>(); + +export async function requestBucketReadToken( + flag: BucketsFlag +): Promise { + const key = bucketsFlagKey(flag); + const existing = inFlightMints.get(key); + if (existing) return existing; + + const mint = (async () => { + const body = await sendBucketsAction({ type: 'issue-bucket-read', flag }); + if (!('token' in body)) { + throw new Error('%buckets issue-bucket-read did not return a token'); + } + return body.token; + })(); + inFlightMints.set(key, mint); + // Cleared on settle, so a failure is retried rather than cached. + void mint + .catch(() => undefined) + .finally(() => { + if (inFlightMints.get(key) === mint) inFlightMints.delete(key); + }); + return mint; +} + +export async function subscribeToBuckets( + handler: (response: BucketsResponse) => void +) { + let activeSubscriptionId: number | null = null; + const subscriptionId = await subscribe( + { app: BUCKETS_APP, path: '/v1' }, + handler, + { + onSubscriptionId: (id) => { + activeSubscriptionId = id; + }, + } + ); + activeSubscriptionId ??= subscriptionId; + + return () => + activeSubscriptionId === null + ? Promise.resolve() + : unsubscribe(activeSubscriptionId); +} diff --git a/packages/api/src/client/channelContentConfig.ts b/packages/api/src/client/channelContentConfig.ts index 6b9623a1d0..d7636dfb65 100644 --- a/packages/api/src/client/channelContentConfig.ts +++ b/packages/api/src/client/channelContentConfig.ts @@ -62,6 +62,10 @@ export const allCollectionRenderers = { displayName: 'Notes', enumTag: 'notes', }, + 'tlon.r0.collection.buckets': { + displayName: 'Buckets', + enumTag: 'buckets', + }, } as const satisfies Record; export const allDraftInputs = { @@ -81,6 +85,10 @@ export const allDraftInputs = { displayName: 'Notes', enumTag: 'notes', }, + 'tlon.r0.input.buckets': { + displayName: 'Buckets', + enumTag: 'buckets', + }, } as const satisfies Record; export const allContentRenderers = { @@ -120,6 +128,10 @@ export const allContentRenderers = { displayName: 'Notes', enumTag: 'notes', }, + 'tlon.r0.content.buckets': { + displayName: 'Buckets', + enumTag: 'buckets', + }, } as const satisfies Record; export const CollectionRendererId = makeEnum(allCollectionRenderers); diff --git a/packages/api/src/client/index.ts b/packages/api/src/client/index.ts index 900e247241..66ea046577 100644 --- a/packages/api/src/client/index.ts +++ b/packages/api/src/client/index.ts @@ -3,6 +3,7 @@ export * from './agentProtocol'; export * from './a2ui'; export * from './channelContentConfig'; export * from './channelsApi'; +export * from './bucketsApi'; export * from './chatApi'; export * from './contactsApi'; export * from './groupsApi'; diff --git a/packages/api/src/client/urbit.subscribe.test.ts b/packages/api/src/client/urbit.subscribe.test.ts new file mode 100644 index 0000000000..cb17ee2280 --- /dev/null +++ b/packages/api/src/client/urbit.subscribe.test.ts @@ -0,0 +1,50 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; + +import type { Urbit } from '../http-api'; +import { + internalConfigureClient, + internalRemoveClient, + subscribe, +} from './urbit'; + +function configureMockClient() { + const client = { + delete: vi.fn(), + on: vi.fn(), + subscribe: vi.fn().mockResolvedValue(42), + }; + internalConfigureClient({ + shipName: '~zod', + shipUrl: 'http://example.test', + client: client as unknown as Urbit, + }); + return client; +} + +afterEach(() => { + internalRemoveClient(); +}); + +describe('client subscribe wrapper', () => { + test('preserves automatic resubscription when options are omitted', async () => { + const client = configureMockClient(); + + await subscribe({ app: 'groups', path: '/v1' }, vi.fn()); + + expect(client.subscribe).toHaveBeenCalledWith( + expect.objectContaining({ resubOnQuit: true }) + ); + }); + + test('allows callers to disable automatic resubscription explicitly', async () => { + const client = configureMockClient(); + + await subscribe({ app: 'groups', path: '/v1' }, vi.fn(), { + resubOnQuit: false, + }); + + expect(client.subscribe).toHaveBeenCalledWith( + expect.objectContaining({ resubOnQuit: false }) + ); + }); +}); diff --git a/packages/api/src/client/urbit.ts b/packages/api/src/client/urbit.ts index 9fd76601d4..d0eedf0019 100644 --- a/packages/api/src/client/urbit.ts +++ b/packages/api/src/client/urbit.ts @@ -89,6 +89,11 @@ interface UrbitEndpoint { path: string; } +type SubscribeOptions = { + onSubscriptionId?: (id: number) => void; + resubOnQuit?: boolean; +}; + export interface ClientParams { shipName: string; shipUrl: string; @@ -332,7 +337,8 @@ function printEndpoint(endpoint: UrbitEndpoint) { export async function subscribe( endpoint: UrbitEndpoint, - handler: (update: T, id?: number) => void + handler: (update: T, id?: number) => void, + options: SubscribeOptions = {} ): Promise { const doSub = async (err?: (error: any, id: string) => void) => { if (!config.client) { @@ -345,6 +351,8 @@ export async function subscribe( return config.client.subscribe({ app: endpoint.app, path: endpoint.path, + onSubscriptionId: options.onSubscriptionId, + resubOnQuit: options.resubOnQuit ?? true, event: (event: any, mark: string, id?: number) => { logger.debug( `got subscription event on ${printEndpoint(endpoint)}:`, diff --git a/packages/api/src/http-api/Urbit.ts b/packages/api/src/http-api/Urbit.ts index b931e901e0..8e0992bcf3 100644 --- a/packages/api/src/http-api/Urbit.ts +++ b/packages/api/src/http-api/Urbit.ts @@ -823,14 +823,15 @@ export class Urbit { * @param handlers Handlers to deal with various events of the subscription */ async subscribe(params: SubscriptionRequestInterface): Promise { - const { app, path, ship, resubOnQuit, err, event, quit } = { - err: () => {}, - event: () => {}, - quit: () => {}, - resubOnQuit: true, - ...params, - ship: desig(params.ship ?? this.nodeId ?? ''), - }; + const { app, path, ship, resubOnQuit, err, event, quit, onSubscriptionId } = + { + err: () => {}, + event: () => {}, + quit: () => {}, + resubOnQuit: true, + ...params, + ship: desig(params.ship ?? this.nodeId ?? ''), + }; if (this.lastEventId === 0) { this.emit('status-update', { status: 'opening' }); @@ -851,6 +852,7 @@ export class Urbit { err, event, quit, + onSubscriptionId, }); this.emit('subscription', { @@ -860,6 +862,10 @@ export class Urbit { status: 'open', }); + // Publish the replacement id as soon as it is allocated. A consumer may + // unmount while the resubscribe request is still in flight and must still + // be able to unsubscribe the replacement rather than the obsolete id. + onSubscriptionId?.(message.id); await this.sendJSONtoChannel(message); return message.id; diff --git a/packages/api/src/http-api/types.ts b/packages/api/src/http-api/types.ts index c21ac9f2e4..3d4c787644 100644 --- a/packages/api/src/http-api/types.ts +++ b/packages/api/src/http-api/types.ts @@ -211,6 +211,11 @@ export interface SubscriptionRequestInterface extends SubscriptionInterface { * Whether to resubscribe this exact subscription on quit */ resubOnQuit?: boolean; + /** + * Receives the active subscription id, including replacement ids allocated + * when Airlock automatically re-subscribes after an Eyre channel reset. + */ + onSubscriptionId?(id: number): void; ship?: string; } diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 73dfe48074..4aefd8b83a 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -28,6 +28,7 @@ export { type Scry, } from './http-api'; export type { Contact, Group, Channel, Post } from './types/models'; +export * from './urbit/buckets'; export type { AppReference, ChannelReference, @@ -37,6 +38,7 @@ export type { export { checkNest, getChannelType, + isThirdPartyChannel, nestToFlag, whomIsDm, whomIsMultiDm, diff --git a/packages/api/src/types/PostCollectionConfiguration.ts b/packages/api/src/types/PostCollectionConfiguration.ts index b503809640..3b4f94c06a 100644 --- a/packages/api/src/types/PostCollectionConfiguration.ts +++ b/packages/api/src/types/PostCollectionConfiguration.ts @@ -38,6 +38,9 @@ export function layoutTypeFromChannel( case CollectionRendererId.gallery: return 'grid'; + case CollectionRendererId.buckets: + return 'comfy-list-top-to-bottom'; + default: // fallthrough to legacy logic break; @@ -65,6 +68,9 @@ export function layoutTypeFromChannel( case 'notes': return 'comfy-list-top-to-bottom'; + + case 'buckets': + return 'comfy-list-top-to-bottom'; } } @@ -198,6 +204,7 @@ export function configurationFromChannel( }; case 'notes': + case 'buckets': return { includeDeletedPosts: false, postActionIds: [], diff --git a/packages/api/src/types/models.ts b/packages/api/src/types/models.ts index c38182b333..bb8b28ae3d 100644 --- a/packages/api/src/types/models.ts +++ b/packages/api/src/types/models.ts @@ -49,7 +49,8 @@ export type ChannelType = | 'gallery' | 'dm' | 'groupDm' - | 'notes'; + | 'notes' + | 'buckets'; export type UnreadChannelType = 'channel' | 'dm'; export type ActivityBucket = 'all' | 'mentions' | 'replies'; export type PinType = 'group' | 'channel' | 'dm' | 'groupDm'; diff --git a/packages/api/src/urbit/buckets.ts b/packages/api/src/urbit/buckets.ts new file mode 100644 index 0000000000..c61e2601a7 --- /dev/null +++ b/packages/api/src/urbit/buckets.ts @@ -0,0 +1,193 @@ +export type BucketsFlag = { + host: string; + name: string; +}; + +export type BucketsBucket = { + id: number; + title: string; + createdBy: string; + createdAt: number; + updatedBy: string; + updatedAt: number; +}; + +export type BucketsFile = { + mime: string; + size: number; + checksum: string | null; + objectKey: string; + status: 'pending' | 'ready' | 'failed'; +}; + +type BucketsEntryBase = { + id: number; + parentId: number | null; + name: string; + createdBy: string; + createdAt: number; + updatedBy: string; + updatedAt: number; +}; + +export type BucketsFolderEntry = BucketsEntryBase & { + kind: 'folder'; +}; + +export type BucketsFileEntry = BucketsEntryBase & { + kind: 'file'; + file: BucketsFile; +}; + +export type BucketsEntry = BucketsFolderEntry | BucketsFileEntry; + +export type BucketsState = { + bucket: BucketsBucket; + group: BucketsFlag; + /** + * Bucket writers only. Readability belongs to %groups: the roles named at + * creation are handed to it with the channel, and the channel record is + * where they live from then on. + */ + writers: string[]; + entries: BucketsEntry[]; + revision: number; +}; + +export type BucketsSnapshot = { + flag: BucketsFlag; + state: BucketsState; +}; + +export type BucketsUpdate = + | { type: 'bucket-created'; bucket: BucketsBucket } + | { type: 'bucket-deleted' } + | { type: 'bucket-updated'; bucket: BucketsBucket } + | { type: 'writers-updated'; writers: string[] } + | { type: 'entry-created'; id: number; entry: BucketsEntry } + | { type: 'entry-updated'; id: number; entry: BucketsEntry } + | { type: 'entries-deleted'; ids: number[] }; + +/** + * A bucket without its entries. + * + * Shares field positions with BucketsSnapshot so code reading the metadata + * finds it in the same place either way; `state.entries` is simply absent. + */ +export type BucketsSummary = { + flag: BucketsFlag; + state: Omit; +}; + +export type BucketsResponse = + | { type: 'snapshot'; flag: BucketsFlag; state: BucketsState } + | { + type: 'update'; + flag: BucketsFlag; + revision: number; + update: BucketsUpdate; + }; + +export type BucketsAction = + | { + type: 'create'; + name: string; + title: string; + group: BucketsFlag; + /** Passed straight to %groups as the new channel's reader roles. */ + readers: string[]; + writers: string[]; + } + | { type: 'delete-bucket'; flag: BucketsFlag } + | { type: 'set-title'; flag: BucketsFlag; title: string } + | { type: 'set-writers'; flag: BucketsFlag; writers: string[] } + | { + type: 'create-folder'; + flag: BucketsFlag; + parentId: number | null; + name: string; + } + | { + type: 'begin-upload'; + flag: BucketsFlag; + parentId: number | null; + name: string; + mime: string; + size: number; + checksum: string | null; + } + | { + type: 'cancel-upload'; + flag: BucketsFlag; + sessionId: string; + reason: string; + } + | { type: 'issue-bucket-read'; flag: BucketsFlag } + | { + type: 'issue-delete'; + flag: BucketsFlag; + id: number; + } + | { type: 'rename-entry'; flag: BucketsFlag; id: number; name: string } + | { + type: 'move-entry'; + flag: BucketsFlag; + id: number; + parentId: number | null; + } + | { + type: 'delete-entry'; + flag: BucketsFlag; + id: number; + recursive: boolean; + }; + +/** + * A host-minted bearer token, returned only to the ship that asked for it. + * + * For an upload the token is the session id; for a read or delete it is a + * freshly minted capability. Either way it is what gets presented to the + * storage broker — the client never invents one. + */ +export type BucketsGrant = { + token: string; + entryId: number; + expiresAt: string; +}; + +export type BucketsActionError = + | 'not-authorized' + | 'not-found' + | 'invalid-input' + | 'unknown'; + +/** + * The terminal answer to one submitted action. + * + * `pending` is emitted by our own ship once it has forwarded the action to the + * bucket's host and is waiting; the host's real answer replaces it. + */ +/** + * A bucket-wide read capability. + * + * Read access is uniform across a bucket, so one token covers every ready + * object in it. Each ship holds its own, refreshed by its own timer, and + * serves it to local clients over a scry — so a read costs no round trip to + * the bucket's host. + */ +export type BucketsReadToken = { + token: string; + expiresAt: string; +}; + +export type BucketsResponseBody = + | { ok: null } + | { grant: BucketsGrant } + | { token: BucketsReadToken } + | { pending: null } + | { error: { type: BucketsActionError; message: string } }; + +export type BucketsRequestResponse = { + requestId: string; + body: BucketsResponseBody; +}; diff --git a/packages/api/src/urbit/channel.ts b/packages/api/src/urbit/channel.ts index c0564f8acc..3f83bb286b 100644 --- a/packages/api/src/urbit/channel.ts +++ b/packages/api/src/urbit/channel.ts @@ -110,7 +110,7 @@ export type KindDataChat = { }; export type KindData = KindDataDiary | KindDataChat | KindDataHeap; -export type Kind = 'heap' | 'diary' | 'chat' | 'notes'; +export type Kind = 'heap' | 'diary' | 'chat' | 'notes' | 'buckets'; export interface PostEssay { content: Story; diff --git a/packages/api/src/urbit/index.ts b/packages/api/src/urbit/index.ts index 45ca477fa9..3aaf70d3cd 100644 --- a/packages/api/src/urbit/index.ts +++ b/packages/api/src/urbit/index.ts @@ -3,6 +3,8 @@ import * as content from './content'; import * as dms from './dms'; import * as groups from './groups'; +export * from './buckets'; + // These re-exports are necessary in order to resolve overlaps between the below // types so that everything can fit in a single namespace. export type Ship = channel.Ship; diff --git a/packages/api/src/urbit/utils.ts b/packages/api/src/urbit/utils.ts index 91ada327e2..a521d5b0ef 100644 --- a/packages/api/src/urbit/utils.ts +++ b/packages/api/src/urbit/utils.ts @@ -13,8 +13,8 @@ import type * as ubg from './groups'; const logger = createDevLogger('urbitUtils', false); -type App = 'chat' | 'heap' | 'diary' | 'notes'; -const APP_PREFIXES = ['chat', 'heap', 'diary', 'notes']; +type App = 'chat' | 'heap' | 'diary' | 'notes' | 'buckets'; +const APP_PREFIXES = ['chat', 'heap', 'diary', 'notes', 'buckets']; export function checkNest(nest: string): boolean { const parts = nest.split('/'); @@ -163,6 +163,8 @@ export function getChannelType(channelId: string) { return 'notebook'; } else if (app === 'notes') { return 'notes'; + } else if (app === 'buckets') { + return 'buckets'; } else { return 'chat'; } @@ -179,6 +181,8 @@ export function getChannelKindFromType( return 'diary'; } else if (type === 'notes') { return 'notes'; + } else if (type === 'buckets') { + return 'buckets'; } else { return 'chat'; } diff --git a/packages/app/features/buckets/BucketsLiveChannel.tsx b/packages/app/features/buckets/BucketsLiveChannel.tsx new file mode 100644 index 0000000000..5197e02a09 --- /dev/null +++ b/packages/app/features/buckets/BucketsLiveChannel.tsx @@ -0,0 +1,632 @@ +import type { BucketsEntry, BucketsFlag } from '@tloncorp/api'; +import * as db from '@tloncorp/shared/db'; +import { ConfirmDialog, Text } from '@tloncorp/ui'; +import * as DocumentPicker from 'expo-document-picker'; +import * as ImagePicker from 'expo-image-picker'; +import { + ReactElement, + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from 'react'; +import { Linking, useWindowDimensions } from 'react-native'; + +import { + BucketFileViewer, + BucketItem, + BucketSearchResult, + BucketUploadCandidate, + BucketsHeaderActions, + BucketsNewSheet, + BucketsPane, + BucketsSearchScreen, + ChannelHeader, + ChannelHeaderItemsProvider, + ScreenHeader, + XStack, + YStack, + canPreviewAsText, + useCanWrite, + useCurrentUserId, + useHideChannelHeader, + useIsWindowNarrow, + useRegisterChannelHeaderItem, +} from '../../ui'; +import { imagePickerAssetsToBucketUploadCandidates } from './bucketMediaPicker'; +import { findUploadShadowEntryIds } from './bucketUploadReconciliation'; +import { + formatBucketTimestamp, + formatFileSize, + useLiveBucket, +} from './useLiveBucket'; + +type SearchOrigin = { + activeFolderId: number | null; + selectedItemId: string | null; +}; + +function toItem( + entry: BucketsEntry, + childCounts: ReadonlyMap +): BucketItem { + if (entry.kind === 'folder') { + return { + author: entry.updatedBy, + id: String(entry.id), + itemCount: childCounts.get(entry.id) ?? 0, + kind: 'folder', + modifiedLabel: formatBucketTimestamp(entry.updatedAt), + name: entry.name, + }; + } + + return { + author: entry.updatedBy, + id: String(entry.id), + kind: 'file', + mimeType: entry.file.mime, + modifiedLabel: formatBucketTimestamp(entry.updatedAt), + name: entry.name, + // Files are always fetched through a short-lived read grant, so there is + // no URL to show until one is issued. + previewUri: undefined, + size: entry.file.size, + sizeLabel: formatFileSize(entry.file.size), + uploadSize: entry.file.status === 'pending' ? entry.file.size : undefined, + uploadError: + entry.file.status === 'failed' + ? 'The object was not finalized' + : undefined, + uploadProgress: entry.file.status === 'pending' ? 0 : undefined, + uploadState: + entry.file.status === 'pending' + ? 'uploading' + : entry.file.status === 'failed' + ? 'failed' + : undefined, + }; +} + +function sortItems(items: BucketItem[]) { + return [...items].sort((left, right) => { + if (left.kind !== right.kind) return left.kind === 'folder' ? -1 : 1; + return left.name.localeCompare(right.name); + }); +} + +function pathLabelFor( + entry: BucketsEntry, + entriesById: ReadonlyMap, + rootLabel: string +) { + const names: string[] = []; + let parentId = entry.parentId; + while (parentId !== null) { + const parent = entriesById.get(parentId); + if (!parent) break; + names.unshift(parent.name); + parentId = parent.parentId; + } + return [rootLabel, ...names].join(' / '); +} + +export function BucketsLiveChannel({ + channel: providedChannel, + embedded = false, + flag, + viewport = 'responsive', +}: { + channel?: db.Channel; + embedded?: boolean; + flag: BucketsFlag; + viewport?: 'mobile' | 'responsive'; +}) { + const { height: windowHeight } = useWindowDimensions(); + const isWindowNarrow = useIsWindowNarrow(); + const isMobileLayout = viewport === 'mobile' || isWindowNarrow; + const live = useLiveBucket(flag); + const [activeFolderId, setActiveFolderId] = useState(null); + const [selectedItemId, setSelectedItemId] = useState(null); + const [newSheetOpen, setNewSheetOpen] = useState(false); + const [searchOpen, setSearchOpen] = useState(false); + const [searchOrigin, setSearchOrigin] = useState(null); + const [query, setQuery] = useState(''); + const [previewItem, setPreviewItem] = useState(null); + const [previewLoading, setPreviewLoading] = useState(false); + const [previewError, setPreviewError] = useState(null); + const previewRequestId = useRef(0); + const [operationError, setOperationError] = useState(null); + const [folderPendingDeletion, setFolderPendingDeletion] = + useState(null); + const currentUserId = useCurrentUserId(); + useHideChannelHeader(embedded && previewItem !== null); + const [mediaLibraryPermissionStatus, requestMediaLibraryPermission] = + ImagePicker.useMediaLibraryPermissions(); + const entries = useMemo( + () => live.snapshot?.state.entries ?? [], + [live.snapshot?.state.entries] + ); + const entriesById = useMemo( + () => new Map(entries.map((entry) => [entry.id, entry])), + [entries] + ); + const childCounts = useMemo(() => { + const counts = new Map(); + for (const entry of entries) { + if (entry.parentId !== null) { + counts.set(entry.parentId, (counts.get(entry.parentId) ?? 0) + 1); + } + } + return counts; + }, [entries]); + const suppressedIds = useMemo( + () => findUploadShadowEntryIds(live.uploads), + [live.uploads] + ); + const serverEntries = useMemo( + () => entries.filter((entry) => !suppressedIds.has(entry.id)), + [entries, suppressedIds] + ); + const rootLabel = live.snapshot?.state.bucket.title ?? 'Bucket'; + const activeFolderCandidate = + activeFolderId === null ? undefined : entriesById.get(activeFolderId); + const activeFolder = + activeFolderCandidate?.kind === 'folder' + ? activeFolderCandidate + : undefined; + // Someone else can delete the folder we are standing in. Without this the + // pane keeps filtering on an id nothing has, so it shows an empty list that + // goBack cannot leave -- it reads the parent off the folder that is gone. + useEffect(() => { + if (activeFolderId === null || activeFolder) return; + if (live.loading || !live.snapshot) return; + setActiveFolderId(null); + setSelectedItemId(null); + }, [activeFolder, activeFolderId, live.loading, live.snapshot]); + const rootFolders = serverEntries.filter( + (entry) => entry.kind === 'folder' && entry.parentId === null + ); + const visibleServerItems = serverEntries + .filter((entry) => entry.parentId === activeFolderId) + .map((entry) => toItem(entry, childCounts)); + const visibleLocalItems = live.uploads + .filter((upload) => upload.parentId === activeFolderId) + .map((upload) => live.localItems.find((item) => item.id === upload.id)) + .filter((item): item is BucketItem => item !== undefined); + const visibleItems = sortItems([...visibleLocalItems, ...visibleServerItems]); + const sidebarItems = sortItems( + rootFolders.map((entry) => toItem(entry, childCounts)) + ); + const allSearchResults = useMemo( + () => + serverEntries.map((entry) => ({ + ...toItem(entry, childCounts), + parentFolderId: entry.parentId === null ? null : String(entry.parentId), + pathLabel: pathLabelFor(entry, entriesById, rootLabel), + })), + [childCounts, entriesById, rootLabel, serverEntries] + ); + const normalizedQuery = query.trim().toLowerCase(); + const searchResults = normalizedQuery + ? allSearchResults.filter((item) => + [item.name, item.pathLabel, item.author, item.mimeType] + .filter(Boolean) + .join(' ') + .toLowerCase() + .includes(normalizedQuery) + ) + : []; + + const fallbackChannel = useMemo( + () => + ({ + description: '', + id: `buckets/${flag.host}/${flag.name}`, + title: rootLabel, + type: 'buckets', + }) as db.Channel, + [flag.host, flag.name, rootLabel] + ); + const channel = providedChannel ?? fallbackChannel; + const canEdit = useCanWrite(channel, currentUserId); + + const reportOperation = async (operation: Promise) => { + try { + setOperationError(null); + await operation; + await live.refresh(); + } catch (cause) { + setOperationError(cause instanceof Error ? cause.message : String(cause)); + } + }; + + const loadPreview = async (item: BucketItem) => { + const requestId = ++previewRequestId.current; + setPreviewItem(item); + setPreviewLoading(true); + setPreviewError(null); + + try { + const previewUri = await live.readUrl(Number(item.id)); + if (previewRequestId.current !== requestId) return; + + const readableItem = { ...item, previewUri }; + setPreviewItem(readableItem); + + // Checked against the manifest size before fetching, not after: the + // read itself is what would exhaust memory. + if ( + canPreviewAsText(readableItem) && + readableItem.textContent === undefined + ) { + const response = await fetch(previewUri); + if (!response.ok) { + throw new Error(`File request failed (${response.status})`); + } + const textContent = await response.text(); + if (previewRequestId.current !== requestId) return; + setPreviewItem({ ...readableItem, textContent }); + } + + if (previewRequestId.current === requestId) { + setPreviewLoading(false); + } + } catch (cause) { + if (previewRequestId.current !== requestId) return; + setPreviewLoading(false); + setPreviewError(cause instanceof Error ? cause.message : String(cause)); + } + }; + + const openItem = (item: BucketItem) => { + if (item.kind === 'folder') { + setActiveFolderId(Number(item.id)); + setSelectedItemId(null); + return; + } + setSelectedItemId(item.id); + setOperationError(null); + void loadPreview(item); + }; + + const closePreview = () => { + previewRequestId.current += 1; + setPreviewItem(null); + setPreviewLoading(false); + setPreviewError(null); + }; + + const chooseUploads = async () => { + const result = await DocumentPicker.getDocumentAsync({ + copyToCacheDirectory: true, + multiple: true, + type: '*/*', + }); + if (!result.assets?.length) return; + // On web the picker hands back the real File. Dropping it would make the + // upload task fetch(uri).blob() first, buffering the whole file in memory + // before the PUT can start -- drag-and-drop already streams the File. + const candidates: BucketUploadCandidate[] = result.assets.map((asset) => ({ + file: typeof File === 'undefined' ? undefined : asset.file, + mimeType: asset.mimeType ?? undefined, + name: asset.name, + size: asset.size ?? -1, + uri: asset.uri, + })); + live.addUploads(candidates, activeFolderId); + }; + + const choosePhotos = async () => { + try { + setOperationError(null); + if (mediaLibraryPermissionStatus?.granted === false) { + const permissionResult = await requestMediaLibraryPermission(); + if (!permissionResult.granted) { + setOperationError( + 'Photo library access is required to choose photos.' + ); + return; + } + } + + const result = await ImagePicker.launchImageLibraryAsync({ + allowsEditing: false, + allowsMultipleSelection: true, + exif: false, + mediaTypes: ['images', 'videos'], + orderedSelection: true, + quality: 1, + selectionLimit: 0, + shouldDownloadFromNetwork: true, + }); + if (result.canceled || !result.assets.length) return; + + live.addUploads( + imagePickerAssetsToBucketUploadCandidates(result.assets), + activeFolderId + ); + } catch (cause) { + setOperationError(cause instanceof Error ? cause.message : String(cause)); + } + }; + + const openNewSheet = useCallback(() => { + setNewSheetOpen(true); + }, []); + + const openSearch = useCallback(() => { + setSearchOrigin({ activeFolderId, selectedItemId }); + setSearchOpen(true); + setQuery(''); + }, [activeFolderId, selectedItemId]); + + const closeSearch = () => { + if (searchOrigin) { + setActiveFolderId(searchOrigin.activeFolderId); + setSelectedItemId(searchOrigin.selectedItemId); + } + setSearchOrigin(null); + setSearchOpen(false); + setQuery(''); + }; + + const openSearchResult = (result: BucketSearchResult) => { + setSearchOrigin((current) => current ?? { activeFolderId, selectedItemId }); + if (result.kind === 'folder') { + setActiveFolderId(Number(result.id)); + setSelectedItemId(null); + } else { + setActiveFolderId( + result.parentFolderId === null ? null : Number(result.parentFolderId) + ); + // The row is labelled Open, so open it. Selecting alone dropped the user + // back on the list to find the same file a second time. + openItem(result); + } + setSearchOpen(false); + }; + + const goBack = () => { + if (searchOrigin) { + setSearchOpen(true); + return; + } + if (activeFolder) { + setActiveFolderId(activeFolder.parentId); + setSelectedItemId(null); + } + }; + + const paneProps = { + canEdit, + // Suppressed only where the ChannelHeader below already names the folder + // and carries its own back button. Embedded is the production channel + // flow, which hides that header, so the breadcrumb is the only way up. + currentFolder: isMobileLayout && !embedded ? undefined : activeFolder?.name, + items: visibleItems, + rootLabel, + selectedItemId, + state: live.loading ? ('loading' as const) : ('populated' as const), + uploadAggregateProgress: live.uploadAggregateProgress, + uploadItems: live.localItems, + onCancelUpload: (item: BucketItem) => void live.cancelUpload(item.id), + onNavigateUp: goBack, + onDeleteItem: (item: BucketItem) => { + if (item.kind === 'folder') { + setFolderPendingDeletion(item); + return; + } + void reportOperation(live.deleteEntry(Number(item.id), false)); + }, + onDownloadItem: (item: BucketItem) => { + void live + .readUrl(Number(item.id)) + .then((url) => Linking.openURL(url)) + .catch((cause) => + setOperationError( + cause instanceof Error ? cause.message : String(cause) + ) + ); + }, + onFilesDropped: (files: BucketUploadCandidate[]) => + live.addUploads(files, activeFolderId), + onOpenItem: (item: BucketItem) => void openItem(item), + onRetryUpload: (item: BucketItem) => void live.retryUpload(item.id), + }; + + const newSheet = ( + + void reportOperation(live.createFolder(activeFolderId, name)) + } + onOpenChange={setNewSheetOpen} + onChoosePhotos={() => void choosePhotos()} + onUploadFiles={() => void chooseUploads()} + /> + ); + + const errorMessage = operationError ?? live.error; + + return ( + + + {previewItem ? ( + void Linking.openURL(previewItem.previewUri!) + : undefined + } + onRetry={() => void loadPreview(previewItem)} + /> + ) : searchOpen ? ( + + ) : isMobileLayout ? ( + + + {!embedded ? ( + + ) : null} + {errorMessage ? : null} + + {newSheet} + + ) : ( + + + + folder.id === activeFolderId) + ? String(activeFolderId) + : null + } + state={live.loading ? 'loading' : 'populated'} + onOpenItem={(item) => void openItem(item)} + /> + + + setNewSheetOpen(true)} + onSearch={openSearch} + /> + } + showSubtitle + subtitle="Bucket" + title={rootLabel} + useHorizontalTitleLayout + /> + {errorMessage ? : null} + + {newSheet} + + + )} + + { + if (!open) setFolderPendingDeletion(null); + }} + title={`Delete ${folderPendingDeletion?.name ?? 'folder'}?`} + description="This folder and everything inside it will be permanently deleted for everyone. This cannot be undone." + confirmText="Delete folder" + cancelText="Cancel" + destructive + onConfirm={() => { + const item = folderPendingDeletion; + setFolderPendingDeletion(null); + if (item) { + void reportOperation(live.deleteEntry(Number(item.id), true)); + } + }} + /> + + ); +} + +function LiveError({ message }: { message: string }) { + return ( + + + {message} + + + ); +} + +function RegisteredLiveHeaderActions({ + canEdit, + onNew, + onSearch, + showSearch, +}: { + canEdit: boolean; + onNew: () => void; + onSearch: () => void; + showSearch: boolean; +}) { + const actions = useMemo( + () => ( + + ), + [canEdit, onNew, onSearch, showSearch] + ); + useRegisterChannelHeaderItem(actions); + return null; +} + +function MaybeChannelHeaderItemsProvider({ + children, + embedded, +}: { + children: ReactElement; + embedded: boolean; +}) { + return embedded ? ( + children + ) : ( + {children} + ); +} diff --git a/packages/app/features/buckets/bucketDeletion.test.ts b/packages/app/features/buckets/bucketDeletion.test.ts new file mode 100644 index 0000000000..a6d6a03f90 --- /dev/null +++ b/packages/app/features/buckets/bucketDeletion.test.ts @@ -0,0 +1,78 @@ +import type { BucketsFileEntry } from '@tloncorp/api'; +import { describe, expect, it, vi } from 'vitest'; + +import { deletePrivateBucketFiles } from './bucketDeletion'; + +function file(id: number, objectKey: string): BucketsFileEntry { + return { + createdAt: 1, + createdBy: '~zod', + file: { + checksum: null, + mime: 'text/plain', + objectKey, + size: 10, + status: 'ready', + }, + id, + kind: 'file', + name: `${id}.txt`, + parentId: null, + updatedAt: 1, + updatedBy: '~zod', + }; +} + +describe('deletePrivateBucketFiles', () => { + it('removes each manifest entry before deleting the next object', async () => { + const calls: string[] = []; + const secondFailure = new Error('second object failed'); + + await expect( + deletePrivateBucketFiles( + [file(1, 'object-1'), file(2, 'object-2')], + { host: '~zod', name: 'files' }, + { + issueDelete: async (id) => { + calls.push(`issue:${id}`); + return `cap-${id}`; + }, + deleteObject: async (_capability, _host, objectId) => { + calls.push(`object:${objectId}`); + if (objectId === 'object-2') throw secondFailure; + }, + deleteManifestEntry: async (id) => { + calls.push(`manifest:${id}`); + }, + isAlreadyDeleted: () => false, + } + ) + ).rejects.toBe(secondFailure); + + expect(calls).toEqual([ + 'issue:1', + 'object:object-1', + 'manifest:1', + 'issue:2', + 'object:object-2', + ]); + }); + + it('continues manifest cleanup when the object was already deleted', async () => { + const deleteManifestEntry = vi.fn().mockResolvedValue(undefined); + const missingObject = new Error('object was not found'); + + await deletePrivateBucketFiles( + [file(1, 'object-1')], + { host: '~zod', name: 'files' }, + { + issueDelete: vi.fn().mockResolvedValue('cap-1'), + deleteObject: vi.fn().mockRejectedValue(missingObject), + deleteManifestEntry, + isAlreadyDeleted: (cause) => cause === missingObject, + } + ); + + expect(deleteManifestEntry).toHaveBeenCalledWith(1); + }); +}); diff --git a/packages/app/features/buckets/bucketDeletion.ts b/packages/app/features/buckets/bucketDeletion.ts new file mode 100644 index 0000000000..8793e7f357 --- /dev/null +++ b/packages/app/features/buckets/bucketDeletion.ts @@ -0,0 +1,35 @@ +import type { BucketsFileEntry, BucketsFlag } from '@tloncorp/api'; + +export type PrivateFileDeletionOperations = { + deleteManifestEntry: (id: number) => Promise; + deleteObject: ( + capability: string, + host: string, + objectId: string + ) => Promise; + isAlreadyDeleted: (cause: unknown) => boolean; + /** Asks the host for a delete grant and returns its bearer token. */ + issueDelete: (id: number) => Promise; + onManifestDelete?: (id: number) => void; +}; + +export async function deletePrivateBucketFiles( + entries: BucketsFileEntry[], + flag: BucketsFlag, + operations: PrivateFileDeletionOperations +) { + for (const entry of entries) { + const capability = await operations.issueDelete(entry.id); + try { + await operations.deleteObject( + capability, + flag.host, + entry.file.objectKey + ); + } catch (cause) { + if (!operations.isAlreadyDeleted(cause)) throw cause; + } + await operations.deleteManifestEntry(entry.id); + operations.onManifestDelete?.(entry.id); + } +} diff --git a/packages/app/features/buckets/bucketMediaPicker.ts b/packages/app/features/buckets/bucketMediaPicker.ts new file mode 100644 index 0000000000..39c9431e99 --- /dev/null +++ b/packages/app/features/buckets/bucketMediaPicker.ts @@ -0,0 +1,49 @@ +import type { ImagePickerAsset } from 'expo-image-picker'; + +import type { BucketUploadCandidate } from '../../ui'; +import { getFileSize } from '../../utils/files'; + +function getAssetSize(asset: ImagePickerAsset) { + if (asset.fileSize != null) return asset.fileSize; + if (asset.file) return asset.file.size; + + try { + return getFileSize(asset.uri) ?? -1; + } catch { + return -1; + } +} + +function fallbackMediaName(asset: ImagePickerAsset, index: number) { + const uriName = asset.uri.split('/').pop()?.split(/[?#]/)[0]?.trim(); + if (uriName) return uriName; + + const mimeSubtype = asset.mimeType?.split('/')[1]?.toLowerCase(); + const extension = + mimeSubtype === 'jpeg' + ? 'jpg' + : mimeSubtype === 'quicktime' + ? 'mov' + : mimeSubtype; + const isVideo = + asset.type === 'video' || + asset.type === 'pairedVideo' || + asset.mimeType?.startsWith('video/'); + + return `${isVideo ? 'video' : 'photo'}-${index + 1}.${extension ?? (isVideo ? 'mov' : 'jpg')}`; +} + +export function imagePickerAssetsToBucketUploadCandidates( + assets: ImagePickerAsset[] +): BucketUploadCandidate[] { + return assets.map((asset, index) => ({ + file: asset.file, + mimeType: asset.mimeType ?? asset.file?.type ?? undefined, + name: + asset.fileName?.trim() || + asset.file?.name || + fallbackMediaName(asset, index), + size: getAssetSize(asset), + uri: asset.uri, + })); +} diff --git a/packages/app/features/buckets/bucketUploadReconciliation.test.ts b/packages/app/features/buckets/bucketUploadReconciliation.test.ts new file mode 100644 index 0000000000..1ec1628034 --- /dev/null +++ b/packages/app/features/buckets/bucketUploadReconciliation.test.ts @@ -0,0 +1,102 @@ +import type { BucketsSnapshot } from '@tloncorp/api'; +import { describe, expect, it } from 'vitest'; + +import { + bucketResponseHasRevisionGap, + findUploadShadowEntryIds, + removeEntryFromBucketSnapshot, +} from './bucketUploadReconciliation'; + +const snapshot = { + flag: { host: '~zod', name: 'project-files' }, + state: { + bucket: { + createdAt: 1, + createdBy: '~zod', + id: 1, + title: 'Project Files', + updatedAt: 1, + updatedBy: '~zod', + }, + entries: [ + { + createdAt: 1, + createdBy: '~zod', + file: { + checksum: null, + mime: 'image/heic', + objectKey: 'first', + size: 42, + status: 'ready', + }, + id: 10, + kind: 'file', + name: 'IMG_0111.heic', + parentId: null, + updatedAt: 1, + updatedBy: '~zod', + }, + ], + group: { host: '~zod', name: 'group' }, + revision: 2, + writers: ['admin'], + }, +} satisfies BucketsSnapshot; + +describe('findUploadShadowEntryIds', () => { + it('hides the entry an upload row already stands for', () => { + expect( + findUploadShadowEntryIds([{ serverEntryId: 10 }, { serverEntryId: 11 }]) + ).toEqual(new Set([10, 11])); + }); + + it('hides nothing for an upload that has no entry yet', () => { + expect(findUploadShadowEntryIds([{}])).toEqual(new Set()); + }); +}); + +describe('removeEntryFromBucketSnapshot', () => { + it('optimistically removes an entry', () => { + const next = removeEntryFromBucketSnapshot(snapshot, 10); + expect(next.state.entries).toEqual([]); + }); + + it('leaves unrelated entries alone', () => { + const next = removeEntryFromBucketSnapshot(snapshot, 999); + expect(next.state.entries).toHaveLength(1); + }); +}); + +describe('bucketResponseHasRevisionGap', () => { + it('requests a refresh when an update skips a revision', () => { + expect( + bucketResponseHasRevisionGap(snapshot, { + type: 'update', + flag: snapshot.flag, + revision: 4, + update: { type: 'entries-deleted', ids: [10] }, + }) + ).toBe(true); + }); + + it('accepts the next revision', () => { + expect( + bucketResponseHasRevisionGap(snapshot, { + type: 'update', + flag: snapshot.flag, + revision: 3, + update: { type: 'entries-deleted', ids: [10] }, + }) + ).toBe(false); + }); + + it('accepts a replacement snapshot at any revision', () => { + expect( + bucketResponseHasRevisionGap(snapshot, { + type: 'snapshot', + flag: snapshot.flag, + state: snapshot.state, + }) + ).toBe(false); + }); +}); diff --git a/packages/app/features/buckets/bucketUploadReconciliation.ts b/packages/app/features/buckets/bucketUploadReconciliation.ts new file mode 100644 index 0000000000..7941a69151 --- /dev/null +++ b/packages/app/features/buckets/bucketUploadReconciliation.ts @@ -0,0 +1,44 @@ +import type { BucketsResponse, BucketsSnapshot } from '@tloncorp/api'; + +export function bucketResponseHasRevisionGap( + snapshot: BucketsSnapshot | null, + response: BucketsResponse +) { + return ( + response.type === 'update' && + snapshot !== null && + response.revision > snapshot.state.revision + 1 + ); +} + +/** + * Server entries that an optimistic upload row is already standing in for. + * + * A pending upload is invisible to the manifest until its object lands, so the + * only overlap is the moment between the entry being published and the local + * row being cleared. The upload knows its own entry id — the host returns it + * when granting the upload — so no matching against snapshot metadata is + * needed to find it. + */ +export function findUploadShadowEntryIds( + uploads: readonly { serverEntryId?: number }[] +): Set { + return new Set( + uploads + .map((upload) => upload.serverEntryId) + .filter((id): id is number => id !== undefined) + ); +} + +export function removeEntryFromBucketSnapshot( + snapshot: BucketsSnapshot, + entryId: number +): BucketsSnapshot { + return { + ...snapshot, + state: { + ...snapshot.state, + entries: snapshot.state.entries.filter((entry) => entry.id !== entryId), + }, + }; +} diff --git a/packages/app/features/buckets/bucketUploadTask.native.ts b/packages/app/features/buckets/bucketUploadTask.native.ts new file mode 100644 index 0000000000..925b95f9f4 --- /dev/null +++ b/packages/app/features/buckets/bucketUploadTask.native.ts @@ -0,0 +1,49 @@ +import * as FileSystem from 'expo-file-system/legacy'; + +import type { CreateBucketUploadTask } from './bucketUploadTask.types'; + +export const createBucketUploadTask: CreateBucketUploadTask = ( + uploadUrl, + candidate, + headers, + onProgress +) => { + if (!candidate.uri) { + return { + cancel: async () => {}, + upload: Promise.reject( + new Error('This file does not have a local upload URI') + ), + }; + } + + const task = FileSystem.createUploadTask( + uploadUrl, + candidate.uri, + { + headers, + httpMethod: 'PUT', + uploadType: FileSystem.FileSystemUploadType.BINARY_CONTENT, + }, + ({ totalBytesExpectedToSend, totalBytesSent }) => { + if (totalBytesExpectedToSend > 0) { + onProgress( + Math.round((totalBytesSent / totalBytesExpectedToSend) * 100) + ); + } + } + ); + + return { + cancel: () => task.cancelAsync(), + upload: task.uploadAsync().then((response) => { + if (!response) { + throw new Error('Object upload was cancelled'); + } + if (response.status < 200 || response.status >= 300) { + throw new Error(`Object upload failed (${response.status})`); + } + onProgress(100); + }), + }; +}; diff --git a/packages/app/features/buckets/bucketUploadTask.ts b/packages/app/features/buckets/bucketUploadTask.ts new file mode 100644 index 0000000000..5ae5a4e06f --- /dev/null +++ b/packages/app/features/buckets/bucketUploadTask.ts @@ -0,0 +1,64 @@ +import type { CreateBucketUploadTask } from './bucketUploadTask.types'; + +export const createBucketUploadTask: CreateBucketUploadTask = ( + uploadUrl, + candidate, + headers, + onProgress +) => { + let request: XMLHttpRequest | null = null; + const abortController = new AbortController(); + + const upload = (async () => { + const body = + candidate.file ?? + (candidate.uri + ? await fetch(candidate.uri, { signal: abortController.signal }).then( + (response) => { + if (!response.ok) { + throw new Error(`Could not read file (${response.status})`); + } + return response.blob(); + } + ) + : null); + + if (!body) { + throw new Error('This file does not have an upload source'); + } + + await new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + request = xhr; + xhr.open('PUT', uploadUrl); + Object.entries(headers).forEach(([name, value]) => { + if (name.toLowerCase() === 'content-length') return; + xhr.setRequestHeader(name, value); + }); + xhr.upload.onprogress = (event) => { + if (event.lengthComputable && event.total > 0) { + onProgress(Math.round((event.loaded / event.total) * 100)); + } + }; + xhr.onerror = () => reject(new Error('Object upload failed')); + xhr.onabort = () => reject(new Error('Upload cancelled')); + xhr.onload = () => { + if (xhr.status >= 200 && xhr.status < 300) { + onProgress(100); + resolve(); + } else { + reject(new Error(`Object upload failed (${xhr.status})`)); + } + }; + xhr.send(body); + }); + })(); + + return { + upload, + cancel: async () => { + abortController.abort(); + request?.abort(); + }, + }; +}; diff --git a/packages/app/features/buckets/bucketUploadTask.types.ts b/packages/app/features/buckets/bucketUploadTask.types.ts new file mode 100644 index 0000000000..dd664f6c0d --- /dev/null +++ b/packages/app/features/buckets/bucketUploadTask.types.ts @@ -0,0 +1,13 @@ +import type { BucketUploadCandidate } from '../../ui'; + +export type BucketUploadTask = { + cancel: () => Promise; + upload: Promise; +}; + +export type CreateBucketUploadTask = ( + uploadUrl: string, + candidate: BucketUploadCandidate, + headers: Record, + onProgress: (progress: number) => void +) => BucketUploadTask; diff --git a/packages/app/features/buckets/useLiveBucket.ts b/packages/app/features/buckets/useLiveBucket.ts new file mode 100644 index 0000000000..a0fce18a77 --- /dev/null +++ b/packages/app/features/buckets/useLiveBucket.ts @@ -0,0 +1,724 @@ +import { + BUCKETS_AUTH_FAILURE_STATUSES, + BucketsEntry, + BucketsFileEntry, + BucketsFlag, + BucketsResponse, + BucketsSnapshot, + bucketsFlagKey, + formatBucketsChannelId, + getBucket, + getBucketReadToken, + getCurrentUserId, + requestBucketReadToken, + requestBucketsGrant, + sendBucketsAction, + subscribeToBuckets, +} from '@tloncorp/api'; +import { + BucketsBrokerError, + brokerRequiredHeaders, + cancelBucketUpload, + completeBucketUpload, + deleteBucketObject, + grantBucketRead, + grantBucketUpload, + isBucketObjectAlreadyDeleted, +} from '@tloncorp/shared'; +import * as db from '@tloncorp/shared/db'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; + +import type { BucketItem, BucketUploadCandidate } from '../../ui'; +import { + calculateBucketUploadProgress, + completeBucketUploadInBatch, + removeBucketUploadFromBatch, +} from '../../utils/bucketUploadProgress'; +import type { BucketUploadBatchItem } from '../../utils/bucketUploadProgress'; +import { deletePrivateBucketFiles } from './bucketDeletion'; +import { + bucketResponseHasRevisionGap, + removeEntryFromBucketSnapshot, +} from './bucketUploadReconciliation'; +import { createBucketUploadTask } from './bucketUploadTask'; +import type { BucketUploadTask } from './bucketUploadTask.types'; + +type LocalUpload = { + brokerObjectId?: string; + brokerReservationId?: string; + candidate: BucketUploadCandidate; + error?: string; + id: string; + parentId: number | null; + progress: number; + serverEntryId?: number; + sessionId?: string; + state: 'queued' | 'uploading' | 'failed'; +}; + +function matchesFlag(left: BucketsFlag, right: BucketsFlag) { + return bucketsFlagKey(left) === bucketsFlagKey(right); +} + +function upsertEntry(entries: BucketsEntry[], entry: BucketsEntry) { + const currentIndex = entries.findIndex( + (candidate) => candidate.id === entry.id + ); + if (currentIndex === -1) return [...entries, entry]; + return entries.map((candidate) => + candidate.id === entry.id ? entry : candidate + ); +} + +function reduceBucketResponse( + current: BucketsSnapshot | null, + response: BucketsResponse +): BucketsSnapshot | null { + if (response.type === 'snapshot') { + return { flag: response.flag, state: response.state }; + } + if (!current || response.revision <= current.state.revision) { + return current; + } + + const update = response.update; + if (update.type === 'bucket-deleted') return null; + + let entries = current.state.entries; + let bucket = current.state.bucket; + let writers = current.state.writers; + + switch (update.type) { + case 'bucket-created': + case 'bucket-updated': + bucket = update.bucket; + break; + case 'writers-updated': + writers = update.writers; + break; + case 'entry-created': + case 'entry-updated': + entries = upsertEntry(entries, update.entry); + break; + case 'entries-deleted': + entries = entries.filter((entry) => !update.ids.includes(entry.id)); + break; + } + + return { + ...current, + state: { + ...current.state, + bucket, + entries, + revision: response.revision, + writers, + }, + }; +} + +function errorMessage(cause: unknown) { + return cause instanceof Error ? cause.message : String(cause); +} + +function delay(milliseconds: number) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +export function useLiveBucket(requestedFlag: BucketsFlag) { + const flag = useMemo( + () => ({ host: requestedFlag.host, name: requestedFlag.name }), + [requestedFlag.host, requestedFlag.name] + ); + const flagKey = bucketsFlagKey(flag); + const [snapshot, setSnapshot] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [uploads, setUploads] = useState([]); + const uploadsRef = useRef([]); + const [uploadBatch, setUploadBatch] = useState([]); + const snapshotRef = useRef(null); + const tasksRef = useRef(new Map()); + const cancelledRef = useRef(new Set()); + + const setCurrentUploads = useCallback( + (update: (current: LocalUpload[]) => LocalUpload[]) => { + const next = update(uploadsRef.current); + uploadsRef.current = next; + setUploads(next); + return next; + }, + [] + ); + + // An upload lives in two structures: the row the file list renders, and the + // batch item the aggregate progress bar sums. Retiring it from one without + // the other is invisible until the bar sticks short of 100% for the rest of + // the session, so both moves happen here and nowhere else. + const retireUpload = useCallback( + (id: string, outcome: 'completed' | 'removed') => { + setCurrentUploads((current) => + current.filter((candidate) => candidate.id !== id) + ); + setUploadBatch((current) => + outcome === 'completed' + ? completeBucketUploadInBatch(current, id) + : removeBucketUploadFromBatch(current, id) + ); + }, + [setCurrentUploads] + ); + + // Retire any local row the manifest has caught up with. + // + // Keyed off the snapshot rather than off an entry-created fact, because a + // fact is only one of the ways an entry becomes visible: a revision gap + // refreshes instead, and a replacement snapshot arrives whole. Matching the + // fact alone left the row standing in exactly those cases, and a row holding + // a published serverEntryId hides the real file and makes Retry delete it. + const reconcileUploads = useCallback( + (next: BucketsSnapshot | null) => { + if (!next) return; + const published = new Set(next.state.entries.map((entry) => entry.id)); + uploadsRef.current + .filter( + (upload) => + upload.serverEntryId !== undefined && + published.has(upload.serverEntryId) + ) + .forEach((upload) => retireUpload(upload.id, 'completed')); + }, + [retireUpload] + ); + + const commitSnapshot = useCallback((next: BucketsSnapshot | null) => { + snapshotRef.current = next; + setSnapshot(next); + if (next) { + const channelId = formatBucketsChannelId(next.flag); + // Writers only: %groups owns the channel's reader roles and the + // groups sync already writes them, so mirroring them from here would + // only add a second copy that can go stale. + void db.updateChannel({ + id: channelId, + writerRoles: next.state.writers.map((roleId) => ({ + channelId, + roleId, + })), + }); + } + }, []); + + // Reads one bucket rather than filtering the whole list. /v1/buckets renders + // every bucket's entire manifest, and this runs after every upload, every + // cancel and every missed update -- so uploading twenty files re-read + // everything on the ship twenty times to learn about twenty entries. + const refresh = useCallback(async () => { + const next = await getBucket(flag); + const current = snapshotRef.current; + // Revisions are monotonic only within one Bucket incarnation. Deleting + // and recreating the same flag allocates a new bucket id at revision 0. + if ( + next && + current && + matchesFlag(next.flag, current.flag) && + next.state.bucket.id === current.state.bucket.id && + next.state.revision <= current.state.revision + ) { + return current; + } + if (!next && !current) return null; + commitSnapshot(next); + return next; + }, [commitSnapshot, flag]); + + useEffect(() => { + let active = true; + let stopSubscription: (() => Promise) | undefined; + const start = async () => { + try { + stopSubscription = await subscribeToBuckets((response) => { + if (!active || !matchesFlag(response.flag, flag)) return; + if (bucketResponseHasRevisionGap(snapshotRef.current, response)) { + void refresh() + .then((next) => { + if (active) setError(null); + reconcileUploads(next); + }) + .catch((cause) => { + if (active) setError(errorMessage(cause)); + }); + return; + } + const next = reduceBucketResponse(snapshotRef.current, response); + commitSnapshot(next); + reconcileUploads(next); + // A gap refresh that failed once left its message up for the rest of + // the mount. Taking a response for this Bucket is proof we are + // synchronized again, so the banner should go with it. + setError(null); + setLoading(false); + }); + if (!active) { + await stopSubscription(); + return; + } + const next = await refresh(); + if (active) { + setLoading(false); + setError(next ? null : `Bucket ${flagKey} was not found`); + } + } catch (cause) { + if (active) { + setError(errorMessage(cause)); + setLoading(false); + } + } + }; + + void start(); + return () => { + active = false; + if (stopSubscription) void stopSubscription(); + }; + }, [commitSnapshot, flag, flagKey, reconcileUploads, refresh]); + + const updateLocalUpload = useCallback( + (id: string, patch: Partial) => { + setCurrentUploads((current) => + current.map((upload) => + upload.id === id ? { ...upload, ...patch } : upload + ) + ); + if (patch.progress !== undefined || patch.state !== undefined) { + setUploadBatch((current) => + current.map((item) => + item.id === id + ? { + ...item, + progress: patch.progress ?? item.progress, + state: + patch.state === undefined + ? item.state + : patch.state === 'failed' + ? 'failed' + : 'active', + } + : item + ) + ); + } + }, + [setCurrentUploads] + ); + + const runUpload = useCallback( + async (upload: LocalUpload) => { + const { candidate, id, parentId } = upload; + let sessionId: string | undefined; + let serverEntryId: number | undefined; + let brokerObjectId: string | undefined; + let brokerReservationId: string | undefined; + let brokerCompleted = false; + + try { + if (candidate.size < 0) { + throw new Error('The file size could not be determined'); + } + const mimeType = candidate.mimeType ?? 'application/octet-stream'; + const current = snapshotRef.current ?? (await refresh()); + if (!current) throw new Error(`Bucket ${flagKey} was not found`); + + updateLocalUpload(id, { + error: undefined, + progress: 1, + state: 'uploading', + }); + // The host mints the upload token and answers with it, along with the + // id of the entry it reserved. Nothing has to be matched against the + // replica afterwards, and nothing is broadcast until the object lands. + const grant = await requestBucketsGrant({ + type: 'begin-upload', + checksum: null, + flag, + mime: mimeType, + name: candidate.name, + parentId, + size: candidate.size, + }); + sessionId = grant.token; + serverEntryId = grant.entryId; + updateLocalUpload(id, { progress: 3, serverEntryId, sessionId }); + + const privateGrant = await grantBucketUpload(grant.token, flag.host); + brokerObjectId = privateGrant.objectId; + brokerReservationId = privateGrant.reservationId; + updateLocalUpload(id, { + brokerObjectId, + brokerReservationId, + progress: 5, + }); + + if (cancelledRef.current.has(id)) { + throw new Error('Upload cancelled'); + } + + const task = createBucketUploadTask( + privateGrant.uploadUrl, + candidate, + brokerRequiredHeaders(privateGrant), + (progress) => + updateLocalUpload(id, { + progress: Math.max(5, Math.round(5 + progress * 0.9)), + }) + ); + tasksRef.current.set(id, task); + await task.upload; + tasksRef.current.delete(id); + + if (cancelledRef.current.has(id)) { + throw new Error('Upload cancelled'); + } + if (!sessionId) { + throw new Error('The upload session was lost'); + } + updateLocalUpload(id, { progress: 96 }); + await completeBucketUpload(brokerReservationId); + brokerCompleted = true; + updateLocalUpload(id, { progress: 100 }); + // Nothing further here on purpose. The host broadcasts the published + // entry and reconcileUploads retires this row once it arrives, with + // the revision-gap path covering a fact we miss. + // + // Refreshing was the one thing that could throw after the upload had + // genuinely succeeded, and the catch then marked the row failed while + // it still held serverEntryId -- which arms Retry to delete the real + // manifest entry and orphan its object. Retiring the row here instead + // would leave a moment showing neither the row nor the entry, and if + // the fact never came, the file would simply be missing; leaving the + // row until the manifest has it makes that case a visible stuck + // upload rather than a vanished file. + } catch (cause) { + tasksRef.current.delete(id); + const cancelled = cancelledRef.current.has(id); + if (sessionId && !brokerCompleted) { + await sendBucketsAction({ + type: 'cancel-upload', + flag, + reason: errorMessage(cause), + sessionId, + }).catch(() => undefined); + } + if (brokerReservationId && !brokerCompleted) { + await cancelBucketUpload(brokerReservationId).catch(() => undefined); + } + if (cancelled && serverEntryId !== undefined) { + await sendBucketsAction({ + type: 'delete-entry', + flag, + id: serverEntryId, + recursive: false, + }).catch(() => undefined); + } + if (!cancelled) { + updateLocalUpload(id, { + brokerReservationId: undefined, + error: errorMessage(cause), + progress: 0, + serverEntryId, + state: 'failed', + }); + } + } + }, + [flag, flagKey, refresh, updateLocalUpload] + ); + + const addUploads = useCallback( + (candidates: BucketUploadCandidate[], parentId: number | null) => { + const now = Date.now(); + const nextUploads = candidates.map( + (candidate, index): LocalUpload => ({ + candidate, + id: `local-upload-${now}-${index}`, + parentId, + progress: 0, + state: 'queued', + }) + ); + setCurrentUploads((current) => [...current, ...nextUploads]); + setUploadBatch((current) => [ + ...current, + ...nextUploads.map((upload) => ({ + id: upload.id, + progress: 0, + size: upload.candidate.size, + state: 'active' as const, + })), + ]); + nextUploads.forEach((upload) => void runUpload(upload)); + }, + [runUpload, setCurrentUploads] + ); + + const cancelUpload = useCallback( + async (id: string) => { + const upload = uploads.find((candidate) => candidate.id === id); + const parsedEntryId = Number(id); + const serverEntryId = + upload?.serverEntryId ?? + (Number.isSafeInteger(parsedEntryId) && parsedEntryId >= 0 + ? parsedEntryId + : undefined); + // Sessions are host-private, so the local row is the only place the + // token lives. A cancel with no row has nothing to fail on the host. + const sessionId = upload?.sessionId; + + if (upload) { + cancelledRef.current.add(id); + } + await tasksRef.current + .get(id) + ?.cancel() + .catch(() => undefined); + tasksRef.current.delete(id); + + retireUpload(id, 'removed'); + if (serverEntryId !== undefined && snapshotRef.current) { + commitSnapshot( + removeEntryFromBucketSnapshot(snapshotRef.current, serverEntryId) + ); + } + + if (sessionId) { + await sendBucketsAction({ + type: 'cancel-upload', + flag, + reason: 'Cancelled', + sessionId, + }).catch(() => undefined); + } + if (upload?.brokerReservationId) { + await cancelBucketUpload(upload.brokerReservationId).catch( + () => undefined + ); + } + if (serverEntryId !== undefined) { + await sendBucketsAction({ + type: 'delete-entry', + flag, + id: serverEntryId, + recursive: false, + }).catch(() => undefined); + } + void refresh(); + }, + [commitSnapshot, flag, refresh, retireUpload, uploads] + ); + + const retryUpload = useCallback( + async (id: string) => { + const upload = uploads.find((candidate) => candidate.id === id); + if (!upload) return; + cancelledRef.current.delete(id); + if (upload.serverEntryId !== undefined) { + if (snapshotRef.current) { + commitSnapshot( + removeEntryFromBucketSnapshot( + snapshotRef.current, + upload.serverEntryId + ) + ); + } + await sendBucketsAction({ + type: 'delete-entry', + flag, + id: upload.serverEntryId, + recursive: false, + }).catch(() => undefined); + } + const next = { + ...upload, + error: undefined, + brokerReservationId: undefined, + brokerObjectId: undefined, + progress: 0, + serverEntryId: undefined, + sessionId: undefined, + state: 'queued' as const, + }; + setCurrentUploads((current) => + current.map((candidate) => (candidate.id === id ? next : candidate)) + ); + setUploadBatch((current) => + current.map((item) => + item.id === id + ? { ...item, progress: 0, state: 'active' as const } + : item + ) + ); + void runUpload(next); + }, + [commitSnapshot, flag, runUpload, setCurrentUploads, uploads] + ); + + const localItems = useMemo( + () => + uploads.map((upload) => ({ + author: snapshot?.state.bucket.updatedBy ?? '', + id: upload.id, + kind: 'file', + mimeType: upload.candidate.mimeType, + modifiedLabel: upload.state === 'failed' ? 'Failed' : 'Uploading', + name: upload.candidate.name, + sizeLabel: formatFileSize(upload.candidate.size), + uploadSize: upload.candidate.size, + uploadError: upload.error, + uploadProgress: upload.progress, + uploadState: upload.state, + })), + [snapshot?.state.bucket.updatedBy, uploads] + ); + const uploadAggregateProgress = useMemo( + () => + uploadBatch.length > 0 + ? calculateBucketUploadProgress(uploadBatch) + : undefined, + [uploadBatch] + ); + + return { + addUploads, + cancelUpload, + createFolder: (parentId: number | null, name: string) => + sendBucketsAction({ type: 'create-folder', flag, name, parentId }), + deleteEntry: async (id: number, recursive: boolean) => { + const current = snapshotRef.current; + const root = current?.state.entries.find((entry) => entry.id === id); + const ids = new Set([id]); + if (root?.kind === 'folder' && recursive && current) { + let changed = true; + while (changed) { + changed = false; + current.state.entries.forEach((entry) => { + if (entry.parentId !== null && ids.has(entry.parentId)) { + if (!ids.has(entry.id)) changed = true; + ids.add(entry.id); + } + }); + } + } + const privateFiles = current?.state.entries.filter( + (entry): entry is BucketsFileEntry => + ids.has(entry.id) && + entry.kind === 'file' && + entry.file.status === 'ready' + ); + await deletePrivateBucketFiles(privateFiles ?? [], flag, { + deleteManifestEntry: (deletedId) => + sendBucketsAction({ + type: 'delete-entry', + flag, + id: deletedId, + recursive: false, + }), + deleteObject: deleteBucketObject, + isAlreadyDeleted: isBucketObjectAlreadyDeleted, + issueDelete: async (entryId) => { + const issued = await requestBucketsGrant({ + type: 'issue-delete', + flag, + id: entryId, + }); + return issued.token; + }, + onManifestDelete: (deletedId) => { + const latest = snapshotRef.current; + if (!latest) return; + commitSnapshot({ + ...latest, + state: { + ...latest.state, + entries: latest.state.entries.filter( + (entry) => entry.id !== deletedId + ), + }, + }); + }, + }); + if ( + root?.kind === 'file' && + privateFiles?.some((entry) => entry.id === id) + ) { + return; + } + return sendBucketsAction({ type: 'delete-entry', flag, id, recursive }); + }, + error, + loading, + localItems, + readUrl: async (id: number) => { + const entry = snapshotRef.current?.state.entries.find( + (candidate) => candidate.id === id + ); + if (!entry || entry.kind !== 'file' || entry.file.status !== 'ready') { + throw new Error('This file is not ready to open'); + } + // One token covers the whole bucket, and our own ship keeps it fresh — + // so this is a local read, and only a cold start has to ask for one. + // requestBucketReadToken shares one in-flight mint per bucket across + // callers, so opening several files at once asks for it once. + const held = + (await getBucketReadToken(flag)) ?? + (await requestBucketReadToken(flag)); + // The entry name is the only place the file's name exists by this point: + // the token is bucket-wide and the broker never stored one. + // + // Retried once on a refused token, because the one we just read can stop + // being the one the broker holds between reading it and using it: the + // host rotates on its own timer, and the local scry will hand back a + // token whose replacement has already been pushed. That is a stale read, + // not a permission problem, and the reader should not see it as one. + const openWith = (token: string) => + grantBucketRead(token, flag.host, entry.file.objectKey, entry.name); + try { + return (await openWith(held.token)).readUrl; + } catch (cause) { + if ( + !(cause instanceof BucketsBrokerError) || + !BUCKETS_AUTH_FAILURE_STATUSES.includes(cause.status) + ) { + throw cause; + } + const minted = await requestBucketReadToken(flag); + return (await openWith(minted.token)).readUrl; + } + }, + refresh, + retryUpload, + snapshot, + uploadAggregateProgress, + uploads, + }; +} + +export function formatFileSize(size: number) { + if (size < 0) return 'Unknown size'; + if (size < 1024) return `${size} B`; + if (size < 1024 * 1024) return `${Math.round(size / 1024)} KB`; + if (size < 1024 * 1024 * 1024) { + return `${(size / (1024 * 1024)).toFixed(1)} MB`; + } + return `${(size / (1024 * 1024 * 1024)).toFixed(1)} GB`; +} + +export function formatBucketTimestamp(timestamp: number) { + const milliseconds = + timestamp < 1_000_000_000_000 ? timestamp * 1000 : timestamp; + const elapsed = Math.max(0, Date.now() - milliseconds); + if (elapsed < 60_000) return 'Just now'; + if (elapsed < 3_600_000) return `${Math.floor(elapsed / 60_000)} min ago`; + if (elapsed < 86_400_000) return `${Math.floor(elapsed / 3_600_000)} hr ago`; + if (elapsed < 172_800_000) return 'Yesterday'; + return new Date(milliseconds).toLocaleDateString(); +} diff --git a/packages/app/features/groups/CreateChannelPermissionsScreen.tsx b/packages/app/features/groups/CreateChannelPermissionsScreen.tsx index d891d918f8..399b76fc88 100644 --- a/packages/app/features/groups/CreateChannelPermissionsScreen.tsx +++ b/packages/app/features/groups/CreateChannelPermissionsScreen.tsx @@ -5,8 +5,8 @@ import { NOTES_PERMISSIONS_COMPAT_NOTICE, notesPermissionsCompatActive, } from '@tloncorp/shared/logic/notesPermissionsCompat'; -import { Button, Text } from '@tloncorp/ui'; -import { useCallback, useEffect } from 'react'; +import { Button, Text, useToast } from '@tloncorp/ui'; +import { useCallback, useEffect, useState } from 'react'; import { FormProvider, useForm } from 'react-hook-form'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; import { ScrollView, View, YStack } from 'tamagui'; @@ -30,6 +30,8 @@ export function CreateChannelPermissionsScreen() { RouteProp >(); const insets = useSafeAreaInsets(); + const toast = useToast(); + const [isCreating, setIsCreating] = useState(false); const { groupId, channelTitle, channelType, createdRoleId, selectedRoleIds } = route.params; @@ -87,7 +89,7 @@ export function CreateChannelPermissionsScreen() { }); }, [navigation, groupId, form, channelTitle, channelType]); - const handleCreateChannel = useCallback(() => { + const handleCreateChannel = useCallback(async () => { const { readers: currentReaders, writers: currentWriters, @@ -100,17 +102,28 @@ export function CreateChannelPermissionsScreen() { channelType ); - createChannel({ - groupId, - title: channelTitle, - channelType, - readers: finalReaders, - writers: finalWriters, - }); + try { + setIsCreating(true); + await createChannel({ + groupId, + title: channelTitle, + channelType, + readers: finalReaders, + writers: finalWriters, + }); - // Navigate back to channel list - navigation.navigate('ManageChannels', { groupId }, { pop: true }); - }, [navigation, groupId, form, channelTitle, channelType]); + navigation.navigate('ManageChannels', { groupId }, { pop: true }); + } catch (cause) { + toast({ + message: + cause instanceof Error + ? cause.message + : 'Could not create this channel', + }); + } finally { + setIsCreating(false); + } + }, [navigation, groupId, form, channelTitle, channelType, toast]); if (!group) { return null; @@ -142,6 +155,8 @@ export function CreateChannelPermissionsScreen() { preset="primary" onPress={handleCreateChannel} label="Create channel" + loading={isCreating} + disabled={isCreating} /> diff --git a/packages/app/features/top/ChannelDetailsScreen.tsx b/packages/app/features/top/ChannelDetailsScreen.tsx index 9471425fee..95cf29a682 100644 --- a/packages/app/features/top/ChannelDetailsScreen.tsx +++ b/packages/app/features/top/ChannelDetailsScreen.tsx @@ -71,7 +71,9 @@ export function ChannelDetailsScreenView({ return group ? group.channels?.length === 1 ? `Group with ${group.members?.length ?? 0} members` - : `Channel in ${groupTitle}` + : channel.type === 'buckets' + ? `Bucket in ${groupTitle}` + : `Channel in ${groupTitle}` : ''; } }, [channel, group, groupTitle]); @@ -138,7 +140,7 @@ export function ChannelDetailsScreenView({ paddingHorizontal="$xl" marginVertical="$l" > - {group ? ( + {group && channel.type !== 'buckets' ? ( ) : ( diff --git a/packages/app/features/top/ChannelScreen.tsx b/packages/app/features/top/ChannelScreen.tsx index 927336866f..630cfd2815 100644 --- a/packages/app/features/top/ChannelScreen.tsx +++ b/packages/app/features/top/ChannelScreen.tsx @@ -115,8 +115,14 @@ export default function ChannelScreen(props: Props) { const notesActivityCapabilitiesEpoch = channel?.type === 'notes' ? activityCapabilitiesEpoch : 0; + // Buckets keep no unread state: nothing posts to them and nothing marks + // them read, so each piece of unread work below is inapplicable rather + // than merely unnecessary. Asked once, since it was three spellings of + // the same question and one of them was missed on the first pass. + const channelTracksUnreads = channel?.type !== 'buckets'; + useEffect(() => { - if (channelIsPending) { + if (channelIsPending || !channelTracksUnreads) { return; } @@ -133,7 +139,12 @@ export default function ChannelScreen(props: Props) { }); return () => abortController.abort(); - }, [channelIsPending, channelId, notesActivityCapabilitiesEpoch]); + }, [ + channelTracksUnreads, + channelIsPending, + channelId, + notesActivityCapabilitiesEpoch, + ]); // Snapshot unread state once per focused entry so the divider does not move // as the channel is marked read. @@ -275,9 +286,11 @@ export default function ChannelScreen(props: Props) { loadOlder, isLoading: isLoadingPosts, } = store.useChannelPosts({ - // Capture the unread cursor before loading posts or mounting Channel, - // which can mark the channel read as soon as cached posts are available. - enabled: unreadDidInitialize && !!channel && !channel?.isPendingChannel, + enabled: + unreadDidInitialize && + !!channel && + !channel.isPendingChannel && + channelTracksUnreads, channelId: currentChannelId, count: 30, filterDeleted: !includeDeletedPosts, @@ -451,7 +464,12 @@ export default function ChannelScreen(props: Props) { ); const handleMarkRead = useCallback(async () => { - if (unreadDidInitialize && channel && !channel.isPendingChannel) { + if ( + unreadDidInitialize && + channel && + !channel.isPendingChannel && + channelTracksUnreads + ) { store.markChannelRead({ id: channel.id, groupId: channel.groupId ?? undefined, diff --git a/packages/app/features/top/chatDetails.tsx b/packages/app/features/top/chatDetails.tsx index 6d63fca4c5..0be0597ba5 100644 --- a/packages/app/features/top/chatDetails.tsx +++ b/packages/app/features/top/chatDetails.tsx @@ -33,7 +33,11 @@ import { useIsAdmin, } from '../../ui'; import { ConnectionIndicatorAction } from '../../ui/components/ConnectionStatus'; -import { getChannelActionCapabilities, getChannelHost } from '../../ui/utils'; +import { + channelSupportsNotifications, + getChannelActionCapabilities, + getChannelHost, +} from '../../ui/utils'; import { useShipConnectionStatus } from './useShipConnectionStatus'; // Utility functions @@ -427,6 +431,7 @@ export function SettingsSection({ }, [entityType, group, channel, onPressChatVolume]); const actions = useMemo(() => { + const supportsNotifications = channelSupportsNotifications(channel); const notificationAction: SettingsActionProps = { title: 'Notifications', description: notificationTitle, @@ -437,7 +442,7 @@ export function SettingsSection({ }; if (!currentUserIsAdmin) { - return [notificationAction]; + return supportsNotifications ? [notificationAction] : []; } if (entityType === 'group' && group) { @@ -472,7 +477,7 @@ export function SettingsSection({ const isPrivate = (channel.readerRoles?.length ?? 0) > 0 || (channel.writerRoles?.length ?? 0) > 0; - return [ + const channelActions: SettingsActionProps[] = [ { title: 'Permissions', endValue: isPrivate ? 'Custom' : 'Public', @@ -480,8 +485,9 @@ export function SettingsSection({ disabled: !actionsEnabled, onPress: handlePressEditChannelPrivacy, }, - notificationAction, ]; + if (supportsNotifications) channelActions.push(notificationAction); + return channelActions; } return [notificationAction]; @@ -568,7 +574,9 @@ export function LeaveActionsSection({ const channelActionCapabilities = getChannelActionCapabilities(channel); const canLeave = !isHost && (entityType !== 'channel' || channelActionCapabilities.canLeave); - const canDelete = isHost || (entityType === 'channel' && currentUserIsAdmin); + const canDelete = + (entityType !== 'channel' || channelActionCapabilities.canDelete) && + (isHost || (entityType === 'channel' && currentUserIsAdmin)); const chatTitle = entityType === 'group' diff --git a/packages/app/fixtures/BucketsChannel.fixture.tsx b/packages/app/fixtures/BucketsChannel.fixture.tsx new file mode 100644 index 0000000000..bbc8b3a0e1 --- /dev/null +++ b/packages/app/fixtures/BucketsChannel.fixture.tsx @@ -0,0 +1,667 @@ +import * as db from '@tloncorp/shared/db'; +import * as DocumentPicker from 'expo-document-picker'; +import * as ImagePicker from 'expo-image-picker'; +import { useEffect, useMemo, useState } from 'react'; +import { Linking, useWindowDimensions } from 'react-native'; + +import { imagePickerAssetsToBucketUploadCandidates } from '../features/buckets/bucketMediaPicker'; +import { + BucketFileViewer, + BucketItem, + BucketSearchResult, + BucketUploadCandidate, + BucketsHeaderActions, + BucketsNewSheet, + BucketsPane, + BucketsPaneState, + BucketsSearchScreen, + ChannelHeader, + ChannelHeaderItemsProvider, + ScreenHeader, + XStack, + YStack, + useIsWindowNarrow, + useRegisterChannelHeaderItem, +} from '../ui'; +import { FixtureWrapper } from './FixtureWrapper'; + +const rootFolders: BucketItem[] = [ + folder('launch', 'Launch', 6), + folder('brand', 'Brand', 12), + folder('research', 'Research', 8), + folder('archive', 'Archive', 24), +]; + +const searchRevealFolder = folder('field-notes', 'Field notes', 48); +const searchRevealFiles = Array.from({ length: 48 }, (_, index) => + file( + `field-record-${index}`, + index === 24 ? 'meadow-survey.pdf' : `site-record-${index + 1}.pdf`, + index === 24 ? '~marzod' : '~zod', + index === 24 ? 'Yesterday' : 'Last week', + index === 24 ? '4.2 MB' : '1.8 MB', + 'application/pdf' + ) +); + +const initialFiles: Record = { + root: [], + launch: [ + file( + 'launch-brief', + 'launch-brief.pdf', + '~zod', + '12 min ago', + '2.4 MB', + 'application/pdf', + false, + 'https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf' + ), + file( + 'homepage-final', + 'homepage-final.fig', + '~marzod', + 'Today', + '18.1 MB', + 'application/x-figma' + ), + file( + 'demo-cut', + 'demo-cut.mp4', + 'Scout', + 'Today', + '84.6 MB', + 'video/mp4', + true, + 'https://d23dyxeqlo5psv.cloudfront.net/big_buck_bunny.mp4' + ), + { + ...file( + 'assets-pack', + 'assets-pack.zip', + '~zod', + 'Uploading', + '146 MB', + 'application/zip' + ), + uploadProgress: 42, + uploadState: 'uploading', + }, + ], + brand: [ + file( + 'brand-system', + 'brand-system.pdf', + '~marzod', + 'Yesterday', + '6.8 MB', + 'application/pdf' + ), + file( + 'wordmark', + 'wordmark.svg', + '~zod', + 'Monday', + '38 KB', + 'image/svg+xml', + false, + 'https://d2w9rnfcy7mm78.cloudfront.net/25296321/original_81eb3ac8a95ce36dc8d64b1038234ec8.jpg' + ), + ], + research: [ + file( + 'member-notes', + 'member-interviews.md', + 'Scout', + 'Yesterday', + '24 KB', + 'text/markdown', + true, + 'data:text/plain,', + '# Field interviews\n\nScout collected these notes during the first research pass.\n\n- Members want one shared place for project files.\n- Bots should use the same permissions as people.\n- Large uploads need visible progress and retry.' + ), + ], + archive: [], +}; + +const sampleUploadQueue: BucketItem[] = [ + { + ...file( + 'upload-field-photo', + 'field-photo.jpg', + '~zod', + 'Uploading', + '4.8 MB', + 'image/jpeg' + ), + uploadProgress: 62, + uploadSize: 4.8 * 1024 * 1024, + uploadState: 'uploading', + }, + { + ...file( + 'upload-interviews', + 'interview-recordings.zip', + '~zod', + 'Waiting', + '184 MB', + 'application/zip' + ), + uploadProgress: 0, + uploadSize: 184 * 1024 * 1024, + uploadState: 'queued', + }, + { + ...file( + 'upload-research-notes', + 'research-notes.pdf', + '~zod', + 'Failed', + '8.1 MB', + 'application/pdf' + ), + uploadError: 'Connection lost', + uploadSize: 8.1 * 1024 * 1024, + uploadState: 'failed', + }, +]; + +const fakeChannel = { + id: 'buckets/~zod/project-files', + type: 'buckets', + title: 'Project Files', + description: '', +} as db.Channel; + +type FixtureState = BucketsPaneState | 'read-only'; +type SearchOrigin = { + activeFolderId: string | null; + selectedItemId: string | null; +}; + +function folder(id: string, name: string, itemCount: number): BucketItem { + return { + id, + kind: 'folder', + name, + author: '', + modifiedLabel: '', + itemCount, + }; +} + +function file( + id: string, + name: string, + author: string, + modifiedLabel: string, + sizeLabel: string, + mimeType: string, + isBot = false, + previewUri?: string, + textContent?: string +): BucketItem { + return { + id, + kind: 'file', + name, + author, + isBot, + mimeType, + modifiedLabel, + previewUri, + sizeLabel, + textContent, + }; +} + +function BucketsFixture({ + initialFolderId, + includeSearchRevealFolder = false, + initialSearchQuery = '', + newSheetInitiallyOpen = false, + searchInitiallyOpen = false, + showUploadQueueInitially = false, + state = 'populated', + viewport, +}: { + initialFolderId?: string | null; + includeSearchRevealFolder?: boolean; + initialSearchQuery?: string; + newSheetInitiallyOpen?: boolean; + searchInitiallyOpen?: boolean; + showUploadQueueInitially?: boolean; + state?: FixtureState; + viewport: 'mobile' | 'desktop' | 'responsive'; +}) { + const { height: windowHeight } = useWindowDimensions(); + const folders = useMemo( + () => + includeSearchRevealFolder + ? [...rootFolders, searchRevealFolder] + : rootFolders, + [includeSearchRevealFolder] + ); + const isWindowNarrow = useIsWindowNarrow(); + const isMobileLayout = + viewport === 'mobile' || (viewport === 'responsive' && isWindowNarrow); + const [activeFolderId, setActiveFolderId] = useState( + initialFolderId === undefined + ? isMobileLayout + ? null + : 'launch' + : initialFolderId + ); + const [selectedItemId, setSelectedItemId] = useState( + activeFolderId === 'launch' ? 'launch-brief' : null + ); + const [newSheetOpen, setNewSheetOpen] = useState(newSheetInitiallyOpen); + const [searchOpen, setSearchOpen] = useState(searchInitiallyOpen); + const [searchOrigin, setSearchOrigin] = useState(null); + const [query, setQuery] = useState(initialSearchQuery); + const [previewItem, setPreviewItem] = useState(null); + const [localFiles, setLocalFiles] = useState(() => { + const baseFiles = includeSearchRevealFolder + ? { ...initialFiles, 'field-notes': searchRevealFiles } + : initialFiles; + if (!showUploadQueueInitially) return baseFiles; + + return { + ...baseFiles, + launch: [...sampleUploadQueue, ...(baseFiles.launch ?? [])], + }; + }); + const canEdit = state !== 'read-only'; + const paneState: BucketsPaneState = + state === 'read-only' ? 'populated' : state; + const activeFolder = folders.find((item) => item.id === activeFolderId); + const currentKey = activeFolderId ?? 'root'; + const rootItems = [...folders, ...(localFiles.root ?? [])]; + const visibleItems = activeFolderId + ? (localFiles[currentKey] ?? []) + : rootItems; + const allSearchResults = useMemo(() => { + const results: BucketSearchResult[] = folders.map((item) => ({ + ...item, + parentFolderId: null, + pathLabel: 'Project Files', + })); + + Object.entries(localFiles).forEach(([folderId, items]) => { + const parentFolder = folders.find((item) => item.id === folderId); + const pathLabel = parentFolder + ? `Project Files / ${parentFolder.name}` + : 'Project Files'; + + items.forEach((item) => { + results.push({ + ...item, + parentFolderId: parentFolder?.id ?? null, + pathLabel, + }); + }); + }); + + return results; + }, [folders, localFiles]); + const normalizedQuery = query.trim().toLowerCase(); + const searchResults = normalizedQuery + ? allSearchResults.filter((item) => + [item.name, item.pathLabel, item.author, item.mimeType] + .filter(Boolean) + .join(' ') + .toLowerCase() + .includes(normalizedQuery) + ) + : []; + + useEffect(() => { + if (!isMobileLayout && activeFolderId === null) { + setActiveFolderId('launch'); + } + }, [activeFolderId, isMobileLayout]); + + const openItem = (item: BucketItem) => { + if (item.kind === 'folder') { + setActiveFolderId(item.id); + setSelectedItemId(null); + setQuery(''); + return; + } + setSelectedItemId(item.id); + if (item.previewUri) { + setPreviewItem(item); + } + }; + + const addFolder = (name: string) => { + const item = folder(`new-folder-${Date.now()}`, name, 0); + setLocalFiles((current) => ({ + ...current, + [currentKey]: [item, ...(current[currentKey] ?? [])], + })); + }; + + const addUploads = (candidates: BucketUploadCandidate[]) => { + const now = Date.now(); + const items = candidates.map((candidate, index): BucketItem => { + const uploadState = + candidates.length > 2 && index === candidates.length - 1 + ? 'failed' + : index % 2 === 1 + ? 'queued' + : 'uploading'; + return { + ...file( + `upload-${now}-${index}`, + candidate.name, + '~zod', + uploadState === 'failed' ? 'Failed' : 'Uploading', + formatFileSize(candidate.size), + candidate.mimeType ?? 'application/octet-stream' + ), + uploadError: uploadState === 'failed' ? 'Connection lost' : undefined, + uploadProgress: uploadState === 'failed' ? undefined : index * 22 + 18, + uploadSize: candidate.size, + uploadState, + }; + }); + + setLocalFiles((current) => ({ + ...current, + [currentKey]: [...items, ...(current[currentKey] ?? [])], + })); + }; + + const chooseUploads = async () => { + const result = await DocumentPicker.getDocumentAsync({ + copyToCacheDirectory: true, + multiple: true, + type: '*/*', + }); + if (!result.assets?.length) return; + + addUploads( + result.assets.map((asset) => ({ + mimeType: asset.mimeType ?? undefined, + name: asset.name, + size: asset.size ?? -1, + uri: asset.uri, + })) + ); + }; + + const choosePhotos = async () => { + const result = await ImagePicker.launchImageLibraryAsync({ + allowsEditing: false, + allowsMultipleSelection: true, + exif: false, + mediaTypes: ['images', 'videos'], + orderedSelection: true, + quality: 1, + selectionLimit: 0, + shouldDownloadFromNetwork: true, + }); + if (result.canceled || !result.assets.length) return; + + addUploads(imagePickerAssetsToBucketUploadCandidates(result.assets)); + }; + + const cancelUpload = (item: BucketItem) => { + setLocalFiles((current) => ({ + ...current, + [currentKey]: (current[currentKey] ?? []).filter( + (candidate) => candidate.id !== item.id + ), + })); + }; + + const retryUpload = (item: BucketItem) => { + setLocalFiles((current) => ({ + ...current, + [currentKey]: (current[currentKey] ?? []).map((candidate) => + candidate.id === item.id + ? { + ...candidate, + modifiedLabel: 'Uploading', + uploadError: undefined, + uploadProgress: 8, + uploadState: 'uploading', + } + : candidate + ), + })); + }; + + const openSearch = () => { + setSearchOrigin({ activeFolderId, selectedItemId }); + setSearchOpen(true); + setQuery(''); + }; + + const closeSearch = () => { + if (searchOrigin) { + setActiveFolderId(searchOrigin.activeFolderId); + setSelectedItemId(searchOrigin.selectedItemId); + } + setSearchOrigin(null); + setSearchOpen(false); + setQuery(''); + }; + + const openSearchResult = (result: BucketSearchResult) => { + setSearchOrigin((current) => current ?? { activeFolderId, selectedItemId }); + if (result.kind === 'folder') { + setActiveFolderId(result.id); + setSelectedItemId(null); + } else { + setActiveFolderId(result.parentFolderId); + setSelectedItemId(result.id); + } + setSearchOpen(false); + }; + + const paneProps = { + canEdit, + currentFolder: isMobileLayout ? undefined : activeFolder?.name, + items: paneState === 'empty' ? [] : visibleItems, + selectedItemId, + state: paneState, + onCancelUpload: cancelUpload, + onFilesDropped: addUploads, + onOpenItem: openItem, + onRetryUpload: retryUpload, + onDeleteItem: (item: BucketItem) => + setLocalFiles((current) => ({ + ...current, + [currentKey]: (current[currentKey] ?? []).filter( + (candidate) => candidate.id !== item.id + ), + })), + onDownloadItem: (item: BucketItem) => setSelectedItemId(item.id), + onMoveItem: (item: BucketItem) => setSelectedItemId(item.id), + onRenameItem: (item: BucketItem) => setSelectedItemId(item.id), + }; + + return ( + + + + {previewItem?.previewUri ? ( + setPreviewItem(null)} + onOpenExternally={() => Linking.openURL(previewItem.previewUri!)} + /> + ) : searchOpen ? ( + + ) : isMobileLayout ? ( + + setNewSheetOpen(true)} + onSearch={openSearch} + /> + { + if (searchOrigin) { + setSearchOpen(true); + return; + } + if (activeFolderId) { + setActiveFolderId(null); + setSelectedItemId(null); + } + }} + hideIdentity + preferProvidedTitle + showSearchButton + title={activeFolder?.name ?? 'Project Files'} + /> + + void choosePhotos()} + onNewFolder={addFolder} + onOpenChange={setNewSheetOpen} + onUploadFiles={() => void chooseUploads()} + /> + + ) : ( + + + setSearchOpen(true) : undefined + } + borderBottom + title="Project Files" + /> + + + + setNewSheetOpen(true)} + onSearch={openSearch} + /> + } + showSubtitle + subtitle="Bucket" + title="Project Files" + useHorizontalTitleLayout + /> + + void choosePhotos()} + onNewFolder={addFolder} + onOpenChange={setNewSheetOpen} + onUploadFiles={() => void chooseUploads()} + /> + + + )} + + + + ); +} + +function formatFileSize(size: number) { + if (size < 0) return 'Unknown size'; + if (size < 1024) return `${size} B`; + if (size < 1024 * 1024) return `${Math.round(size / 1024)} KB`; + if (size < 1024 * 1024 * 1024) { + return `${(size / (1024 * 1024)).toFixed(1)} MB`; + } + return `${(size / (1024 * 1024 * 1024)).toFixed(1)} GB`; +} + +function RegisteredHeaderActions({ + canEdit, + onNew, + onSearch, +}: { + canEdit: boolean; + onNew: () => void; + onSearch: () => void; +}) { + const actions = useMemo( + () => ( + + ), + [canEdit, onNew, onSearch] + ); + useRegisterChannelHeaderItem(actions); + return null; +} + +export default { + Mobile: , + 'Mobile folder': ( + + ), + Desktop: , + Search: , + 'Search reveal': ( + + ), + 'New action': , + 'Upload queue': ( + + ), + Empty: , + Loading: , + 'Read only': , +}; diff --git a/packages/app/fixtures/BucketsChannel.live.fixture.tsx b/packages/app/fixtures/BucketsChannel.live.fixture.tsx new file mode 100644 index 0000000000..2f744dc0da --- /dev/null +++ b/packages/app/fixtures/BucketsChannel.live.fixture.tsx @@ -0,0 +1,65 @@ +import { Text } from '@tloncorp/ui'; + +import { BucketsLiveChannel } from '../features/buckets/BucketsLiveChannel'; +import { + DEFAULT_SHIP_LOGIN_ACCESS_CODE, + DEFAULT_SHIP_LOGIN_URL, +} from '../lib/envVars'; +import { YStack } from '../ui'; +import { FixtureUrbitClient, FixtureWrapper } from './FixtureWrapper'; + +function shipFromUrl(shipUrl: string) { + try { + return `~${new URL(shipUrl).hostname.split('.')[0]}`; + } catch { + return ''; + } +} + +const shipName = shipFromUrl(DEFAULT_SHIP_LOGIN_URL); +const urbitClient: FixtureUrbitClient | undefined = + shipName && DEFAULT_SHIP_LOGIN_URL && DEFAULT_SHIP_LOGIN_ACCESS_CODE + ? { + accessCode: DEFAULT_SHIP_LOGIN_ACCESS_CODE, + shipName, + shipUrl: DEFAULT_SHIP_LOGIN_URL, + } + : undefined; + +function LiveHostedBucket() { + if (!urbitClient) { + return ( + + + + Start Cosmos with the hosted ship login environment variables to use + this fixture. + + + + ); + } + + return ( + + + + ); +} + +export default { + 'Live hosted ship': , +}; diff --git a/packages/app/fixtures/FixtureWrapper.tsx b/packages/app/fixtures/FixtureWrapper.tsx index 21b2b1aad4..feca58ba8d 100644 --- a/packages/app/fixtures/FixtureWrapper.tsx +++ b/packages/app/fixtures/FixtureWrapper.tsx @@ -4,10 +4,15 @@ import { NavigationContainer, NavigationIndependentTree, } from '@react-navigation/native'; -import { internalConfigureClient } from '@tloncorp/api'; +import { + configureClient, + internalConfigureClient, + internalRemoveClient, +} from '@tloncorp/api'; import { QueryClientProvider, queryClient } from '@tloncorp/shared'; import { type PropsWithChildren, useEffect, useState } from 'react'; import { useFixtureSelect } from 'react-cosmos/client'; +import { Text as NativeText } from 'react-native'; import { GestureHandlerRootView } from 'react-native-gesture-handler'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; @@ -22,6 +27,13 @@ import { View, } from '../ui'; import { initialContacts } from './fakeData'; +import { createFixtureUrbitFetch } from './fixtureUrbitFetch'; + +export type FixtureUrbitClient = { + shipName: string; + shipUrl: string; + accessCode: string; +}; type FixtureWrapperProps = PropsWithChildren<{ fillWidth?: boolean; @@ -31,17 +43,56 @@ type FixtureWrapperProps = PropsWithChildren<{ backgroundColor?: ColorProp; innerBackgroundColor?: ColorProp; safeArea?: boolean; + currentUserId?: string; + urbitClient?: FixtureUrbitClient; }>; -function MockedUrbitClientProvider({ children }: PropsWithChildren) { +function FixtureUrbitClientProvider({ + children, + urbitClient, +}: PropsWithChildren<{ urbitClient?: FixtureUrbitClient }>) { const [ready, setReady] = useState(false); + const [error, setError] = useState(null); + useEffect(() => { - internalConfigureClient({ - shipName: 'zod', - shipUrl: 'whitehouse.com', - }); - setReady(true); - }, []); + let active = true; + internalRemoveClient(); + setReady(false); + setError(null); + + const setup = async () => { + try { + if (urbitClient) { + await configureClient({ + fetchFn: createFixtureUrbitFetch(urbitClient.shipUrl), + shipName: urbitClient.shipName, + shipUrl: urbitClient.shipUrl, + getCode: async () => urbitClient.accessCode, + }); + } else { + internalConfigureClient({ + shipName: 'zod', + shipUrl: 'whitehouse.com', + }); + } + if (active) setReady(true); + } catch (cause) { + if (active) { + setError(cause instanceof Error ? cause.message : String(cause)); + } + } + }; + + void setup(); + return () => { + active = false; + internalRemoveClient(); + }; + }, [urbitClient]); + + if (error) { + return Could not connect to the ship: {error}; + } return <>{ready ? children : null}; } @@ -51,9 +102,9 @@ export const FixtureWrapper = (props: FixtureWrapperProps) => { - + - + @@ -70,6 +121,7 @@ const InnerWrapper = ({ backgroundColor, innerBackgroundColor, safeArea, + currentUserId = '~zod', children, }: FixtureWrapperProps) => { const insets = useSafeAreaInsets(); @@ -83,7 +135,7 @@ const InnerWrapper = ({ + name.toLowerCase() === 'set-cookie' ? setCookie : target.get(name); + } + const value = Reflect.get(target, property, target); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); +} + +function proxyResponse(response: Response, setCookie: string) { + const headers = proxyHeaders(response.headers, setCookie); + return new Proxy(response, { + get(target, property) { + if (property === 'headers') return headers; + const value = Reflect.get(target, property, target); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); +} + +export function createFixtureUrbitFetch(shipUrl: string): typeof fetch { + const ship = new URL(shipUrl).hostname.split('.')[0]; + const authCookieName = `urbauth-~${ship}`; + + return async (input, init) => { + const headers = new Headers(init?.headers); + headers.delete('Cookie'); + const response = await fetch(input, { + ...init, + credentials: undefined, + headers, + }); + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : input.url; + if (!url.endsWith('/~/login') || !response.ok) return response; + const responseCookie = response.headers.get('set-cookie'); + + const cookies = await CookieManager.get(shipUrl); + if (responseCookie?.startsWith(`${authCookieName}=`)) return response; + const authCookie = cookies[authCookieName]; + if (!authCookie) return response; + + return proxyResponse( + response, + `${authCookie.name}=${authCookie.value}; Path=/; HttpOnly` + ); + }; +} diff --git a/packages/app/fixtures/fixtureUrbitFetch.ts b/packages/app/fixtures/fixtureUrbitFetch.ts new file mode 100644 index 0000000000..83666659ef --- /dev/null +++ b/packages/app/fixtures/fixtureUrbitFetch.ts @@ -0,0 +1,3 @@ +export function createFixtureUrbitFetch(_shipUrl: string): typeof fetch { + return fetch; +} diff --git a/packages/app/navigation/types.ts b/packages/app/navigation/types.ts index 9f653abec4..123095153e 100644 --- a/packages/app/navigation/types.ts +++ b/packages/app/navigation/types.ts @@ -250,7 +250,7 @@ export type RoleSelectionReturn = returnParams: { groupId: string; channelTitle: string; - channelType: 'chat' | 'notebook' | 'gallery' | 'notes'; + channelType: 'chat' | 'notebook' | 'gallery' | 'notes' | 'buckets'; }; } | { @@ -331,7 +331,7 @@ export type GroupSettingsStackParamList = { CreateChannelPermissions: { groupId: string; channelTitle: string; - channelType: 'chat' | 'notebook' | 'gallery' | 'notes'; + channelType: 'chat' | 'notebook' | 'gallery' | 'notes' | 'buckets'; createdRoleId?: string; selectedRoleIds?: string[]; }; diff --git a/packages/app/ui/components/BucketsChannel/BucketFileViewer.native.tsx b/packages/app/ui/components/BucketsChannel/BucketFileViewer.native.tsx new file mode 100644 index 0000000000..84de4650f7 --- /dev/null +++ b/packages/app/ui/components/BucketsChannel/BucketFileViewer.native.tsx @@ -0,0 +1,205 @@ +import { FilePreview, Image, Pressable, Text } from '@tloncorp/ui'; +import { VideoView, useVideoPlayer } from 'expo-video'; +import { Platform } from 'react-native'; +import { WebView } from 'react-native-webview'; +import { ScrollView, Spinner, View, YStack } from 'tamagui'; + +import { useWebView } from '../../../hooks/useWebview'; +import { ScreenHeader } from '../ScreenHeader'; +import { + BucketFileViewerItem, + getBucketPreviewKind, +} from './BucketFileViewer.shared'; + +export function BucketFileViewer({ + error, + item, + loading = false, + onClose, + onOpenExternally, + onRetry, +}: { + error?: string | null; + item: BucketFileViewerItem; + loading?: boolean; + onClose: () => void; + onOpenExternally?: () => void; + onRetry?: () => void; +}) { + const previewKind = getBucketPreviewKind(item); + const webview = useWebView(); + + return ( + + + Open + + ) : null + } + showSubtitle + subtitle={item.sizeLabel ?? 'File'} + title={item.name} + /> + + {loading ? ( + + ) : error ? ( + + ) : !item.uri ? ( + + ) : previewKind === 'image' ? ( + {item.name} + ) : previewKind === 'video' ? ( + + ) : previewKind === 'pdf' && Platform.OS === 'ios' && webview ? ( + + ) : previewKind === 'text' && item.textContent !== undefined ? ( + + + {item.textContent} + + + ) : ( + + )} + + + ); +} + +function LoadingPreview() { + return ( + + + + Loading file… + + + ); +} + +function FailedPreview({ onRetry }: { onRetry?: () => void }) { + return ( + + + + Couldn’t load this file + + + Check your connection and try again. + + + {onRetry ? ( + + + Try again + + + ) : null} + + ); +} + +function NativeVideoPreview({ uri }: { uri: string }) { + const player = useVideoPlayer({ uri }, (videoPlayer) => { + videoPlayer.play(); + }); + + return ( + + + + ); +} + +function UnsupportedPreview({ + isAndroidPdf, + item, + onOpen, +}: { + isAndroidPdf: boolean; + item: BucketFileViewerItem; + onOpen?: () => void; +}) { + return ( + + + + + {isAndroidPdf ? 'Open PDF to view' : 'Preview unavailable'} + + + {isAndroidPdf + ? 'PDFs open in your device viewer.' + : 'Open this file in another app to view it.'} + + + {onOpen ? ( + + + Open file + + + ) : null} + + ); +} diff --git a/packages/app/ui/components/BucketsChannel/BucketFileViewer.shared.test.ts b/packages/app/ui/components/BucketsChannel/BucketFileViewer.shared.test.ts new file mode 100644 index 0000000000..88cd608410 --- /dev/null +++ b/packages/app/ui/components/BucketsChannel/BucketFileViewer.shared.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest'; + +import { + MAX_TEXT_PREVIEW_BYTES, + canPreviewAsText, + getBucketPreviewKind, +} from './BucketFileViewer.shared'; + +describe('getBucketPreviewKind', () => { + it.each([ + ['photo.jpg', 'image/jpeg', 'image'], + ['demo.mp4', 'video/mp4', 'video'], + ['notes.md', undefined, 'text'], + ['report.pdf', undefined, 'pdf'], + ['archive.zip', 'application/zip', 'unsupported'], + ] as const)('classifies %s as %s', (name, mimeType, expected) => { + expect(getBucketPreviewKind({ name, mimeType })).toBe(expected); + }); +}); + +describe('canPreviewAsText', () => { + // A text preview reads the whole object into memory, and the backend accepts + // objects up to 5 GiB, so the size is the gate rather than the type. + it('refuses a text file past the cap', () => { + const item = { name: 'export.csv', mimeType: 'text/csv' }; + expect(canPreviewAsText({ ...item, size: MAX_TEXT_PREVIEW_BYTES })).toBe( + true + ); + expect( + canPreviewAsText({ ...item, size: MAX_TEXT_PREVIEW_BYTES + 1 }) + ).toBe(false); + }); + + // Extension alone makes something text, so a renamed dump reaches this path. + it('gates on size even when only the extension says text', () => { + expect( + canPreviewAsText({ name: 'dump.txt', size: 4 * 1024 * 1024 * 1024 }) + ).toBe(false); + }); + + it('still refuses anything that is not text', () => { + expect(canPreviewAsText({ name: 'clip.mp4', mimeType: 'video/mp4' })).toBe( + false + ); + }); + + // An entry with no size recorded is allowed through rather than blocked. + it('allows a text file whose size is unknown', () => { + expect(canPreviewAsText({ name: 'notes.md' })).toBe(true); + }); +}); diff --git a/packages/app/ui/components/BucketsChannel/BucketFileViewer.shared.ts b/packages/app/ui/components/BucketsChannel/BucketFileViewer.shared.ts new file mode 100644 index 0000000000..44c3526f73 --- /dev/null +++ b/packages/app/ui/components/BucketsChannel/BucketFileViewer.shared.ts @@ -0,0 +1,67 @@ +export type BucketPreviewKind = + | 'image' + | 'video' + | 'text' + | 'pdf' + | 'unsupported'; + +export type BucketFileViewerItem = { + name: string; + mimeType?: string; + size?: number; + sizeLabel?: string; + uri?: string; + textContent?: string; +}; + +/** + * Largest object we will read into memory to preview as text. + * + * A text preview is `response.text()`, so the whole object becomes a JS + * string — at roughly twice its byte size in UTF-16, before rendering it. + * The backend accepts objects up to 5 GiB and a file counts as text on its + * extension alone, so a log or a database dump named `.csv` is an ordinary + * thing to find in a Bucket and an unbounded read of one takes the client + * down. Two megabytes is already tens of thousands of lines, well past what + * anyone reads in a preview pane. + */ +export const MAX_TEXT_PREVIEW_BYTES = 2 * 1024 * 1024; + +/** + * Whether this file can be previewed as text, or is only too large to be. + * + * Refused rather than truncated: a partial JSON or CSV looks like a whole + * one, and a preview that silently lies is worse than one that declines. + */ +export function canPreviewAsText( + item: Pick +): boolean { + if (getBucketPreviewKind(item) !== 'text') return false; + return item.size === undefined || item.size <= MAX_TEXT_PREVIEW_BYTES; +} + +export function getBucketPreviewKind({ + mimeType, + name, +}: Pick): BucketPreviewKind { + const normalizedMimeType = mimeType?.toLowerCase() ?? ''; + const extension = name.split('.').pop()?.toLowerCase(); + + if (normalizedMimeType.startsWith('image/')) { + return 'image'; + } + if (normalizedMimeType.startsWith('video/')) { + return 'video'; + } + if ( + normalizedMimeType.startsWith('text/') || + ['md', 'markdown', 'txt', 'json', 'csv'].includes(extension ?? '') + ) { + return 'text'; + } + if (normalizedMimeType === 'application/pdf' || extension === 'pdf') { + return 'pdf'; + } + + return 'unsupported'; +} diff --git a/packages/app/ui/components/BucketsChannel/BucketFileViewer.tsx b/packages/app/ui/components/BucketsChannel/BucketFileViewer.tsx new file mode 100644 index 0000000000..fc56a8c9ff --- /dev/null +++ b/packages/app/ui/components/BucketsChannel/BucketFileViewer.tsx @@ -0,0 +1,192 @@ +import { FilePreview, Image, Pressable, Text } from '@tloncorp/ui'; +import { ScrollView, Spinner, View, YStack } from 'tamagui'; + +import { ScreenHeader } from '../ScreenHeader'; +import { + BucketFileViewerItem, + getBucketPreviewKind, +} from './BucketFileViewer.shared'; + +export function BucketFileViewer({ + error, + item, + loading = false, + onClose, + onOpenExternally, + onRetry, +}: { + error?: string | null; + item: BucketFileViewerItem; + loading?: boolean; + onClose: () => void; + onOpenExternally?: () => void; + onRetry?: () => void; +}) { + const previewKind = getBucketPreviewKind(item); + + return ( + + + Open + + ) : null + } + showSubtitle + subtitle={item.sizeLabel ?? 'File'} + title={item.name} + useHorizontalTitleLayout + /> + + {loading ? ( + + ) : error ? ( + + ) : !item.uri ? ( + + ) : previewKind === 'image' ? ( + {item.name} + ) : previewKind === 'video' ? ( + + + ) : previewKind === 'pdf' ? ( +