diff --git a/.github/workflows/audit.yaml b/.github/workflows/audit.yaml index bb930adb..dd2db20c 100644 --- a/.github/workflows/audit.yaml +++ b/.github/workflows/audit.yaml @@ -22,7 +22,7 @@ jobs: with: path: ~/.cargo/bin/cargo-audit key: cargo-audit-${{ runner.os }} - - run: cargo install cargo-audit --locked + - run: cargo install cargo-audit --locked --force - run: cargo audit cargo-vet: diff --git a/CHANGELOG.md b/CHANGELOG.md index ae6d3562..86cb0fd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,10 +49,11 @@ All notable changes to this project are documented in this file. ### Breaking Changes -- **api**: `Parts` becomes to `ExecutionContext` and merges Ctx into the task ([#751](https://github.com/apalis-dev/apalis/pull/751)) -- **api**: Changed `Task` to `Task` ([#751](https://github.com/apalis-dev/apalis/pull/751)) +- **api**: `Parts` becomes to `ExecutionContext` and merges Ctx into the task ([#751](https://github.com/apalis-dev/apalis/pull/751)) +- **api**: Changed `Task` to `Task` ([#751](https://github.com/apalis-dev/apalis/pull/751)) - **crates**: Moved backend crates to respective repos ([#586](https://github.com/apalis-dev/apalis/pull/586)) - **api**: `Backend` must be the second input in `WorkerBuilder` ([#586](https://github.com/apalis-dev/apalis/pull/586)) +- **api**: `Backend` now uses rust default poll based mechanism and is standardized: ([#755](https://github.com/apalis-dev/apalis/pull/755)) ```rust let worker = WorkerBuilder::new("tasty-banana") diff --git a/Cargo.lock b/Cargo.lock index 0b1a4a98..9b5c9232 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,7 +8,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "futures-core", "futures-sink", @@ -29,7 +29,7 @@ dependencies = [ "actix-rt", "actix-service", "actix-utils", - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "bytestring", "derive_more", @@ -144,7 +144,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "smallvec", - "socket2 0.6.4", + "socket2 0.6.5", "time", "tracing", "url", @@ -282,7 +282,6 @@ dependencies = [ name = "apalis-core" version = "1.0.0-rc.9" dependencies = [ - "bytes", "document-features", "futures-channel", "futures-core", @@ -338,7 +337,7 @@ dependencies = [ "apalis-file-storage", "futures", "petgraph", - "rand", + "rand 0.9.5", "serde", "serde_json", "thiserror", @@ -531,9 +530,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "block2" @@ -558,12 +557,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" -dependencies = [ - "serde", -] +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "bytestring" @@ -589,9 +585,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.65" +version = "1.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" dependencies = [ "find-msvc-tools", "jobserver", @@ -607,9 +603,20 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" -version = "0.2.1" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core 0.10.1", +] [[package]] name = "chrono" @@ -665,20 +672,29 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "dag-workflow" @@ -707,9 +723,9 @@ dependencies = [ [[package]] name = "defmt" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e524506490a1953d237cb87b1cfc1e46f88c18f10a22dfe0f507dc6bfc7f7f" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" dependencies = [ "bitflags 1.3.2", "defmt-macros", @@ -717,12 +733,11 @@ dependencies = [ [[package]] name = "defmt-macros" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0a27770e9c8f719a79d8b638281f4d828f77d8fd61e0bd94451b9b85e576a0b" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" dependencies = [ "defmt-parser", - "proc-macro-error2", "proc-macro2", "quote", "syn", @@ -739,9 +754,9 @@ dependencies = [ [[package]] name = "der" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71fd89660b2dc699704064e59e9dba0147b903e85319429e131620d022be411b" +checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" dependencies = [ "pem-rfc7468", "zeroize", @@ -785,7 +800,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "objc2", ] @@ -1174,6 +1189,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", ] [[package]] @@ -1296,9 +1312,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http 1.4.2", @@ -1306,9 +1322,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", @@ -1400,7 +1416,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.4", + "socket2 0.6.5", "tokio", "tower-service", "tracing", @@ -1578,9 +1594,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccfe6121cbe750cf81efa362d85c0bde7ea298ec43092d3a193baca59cdbd634" +checksum = "961d16382652bfdd8c6f68b223b26a8c93e0d475c672f414411db31c6c5c900e" dependencies = [ "defmt", "jiff-static", @@ -1592,9 +1608,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2" +checksum = "d0879bd39df99c4c5e2c6615ccc026391a423dde10532c573e6086eb94a802cc" dependencies = [ "proc-macro2", "quote", @@ -1603,11 +1619,11 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.3", "libc", ] @@ -1733,9 +1749,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "memchr" -version = "2.8.2" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "metrics" @@ -1781,7 +1797,7 @@ dependencies = [ "hashbrown 0.16.1", "metrics", "quanta", - "rand", + "rand 0.9.5", "rand_xoshiro", "rapidhash", "sketches-ddsketch", @@ -1804,9 +1820,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "log", @@ -1854,7 +1870,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "cfg_aliases", "libc", @@ -1899,7 +1915,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73ad74d880bb43877038da939b7427bba67e9dd42004a18b809ba7d87cee241c" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "objc2", "objc2-foundation", ] @@ -1920,7 +1936,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "dispatch2", "objc2", ] @@ -1931,7 +1947,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "dispatch2", "objc2", "objc2-core-foundation", @@ -1964,7 +1980,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cde0dfb48d25d2b4862161a4d5fcc0e3c24367869ad306b0c9ec0073bfed92d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "objc2", "objc2-core-foundation", "objc2-core-graphics", @@ -1982,7 +1998,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "block2", "libc", "objc2", @@ -1995,7 +2011,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "objc2", "objc2-core-foundation", ] @@ -2006,7 +2022,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96c1358452b371bf9f104e21ec536d37a650eb10f7ee379fff67d2e08d537f1f" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "objc2", "objc2-core-foundation", "objc2-foundation", @@ -2018,7 +2034,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d87d638e33c06f577498cbcc50491496a3ed4246998a7fbba7ccb98b1e7eab22" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "block2", "objc2", "objc2-cloud-kit", @@ -2070,7 +2086,7 @@ version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "foreign-types", "libc", @@ -2204,7 +2220,7 @@ dependencies = [ "opentelemetry", "percent-encoding", "portable-atomic", - "rand", + "rand 0.9.5", "thiserror", ] @@ -2346,28 +2362,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "proc-macro-error-attr2" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" -dependencies = [ - "proc-macro2", - "quote", -] - -[[package]] -name = "proc-macro-error2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" -dependencies = [ - "proc-macro-error-attr2", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "proc-macro2" version = "1.0.106" @@ -2470,12 +2464,23 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha", - "rand_core", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -2485,7 +2490,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.5", ] [[package]] @@ -2497,20 +2502,26 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_xoshiro" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" dependencies = [ - "rand_core", + "rand_core 0.9.5", ] [[package]] name = "rapidhash" -version = "4.4.2" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b266a82f4aa99bb5c25e28d11cc44ace63d91adbcbcee4d323e2ae3d49ef37" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" dependencies = [ "rustversion", ] @@ -2521,7 +2532,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -2530,14 +2541,14 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] name = "regex" -version = "1.12.4" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -2547,9 +2558,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -2656,9 +2667,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" [[package]] name = "rustc_version" @@ -2675,7 +2686,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys", @@ -2684,9 +2695,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.41" +version = "0.23.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "aws-lc-rs", "once_cell", @@ -2710,9 +2721,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" dependencies = [ "zeroize", ] @@ -2731,9 +2742,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "ryu" @@ -2768,7 +2779,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "core-foundation", "core-foundation-sys", "libc", @@ -2793,9 +2804,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "sentry" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1975064d9f3d9d87a7c8f0371f7fac10d876906ca3e39faad72e61c263ff9b87" +checksum = "d631477761f57c76148456e55e80e9a479ff3fa4c65b2b4a0c3acf1167fd4638" dependencies = [ "cfg_aliases", "httpdate", @@ -2814,9 +2825,9 @@ dependencies = [ [[package]] name = "sentry-actix" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488b43adc03f07b01563ea078c33285c5a9bbbc34d07aaa483d82922db959637" +checksum = "119ede4e37790ec04e8a14073c8414f0a4b2648402856c0563495a9a5cf54d57" dependencies = [ "actix-http", "actix-web", @@ -2827,9 +2838,9 @@ dependencies = [ [[package]] name = "sentry-backtrace" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134f552b6b147f77aeee46ece875d67a8bfc1d56fe3860d78446ed4c25bb5f9f" +checksum = "448b0981fbde6cdc9eb087ba3dc01035a253fef9a0d9e79aaf198fc26acb2e64" dependencies = [ "backtrace", "regex", @@ -2838,9 +2849,9 @@ dependencies = [ [[package]] name = "sentry-contexts" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98482b938fb1f31ecd23506fb7f08b2ba20d60b9cc72581b1205a9acd31d32fa" +checksum = "9e5e909d02170ba6d1dc5ebd05bef7999280f5d960e3eaee5d1a18d88d41b334" dependencies = [ "hostname", "libc", @@ -2852,11 +2863,11 @@ dependencies = [ [[package]] name = "sentry-core" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cff96247f4dc36867511d108709e703eb354143e7893319d763d2dd1edb4f48" +checksum = "ecf0b1a4a4e9ec88395b52e6fa4868b95564c1fa96b26a0606f5f6288a0f7149" dependencies = [ - "rand", + "rand 0.9.5", "sentry-types", "serde", "serde_json", @@ -2865,9 +2876,9 @@ dependencies = [ [[package]] name = "sentry-debug-images" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b603a51b083141062a142d73e36391b14244b4bd0bfe28bcd80e8327bb1d7698" +checksum = "f81d7749c57fc78ed52134889e8121da874065bc40da788d5798cce0f8c19f15" dependencies = [ "findshlibs", "sentry-core", @@ -2893,9 +2904,9 @@ dependencies = [ [[package]] name = "sentry-panic" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61de3f4a1fc77d4c57e8bacd8ef064c26aaa88ea5b2541a761d37c7c3703b9a9" +checksum = "0b7d93d6ecb55d2251c5fc084c55c03a2bc68904918d76117e602c999e92f00f" dependencies = [ "sentry-backtrace", "sentry-core", @@ -2903,9 +2914,9 @@ dependencies = [ [[package]] name = "sentry-tower" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5cb61301e328cbd42d2fede094aca746674b359fcd9c44691f027820e8fe59" +checksum = "a64a8a0a33239c65f95ff719fcb56655f398f4e4e74a01af8daca725cf7c2b5f" dependencies = [ "sentry-core", "tower-layer", @@ -2914,11 +2925,11 @@ dependencies = [ [[package]] name = "sentry-tracing" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc59b27dae3bb495e37e6d62d252214840a2e7e1875531ee9fa2953df8985537" +checksum = "58d26379236c4ef97eaf081bca3c7bf0ef6f06b3aa881eca2ee8e8dbc923e5b8" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "sentry-backtrace", "sentry-core", "tracing-core", @@ -2927,13 +2938,13 @@ dependencies = [ [[package]] name = "sentry-types" -version = "0.48.3" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d7e78132ccfb4a1c777b959fb2944d1f6d5145bffe6be2a98ee6b25d244f72c" +checksum = "2239099d47e76857b0825a182ecdb90159add7aa1097b60247c6e7ca6bf49c6a" dependencies = [ "debugid", "hex", - "rand", + "rand 0.9.5", "serde", "serde_json", "thiserror", @@ -3063,9 +3074,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -3099,9 +3110,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", @@ -3163,18 +3174,18 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" dependencies = [ "cfg-if", ] [[package]] name = "time" -version = "0.3.52" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e48db7b415311b615f910b3dcaa4557bcd4bf1982379c95c223fd8c2a20e210" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" dependencies = [ "deranged", "num-conv", @@ -3212,9 +3223,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee" dependencies = [ "bytes", "libc", @@ -3222,16 +3233,16 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.4", + "socket2 0.6.5", "tokio-macros", "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" dependencies = [ "proc-macro2", "quote", @@ -3297,7 +3308,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "futures-util", "http 1.4.2", @@ -3437,11 +3448,11 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "ulid" -version = "1.2.1" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" +checksum = "39493d572b98fe1d142e132e7a726c50c68121bce84429216a5ec20e1ad7fec5" dependencies = [ - "rand", + "rand 0.10.2", "uuid", "web-time", ] @@ -3574,9 +3585,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.4" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -3919,18 +3930,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.52" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.52" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", @@ -3999,6 +4010,6 @@ dependencies = [ [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/apalis-core/Cargo.toml b/apalis-core/Cargo.toml index 6b69796e..0d62057a 100644 --- a/apalis-core/Cargo.toml +++ b/apalis-core/Cargo.toml @@ -33,7 +33,6 @@ pin-project = "1" thiserror = "2.0.0" futures-timer = { version = "3.0.3", optional = true } tracing = { version = "0.1.41", default-features = false, optional = true } -bytes = { version = "1.12.0", features = ["serde"] } [dependencies.document-features] version = "0.2" diff --git a/apalis-core/README.md b/apalis-core/README.md index f65d8c55..1cba3c54 100644 --- a/apalis-core/README.md +++ b/apalis-core/README.md @@ -60,7 +60,7 @@ It defines task polling mechanisms, streaming interfaces, and middleware integra - `Layer` - Specifies the middleware layer stack for the backend - `Codec` - Determines serialization format for task data persistence - `Beat` - Heartbeat stream for worker liveness checks -- `IdType` - Type used for unique task identifiers +- `Id` - Type used for unique task identifiers - `Conn` - Context associated with tasks - `Error` - Error type for backend operations @@ -194,9 +194,9 @@ pub struct LoggingService { inner: S, } -impl Service> for LoggingService +impl Service> for LoggingService where - S: Service, Response = Res, Error = Err>, + S: Service, Response = Res, Error = Err>, Req: std::fmt::Debug, { type Response = Res; @@ -207,7 +207,7 @@ where self.inner.poll_ready(cx) } - fn call(&mut self, req: Task) -> Self::Future { + fn call(&mut self, req: Task) -> Self::Future { println!("Processing task: {:?}", req.args); self.inner.call(req) } diff --git a/apalis-core/src/backend/codec.rs b/apalis-core/src/backend/codec.rs index 255ba81e..74ab2dff 100644 --- a/apalis-core/src/backend/codec.rs +++ b/apalis-core/src/backend/codec.rs @@ -10,10 +10,6 @@ //! The module includes several implementations of the `Codec` trait, such as `IdentityCodec` //! and `NoopCodec`, as well as a JSON codec when the `json` feature is enabled. -use crate::{ - backend::{Backend, BackendExt}, - worker::context::WorkerContext, -}; /// A trait for converting values between a type `T` and a more compact or /// transport-friendly representation for a `Backend`. Examples include json /// and bytes. @@ -41,14 +37,14 @@ pub trait Codec { /// /// # Errors /// Returns [`Self::Error`] if the value cannot be encoded. - fn encode(val: &T) -> Result; + fn encode(&self, val: &T) -> Result; /// Decode a compact representation back into a value of type `T`. /// /// # Errors /// Returns [`Self::Error`] if the compact representation cannot /// be decoded into a valid `T`. - fn decode(val: &Self::Compact) -> Result; + fn decode(&self, val: &Self::Compact) -> Result; } /// A codec that performs no transformation, returning the input value as-is. @@ -62,52 +58,11 @@ where type Compact = T; type Error = std::convert::Infallible; - fn encode(val: &T) -> Result { + fn encode(&self, val: &T) -> Result { Ok(val.clone()) } - fn decode(val: &Self::Compact) -> Result { + fn decode(&self, val: &Self::Compact) -> Result { Ok(val.clone()) } } - -/// Wrapper that skips decoding and works directly with compact representation. -/// -/// This is useful for backends that natively handle compact types and do not know the types at compile time. -/// Examples include backends that work with raw bytes or JSON values like workflows that manipulate dynamic data. -#[derive(Debug, Clone)] -pub struct RawDataBackend { - inner: B, -} - -impl RawDataBackend { - /// Create a new `RawDataBackend` wrapping the given backend. - pub fn new(backend: B) -> Self { - Self { inner: backend } - } -} - -impl Backend for RawDataBackend -where - B: BackendExt, -{ - type Args = B::Compact; - type IdType = B::IdType; - type Connection = B::Connection; - type Error = B::Error; - type Stream = B::CompactStream; - type Beat = B::Beat; - type Layer = B::Layer; - - fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat { - self.inner.heartbeat(worker) - } - - fn middleware(&self) -> Self::Layer { - self.inner.middleware() - } - - fn poll(self, worker: &WorkerContext) -> Self::Stream { - self.inner.poll_compact(worker) - } -} diff --git a/apalis-core/src/backend/custom.rs b/apalis-core/src/backend/custom.rs index b71c3a6e..ce05b05b 100644 --- a/apalis-core/src/backend/custom.rs +++ b/apalis-core/src/backend/custom.rs @@ -89,7 +89,6 @@ //! - **Custom Sink**: Define how jobs are persisted to your storage. //! - **Configurable**: Pass custom configuration to your backend. //! -use futures_core::stream::BoxStream; use futures_sink::Sink; use futures_util::SinkExt; use futures_util::{Stream, StreamExt}; @@ -100,7 +99,8 @@ use std::{fmt, marker::PhantomData}; use thiserror::Error; use tower_layer::Identity; -use crate::backend::TaskStream; +use crate::backend::codec::IdentityCodec; +use crate::backend::queue::Queue; use crate::error::BoxDynError; use crate::features_table; use crate::{backend::Backend, task::Task, worker::context::WorkerContext}; @@ -141,18 +141,19 @@ type Sinker = Arc Sink + Send }] #[pin_project::pin_project] #[must_use = "Custom backends must be polled or used as a sink"] -pub struct CustomBackend { - _marker: PhantomData<(Args, IdType)>, +pub struct CustomBackend { + _marker: PhantomData<(Args, Id)>, db: DB, fetcher: Fetcher, sinker: Sinker, #[pin] current_sink: Sink, config: Config, + + stream: Option, } -impl Clone - for CustomBackend +impl Clone for CustomBackend where DB: Clone, Config: Clone, @@ -167,12 +168,14 @@ where sinker: Arc::clone(&self.sinker), current_sink, config: self.config.clone(), + + stream: None, } } } -impl fmt::Debug - for CustomBackend +impl fmt::Debug + for CustomBackend where DB: fmt::Debug, Config: fmt::Debug, @@ -184,7 +187,7 @@ where &format_args!( "PhantomData<({}, {})>", std::any::type_name::(), - std::any::type_name::() + std::any::type_name::() ), ) .field("db", &self.db) @@ -204,16 +207,16 @@ type SinkerBuilder = /// Builder for [`CustomBackend`] /// /// Lets you set the database, fetcher, sink, codec, and config -pub struct BackendBuilder { - _marker: PhantomData<(Args, IdType)>, +pub struct BackendBuilder { + _marker: PhantomData<(Args, Id)>, database: Option, fetcher: Option>, sink: Option>, config: Option, } -impl fmt::Debug - for BackendBuilder +impl fmt::Debug + for BackendBuilder where DB: fmt::Debug, Config: fmt::Debug, @@ -225,7 +228,7 @@ where &format_args!( "PhantomData<({}, {})>", std::any::type_name::(), - std::any::type_name::() + std::any::type_name::() ), ) .field("database", &self.database) @@ -236,8 +239,8 @@ where } } -impl Default - for BackendBuilder +impl Default + for BackendBuilder { fn default() -> Self { Self { @@ -250,7 +253,7 @@ impl Default } } -impl BackendBuilder { +impl BackendBuilder { /// Create a new `BackendBuilder` instance #[must_use] pub fn new() -> Self { @@ -260,7 +263,7 @@ impl BackendBuilder( config: Config, - ) -> BackendBuilder { + ) -> BackendBuilder { BackendBuilder { config: Some(config), ..Default::default() @@ -268,7 +271,7 @@ impl BackendBuilder BackendBuilder { +impl BackendBuilder { /// The custom backend persistence engine #[must_use] pub fn database(mut self, db: DB) -> Self { @@ -298,7 +301,7 @@ impl BackendBuilder Result, BuildError> { + pub fn build(self) -> Result, BuildError> { let mut db = self.database.ok_or(BuildError::MissingDb)?; let config = self.config.ok_or(BuildError::MissingConfig)?; let sink_fn = self.sink.ok_or(BuildError::MissingSink)?; @@ -314,6 +317,7 @@ impl BackendBuilder Backend - for CustomBackend +impl Backend + for CustomBackend where - Fetch: Stream>, E>> + Send + 'static, + Fetch: Stream>, E>> + Unpin + Send + 'static, + S: Sink, Error = E> + Unpin + Send + 'static, E: Into, + Id: Clone + Send + Sync + 'static, + Args: Clone, { type Args = Args; - type IdType = IdType; + type Id = Id; type Connection = Conn; type Error = CustomBackendError; - type Stream = TaskStream, CustomBackendError>; + type Layer = Identity; - type Beat = BoxStream<'static, Result<(), Self::Error>>; + type Codec = IdentityCodec; - type Layer = Identity; + type Compact = Args; - fn heartbeat(&self, _: &WorkerContext) -> Self::Beat { - futures_util::stream::once(async { Ok(()) }).boxed() + fn codec(&self) -> &Self::Codec { + &IdentityCodec + } + + fn queue(&self) -> Queue { + Queue::from(std::any::type_name::()) } fn middleware(&self) -> Self::Layer { Identity::new() } - fn poll(mut self, worker: &WorkerContext) -> Self::Stream { - (self.fetcher)(&mut self.db, &self.config, worker) - .map(|task| match task { - Ok(Some(t)) => Ok(Some(t)), - Ok(None) => Ok(None), - Err(e) => Err(e.into().into()), - }) - .boxed() + fn poll_ready( + &mut self, + _: &mut Context<'_>, + _: &WorkerContext, + ) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_next( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll, Self::Error>>> { + if self.stream.is_none() { + self.stream = Some((self.fetcher)(&mut self.db, &self.config, worker)); + } + let stream = self.stream.as_mut().unwrap(); + stream.poll_next_unpin(cx).map(|item| { + item.transpose() + .map(|res| res.flatten()) + .map_err(|e| CustomBackendError::Inner(e.into())) + .transpose() + }) + } + + fn poll_close( + &mut self, + cx: &mut Context<'_>, + _: &WorkerContext, + ) -> Poll> { + self.current_sink + .poll_close_unpin(cx) + .map_err(|e| CustomBackendError::Inner(e.into())) } } -impl Sink> - for CustomBackend +impl Sink> + for CustomBackend where - S: Sink>, + S: Sink>, S::Error: Into, { type Error = CustomBackendError; @@ -396,7 +432,7 @@ where .map_err(|e| CustomBackendError::Inner(e.into())) } - fn start_send(self: Pin<&mut Self>, item: Task) -> Result<(), Self::Error> { + fn start_send(self: Pin<&mut Self>, item: Task) -> Result<(), Self::Error> { self.project() .current_sink .start_send_unpin(item) diff --git a/apalis-core/src/backend/expose.rs b/apalis-core/src/backend/expose.rs index 7f95f11f..52d53a6b 100644 --- a/apalis-core/src/backend/expose.rs +++ b/apalis-core/src/backend/expose.rs @@ -1,7 +1,7 @@ use std::str::FromStr; use crate::{ - backend::{Backend, BackendExt, TaskSink}, + backend::{Backend, TaskSink}, task::{Task, status::Status}, }; @@ -43,18 +43,18 @@ pub trait ListTasks: Backend { fn list_tasks( &self, filter: &Filter, - ) -> impl Future>, Self::Error>> + Send; + ) -> impl Future>, Self::Error>> + Send; } /// Allows listing tasks across all queues with optional filtering -pub trait ListAllTasks: BackendExt { +pub trait ListAllTasks: Backend { /// List tasks matching the given filter in all queues #[allow(clippy::type_complexity)] fn list_all_tasks( &self, filter: &Filter, ) -> impl Future< - Output = Result>, Self::Error>, + Output = Result>, Self::Error>, > + Send; } diff --git a/apalis-core/src/backend/ext/delegate.rs b/apalis-core/src/backend/ext/delegate.rs new file mode 100644 index 00000000..f435b089 --- /dev/null +++ b/apalis-core/src/backend/ext/delegate.rs @@ -0,0 +1,230 @@ +//! A helper macros to delegate `Backend` trait methods from a wrapper type to its inner backend field. + +/// This macro is used to implement the `Sink` trait for a wrapper type that contains a backend field. It delegates the `Sink` methods to the inner backend field, allowing the wrapper type to behave like a `Sink` without having to manually implement each method. +#[macro_export] +macro_rules! delegate_sink { + ($wrapper:ident<$backend:ident $(, $rest:ident)* $(,)?>, $field:ident) => { + impl<$backend, $($rest,)* T, Err> Sink for $wrapper<$backend, $($rest,)*> + where + $backend: Sink + Unpin, + $($rest: Unpin,)* + { + type Error = Err; + + fn start_send(self: Pin<&mut Self>, item: T) -> Result<(), Self::Error> { + self.get_mut().$field.start_send_unpin(item) + } + + fn poll_ready( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + self.get_mut().$field.poll_ready_unpin(cx) + } + + fn poll_flush( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + self.get_mut().$field.poll_flush_unpin(cx) + } + + fn poll_close( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + self.get_mut().$field.poll_close_unpin(cx) + } + } + }; +} +/// Delegates FetchById, Update, Reschedule, Vacuum, ResumeById, +/// ResumeAbandoned, RegisterWorker, and WaitForCompletion to a named field +/// in one call. The caller supplies shared extra bounds, and optionally a +/// `wrap` expression applied to every delegated method's `Result`. +/// +/// `wrap` receives the `Result` returned by the inner call and +/// must produce the wrapper's own `Result` — use it for +/// `.map_err(...)`, logging, retries, or any other postprocessing. +/// +/// ```ignore +/// // plain forwarding +/// delegate_expose!( +/// impl for RawDataBackend +/// where { +/// B: Send + 'static, +/// B::Compact: Send + Clone + 'static, +/// } +/// => inner +/// ); +/// +/// // with postprocessing +/// delegate_expose!( +/// impl for MapErr +/// where { +/// B: Backend, +/// F: Fn(B::Error) -> E2, +/// E2: std::error::Error + Send + Sync + 'static, +/// } +/// => backend, +/// wrap = |this, result| result.map_err(|err| (this.f)(err)) +/// ); +/// ``` +#[macro_export] +macro_rules! delegate_expose { + ( + impl<$($generic:ident),+> for $wrapper:ty + where { $($bound:tt)+ } + => $field:ident + ) => { + $crate::delegate_expose!( + impl<$($generic),+> for $wrapper + where { $($bound)+ } + => $field, + wrap = |this, result| result + ); + }; + ( + impl<$($generic:ident),+> for $wrapper:ty + where { $($bound:tt)+ } + => $field:ident, + wrap = |$this:ident, $result:ident| $wrap_body:expr + ) => { + impl<$($generic),+> FetchById for $wrapper + where + B: FetchById, + B::Compact: Send, + $($bound)+ + { + async fn fetch_by_id( + &mut self, + task_id: &$crate::task::task_id::TaskId, + ) -> Result>, Self::Error> { + let $result = self.$field.fetch_by_id(task_id).await; + #[allow(unused_variables)] + let $this = &*self; + $wrap_body + } + } + + impl<$($generic),+> Update for $wrapper + where + B: Update + Send, + B::Compact: Send, + $($bound)+ + { + async fn update( + &mut self, + task: Task, + ) -> Result<(), Self::Error> { + let $result = self.$field.update(task).await; + #[allow(unused_variables)] + let $this = &*self; + $wrap_body + } + } + + impl<$($generic),+> Reschedule for $wrapper + where + B: Reschedule, + B::Compact: Send, + $($bound)+ + { + async fn reschedule( + &mut self, + task: Task, + wait: std::time::Duration, + ) -> Result<(), Self::Error> { + let $result = self.$field.reschedule(task, wait).await; + #[allow(unused_variables)] + let $this = &*self; + $wrap_body + } + } + + impl<$($generic),+> Vacuum for $wrapper + where + B: Vacuum, + $($bound)+ + { + async fn vacuum(&mut self) -> Result { + let $result = self.$field.vacuum().await; + #[allow(unused_variables)] + let $this = &*self; + $wrap_body + } + } + + impl<$($generic),+> ResumeById for $wrapper + where + B: ResumeById, + $($bound)+ + { + async fn resume_by_id( + &mut self, + id: TaskId, + ) -> Result { + let $result = self.$field.resume_by_id(id).await; + #[allow(unused_variables)] + let $this = &*self; + $wrap_body + } + } + + impl<$($generic),+> ResumeAbandoned for $wrapper + where + B: ResumeAbandoned, + $($bound)+ + { + async fn resume_abandoned(&mut self) -> Result { + let $result = self.$field.resume_abandoned().await; + #[allow(unused_variables)] + let $this = &*self; + $wrap_body + } + } + + impl<$($generic),+> RegisterWorker for $wrapper + where + B: RegisterWorker, + $($bound)+ + { + async fn register_worker(&mut self, worker_id: String) -> Result<(), Self::Error> { + let $result = self.$field.register_worker(worker_id).await; + #[allow(unused_variables)] + let $this = &*self; + $wrap_body + } + } + + impl<$($generic),+> WaitForCompletion for $wrapper + where + B: WaitForCompletion + Sync, + B::Compact: 'static, + $($bound)+ + { + type ResultStream = futures_core::stream::BoxStream<'static, Result, Self::Error>>; + + fn wait_for( + &self, + task_ids: impl IntoIterator>, + ) -> Self::ResultStream { + use futures_util::StreamExt; + let $result = self.$field.wait_for(task_ids); + #[allow(unused_variables)] + let $this = &*self; + $wrap_body.boxed() + } + + async fn check_status( + &self, + task_ids: impl IntoIterator> + Send, + ) -> Result>, Self::Error> { + let $result = self.$field.check_status(task_ids).await; + #[allow(unused_variables)] + let $this = &*self; + $wrap_body + } + } + }; +} diff --git a/apalis-core/src/backend/ext/inspect_err.rs b/apalis-core/src/backend/ext/inspect_err.rs new file mode 100644 index 00000000..9596b575 --- /dev/null +++ b/apalis-core/src/backend/ext/inspect_err.rs @@ -0,0 +1,95 @@ +use std::{ + pin::Pin, + task::{Context, Poll}, +}; + +use futures_sink::Sink; +use futures_util::SinkExt; +use futures_util::TryStreamExt; + +use crate::{ + backend::{queue::Queue, *}, + task::Task, + worker::context::WorkerContext, +}; + +/// A `Backend` wrapper that runs a callback `F` on each error yielded by the poll stream. +#[derive(Debug, Clone)] +pub struct InspectErr { + pub(super) backend: B, + pub(super) f: F, +} + +impl Backend for InspectErr +where + B: Backend, + F: FnMut(&B::Error), +{ + type Args = B::Args; + type Id = B::Id; + type Connection = B::Connection; + type Error = B::Error; + type Codec = B::Codec; + type Compact = B::Compact; + type Layer = B::Layer; + + fn codec(&self) -> &Self::Codec { + self.backend.codec() + } + + fn queue(&self) -> Queue { + self.backend.queue() + } + + fn middleware(&self) -> Self::Layer { + self.backend.middleware() + } + + fn poll_ready( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll> { + self.backend.poll_ready(cx, worker).map_err(|err| { + (self.f)(&err); + err + }) + } + + fn poll_next( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll, Self::Error>>> { + self.backend.poll_next(cx, worker).map(|opt| { + opt.map(|res| { + res.inspect_err(|err| { + (self.f)(err); + }) + }) + }) + } + + fn poll_close( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll> { + self.backend.poll_close(cx, worker).map_err(|err| { + (self.f)(&err); + err + }) + } +} + +delegate_sink!(InspectErr, backend); + +delegate_expose!( + impl for InspectErr + where { + B: Backend + Send, + F: FnMut(&B::Error) + Send + Sync + Clone + 'static, + } + => backend, + wrap = |this, result| { let mut f = this.f.clone(); result.inspect_err(move |err| {(f)(err)}) } +); diff --git a/apalis-core/src/backend/ext/map_err.rs b/apalis-core/src/backend/ext/map_err.rs new file mode 100644 index 00000000..073abd99 --- /dev/null +++ b/apalis-core/src/backend/ext/map_err.rs @@ -0,0 +1,95 @@ +use std::{ + pin::Pin, + task::{Context, Poll}, +}; + +use futures_sink::Sink; +use futures_util::SinkExt; +use futures_util::TryStreamExt; + +use crate::{ + backend::{queue::Queue, *}, + task::Task, + worker::context::WorkerContext, +}; + +/// A `Backend` wrapper that maps the backend's error type `Self::Error` into `E2`. +#[derive(Debug, Clone)] +pub struct MapErr { + pub(super) backend: B, + pub(super) f: F, +} + +impl Backend for MapErr +where + B: Backend, + F: Fn(B::Error) -> E2, + E2: std::error::Error + Send + Sync + 'static, +{ + type Args = B::Args; + type Id = B::Id; + type Connection = B::Connection; + type Error = E2; + type Codec = B::Codec; + type Compact = B::Compact; + type Layer = B::Layer; + + fn codec(&self) -> &Self::Codec { + self.backend.codec() + } + + fn queue(&self) -> Queue { + self.backend.queue() + } + + fn middleware(&self) -> Self::Layer { + self.backend.middleware() + } + + fn poll_ready( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll> { + self.backend + .poll_ready(cx, worker) + .map_err(|err| (self.f)(err)) + } + + fn poll_next( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll, Self::Error>>> { + self.backend + .poll_next(cx, worker) + .map(|opt| opt.map(|res| res.map_err(|err| (self.f)(err)))) + } + + fn poll_close( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll> { + self.backend + .poll_close(cx, worker) + .map_err(|err| (self.f)(err)) + } +} + +delegate_sink!(MapErr, backend); + +delegate_expose!( + impl for MapErr + where { + B: Backend + Send, + F: Fn(B::Error) -> E2 + Send + Sync + Clone + 'static, + E2: std::error::Error + Send + Sync + 'static, + } + => backend, + wrap = |this, result| { + let f = this.f.clone(); + #[allow(clippy::manual_inspect, clippy::redundant_closure)] + result.map_err(move |err| (f)(err)) + } +); diff --git a/apalis-core/src/backend/ext/mod.rs b/apalis-core/src/backend/ext/mod.rs new file mode 100644 index 00000000..0a25ac3f --- /dev/null +++ b/apalis-core/src/backend/ext/mod.rs @@ -0,0 +1,143 @@ +//! Extension traits and combinators for `Backend` implementations. +//! +//! This module provides additional functionality for `Backend` implementations, including: +//! - [`BackendExt`]: An extension trait that adds combinators for transforming and composing backends. +//! - [`InspectErr`]: A wrapper that allows inspection of errors produced by a backend +//! - [`MapErr`]: A wrapper that allows mapping errors from one type to another +//! - [`Pipe`]: A utility for piping tasks from one backend to another +//! - [`RawDataBackend`]: A wrapper that allows a backend to be used as a stream of tasks without decoding +use std::task::{Context, Poll}; + +use crate::{ + backend::{ + Backend, + codec::Codec, + ext::{ + inspect_err::InspectErr, + map_err::MapErr, + pipe::{FromBackend, Pipe}, + raw::RawDataBackend, + with_codec::WithCodec, + with_poll_strategy::WithPollStrategy, + }, + }, + task::Task, + worker::context::WorkerContext, +}; + +#[macro_use] +pub mod delegate; +/// A wrapper that allows inspecting errors produced by a backend. +pub mod inspect_err; +/// A wrapper that allows mapping the error type of a backend from `Self::Error` to another error type `E2`. +pub mod map_err; +pub mod pipe; +pub mod raw; +pub mod with_codec; + +pub mod with_poll_strategy; + +/// A wrapper that allows a backend to be used as a stream of tasks, without needing to know the concrete backend type at compile time. +#[derive(Debug, thiserror::Error)] +pub enum PollNextArgsError +where + >::Error: std::error::Error, +{ + /// The backend produced an error while polling for the next task. + #[error("backend error: {0}")] + BackendError(B::Error), + /// The backend produced a task, but the task's arguments could not be decoded. + #[error("failed to decode task args: {0}")] + DecodeError(>::Error), +} + +/// Extension trait for `Backend` that provides additional combinators and utilities. +pub trait BackendExt: Backend { + /// Skips decoding entirely: `poll` yields `Task` + /// directly from `poll_compact`, and `Args` becomes `Self::Compact`. + fn raw(self) -> RawDataBackend + where + Self: Sized, + { + RawDataBackend::new(self) + } + + /// A convenience method for calling `poll_next` and decoding the `Args` in one step, + /// returning a `Task` instead of `Task`. + fn poll_next_args( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll, PollNextArgsError>>> + where + Self: Sized, + >::Error: std::error::Error, + { + let next = self.poll_next(cx, worker); + let codec = self.codec(); + next.map(move |item| match item { + Some(Ok(task)) => { + let task = task.try_map_args(|compact| codec.decode(&compact)); + Some(task.map_err(|e| PollNextArgsError::DecodeError(e))) + } + Some(Err(e)) => Some(Err(PollNextArgsError::BackendError(e))), + None => None, + }) + } + + /// Pipes every task polled from this backend into `sink`, then serves + /// from `sink` going forward. Wraps `self` in [`FromBackend`] + /// automatically so it can be piped through the same [`Pipe`] used for + /// plain streams. + /// + /// Useful for bridging two backend implementations — e.g. draining an + /// ephemeral/legacy queue into a durable one, or fanning a lightweight + /// source into a shared sink that multiple producers write into. + fn pipe_to(self, backend: Dst) -> Pipe, Self::Args, Dst::Connection> + where + Self: Sized, + Dst: Backend, + { + Pipe::new(FromBackend(self), backend) + } + + /// Attaches a callback `F` to be run on each error produced while polling the backend. + fn inspect_err(self, f: F) -> InspectErr + where + Self: Sized, + F: Fn(&Self::Error), + { + InspectErr { backend: self, f } + } + + /// Maps errors produced by the backend from `Self::Error` into `E2`, useful for + /// heterogeneous composed backends. + fn map_err(self, f: F) -> MapErr + where + Self: Sized, + F: Fn(Self::Error) -> E2, + { + MapErr { backend: self, f } + } + + /// Swaps out the backend's serialization codec entirely (JSON, + /// MessagePack, Protobuf, ...) without touching storage logic. + fn with_codec(self, codec: NewCodec) -> WithCodec + where + Self: Sized, + NewCodec: Codec, + { + WithCodec::new(self, codec) + } + + /// Swaps out the backend's polling strategy entirely (e.g. for rate limiting, backoff, etc.) + /// without touching storage logic. + fn with_poll_strategy(self, strategy: S) -> WithPollStrategy + where + Self: Sized, + { + WithPollStrategy::new(self, strategy) + } +} + +impl BackendExt for B {} diff --git a/apalis-core/src/backend/ext/pipe.rs b/apalis-core/src/backend/ext/pipe.rs new file mode 100644 index 00000000..a8f837e8 --- /dev/null +++ b/apalis-core/src/backend/ext/pipe.rs @@ -0,0 +1,480 @@ +//! # Pipe streams to backends +//! +//! This backend allows you to pipe tasks from any stream into another backend. +//! It is useful for connecting different backends together, such as piping tasks +//! from a cron stream into a database backend, or transforming and forwarding tasks +//! between systems. +//! +//! ## Example +//! +//! ```rust +//! # use futures_util::stream; +//! # use apalis_core::backend::{ext::pipe::PipeExt, dequeue::VecDequeBackend}; +//! # use apalis_core::worker::{builder::WorkerBuilder, context::WorkerContext}; +//! # use apalis_core::error::BoxDynError; +//! # use std::time::Duration; +//! # use futures_util::StreamExt; +//! # use crate::apalis_core::worker::ext::event_listener::EventListenerExt; +//! #[tokio::main] +//! async fn main() { +//! let stm = stream::iter(0..10).map(|s| Ok::<_, std::io::Error>(s)); +//! +//! let in_memory = VecDequeBackend::new(); +//! let backend = stm.pipe_to(in_memory); +//! +//! async fn task(task: u32, ctx: WorkerContext) -> Result<(), BoxDynError> { +//! tokio::time::sleep(Duration::from_secs(1)).await; +//! # if task == 9 { +//! # ctx.stop().unwrap(); +//! # } +//! Ok(()) +//! } +//! +//! let worker = WorkerBuilder::new("rango-tango") +//! .backend(backend) +//! .on_event(|_ctx, ev| { +//! println!("On Event = {:?}", ev); +//! }) +//! .build(task); +//! worker.run().await.unwrap(); +//! } +//! ``` +//! +//! This example pipes a stream of numbers into an in-memory backend and processes them with a worker. +//! +//! See also: +//! - [`apalis-cron`](https://docs.rs/apalis-cron) +use std::fmt::Debug; +use std::fmt::{self, Display}; +use std::marker::PhantomData; +use std::ops::{Deref, DerefMut}; +use std::{ + pin::Pin, + task::{Context, Poll}, +}; + +use crate::backend::ext::{BackendExt, PollNextArgsError}; +use crate::backend::queue::Queue; +use crate::backend::*; +use crate::error::BoxDynError; +use crate::features_table; +use crate::task::Task; +use crate::task::builder::TaskBuilder; +use crate::{backend::codec::Codec, worker::context::WorkerContext}; +use futures_core::{Stream, ready}; +use futures_sink::Sink; +use futures_util::SinkExt; +use futures_util::TryStreamExt; + +/// A generic pipe that wraps a [`Stream`] and passes it to a backend +#[doc = features_table! { + setup = "{ unreachable!() }", + TaskSink => supported("Ability to push new tasks", false), + InheritsFeatures => limited("Inherits features from the underlying backend", false), +}] +pub struct Pipe { + pub(crate) from: S, + pub(crate) into: Dst, + pub(crate) _req: PhantomData<(Args, Conn)>, +} + +/// Adapts a [`Backend`] into an [`IntoArgsStream`] source, so it can be +/// used wherever a plain `Stream>` would be — +/// specifically, as the `from` side of a [`Pipe`]. +/// +/// This exists because a blanket `IntoArgsStream` impl can't be written for +/// both "any stream of `Result`" and "any `Backend`" at once — +/// nothing stops a type from satisfying both bounds simultaneously, so the +/// two blanket impls would conflict under coherence. Wrapping the backend +/// in this newtype sidesteps that: `FromBackend` is a distinct type from +/// `B`, so it only ever matches the backend-flavored impl. +pub struct FromBackend(pub B); + +impl Clone for FromBackend { + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + +impl fmt::Debug for FromBackend { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("FromBackend").field(&self.0).finish() + } +} + +/// Something that can be converted into a stream of `Result`, +/// suitable for piping into a sink backend via [`Pipe`]. +/// +/// Implemented for: +/// - any `Stream>` directly (the "plain stream" +/// case), and +/// - [`FromBackend`] for any `Backend` (the "backend as a +/// source" case), which polls the backend and unwraps each task down to +/// its args. +/// +/// [`Pipe`] is generic over this trait rather than over `Stream` directly, +/// which is what lets one `Pipe` type serve both streams and backends. +pub trait PipeNextStream { + /// The error type yielded alongside `Args` on failure. + type Error; + + /// Polls for the next task from the underlying source. + /// Takes a [`WorkerContext`] since some sources need it to poll + /// (e.g. backend-backed sources). + fn poll_pipe_next( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll, Self::Error>>>; +} + +// Plain-stream case: any stream of `Result` is already exactly +// what we need, so this is just a passthrough. +impl PipeNextStream for S +where + S: Stream> + Send + Unpin + 'static, + Args: 'static, + Conn: 'static, + Id: 'static, + Err: 'static, +{ + type Error = Err; + + fn poll_pipe_next( + &mut self, + cx: &mut Context<'_>, + _: &WorkerContext, + ) -> Poll, Self::Error>>> { + let next = Stream::poll_next(Pin::new(self), cx); + next.map(|item| item.map(|res| res.map(TaskBuilder::new).map(|t| t.build()))) + } +} + +// Backend case: poll the backend, drop `None`s (no task available right +// now), and project each `Task` down to its `args`. +impl PipeNextStream for FromBackend +where + B: Backend + Send + 'static, + B::Error: std::error::Error + Send + Sync + 'static, + B::Args: Send + 'static, + B::Id: Display, + Id: 'static, + Conn: 'static, + >::Error: std::error::Error + Send + Sync + 'static, +{ + type Error = PollNextArgsError; + + fn poll_pipe_next( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll, Self::Error>>> { + let next = self.0.poll_next_args(cx, worker); + next.map(|item| match item { + Some(Ok(task)) => Some(Ok(task.map_backend::())), + Some(Err(e)) => Some(Err(e)), + None => None, + }) + } +} + +impl Deref for Pipe { + type Target = Dst; + + fn deref(&self) -> &Self::Target { + &self.into + } +} + +impl DerefMut for Pipe { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.into + } +} + +impl Pipe { + /// Create a new `Pipe` from a raw `from` source and an `into` sink. + /// Prefer [`PipeExt::pipe_to`] or [`BackendExt::pipe_to`] over calling + /// this directly. + pub fn new(from: S, into: Dst) -> Self { + Self { + from, + into, + _req: PhantomData, + } + } +} + +impl fmt::Debug for Pipe { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Pipe") + .field("from", &self.from) + .field("into", &self.into) + .finish() + } +} + +impl Backend for Pipe +where + S: PipeNextStream + Send + 'static, + S::Error: std::error::Error + Send + Sync + 'static, + TSink: Backend + + TaskSink + + Unpin + + Send + + 'static + + Sink>, + ::Error: std::error::Error + Send + Sync + 'static, + TSink::Id: Display + Send + Sync + 'static, + TSink::Codec: Codec + Send + Sync + 'static, + >>::Error: + std::error::Error + Send + Sync + 'static, + Args: Send + 'static, + CdcErr: std::error::Error + Send + Sync + 'static, + TSink::Compact: Send, + Conn: 'static, +{ + type Args = Args; + type Id = TSink::Id; + type Connection = Conn; + type Layer = TSink::Layer; + type Error = PipeError; + type Codec = TSink::Codec; + type Compact = TSink::Compact; + + fn codec(&self) -> &Self::Codec { + self.into.codec() + } + + fn queue(&self) -> Queue { + self.into.queue() + } + + fn middleware(&self) -> Self::Layer { + self.into.middleware() + } + + fn poll_ready( + &mut self, + cx: &mut Context<'_>, + _: &WorkerContext, + ) -> Poll> { + ready!(Sink::poll_ready(Pin::new(&mut self.into), cx)) + .map_err(|e| PipeError::Inner(e.into()))?; + + Poll::Ready(Ok(())) + } + + fn poll_next( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll, Self::Error>>> { + let mut source_done = false; + + loop { + match Sink::poll_ready(Pin::new(&mut self.into), cx) { + Poll::Ready(Ok(())) => {} + Poll::Ready(Err(e)) => { + return Poll::Ready(Some(Err(PipeError::Inner(e.into())))); + } + Poll::Pending => break, + } + + match self.from.poll_pipe_next(cx, worker) { + Poll::Ready(Some(Ok(task))) => { + let codec = self.into.codec(); + let encoded = task.try_map_args(|args| { + codec.encode(&args).map_err(|e| PipeError::Inner(e.into())) + }); + + match encoded { + Ok(t) => { + if let Err(e) = Sink::start_send(Pin::new(&mut self.into), t) { + return Poll::Ready(Some(Err(PipeError::Inner(e.into())))); + } + } + Err(e) => return Poll::Ready(Some(Err(e))), + } + } + Poll::Ready(Some(Err(e))) => { + return Poll::Ready(Some(Err(PipeError::Inner(e.into())))); + } + Poll::Ready(None) => { + source_done = true; + break; + } + Poll::Pending => break, + } + } + + match Sink::poll_flush(Pin::new(&mut self.into), cx) { + Poll::Ready(Err(e)) => return Poll::Ready(Some(Err(PipeError::Inner(e.into())))), + Poll::Ready(Ok(())) | Poll::Pending => {} + } + + if source_done { + let _ = Sink::poll_close(Pin::new(&mut self.into), cx); + } + + if let Poll::Ready(Err(e)) = self.into.poll_ready(cx, worker) { + return Poll::Ready(Some(Err(PipeError::Inner(e.into())))); + } + + self.into + .poll_next(cx, worker) + .map_ok(|task| task.map_backend()) + .map_err(|e| PipeError::Inner(e.into())) + } + + fn poll_close( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll> { + self.into + .poll_close(cx, worker) + .map_err(|e| PipeError::Inner(e.into())) + } +} + +/// Utility for piping a plain stream of `Result` into a backend. +pub trait PipeExt +where + B: Backend, + Self: PipeNextStream + Sized, +{ + /// Pipe the current stream into the provided sink backend. + fn pipe_to(self, backend: B) -> Pipe; +} + +impl PipeExt for S +where + S: Stream> + Send + Unpin + 'static, + B::Error: Into + Send + Sync + 'static, + B: Backend + TaskSink, + Err: 'static, + Args: 'static, + Conn: 'static, +{ + fn pipe_to(self, backend: B) -> Pipe { + Pipe::new(self, backend) + } +} + +/// Error encountered while piping streams +#[derive(Debug, thiserror::Error)] +pub enum PipeError { + /// The cron stream provided a None + #[error("The inner stream provided a None")] + EmptyStream, + /// An inner stream error occurred + #[error("The inner stream error: {0}")] + Inner(BoxDynError), +} + +delegate_sink!(Pipe, into); + +delegate_expose!( + impl for Pipe + where { + B: Send + Sync, + S: PipeNextStream + Send + Sync + 'static, + S::Error: std::error::Error + Send + Sync + 'static, + B: Backend + + TaskSink + + Unpin + + Send + + 'static + + Sink>, + ::Error: std::error::Error + Send + Sync + 'static, + B::Id: Display + Send + Sync + 'static, + B::Codec: Codec + Send + Sync + 'static, + >>::Error: + std::error::Error + Send + Sync + 'static, + Args: Send + Sync + 'static, + CdcErr: std::error::Error + Send + Sync + 'static, + B::Compact: Send, + Conn: Send + Sync + 'static, + } + => into, + wrap = |this, result| result.map_err(|e| PipeError::Inner(e.into())) +); + +#[cfg(test)] +mod tests { + use std::{io, time::Duration}; + + use futures_util::{StreamExt, stream}; + + use crate::{ + backend::{ + dequeue::{self, VecDequeBackend}, + ext::BackendExt, + memory::MemoryStorage, + }, + error::BoxDynError, + worker::{ + builder::WorkerBuilder, context::WorkerContext, ext::event_listener::EventListenerExt, + }, + }; + + use super::*; + + const ITEMS: u32 = 10; + + #[tokio::test] + async fn basic_worker() { + let stm = stream::iter(0..ITEMS).map(Ok::<_, io::Error>); + let in_memory = dequeue::VecDequeBackend::new(); + + let backend = stm.pipe_to(in_memory); + + async fn task(task: u32, ctx: WorkerContext) -> Result<(), BoxDynError> { + tokio::time::sleep(Duration::from_secs(1)).await; + if task == ITEMS - 1 { + ctx.stop().unwrap(); + return Err("Graceful Exit".into()); + } + Ok(()) + } + + let worker = WorkerBuilder::new("rango-tango") + .backend(backend) + .on_event(|_ctx, ev| { + println!("On Event = {ev:?}"); + }) + .build(task); + worker.run().await.unwrap(); + } + + #[tokio::test] + async fn dequeue_to_memory_worker() { + let dequeue = VecDequeBackend::new(); + + let mut in_memory = MemoryStorage::new(); + + in_memory.push(42).await.unwrap(); + + let mut backend = in_memory.pipe_to(dequeue); + + backend.push(43).await.unwrap(); + + async fn task(task: u32, ctx: WorkerContext) -> Result<(), BoxDynError> { + tokio::time::sleep(Duration::from_secs(1)).await; + if task == 43 { + ctx.stop().unwrap(); + return Err("Graceful Exit".into()); + } + Ok(()) + } + + let worker = WorkerBuilder::new("rango-tango") + .backend(backend) + .on_event(|_ctx, ev| { + println!("On Event = {ev:?}"); + }) + .build(task); + worker.run().await.unwrap(); + } +} diff --git a/apalis-core/src/backend/ext/raw.rs b/apalis-core/src/backend/ext/raw.rs new file mode 100644 index 00000000..97608742 --- /dev/null +++ b/apalis-core/src/backend/ext/raw.rs @@ -0,0 +1,107 @@ +//! A wrapper that allows a backend to be used as a stream of tasks, without needing to know the concrete backend type at compile time. + +use std::{ + pin::Pin, + task::{Context, Poll}, +}; + +use futures_sink::Sink; +use futures_util::SinkExt; + +use crate::{ + backend::{codec::IdentityCodec, *}, + features_table, + task::Task, + worker::context::WorkerContext, +}; + +/// Wrapper that skips decoding and works directly with the compact +/// representation. `Args` becomes `B::Compact`, and `poll` delegates +/// straight to `poll_compact` instead of decoding through `B::Codec`. +/// +/// Useful for backends that natively handle compact types and don't know +/// the concrete `Args` type at compile time — e.g. workflow engines or +/// backends manipulating dynamic/raw payloads. +#[doc = features_table! { + setup = r#" + # { + # use apalis_core::backend::memory::MemoryStorage; + # use apalis_core::backend::ext::raw::RawDataBackend; + # let memory = MemoryStorage::new(); + # RawDataBackend::new(memory) + # }; + "#, + Backend => supported("Basic Backend functionality", true), + TaskSink => supported("Ability to push new tasks", true), + InheritsFeatures => limited("Inherits features from the underlying backend", false), +}] +#[derive(Debug, Clone)] +pub struct RawDataBackend { + inner: B, +} + +impl RawDataBackend { + /// Create a new `RawDataBackend` wrapping the given backend. + pub fn new(backend: B) -> Self { + Self { inner: backend } + } +} + +impl Backend for RawDataBackend +where + B: Backend, + B::Compact: Clone + 'static, +{ + type Args = B::Compact; + type Id = B::Id; + type Connection = B::Connection; + type Error = B::Error; + type Layer = B::Layer; + type Codec = IdentityCodec; + type Compact = B::Compact; + fn codec(&self) -> &Self::Codec { + &IdentityCodec + } + + fn queue(&self) -> Queue { + self.inner.queue() + } + + fn middleware(&self) -> Self::Layer { + self.inner.middleware() + } + + fn poll_ready( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll> { + self.inner.poll_ready(cx, worker) + } + + fn poll_next( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll, Self::Error>>> { + self.inner.poll_next(cx, worker) + } + fn poll_close( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll> { + self.inner.poll_close(cx, worker) + } +} + +delegate_sink!(RawDataBackend, inner); + +delegate_expose!( + impl for RawDataBackend + where { + B: Send + Sync, + B::Compact: Send + Clone + 'static, + } + => inner +); diff --git a/apalis-core/src/backend/ext/with_codec.rs b/apalis-core/src/backend/ext/with_codec.rs new file mode 100644 index 00000000..edd31ebf --- /dev/null +++ b/apalis-core/src/backend/ext/with_codec.rs @@ -0,0 +1,107 @@ +//! A utility module for modifying the codec used by a backend. +//! +//! This is useful for changing the serialization format of task arguments without altering the underlying backend logic. +use std::{ + pin::Pin, + task::{Context, Poll}, +}; + +use futures_sink::Sink; +use futures_util::SinkExt; + +use crate::{ + backend::{queue::Queue, *}, + features_table, + task::Task, + worker::context::WorkerContext, +}; + +/// A `Backend` wrapper that swaps out the serialization codec entirely (JSON, +/// MessagePack, Protobuf, ...) without touching storage logic. +#[derive(Debug, Clone)] +#[doc = features_table! { + setup = r#" + # { + # use apalis_core::backend::memory::MemoryStorage; + # use apalis_core::backend::ext::with_codec::WithCodec; + # use apalis_core::backend::codec::IdentityCodec; + # let memory = MemoryStorage::new(); + # WithCodec::new(memory, IdentityCodec) + # }; + "#, + Backend => supported("Basic Backend functionality", true), + TaskSink => supported("Ability to push new tasks", true), + InheritsFeatures => limited("Inherits features from the underlying backend", false), +}] +pub struct WithCodec { + backend: B, + codec: NewCodec, +} + +impl WithCodec { + /// Create a new `WithCodec` wrapping the given backend and codec. + pub fn new(backend: B, codec: NewCodec) -> Self { + Self { backend, codec } + } +} + +impl Backend for WithCodec +where + B: Backend, + NewCodec: Codec + Send + 'static, +{ + type Args = B::Args; + type Id = B::Id; + type Connection = B::Connection; + type Error = B::Error; + type Codec = NewCodec; + type Compact = B::Compact; + type Layer = B::Layer; + + fn codec(&self) -> &Self::Codec { + &self.codec + } + + fn queue(&self) -> Queue { + self.backend.queue() + } + + fn middleware(&self) -> Self::Layer { + self.backend.middleware() + } + + fn poll_ready( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll> { + self.backend.poll_ready(cx, worker) + } + + fn poll_next( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll, Self::Error>>> { + self.backend.poll_next(cx, worker) + } + fn poll_close( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll> { + self.backend.poll_close(cx, worker) + } +} + +delegate_sink!(WithCodec, backend); + +delegate_expose!( + impl for WithCodec + where { + B: Send + Sync, + C: Codec + Send + Sync + 'static, + B::Compact: Send + } + => backend +); diff --git a/apalis-core/src/backend/ext/with_poll_strategy.rs b/apalis-core/src/backend/ext/with_poll_strategy.rs new file mode 100644 index 00000000..1a0c17df --- /dev/null +++ b/apalis-core/src/backend/ext/with_poll_strategy.rs @@ -0,0 +1,118 @@ +//! A utility module for modifying the codec used by a backend. +//! +//! This is useful for changing the serialization format of task arguments without altering the underlying backend logic. +use std::{ + pin::Pin, + task::{Context, Poll}, +}; + +use futures_sink::Sink; +use futures_util::SinkExt; + +use crate::{ + backend::{ + poll_strategy::{PollMetrics, PollStrategy}, + queue::Queue, + *, + }, + task::Task, + worker::context::WorkerContext, +}; + +/// A `Backend` wrapper that swaps out the serialization codec entirely (JSON, +/// MessagePack, Protobuf, ...) without touching storage logic. +#[derive(Debug, Clone)] +pub struct WithPollStrategy { + backend: B, + strategy: S, + poll_metrics: PollMetrics, +} + +impl WithPollStrategy { + /// Create a new `WithPollStrategy` wrapping the given backend and strategy. + pub fn new(backend: B, strategy: S) -> Self { + Self { + backend, + strategy, + poll_metrics: PollMetrics::default(), + } + } +} + +impl Backend for WithPollStrategy +where + B: Backend, + S: PollStrategy, +{ + type Args = B::Args; + type Id = B::Id; + type Connection = B::Connection; + type Error = B::Error; + type Codec = B::Codec; + type Compact = B::Compact; + type Layer = B::Layer; + + fn codec(&self) -> &Self::Codec { + self.backend.codec() + } + + fn queue(&self) -> Queue { + self.backend.queue() + } + + fn middleware(&self) -> Self::Layer { + self.backend.middleware() + } + + fn poll_ready( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll> { + let snapshot = self.poll_metrics.snapshot(); + if self.strategy.poll_gate(cx, &snapshot).is_ready() { + self.backend.poll_ready(cx, worker) // Poll the inner backend + } else { + Poll::Pending + } + } + + fn poll_next( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll, Self::Error>>> { + match self.backend.poll_next(cx, worker) { + Poll::Ready(task) => { + self.poll_metrics.on_ready(); + self.strategy.on_poll(&self.poll_metrics.snapshot()); + Poll::Ready(task) + } + + Poll::Pending => { + self.poll_metrics.on_pending(); + self.strategy.on_poll(&self.poll_metrics.snapshot()); + Poll::Pending + } + } + } + fn poll_close( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll> { + self.backend.poll_close(cx, worker) + } +} + +delegate_sink!(WithPollStrategy, backend); + +delegate_expose!( + impl for WithPollStrategy + where { + B: Send + Sync, + S: PollStrategy + Send + Sync, + B::Compact: Send + } + => backend +); diff --git a/apalis-core/src/backend/impls/dequeue.rs b/apalis-core/src/backend/impls/dequeue.rs index bd5bf71b..87288cc0 100644 --- a/apalis-core/src/backend/impls/dequeue.rs +++ b/apalis-core/src/backend/impls/dequeue.rs @@ -1,118 +1,56 @@ -#![cfg(feature = "sleep")] use std::{ collections::VecDeque, - fmt, - fmt::Debug, pin::Pin, - sync::{ - Arc, - atomic::{AtomicUsize, Ordering}, - }, - time::Duration, + sync::{Arc, Mutex}, + task::{Context, Poll, Waker}, }; -use futures_core::stream::BoxStream; use futures_sink::Sink; -use futures_util::{ - FutureExt, StreamExt, TryStreamExt, - lock::Mutex, - sink::{self}, - stream, -}; use tower_layer::Identity; use crate::{ - backend::{ - Backend, BackendExt, - codec::IdentityCodec, - custom::{BackendBuilder, CustomBackend}, - poll_strategy::{ - IntervalStrategy, MultiStrategy, PollContext, PollStrategyExt, StrategyBuilder, - }, - }, + backend::{Backend, codec::IdentityCodec, queue::Queue}, error::BoxDynError, task::{Task, task_id::RandomId}, worker::context::WorkerContext, }; -/// Wrapper type for the shared -#[derive(Debug)] -pub struct InMemoryDb { - inner: Arc>>>, +/// A simple in-memory backend that uses a `VecDeque` to store tasks. +/// +/// This backend is primarily intended for testing and demonstration purposes. It does not persist tasks and is not suitable for production use. +#[derive(Debug, Clone)] +pub struct VecDequeBackend { + queue: Arc>>>, + waker: Arc>>, } -impl Clone for InMemoryDb { - fn clone(&self) -> Self { +impl Default for VecDequeBackend { + fn default() -> Self { Self { - inner: self.inner.clone(), + queue: Arc::new(Mutex::new(VecDeque::new())), + waker: Arc::new(Mutex::new(None)), } } } -impl InMemoryDb { - /// Create a new InMemoryDb instance +impl VecDequeBackend { + /// Create a new `VecDequeBackend` with default capacity. #[must_use] pub fn new() -> Self { - Self { - inner: Arc::new(Mutex::new(VecDeque::new())), - } - } - - /// Consume the InMemoryDb and return the inner Arc> - #[must_use] - pub fn into_inner(self) -> Arc>>> { - self.inner + Self::default() } - /// Get a reference to the inner Arc> + /// Create a new `VecDequeBackend` with a specified capacity for the internal queue. #[must_use] - pub fn as_arc(&self) -> &Arc>>> { - &self.inner - } -} - -impl Default for InMemoryDb { - fn default() -> Self { - Self::new() - } -} - -/// Configuration for the in-memory VecDeque backend -#[derive(Debug, Clone)] -pub struct Config { - strategy: MultiStrategy, - prev_count: Arc, -} -/// Type alias for the boxed sink type -pub type BoxSink<'a, T> = Pin + Send + Sync + 'a>>; -/// Type alias for the sink type -pub type InMemorySink = BoxSink<'static, Task>; - -/// Type alias for the complete in-memory backend -pub struct VecDequeBackend( - CustomBackend< - T, - InMemoryDb, - BoxStream<'static, Result>, VecDequeError>>, - InMemorySink, - RandomId, - Config, - >, -); - -impl Debug for VecDequeBackend { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("VecDequeBackend").finish() - } -} - -impl Clone for VecDequeBackend { - fn clone(&self) -> Self { - Self(self.0.clone()) + pub fn with_capacity(capacity: usize) -> Self { + Self { + queue: Arc::new(Mutex::new(VecDeque::with_capacity(capacity))), + waker: Arc::new(Mutex::new(None)), + } } } -/// Errors encountered while using the `VecDequeBackend` +/// Error type for VecDequeBackend #[derive(Debug, thiserror::Error, Clone)] pub enum VecDequeError { /// Error occurred during polling @@ -125,193 +63,126 @@ pub enum VecDequeError { impl Backend for VecDequeBackend where - T: Send + 'static, + T: Send + Clone + 'static, { type Args = T; - - type IdType = RandomId; - + type Id = RandomId; type Connection = (); - - type Stream = BoxStream<'static, Result>, VecDequeError>>; - type Layer = Identity; - - type Beat = BoxStream<'static, Result<(), VecDequeError>>; - type Error = VecDequeError; - - fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat { - self.0 - .heartbeat(worker) - .map_err(|e| VecDequeError::PollError(Arc::new(e.into()))) - .boxed() - } - - fn middleware(&self) -> Self::Layer { - self.0.middleware() - } - fn poll(self, worker: &WorkerContext) -> Self::Stream { - self.0 - .poll(worker) - .map_err(|e| VecDequeError::PollError(Arc::new(e.into()))) - .boxed() - } -} - -impl BackendExt for VecDequeBackend -where - T: Send + 'static, -{ type Codec = IdentityCodec; type Compact = T; - type CompactStream = Self::Stream; - fn get_queue(&self) -> crate::backend::queue::Queue { - std::any::type_name::().into() + fn codec(&self) -> &Self::Codec { + &IdentityCodec } - fn poll_compact(self, worker: &WorkerContext) -> Self::CompactStream { - self.0 - .poll(worker) - .map_err(|e| VecDequeError::PollError(Arc::new(e.into()))) - .boxed() + fn queue(&self) -> Queue { + Queue::from(std::any::type_name::()) } -} - -impl Sink> for VecDequeBackend -where - T: Send + 'static, -{ - type Error = VecDequeError; fn poll_ready( - self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - Pin::new(&mut self.get_mut().0) - .poll_ready(cx) - .map_err(|e| VecDequeError::SendError(Arc::new(e.into()))) - } - - fn start_send(self: Pin<&mut Self>, item: Task) -> Result<(), Self::Error> { - Pin::new(&mut self.get_mut().0) - .start_send(item) - .map_err(|e| VecDequeError::SendError(Arc::new(e.into()))) + &mut self, + cx: &mut Context<'_>, + _worker: &WorkerContext, + ) -> Poll> { + if self + .queue + .lock() + .map_err(|e| VecDequeError::PollError(Arc::new(e.to_string().into())))? + .is_empty() + { + *self.waker.lock().unwrap() = Some(cx.waker().clone()); + Poll::Pending + } else { + Poll::Ready(Ok(())) + } } - fn poll_flush( - self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - Pin::new(&mut self.get_mut().0) - .poll_flush(cx) - .map_err(|e| VecDequeError::SendError(Arc::new(e.into()))) + fn middleware(&self) -> Self::Layer { + Identity::new() + } + + fn poll_next( + &mut self, + _cx: &mut Context<'_>, + _worker: &WorkerContext, + ) -> Poll, Self::Error>>> { + match self + .queue + .lock() + .map_err(|e| VecDequeError::PollError(Arc::new(e.to_string().into())))? + .pop_front() + { + Some(task) => Poll::Ready(Some(Ok(task))), + None => Poll::Ready(None), + } } fn poll_close( - self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - Pin::new(&mut self.get_mut().0) - .poll_close(cx) - .map_err(|e| VecDequeError::SendError(Arc::new(e.into()))) + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll> { + if self + .queue + .lock() + .map_err(|e| VecDequeError::PollError(Arc::new(e.to_string().into())))? + .is_empty() + { + Poll::Ready(Ok(())) + } else { + self.poll_ready(cx, worker) + } } } -/// Create an in-memory `VecDeque` backend with polling strategy for tasks of type T -#[must_use] -pub fn backend(poll_interval: Duration) -> VecDequeBackend +impl Sink> for VecDequeBackend where - T: Send + 'static, + T: Send + Unpin + 'static, { - let memory = InMemoryDb::new(); - - let strategy = StrategyBuilder::new() - .apply(IntervalStrategy::new(poll_interval)) - .build(); + type Error = VecDequeError; - let config = Config { - strategy, - prev_count: Arc::new(AtomicUsize::new(1)), - }; + fn poll_ready(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } - let backend = BackendBuilder::new_with_cfg(config) - .database(memory) - .fetcher( - |db: &mut InMemoryDb, config: &Config, worker: &WorkerContext| { - let poll_strategy = config.strategy.clone(); - let poll_ctx = PollContext::new(worker.clone(), config.prev_count.clone()); - let poller = poll_strategy.build_stream(&poll_ctx); - stream::unfold( - (db.clone(), config.clone(), poller, worker.clone()), - |(p, config, mut poller, ctx)| async move { - poller.next().await; - let Some(mut db) = p.inner.try_lock() else { - return Some(( - Err::>, VecDequeError>( - VecDequeError::PollError(Arc::new( - "Failed to acquire lock".into(), - )), - ), - (p, config, poller, ctx), - )); - }; - let item = db.pop_front(); - drop(db); - if let Some(item) = item { - config.prev_count.store(1, Ordering::Relaxed); - Some((Ok::<_, VecDequeError>(Some(item)), (p, config, poller, ctx))) - } else { - config.prev_count.store(0, Ordering::Relaxed); - Some(( - Ok::>, VecDequeError>(None), - (p, config, poller, ctx), - )) - } - }, - ) - .boxed() - }, - ) - .sink(|db, _| { - Box::pin(sink::unfold( - db.clone(), - move |p, item: Task| { - async move { - let Some(mut db) = p.inner.try_lock() else { - return Err(VecDequeError::PollError(Arc::new( - "Failed to acquire lock".into(), - ))); - }; + fn start_send(self: Pin<&mut Self>, item: Task) -> Result<(), Self::Error> { + let this = self.get_mut(); + + let mut tasks = this + .queue + .lock() + .map_err(|e| VecDequeError::SendError(Arc::new(e.to_string().into())))?; + + if let Some(ref key) = item.ctx.idempotency_key { + let exists = tasks.iter().any(|task| { + task.ctx + .idempotency_key + .as_ref() + .map(|existing| existing == key) + .unwrap_or(false) + }); + + if exists { + return Ok(()); + } + } - if let Some(ref key) = item.ctx.idempotency_key { - let exists = db.iter().any(|task| { - task.ctx - .idempotency_key - .as_ref() - .map(|existing| existing == key) - .unwrap_or(false) - }); + tasks.push_back(item); - if exists { - drop(db); - return Ok::<_, VecDequeError>(p); - } - } + if let Some(waker) = this.waker.lock().unwrap().take() { + waker.wake(); + } - db.push_back(item); - drop(db); + Ok(()) + } - Ok::<_, VecDequeError>(p) - } - .boxed() - .shared() - }, - )) as _ - }) - .build() - .unwrap(); + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } - VecDequeBackend(backend) + fn poll_close(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } } diff --git a/apalis-core/src/backend/impls/memory.rs b/apalis-core/src/backend/impls/memory.rs index a78ef5c0..cd7b6cf5 100644 --- a/apalis-core/src/backend/impls/memory.rs +++ b/apalis-core/src/backend/impls/memory.rs @@ -37,12 +37,11 @@ //! ## See Also //! - [`Backend`] //! - [`WorkerContext`] -use crate::backend::BackendExt; +use crate::backend::Backend; use crate::backend::codec::IdentityCodec; use crate::error::BoxDynError; use crate::features_table; use crate::{ - backend::{Backend, TaskStream}, task::{ Task, task_id::{RandomId, TaskId}, @@ -53,10 +52,7 @@ use futures_channel::mpsc::{SendError, unbounded}; use futures_core::ready; use futures_sink::Sink; use futures_util::lock::Mutex; -use futures_util::{ - FutureExt, SinkExt, Stream, StreamExt, - stream::{self, BoxStream}, -}; +use futures_util::{FutureExt, SinkExt, Stream, StreamExt}; use std::collections::HashSet; use std::{ pin::Pin, @@ -297,41 +293,55 @@ impl Stream for MemoryStorage { } // MemoryStorage as a Backend -impl Backend for MemoryStorage { +impl Backend for MemoryStorage +where + Args: 'static + Clone + Send, + Conn: 'static, +{ type Args = Args; - type IdType = RandomId; + type Id = RandomId; type Connection = Conn; type Error = MemoryStorageError; - type Stream = TaskStream, MemoryStorageError>; type Layer = Identity; - type Beat = BoxStream<'static, Result<(), Self::Error>>; - fn heartbeat(&self, _: &WorkerContext) -> Self::Beat { - stream::once(async { Ok(()) }).boxed() + type Codec = IdentityCodec; + type Compact = Args; + + fn codec(&self) -> &Self::Codec { + &IdentityCodec } - fn middleware(&self) -> Self::Layer { - Identity::new() + + fn queue(&self) -> crate::backend::queue::Queue { + std::any::type_name::().into() } - fn poll(self, _worker: &WorkerContext) -> Self::Stream { - (self.receiver.boxed().map(|r| Ok(Some(r))).boxed()) as _ + fn poll_ready( + &mut self, + _: &mut Context<'_>, + _: &WorkerContext, + ) -> Poll> { + Poll::Ready(Ok(())) } -} -impl BackendExt - for MemoryStorage -{ - type Codec = IdentityCodec; - type Compact = Args; - type CompactStream = TaskStream, Self::Error>; + fn middleware(&self) -> Self::Layer { + Identity::new() + } - fn get_queue(&self) -> crate::backend::queue::Queue { - std::any::type_name::().into() + fn poll_next( + &mut self, + cx: &mut Context<'_>, + _: &WorkerContext, + ) -> Poll, Self::Error>>> { + self.receiver.poll_next_unpin(cx).map(|item| item.map(Ok)) } - fn poll_compact(self, _worker: &WorkerContext) -> Self::CompactStream { - (self.receiver.map(|task| Ok(Some(task))).boxed()) as _ + fn poll_close( + &mut self, + cx: &mut Context<'_>, + _: &WorkerContext, + ) -> Poll> { + self.sender.poll_close_unpin(cx) } } diff --git a/apalis-core/src/backend/mod.rs b/apalis-core/src/backend/mod.rs index afb763c4..6041cecd 100644 --- a/apalis-core/src/backend/mod.rs +++ b/apalis-core/src/backend/mod.rs @@ -18,20 +18,22 @@ //! - [`MemoryStorage`](memory::MemoryStorage): An in-memory backend for testing and lightweight use cases //! - [`Pipe`](pipe::Pipe): A simple pipe-based backend for inter-thread communication //! - [`CustomBackend`](custom::CustomBackend): A flexible backend allowing custom functions for task management -use std::{future::Future, time::Duration}; - -use futures_util::{Stream, stream::BoxStream}; +use futures_util::Stream; +use std::{ + future::Future, + task::{Context, Poll}, + time::Duration, +}; use crate::{ backend::{codec::Codec, queue::Queue}, - error::BoxDynError, task::{Task, status::Status, task_id::TaskId}, worker::context::WorkerContext, }; pub mod codec; pub mod custom; -pub mod pipe; +pub mod ext; pub mod poll_strategy; pub mod queue; pub mod shared; @@ -51,7 +53,6 @@ pub mod memory { } /// In-memory dequeue backend -#[cfg(feature = "sleep")] pub mod dequeue { pub use crate::backend::impls::dequeue::*; } @@ -59,61 +60,66 @@ pub mod dequeue { /// The `Backend` trait defines how workers get and manage tasks from a backend. /// /// In other languages, this might be called a "Queue", "Broker", etc. -pub trait Backend { +pub trait Backend: Sized { /// The type of arguments the backend handles. type Args; /// The type used to uniquely identify tasks. - type IdType: Clone; + type Id: Clone + Send + Sync + 'static; /// The type of connection used by the backend. type Connection; /// The error type returned by backend operations - type Error; - /// A stream of tasks provided by the backend. - type Stream: Stream< - Item = Result>, Self::Error>, - >; - /// A stream representing heartbeat signals. - type Beat: Stream>; - /// The type representing backend middleware layer. - type Layer; - - /// Returns a heartbeat stream for the given worker. - fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat; - /// Returns the backend's middleware layer. - fn middleware(&self) -> Self::Layer; - /// Polls the backend for tasks for the given worker. - fn poll(self, worker: &WorkerContext) -> Self::Stream; -} + type Error: std::error::Error + Send + Sync + 'static; -/// Defines the encoding/serialization aspects of a backend. -pub trait BackendExt: Backend { /// The codec used for serialization/deserialization of tasks. - type Codec: Codec; + type Codec: Codec + Send + 'static; /// The compact representation of task arguments. type Compact; - /// A stream of encoded tasks provided by the backend. - type CompactStream: Stream< - Item = Result>, Self::Error>, - >; + /// The type representing backend middleware layer. + type Layer; + + /// The encoding and decoding mechanism + fn codec(&self) -> &Self::Codec; /// Returns the queue associated with the backend. - fn get_queue(&self) -> Queue; + fn queue(&self) -> Queue; - /// Polls the backend for encoded tasks for the given worker. - fn poll_compact(self, worker: &WorkerContext) -> Self::CompactStream; -} + /// Returns the backend's middleware layer. + fn middleware(&self) -> Self::Layer; + + /// Drives the readiness of the backend to poll for new tasks + /// + /// This can be important for flushing any pending actions + fn poll_ready( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll>; -/// Represents a stream for T. -pub type TaskStream = BoxStream<'static, Result, E>>; + /// Polls the backend for tasks for the given worker. + fn poll_next( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll, Self::Error>>>; + + /// Flushes/releases any resources the backend holds (pending acks, + /// open subscriptions, connections) before the worker fully shuts down. + /// Default no-op for backends with nothing to flush. + fn poll_close( + &mut self, + cx: &mut Context<'_>, + worker: &WorkerContext, + ) -> Poll>; +} /// Allows fetching a task by its ID -pub trait FetchById: Backend { +pub trait FetchById: Backend { /// Fetch a task by its unique identifier #[allow(clippy::type_complexity)] fn fetch_by_id( &mut self, - task_id: &TaskId, + task_id: &TaskId, ) -> impl Future< - Output = Result>, Self::Error>, + Output = Result>, Self::Error>, > + Send; } @@ -122,7 +128,7 @@ pub trait Update: Backend { /// Update the given task fn update( &mut self, - task: Task, + task: Task, ) -> impl Future> + Send; } @@ -131,7 +137,7 @@ pub trait Reschedule: Backend { /// Reschedule the task after a specified duration fn reschedule( &mut self, - task: Task, + task: Task, wait: Duration, ) -> impl Future> + Send; } @@ -147,11 +153,11 @@ pub trait ResumeById: Backend { /// Resume a task by its ID fn resume_by_id( &mut self, - id: TaskId, + id: TaskId, ) -> impl Future> + Send; } -/// Allows fetching multiple tasks by their IDs +/// Allows resuming all abandoned tasks in the backend pub trait ResumeAbandoned: Backend { /// Resume all abandoned tasks fn resume_abandoned(&mut self) -> impl Future> + Send; @@ -169,18 +175,18 @@ pub trait RegisterWorker: Backend { /// Represents the result of a task execution #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(Debug, Clone)] -pub struct TaskResult { +pub struct TaskResult { /// The unique identifier of the task - pub task_id: TaskId, + pub task_id: TaskId, /// The status of the task pub status: Status, /// The result of the task execution pub result: Result, } -impl TaskResult { +impl TaskResult { /// Create a new TaskResult - pub fn new(task_id: TaskId, status: Status, result: Result) -> Self { + pub fn new(task_id: TaskId, status: Status, result: Result) -> Self { Self { task_id, status, @@ -188,7 +194,7 @@ impl TaskResult { } } /// Get the ID of the task - pub fn task_id(&self) -> &TaskId { + pub fn task_id(&self) -> &TaskId { &self.task_id } @@ -209,52 +215,23 @@ impl TaskResult { } /// Allows waiting for tasks to complete and checking their status -pub trait WaitForCompletion: Backend { +pub trait WaitForCompletion: Backend { /// The result stream type yielding task results - type ResultStream: Stream, Self::Error>> + type ResultStream: Stream, Self::Error>> + Send + 'static; /// Wait for multiple tasks to complete, yielding results as they become available - fn wait_for( - &self, - task_ids: impl IntoIterator>, - ) -> Self::ResultStream; + fn wait_for(&self, task_ids: impl IntoIterator>) -> Self::ResultStream; /// Wait for a single task to complete, yielding its result - fn wait_for_single(&self, task_id: TaskId) -> Self::ResultStream { + fn wait_for_single(&self, task_id: TaskId) -> Self::ResultStream { self.wait_for(std::iter::once(task_id)) } /// Check current status of tasks without waiting fn check_status( &self, - task_ids: impl IntoIterator> + Send, - ) -> impl Future>, Self::Error>> + Send; -} - -/// A helper trait to build connections and pollers for backends -/// This should be used in crates implementing Backend rather than end users. -pub trait TryIntoConnectionParts { - /// The config for the backend - type Config; - /// The connection for the backend - type Connection; - /// The poller to be used by the backend - type Fetcher; - /// The error emitted during creation - type Error: std::error::Error + Send + Sync + 'static; - /// Generate the parts needed to build a Backend - fn try_into_parts( - self, - config: &Self::Config, - ) -> Result<(Self::Connection, Self::Fetcher), Self::Error>; -} - -/// A helper to standardize building new backends -pub trait TryNewBackend: Backend + Sized { - /// Build a new backend given a connection source and a config - fn try_new

(src: P, config: P::Config) -> Result - where - P: TryIntoConnectionParts; + task_ids: impl IntoIterator> + Send, + ) -> impl Future>, Self::Error>> + Send; } diff --git a/apalis-core/src/backend/pipe.rs b/apalis-core/src/backend/pipe.rs deleted file mode 100644 index 03c96581..00000000 --- a/apalis-core/src/backend/pipe.rs +++ /dev/null @@ -1,270 +0,0 @@ -//! # Pipe streams to backends -//! -//! This backend allows you to pipe tasks from any stream into another backend. -//! It is useful for connecting different backends together, such as piping tasks -//! from a cron stream into a database backend, or transforming and forwarding tasks -//! between systems. -//! -//! ## Example -//! -//! ```rust -//! # use futures_util::stream; -//! # use apalis_core::backend::{pipe::PipeExt, dequeue}; -//! # use apalis_core::worker::{builder::WorkerBuilder, context::WorkerContext}; -//! # use apalis_core::error::BoxDynError; -//! # use std::time::Duration; -//! # use futures_util::StreamExt; -//! # use crate::apalis_core::worker::ext::event_listener::EventListenerExt; -//! #[tokio::main] -//! async fn main() { -//! let stm = stream::iter(0..10).map(|s| Ok::<_, std::io::Error>(s)); -//! -//! let in_memory = dequeue::backend::(Duration::from_secs(1)); -//! let backend = stm.pipe_to(in_memory); -//! -//! async fn task(task: u32, ctx: WorkerContext) -> Result<(), BoxDynError> { -//! tokio::time::sleep(Duration::from_secs(1)).await; -//! # if task == 9 { -//! # ctx.stop().unwrap(); -//! # } -//! Ok(()) -//! } -//! -//! let worker = WorkerBuilder::new("rango-tango") -//! .backend(backend) -//! .on_event(|_ctx, ev| { -//! println!("On Event = {:?}", ev); -//! }) -//! .build(task); -//! worker.run().await.unwrap(); -//! } -//! ``` -//! -//! This example pipes a stream of numbers into an in-memory backend and processes them with a worker. -//! -//! See also: -//! - [`apalis-cron`](https://docs.rs/apalis-cron) - -use crate::backend::{BackendExt, TaskSink}; -use crate::error::BoxDynError; -use crate::features_table; -use crate::task::Task; -use crate::task::builder::TaskBuilder; -use crate::{backend::Backend, backend::codec::Codec, worker::context::WorkerContext}; -use futures_sink::Sink; -use futures_util::stream::{once, select}; -use futures_util::{SinkExt, Stream, TryStreamExt}; -use futures_util::{StreamExt, stream::BoxStream}; -use std::fmt; -use std::fmt::Debug; -use std::marker::PhantomData; -use std::ops::{Deref, DerefMut}; - -/// A generic pipe that wraps a [`Stream`] and passes it to a backend -#[doc = features_table! { - setup = "{ unreachable!() }", - TaskSink => supported("Ability to push new tasks", false), - InheritsFeatures => limited("Inherits features from the underlying backend", false), -}] -pub struct Pipe { - pub(crate) from: S, - pub(crate) into: Into, - pub(crate) _req: PhantomData<(Args, Conn)>, -} - -impl Clone for Pipe { - fn clone(&self) -> Self { - Self { - from: self.from.clone(), - into: self.into.clone(), - _req: PhantomData, - } - } -} - -impl Deref for Pipe { - type Target = Into; - - fn deref(&self) -> &Self::Target { - &self.into - } -} - -impl DerefMut for Pipe { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.into - } -} - -impl Pipe { - /// Create a new Pipe instance - pub fn new(stream: S, backend: Into) -> Self { - Self { - from: stream, - into: backend, - _req: PhantomData, - } - } -} - -impl fmt::Debug for Pipe { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Pipe") - .field("inner", &self.from) - .field("into", &self.into) - .finish() - } -} - -impl Backend for Pipe -where - S: Stream> + Send + 'static, - TSink: Backend - + BackendExt - + TaskSink - + Clone - + Unpin - + Send - + 'static - + Sink>, - ::Error: std::error::Error + Send + Sync + 'static, - TSink::Beat: Send + 'static, - TSink::IdType: Send + Sync + 'static, - TSink::Stream: Send + 'static, - Args: Send + 'static, - Conn: Send + Sync + 'static, - Err: std::error::Error + Send + Sync + 'static, - >>::Error: - std::error::Error + Send + Sync + 'static, - <::Codec as Codec>::Error: std::error::Error + Send + Sync + 'static, - TSink::Compact: Send, -{ - type Args = Args; - - type IdType = TSink::IdType; - - type Connection = Conn; - - type Stream = BoxStream<'static, Result>, PipeError>>; - - type Layer = TSink::Layer; - - type Beat = BoxStream<'static, Result<(), PipeError>>; - - type Error = PipeError; - - fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat { - self.into - .heartbeat(worker) - .map_err(|e| PipeError::Inner(e.into())) - .boxed() - } - - fn middleware(&self) -> Self::Layer { - self.into.middleware() - } - - fn poll(self, worker: &WorkerContext) -> Self::Stream { - let mut sink = self - .into - .clone() - .sink_map_err(|e| PipeError::Inner(e.into())); - - let mut sink_stream = self - .from - .map_err(|e| PipeError::Inner(e.into())) - .map_ok(|s| { - TaskBuilder::new(s) - .try_map(|s| TSink::Codec::encode(&s).map_err(|e| PipeError::Inner(e.into()))) - .map(|s| s.build()) - }) - .map(|t| t.and_then(|t| t)) - .boxed(); - - let sender_stream = self.into.poll(worker); - select( - once(async move { - let fut = sink.send_all(&mut sink_stream); - fut.await.map_err(|e| PipeError::Inner(e.into()))?; - Ok(None) - }), - sender_stream.map_err(|e| PipeError::Inner(e.into())), - ) - .boxed() - } -} - -/// Represents utility for piping streams into a backend -pub trait PipeExt -where - Self: Sized, -{ - /// Pipe the current stream into the provided backend - fn pipe_to(self, backend: B) -> Pipe; -} - -impl PipeExt for S -where - S: Stream> + Send + 'static, - ::Error: Into + Send + Sync + 'static, - B: Backend + TaskSink, -{ - fn pipe_to(self, backend: B) -> Pipe { - Pipe::new(self, backend) - } -} - -/// Error encountered while piping streams -#[derive(Debug, thiserror::Error)] -pub enum PipeError { - /// The cron stream provided a None - #[error("The inner stream provided a None")] - EmptyStream, - /// An inner stream error occurred - #[error("The inner stream error: {0}")] - Inner(BoxDynError), -} - -#[cfg(test)] -mod tests { - use std::{io, time::Duration}; - - use futures_util::stream; - - use crate::{ - backend::dequeue, - error::BoxDynError, - worker::{ - builder::WorkerBuilder, context::WorkerContext, ext::event_listener::EventListenerExt, - }, - }; - - use super::*; - - const ITEMS: u32 = 10; - - #[tokio::test] - async fn basic_worker() { - let stm = stream::iter(0..ITEMS).map(Ok::<_, io::Error>); - let in_memory = dequeue::backend::(Duration::from_secs(1)); - - let backend = Pipe::new(stm, in_memory); - - async fn task(task: u32, ctx: WorkerContext) -> Result<(), BoxDynError> { - tokio::time::sleep(Duration::from_secs(1)).await; - if task == ITEMS - 1 { - ctx.stop().unwrap(); - return Err("Graceful Exit".into()); - } - Ok(()) - } - - let worker = WorkerBuilder::new("rango-tango") - .backend(backend) - .on_event(|_ctx, ev| { - println!("On Event = {ev:?}"); - }) - .build(task); - worker.run().await.unwrap(); - } -} diff --git a/apalis-core/src/backend/poll_strategy/builder.rs b/apalis-core/src/backend/poll_strategy/builder.rs index 272920db..a5981ab2 100644 --- a/apalis-core/src/backend/poll_strategy/builder.rs +++ b/apalis-core/src/backend/poll_strategy/builder.rs @@ -1,11 +1,10 @@ -use futures_core::Stream; -use futures_util::StreamExt; -use std::{pin::Pin, sync::Arc}; - -use crate::backend::poll_strategy::{ - BoxedPollStrategy, PollContext, PollStrategy, RaceNext, WrapperStrategy, +use std::{ + sync::Arc, + task::{Context, Poll}, }; +use crate::backend::poll_strategy::{BoxedPollStrategy, PollSnapshot, PollStrategy}; + /// Builder for composing multiple polling strategies pub struct StrategyBuilder { strategies: Vec, @@ -38,19 +37,16 @@ impl StrategyBuilder { /// Strategies are executed in the order they are added, with the first strategy having the highest priority /// In case of multiple strategies being ready at the same time, the first one added will be chosen #[must_use] - pub fn apply(mut self, strategy: S) -> Self + pub fn apply(mut self, strategy: S) -> Self where - S: PollStrategy + 'static + Sync + Send, - Stm: Stream + Send + 'static, + S: PollStrategy + Send + Sync + 'static, { - self.strategies - .push(Box::new(WrapperStrategy::new(strategy))); + self.strategies.push(Box::new(strategy)); self } /// Build the MultiStrategy from the builder /// Consumes the builder and returns a MultiStrategy - /// The MultiStrategy will contain all the strategies added to the builder #[must_use] pub fn build(self) -> MultiStrategy { MultiStrategy { @@ -60,7 +56,7 @@ impl StrategyBuilder { } /// A polling strategy that combines multiple strategies -/// The strategies are executed in the order they were added to the builder +/// The strategies are polled in the order they were added to the builder /// In case of multiple strategies being ready at the same time, the first one added will be chosen #[derive(Clone)] pub struct MultiStrategy { @@ -76,22 +72,14 @@ impl std::fmt::Debug for MultiStrategy { } impl PollStrategy for MultiStrategy { - type Stream = Pin + Send>>; - - fn poll_strategy(self: Box, ctx: &PollContext) -> Self::Stream { - let ctx = ctx.clone(); - let mut streams = self - .strategies - .lock() - .unwrap() - .drain(..) - .map(move |s| { - let ctx = ctx.clone(); - s.poll_strategy(&ctx) - }) - .collect::>(); - // Reverse to give priority to strategies in the order they were added - streams.reverse(); - RaceNext::new(streams).map(|(_idx, _)| ()).boxed() + fn poll_gate(&mut self, cx: &mut Context<'_>, worker: &PollSnapshot) -> Poll<()> { + // Priority order: first-added strategy wins if multiple are ready. + for strategy in &mut self.strategies.lock().unwrap().iter_mut() { + if strategy.poll_gate(cx, worker).is_ready() { + return Poll::Ready(()); + } + } + cx.waker().wake_by_ref(); + Poll::Pending } } diff --git a/apalis-core/src/backend/poll_strategy/mod.rs b/apalis-core/src/backend/poll_strategy/mod.rs index 2b95cb60..ccd07b02 100644 --- a/apalis-core/src/backend/poll_strategy/mod.rs +++ b/apalis-core/src/backend/poll_strategy/mod.rs @@ -19,12 +19,9 @@ //! task counts. //! //! See submodules for available strategies and builder utilities. -use crate::worker::context::WorkerContext; -use futures_core::Stream; -use futures_util::StreamExt; use std::{ - pin::Pin, - sync::{Arc, atomic::AtomicUsize}, + task::{Context, Poll}, + time::{Duration, Instant}, }; mod strategies; @@ -35,66 +32,209 @@ mod race_next; pub use race_next::*; /// A boxed poll strategy -pub type BoxedPollStrategy = - Box + Send>>> + Send + Sync + 'static>; +pub type BoxedPollStrategy = Box; /// A trait for different polling strategies /// All strategies can be combined in a race condition pub trait PollStrategy { - /// The stream returned by the strategy - type Stream: Stream + Send; + /// Poll to determine whether the next poll cycle should occur now. + /// + /// Returns `Poll::Ready(())` when it's time to poll, registering the + /// waker via `cx` if not ready. + /// + /// # Contract + /// If this returns `Poll::Pending`, the implementation **must** arrange + /// for `cx.waker()` to be woken when it becomes ready to poll again. + fn poll_gate(&mut self, cx: &mut Context<'_>, snapshot: &PollSnapshot) -> Poll<()>; + + /// Signifies a poll was made + #[allow(unused_variables)] + fn on_poll(&mut self, snapshot: &PollSnapshot) {} +} + +/// Snapshot of the current polling state. +/// +/// This is intended to be passed to adaptive polling strategies. +#[derive(Debug, Clone, Copy, Default)] +pub struct PollSnapshot { + /// Number of consecutive successful polls since the last `Pending`. + pub consecutive_ready: usize, + + /// Total number of successful polls. + pub total_ready: u64, - /// Create a stream that completes when the next poll should occur - fn poll_strategy(self: Box, ctx: &PollContext) -> Self::Stream; + /// Total number of pending polls. + pub total_pending: u64, + + /// Duration since the last successful poll. + pub idle_for: Duration, } -impl PollStrategyExt for T where T: PollStrategy {} - -/// Extension trait for PollStrategy -pub trait PollStrategyExt: PollStrategy + Sized { - /// Build a boxed stream from the strategy - /// This is a convenience method that boxes the strategy and calls `poll_strategy` - fn build_stream(self, ctx: &PollContext) -> Pin + Send>> - where - Self::Stream: 'static, - { - let this = Box::new(self); - this.poll_strategy(ctx).map(|_| ()).boxed() - } +/// Keeps a record of the current poll status +#[derive(Debug, Default, Clone)] +pub struct PollMetrics { + consecutive_ready: usize, + total_ready: u64, + total_pending: u64, + last_ready: Option, } -/// Context provided to the polling strategies -/// Includes the worker context and a reference to the previous count of tasks received -#[derive(Debug, Clone)] -pub struct PollContext { - worker: WorkerContext, - prev_count: Arc, +/// The worker's activity classification based on consecutive successful polls. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum ActivityState { + /// No work has been produced since the last `Pending` (or ever). + #[default] + Idle, + /// Exactly one successful poll since the last idle period — just started working. + WakingUp, + /// More than one consecutive successful poll — sustained activity. + Saturated, +} + +impl ActivityState { + #[must_use] + const fn from_consecutive_ready(consecutive_ready: usize) -> Self { + match consecutive_ready { + 0 => Self::Idle, + 1 => Self::WakingUp, + _ => Self::Saturated, + } + } } -impl PollContext { - /// Create a new PollContext - pub fn new(worker: WorkerContext, prev_count: Arc) -> Self { - Self { worker, prev_count } +impl PollSnapshot { + /// Create a new polling context. + #[must_use] + pub const fn new() -> Self { + Self { + consecutive_ready: 0, + total_ready: 0, + total_pending: 0, + idle_for: Duration::from_secs(0), + } + } + + /// Total number of polls performed. + #[must_use] + pub const fn total_polls(&self) -> u64 { + self.total_ready + self.total_pending + } + + /// Returns true if no polls have been performed yet. + #[must_use] + pub const fn is_empty(&self) -> bool { + self.total_polls() == 0 + } + + /// Fraction of polls that returned work. + /// + /// Returns `1.0` if no polls have occurred yet. + #[must_use] + pub fn hit_rate(&self) -> f64 { + let total = self.total_polls(); + + if total == 0 { + 1.0 + } else { + self.total_ready as f64 / total as f64 + } + } + + /// Fraction of polls that returned `Pending`. + #[must_use] + pub fn miss_rate(&self) -> f64 { + 1.0 - self.hit_rate() + } + + /// Returns whether the worker has been idle for at least `duration`. + #[must_use] + pub fn idle_for_at_least(&self, duration: Duration) -> bool { + self.idle_for >= duration + } + + /// Returns whether the worker has been idle for less than `duration`. + #[must_use] + pub fn idle_for_less_than(&self, duration: Duration) -> bool { + self.idle_for < duration + } + + /// Returns true if enough consecutive jobs have been received to + /// consider switching to an aggressive polling strategy. + #[must_use] + pub const fn should_poll_aggressively(&self, threshold: usize) -> bool { + self.consecutive_ready >= threshold + } + + /// Returns true if the worker should consider backing off after + /// remaining idle for the given duration. + #[must_use] + pub fn should_backoff(&self, max_idle: Duration) -> bool { + self.consecutive_ready == 0 && self.idle_for >= max_idle + } + + /// Classify the worker's current activity state. + #[must_use] + pub const fn activity(&self) -> ActivityState { + ActivityState::from_consecutive_ready(self.consecutive_ready) + } + + /// Returns true if the previous poll produced work (not idle). + #[must_use] + pub const fn is_busy(&self) -> bool { + !matches!(self.activity(), ActivityState::Idle) + } + + /// Returns true if the worker is currently idle. + #[must_use] + pub const fn is_idle(&self) -> bool { + matches!(self.activity(), ActivityState::Idle) } - /// Get a reference to the worker context + + /// Returns whether this is the first successful poll after an idle period. #[must_use] - pub fn worker(&self) -> &WorkerContext { - &self.worker + pub const fn is_waking_up(&self) -> bool { + matches!(self.activity(), ActivityState::WakingUp) + } + + /// Returns whether the worker has sustained activity. + #[must_use] + pub const fn is_saturated(&self) -> bool { + matches!(self.activity(), ActivityState::Saturated) + } +} + +impl PollMetrics { + /// Register a ready call response + pub fn on_ready(&mut self) { + self.consecutive_ready = self.consecutive_ready.saturating_add(1); + self.total_ready = self.total_ready.saturating_add(1); + self.last_ready = Some(Instant::now()); } - /// Get a reference to the previous count of tasks received + + /// Register a pending poll response + pub fn on_pending(&mut self) { + self.consecutive_ready = 0; + self.total_pending = self.total_pending.saturating_add(1); + } + + /// Get the snapshot of the current poll status #[must_use] - pub fn prev_count(&self) -> &Arc { - &self.prev_count + pub fn snapshot(&self) -> PollSnapshot { + PollSnapshot { + consecutive_ready: self.consecutive_ready, + total_ready: self.total_ready, + total_pending: self.total_pending, + idle_for: self + .last_ready + .map(|instant| instant.elapsed()) + .unwrap_or_default(), + } } } #[cfg(test)] mod tests { - use std::{ - collections::VecDeque, - sync::{Arc, atomic::Ordering}, - time::Duration, - }; + use std::{collections::VecDeque, sync::Arc, time::Duration}; use futures_channel::mpsc; @@ -120,9 +260,13 @@ mod tests { #[tokio::test] #[cfg(feature = "sleep")] async fn basic_strategy_backend() { - use crate::backend::impls::dequeue::backend; + use crate::backend::{dequeue::VecDequeBackend, ext::BackendExt}; + let backoff = BackoffConfig::new(Duration::from_secs(5)) + .with_multiplier(1.5) + .with_jitter(0.2); + let interval = IntervalStrategy::new(Duration::from_millis(200)).with_backoff(backoff); - let mut backend = backend(Duration::from_secs(1)); + let mut backend = VecDequeBackend::new().with_poll_strategy(interval); for i in 0..ITEMS { use crate::task::builder::TaskBuilder; @@ -152,26 +296,17 @@ mod tests { #[tokio::test] #[cfg(feature = "sleep")] async fn custom_strategy_backend() { - use crate::backend::custom::BackendBuilder; + use crate::backend::{custom::BackendBuilder, ext::BackendExt}; let memory = Arc::new(Mutex::new(VecDeque::new())); - - #[derive(Clone)] - struct Config { - strategy: MultiStrategy, - prev_count: Arc, - } - let backoff = BackoffConfig::new(Duration::from_secs(5)) .with_multiplier(1.5) .with_jitter(0.2); let interval = IntervalStrategy::new(Duration::from_millis(200)).with_backoff(backoff); - let when_i_am_ready = FutureStrategy::new(|_ctx: WorkerContext, _prev: usize| { + let when_i_am_ready = FutureStrategy::new(move |_| { // println!("Waiting to be ready..."); - tokio::time::sleep(Duration::from_millis(1500)).map(|_| { - // println!("I am ready now!"); - }) + crate::timer::sleep(Duration::from_millis(1500)) }); let (mut tx, rx) = mpsc::channel(1); @@ -191,36 +326,22 @@ mod tests { .apply(StreamStrategy::new(rx)) .build(); - let config = Config { - strategy, - prev_count: Arc::new(AtomicUsize::new(1)), - }; - - let mut backend = BackendBuilder::new_with_cfg(config) + let mut backend = BackendBuilder::new_with_cfg(()) .database(memory) - .fetcher(|db, config, worker| { - let poll_strategy = config.strategy.clone(); - let poll_ctx = PollContext::new(worker.clone(), config.prev_count.clone()); - let poller = poll_strategy.build_stream(&poll_ctx); - stream::unfold( - (db.clone(), config.clone(), poller, worker.clone()), - |(p, config, mut poller, ctx)| async move { - poller.next().await; - let mut db = p.lock().await; - let item = db.pop_front(); - drop(db); - if let Some(item) = item { - config.prev_count.store(1, Ordering::Relaxed); - Some((Ok::<_, BoxDynError>(Some(item)), (p, config, poller, ctx))) - } else { - config.prev_count.store(0, Ordering::Relaxed); - Some(( - Ok::>, BoxDynError>(None), - (p, config, poller, ctx), - )) - } - }, - ) + .fetcher(|db, _, worker| { + stream::unfold((db.clone(), worker.clone()), |(p, ctx)| async move { + let mut db = p.lock().await; + let item = db.pop_front(); + drop(db); + if let Some(item) = item { + Some((Ok::<_, BoxDynError>(Some(item)), (p, ctx))) + } else { + Some(( + Ok::>, BoxDynError>(None), + (p, ctx), + )) + } + }) .boxed() }) .sink(|db, _| { @@ -235,7 +356,8 @@ mod tests { }) }) .build() - .unwrap(); + .unwrap() + .with_poll_strategy(strategy); for i in 0..ITEMS { use crate::task::builder::TaskBuilder; diff --git a/apalis-core/src/backend/poll_strategy/strategies/backoff.rs b/apalis-core/src/backend/poll_strategy/strategies/backoff.rs index f443de12..52e61852 100644 --- a/apalis-core/src/backend/poll_strategy/strategies/backoff.rs +++ b/apalis-core/src/backend/poll_strategy/strategies/backoff.rs @@ -1,57 +1,52 @@ use std::{ sync::atomic::{AtomicU64, Ordering}, + task::{Context, Poll}, time::Duration, }; -use futures_core::stream::BoxStream; -use futures_util::{StreamExt, stream}; +use futures_util::FutureExt; -use crate::backend::poll_strategy::{IntervalStrategy, PollContext, PollStrategy}; +use crate::backend::poll_strategy::{PollSnapshot, PollStrategy}; // Simple PRNG state for jitter (thread-safe) static JITTER_STATE: AtomicU64 = AtomicU64::new(1); /// A polling strategy that applies exponential backoff to an inner interval strategy -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct BackoffStrategy { - interval: IntervalStrategy, - backoff_config: BackoffConfig, default_delay: Duration, + current_delay: Duration, + backoff_config: BackoffConfig, + delay: futures_timer::Delay, } + impl BackoffStrategy { - /// Create a new BackoffStrategy wrapping an IntervalStrategy with the given BackoffConfig + /// Create a new BackoffStrategy with the given BackoffConfig #[must_use] - pub fn new(inner: IntervalStrategy, config: BackoffConfig) -> Self { + pub fn new(default_delay: Duration, config: BackoffConfig) -> Self { Self { - default_delay: inner.poll_interval, - interval: inner, + default_delay, + current_delay: default_delay, backoff_config: config, + delay: futures_timer::Delay::new(default_delay), } } } impl PollStrategy for BackoffStrategy { - type Stream = BoxStream<'static, ()>; - - fn poll_strategy(self: Box, ctx: &PollContext) -> Self::Stream { - let backoff_config = self.backoff_config.clone(); - let current_delay = self.interval.poll_interval; - let default_delay = self.default_delay; - - stream::unfold( - (ctx.clone(), current_delay), - move |(ctx, mut current_delay)| { - let fut = futures_timer::Delay::new(current_delay); - let backoff_config = backoff_config.clone(); - async move { - fut.await; - let failed = ctx.prev_count.load(Ordering::Relaxed) == 0; - current_delay = backoff_config.next_delay(default_delay, current_delay, failed); - Some(((), (ctx, current_delay))) - } - }, - ) - .boxed() + fn poll_gate(&mut self, cx: &mut Context<'_>, _: &PollSnapshot) -> Poll<()> { + self.delay.poll_unpin(cx) + } + + fn on_poll(&mut self, ctx: &PollSnapshot) { + self.current_delay = if ctx.is_idle() { + self.backoff_config + .next_delay(self.default_delay, self.current_delay, true) + } else { + self.default_delay + }; + + self.delay = futures_timer::Delay::new(self.current_delay); } } @@ -98,32 +93,20 @@ impl BackoffConfig { } /// Calculate the next delay with backoff and jitter - fn next_delay( - &self, - default_delay: Duration, - current_delay: Duration, - failed: bool, - ) -> Duration { - let base_delay = if failed { - // Exponential backoff on failure + fn next_delay(&self, default_delay: Duration, current_delay: Duration, idle: bool) -> Duration { + let base_delay = if idle { let next = Duration::from_secs_f64(current_delay.as_secs_f64() * self.multiplier); next.min(self.max_delay) } else { - // Reset to initial on success default_delay }; - // Add jitter using a simple LCG (Linear Congruential Generator) if self.jitter_factor > 0.0 { - // Simple deterministic pseudo-random number generation let mut state = JITTER_STATE.load(Ordering::Relaxed); state = state.wrapping_mul(1103515245).wrapping_add(12345); JITTER_STATE.store(state, Ordering::Relaxed); - // Convert to 0.0-1.0 range let normalized = (state as f64) / (u64::MAX as f64); - - // Apply jitter: -jitter_factor to +jitter_factor let jitter_range = base_delay.as_secs_f64() * self.jitter_factor; let jitter = (normalized - 0.5) * 2.0 * jitter_range; let jittered = base_delay.as_secs_f64() + jitter; diff --git a/apalis-core/src/backend/poll_strategy/strategies/future.rs b/apalis-core/src/backend/poll_strategy/strategies/future.rs index d2f65b7d..7209ce5c 100644 --- a/apalis-core/src/backend/poll_strategy/strategies/future.rs +++ b/apalis-core/src/backend/poll_strategy/strategies/future.rs @@ -1,21 +1,16 @@ use core::fmt; use std::{ pin::Pin, - sync::{Arc, atomic::Ordering}, -}; - -use futures_core::Stream; -use futures_util::{StreamExt, lock::Mutex, stream}; - -use crate::{ - backend::poll_strategy::{PollContext, PollStrategy}, - worker::context::WorkerContext, + sync::{Arc, Mutex}, + task::{Context, Poll}, }; +use crate::backend::poll_strategy::{PollSnapshot, PollStrategy}; /// A polling strategy that uses a future factory to create futures for polling #[derive(Clone)] pub struct FutureStrategy { - future_factory: Arc F + Send>>, + future_factory: Arc F + Send>>, + current: Option>>, } impl fmt::Debug for FutureStrategy { @@ -33,34 +28,35 @@ where /// Create a new FutureStrategy from a future factory pub fn new(factory: Factory) -> Self where - Factory: FnMut(WorkerContext, usize) -> F + Send + 'static, + Factory: Fn(&PollSnapshot) -> F + Send + 'static, { Self { - future_factory: Arc::new(Mutex::new(Box::new(factory))), + future_factory: Arc::new(Mutex::new(factory)), + current: None, } } } impl PollStrategy for FutureStrategy where - F: Future + Send + 'static, + F: Future + Send + Sync + 'static, { - type Stream = Pin + Send>>; - fn poll_strategy(self: Box, ctx: &PollContext) -> Self::Stream { - let factory = self.future_factory; - let ctx = ctx.clone(); + fn poll_gate(&mut self, cx: &mut Context<'_>, snapshot: &PollSnapshot) -> Poll<()> { + if self.current.is_none() { + let Ok(factory) = (self.future_factory).try_lock() else { + return Poll::Pending; + }; + let fut = (factory)(snapshot); + self.current = Some(Box::pin(fut)); + } - stream::unfold(ctx, move |ctx| { - let factory = factory.clone(); - async move { - let fut = { - let mut lock = factory.try_lock().unwrap(); - (lock)(ctx.worker.clone(), ctx.prev_count.load(Ordering::Relaxed)) - }; - fut.await; - Some(((), ctx)) + let fut = self.current.as_mut().unwrap(); + match fut.as_mut().poll(cx) { + Poll::Ready(()) => { + self.current = None; + Poll::Ready(()) } - }) - .boxed() + Poll::Pending => Poll::Pending, + } } } diff --git a/apalis-core/src/backend/poll_strategy/strategies/interval.rs b/apalis-core/src/backend/poll_strategy/strategies/interval.rs index c0ee9770..85cbf943 100644 --- a/apalis-core/src/backend/poll_strategy/strategies/interval.rs +++ b/apalis-core/src/backend/poll_strategy/strategies/interval.rs @@ -1,46 +1,53 @@ -use std::time::Duration; +use std::{ + pin::Pin, + task::{Context, Poll}, + time::Duration, +}; -use futures_core::stream::BoxStream; -use futures_util::{StreamExt, stream}; - -use crate::backend::poll_strategy::{BackoffConfig, BackoffStrategy, PollContext, PollStrategy}; +use crate::backend::poll_strategy::{BackoffConfig, BackoffStrategy, PollSnapshot, PollStrategy}; /// Interval-based polling strategy with optional backoff -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct IntervalStrategy { - pub(super) poll_interval: Duration, + poll_interval: Duration, + delay: futures_timer::Delay, } impl IntervalStrategy { /// Create a new IntervalStrategy with the specified interval #[must_use] - pub fn new(interval: Duration) -> Self { + pub fn new(poll_interval: Duration) -> Self { Self { - poll_interval: interval, + poll_interval, + delay: futures_timer::Delay::new(poll_interval), } } + /// Get the current polling interval + #[must_use] + pub fn poll_interval(&self) -> Duration { + self.poll_interval + } + /// Wrap the IntervalStrategy with a BackoffStrategy /// This will apply exponential backoff to the polling interval /// based on the provided [`BackoffConfig`].` #[must_use] pub fn with_backoff(self, config: BackoffConfig) -> BackoffStrategy { - BackoffStrategy::new(self, config) + BackoffStrategy::new(self.poll_interval(), config) } } impl PollStrategy for IntervalStrategy { - type Stream = BoxStream<'static, ()>; - - fn poll_strategy(self: Box, _: &PollContext) -> Self::Stream { - let interval = self.poll_interval; - stream::unfold((), move |()| { - let fut = futures_timer::Delay::new(interval); - async move { - fut.await; - Some(((), ())) + fn poll_gate(&mut self, cx: &mut Context<'_>, _: &PollSnapshot) -> Poll<()> { + match Pin::new(&mut self.delay).poll(cx) { + Poll::Ready(()) => { + self.delay = futures_timer::Delay::new(self.poll_interval); + // Wake immediately to register the new delay's waker so it makes progress. + cx.waker().wake_by_ref(); + Poll::Ready(()) } - }) - .boxed() + Poll::Pending => Poll::Pending, + } } } diff --git a/apalis-core/src/backend/poll_strategy/strategies/mod.rs b/apalis-core/src/backend/poll_strategy/strategies/mod.rs index ae5f84be..1ea5883b 100644 --- a/apalis-core/src/backend/poll_strategy/strategies/mod.rs +++ b/apalis-core/src/backend/poll_strategy/strategies/mod.rs @@ -1,9 +1,6 @@ -use std::pin::Pin; +use std::task::{Context, Poll}; -use futures_core::Stream; -use futures_util::StreamExt; - -use crate::backend::poll_strategy::{PollContext, PollStrategy}; +use crate::backend::poll_strategy::{PollSnapshot, PollStrategy}; mod stream; pub use stream::*; @@ -42,12 +39,7 @@ impl PollStrategy for WrapperStrategy where S: PollStrategy + Send + 'static, { - type Stream = Pin + Send>>; - - fn poll_strategy(self: Box, ctx: &PollContext) -> Self::Stream { - Box::new(self.strategy) - .poll_strategy(ctx) - .map(|_| ()) - .boxed() + fn poll_gate(&mut self, cx: &mut Context<'_>, worker: &PollSnapshot) -> Poll<()> { + self.strategy.poll_gate(cx, worker) } } diff --git a/apalis-core/src/backend/poll_strategy/strategies/stream.rs b/apalis-core/src/backend/poll_strategy/strategies/stream.rs index f3f6e407..a9061e74 100644 --- a/apalis-core/src/backend/poll_strategy/strategies/stream.rs +++ b/apalis-core/src/backend/poll_strategy/strategies/stream.rs @@ -1,6 +1,9 @@ +use std::task::{Context, Poll}; + use futures_core::Stream; +use futures_util::StreamExt; -use crate::backend::poll_strategy::{PollContext, PollStrategy}; +use crate::backend::poll_strategy::{PollSnapshot, PollStrategy}; /// A polling strategy that uses a provided stream #[derive(Debug, Clone)] @@ -10,7 +13,7 @@ pub struct StreamStrategy { impl StreamStrategy where - S: Stream + Unpin + Send + 'static, + S: Stream + Unpin + Send + 'static, { /// Create a new StreamStrategy from a stream pub fn new(stm: S) -> Self { @@ -18,13 +21,8 @@ where } } -impl PollStrategy for StreamStrategy -where - S: Stream + Unpin + Send + 'static, -{ - type Stream = S; - - fn poll_strategy(self: Box, _ctx: &PollContext) -> Self::Stream { - self.stm +impl PollStrategy for StreamStrategy { + fn poll_gate(&mut self, cx: &mut Context<'_>, _: &PollSnapshot) -> Poll<()> { + self.stm.poll_next_unpin(cx).map(|_| ()) } } diff --git a/apalis-core/src/backend/queue.rs b/apalis-core/src/backend/queue.rs index f9548961..4755c93f 100644 --- a/apalis-core/src/backend/queue.rs +++ b/apalis-core/src/backend/queue.rs @@ -71,15 +71,15 @@ impl<'de> serde::Deserialize<'de> for Queue { } } -impl FromRequest> for Queue +impl FromRequest> for Queue where Args: Sync, Conn: Send + Sync, - IdType: Sync + Send, + Id: Sync + Send, { type Error = QueueError; - async fn from_request(req: &Task) -> Result { + async fn from_request(req: &Task) -> Result { let queue = req.ctx.queue.clone().ok_or(QueueError::NotFound)?; Ok(queue) } diff --git a/apalis-core/src/backend/sink.rs b/apalis-core/src/backend/sink.rs index 5943e219..4fd2f469 100644 --- a/apalis-core/src/backend/sink.rs +++ b/apalis-core/src/backend/sink.rs @@ -1,5 +1,5 @@ use crate::{ - backend::{Backend, BackendExt, codec::Codec}, + backend::{Backend, codec::Codec}, error::BoxDynError, task::{Task, builder::TaskBuilder}, }; @@ -43,33 +43,36 @@ pub trait TaskSink: Backend { /// Allows pushing a fully constructed task into the backend fn push_task( &mut self, - task: Task, + task: Task, ) -> impl Future>> + Send; /// Allows pushing a fully constructed task into the backend fn push_all( &mut self, - tasks: impl Stream> + Unpin + Send, + tasks: impl Stream> + Unpin + Send, ) -> impl Future>> + Send; } impl TaskSink for S where - S: Sink, Error = E> + S: Sink, Error = E> + Unpin - + BackendExt + + Backend + Send, Args: Send, C::Compact: Send, S::Connection: Send + Sync, - S::IdType: Send + Sync + 'static, - C: Codec, + S::Id: Send + Sync + 'static, + C: Codec + Clone + Send + Sync, E: Send, C::Error: std::error::Error + Send + Sync + 'static, { async fn push(&mut self, task: Args) -> Result<(), TaskSinkError> { use futures_util::SinkExt; - let encoded = C::encode(&task).map_err(|e| TaskSinkError::CodecError(e.into()))?; + let encoded = self + .codec() + .encode(&task) + .map_err(|e| TaskSinkError::CodecError(e.into()))?; self.send(TaskBuilder::new(encoded).build()).await?; Ok(()) } @@ -80,8 +83,12 @@ where .into_iter() .map(TaskBuilder::new) .map(|task| { - task.try_map(|t| C::encode(&t).map_err(|e| TaskSinkError::CodecError(e.into()))) - .map(|t| t.build()) + task.try_map_args(|t| { + self.codec() + .encode(&t) + .map_err(|e| TaskSinkError::CodecError(e.into())) + }) + .map(|t| t.build()) }) .collect::, _>>()?; self.send_all(&mut stream::iter(tasks.into_iter().map(Ok))) @@ -93,38 +100,48 @@ where &mut self, tasks: impl Stream + Unpin + Send, ) -> Result<(), TaskSinkError> { + let codec = self.codec().clone(); self.sink_map_err(|e| TaskSinkError::PushError(e)) .send_all(&mut tasks.map(TaskBuilder::new).map(|task| { - task.try_map(|t| C::encode(&t).map_err(|e| TaskSinkError::CodecError(e.into()))) - .map(|t| t.build()) + task.try_map_args(|t| { + codec + .encode(&t) + .map_err(|e| TaskSinkError::CodecError(e.into())) + }) + .map(|t| t.build()) })) .await } async fn push_task( &mut self, - task: Task, + task: Task, ) -> Result<(), TaskSinkError> { use futures_util::SinkExt; + let codec = self.codec().clone(); + self.sink_map_err(|e| TaskSinkError::PushError(e)) - .send( - task.into_builder() - .try_map(|t| C::encode(&t).map_err(|e| TaskSinkError::CodecError(e.into()))) - .map(|t| t.build())?, - ) + .send(task.try_map_args(|t| { + codec + .encode(&t) + .map_err(|e| TaskSinkError::CodecError(e.into())) + })?) .await } async fn push_all( &mut self, - tasks: impl Stream> + Unpin + Send, + tasks: impl Stream> + Unpin + Send, ) -> Result<(), TaskSinkError> { use futures_util::SinkExt; + let codec = self.codec().clone(); self.sink_map_err(|e| TaskSinkError::PushError(e)) .send_all(&mut tasks.map(|task| { - task.into_builder() - .try_map(|t| C::encode(&t).map_err(|e| TaskSinkError::CodecError(e.into()))) - .map(|t| t.build()) + task.try_map_args(|t| { + codec + .encode(&t) + .map_err(|e| TaskSinkError::CodecError(e.into())) + }) })) .await } diff --git a/apalis-core/src/error.rs b/apalis-core/src/error.rs index 06d1c406..d96fbc43 100644 --- a/apalis-core/src/error.rs +++ b/apalis-core/src/error.rs @@ -60,14 +60,15 @@ pub struct DeferredError { } /// Possible errors that can occur when running a worker. +#[non_exhaustive] #[derive(Error, Debug)] pub enum WorkerError { - /// An error occurred while consuming the task stream. - #[error("Failed to consume task stream: {0}")] - StreamError(BoxDynError), - /// An error occurred in the worker's heartbeat. - #[error("Heartbeat error: {0}")] - HeartbeatError(BoxDynError), + /// An error occurred while polling for new tasks. + #[error("Failed to poll for new tasks: {0}")] + PollError(BoxDynError), + /// An error occurred while driving the backend. + #[error("The backend is cannot handle new polls: {0}")] + PollReadyError(BoxDynError), /// An error occurred while trying to change the state of the worker. #[error("Failed to handle the new state: {0}")] StateError(#[from] WorkerStateError), @@ -80,6 +81,10 @@ pub enum WorkerError { /// An error occurred while handling io #[error("IO error: {0}")] IoError(#[from] std::io::Error), + + /// Error originating from the decoding of the task + #[error("Task decoding error: {0}")] + CodecError(BoxDynError), } /// Errors related to worker state transitions diff --git a/apalis-core/src/lib.rs b/apalis-core/src/lib.rs index 0bbac214..e35a76b7 100644 --- a/apalis-core/src/lib.rs +++ b/apalis-core/src/lib.rs @@ -80,7 +80,7 @@ //! - `Layer` - Specifies the middleware layer stack for the backend //! - `Codec` - Determines serialization format for task data persistence //! - `Beat` - Heartbeat stream for worker liveness checks -//! - `IdType` - Type used for unique task identifiers +//! - `Id` - Type used for unique task identifiers //! - `Conn` - Context associated with tasks //! - `Error` - Error type for backend operations //! @@ -169,13 +169,13 @@ //! # use apalis_core::monitor::Monitor; //! # use apalis_core::worker::builder::WorkerBuilder; //! # use apalis_core::task::Task; -//! # use apalis_core::backend::{TaskSink, dequeue}; +//! # use apalis_core::backend::{TaskSink, dequeue::VecDequeBackend}; //! # use tower::service_fn; //! # use std::time::Duration; //! # use apalis_core::worker::context::WorkerContext; //! #[tokio::main] //! async fn main() { -//! let mut storage = dequeue::backend::(Duration::from_secs(1)); +//! let mut storage = VecDequeBackend::new(); //! storage.push(1u32).await.unwrap(); //! //! let monitor = Monitor::new() @@ -185,7 +185,7 @@ //! .backend(storage.clone()) //! .build(|req: u32, ctx: WorkerContext| async move { //! println!("Processing task: {:?}", req); -//! # ctx.stop().unwrap(); +//! # ctx.stop().unwrap(); //! Ok::<_, std::io::Error>(req) //! }) //! }); @@ -228,9 +228,9 @@ //! inner: S, //! } //! -//! impl Service> for LoggingService +//! impl Service> for LoggingService //! where -//! S: Service, Response = Res, Error = Err>, +//! S: Service, Response = Res, Error = Err>, //! Req: std::fmt::Debug, //! { //! type Response = Res; @@ -241,7 +241,7 @@ //! self.inner.poll_ready(cx) //! } //! -//! fn call(&mut self, req: Task) -> Self::Future { +//! fn call(&mut self, req: Task) -> Self::Future { //! println!("Processing task: {:?}", req.args); //! self.inner.call(req) //! } diff --git a/apalis-core/src/macros.rs b/apalis-core/src/macros.rs index 02daf7b1..6d9d2ed0 100644 --- a/apalis-core/src/macros.rs +++ b/apalis-core/src/macros.rs @@ -215,8 +215,8 @@ macro_rules! features_table { ) }; (@assert_function Serialization) => { concat!( " # use apalis_core::backend::codec::Codec;\n", - " # use apalis_core::backend::BackendExt;\n", - " fn assert_codec>(backend: B) \n", + " # use apalis_core::backend::Backend;\n", + " fn assert_codec>(backend: B) \n", " where\n", " B::Codec: Codec<(), Compact=Vec>,\n", " {\n", @@ -226,7 +226,7 @@ macro_rules! features_table { ) }; (@assert_function WaitForCompletion) => { concat!( "# use apalis_core::backend::WaitForCompletion;\n", - " fn assert_wait_for_completion>(backend: B) {};\n", + " fn assert_wait_for_completion>(backend: B) {};\n", " assert_wait_for_completion(backend);\n", "}\n" ) }; diff --git a/apalis-core/src/monitor/mod.rs b/apalis-core/src/monitor/mod.rs index e14a0762..9e5e5d79 100644 --- a/apalis-core/src/monitor/mod.rs +++ b/apalis-core/src/monitor/mod.rs @@ -146,7 +146,7 @@ use tower_layer::Layer; use tower_service::Service; use crate::{ - backend::Backend, + backend::{Backend, codec::Codec}, error::{BoxDynError, WorkerError}, monitor::shutdown::Shutdown, task::Task, @@ -286,23 +286,23 @@ impl Monitor { worker: Worker, ) -> BoxFuture<'static, Result<(), WorkerError>> where - S: Service> + Send + 'static, + S: Service> + Send + 'static, S::Future: Send, S::Error: Send + Sync + 'static + Into, - B: Backend + Send + 'static, + B: Backend + Send + Unpin + 'static, B::Error: Into + Send + 'static, B::Layer: Layer>> + 'static, M: Layer<<::Layer as Layer>>>::Service> + 'static, ::Layer as Layer>>>::Service, - >>::Service: Service> + Send + 'static, - <>>>::Service>>::Service as Service>>::Future: Send, - <>>>::Service>>::Service as Service>>::Error: Into + Send + Sync + 'static, - B::Stream: Unpin + Send + 'static, - B::Beat: Unpin + Send, + >>::Service: Service> + Send + 'static, + <>>>::Service>>::Service as Service>>::Future: Send, + <>>>::Service>>::Service as Service>>::Error: Into + Send + Sync + 'static, Args: Send + 'static, B::Connection: Send + Sync + 'static, - B::IdType: Sync + Send + 'static, + B::Id: Sync + Send + 'static, + >::Error: Into, + { let mut stream = worker.stream_with_ctx(&mut ctx); async move { @@ -342,24 +342,24 @@ impl Monitor { factory: impl Fn(usize) -> Worker + 'static + Send + Sync, ) -> Self where - S: Service> + Send + 'static, + S: Service> + Send + 'static, S::Future: Send, S::Error: Send + Sync + 'static + Into, - B: Backend + Send + 'static, + B: Backend + Send + Unpin + 'static, B::Error: Into + Send + 'static, - B::Stream: Unpin + Send + 'static, - B::Beat: Unpin + Send, Args: Send + 'static, B::Connection: Send + Sync + 'static, B::Layer: Layer>> + 'static, M: Layer<<::Layer as Layer>>>::Service> + 'static, ::Layer as Layer>>>::Service, - >>::Service: Service> + Send + 'static, - <>>>::Service>>::Service as Service>>::Future: Send, - <>>>::Service>>::Service as Service>>::Error: + >>::Service: Service> + Send + 'static, + <>>>::Service>>::Service as Service>>::Future: Send, + <>>>::Service>>::Service as Service>>::Error: Into + Send + Sync + 'static, - B::IdType: Send + Sync + 'static, + B::Id: Send + Sync + 'static, + >::Error: Into, + { let shutdown = Some(self.shutdown.clone()); let handler = self.event_handler.clone(); @@ -586,7 +586,7 @@ impl std::fmt::Display for ExitError { mod tests { use super::*; use crate::{ - backend::{TaskSink, dequeue::backend}, + backend::{TaskSink, dequeue::VecDequeBackend}, task::task_id::TaskId, worker::context::WorkerContext, }; @@ -600,7 +600,7 @@ mod tests { #[tokio::test] async fn basic_with_workers() { - let mut backend = backend(Duration::from_millis(100)); + let mut backend = VecDequeBackend::new(); for i in 0..10 { backend.push(i).await.unwrap(); @@ -625,7 +625,7 @@ mod tests { } #[tokio::test] async fn test_monitor_run() { - let mut backend = backend(Duration::from_millis(100)); + let mut backend = VecDequeBackend::new(); for i in 0..10 { backend.push(i).await.unwrap(); @@ -671,9 +671,9 @@ mod tests { #[tokio::test] async fn test_monitor_register_multiple() { - let mut int_backend = backend(Duration::from_millis(500)); + let mut int_backend = VecDequeBackend::new(); - let mut str_backend = backend(Duration::from_millis(500)); + let mut str_backend = VecDequeBackend::new(); for i in 0..10 { int_backend.push(i).await.unwrap(); diff --git a/apalis-core/src/task/attempt.rs b/apalis-core/src/task/attempt.rs index 6c9a5b8f..08751ea9 100644 --- a/apalis-core/src/task/attempt.rs +++ b/apalis-core/src/task/attempt.rs @@ -49,11 +49,13 @@ impl Attempt { } } -impl FromRequest> - for Attempt +impl FromRequest> for Attempt +where + Args: Sync, + Id: Sync + Send, { type Error = Infallible; - async fn from_request(task: &Task) -> Result { + async fn from_request(task: &Task) -> Result { Ok(task.ctx.attempt.clone()) } } diff --git a/apalis-core/src/task/builder.rs b/apalis-core/src/task/builder.rs index ed78eab6..8a5b6c11 100644 --- a/apalis-core/src/task/builder.rs +++ b/apalis-core/src/task/builder.rs @@ -44,16 +44,16 @@ use std::{ /// Builder for creating [`Task`] instances with optional configuration #[derive(Debug)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct TaskBuilder { +#[must_use = "TaskBuilder is used to construct a Task. Ensure to call `.build()` to create the Task instance."] +pub struct TaskBuilder { /// The arguments for the task pub args: Args, /// Execution context for the task, including metadata and extensions - pub ctx: ExecutionContext, + pub ctx: ExecutionContext, } -impl TaskBuilder { +impl TaskBuilder { /// Create a new TaskBuilder with the required args - #[must_use] pub fn new(args: Args) -> Self { Self { args, @@ -62,28 +62,24 @@ impl TaskBuilder { } /// Set the task's metadata - #[must_use] pub fn with_metadata(mut self, metadata: MetadataStore) -> Self { self.ctx.metadata = metadata; self } /// Set the task's runtime data - #[must_use] pub fn with_data(mut self, data: Extensions) -> Self { self.ctx.data = data; self } /// Insert a value into the task's data context - #[must_use] pub fn data(mut self, value: D) -> Self { self.ctx.data.insert(value); self } /// Insert a value into the task's metadata - #[must_use] pub fn metadata(mut self, value: &M) -> Self where M: Metadata, @@ -96,56 +92,48 @@ impl TaskBuilder { } /// Set the task ID - #[must_use] - pub fn task_id(mut self, task_id: TaskId) -> Self { + pub fn task_id(mut self, task_id: TaskId) -> Self { self.ctx.task_id = Some(task_id); self } /// Set the attempt information - #[must_use] pub fn attempt(mut self, attempt: Attempt) -> Self { self.ctx.attempt = attempt; self } /// Set the task status - #[must_use] pub fn status(mut self, status: Status) -> Self { self.ctx.status = status.into(); self } /// Set the maximum number of attempts allowed for the task - #[must_use] pub fn max_attempts(mut self, max_attempts: usize) -> Self { self.ctx.max_attempts = Some(max_attempts); self } /// Set the priority of the task - #[must_use] pub fn priority(mut self, priority: usize) -> Self { self.ctx.priority = Some(priority); self } /// Set the queue this task belongs to - #[must_use] pub fn queue(mut self, queue: Queue) -> Self { self.ctx.queue = Some(queue); self } /// Schedule the task to run at a specific Unix timestamp - #[must_use] pub fn run_at_timestamp(mut self, timestamp: u64) -> Self { self.ctx.run_at = Some(timestamp); self } /// Schedule the task to run at a specific SystemTime - #[must_use] pub fn run_at_time(mut self, time: SystemTime) -> Self { let timestamp = time .duration_since(UNIX_EPOCH) @@ -156,7 +144,6 @@ impl TaskBuilder { } /// Schedule the task to run after a delay from now - #[must_use] pub fn run_after(mut self, delay: Duration) -> Self { let now = SystemTime::now(); let run_time = now + delay; @@ -169,54 +156,46 @@ impl TaskBuilder { } /// Schedule the task to run in the specified number of seconds - #[must_use] pub fn run_in_seconds(self, seconds: u64) -> Self { self.run_after(Duration::from_secs(seconds)) } /// Schedule the task to run in the specified number of minutes - #[must_use] pub fn run_in_minutes(self, minutes: u64) -> Self { self.run_after(Duration::from_secs(minutes * 60)) } /// Schedule the task to run in the specified number of hours - #[must_use] pub fn run_in_hours(self, hours: u64) -> Self { self.run_after(Duration::from_secs(hours * 3600)) } /// Set the idempotency key - #[must_use] pub fn idempotency_key>(mut self, idempotency_key: S) -> Self { self.ctx.idempotency_key = Some(idempotency_key.as_ref().to_owned()); self } /// Set the time the task was completed - #[must_use] pub fn done_at(mut self, done_at: Option) -> Self { self.ctx.done_at = done_at; self } /// Set the time the task was locked - #[must_use] pub fn lock_at(mut self, lock_at: Option) -> Self { self.ctx.lock_at = lock_at; self } /// Set the worker/process identifier holding the lock on this task - #[must_use] pub fn lock_by(mut self, lock_by: Option) -> Self { self.ctx.lock_by = lock_by; self } /// Build the Task with default context - #[must_use] - pub fn build(self) -> Task { + pub fn build(self) -> Task { Task { args: self.args, ctx: Arc::new(self.ctx), @@ -224,9 +203,9 @@ impl TaskBuilder { } } -impl TaskBuilder { +impl TaskBuilder { /// Maps the `args` field using the provided function, consuming the task. - pub fn try_map(self, f: F) -> Result, Err> + pub fn try_map_args(self, f: F) -> Result, Err> where F: FnOnce(Args) -> Result, { @@ -236,7 +215,7 @@ impl TaskBuilder { }) } /// Maps the `args` field using the provided function, consuming the task. - pub fn map(self, f: F) -> TaskBuilder + pub fn map_args(self, f: F) -> TaskBuilder where F: FnOnce(Args) -> NewArgs, { @@ -246,27 +225,12 @@ impl TaskBuilder { } } - /// Maps both `args` and `execution_context` together. - pub fn map_all(self, f: F) -> TaskBuilder - where - F: FnOnce( - Args, - ExecutionContext, - ) -> (NewArgs, ExecutionContext), - { - let (args, execution_context) = f(self.args, self.ctx); - TaskBuilder { - args, - ctx: execution_context, - } - } - /// Maps only the `execution_context` field. - pub fn map_ctx(self, f: F) -> TaskBuilder + pub fn map_context(self, f: F) -> Self where - F: FnOnce(ExecutionContext) -> ExecutionContext, + F: FnOnce(ExecutionContext) -> ExecutionContext, { - TaskBuilder { + Self { args: self.args, ctx: f(self.ctx), } diff --git a/apalis-core/src/task/data.rs b/apalis-core/src/task/data.rs index b6483f5c..adb53cad 100644 --- a/apalis-core/src/task/data.rs +++ b/apalis-core/src/task/data.rs @@ -116,9 +116,9 @@ pub struct AddExtension { value: T, } -impl Service> for AddExtension +impl Service> for AddExtension where - S: Service>, + S: Service>, T: Clone + Send + Sync + 'static, { type Response = S::Response; @@ -130,7 +130,7 @@ where self.inner.poll_ready(cx) } - fn call(&mut self, mut task: Task) -> Self::Future { + fn call(&mut self, mut task: Task) -> Self::Future { if let Some(ctx) = Arc::get_mut(&mut task.ctx) { ctx.data.insert(self.value.clone()); } @@ -146,11 +146,11 @@ pub enum MissingDataError { NotFound(String), } -impl - FromRequest> for Data +impl + FromRequest> for Data { type Error = MissingDataError; - async fn from_request(task: &Task) -> Result { + async fn from_request(task: &Task) -> Result { task.ctx.data.get_checked().cloned().map(Self::new) } } diff --git a/apalis-core/src/task/metadata.rs b/apalis-core/src/task/metadata.rs index c3d1abd5..8fa3d442 100644 --- a/apalis-core/src/task/metadata.rs +++ b/apalis-core/src/task/metadata.rs @@ -296,24 +296,24 @@ pub trait Metadata: Sized { fn inject(&self, map: &mut MetadataStore) -> Result<(), Self::Error>; } -impl - FromRequest> for Meta +impl + FromRequest> for Meta { type Error = T::Error; - async fn from_request(task: &Task) -> Result { + async fn from_request(task: &Task) -> Result { let metadata = &task.ctx.metadata; let value = T::extract(metadata)?; Ok(Self(value)) } } -impl - FromRequest> for MetadataStore +impl FromRequest> + for MetadataStore { type Error = Infallible; - async fn from_request(task: &Task) -> Result { + async fn from_request(task: &Task) -> Result { Ok(task.ctx.metadata.clone()) } } @@ -545,10 +545,10 @@ mod tests { } } - impl - Service> for ExampleService + impl + Service> for ExampleService where - S: Service> + Clone + Send + 'static, + S: Service> + Clone + Send + 'static, Conn: Send, S::Future: Send + 'static, { @@ -560,7 +560,7 @@ mod tests { self.service.poll_ready(cx) } - fn call(&mut self, request: Task) -> Self::Future { + fn call(&mut self, request: Task) -> Self::Future { let mut svc = self.service.clone(); // Do something with config diff --git a/apalis-core/src/task/mod.rs b/apalis-core/src/task/mod.rs index b2660717..07a4ee85 100644 --- a/apalis-core/src/task/mod.rs +++ b/apalis-core/src/task/mod.rs @@ -15,7 +15,7 @@ //! The [`Task`] struct is generic over: //! - `Args`: The type of arguments or payload for the task. //! - `Conn`: Backend-specific marker for a task. -//! - `IdType`: The type used for uniquely identifying the task (defaults to [`RandomId`]). +//! - `Id`: The type used for uniquely identifying the task (defaults to [`RandomId`]). //! //! ## [`ExecutionContext`] //! @@ -139,7 +139,11 @@ //! [`IntoResponse`]: crate::task_fn::into_response::IntoResponse //! [`FromRequest`]: crate::task_fn::from_request::FromRequest -use std::{fmt::Debug, marker::PhantomData, sync::Arc}; +use std::{ + fmt::{Debug, Display}, + marker::PhantomData, + sync::Arc, +}; use crate::{ backend::queue::Queue, @@ -167,18 +171,18 @@ pub mod task_id; /// Represents a task which will be executed /// Should be considered a single unit of work #[derive(Debug, Clone, Default)] -pub struct Task { +pub struct Task { /// The argument task part pub args: Args, /// ExecutionContext of the task eg id, attempts and context - pub ctx: Arc>, + pub ctx: Arc>, } /// Execution context of a `Task` #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct ExecutionContext { +pub struct ExecutionContext { /// The task's id if allocated - pub task_id: Option>, + pub task_id: Option>, /// The tasks's extensions #[cfg_attr(feature = "serde", serde(skip))] @@ -222,10 +226,10 @@ pub struct ExecutionContext { pub runs: Vec, /// A marker to indicate the type of connection used by the backend. - pub connection: PhantomData, + pub connection: PhantomData Connection>, } -impl Default for ExecutionContext { +impl Default for ExecutionContext { fn default() -> Self { Self { task_id: None, @@ -247,7 +251,7 @@ impl Default for ExecutionContext { } } -impl Debug for ExecutionContext { +impl Debug for ExecutionContext { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("ExecutionContext") .field("task_id", &self.task_id) @@ -269,7 +273,7 @@ impl Debug for ExecutionContext { } } -impl Clone for ExecutionContext { +impl Clone for ExecutionContext { fn clone(&self) -> Self { Self { task_id: self.task_id.clone(), @@ -291,23 +295,95 @@ impl Clone for ExecutionContext { } } -impl Task { +impl Task { /// Take the task into its parts - pub fn take(self) -> (Args, Arc>) { + #[must_use] + pub fn take(self) -> (Args, Arc>) { (self.args, self.ctx) } /// Extract a value of type `T` from the task's context /// /// Uses [FromRequest] trait to extract the value. + #[must_use = "An extracted value should be used or handled to avoid unused value warnings."] pub async fn extract>(&self) -> Result { T::from_request(self).await } + /// Maps the `args` field using the provided function, consuming the task. + pub fn map_args(self, f: F) -> Task + where + F: FnOnce(Args) -> NewArgs, + { + Task { + args: f(self.args), + ctx: self.ctx, + } + } + + /// Maps the `args` field using the provided function, consuming the task. + #[must_use = "A mapped task should be used or handled to avoid unused value warnings."] + pub fn try_map_args(self, f: F) -> Result, Err> + where + F: FnOnce(Args) -> Result, + { + Ok(Task { + args: f(self.args)?, + ctx: self.ctx, + }) + } + + /// Maps the `execution_context` using the provided function, consuming the task + #[must_use] + pub fn map_context(self, f: F) -> Self + where + F: FnOnce(Arc>) -> Arc>, + { + Self { + args: self.args, + ctx: f(self.ctx), + } + } + + /// Modifies the relevant backend types, consuming the task + #[must_use] + pub fn map_backend(self) -> Task + where + Id: Clone, + Id: Display, + { + let mut ctx = Arc::unwrap_or_clone(self.ctx); + let _ = ctx.metadata.insert( + "apalis_core.transform.old_id", + ctx.task_id.map(|id| id.to_string()).unwrap_or_default(), + ); + Task { + args: self.args, + ctx: Arc::new(ExecutionContext { + task_id: None, + data: ctx.data, + attempt: ctx.attempt, + status: ctx.status, + run_at: ctx.run_at, + done_at: ctx.done_at, + lock_at: ctx.lock_at, + lock_by: ctx.lock_by, + idempotency_key: ctx.idempotency_key, + metadata: ctx.metadata, + max_attempts: ctx.max_attempts, + priority: ctx.priority, + queue: ctx.queue, + runs: ctx.runs, + connection: Default::default(), + }), + } + } + /// Converts the task into a [`TaskBuilder`] - pub fn into_builder(self) -> TaskBuilder + #[must_use = "Converting a task into a builder allows for further modifications before rebuilding the task."] + pub fn into_builder(self) -> TaskBuilder where - IdType: Clone, + Id: Clone, { TaskBuilder { args: self.args, diff --git a/apalis-core/src/task/runs.rs b/apalis-core/src/task/runs.rs index 83e0292b..427db92b 100644 --- a/apalis-core/src/task/runs.rs +++ b/apalis-core/src/task/runs.rs @@ -1,12 +1,10 @@ //! This module contains the definition of a `Run` struct, which represents a single execution attempt of a task. //! It includes information about the attempt number, the serialized value of the run, timestamps for when the task was locked and completed, and the identifier of the worker that executed the task. -use bytes::Bytes; - /// Represents a run of a task #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(Debug, Clone)] -pub struct Run { +pub struct Run> { /// The attempt number of the run pub attempt: usize, /// The serialized value of the run diff --git a/apalis-core/src/task/task_id.rs b/apalis-core/src/task/task_id.rs index 0ae6d03b..93dd98af 100644 --- a/apalis-core/src/task/task_id.rs +++ b/apalis-core/src/task/task_id.rs @@ -18,15 +18,15 @@ pub use random_id::RandomId; /// A wrapper type that defines a task id. #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(Debug, Clone, Copy, Eq, Hash, PartialEq, PartialOrd, Ord)] -pub struct TaskId(IdType); +pub struct TaskId(Id); -impl TaskId { +impl TaskId { /// Generate a new [`TaskId`] - pub fn new(id: IdType) -> Self { + pub fn new(id: Id) -> Self { Self(id) } /// Get the inner value - pub fn inner(&self) -> &IdType { + pub fn inner(&self) -> &Id { &self.0 } } @@ -39,32 +39,32 @@ pub enum TaskIdError { Decode(E), } -impl FromStr for TaskId { - type Err = TaskIdError; +impl FromStr for TaskId { + type Err = TaskIdError; fn from_str(s: &str) -> Result { - Ok(Self::new(IdType::from_str(s).map_err(TaskIdError::Decode)?)) + Ok(Self::new(Id::from_str(s).map_err(TaskIdError::Decode)?)) } } -impl TryFrom<&'_ str> for TaskId { - type Error = TaskIdError; +impl TryFrom<&'_ str> for TaskId { + type Error = TaskIdError; fn try_from(value: &'_ str) -> Result { Self::from_str(value) } } -impl Display for TaskId { +impl Display for TaskId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { Display::fmt(&self.0, f) } } -impl - FromRequest> for TaskId +impl FromRequest> + for TaskId { type Error = MissingDataError; - async fn from_request(task: &Task) -> Result { + async fn from_request(task: &Task) -> Result { task.ctx.task_id.clone().ok_or(MissingDataError::NotFound( std::any::type_name::().to_owned(), )) diff --git a/apalis-core/src/task_fn/from_request.rs b/apalis-core/src/task_fn/from_request.rs index 0789a7cd..430120fb 100644 --- a/apalis-core/src/task_fn/from_request.rs +++ b/apalis-core/src/task_fn/from_request.rs @@ -52,9 +52,9 @@ //! # } //! # } //! # struct Email { user_id: String }; -//! impl FromRequest> for User { +//! impl FromRequest> for User { //! type Error = BoxDynError; -//! async fn from_request(req: &Task) -> Result { +//! async fn from_request(req: &Task) -> Result { //! let user_id = &req.args.user_id; //! let user = User::find_by_id(user_id).await?; //! Ok(user) diff --git a/apalis-core/src/task_fn/guide.rs b/apalis-core/src/task_fn/guide.rs index 72fbe6fd..b2f2e3ff 100644 --- a/apalis-core/src/task_fn/guide.rs +++ b/apalis-core/src/task_fn/guide.rs @@ -105,9 +105,9 @@ //! # // other fields... //! # } //! -//! impl FromRequest> for User { +//! impl FromRequest> for User { //! type Error = BoxDynError; -//! async fn from_request(req: &Task) -> Result { +//! async fn from_request(req: &Task) -> Result { //! let user_id = req.args.user_id.clone(); //! // Simulate fetching user from DB //! Ok(User { id: user_id }) diff --git a/apalis-core/src/task_fn/mod.rs b/apalis-core/src/task_fn/mod.rs index 83556e23..01d594c3 100644 --- a/apalis-core/src/task_fn/mod.rs +++ b/apalis-core/src/task_fn/mod.rs @@ -134,14 +134,14 @@ type FnFuture = Map std::result::Result>; macro_rules! impl_service_fn { ($($K:ident),+) => { #[allow(unused_parens)] - impl Service> for TaskFn + impl Service> for TaskFn where T: FnMut(Args, $($K),+) -> F + Send + Clone + 'static, F: Future + Send, F::Output: IntoResponse, $( - $K: FromRequest> + Send, - < $K as FromRequest> >::Error: std::error::Error + 'static + Send + Sync, + $K: FromRequest> + Send, + < $K as FromRequest> >::Error: std::error::Error + 'static + Send + Sync, )+ { type Response = R; @@ -152,7 +152,7 @@ macro_rules! impl_service_fn { Poll::Ready(Ok(())) } - fn call(&mut self, task: Task) -> Self::Future { + fn call(&mut self, task: Task) -> Self::Future { let mut svc = self.f.clone(); #[allow(non_snake_case)] let fut = async move { @@ -179,11 +179,11 @@ macro_rules! impl_service_fn { F: Future + Send, Args: Send + 'static, Conn: Send + Sync + 'static, - B::IdType: Send + Sync + 'static, + B::Id: Send + Sync + 'static, F::Output: IntoResponse, $( - $K: FromRequest> + Send, - < $K as FromRequest> >::Error: std::error::Error + 'static + Send + Sync, + $K: FromRequest> + Send, + < $K as FromRequest> >::Error: std::error::Error + 'static + Send + Sync, )+ { type Backend = B; @@ -197,7 +197,7 @@ macro_rules! impl_service_fn { }; } -impl Service> for TaskFn +impl Service> for TaskFn where T: FnMut(Args) -> F, F: Future, @@ -211,7 +211,7 @@ where Poll::Ready(Ok(())) } - fn call(&mut self, task: Task) -> Self::Future { + fn call(&mut self, task: Task) -> Self::Future { let fut = (self.f)(task.args); fut.map(F::Output::into_response) @@ -238,7 +238,7 @@ where impl IntoWorkerService for S where - S: Service>, + S: Service>, B: Backend, { type Backend = B; diff --git a/apalis-core/src/worker/builder.rs b/apalis-core/src/worker/builder.rs index a6930401..7e9e982a 100644 --- a/apalis-core/src/worker/builder.rs +++ b/apalis-core/src/worker/builder.rs @@ -198,7 +198,7 @@ where /// Consumes the builder and a service to construct the final worker pub fn build(self, service: W) -> Worker where - Svc: Service>, + Svc: Service>, W: IntoWorkerServiceExt, { service.build_with(self) @@ -218,7 +218,7 @@ pub struct WorkerService { pub trait IntoWorkerService where B: crate::backend::Backend, - Svc: Service>, + Svc: Service>, { /// The backend type for the worker type Backend; @@ -230,7 +230,7 @@ where pub trait IntoWorkerServiceExt: Sized where Backend: crate::backend::Backend, - Svc: Service>, + Svc: Service>, Self: IntoWorkerService, { /// Consumes the builder and returns a worker @@ -248,7 +248,7 @@ impl IntoWorkerServiceExt for T where T: IntoWorkerService, B: Backend, - Svc: Service>, + Svc: Service>, { fn build_with( self, @@ -288,11 +288,11 @@ pub mod task_fn_validator { #[doc = concat!("A helper for checking that the builder can build a worker with the provided service (", stringify!($num), " arguments)")] pub fn $num< F, B, Args, Conn, - $($arg: FromRequest>),+ + $($arg: FromRequest>),+ >( _: F, ) where - TaskFn: Service>, + TaskFn: Service>, B: Backend { } diff --git a/apalis-core/src/worker/call_all.rs b/apalis-core/src/worker/call_all.rs index 810e05ec..a422f80c 100644 --- a/apalis-core/src/worker/call_all.rs +++ b/apalis-core/src/worker/call_all.rs @@ -1,11 +1,10 @@ -//! Utilities for executing all tasks from a stream to a service. +//! Utilities for executing all tasks from a backend to a service. //! -//! A combinator for calling all requests from a stream to a service, yielding responses +//! A combinator for calling all requests from a `Backend` to a service, yielding responses //! as they arrive. It supports both ordered and unordered response handling, allowing for flexible integration //! with asynchronous services. use futures_util::{Stream, ready, stream::FuturesUnordered}; use std::{ - error::Error, fmt, future::Future, pin::Pin, @@ -13,38 +12,45 @@ use std::{ }; use tower_service::Service; -use crate::error::BoxDynError; +use crate::{ + backend::{Backend, codec::Codec}, + error::BoxDynError, + task::Task, + worker::WorkerContext, +}; -/// A stream of responses received from the inner service in received order. +/// A stream of responses received from the inner service in received order, +/// driven by a `Backend` directly instead of a plain `Stream`. #[derive(Debug)] #[pin_project::pin_project] -pub(super) struct CallAllUnordered +pub(super) struct CallAllUnordered where - Svc: Service, - S: Stream, E>>, + Svc: Service::Args, ::Connection, ::Id>>, + B: Backend, { #[pin] - inner: CallAll, E>, + inner: CallAll>, } -impl CallAllUnordered +impl CallAllUnordered where - Svc: Service, - S: Stream, E>>, + Svc: Service>, + B: Backend + Unpin, { /// Create new [`CallAllUnordered`] combinator. - pub(super) fn new(service: Svc, stream: S) -> Self { + pub(super) fn new(service: Svc, backend: B, worker: WorkerContext) -> Self { Self { - inner: CallAll::new(service, stream, FuturesUnordered::new()), + inner: CallAll::new(service, backend, worker, FuturesUnordered::new()), } } } -impl Stream for CallAllUnordered +impl Stream for CallAllUnordered where - Svc: Service, - S: Stream, E>>, - E: Into, + Svc: Service>, + B: Backend + Unpin, + B::Error: Into, + >::Error: Into, { type Item = Result, CallAllError>; @@ -53,31 +59,18 @@ where } } -/// Error type that combines stream errors and service errors -#[derive(Debug)] +/// Error type that combines backend errors and service errors +#[derive(Debug, thiserror::Error)] pub enum CallAllError { - /// Error originating from the request stream - StreamError(BoxDynError), + /// Error originating from `Backend::poll_ready` or `Backend::poll` + #[error("Backend error: {0}")] + PollError(BoxDynError), /// Error originating from the service + #[error("Service error: {0}")] ServiceError(ServiceError), -} - -impl fmt::Display for CallAllError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::StreamError(e) => write!(f, "Stream error: {e}"), - Self::ServiceError(e) => write!(f, "Service error: {e}"), - } - } -} - -impl Error for CallAllError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::StreamError(e) => Some(e.as_ref()), - Self::ServiceError(e) => Some(e), - } - } + /// Error originating from the decoding of the task + #[error("Task decoding error: {0}")] + CodecError(BoxDynError), } impl Drive for FuturesUnordered { @@ -94,30 +87,32 @@ impl Drive for FuturesUnordered { } } -/// The [`Future`] returned by the [`ServiceExt::call_all`](https://docs.rs/tower/latest/tower/trait.ServiceExt.html#method.call_all) combinator. -/// +/// The [`Future`]/[`Stream`] that sequences `backend.poll_ready`, +/// `svc.poll_ready`, and `backend.poll` on every iteration, guaranteeing +/// the backend never claims a task the service isn't yet ready to accept. #[pin_project::pin_project] -pub(crate) struct CallAll +pub(crate) struct CallAll where - S: Stream, E>>, + B: Backend, { service: Option, #[pin] - stream: S, + backend: B, + worker: WorkerContext, queue: Q, eof: bool, - curr_req: Option, + curr_req: Option>, } -impl fmt::Debug for CallAll +impl fmt::Debug for CallAll where Svc: fmt::Debug, - S: Stream, E>> + fmt::Debug, + B: Backend + fmt::Debug, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("CallAll") .field("service", &self.service) - .field("stream", &self.stream) + .field("backend", &self.backend) .field("eof", &self.eof) .finish() } @@ -131,16 +126,17 @@ pub(crate) trait Drive { fn poll(&mut self, cx: &mut Context<'_>) -> Poll>; } -impl CallAll +impl CallAll where - Svc: Service, - S: Stream, E>>, + Svc: Service>, + B: Backend + Unpin, Q: Drive, { - pub(crate) const fn new(service: Svc, stream: S, queue: Q) -> Self { + pub(crate) const fn new(service: Svc, backend: B, worker: WorkerContext, queue: Q) -> Self { Self { service: Some(service), - stream, + backend, + worker, queue, eof: false, curr_req: None, @@ -148,12 +144,13 @@ where } } -impl Stream for CallAll +impl Stream for CallAll where - Svc: Service, - S: Stream, E>>, + Svc: Service>, + B: Backend + Unpin, Q: Drive, - E: Into, + B::Error: Into, + >::Error: Into, { type Item = Result, CallAllError>; @@ -166,43 +163,65 @@ where return Poll::Ready(Some(result.map_err(CallAllError::ServiceError).map(Some))); } - // If there are no more requests coming, check if we're done - if *this.eof { - if this.queue.is_empty() { - return Poll::Ready(None); - } else { + // Shutdown requested (or backend naturally exhausted via eof) AND + // no in-flight work left: time to close. + let shutting_down = *this.eof || this.worker.is_shutting_down(); + + if shutting_down { + if !this.queue.is_empty() { return Poll::Pending; } + return match this.backend.as_mut().get_mut().poll_close(cx, this.worker) { + Poll::Pending => Poll::Pending, + Poll::Ready(Err(e)) => { + Poll::Ready(Some(Err(CallAllError::PollError(e.into())))) + } + Poll::Ready(Ok(())) => Poll::Ready(None), + }; } - // Then, see that the service is ready for another request let svc = this .service .as_mut() .expect("Using CallAll after extracting inner Service"); + // 1. backend.poll_ready — must succeed before we even ask svc. + match this.backend.as_mut().get_mut().poll_ready(cx, this.worker) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => { + *this.eof = true; + return Poll::Ready(Some(Err(CallAllError::PollError(e.into())))); + } + Poll::Ready(Ok(())) => {} + } + + // 2. svc.poll_ready — still gates step 3, same as before. if let Err(e) = ready!(svc.poll_ready(cx)) { - // Set eof to prevent the service from being called again after a `poll_ready` error *this.eof = true; return Poll::Ready(Some(Err(CallAllError::ServiceError(e)))); } - // If not done, and we don't have a stored request, gather the next request from the - // stream (if there is one), or return `Pending` if the stream is not ready. + // 3. backend.poll — only now pull a task, having confirmed both + // the backend and the service are ready. if this.curr_req.is_none() { - match ready!(this.stream.as_mut().poll_next(cx)) { - Some(Ok(Some(next_req))) => { - *this.curr_req = Some(next_req); - } - Some(Ok(None)) => { - return Poll::Ready(Some(Ok(None))); + match ready!(this.backend.as_mut().get_mut().poll_next(cx, this.worker)) { + Some(Ok(next_req)) => { + let codec = this.backend.codec(); + let res = next_req.try_map_args(|args| codec.decode(&args)); + match res { + Err(e) => { + *this.eof = true; + return Poll::Ready(Some(Err(CallAllError::CodecError(e.into())))); + } + Ok(next) => { + *this.curr_req = Some(next); + } + } } Some(Err(e)) => { - // Stream error, propagate it - return Poll::Ready(Some(Err(CallAllError::StreamError(e.into())))); + return Poll::Ready(Some(Err(CallAllError::PollError(e.into())))); } None => { - // Stream is exhausted, mark EOF *this.eof = true; continue; } diff --git a/apalis-core/src/worker/context.rs b/apalis-core/src/worker/context.rs index d7ff9920..94d3efd0 100644 --- a/apalis-core/src/worker/context.rs +++ b/apalis-core/src/worker/context.rs @@ -78,6 +78,7 @@ use crate::{ pub struct WorkerContext { pub(super) name: Arc, task_count: Arc, + /// The waker used to wake the worker when tasks complete or shutdown is triggered. waker: Arc>>, state: Arc, pub(crate) shutdown: Option, @@ -160,7 +161,6 @@ impl WorkerContext { self.state .store(InnerWorkerState::Running, Ordering::SeqCst); self.is_ready.store(false, Ordering::SeqCst); - self.emit(&Event::Start); info!("Worker {} started", self.name()); Ok(()) } @@ -296,27 +296,20 @@ impl WorkerContext { self.event_handler = Arc::new(new); } - pub(crate) fn add_waker(&self, cx: &Context<'_>) { - if let Ok(mut waker_guard) = self.waker.lock() { - if waker_guard + /// Register the current waker for the worker + /// + /// This is used to wake the worker when tasks complete or shutdown is triggered. + pub(crate) fn register_waker(&self, cx: &Context<'_>) { + if let Ok(mut guard) = self.waker.lock() { + if guard .as_ref() - .is_none_or(|stored_waker| !stored_waker.will_wake(cx.waker())) + .is_none_or(|stored| !stored.will_wake(cx.waker())) { - *waker_guard = Some(cx.waker().clone()); + *guard = Some(cx.waker().clone()); } } } - /// Checks if the stored waker matches the current one. - fn has_recent_waker(&self, cx: &Context<'_>) -> bool { - if let Ok(waker_guard) = self.waker.lock() { - if let Some(stored_waker) = &*waker_guard { - return stored_waker.will_wake(cx.waker()); - } - } - false - } - fn start_task(&self) { self.task_count.fetch_add(1, Ordering::Relaxed); } @@ -336,31 +329,54 @@ impl WorkerContext { } } -impl Future for WorkerContext { +/// Internal handle used to await worker shutdown completion. +/// +/// This owns the `Future` impl that used to live on `WorkerContext` directly. +/// `WorkerContext` itself is `Clone` and handed out freely to backends, +/// middleware, and user code — it deliberately does NOT implement `Future`, +/// so nothing outside this module can accidentally spawn a second task +/// awaiting worker completion with its own independent waker. Only +/// `stream_with_ctx`'s single unified `stream_select!` loop drives shutdown +/// awaiting, via this private handle. +pub(super) struct WorkerHandle { + inner: WorkerContext, +} + +impl WorkerHandle { + pub(super) fn new(ctx: WorkerContext) -> Self { + Self { inner: ctx } + } +} + +impl Future for WorkerHandle { type Output = Result<(), WorkerError>; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let task_count = self.task_count.load(Ordering::Relaxed); - let state = self.state.load(Ordering::SeqCst); + let this = &self.inner; + let task_count = this.task_count.load(Ordering::Relaxed); + let state = this.state.load(Ordering::SeqCst); + if state == InnerWorkerState::Pending { return Poll::Ready(Err(WorkerError::StateError(WorkerStateError::NotStarted))); } - if self.is_shutting_down() && task_count == 0 { + if this.is_shutting_down() && task_count == 0 { Poll::Ready(Ok(())) } else { - if !self.has_recent_waker(cx) { - self.add_waker(cx); - } + // Same single shared waker slot as before — sound here because + // this is the only place in the codebase that ever polls a + // worker-completion future or drives backend polling for this + // worker; both live under the one stream_select! task below. + this.register_waker(cx); Poll::Pending } } } -impl FromRequest> +impl FromRequest> for WorkerContext { type Error = MissingDataError; - async fn from_request(task: &Task) -> Result { + async fn from_request(task: &Task) -> Result { task.ctx.data.get_checked().cloned() } } diff --git a/apalis-core/src/worker/ext/ack/mod.rs b/apalis-core/src/worker/ext/ack/mod.rs index febdeee6..e5df07dd 100644 --- a/apalis-core/src/worker/ext/ack/mod.rs +++ b/apalis-core/src/worker/ext/ack/mod.rs @@ -35,13 +35,13 @@ //! #[derive(Debug, Clone)] //! struct MyAcknowledger; //! -//! impl Acknowledge<(), Conn, IdType> for MyAcknowledger { +//! impl Acknowledge<(), Conn, Id> for MyAcknowledger { //! type Error = SendError<()>; //! type Future = BoxFuture<'static, Result<(), Self::Error>>; //! fn ack( //! &mut self, //! res: &Result<(), BoxDynError>, -//! ctx: &ExecutionContext, +//! ctx: &ExecutionContext, //! ) -> Self::Future { //! println!("{res:?}, {ctx:?}"); //! ready(Ok(())).boxed() @@ -77,7 +77,7 @@ use crate::{ pub trait AcknowledgementExt: Sized where Source: Backend, - Ack: Acknowledge, + Ack: Acknowledge, { /// Add an acknowledgment handler to the worker fn ack_with( @@ -89,7 +89,7 @@ where /// Acknowledge the result of a task processing /// /// See [module level documentation](self) for more details. -pub trait Acknowledge { +pub trait Acknowledge { /// The error type returned by the acknowledgment process type Error; /// The future returned by the `ack` method @@ -98,13 +98,13 @@ pub trait Acknowledge { fn ack( &mut self, res: &Result, - ctx: &ExecutionContext, + ctx: &ExecutionContext, ) -> Self::Future; } -impl Acknowledge for F +impl Acknowledge for F where - F: FnMut(&Result, &ExecutionContext) -> Fut, + F: FnMut(&Result, &ExecutionContext) -> Fut, Fut: Future>, { type Error = E; @@ -113,7 +113,7 @@ where fn ack( &mut self, res: &Result, - ctx: &ExecutionContext, + ctx: &ExecutionContext, ) -> Self::Future { (self)(res, ctx) } @@ -158,17 +158,17 @@ pub struct AcknowledgeService { acknowledger: A, } -impl Service> for AcknowledgeService +impl Service> for AcknowledgeService where - S: Service, Response = Res>, - A: Acknowledge + Clone + Send + 'static, + S: Service, Response = Res>, + A: Acknowledge + Clone + Send + 'static, S::Error: Into, A::Error: std::error::Error + Send + Sync + 'static, S::Future: Send + 'static, A::Future: Send + 'static, Conn: Clone + Send + Sync + 'static + Send, Res: Send, - IdType: Send + Sync + 'static, + Id: Send + Sync + 'static, { type Response = Res; type Error = BoxDynError; @@ -178,7 +178,7 @@ where self.inner.poll_ready(cx).map_err(|e| e.into()) } - fn call(&mut self, task: Task) -> Self::Future { + fn call(&mut self, task: Task) -> Self::Future { let ctx = task.ctx.clone(); let worker: WorkerContext = ctx.data.get().cloned().unwrap(); let future = self.inner.call(task); @@ -195,7 +195,7 @@ impl AcknowledgementExt for WorkerBuilder where M: Layer>, - Ack: Acknowledge, + Ack: Acknowledge, B: Backend, { fn ack_with(self, ack: Ack) -> WorkerBuilder, M>> { diff --git a/apalis-core/src/worker/ext/event_listener/mod.rs b/apalis-core/src/worker/ext/event_listener/mod.rs index 739e70d6..192f8e7d 100644 --- a/apalis-core/src/worker/ext/event_listener/mod.rs +++ b/apalis-core/src/worker/ext/event_listener/mod.rs @@ -64,9 +64,9 @@ pub struct EventListenerService { service: S, } -impl Service> for EventListenerService +impl Service> for EventListenerService where - S: Service>, + S: Service>, { type Response = S::Response; type Error = S::Error; @@ -79,7 +79,7 @@ where self.service.poll_ready(cx) } - fn call(&mut self, request: Task) -> Self::Future { + fn call(&mut self, request: Task) -> Self::Future { self.service.call(request) } } diff --git a/apalis-core/src/worker/ext/long_running/mod.rs b/apalis-core/src/worker/ext/long_running/mod.rs index 4a2eef54..e039329b 100644 --- a/apalis-core/src/worker/ext/long_running/mod.rs +++ b/apalis-core/src/worker/ext/long_running/mod.rs @@ -216,11 +216,9 @@ impl RunnerContext { } } -impl FromRequest> - for Runner -{ +impl FromRequest> for Runner { type Error = MissingDataError; - async fn from_request(task: &Task) -> Result { + async fn from_request(task: &Task) -> Result { let runner: &Self = task.ctx.data.get_checked()?; Ok(runner.clone()) } @@ -261,9 +259,9 @@ pub struct LongRunningService { config: LongRunningConfig, } -impl Service> for LongRunningService +impl Service> for LongRunningService where - S: Service>, + S: Service>, S::Future: Send + 'static, S::Response: Send, S::Error: Send, @@ -279,7 +277,7 @@ where self.service.poll_ready(cx) } - fn call(&mut self, mut task: Task) -> Self::Future { + fn call(&mut self, mut task: Task) -> Self::Future { let tracker = TaskTracker::new(); let worker: WorkerContext = task .ctx diff --git a/apalis-core/src/worker/ext/parallelize/mod.rs b/apalis-core/src/worker/ext/parallelize/mod.rs index 5c224a75..d57fc6a9 100644 --- a/apalis-core/src/worker/ext/parallelize/mod.rs +++ b/apalis-core/src/worker/ext/parallelize/mod.rs @@ -76,10 +76,10 @@ pub struct ParallelizeService { executor: Executor, } -impl Service> +impl Service> for ParallelizeService where - S: Service, Future = Fut>, + S: Service, Future = Fut>, Executor: Fn(Fut) -> T + Send + 'static, Fut: Future> + Send + 'static, T: Future, ExecErr>> + Send + 'static, @@ -98,7 +98,7 @@ where self.service.poll_ready(cx).map_err(|e| e.into()) } - fn call(&mut self, request: Task) -> Self::Future { + fn call(&mut self, request: Task) -> Self::Future { (self.executor)(self.service.call(request)) .map_err(|e| e.into()) .and_then(|s| ready(s.map_err(|e| e.into()))) diff --git a/apalis-core/src/worker/mod.rs b/apalis-core/src/worker/mod.rs index 07054d39..9635eb24 100644 --- a/apalis-core/src/worker/mod.rs +++ b/apalis-core/src/worker/mod.rs @@ -86,13 +86,14 @@ //! # Test Utilities //! The [`test_worker`] module includes utilities for unit tests and validation of worker behavior. use crate::backend::Backend; +use crate::backend::codec::Codec; use crate::error::{BoxDynError, WorkerError}; use crate::monitor::shutdown::Shutdown; use crate::task::Task; use crate::task::attempt::Attempt; use crate::task::data::Data; use crate::worker::call_all::{CallAllError, CallAllUnordered}; -use crate::worker::context::{Tracked, WorkerContext}; +use crate::worker::context::{Tracked, WorkerContext, WorkerHandle}; use crate::worker::event::{Event, RawEventListener}; use futures_core::stream::BoxStream; use futures_util::{Future, FutureExt, Stream, StreamExt, TryFutureExt}; @@ -184,20 +185,20 @@ impl Worker { impl Worker where - B: Backend, - S: Service> + Send + 'static, - B::Stream: Unpin + Send + 'static, - B::Beat: Unpin + Send + 'static, + B: Backend + Send + Unpin + 'static, + S: Service> + Send + 'static, Args: Send + 'static, B::Connection: Send + Sync + 'static, B::Error: Into + Send + 'static, B::Layer: Layer>>, M: Layer<>>>::Service>, >>>::Service>>::Service: - Service> + Send + 'static, - <>>>::Service>>::Service as Service>>::Future: Send, - <>>>::Service>>::Service as Service>>::Error: Into + Send + Sync + 'static, - B::IdType: Send + Sync + 'static, + Service> + Send + 'static, + <>>>::Service>>::Service as Service>>::Future: Send, + <>>>::Service>>::Service as Service>>::Error: Into + Send + Sync + 'static, + B::Id: Send + Sync + 'static, + >::Error: Into, + { /// Run the worker until completion /// @@ -357,15 +358,13 @@ where /// with a [`WorkerContext`]. /// /// See [`stream`](Self::stream) for an example. - pub fn stream_with_ctx( - self, - ctx: &mut WorkerContext, - ) -> impl Stream> + use { + pub fn stream_with_ctx(self, ctx: &mut WorkerContext) -> impl Stream> + use { let backend = self.backend; let event_handler = self.event_handler; ctx.wrap_listener(event_handler); let worker = ctx.clone(); let backend_middleware = backend.middleware(); + struct ServiceBuilder { layer: L, } @@ -383,6 +382,7 @@ where self.layer.layer(service) } } + let svc = ServiceBuilder { layer: Data::new(worker.clone()), }; @@ -398,31 +398,38 @@ where // this ensures that the attempt count is accurate even if the task fails before completion .layer(TrackerLayer::new(worker.clone())) .service(self.service); - let heartbeat = backend.heartbeat(&worker).map(|res| match res { - Ok(_) => Ok(Event::HeartBeat), - Err(e) => Err(WorkerError::HeartbeatError(e.into())), - }); - let stream = backend.poll(&worker); + // One `poll_fn` owns `backend` and drives poll_ready -> poll_compact in + // order every time it's polled, registering the current task's waker + // into `worker` first so a backend's internal future resolving can wake + // this exact task. + let worker_ctx = worker.clone(); + + let tasks = Self::poll_tasks(service, backend, worker_ctx); - let tasks = Self::poll_tasks(service, stream); let mut w = worker.clone(); let mut ww = w.clone(); let starter: BoxStream<'static, _> = futures_util::stream::once(async move { - if !ww.is_running() { - ww.start()?; - } - Ok(None) + + ww.start()?; + Ok(Some(Event::Start)) }) .filter_map(|res: Result, WorkerError>| async move { + match res { - Ok(_) => None, + Ok(res) => res.map(Ok), Err(e) => Some(Err(e)), } }) .boxed(); + + // Shutdown-completion awaiting now goes through the private `WorkerHandle` + // instead of polling `WorkerContext` directly — `WorkerContext` no + // longer implements `Future` at all, so this is the only path capable of + // awaiting worker completion. + let handle = WorkerHandle::new(worker); let wait_for_exit: BoxStream<'static, _> = futures_util::stream::once(async move { - match worker.await { + match handle.await { Ok(_) => Err(WorkerError::GracefulExit), Err(e) => Err(e), } @@ -430,33 +437,35 @@ where .boxed(); #[allow(clippy::needless_continue)] - let work_stream = - futures_util::stream_select!(wait_for_exit, heartbeat, tasks).map(move |res| { - if let Ok(e) = &res { - w.emit(e); - } - res - }); - starter.chain(work_stream) - } - fn poll_tasks( + let work_stream = futures_util::stream_select!(wait_for_exit, tasks); + + starter.chain(work_stream).map(move |res| { + if let Ok(e) = &res { + w.emit(e); + } + res + }) +} + fn poll_tasks( service: Svc, - stream: Stm, + backend: B, + worker: WorkerContext, ) -> BoxStream<'static, Result> where - Svc: Service> + Send + 'static, - Stm: Stream>, E>> + Send + Unpin + 'static, + Svc: Service> + Send + 'static, Args: Send + 'static, Svc::Future: Send, - Conn: Send + Sync + 'static, + B::Connection: Send + Sync + 'static, Svc::Error: Into + Sync + Send, - E: Into + Send + 'static, + >::Error: Into, + { - let stream = CallAllUnordered::new(service, stream).map(|r| match r { + let stream = CallAllUnordered::new(service, backend, worker).map(|r| match r { Ok(Some(_)) => Ok(Event::Success), Ok(None) => Ok(Event::Idle), Err(CallAllError::ServiceError(err)) => Ok(Event::Error(err.into().into())), - Err(CallAllError::StreamError(err)) => Err(WorkerError::StreamError(err)), + Err(CallAllError::PollError(err)) => Err(WorkerError::PollError(err)), + Err(CallAllError::CodecError(err)) => Err(WorkerError::CodecError(err)), }); stream.boxed() } @@ -490,9 +499,9 @@ pub struct TrackerService { service: S, } -impl Service> for TrackerService +impl Service> for TrackerService where - S: Service>, + S: Service>, { type Response = S::Response; type Error = S::Error; @@ -502,7 +511,7 @@ where self.service.poll_ready(cx) } - fn call(&mut self, task: Task) -> Self::Future { + fn call(&mut self, task: Task) -> Self::Future { let attempt = task.ctx.attempt.clone(); self.ctx.track(AttemptOnPollFuture { attempt, @@ -670,13 +679,13 @@ mod tests { #[derive(Debug, Clone)] struct MyAcknowledger; - impl Acknowledge<(), Conn, IdType> for MyAcknowledger { + impl Acknowledge<(), Conn, Id> for MyAcknowledger { type Error = SendError; type Future = BoxFuture<'static, Result<(), SendError>>; fn ack( &mut self, res: &Result<(), BoxDynError>, - ctx: &ExecutionContext, + ctx: &ExecutionContext, ) -> Self::Future { println!("{res:?}, {ctx:?}"); // Call webhook with the result and ctx? diff --git a/apalis-core/src/worker/test_worker.rs b/apalis-core/src/worker/test_worker.rs index 3bb95ae0..7511a955 100644 --- a/apalis-core/src/worker/test_worker.rs +++ b/apalis-core/src/worker/test_worker.rs @@ -35,6 +35,7 @@ //! This module is intended for use in tests and local development. use crate::backend::Backend; +use crate::backend::codec::Codec; use crate::error::BoxDynError; use crate::task::Task; use crate::task::task_id::{RandomId, TaskId}; @@ -51,8 +52,8 @@ use std::task::{Context, Poll}; use tower_layer::Layer; use tower_service::Service; -type TestStream = - BoxStream<'static, Result<(TaskId, Result), WorkerError>>; +type TestStream = + BoxStream<'static, Result<(TaskId, Result), WorkerError>>; /// A test worker to allow you to test services. /// Important for testing backends and tasks /// # Example @@ -80,19 +81,19 @@ type TestStream = /// } ///} /// ```` -pub struct TestWorker { - stream: TestStream, +pub struct TestWorker { + stream: TestStream, backend: PhantomData, service: PhantomData<(S, Res)>, } -impl fmt::Debug for TestWorker { +impl fmt::Debug for TestWorker { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("TestWorker") .field("stream", &"BoxStream<...>") // can't really debug streams .field("backend", &std::any::type_name::()) .field("service", &std::any::type_name::<(S, Res)>()) - .field("id_type", &std::any::type_name::()) + .field("id_type", &std::any::type_name::()) .finish() } } @@ -106,13 +107,13 @@ pub trait ExecuteNext { fn execute_next(&mut self) -> impl Future + Send; } -impl ExecuteNext for TestWorker +impl ExecuteNext for TestWorker where - S: Service, Response = Res> + Send + 'static, + S: Service, Response = Res> + Send + 'static, B: Send, Res: Send, { - type Result = Option, Result), WorkerError>>; + type Result = Option, Result), WorkerError>>; async fn execute_next(&mut self) -> Self::Result { self.stream.next().await } @@ -120,44 +121,38 @@ where impl TestWorker { /// Create a new test worker - pub fn new(backend: B, factory: W) -> TestWorker + pub fn new(backend: B, factory: W) -> TestWorker where W: IntoWorkerService, - W::Backend: Backend< - Args = B::Args, - Connection = B::Connection, - IdType = B::IdType, - Error = B::Error, - Stream = B::Stream, - Beat = B::Beat, - Layer = B::Layer, - > + 'static, - B: Backend + 'static, - S: Service, Response = Res> + Send + 'static, - B::Stream: Unpin + Send + 'static, - B::Beat: Unpin + Send + 'static, + W::Backend: Backend + + Unpin + + Send + + 'static, + B: Backend + Send + Unpin + 'static, + S: Service, Response = Res> + Send + 'static, Args: Send + 'static, Conn: Send + Sync + 'static, B::Error: Into + Send + 'static, - B::Layer: Layer>>>, + B::Layer: Layer>>>, S::Future: Send, S::Error: Into + Send + Sync, S::Response: Clone + Send, Res: 'static, <::Layer as Layer< - ReadinessService>>, - >>::Service: Service>, + ReadinessService>>, + >>::Service: Service>, <<::Layer as Layer< - ReadinessService>>, - >>::Service as Service>>::Error: - Into + Sync + Send, + ReadinessService>>, + >>::Service as Service>>::Error: Into + Sync + Send, <<::Layer as Layer< - ReadinessService>>, - >>::Service as Service>>::Future: Send, + ReadinessService>>, + >>::Service as Service>>::Future: Send, <::Layer as Layer< - ReadinessService>>, + ReadinessService>>, >>::Service: std::marker::Send + 'static, - B::IdType: Send + Sync + 'static, + B::Id: Send + Sync + 'static, + >::Error: std::error::Error + Send + Sync + 'static, + <<>::Backend as Backend>::Codec as Codec>::Error: std::error::Error + Send + Sync + 'static { let worker_service = factory.into_service(backend); TestWorker::::new_with_svc( @@ -169,42 +164,40 @@ impl TestWorker { impl TestWorker { /// Create a new test worker with a service - pub fn new_with_svc(backend: B, service: S) -> TestWorker + pub fn new_with_svc(backend: B, service: S) -> TestWorker where - B: Backend + 'static, - S: Service, Response = Res> + Send + 'static, - B::Stream: Unpin + Send + 'static, - B::Beat: Unpin + Send + 'static, + B: Backend + Send + Unpin + 'static, + S: Service, Response = Res> + Send + 'static, Args: Send + 'static, Conn: Send + Sync + 'static, B::Error: Into + Send + 'static, - B::Layer: Layer>>>, + B::Layer: Layer>>>, S::Future: Send, S::Error: Into + Send + Sync, S::Response: Clone + Send, Res: 'static, <::Layer as Layer< - ReadinessService>>, - >>::Service: Service>, + ReadinessService>>, + >>::Service: Service>, <<::Layer as Layer< - ReadinessService>>, - >>::Service as Service>>::Error: - Into + Sync + Send, + ReadinessService>>, + >>::Service as Service>>::Error: Into + Sync + Send, <<::Layer as Layer< - ReadinessService>>, - >>::Service as Service>>::Future: Send, + ReadinessService>>, + >>::Service as Service>>::Future: Send, <::Layer as Layer< - ReadinessService>>, + ReadinessService>>, >>::Service: std::marker::Send + 'static, - B::IdType: Send + Sync + 'static, + B::Id: Send + Sync + 'static, + >::Error: std::error::Error + Send + Sync + 'static, { - enum Item { + enum Item { Ev(Result), - Res((TaskId, Result)), + Res((TaskId, Result)), } let (tx, rx) = channel(1); let sender = tx.clone(); - let service: TestEmitService = TestEmitService { + let service: TestEmitService = TestEmitService { service, tx: tx.clone(), }; @@ -247,21 +240,20 @@ impl TestWorker { /// A generic service that emits the result of a test #[derive(Debug, Clone)] -pub struct TestEmitService { - tx: mpsc::Sender<(TaskId, Result)>, +pub struct TestEmitService { + tx: mpsc::Sender<(TaskId, Result)>, service: S, } -impl Service> - for TestEmitService +impl Service> for TestEmitService where - S: Service, Response = Res> + Send + 'static, + S: Service, Response = Res> + Send + 'static, S::Future: Send + 'static, Args: Send + 'static, Conn: Send + 'static, S::Response: Send + Clone + 'static, S::Error: Into + Send, - IdType: Send + 'static + Clone, + Id: Send + 'static + Clone, { type Response = Res; type Error = String; @@ -273,7 +265,7 @@ where .map_err(|e| e.into().to_string()) } - fn call(&mut self, task: Task) -> Self::Future { + fn call(&mut self, task: Task) -> Self::Future { let task_id = task.ctx.task_id.clone().unwrap(); let mut tx = Clone::clone(&self.tx); let fut = self.service.call(task); diff --git a/apalis-sql/src/from_row.rs b/apalis-sql/src/from_row.rs index b5f6c5e3..03078b5c 100644 --- a/apalis-sql/src/from_row.rs +++ b/apalis-sql/src/from_row.rs @@ -62,17 +62,18 @@ pub struct TaskRow { impl TaskRow { /// Convert the TaskRow into a Task with decoded arguments - pub fn try_into_task( + pub fn try_into_task( self, - ) -> Result, FromRowError> + codec: &D, + ) -> Result, FromRowError> where D::Error: Into + Send + Sync + 'static, - IdType: FromStr, - ::Err: std::error::Error + Send + Sync + 'static, + Id: FromStr, + ::Err: std::error::Error + Send + Sync + 'static, D: Codec>, Args: 'static, { - let args = D::decode(&self.job).map_err(|e| FromRowError::DecodeError(e.into()))?; + let args = D::decode(codec, &self.job).map_err(|e| FromRowError::DecodeError(e.into()))?; let mut task = TaskBuilder::new(args) .done_at(self.done_at.map(|dt| dt.to_unix_timestamp() as u64)) .lock_by(self.lock_by) @@ -99,12 +100,10 @@ impl TaskRow { } /// Convert the TaskRow into a Task with compacted arguments - pub fn try_into_task_compact( - self, - ) -> Result, Conn, IdType>, FromRowError> + pub fn try_into_task_compact(self) -> Result, Conn, Id>, FromRowError> where - IdType: FromStr, - ::Err: std::error::Error + Send + Sync + 'static, + Id: FromStr, + ::Err: std::error::Error + Send + Sync + 'static, { let mut task = TaskBuilder::new(self.job) .done_at(self.done_at.map(|dt| dt.to_unix_timestamp() as u64)) diff --git a/apalis-workflow/Cargo.toml b/apalis-workflow/Cargo.toml index 3d285801..ee5f546a 100644 --- a/apalis-workflow/Cargo.toml +++ b/apalis-workflow/Cargo.toml @@ -31,8 +31,8 @@ futures = "0.3.30" thiserror = "2.0.0" # TODO: move to workspace dependencies tracing = { version = "0.1.41", default-features = false, optional = true } -ulid = { version = "1", optional = true } -uuid = { version = "1", features = ["v4"], optional = true } +ulid = { version = "2.0.1", optional = true } +uuid = { version = "1.23.5", features = ["v4"], optional = true } rand = { version = "0.9", features = ["std"], optional = true } [dev-dependencies] diff --git a/apalis-workflow/src/dag/context.rs b/apalis-workflow/src/dag/context.rs index f09a20d7..da477554 100644 --- a/apalis-workflow/src/dag/context.rs +++ b/apalis-workflow/src/dag/context.rs @@ -17,7 +17,7 @@ use serde::{Deserialize, Serialize}; /// Metadata stored in each task for workflow processing #[derive(Debug, Deserialize, Serialize, Default)] -pub struct DagFlowContext { +pub struct DagFlowContext { /// Previous node executed in the DAG /// This is the source node that led to the current node's execution pub prev_node: Option, @@ -28,7 +28,7 @@ pub struct DagFlowContext { pub completed_nodes: HashSet, /// Map of node indices to their task IDs for result lookup - pub node_task_ids: HashMap>, + pub node_task_ids: HashMap>, /// Current position in the topological order pub current_position: usize, @@ -37,10 +37,10 @@ pub struct DagFlowContext { pub is_initial: bool, /// The original task ID that started this DAG execution - pub root_task_id: Option>, + pub root_task_id: Option>, } -impl Clone for DagFlowContext { +impl Clone for DagFlowContext { fn clone(&self) -> Self { Self { prev_node: self.prev_node, @@ -54,9 +54,9 @@ impl Clone for DagFlowContext { } } -impl DagFlowContext { +impl DagFlowContext { /// Create initial context for DAG execution - pub fn new(root_task_id: Option>) -> Self { + pub fn new(root_task_id: Option>) -> Self { Self { prev_node: None, current_node: NodeIndex::new(0), @@ -71,7 +71,7 @@ impl DagFlowContext { pub fn get_dependency_task_ids( &self, dependencies: &[NodeIndex], - ) -> HashMap> { + ) -> HashMap> { dependencies .iter() .filter_map(|dep| { @@ -122,10 +122,10 @@ pub enum DagFlowContextError { DuplicateEntry(#[from] MetadataError), } -impl Metadata for DagFlowContext +impl Metadata for DagFlowContext where - IdType: FromStr + Display, - ::Err: std::error::Error + Send + Sync + 'static, + Id: FromStr + Display, + ::Err: std::error::Error + Send + Sync + 'static, { type Error = DagFlowContextError; @@ -166,7 +166,7 @@ where .map(NodeIndex::new) .map_err(DagFlowContextError::ParseNodeIndex)?; let task_id = v - .parse::>() + .parse::>() .map_err(|e| DagFlowContextError::ParseTaskId(e.into()))?; Ok((node, task_id)) }) @@ -190,7 +190,7 @@ where let root_task_id = map .get(DAG_FLOW_ROOT_TASK_ID_KEY) - .map(|v| v.parse::().map(TaskId::new)) + .map(|v| v.parse::().map(TaskId::new)) .transpose() .map_err(|e| DagFlowContextError::ParseTaskId(e.into()))?; diff --git a/apalis-workflow/src/dag/decode.rs b/apalis-workflow/src/dag/decode.rs index 8233e14c..62fce548 100644 --- a/apalis-workflow/src/dag/decode.rs +++ b/apalis-workflow/src/dag/decode.rs @@ -1,36 +1,36 @@ -use apalis_core::backend::{BackendExt, codec::Codec}; +use apalis_core::backend::{Backend, codec::Codec}; /// An entry trait that handles the values of a fan-in/fanout entry -pub trait DagCodec: Sized { +pub trait DagCodec: Sized { /// The codec error type Error; - /// Encode the input into a Vec - fn encode(self) -> Result; + /// Encode the input into a Compact + fn encode(self, codec: &B::Codec) -> Result; /// Decode the previous input - fn decode(response: &B::Compact) -> Result; + fn decode(response: &B::Compact, codec: &B::Codec) -> Result; } impl DagCodec for Vec where - B: BackendExt, + B: Backend, B::Codec: Codec + Codec, Compact = B::Compact, Error = Err> + Codec, { type Error = Err; - fn encode(self) -> Result>::Error> { + fn encode(self, codec: &B::Codec) -> Result>::Error> { let mut result = Vec::new(); for input in self { - result.push(B::Codec::encode(&input)?); + result.push(B::Codec::encode(codec, &input)?); } - let compact = B::Codec::encode(&result)?; + let compact = B::Codec::encode(codec, &result)?; Ok(compact) } - fn decode(response: &B::Compact) -> Result { - let decoded = B::Codec::decode(response)?; + fn decode(response: &B::Compact, codec: &B::Codec) -> Result { + let decoded = B::Codec::decode(codec, response)?; Ok(decoded) } } @@ -40,18 +40,18 @@ macro_rules! impl_entry_for_tuple { ($T1:ident) => { impl DagCodec for ($T1,) where - B: BackendExt, + B: Backend, B::Codec: Codec<($T1,), Compact = B::Compact, Error = Err> + Codec, Compact = B::Compact, Error = Err>, { type Error = Err; - fn encode(self) -> Result { - let result = vec![B::Codec::encode(&self)?]; - let compact = B::Codec::encode(&result)?; + fn encode(self, codec: &B::Codec) -> Result { + let result = vec![B::Codec::encode(codec, &self)?]; + let compact = B::Codec::encode(codec, &result)?; Ok(compact) } - fn decode(response: &B::Compact) -> Result { - let decoded = B::Codec::decode(response)?; + fn decode(response: &B::Compact, codec: &B::Codec) -> Result { + let decoded = B::Codec::decode(codec, response)?; Ok(decoded) } } @@ -61,7 +61,7 @@ macro_rules! impl_entry_for_tuple { ($T1:ident, $($T:ident),+) => { impl DagCodec for ($T1, $($T),+) where - B: BackendExt, + B: Backend, B::Codec: Codec<$T1, Compact = B::Compact, Error = Err> + Codec, Compact = B::Compact, Error = Err>, $(B::Codec: Codec<$T, Compact = B::Compact, Error = Err>,)+ // Ensure all errors are the same type as T1's error @@ -71,20 +71,20 @@ macro_rules! impl_entry_for_tuple { { type Error = >::Error; - fn encode(self) -> Result { + fn encode(self, codec: &B::Codec) -> Result { #[allow(non_snake_case)] let ($T1, $($T),+) = self; - let mut result = vec![B::Codec::encode(&$T1)?]; + let mut result = vec![B::Codec::encode(codec, &$T1)?]; $( - result.push(B::Codec::encode(&$T).map_err(Into::into)?); + result.push(B::Codec::encode(codec, &$T).map_err(Into::into)?); )+ - let compact = B::Codec::encode(&result)?; + let compact = B::Codec::encode(codec, &result)?; Ok(compact) } - fn decode(response: &B::Compact) -> Result { - let decoded: Vec = B::Codec::decode(response)?; + fn decode(response: &B::Compact, codec: &B::Codec) -> Result { + let decoded: Vec = B::Codec::decode(codec, response)?; // Count the number of types in the tuple let expected_len = 1 $(+ {let _ = stringify!($T); 1})+; if decoded.len() != expected_len { @@ -94,10 +94,10 @@ macro_rules! impl_entry_for_tuple { let mut iter = decoded.into_iter(); #[allow(non_snake_case)] - let $T1 = B::Codec::decode(&iter.next().unwrap())?; + let $T1 = B::Codec::decode(codec, &iter.next().unwrap())?; $( #[allow(non_snake_case)] - let $T = B::Codec::decode(&iter.next().unwrap()).map_err(Into::into)?; + let $T = B::Codec::decode(codec, &iter.next().unwrap()).map_err(Into::into)?; )+ Ok(($T1, $($T),+)) @@ -124,17 +124,17 @@ macro_rules! impl_entry_passthrough { $( impl DagCodec for $T where - B: BackendExt, + B: Backend, B::Codec: Codec<$T, Compact = B::Compact, Error = Err>, { type Error = Err; - fn encode(self) -> Result { - B::Codec::encode(&self) + fn encode(self, codec: &B::Codec) -> Result { + B::Codec::encode(codec, &self) } - fn decode(response: &B::Compact) -> Result { - B::Codec::decode(response) + fn decode(response: &B::Compact, codec: &B::Codec) -> Result { + B::Codec::decode(codec, response) } } )+ @@ -165,32 +165,32 @@ impl_entry_passthrough!( impl DagCodec for Option where - B: BackendExt, + B: Backend, B::Codec: Codec, { type Error = Err; - fn encode(self) -> Result { - B::Codec::encode(&self) + fn encode(self, codec: &B::Codec) -> Result { + B::Codec::encode(codec, &self) } - fn decode(response: &B::Compact) -> Result { - B::Codec::decode(response) + fn decode(response: &B::Compact, codec: &B::Codec) -> Result { + B::Codec::decode(codec, response) } } impl DagCodec for Result where - B: BackendExt, + B: Backend, B::Codec: Codec, { type Error = Err; - fn encode(self) -> Result { - B::Codec::encode(&self) + fn encode(self, codec: &B::Codec) -> Result { + B::Codec::encode(codec, &self) } - fn decode(response: &B::Compact) -> Result { - B::Codec::decode(response) + fn decode(response: &B::Compact, codec: &B::Codec) -> Result { + B::Codec::decode(codec, response) } } diff --git a/apalis-workflow/src/dag/error.rs b/apalis-workflow/src/dag/error.rs index 8a07a1ba..16558776 100644 --- a/apalis-workflow/src/dag/error.rs +++ b/apalis-workflow/src/dag/error.rs @@ -100,4 +100,8 @@ pub enum DagServiceError { /// Dependency task failed #[error("Dependency task failed: {0:?}")] DependencyTaskFailed(#[source] BoxDynError), + + /// An error related to codec operations. + #[error("Codec error: {0}")] + Codec(#[source] BoxDynError), } diff --git a/apalis-workflow/src/dag/executor.rs b/apalis-workflow/src/dag/executor.rs index 2b098bbf..ed00c006 100644 --- a/apalis-workflow/src/dag/executor.rs +++ b/apalis-workflow/src/dag/executor.rs @@ -7,7 +7,10 @@ use std::{ }; use apalis_core::{ - backend::{BackendExt, codec::RawDataBackend}, + backend::{ + Backend, + ext::{BackendExt, raw::RawDataBackend}, + }, task::{Task, metadata::Meta}, worker::builder::{IntoWorkerService, WorkerService}, }; @@ -27,19 +30,20 @@ use crate::{ #[derive(Debug)] pub struct DagExecutor where - B: BackendExt, + B: Backend, { - pub(super) graph: DiGraph, ()>, + pub(super) graph: DiGraph, ()>, pub(super) node_mapping: HashMap, pub(super) topological_order: Vec, pub(super) start_nodes: Vec, pub(super) end_nodes: Vec, pub(super) not_ready: VecDeque, + pub(super) backend: B, } impl Clone for DagExecutor where - B: BackendExt, + B: Backend + Clone, { fn clone(&self) -> Self { Self { @@ -49,32 +53,33 @@ where start_nodes: self.start_nodes.clone(), end_nodes: self.end_nodes.clone(), not_ready: self.not_ready.clone(), + backend: self.backend.clone(), } } } impl DagExecutor where - B: BackendExt, + B: Backend, { /// Get a node by name pub fn get_node_by_name_mut( &mut self, name: &str, - ) -> Option<&mut DagService> { + ) -> Option<&mut DagService> { self.node_mapping .get(name) .and_then(|&idx| self.graph.node_weight_mut(idx)) } } -impl Service> for DagExecutor +impl Service> for DagExecutor where - B: BackendExt, + B: Backend, B::Connection: Send + Sync + 'static, - B::IdType: Clone + Send + Sync + 'static + GenerateId + Debug + FromStr + Display, + B::Id: Clone + Send + Sync + 'static + GenerateId + Debug + FromStr + Display, B::Compact: Send + Sync + 'static, - ::Err: std::error::Error + Send + Sync + 'static, + ::Err: std::error::Error + Send + Sync + 'static, { type Response = B::Compact; type Error = DagFlowError; @@ -104,12 +109,12 @@ where } } - fn call(&mut self, req: Task) -> Self::Future { + fn call(&mut self, req: Task) -> Self::Future { let mut graph = self.graph.clone(); Box::pin(async move { let context = req - .extract::>>() + .extract::>>() .await .map_err(|e| DagFlowError::Metadata(e.into()))? .0; @@ -131,19 +136,19 @@ where impl IntoWorkerService, B::Compact, B::Connection> for DagFlow where - B: BackendExt + Clone, + B: Backend + Clone, Err: std::error::Error + Send + Sync + 'static, B::Connection: Send + Sync + 'static, - B::IdType: Send + Sync + 'static + Default + GenerateId + PartialEq + Debug, + B::Id: Send + Sync + 'static + Default + GenerateId + PartialEq + Debug, B::Compact: Send + Sync + 'static + Clone, - RootDagService: Service>, + RootDagService: Service>, { type Backend = RawDataBackend; fn into_service(self, b: B) -> WorkerService, RootDagService> { - let executor = self.build().expect("Execution should be valid"); + let executor = self.build(b.clone()).expect("Execution should be valid"); WorkerService { - backend: RawDataBackend::new(b.clone()), - service: RootDagService::new(executor, b), + backend: b.raw(), + service: RootDagService::new(executor), } } } diff --git a/apalis-workflow/src/dag/mod.rs b/apalis-workflow/src/dag/mod.rs index a6ede5f7..28a03a49 100644 --- a/apalis-workflow/src/dag/mod.rs +++ b/apalis-workflow/src/dag/mod.rs @@ -6,7 +6,7 @@ use std::{ }; use apalis_core::{ - backend::{BackendExt, codec::Codec}, + backend::{Backend, codec::Codec}, error::BoxDynError, task::Task, task_fn::{TaskFn, task_fn}, @@ -51,16 +51,16 @@ pub use service::RootDagService; #[derive(Debug)] pub struct DagFlow where - B: BackendExt, + B: Backend, { name: String, - graph: Mutex, ()>>, + graph: Mutex, ()>>, node_mapping: Mutex>, } impl fmt::Display for DagFlow where - B: BackendExt, + B: Backend, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { writeln!(f, "DAG name: {}", self.name)?; @@ -71,7 +71,7 @@ where impl DagFlow where - B: BackendExt, + B: Backend, { /// Create a new DAG workflow builder #[must_use] @@ -91,10 +91,12 @@ where service: S, ) -> NodeBuilder<'_, Input, S::Response, B> where - S: Service> + Send + 'static + Sync + Clone, + S: Service> + Send + 'static + Sync + Clone, S::Future: Send + 'static, B::Codec: Codec + Codec + + Send + + Clone + 'static, CodecError: Into + Send + 'static, S::Error: Into, @@ -125,16 +127,16 @@ where ) -> NodeBuilder<'_, Input, O, B> where TaskFn: - Service, Response = O, Error = Err> + Clone, + Service, Response = O, Error = Err> + Clone, F: Send + 'static + Sync, Input: Send + 'static + Sync, FnArgs: Send + 'static + Sync, B::Connection: Send + Sync + 'static, as Service< - Task, + Task, >>::Future: Send + 'static, B::Codec: Codec + 'static, - B::Codec: Codec + 'static, + B::Codec: Codec + Send + Clone + 'static, CodecError: Into + Send + 'static, Err: Into, B: Send + Sync + 'static, @@ -150,17 +152,17 @@ where ) -> NodeBuilder<'_, Input, O, B> where TaskFn: - Service, Response = O, Error = Err> + Clone, + Service, Response = O, Error = Err> + Clone, F: Send + 'static + Sync, Input: Send + 'static + Sync, FnArgs: Send + 'static + Sync, as Service< - Task, + Task, >>::Future: Send + 'static, O: Into, B::Connection: Send + Sync + 'static, B::Codec: Codec + 'static, - B::Codec: Codec + 'static, + B::Codec: Codec + Send + Clone + 'static, CodecError: Into + Send + 'static, Err: Into, B: Send + Sync + 'static, @@ -209,7 +211,7 @@ where } /// Build the DAG executor - pub(crate) fn build(self) -> Result, DagFlowError> { + pub(crate) fn build(self, backend: B) -> Result, DagFlowError> { // Validate DAG (check for cycles) let sorted = toposort( &*self.graph.lock().expect("Failed to lock graph mutex"), @@ -239,6 +241,7 @@ where .expect("Failed to unlock node_mapping mutex"), topological_order: sorted, not_ready: VecDeque::new(), + backend, }) } } @@ -246,7 +249,7 @@ where /// Builder for a node in the DAG pub struct NodeBuilder<'a, Input, Output, B> where - B: BackendExt, + B: Backend, { pub(crate) id: NodeIndex, pub(crate) dag: &'a DagFlow, @@ -255,7 +258,7 @@ where impl<'a, Input, Output, B> std::fmt::Debug for NodeBuilder<'a, Input, Output, B> where - B: BackendExt, + B: Backend, { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("NodeBuilder") @@ -266,7 +269,7 @@ where impl<'a, Input, Output, B> Clone for NodeBuilder<'a, Input, Output, B> where - B: BackendExt, + B: Backend, { fn clone(&self) -> Self { Self { @@ -279,7 +282,7 @@ where impl NodeBuilder<'_, Input, Output, B> where - B: BackendExt, + B: Backend, { /// Specify dependencies for this node #[allow(clippy::needless_pass_by_value)] @@ -335,7 +338,7 @@ impl DepsCheck<()> for () { impl<'a, Input, Output, B> DepsCheck for &NodeBuilder<'a, Input, Output, B> where - B: BackendExt, + B: Backend, { fn to_node_indices(&self) -> Vec { vec![self.id] @@ -356,7 +359,7 @@ impl DepsCheck for (&NodeHandle,) { impl<'a, Input, Output, B> DepsCheck for (&NodeBuilder<'a, Input, Output, B>,) where - B: BackendExt, + B: Backend, { fn to_node_indices(&self) -> Vec { vec![self.0.id] @@ -375,7 +378,7 @@ macro_rules! impl_deps_check { ($( $len:literal => ( $( $in:ident $out:ident $idx:tt ),+ ) ),+ $(,)?) => { $( impl<'a, $( $in, )+ $( $out, )+ B> DepsCheck<( $( $out, )+ )> - for ( $( &NodeBuilder<'a, $in, $out, B>, )+ ) where B: BackendExt + for ( $( &NodeBuilder<'a, $in, $out, B>, )+ ) where B: Backend { fn to_node_indices(&self) -> Vec { vec![ $( self.$idx.id ),+ ] diff --git a/apalis-workflow/src/dag/node.rs b/apalis-workflow/src/dag/node.rs index 32aaa518..c978c8da 100644 --- a/apalis-workflow/src/dag/node.rs +++ b/apalis-workflow/src/dag/node.rs @@ -1,4 +1,4 @@ -use apalis_core::backend::BackendExt; +use apalis_core::backend::Backend; use apalis_core::backend::codec::Codec; use apalis_core::error::BoxDynError; use apalis_core::task::Task; @@ -7,14 +7,15 @@ use std::pin::Pin; use std::task::{Context, Poll}; use tower::Service; +use crate::DagExecutor; use crate::dag::decode::DagCodec; /// A service that wraps another service to handle encoding and decoding /// of task inputs and outputs using the backend's codec. pub struct NodeService where - S: Service>, - B: BackendExt, + S: Service>, + B: Backend, { inner: S, _phantom: std::marker::PhantomData<(B, Input)>, @@ -22,8 +23,8 @@ where impl std::fmt::Debug for NodeService where - S: Service>, - B: BackendExt, + S: Service>, + B: Backend, { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("NodeService") @@ -35,8 +36,8 @@ where impl Clone for NodeService where - S: Service> + Clone, - B: BackendExt, + S: Service> + Clone, + B: Backend, { fn clone(&self) -> Self { Self { @@ -48,8 +49,8 @@ where impl NodeService where - S: Service>, - B: BackendExt, + S: Service>, + B: Backend, { /// Creates a new `NodeService` wrapping the provided service. pub fn new(inner: S) -> Self { @@ -60,14 +61,16 @@ where } } -impl Service> +impl Service> for NodeService where - S: Service>, + S: Service>, S::Error: Into, - B: BackendExt, + B: Backend + Send + Sync + 'static, B::Codec: Codec - + Codec, + + Codec + + Send + + Clone, Input: DagCodec, CdcErr: Into + Send + 'static, S::Future: Send + 'static, @@ -80,9 +83,16 @@ where self.inner.poll_ready(cx).map_err(|e| e.into()) } - fn call(&mut self, req: Task) -> Self::Future { - let decoded_req = match Input::decode(&req.args) { - Ok(decoded) => req.into_builder().map(|_| decoded).build(), + fn call(&mut self, req: Task) -> Self::Future { + let executor = req + .ctx + .data + .get::>() + .expect("DagExecutor should be injected"); + + let codec = executor.backend.codec().clone(); + let decoded_req = match Input::decode(&req.args, &codec) { + Ok(decoded) => req.map_args(|_| decoded), Err(e) => { return Box::pin(async move { Err(CdcErr::into(e)) }); } @@ -92,7 +102,7 @@ where Box::pin(async move { let response = fut.await.map_err(|e| e.into())?; - B::Codec::encode(&response).map_err(|e| e.into()) + B::Codec::encode(&codec, &response).map_err(|e| e.into()) }) } } diff --git a/apalis-workflow/src/dag/response.rs b/apalis-workflow/src/dag/response.rs index 88ce4101..700aec62 100644 --- a/apalis-workflow/src/dag/response.rs +++ b/apalis-workflow/src/dag/response.rs @@ -6,18 +6,18 @@ use serde::{Deserialize, Serialize}; /// Response from DAG execution step #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub enum DagExecutionResponse { +pub enum DagExecutionResponse { /// Entry nodes have been fanned out EntryFanOut { /// Map of node indices to their task IDs - node_task_ids: HashMap>, + node_task_ids: HashMap>, }, /// Next tasks have been fanned out FanOut { /// Result of the current task response: Compact, /// Map of node indices to their task IDs - node_task_ids: HashMap>, + node_task_ids: HashMap>, }, /// Next task has been enqueued EnqueuedNext { @@ -27,7 +27,7 @@ pub enum DagExecutionResponse { /// Waiting for dependencies to complete WaitingForDependencies { /// Map of pending dependency node indices to their task IDs - pending_dependencies: HashMap>, + pending_dependencies: HashMap>, }, /// DAG execution is complete diff --git a/apalis-workflow/src/dag/service.rs b/apalis-workflow/src/dag/service.rs index 50105b8f..b81df45f 100644 --- a/apalis-workflow/src/dag/service.rs +++ b/apalis-workflow/src/dag/service.rs @@ -1,4 +1,4 @@ -use apalis_core::backend::BackendExt; +use apalis_core::backend::Backend; use apalis_core::backend::codec::Codec; use apalis_core::task::builder::TaskBuilder; use apalis_core::task::metadata::Meta; @@ -28,15 +28,14 @@ use crate::id_generator::GenerateId; /// Service that manages the execution of a DAG workflow pub struct RootDagService where - B: BackendExt, + B: Backend, { executor: DagExecutor, - backend: B, } impl std::fmt::Debug for RootDagService where - B: BackendExt, + B: Backend, { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("RootDagService") @@ -48,21 +47,20 @@ where impl RootDagService where - B: BackendExt, + B: Backend, { - pub(crate) fn new(executor: DagExecutor, backend: B) -> Self { - Self { executor, backend } + pub(crate) fn new(executor: DagExecutor) -> Self { + Self { executor } } } impl Clone for RootDagService where - B: BackendExt + Clone, + B: Backend + Clone, { fn clone(&self) -> Self { Self { executor: self.executor.clone(), - backend: self.backend.clone(), } } } @@ -75,27 +73,28 @@ fn find_designated_fan_in_handler( designated_handler.ok_or(DagFlowError::Service(DagServiceError::MissingFaninHandler)) } -impl Service> - for RootDagService +impl Service> for RootDagService where - B: BackendExt + B: Backend + Send + Sync + 'static + Clone - + WaitForCompletion>, - IdType: GenerateId + Send + Sync + 'static + PartialEq + Debug + Clone + FromStr + Display, - B::Compact: Send + Sync + 'static + Clone, + + WaitForCompletion, + Id: GenerateId + Send + Sync + 'static + PartialEq + Debug + Clone + FromStr + Display, + Compact: Send + Sync + 'static + Clone, B::Connection: Send + Sync + Default + 'static, Err: std::error::Error + Send + Sync + 'static, - B: Sink, Error = Err> + Unpin, - B::Codec: Codec, Compact = B::Compact, Error = CdcErr> + B: Sink, Error = Err> + Unpin, + B::Codec: Codec, Compact = Compact, Error = CdcErr> + + Send + + Clone + 'static - + Codec, Compact = B::Compact, Error = CdcErr>, + + Codec, Compact = B::Compact, Error = CdcErr>, CdcErr: Into, - ::Err: std::error::Error + Send + Sync + 'static, + ::Err: std::error::Error + Send + Sync + 'static, { - type Response = DagExecutionResponse; + type Response = DagExecutionResponse; type Error = DagFlowError; type Future = BoxFuture<'static, Result>; @@ -106,13 +105,19 @@ where self.executor.poll_ready(cx) } - fn call(&mut self, mut req: Task) -> Self::Future { + fn call(&mut self, req: Task) -> Self::Future { let mut executor = self.executor.clone(); - let mut backend = self.backend.clone(); + let codec = executor.backend.codec().clone(); let start_nodes = executor.start_nodes.clone(); let end_nodes = executor.end_nodes.clone(); + + let mut task = req.into_builder(); + task.ctx.data.insert(executor.clone()); // Inject executor + + let mut req = task.build(); + async move { - let ctx = req.extract::>>().await; + let ctx = req.extract::>>().await; let (response, context) = if let Ok(Meta(context)) = ctx { #[cfg(feature = "tracing")] tracing::debug!( @@ -160,7 +165,8 @@ where }); } - let results = backend + let results = executor + .backend .wait_for(dependency_task_ids.values().cloned().collect::>()) .collect::>() .await @@ -206,27 +212,32 @@ where let res = sorted_results .iter() .map(|s| match &s.result { - Ok(val) => match val { - DagExecutionResponse::FanOut { response, .. } => { - Ok(response.clone()) - } - DagExecutionResponse::EnqueuedNext { result } - | DagExecutionResponse::Complete { result } => { - Ok(result.clone()) + Ok(val) => { + let res = codec.decode(val).map_err(|e: CdcErr| { + DagServiceError::Codec(e.into()) + })?; + match res { + DagExecutionResponse::FanOut { response, .. } => { + Ok(response) + } + DagExecutionResponse::EnqueuedNext { result } + | DagExecutionResponse::Complete { result } => { + Ok(result) + } + _ => Err(DagFlowError::Service( + DagServiceError::InvalidFanInDependencyResult, + )), } - _ => Err(DagFlowError::Service( - DagServiceError::InvalidFanInDependencyResult, - )), - }, + } Err(e) => Err(DagFlowError::Service( DagServiceError::DependencyTaskFailed(e.as_str().into()), )), }) .collect::, _>>()?; - let encoded_input = B::Codec::encode(&res) + let encoded_input = B::Codec::encode(&codec, &res) .map_err(|e| DagFlowError::Codec(e.into()))?; - let req = req.into_builder().map(|_| encoded_input).build(); // Replace args with fan-in input + let req = req.map_args(|_| encoded_input); // Replace args with fan-in input let response = executor.call(req).await?; (response, context) } else { @@ -257,7 +268,6 @@ where } else { let new_node_task_ids = fan_out_entry_nodes( &executor, - &backend, &DagFlowContext::new(req.ctx.task_id.clone()), &req.args, ) @@ -294,10 +304,11 @@ where new_context.is_initial = false; let task = TaskBuilder::new(response.clone()) - .task_id(TaskId::new(B::IdType::generate())) + .task_id(TaskId::new(B::Id::generate())) .metadata(&new_context) .build(); - backend + executor + .backend .send(task) .await .map_err(|e| DagFlowError::Backend(e.into()))?; @@ -309,14 +320,9 @@ where new_context.current_position += 1; new_context.is_initial = false; - let next_task_ids = fan_out_next_nodes( - &executor, - outgoing_nodes, - &backend, - &new_context, - &response, - ) - .await?; + let next_task_ids = + fan_out_next_nodes(&executor, outgoing_nodes, &new_context, &response) + .await?; return Ok(DagExecutionResponse::FanOut { response, node_task_ids: next_task_ids, @@ -330,29 +336,28 @@ where } async fn fan_out_next_nodes( - _executor: &DagExecutor, + executor: &DagExecutor, outgoing_nodes: Vec, - backend: &B, - context: &DagFlowContext, + context: &DagFlowContext, input: &B::Compact, -) -> Result>, DagFlowError> +) -> Result>, DagFlowError> where - B::IdType: GenerateId + Send + Sync + 'static + PartialEq, + B::Id: GenerateId + Send + Sync + 'static + PartialEq, B::Compact: Send + Sync + 'static + Clone, B::Connection: Send + Sync + Default + 'static, - B: Sink, Error = Err> + Unpin, + B: Sink, Error = Err> + Unpin, Err: std::error::Error + Send + Sync + 'static, - B: BackendExt + Send + Sync + 'static + Clone, + B: Backend + Send + Sync + 'static + Clone, B::Codec: Codec, Compact = B::Compact, Error = CdcErr>, CdcErr: Into, - B::IdType: FromStr + Display, - ::Err: std::error::Error + Send + Sync + 'static, + B::Id: FromStr + Display, + ::Err: std::error::Error + Send + Sync + 'static, { let mut enqueue_futures = vec![]; let next_nodes = outgoing_nodes .iter() - .map(|node| (*node, TaskId::new(B::IdType::generate()))) - .collect::>>(); + .map(|node| (*node, TaskId::new(B::Id::generate()))) + .collect::>>(); let mut node_task_ids = next_nodes.clone(); node_task_ids.extend(context.node_task_ids.clone()); for outgoing_node in outgoing_nodes.into_iter() { @@ -372,7 +377,7 @@ where root_task_id: context.root_task_id.clone(), }) .build(); - let mut b = backend.clone(); + let mut b = executor.backend.clone(); enqueue_futures.push( async move { b.send(task) @@ -389,24 +394,24 @@ where async fn fan_out_entry_nodes( executor: &DagExecutor, - backend: &B, - context: &DagFlowContext, + context: &DagFlowContext, input: &B::Compact, -) -> Result>, DagFlowError> +) -> Result>, DagFlowError> where - B::IdType: GenerateId + Send + Sync + 'static + PartialEq + Debug, + B::Id: GenerateId + Send + Sync + 'static + PartialEq + Debug, B::Compact: Send + Sync + 'static + Clone, B::Connection: Send + Sync + Default + 'static, - B: Sink, Error = Err> + Unpin, + B: Sink, Error = Err> + Unpin, Err: std::error::Error + Send + Sync + 'static, - B: BackendExt + Send + Sync + 'static + Clone, - B::Codec: Codec, Compact = B::Compact, Error = CdcErr>, + B: Backend + Send + Sync + 'static + Clone, + B::Codec: Codec, Compact = B::Compact, Error = CdcErr> + Clone, CdcErr: Into, - B::IdType: FromStr + Display, - ::Err: std::error::Error + Send + Sync + 'static, + B::Id: FromStr + Display, + ::Err: std::error::Error + Send + Sync + 'static, { + let codec = executor.backend.codec().clone(); let values: Vec = - B::Codec::decode(input).map_err(|e: CdcErr| DagFlowError::Codec(e.into()))?; + B::Codec::decode(&codec, input).map_err(|e: CdcErr| DagFlowError::Codec(e.into()))?; let start_nodes = executor.start_nodes.clone(); if values.len() != start_nodes.len() { return Err(DagFlowError::InputCountMismatch { @@ -417,8 +422,8 @@ where let mut enqueue_futures = vec![]; let next_nodes = start_nodes .iter() - .map(|node| (*node, TaskId::new(B::IdType::generate()))) - .collect::>>(); + .map(|node| (*node, TaskId::new(B::Id::generate()))) + .collect::>>(); let mut node_task_ids = next_nodes.clone(); node_task_ids.extend(context.node_task_ids.clone()); for (outgoing_node, input) in start_nodes.into_iter().zip(values) { @@ -437,7 +442,7 @@ where root_task_id: context.root_task_id.clone(), }) .build(); - let mut b = backend.clone(); + let mut b = executor.backend.clone(); enqueue_futures.push( async move { b.send(task) diff --git a/apalis-workflow/src/id_generator.rs b/apalis-workflow/src/id_generator.rs index 19ae28cd..6a2d8957 100644 --- a/apalis-workflow/src/id_generator.rs +++ b/apalis-workflow/src/id_generator.rs @@ -16,7 +16,7 @@ impl GenerateId for uuid::Uuid { #[cfg(feature = "ulid")] impl GenerateId for ulid::Ulid { fn generate() -> Self { - Self::new() + Self::r#gen() } } diff --git a/apalis-workflow/src/lib.rs b/apalis-workflow/src/lib.rs index 16ebe650..17954ac2 100644 --- a/apalis-workflow/src/lib.rs +++ b/apalis-workflow/src/lib.rs @@ -12,10 +12,10 @@ use apalis_core::{error::BoxDynError, task::Task}; use crate::sequential::router::{GoTo, StepResult}; type BoxedService = tower::util::BoxCloneSyncService; -type SteppedService = - BoxedService, GoTo>>; +type SteppedService = + BoxedService, GoTo>>; -type DagService = BoxedService, Compact>; +type DagService = BoxedService, Compact>; /// combinator for chaining multiple workflows. pub mod composite; diff --git a/apalis-workflow/src/sequential/and_then/mod.rs b/apalis-workflow/src/sequential/and_then/mod.rs index 5060d466..df3e11c9 100644 --- a/apalis-workflow/src/sequential/and_then/mod.rs +++ b/apalis-workflow/src/sequential/and_then/mod.rs @@ -1,7 +1,7 @@ use std::marker::PhantomData; use apalis_core::{ - backend::{BackendExt, codec::Codec}, + backend::{Backend, codec::Codec}, error::BoxDynError, task::Task, task_fn::{TaskFn, task_fn}, @@ -58,14 +58,14 @@ where impl Step for AndThenStep where - B: BackendExt + B: Backend + Send + Sync + 'static + Clone - + Sink, Error = SinkError> + + Sink, Error = SinkError> + Unpin, - F: Service, Error = BoxDynError> + F: Service, Error = BoxDynError> + Send + Sync + 'static @@ -77,9 +77,11 @@ where B::Codec: Codec + Codec + Codec + + Send + + Clone + 'static, CodecError: std::error::Error + Send + Sync + 'static, - B::IdType: GenerateId + Send + Sync + 'static, + B::Id: GenerateId + Send + Sync + 'static, S::Response: Send + 'static, B::Compact: Send + 'static, B::Connection: Send + Sync + 'static, @@ -96,7 +98,7 @@ where })) .map_response(|res: F::Response| GoTo::Next(res)) .service(self.then_fn.clone()); - let svc = SteppedService::::new(svc); + let svc = SteppedService::::new(svc); let count = ctx.steps.len(); ctx.steps.insert(count, svc); self.step.register(ctx) @@ -129,30 +131,32 @@ impl AndThenService { } } -impl Service> +impl Service> for AndThenService where - S: Service, Response = GoTo>, + S: Service, Response = GoTo>, S::Future: Send + 'static, - B: BackendExt + B: Backend + Sync + Send + 'static + Clone - + Sink, Error = SinkError> + + Sink, Error = SinkError> + Unpin, B::Codec: Codec - + Codec, + + Codec + + Send + + Clone, S::Error: Into + Send + 'static, CodecErr: Into + Send + 'static, Cur: Send + 'static, - B::IdType: GenerateId + Send + Sync + 'static, + B::Id: GenerateId + Send + Sync + 'static, SinkError: std::error::Error + Send + Sync + 'static, Res: Send + 'static, B::Compact: Send + 'static, B::Connection: Send + Sync + 'static, { - type Response = GoTo>; + type Response = GoTo>; type Error = BoxDynError; type Future = BoxFuture<'static, Result>; @@ -163,12 +167,10 @@ where self.service.poll_ready(cx).map_err(|e| e.into()) } - fn call(&mut self, request: Task) -> Self::Future { + fn call(&mut self, request: Task) -> Self::Future { let mut ctx = request.ctx.data.get::>().cloned().unwrap(); - let compacted = request - .into_builder() - .try_map(|t| B::Codec::decode(&t)) - .map(|t| t.build()); + let codec = ctx.backend.codec().clone(); + let compacted = request.try_map_args(|t| B::Codec::decode(&codec, &t)); match compacted { Ok(task) => { let fut = self.service.call(task); @@ -185,7 +187,7 @@ where impl Workflow where - B: BackendExt, + B: Backend, { /// Adds a transformation step to the workflow that processes the output of the previous step. /// @@ -206,7 +208,7 @@ where ) -> Workflow>, L>> where TaskFn: - Service, Response = O>, + Service, Response = O>, { self.add_step(AndThen { then_fn: task_fn(and_then), diff --git a/apalis-workflow/src/sequential/delay/mod.rs b/apalis-workflow/src/sequential/delay/mod.rs index 79903bd3..9898a528 100644 --- a/apalis-workflow/src/sequential/delay/mod.rs +++ b/apalis-workflow/src/sequential/delay/mod.rs @@ -1,7 +1,7 @@ use std::time::Duration; use apalis_core::{ - backend::{BackendExt, codec::Codec}, + backend::{Backend, codec::Codec}, error::BoxDynError, task::{Task, builder::TaskBuilder, task_id::TaskId}, }; @@ -46,9 +46,9 @@ pub struct DelayForStep { impl Step for DelayForStep where - B::IdType: GenerateId + Send + Sync + 'static, + B::Id: GenerateId + Send + Sync + 'static, B::Compact: Send + 'static, - B: Sink, Error = Err> + B: Sink, Error = Err> + Unpin + Send + Sync @@ -57,12 +57,16 @@ where Err: std::error::Error + Send + Sync + 'static, S: Clone + Send + Sync + 'static, S::Response: Send + 'static, - B::Codec: Codec + Codec + 'static, + B::Codec: Codec + + Codec + + Send + + Clone + + 'static, >::Error: Into, B::Connection: Send + Sync + 'static, Input: Send + Sync + 'static, >::Error: Into, - B: BackendExt, + B: Backend, S: Step, { type Response = Input; @@ -119,10 +123,10 @@ impl Clone for DelayWithStep { impl Step for DelayWithStep where - F: FnMut(Task) -> Duration + Send + Sync + 'static + Clone, - B::IdType: GenerateId + Sync + Send + 'static, + F: FnMut(Task) -> Duration + Send + Sync + 'static + Clone, + B::Id: GenerateId + Sync + Send + 'static, B::Compact: Send + 'static, - B: Sink, Error = Err> + B: Sink, Error = Err> + Unpin + Send + Sync @@ -131,12 +135,16 @@ where Err: std::error::Error + Send + Sync + 'static, S: Clone + Send + Sync + 'static, S::Response: Send + 'static, - B::Codec: Codec + Codec + 'static, + B::Codec: Codec + + Codec + + Send + + Clone + + 'static, >::Error: Into, B::Connection: Send + Sync + 'static, Input: Send + Sync + 'static, >::Error: Into, - B: BackendExt, + B: Backend, S: Step, { type Response = Input; @@ -153,22 +161,26 @@ where } } -impl - Service> for DelayWithStep +impl + Service> for DelayWithStep where - F: FnMut(Task) -> Duration + Send + 'static + Clone, + F: FnMut(Task) -> Duration + Send + 'static + Clone, S: Step + Send + 'static, S::Response: Send + 'static, - B::IdType: GenerateId + Sync + Send + 'static, + B::Id: GenerateId + Sync + Send + 'static, B::Compact: Send + 'static, - B: Sink, Error = Err> + Unpin + Send + Sync, + B: Sink, Error = Err> + Unpin + Send + Sync, Err: std::error::Error + Send + Sync + 'static, - B::Codec: Codec + Codec + 'static, + B::Codec: Codec + + Codec + + Send + + Clone + + 'static, >::Error: Into, >::Error: Into, B::Connection: Send + Sync + 'static, { - type Response = GoTo>; + type Response = GoTo>; type Error = BoxDynError; type Future = BoxFuture<'static, Result>; @@ -179,13 +191,13 @@ where std::task::Poll::Ready(Ok(())) } - fn call(&mut self, req: Task) -> Self::Future { + fn call(&mut self, req: Task) -> Self::Future { let mut step_context: StepContext = req.ctx.data.get().cloned().unwrap(); let mut f = self.f.clone(); - - let task_id = TaskId::new(B::IdType::generate()); + let codec = step_context.backend.codec().clone(); + let task_id = TaskId::new(B::Id::generate()); async move { - let decoded: Input = B::Codec::decode(&req.args) + let decoded: Input = B::Codec::decode(&codec, &req.args) .map_err(|e: >::Error| e.into())?; let (args, ctx) = req.take(); let delay_duration = f(Task { args: decoded, ctx }); @@ -202,7 +214,7 @@ where .await .map_err(|e| BoxDynError::from(e))?; Ok(GoTo::Next(StepResult { - result: B::Codec::encode(&delay_duration).map_err(|e| e.into())?, + result: B::Codec::encode(&codec, &delay_duration).map_err(|e| e.into())?, next_task_id: Some(task_id), })) } diff --git a/apalis-workflow/src/sequential/filter_map/mod.rs b/apalis-workflow/src/sequential/filter_map/mod.rs index 7c2219dd..c39d62e1 100644 --- a/apalis-workflow/src/sequential/filter_map/mod.rs +++ b/apalis-workflow/src/sequential/filter_map/mod.rs @@ -1,7 +1,7 @@ use std::{fmt::Display, marker::PhantomData, str::FromStr}; use apalis_core::{ - backend::{BackendExt, TaskSinkError, WaitForCompletion, codec::Codec}, + backend::{Backend, TaskSinkError, WaitForCompletion, codec::Codec}, error::BoxDynError, task::{ Task, @@ -161,8 +161,8 @@ impl Metadata for FilterState { /// The context for the filter operation #[derive(Debug, Clone, Deserialize, Serialize)] -pub struct FilterContext { - task_ids: Vec>, +pub struct FilterContext { + task_ids: Vec>, } const FILTER_CONTEXT_TASK_IDS_KEY: &str = "apalis_workflow.filter.task_ids"; @@ -183,9 +183,9 @@ pub enum FilterContextError { DuplicateEntry(#[from] MetadataError), } -impl Metadata for FilterContext +impl Metadata for FilterContext where - IdType: std::str::FromStr + Display, + Id: std::str::FromStr + Display, { type Error = FilterContextError; @@ -200,7 +200,7 @@ where value .split(',') .map(|id| { - id.parse::() + id.parse::() .map(TaskId::new) .map_err(|_| FilterContextError::ParseTaskId) }) @@ -224,25 +224,28 @@ where } } -impl - Service> for FilterService +impl + Service> for FilterService where - F: Service, Response = Option>, - B: BackendExt + F: Service, Response = Option>, + B: Backend + Send + Sync + 'static + Clone - + Sink, Error = Err> - + WaitForCompletion>> + + Sink, Error = Err> + + WaitForCompletion + Unpin, B::Codec: Codec, Error = CodecError, Compact = B::Compact> + Codec + Codec + Codec + Codec, Error = CodecError, Compact = B::Compact> + + Codec>, Error = CodecError, Compact = B::Compact> + + Send + + Clone + 'static, - IdType: GenerateId + FromStr + Display + Send + 'static + Clone + Sync, + Id: GenerateId + FromStr + Display + Send + 'static + Clone + Sync, B::Connection: Send + Sync + 'static, Err: std::error::Error + Send + Sync + 'static, CodecError: std::error::Error + Send + Sync + 'static, @@ -253,7 +256,7 @@ where Output: Send + 'static, Iter: IntoIterator + Send + 'static, { - type Response = GoTo>; + type Response = GoTo>; type Error = BoxDynError; type Future = BoxFuture<'static, Result>; @@ -264,29 +267,27 @@ where self.service.poll_ready(cx).map_err(|e| e.into()) } - fn call(&mut self, request: Task) -> Self::Future { + fn call(&mut self, request: Task) -> Self::Future { let filter_state: FilterState = Metadata::extract(&request.ctx.metadata).unwrap_or(FilterState::Init); let mut ctx = request.ctx.data.get::>().cloned().unwrap(); + let codec = ctx.backend.codec().clone(); use futures::SinkExt; match filter_state { FilterState::Init => { // Handle unknown state async move { let main_args: Vec = vec![]; - let steps: Task = request - .into_builder() - .try_map(|arg| B::Codec::decode(&arg)) - .unwrap() - .build(); + let steps: Task = + request.try_map_args(|arg| B::Codec::decode(&codec, &arg))?; let steps = steps.args.into_iter().collect::>(); #[cfg(feature = "tracing")] tracing::debug!(step_count = ?steps.len(), "Enqueuing FilterMap steps"); let mut task_ids = Vec::new(); for step in steps { - let task_id = TaskId::new(B::IdType::generate()); + let task_id = TaskId::new(B::Id::generate()); - let task = TaskBuilder::new(B::Codec::encode(&step)?) + let task = TaskBuilder::new(B::Codec::encode(&codec, &step)?) .metadata(&WorkflowContext { step_index: ctx.current_step, }) @@ -300,8 +301,8 @@ where task_ids.push(task_id); } - let task_id = TaskId::new(B::IdType::generate()); - let task = TaskBuilder::new(B::Codec::encode(&main_args)?) + let task_id = TaskId::new(B::Id::generate()); + let task = TaskBuilder::new(B::Codec::encode(&codec, &main_args)?) .task_id(task_id.clone()) .metadata(&WorkflowContext { step_index: ctx.current_step, @@ -321,15 +322,13 @@ where } FilterState::SingleStep => { let step: Task = request - .into_builder() - .try_map(|arg| B::Codec::decode(&arg)) - .unwrap() - .build(); + .try_map_args(|arg| B::Codec::decode(&codec, &arg)) + .unwrap(); let fut = self.service.call(step); async move { let res = fut.await.map_err(|e| e.into())?; Ok(GoTo::Break(StepResult { - result: B::Codec::encode(&res) + result: B::Codec::encode(&codec, &res) .map_err(|e| TaskSinkError::CodecError::(e.into()))?, next_task_id: None, })) @@ -339,7 +338,7 @@ where FilterState::Collector => { // Handle collector state async move { - let filter_ctx: FilterContext = + let filter_ctx: FilterContext = Metadata::extract(&request.ctx.metadata)?; let res: Vec = ctx .backend @@ -352,10 +351,16 @@ where .filter_map(|res| { let res = res.take().ok(); match res { - Some(GoTo::Break(val)) => { - let opt: Result, _> = - B::Codec::decode(&val.result); - opt.ok().flatten() + Some(val) => { + let decoded = codec.decode(&val).ok(); + match decoded { + Some(GoTo::Break(val)) => { + let opt: Result, _> = + B::Codec::decode(&codec, &val.result); + opt.ok().flatten() + } + _ => None, + } } _ => None, } @@ -363,7 +368,7 @@ where .collect(); if res.is_empty() { return Ok(GoTo::Break(StepResult { - result: B::Codec::encode(&res) + result: B::Codec::encode(&codec, &res) .map_err(|e| TaskSinkError::CodecError::(e.into()))?, next_task_id: None, })); @@ -378,19 +383,19 @@ where } } -impl Step +impl Step for FilterMapStep where I: IntoIterator + Send + Sync + 'static, - B: BackendExt + B: Backend + Send + Sync + 'static + Clone - + Sink, Error = SinkError> - + WaitForCompletion>> + + Sink, Error = SinkError> + + WaitForCompletion + Unpin, - F: Service, Error = BoxDynError, Response = Option> + F: Service, Error = BoxDynError, Response = Option> + Send + Sync + 'static @@ -401,9 +406,11 @@ where F::Error: Into + Send + 'static, B::Codec: Codec + Codec + + Send + + Clone + 'static, CodecError: std::error::Error + Send + Sync + 'static, - B::IdType: GenerateId + Send + 'static + Clone, + B::Id: GenerateId + Send + 'static + Clone, S::Response: Send + 'static, B::Compact: Send + 'static, SinkError: std::error::Error + Send + Sync + 'static, @@ -413,14 +420,16 @@ where + Codec + Codec + Codec, Error = CodecError, Compact = B::Compact> + + Codec>, Error = CodecError, Compact = B::Compact> + 'static, - B::IdType: GenerateId + Send + Sync + 'static, + B::Id: GenerateId + Send + Sync + 'static, B::Connection: Send + Sync + 'static, CodecError: std::error::Error + Send + Sync + 'static, F::Future: Send + 'static, B::Compact: Send + 'static, Output: Send + 'static, - IdType: FromStr + Display, + Id: FromStr + Display, + Output: 'static, { type Response = Vec; type Error = F::Error; @@ -435,7 +444,7 @@ where } } -impl, B: BackendExt> Workflow { +impl, B: Backend> Workflow { /// Adds a filter and map step to the workflow. pub fn filter_map( self, @@ -443,7 +452,7 @@ impl, B: BackendExt> Workflow Workflow, B, Stack, I>, L>> where TaskFn: - Service, Response = Option>, + Service, Response = Option>, { self.add_step(FilterMap { filter_map: task_fn(filter_map), diff --git a/apalis-workflow/src/sequential/fold/mod.rs b/apalis-workflow/src/sequential/fold/mod.rs index 6b1eb1b7..36d62db1 100644 --- a/apalis-workflow/src/sequential/fold/mod.rs +++ b/apalis-workflow/src/sequential/fold/mod.rs @@ -1,7 +1,7 @@ use std::{marker::PhantomData, task::Context}; use apalis_core::{ - backend::{BackendExt, TaskSinkError, codec::Codec}, + backend::{Backend, TaskSinkError, codec::Codec}, error::BoxDynError, task::{ Task, @@ -46,7 +46,7 @@ where } } } -impl, B: BackendExt> Workflow { +impl, B: Backend> Workflow { /// Folds over a collection of items in the workflow. pub fn fold( self, @@ -54,7 +54,7 @@ impl, B: BackendExt> Workflow Workflow, Init>, L>> where TaskFn: - Service, Response = Output>, + Service, Response = Output>, { self.add_step(Fold { fold: task_fn(fold), @@ -74,17 +74,17 @@ pub struct FoldStep { impl, Init, B, Err, CodecError> Step for FoldStep where - F: Service, Response = Init> + F: Service, Response = Init> + Send + Sync + 'static + Clone, S: Step, - B: BackendExt + B: Backend + Send + Sync + Clone - + Sink, Error = Err> + + Sink, Error = Err> + Unpin + 'static, I: IntoIterator + Send + Sync + 'static, @@ -93,8 +93,10 @@ where + Codec + Codec + Codec<(Init, Input), Error = CodecError, Compact = B::Compact> + + Send + + Clone + 'static, - B::IdType: GenerateId + Sync + Send + 'static + Clone, + B::Id: GenerateId + Sync + Send + 'static + Clone, Init: Default + Send + Sync + 'static, Err: std::error::Error + Send + Sync + 'static, CodecError: std::error::Error + Send + Sync + 'static, @@ -142,18 +144,15 @@ impl FoldService { } } -impl Service> +impl Service> for FoldService where - F: Service, Response = Init> - + Send - + 'static - + Clone, - B: BackendExt + F: Service, Response = Init> + Send + 'static + Clone, + B: Backend + Send + Sync + Clone - + Sink, Error = Err> + + Sink, Error = Err> + Unpin + 'static, I: IntoIterator + Send + 'static, @@ -162,8 +161,10 @@ where + Codec + Codec + Codec<(Init, Input), Error = CodecError, Compact = B::Compact> + + Send + + Clone + 'static, - B::IdType: GenerateId + Sync + Send + 'static, + B::Id: GenerateId + Sync + Send + 'static, Init: Default + Send + 'static, Err: std::error::Error + Send + Sync + 'static, CodecError: std::error::Error + Send + Sync + 'static, @@ -172,7 +173,7 @@ where B::Compact: Send + 'static, Input: Send + 'static, { - type Response = GoTo>; + type Response = GoTo>; type Error = BoxDynError; type Future = BoxFuture<'static, Result>; @@ -180,20 +181,19 @@ where self.fold.poll_ready(cx).map_err(|e| e.into()) } - fn call(&mut self, task: Task) -> Self::Future { + fn call(&mut self, task: Task) -> Self::Future { let state = FoldState::extract(&task.ctx.metadata).unwrap_or(FoldState::Init); let mut ctx = task.ctx.data.get::>().cloned().unwrap(); + let codec = ctx.backend.codec().clone(); let mut fold = self.fold.clone(); match state { FoldState::Init => async move { - let task_id = TaskId::new(B::IdType::generate()); - let steps: Task = task - .into_builder() - .try_map(|arg| B::Codec::decode(&arg))? - .build(); + let task_id = TaskId::new(B::Id::generate()); + let steps: Task = + task.try_map_args(|arg| B::Codec::decode(&codec, &arg))?; let steps = steps.args.into_iter().collect::>(); - let task = TaskBuilder::new(B::Codec::encode(&(Init::default(), steps))?) + let task = TaskBuilder::new(B::Codec::encode(&codec, &(Init::default(), steps))?) .metadata(&WorkflowContext { step_index: ctx.current_step, }) @@ -205,25 +205,25 @@ where .await .map_err(|e| TaskSinkError::PushError(e))?; Ok(GoTo::Next(StepResult { - result: B::Codec::encode(&Init::default())?, + result: B::Codec::encode(&codec, &Init::default())?, next_task_id: Some(task_id), })) } .boxed(), FoldState::Collection => async move { - let args: (Init, Vec) = B::Codec::decode(&task.args)?; + let args: (Init, Vec) = B::Codec::decode(&codec, &task.args)?; let (acc, items) = args; let mut items = items.into_iter(); let next = items.next().unwrap(); let rest = items.collect::>(); - let fold_task = task.into_builder().map(|_| (acc, next)).build(); + let fold_task = task.map_args(|_| (acc, next)); let response = fold.call(fold_task).await.map_err(|e| e.into())?; match rest.len() { 0 if ctx.has_next => { - let task_id = TaskId::new(B::IdType::generate()); - let result = B::Codec::encode(&response)?; + let task_id = TaskId::new(B::Id::generate()); + let result = B::Codec::encode(&codec, &response)?; let next_step = TaskBuilder::new(result) .task_id(task_id.clone()) .metadata(&WorkflowContext { @@ -235,19 +235,19 @@ where .await .map_err(|e| TaskSinkError::PushError(e))?; Ok(GoTo::Break(StepResult { - result: B::Codec::encode(&response)?, + result: B::Codec::encode(&codec, &response)?, next_task_id: Some(task_id), })) } 0 => Ok(GoTo::Break(StepResult { - result: B::Codec::encode(&response)?, + result: B::Codec::encode(&codec, &response)?, next_task_id: None, })), 1.. => { // Shouldn't this be limited? - let task_id = TaskId::new(B::IdType::generate()); - let result = B::Codec::encode(&response)?; - let steps = TaskBuilder::new(B::Codec::encode(&(response, rest))?) + let task_id = TaskId::new(B::Id::generate()); + let result = B::Codec::encode(&codec, &response)?; + let steps = TaskBuilder::new(B::Codec::encode(&codec, &(response, rest))?) .task_id(task_id.clone()) .metadata(&WorkflowContext { step_index: ctx.current_step, diff --git a/apalis-workflow/src/sequential/repeat_until/mod.rs b/apalis-workflow/src/sequential/repeat_until/mod.rs index 7dab9904..96788790 100644 --- a/apalis-workflow/src/sequential/repeat_until/mod.rs +++ b/apalis-workflow/src/sequential/repeat_until/mod.rs @@ -11,7 +11,7 @@ use apalis_core::task::builder::TaskBuilder; use apalis_core::task::metadata::{Metadata, MetadataError, MetadataStore}; use apalis_core::task::task_id::TaskId; use apalis_core::task_fn::{TaskFn, task_fn}; -use apalis_core::{backend::BackendExt, task::Task}; +use apalis_core::{backend::Backend, task::Task}; use futures::future::BoxFuture; use futures::{FutureExt, Sink, SinkExt}; use serde::{Deserialize, Serialize}; @@ -43,7 +43,7 @@ where } } } -impl Workflow { +impl Workflow { /// Folds over a collection of items in the workflow. pub fn repeat_until( self, @@ -56,7 +56,7 @@ impl Workflow { > where TaskFn: - Service, Response = Option>, + Service, Response = Option>, { self.add_step(RepeatUntil { repeater: task_fn(repeater), @@ -92,26 +92,25 @@ where } } -impl Service> +impl Service> for RepeatUntilService where - F: Service, Response = Option> - + Send - + 'static - + Clone, - B: BackendExt + F: Service, Response = Option> + Send + 'static + Clone, + B: Backend + Send + Sync + Clone - + Sink, Error = Err> + + Sink, Error = Err> + Unpin + 'static, B::Connection: Send + Sync + 'static, B::Codec: Codec + Codec + Codec, Error = CodecError, Compact = B::Compact> + + Send + + Clone + 'static, - B::IdType: GenerateId + Send + Sync + Display + FromStr + 'static, + B::Id: GenerateId + Send + Sync + Display + FromStr + 'static, Err: std::error::Error + Send + Sync + 'static, CodecError: std::error::Error + Send + Sync + 'static, F::Error: Into + Send + 'static, @@ -120,7 +119,7 @@ where Input: Send + 'static, // We don't need Clone because decoding just needs a reference Res: Send + 'static, { - type Response = GoTo>; + type Response = GoTo>; type Error = BoxDynError; type Future = BoxFuture<'static, Result>; @@ -128,31 +127,28 @@ where self.repeater.poll_ready(cx).map_err(|e| e.into()) } - fn call(&mut self, task: Task) -> Self::Future { - let state: RepeaterState = - Metadata::extract(&task.ctx.metadata).unwrap_or_default(); + fn call(&mut self, task: Task) -> Self::Future { + let state: RepeaterState = Metadata::extract(&task.ctx.metadata).unwrap_or_default(); let mut ctx = task.ctx.data.get::>().cloned().expect( "StepContext missing, Did you call the repeater outside of a workflow step?", ); let mut repeater = self.repeater.clone(); + let codec = ctx.backend.codec().clone(); (async move { let mut compact = None; - let decoded: Input = B::Codec::decode(&task.args)?; + let decoded: Input = B::Codec::decode(&codec, &task.args)?; let prev_task_id = task.ctx.task_id.clone(); - let repeat_task = task - .into_builder() - .map(|c| { - compact = Some(c); - decoded - }) - .build(); + let repeat_task = task.map_args(|c| { + compact = Some(c); + decoded + }); let response = repeater.call(repeat_task).await.map_err(|e| e.into())?; Ok(match response { Some(res) if ctx.has_next => { - let task_id = TaskId::new(B::IdType::generate()); - let next_step = TaskBuilder::new(B::Codec::encode(&res)?) + let task_id = TaskId::new(B::Id::generate()); + let next_step = TaskBuilder::new(B::Codec::encode(&codec, &res)?) .task_id(task_id.clone()) .metadata(&WorkflowContext { step_index: ctx.current_step + 1, @@ -163,16 +159,16 @@ where .await .map_err(|e| TaskSinkError::PushError(e))?; GoTo::Next(StepResult { - result: B::Codec::encode(&res)?, + result: B::Codec::encode(&codec, &res)?, next_task_id: Some(task_id), }) } Some(res) => GoTo::Break(StepResult { - result: B::Codec::encode(&res)?, + result: B::Codec::encode(&codec, &res)?, next_task_id: None, }), None => { - let task_id = TaskId::new(B::IdType::generate()); + let task_id = TaskId::new(B::Id::generate()); let next_step = TaskBuilder::new(compact.take().expect("Compact args should be set")) .task_id(task_id.clone()) @@ -189,7 +185,7 @@ where .await .map_err(|e| TaskSinkError::PushError(e))?; GoTo::Break(StepResult { - result: B::Codec::encode(&None::)?, + result: B::Codec::encode(&codec, &None::)?, next_task_id: Some(task_id), }) } @@ -201,12 +197,12 @@ where /// The state of the repeat operation #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct RepeaterState { +pub struct RepeaterState { iterations: usize, - prev_task_id: Option>, + prev_task_id: Option>, } -impl Default for RepeaterState { +impl Default for RepeaterState { fn default() -> Self { Self { iterations: 0, @@ -215,14 +211,14 @@ impl Default for RepeaterState { } } -impl RepeaterState { +impl RepeaterState { /// Get the number of iterations completed so far. pub fn iterations(&self) -> usize { self.iterations } /// Get the previous task id. - pub fn previous_task_id(&self) -> Option<&TaskId> { + pub fn previous_task_id(&self) -> Option<&TaskId> { self.prev_task_id.as_ref() } } @@ -250,9 +246,9 @@ pub enum RepeaterStateError { const REPEATER_ITERATIONS_KEY: &str = "apalis_workflow.repeater.iterations"; const REPEATER_PREV_TASK_ID_KEY: &str = "apalis_workflow.repeater.prev_task_id"; -impl Metadata for RepeaterState +impl Metadata for RepeaterState where - IdType: std::str::FromStr + ToString, + Id: std::str::FromStr + ToString, { type Error = RepeaterStateError; @@ -266,7 +262,7 @@ where .get(REPEATER_PREV_TASK_ID_KEY) .map(|value| { value - .parse::() + .parse::() .map(TaskId::new) .map_err(|_| RepeaterStateError::ParseTaskId) }) @@ -290,24 +286,26 @@ where impl Step for RepeatUntilStep where - F: Service, Response = Option> + F: Service, Response = Option> + Send + Sync + 'static + Clone, - B: BackendExt + B: Backend + Send + Sync + Clone - + Sink, Error = Err> + + Sink, Error = Err> + Unpin + 'static, B::Connection: Send + Sync + 'static, B::Codec: Codec + Codec + Codec, Error = CodecError, Compact = B::Compact> + + Send + + Clone + 'static, - B::IdType: GenerateId + Send + 'static, + B::Id: GenerateId + Send + 'static, Err: std::error::Error + Send + Sync + 'static, CodecError: std::error::Error + Send + Sync + 'static, F::Error: Into + Send + 'static, @@ -316,7 +314,7 @@ where Input: Send + Sync + 'static, // We don't need Clone because decoding just needs a reference Res: Send + Sync + 'static, S: Step + Send + 'static, - B::IdType: FromStr + Display + Sync, + B::Id: FromStr + Display + Sync, { type Response = Res; type Error = F::Error; diff --git a/apalis-workflow/src/sequential/router.rs b/apalis-workflow/src/sequential/router.rs index 7ef5b5f6..3d387069 100644 --- a/apalis-workflow/src/sequential/router.rs +++ b/apalis-workflow/src/sequential/router.rs @@ -1,23 +1,22 @@ use std::{collections::HashMap, time::Duration}; -use apalis_core::{backend::BackendExt, task::task_id::TaskId}; +use apalis_core::{backend::Backend, task::task_id::TaskId}; use serde::{Deserialize, Serialize}; use crate::SteppedService; /// Router for workflow steps #[derive(Debug, Default)] -pub struct WorkflowRouter +pub struct WorkflowRouter where - Backend: BackendExt, + B: Backend, { - pub(super) steps: - HashMap>, + pub(super) steps: HashMap>, } -impl WorkflowRouter +impl WorkflowRouter where - Backend: BackendExt, + B: Backend, { /// Create a new workflow router #[must_use] @@ -29,11 +28,11 @@ where } /// Result information for workflow steps #[derive(Debug, Clone, Deserialize, Serialize, Default)] -pub struct StepResult { +pub struct StepResult { /// Result produced by the step pub result: Res, /// Optional ID of the next task to execute - pub next_task_id: Option>, + pub next_task_id: Option>, } /// Enum representing the possible transitions in a workflow diff --git a/apalis-workflow/src/sequential/service.rs b/apalis-workflow/src/sequential/service.rs index c9b9e3c1..4dd268ef 100644 --- a/apalis-workflow/src/sequential/service.rs +++ b/apalis-workflow/src/sequential/service.rs @@ -1,5 +1,5 @@ use apalis_core::{ - backend::{BackendExt, TaskSinkError, codec::Codec}, + backend::{Backend, TaskSinkError, codec::Codec}, error::BoxDynError, task::{Task, builder::TaskBuilder, metadata::Metadata, task_id::TaskId}, }; @@ -23,20 +23,20 @@ use crate::{ #[derive(Debug, Clone)] pub struct WorkflowService where - B: BackendExt, + B: Backend, { - services: HashMap>, + services: HashMap>, not_ready: VecDeque, backend: B, _marker: PhantomData, } impl WorkflowService where - B: BackendExt, + B: Backend, { /// Creates a new `WorkflowService` with the given services and backend. pub fn new( - services: HashMap>, + services: HashMap>, backend: B, ) -> Self { Self { @@ -48,18 +48,17 @@ where } } -impl Service> - for WorkflowService +impl Service> for WorkflowService where B::Compact: Send + 'static, B: Sync, B::Connection: Send, Err: std::error::Error + Send + Sync + 'static, - B::IdType: GenerateId + Send + 'static, - B: Sink, Error = Err> + Unpin, - B: Clone + Send + Sync + 'static + BackendExt, + B::Id: GenerateId + Send + 'static, + B: Sink, Error = Err> + Unpin, + B: Clone + Send + Sync + 'static + Backend, { - type Response = GoTo>; + type Response = GoTo>; type Error = BoxDynError; type Future = BoxFuture<'static, Result>; @@ -85,7 +84,7 @@ where } } - fn call(&mut self, req: Task) -> Self::Future { + fn call(&mut self, req: Task) -> Self::Future { assert!( self.not_ready.is_empty(), "Workflow must wait for all services to be ready. Did you forget to call poll_ready()?" @@ -113,24 +112,25 @@ where pub async fn handle_step_result( ctx: &mut StepContext, result: GoTo, -) -> Result>, TaskSinkError> +) -> Result>, TaskSinkError> where - B: Sink, Error = Err> - + BackendExt + B: Sink, Error = Err> + + Backend + Send + Unpin, - B::Codec: Codec, + B::Codec: Codec + Clone, >::Error: Into, Compact: 'static, N: 'static, - B::IdType: GenerateId + Send + 'static, + B::Id: GenerateId + Send + 'static, { + let codec = ctx.backend.codec().clone(); match result { GoTo::Next(next) if ctx.has_next => { - let task_id = B::IdType::generate(); + let task_id = B::Id::generate(); let task_id = TaskId::new(task_id); let task = TaskBuilder::new( - B::Codec::encode(&next).map_err(|e| TaskSinkError::CodecError(e.into()))?, + B::Codec::encode(&codec, &next).map_err(|e| TaskSinkError::CodecError(e.into()))?, ) .task_id(task_id.clone()) .metadata(&WorkflowContext { @@ -139,15 +139,16 @@ where .build(); ctx.backend.send(task).await?; Ok(GoTo::Next(StepResult { - result: B::Codec::encode(&next).map_err(|e| TaskSinkError::CodecError(e.into()))?, + result: B::Codec::encode(&codec, &next) + .map_err(|e| TaskSinkError::CodecError(e.into()))?, next_task_id: Some(task_id), })) } GoTo::DelayFor(delay, next) if ctx.has_next => { - let task_id = B::IdType::generate(); + let task_id = B::Id::generate(); let task_id = TaskId::new(task_id); let task = TaskBuilder::new( - B::Codec::encode(&next).map_err(|e| TaskSinkError::CodecError(e.into()))?, + B::Codec::encode(&codec, &next).map_err(|e| TaskSinkError::CodecError(e.into()))?, ) .run_after(delay) .task_id(task_id.clone()) @@ -159,7 +160,7 @@ where Ok(GoTo::DelayFor( delay, StepResult { - result: B::Codec::encode(&next) + result: B::Codec::encode(&codec, &next) .map_err(|e| TaskSinkError::CodecError(e.into()))?, next_task_id: Some(task_id), }, @@ -168,7 +169,8 @@ where #[allow(clippy::match_same_arms)] GoTo::Done => Ok(GoTo::Done), GoTo::Break(res) => Ok(GoTo::Break(StepResult { - result: B::Codec::encode(&res).map_err(|e| TaskSinkError::CodecError(e.into()))?, + result: B::Codec::encode(&codec, &res) + .map_err(|e| TaskSinkError::CodecError(e.into()))?, next_task_id: None, })), _ => Ok(GoTo::Done), diff --git a/apalis-workflow/src/sequential/step.rs b/apalis-workflow/src/sequential/step.rs index 62619043..7802faec 100644 --- a/apalis-workflow/src/sequential/step.rs +++ b/apalis-workflow/src/sequential/step.rs @@ -1,4 +1,4 @@ -use apalis_core::{backend::BackendExt, error::BoxDynError}; +use apalis_core::{backend::Backend, error::BoxDynError}; use crate::sequential::router::WorkflowRouter; @@ -15,7 +15,7 @@ pub trait Layer { /// A single unit of work in a sequential workflow pipeline. pub trait Step where - B: BackendExt, + B: Backend, { /// The response type produced by the step. type Response; diff --git a/apalis-workflow/src/sequential/workflow.rs b/apalis-workflow/src/sequential/workflow.rs index 0e9f88f4..f3fb008f 100644 --- a/apalis-workflow/src/sequential/workflow.rs +++ b/apalis-workflow/src/sequential/workflow.rs @@ -1,7 +1,10 @@ use std::marker::PhantomData; use apalis_core::{ - backend::{Backend, BackendExt, codec::RawDataBackend}, + backend::{ + Backend, + ext::{BackendExt, raw::RawDataBackend}, + }, error::BoxDynError, task::Task, worker::builder::{IntoWorkerService, WorkerService}, @@ -55,7 +58,7 @@ impl Workflow { where S: Step, L: Layer, - B: BackendExt, + B: Backend, { Workflow { inner: self.inner.layer(root), @@ -67,7 +70,7 @@ impl Workflow { impl Workflow where - B: BackendExt, + B: Backend, { /// Builds the workflow by layering the root step. pub fn build(self) -> L::Step @@ -89,7 +92,7 @@ impl Default for RootStep { } } -impl Step for RootStep { +impl Step for RootStep { type Response = Current; type Error = BoxDynError; fn register(&mut self, _ctx: &mut WorkflowRouter) -> Result<(), BoxDynError> { @@ -102,16 +105,16 @@ impl IntoWorkerService, Compact, B::Connection> for Workflow where - B: BackendExt + B: Backend + Send + Sync + 'static - + Sink, Error = Err> + + Sink, Error = Err> + Unpin + Clone, Err: std::error::Error + Send + Sync + 'static, B::Connection: Send + Sync + 'static, - B::IdType: Send + 'static + Default + GenerateId, + B::Id: Send + 'static + Default + GenerateId, B: Sync + Backend, B::Compact: Send + Sync + 'static, L: Layer>, @@ -127,8 +130,8 @@ where .register(&mut ctx) .expect("Failed to register workflow steps"); WorkerService { - backend: RawDataBackend::new(b.clone()), - service: WorkflowService::new(ctx.steps, b), + service: WorkflowService::new(ctx.steps, b.clone()), + backend: b.raw(), } } } diff --git a/apalis-workflow/src/sink.rs b/apalis-workflow/src/sink.rs index 63fc1f74..5c3710de 100644 --- a/apalis-workflow/src/sink.rs +++ b/apalis-workflow/src/sink.rs @@ -1,7 +1,7 @@ use std::{fmt::Display, str::FromStr}; use apalis_core::{ - backend::{BackendExt, TaskSinkError, codec::Codec}, + backend::{Backend, TaskSinkError, codec::Codec}, error::BoxDynError, task::{Task, builder::TaskBuilder, task_id::TaskId}, }; @@ -15,7 +15,7 @@ use crate::{ }; /// Extension trait for pushing tasks into a workflow -pub trait WorkflowSink: BackendExt + Sized +pub trait WorkflowSink: Backend + Sized where Self::Codec: Codec, { @@ -61,21 +61,23 @@ where impl WorkflowSink for S where - S: Sink, Error = Err> - + BackendExt + S: Sink, Error = Err> + + Backend + Unpin, - S::IdType: GenerateId + Send + Sync + FromStr + Display, + S::Id: GenerateId + Send + Sync + FromStr + Display, S::Codec: Codec, S::Connection: Send + Sync, Err: std::error::Error + Send + Sync + 'static, >::Error: Into + Send + Sync + 'static, Compact: Send + 'static, - ::Err: std::error::Error + Send + Sync + 'static, + ::Err: std::error::Error + Send + Sync + 'static, { async fn push_start(&mut self, args: Args) -> Result<(), TaskSinkError> { use futures::SinkExt; - let task_id = TaskId::new(S::IdType::generate()); - let compact = S::Codec::encode(&args).map_err(|e| TaskSinkError::CodecError(e.into()))?; + let codec = self.codec(); + let task_id = TaskId::new(S::Id::generate()); + let compact = + S::Codec::encode(codec, &args).map_err(|e| TaskSinkError::CodecError(e.into()))?; let task = TaskBuilder::new(compact).task_id(task_id.clone()).build(); self.send(task) .await @@ -88,9 +90,9 @@ where Args::Error: std::error::Error + Send + Sync + 'static, { use futures::SinkExt; - let task_id = TaskId::new(S::IdType::generate()); - - let compact = Args::encode(args).map_err(|e| TaskSinkError::CodecError(e.into()))?; + let task_id = TaskId::new(S::Id::generate()); + let codec = self.codec(); + let compact = Args::encode(args, codec).map_err(|e| TaskSinkError::CodecError(e.into()))?; let task = TaskBuilder::new(compact).task_id(task_id.clone()).build(); self.send(task) .await @@ -103,8 +105,10 @@ where index: usize, ) -> Result<(), TaskSinkError> { use futures::SinkExt; - let task_id = TaskId::new(S::IdType::generate()); - let compact = S::Codec::encode(&step).map_err(|e| TaskSinkError::CodecError(e.into()))?; + let task_id = TaskId::new(S::Id::generate()); + let codec = self.codec(); + let compact = + S::Codec::encode(codec, &step).map_err(|e| TaskSinkError::CodecError(e.into()))?; let task = TaskBuilder::new(compact) .metadata(&WorkflowContext { step_index: index }) .task_id(task_id.clone()) @@ -120,8 +124,10 @@ where index: NodeIndex, ) -> Result<(), TaskSinkError> { use futures::SinkExt; - let task_id = TaskId::new(S::IdType::generate()); - let compact = S::Codec::encode(&node).map_err(|e| TaskSinkError::CodecError(e.into()))?; + let task_id = TaskId::new(S::Id::generate()); + let codec = self.codec(); + let compact = + S::Codec::encode(codec, &node).map_err(|e| TaskSinkError::CodecError(e.into()))?; let task = TaskBuilder::new(compact) .metadata(&DagFlowContext { current_node: index, diff --git a/apalis/Cargo.toml b/apalis/Cargo.toml index d3552bfa..4ef8dbfb 100644 --- a/apalis/Cargo.toml +++ b/apalis/Cargo.toml @@ -72,7 +72,7 @@ optional = true [dependencies] tower = { version = "0.5.2", features = ["util"], default-features = false } -sentry-core = { version = "0.48.0", optional = true, default-features = false } +sentry-core = { version = "0.48.4", optional = true, default-features = false } metrics = { version = "0.24.5", optional = true, default-features = false } metrics-exporter-prometheus = { version = "0.18.1", optional = true, default-features = false } opentelemetry = { version = "0.32.0", optional = true, default-features = false, features = ["metrics"] } @@ -82,8 +82,8 @@ thiserror = "2.0.0" pin-project = "1" # Needed only for sentry reporting -uuid = { version = "1.8", optional = true } -ulid = { version = "1", optional = true } +uuid = { version = "1.23.5", optional = true } +ulid = { version = "2.0.1", optional = true } [dependencies.tracing] default-features = false diff --git a/apalis/src/layers/catch_panic/mod.rs b/apalis/src/layers/catch_panic/mod.rs index 018eee87..ecf45489 100644 --- a/apalis/src/layers/catch_panic/mod.rs +++ b/apalis/src/layers/catch_panic/mod.rs @@ -143,10 +143,9 @@ pub struct CatchPanicService { on_panic: F, } -impl Service> - for CatchPanicService +impl Service> for CatchPanicService where - S: Service, Response = Res>, + S: Service, Response = Res>, F: FnMut(Box) -> PanicErr + Clone, S::Error: Into, PanicErr: Into, @@ -159,7 +158,7 @@ where self.service.poll_ready(cx).map_err(Into::into) } - fn call(&mut self, task: Task) -> Self::Future { + fn call(&mut self, task: Task) -> Self::Future { match std::panic::catch_unwind(AssertUnwindSafe(|| self.service.call(task))) { Ok(future) => CatchPanicFuture { kind: Kind::Future { diff --git a/apalis/src/layers/opentelemetry/mod.rs b/apalis/src/layers/opentelemetry/mod.rs index 6cc7476c..10097f64 100644 --- a/apalis/src/layers/opentelemetry/mod.rs +++ b/apalis/src/layers/opentelemetry/mod.rs @@ -52,10 +52,10 @@ pub struct OpenTelemetryMetricsService { duration_histogram: Histogram, } -impl Service> +impl Service> for OpenTelemetryMetricsService where - Svc: Service, Response = Res, Error = Err, Future = Fut>, + Svc: Service, Response = Res, Error = Err, Future = Fut>, Fut: Future> + 'static, { type Response = Svc::Response; @@ -66,7 +66,7 @@ where self.service.poll_ready(cx) } - fn call(&mut self, request: Task) -> Self::Future { + fn call(&mut self, request: Task) -> Self::Future { let start = Instant::now(); let worker = request .ctx diff --git a/apalis/src/layers/prometheus/mod.rs b/apalis/src/layers/prometheus/mod.rs index 969332bd..de3fad59 100644 --- a/apalis/src/layers/prometheus/mod.rs +++ b/apalis/src/layers/prometheus/mod.rs @@ -27,10 +27,9 @@ pub struct PrometheusService { service: S, } -impl Service> - for PrometheusService +impl Service> for PrometheusService where - Svc: Service, Response = Res, Error = Err, Future = Fut>, + Svc: Service, Response = Res, Error = Err, Future = Fut>, Fut: Future> + 'static, { type Response = Svc::Response; @@ -41,7 +40,7 @@ where self.service.poll_ready(cx) } - fn call(&mut self, request: Task) -> Self::Future { + fn call(&mut self, request: Task) -> Self::Future { let start = Instant::now(); let worker = request .ctx diff --git a/apalis/src/layers/retry/mod.rs b/apalis/src/layers/retry/mod.rs index ffd61246..08747a0e 100644 --- a/apalis/src/layers/retry/mod.rs +++ b/apalis/src/layers/retry/mod.rs @@ -122,12 +122,11 @@ impl BackoffRetryPolicy { } } -impl Policy, Res, Err> - for BackoffRetryPolicy +impl Policy, Res, Err> for BackoffRetryPolicy where T: Clone, Conn: Clone, - IdType: Clone, + Id: Clone, B: Backoff, B::Future: Send + 'static, { @@ -135,7 +134,7 @@ where fn retry( &mut self, - req: &mut Task, + req: &mut Task, result: &mut Result, ) -> Option { let attempt = req.ctx.attempt.current(); @@ -160,7 +159,7 @@ where } } - fn clone_request(&mut self, req: &Task) -> Option> { + fn clone_request(&mut self, req: &Task) -> Option> { let req = req.clone(); Some(req) } @@ -206,17 +205,17 @@ impl RetryPolicy { } } -impl Policy, Res, Err> for RetryPolicy +impl Policy, Res, Err> for RetryPolicy where T: Clone, Conn: Clone, - IdType: Clone, + Id: Clone, { type Future = std::future::Ready<()>; fn retry( &mut self, - req: &mut Task, + req: &mut Task, result: &mut Result, ) -> Option { let attempt = req.ctx.attempt.current(); @@ -239,7 +238,7 @@ where } } - fn clone_request(&mut self, req: &Task) -> Option> { + fn clone_request(&mut self, req: &Task) -> Option> { let req = req.clone(); Some(req) } @@ -265,19 +264,18 @@ impl RetryIfPolicy { FromTaskConfigPolicy::new(self) } } -impl Policy, Res, Err> - for RetryIfPolicy +impl Policy, Res, Err> for RetryIfPolicy where T: Clone, Conn: Clone, - P: Policy, Res, Err>, + P: Policy, Res, Err>, F: Fn(&Err) -> bool + Send + Sync + 'static, { type Future = P::Future; fn retry( &mut self, - req: &mut Task, + req: &mut Task, result: &mut Result, ) -> Option { let worker = req.ctx.data.get::()?; @@ -295,7 +293,7 @@ where } } - fn clone_request(&mut self, req: &Task) -> Option> { + fn clone_request(&mut self, req: &Task) -> Option> { self.inner.clone_request(req) } } @@ -372,18 +370,17 @@ impl Default for FromTaskConfigPolicy { } } -impl Policy, Res, Err> - for FromTaskConfigPolicy

+impl Policy, Res, Err> for FromTaskConfigPolicy

where T: Clone, Conn: Clone, - P: Policy, Res, Err>, + P: Policy, Res, Err>, { type Future = P::Future; fn retry( &mut self, - req: &mut Task, + req: &mut Task, result: &mut Result, ) -> Option { let worker = req.ctx.data.get::()?; @@ -406,7 +403,7 @@ where } } - fn clone_request(&mut self, req: &Task) -> Option> { + fn clone_request(&mut self, req: &Task) -> Option> { self.inner.clone_request(req) } } @@ -417,7 +414,7 @@ pub trait RetryMetadataExt { fn retries(self, retries: usize) -> Self; } -impl RetryMetadataExt for TaskBuilder { +impl RetryMetadataExt for TaskBuilder { /// Set number of retries in the metadata fn retries(self, retries: usize) -> Self { self.metadata(&RetryConfig { retries }) diff --git a/apalis/src/layers/sentry/mod.rs b/apalis/src/layers/sentry/mod.rs index 07f636d9..29d4898e 100644 --- a/apalis/src/layers/sentry/mod.rs +++ b/apalis/src/layers/sentry/mod.rs @@ -153,12 +153,11 @@ where } } -impl Service> - for SentryTaskService +impl Service> for SentryTaskService where - Svc: Service, Response = Res, Error = Err, Future = Fut>, + Svc: Service, Response = Res, Error = Err, Future = Fut>, Fut: Future> + 'static, - IdType: ToUuid, + Id: ToUuid, Err: Into + 'static, { type Response = Svc::Response; @@ -169,7 +168,7 @@ where self.service.poll_ready(cx).map_err(|e| e.into()) } - fn call(&mut self, task: Task) -> Self::Future { + fn call(&mut self, task: Task) -> Self::Future { let task_type = std::any::type_name::().to_string(); let attempt = &task.ctx.attempt; let task_id = task diff --git a/apalis/src/layers/tracing/contextual_span.rs b/apalis/src/layers/tracing/contextual_span.rs index 565311f2..a3767f96 100644 --- a/apalis/src/layers/tracing/contextual_span.rs +++ b/apalis/src/layers/tracing/contextual_span.rs @@ -43,11 +43,11 @@ impl Default for ContextualTaskSpan { } } -impl MakeSpan for ContextualTaskSpan +impl MakeSpan for ContextualTaskSpan where - IdType: Display, + Id: Display, { - fn make_span(&mut self, req: &Task) -> Span { + fn make_span(&mut self, req: &Task) -> Span { let task_id = req .ctx .task_id diff --git a/apalis/src/layers/tracing/make_span.rs b/apalis/src/layers/tracing/make_span.rs index 56d3d666..4ca9ee7f 100644 --- a/apalis/src/layers/tracing/make_span.rs +++ b/apalis/src/layers/tracing/make_span.rs @@ -10,22 +10,22 @@ use super::DEFAULT_MESSAGE_LEVEL; /// /// [`Span`]: tracing::Span /// [`Trace`]: super::Trace -pub trait MakeSpan { +pub trait MakeSpan { /// Make a span from a request. - fn make_span(&mut self, request: &Task) -> Span; + fn make_span(&mut self, request: &Task) -> Span; } -impl MakeSpan for Span { - fn make_span(&mut self, _request: &Task) -> Span { +impl MakeSpan for Span { + fn make_span(&mut self, _request: &Task) -> Span { self.clone() } } -impl MakeSpan for F +impl MakeSpan for F where - F: FnMut(&Task) -> Span, + F: FnMut(&Task) -> Span, { - fn make_span(&mut self, request: &Task) -> Span { + fn make_span(&mut self, request: &Task) -> Span { self(request) } } @@ -64,8 +64,8 @@ impl Default for DefaultMakeSpan { } } -impl MakeSpan for DefaultMakeSpan { - fn make_span(&mut self, req: &Task) -> Span { +impl MakeSpan for DefaultMakeSpan { + fn make_span(&mut self, req: &Task) -> Span { // This ugly macro is needed, unfortunately, because `tracing::span!` // required the level argument to be static. Meaning we can't just pass // `self.level`. diff --git a/apalis/src/layers/tracing/mod.rs b/apalis/src/layers/tracing/mod.rs index 7cea3319..a244f9ac 100644 --- a/apalis/src/layers/tracing/mod.rs +++ b/apalis/src/layers/tracing/mod.rs @@ -293,13 +293,13 @@ impl } } -impl - Service> for Trace +impl + Service> for Trace where - S: Service, Response = Res, Future = F> + Unpin + Send + 'static, + S: Service, Response = Res, Future = F> + Unpin + Send + 'static, S::Error: fmt::Display + 'static, - MakeSpanT: MakeSpan, - OnRequestT: OnRequest, + MakeSpanT: MakeSpan, + OnRequestT: OnRequest, OnResponseT: OnResponse + Clone + 'static, F: Future> + 'static, OnFailureT: OnFailure + Clone + 'static, @@ -312,7 +312,7 @@ where self.inner.poll_ready(cx) } - fn call(&mut self, req: Task) -> Self::Future { + fn call(&mut self, req: Task) -> Self::Future { let span = self.make_span.make_span(&req); let start = Instant::now(); let job = { diff --git a/apalis/src/layers/tracing/on_request.rs b/apalis/src/layers/tracing/on_request.rs index d44ad3c1..0b142710 100644 --- a/apalis/src/layers/tracing/on_request.rs +++ b/apalis/src/layers/tracing/on_request.rs @@ -10,7 +10,7 @@ use tracing::Span; /// `on_request` callback is called. /// /// [`Trace`]: super::Trace -pub trait OnRequest { +pub trait OnRequest { /// Do the thing. /// /// `span` is the `tracing` [`Span`], corresponding to this request, produced by the closure @@ -20,14 +20,14 @@ pub trait OnRequest { /// [`Span`]: https://docs.rs/tracing/latest/tracing/span/index.html /// [record]: https://docs.rs/tracing/latest/tracing/span/struct.Span.html#method.record /// [`TraceLayer::make_span_with`]: crate::layers::tracing::TraceLayer::make_span_with - fn on_request(&mut self, request: &Task, span: &Span); + fn on_request(&mut self, request: &Task, span: &Span); } -impl OnRequest for F +impl OnRequest for F where - F: for<'a> FnMut(&'a Task, &'a Span), + F: for<'a> FnMut(&'a Task, &'a Span), { - fn on_request(&mut self, request: &Task, span: &Span) { + fn on_request(&mut self, request: &Task, span: &Span) { self(request, span) } } @@ -71,8 +71,8 @@ impl DefaultOnRequest { } } -impl OnRequest for DefaultOnRequest { - fn on_request(&mut self, _: &Task, _: &Span) { +impl OnRequest for DefaultOnRequest { + fn on_request(&mut self, _: &Task, _: &Span) { match self.level { Level::ERROR => { tracing::event!(Level::ERROR, "task.start",); diff --git a/apalis/src/lib.rs b/apalis/src/lib.rs index c986061f..bb6af215 100644 --- a/apalis/src/lib.rs +++ b/apalis/src/lib.rs @@ -29,12 +29,12 @@ pub mod prelude { }; pub use apalis_core::{ backend::{ - Backend, BackendExt, Expose, FetchById, Filter, ListAllTasks, ListQueues, ListTasks, - ListWorkers, Metrics, QueueInfo, RegisterWorker, Reschedule, ResumeAbandoned, - ResumeById, RunningWorker, StatType, Statistic, TaskResult, TaskSink, TaskSinkError, - TaskStream, Update, WaitForCompletion, + Backend, Expose, FetchById, Filter, ListAllTasks, ListQueues, ListTasks, ListWorkers, + Metrics, QueueInfo, RegisterWorker, Reschedule, ResumeAbandoned, ResumeById, + RunningWorker, StatType, Statistic, TaskResult, TaskSink, TaskSinkError, Update, + WaitForCompletion, }, - backend::{codec::*, custom::*, memory::*, pipe::*, poll_strategy::*, shared::*}, + backend::{codec::*, custom::*, ext::pipe::*, memory::*, poll_strategy::*, shared::*}, error::*, layers::*, monitor::{ExitError, Monitor, MonitorError, MonitoredWorkerError, shutdown::Shutdown}, diff --git a/examples/sentry/Cargo.toml b/examples/sentry/Cargo.toml index 656c5ae3..4c7b9582 100644 --- a/examples/sentry/Cargo.toml +++ b/examples/sentry/Cargo.toml @@ -11,9 +11,9 @@ apalis = { path = "../../apalis", features = ["sentry"] } serde = "1" env_logger = "0.11" tracing-subscriber = { version = "0.3.20", features = ["env-filter"] } -sentry = "0.48.0" -sentry-tower = "0.48.0" -sentry-tracing = "0.48.0" +sentry = "0.48.4" +sentry-tower = "0.48.4" +sentry-tracing = "0.48.4" chrono = { version = "0.4", default-features = false, features = ["clock"] } tokio = { version = "1", features = ["full"] } email-service = { path = "../email-service" } diff --git a/supply-chain/config.toml b/supply-chain/config.toml index bfb07644..3310ae1a 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -161,7 +161,7 @@ version = "1.3.2" criteria = "safe-to-deploy" [[exemptions.bitflags]] -version = "2.13.0" +version = "2.13.1" criteria = "safe-to-deploy" [[exemptions.block2]] @@ -177,7 +177,7 @@ version = "1.5.0" criteria = "safe-to-deploy" [[exemptions.bytes]] -version = "1.12.0" +version = "1.12.1" criteria = "safe-to-deploy" [[exemptions.bytestring]] @@ -185,7 +185,7 @@ version = "1.5.1" criteria = "safe-to-deploy" [[exemptions.cc]] -version = "1.2.65" +version = "1.2.67" criteria = "safe-to-deploy" [[exemptions.cfg-if]] @@ -193,7 +193,11 @@ version = "1.0.4" criteria = "safe-to-deploy" [[exemptions.cfg_aliases]] -version = "0.2.1" +version = "0.2.2" +criteria = "safe-to-deploy" + +[[exemptions.chacha20]] +version = "0.10.1" criteria = "safe-to-deploy" [[exemptions.chrono]] @@ -220,12 +224,16 @@ criteria = "safe-to-deploy" version = "0.8.7" criteria = "safe-to-deploy" +[[exemptions.cpufeatures]] +version = "0.3.0" +criteria = "safe-to-deploy" + [[exemptions.crossbeam-epoch]] -version = "0.9.18" +version = "0.9.20" criteria = "safe-to-deploy" [[exemptions.crossbeam-utils]] -version = "0.8.21" +version = "0.8.22" criteria = "safe-to-deploy" [[exemptions.debugid]] @@ -233,11 +241,11 @@ version = "0.8.0" criteria = "safe-to-deploy" [[exemptions.defmt]] -version = "1.1.0" +version = "1.1.1" criteria = "safe-to-deploy" [[exemptions.defmt-macros]] -version = "1.1.0" +version = "1.1.1" criteria = "safe-to-deploy" [[exemptions.defmt-parser]] @@ -245,7 +253,7 @@ version = "1.0.0" criteria = "safe-to-deploy" [[exemptions.der]] -version = "0.8.0" +version = "0.8.1" criteria = "safe-to-deploy" [[exemptions.deranged]] @@ -461,11 +469,11 @@ version = "1.4.2" criteria = "safe-to-deploy" [[exemptions.http-body]] -version = "1.0.1" +version = "1.1.0" criteria = "safe-to-deploy" [[exemptions.http-body-util]] -version = "0.1.3" +version = "0.1.4" criteria = "safe-to-deploy" [[exemptions.httparse]] @@ -561,15 +569,15 @@ version = "1.0.18" criteria = "safe-to-deploy" [[exemptions.jiff]] -version = "0.2.31" +version = "0.2.32" criteria = "safe-to-deploy" [[exemptions.jiff-static]] -version = "0.2.31" +version = "0.2.32" criteria = "safe-to-deploy" [[exemptions.jobserver]] -version = "0.1.34" +version = "0.1.35" criteria = "safe-to-deploy" [[exemptions.js-sys]] @@ -629,7 +637,7 @@ version = "0.8.4" criteria = "safe-to-deploy" [[exemptions.memchr]] -version = "2.8.2" +version = "2.8.3" criteria = "safe-to-deploy" [[exemptions.metrics]] @@ -653,7 +661,7 @@ version = "0.8.9" criteria = "safe-to-deploy" [[exemptions.mio]] -version = "1.2.1" +version = "1.2.2" criteria = "safe-to-deploy" [[exemptions.native-tls]] @@ -844,14 +852,6 @@ criteria = "safe-to-deploy" version = "0.2.21" criteria = "safe-to-deploy" -[[exemptions.proc-macro-error-attr2]] -version = "2.0.0" -criteria = "safe-to-deploy" - -[[exemptions.proc-macro-error2]] -version = "2.0.1" -criteria = "safe-to-deploy" - [[exemptions.proc-macro2]] version = "1.0.106" criteria = "safe-to-deploy" @@ -885,7 +885,11 @@ version = "6.0.0" criteria = "safe-to-deploy" [[exemptions.rand]] -version = "0.9.4" +version = "0.9.5" +criteria = "safe-to-deploy" + +[[exemptions.rand]] +version = "0.10.2" criteria = "safe-to-deploy" [[exemptions.rand_chacha]] @@ -896,12 +900,16 @@ criteria = "safe-to-deploy" version = "0.9.5" criteria = "safe-to-deploy" +[[exemptions.rand_core]] +version = "0.10.1" +criteria = "safe-to-deploy" + [[exemptions.rand_xoshiro]] version = "0.7.0" criteria = "safe-to-deploy" [[exemptions.rapidhash]] -version = "4.4.2" +version = "4.5.1" criteria = "safe-to-deploy" [[exemptions.raw-cpuid]] @@ -913,11 +921,11 @@ version = "0.5.18" criteria = "safe-to-deploy" [[exemptions.regex]] -version = "1.12.4" +version = "1.13.1" criteria = "safe-to-deploy" [[exemptions.regex-automata]] -version = "0.4.14" +version = "0.4.16" criteria = "safe-to-deploy" [[exemptions.regex-lite]] @@ -945,7 +953,7 @@ version = "1.3.1" criteria = "safe-to-deploy" [[exemptions.rustc-demangle]] -version = "0.1.27" +version = "0.1.28" criteria = "safe-to-deploy" [[exemptions.rustc_version]] @@ -957,7 +965,7 @@ version = "1.1.4" criteria = "safe-to-deploy" [[exemptions.rustls]] -version = "0.23.41" +version = "0.23.42" criteria = "safe-to-deploy" [[exemptions.rustls-native-certs]] @@ -965,7 +973,7 @@ version = "0.8.4" criteria = "safe-to-deploy" [[exemptions.rustls-pki-types]] -version = "1.14.1" +version = "1.15.0" criteria = "safe-to-deploy" [[exemptions.rustls-webpki]] @@ -973,7 +981,7 @@ version = "0.103.13" criteria = "safe-to-deploy" [[exemptions.rustversion]] -version = "1.0.22" +version = "1.0.23" criteria = "safe-to-deploy" [[exemptions.ryu]] @@ -1005,43 +1013,43 @@ version = "1.0.28" criteria = "safe-to-deploy" [[exemptions.sentry]] -version = "0.48.3" +version = "0.48.5" criteria = "safe-to-deploy" [[exemptions.sentry-actix]] -version = "0.48.3" +version = "0.48.5" criteria = "safe-to-deploy" [[exemptions.sentry-backtrace]] -version = "0.48.3" +version = "0.48.5" criteria = "safe-to-deploy" [[exemptions.sentry-contexts]] -version = "0.48.3" +version = "0.48.5" criteria = "safe-to-deploy" [[exemptions.sentry-core]] -version = "0.48.3" +version = "0.48.5" criteria = "safe-to-deploy" [[exemptions.sentry-debug-images]] -version = "0.48.3" +version = "0.48.5" criteria = "safe-to-deploy" [[exemptions.sentry-panic]] -version = "0.48.3" +version = "0.48.5" criteria = "safe-to-deploy" [[exemptions.sentry-tower]] -version = "0.48.3" +version = "0.48.5" criteria = "safe-to-deploy" [[exemptions.sentry-tracing]] -version = "0.48.3" +version = "0.48.5" criteria = "safe-to-deploy" [[exemptions.sentry-types]] -version = "0.48.3" +version = "0.48.5" criteria = "safe-to-deploy" [[exemptions.serde]] @@ -1097,7 +1105,7 @@ version = "0.5.10" criteria = "safe-to-deploy" [[exemptions.socket2]] -version = "0.6.4" +version = "0.6.5" criteria = "safe-to-deploy" [[exemptions.stable_deref_trait]] @@ -1109,7 +1117,7 @@ version = "2.6.1" criteria = "safe-to-deploy" [[exemptions.syn]] -version = "2.0.118" +version = "2.0.119" criteria = "safe-to-deploy" [[exemptions.sync_wrapper]] @@ -1133,11 +1141,11 @@ version = "2.0.18" criteria = "safe-to-deploy" [[exemptions.thread_local]] -version = "1.1.9" +version = "1.1.10" criteria = "safe-to-deploy" [[exemptions.time]] -version = "0.3.52" +version = "0.3.53" criteria = "safe-to-deploy" [[exemptions.time-core]] @@ -1153,11 +1161,11 @@ version = "0.8.3" criteria = "safe-to-deploy" [[exemptions.tokio]] -version = "1.52.3" +version = "1.53.0" criteria = "safe-to-deploy" [[exemptions.tokio-macros]] -version = "2.7.0" +version = "2.7.1" criteria = "safe-to-deploy" [[exemptions.tokio-native-tls]] @@ -1221,7 +1229,7 @@ version = "0.2.5" criteria = "safe-to-deploy" [[exemptions.ulid]] -version = "1.2.1" +version = "2.0.1" criteria = "safe-to-deploy" [[exemptions.uname]] @@ -1273,7 +1281,7 @@ version = "0.2.2" criteria = "safe-to-deploy" [[exemptions.uuid]] -version = "1.23.4" +version = "1.24.0" criteria = "safe-to-deploy" [[exemptions.valuable]] @@ -1433,11 +1441,11 @@ version = "0.8.2" criteria = "safe-to-deploy" [[exemptions.zerocopy]] -version = "0.8.52" +version = "0.8.54" criteria = "safe-to-deploy" [[exemptions.zerocopy-derive]] -version = "0.8.52" +version = "0.8.54" criteria = "safe-to-deploy" [[exemptions.zerofrom]] @@ -1465,5 +1473,5 @@ version = "0.11.3" criteria = "safe-to-deploy" [[exemptions.zmij]] -version = "1.0.21" +version = "1.0.23" criteria = "safe-to-deploy" diff --git a/utils/apalis-codec/README.md b/utils/apalis-codec/README.md index 1d63d18d..6fbe0524 100644 --- a/utils/apalis-codec/README.md +++ b/utils/apalis-codec/README.md @@ -12,7 +12,7 @@ It handles the conversion of task data to and from various formats for reliable - Multiple codec support (JSON, MessagePack, etc.) - Type-safe serialization - Error handling for codec operations -- Integration with all apalis backends with codec support via `BackendExt` +- Integration with all apalis backends with codec support via `Backend` ## Usage diff --git a/utils/apalis-codec/src/bincode.rs b/utils/apalis-codec/src/bincode.rs index 9b66eac9..3b87246f 100644 --- a/utils/apalis-codec/src/bincode.rs +++ b/utils/apalis-codec/src/bincode.rs @@ -19,12 +19,12 @@ pub enum BincodeCodecError { impl> Codec for BincodeCodec { type Compact = Vec; type Error = BincodeCodecError; - fn encode(input: &T) -> Result, Self::Error> { + fn encode(&self, input: &T) -> Result, Self::Error> { let config = bincode::config::standard(); Ok(bincode::encode_to_vec(input, config)?) } - fn decode(compact: &Vec) -> Result { + fn decode(&self, compact: &Vec) -> Result { let config = bincode::config::standard(); Ok(bincode::decode_from_slice(compact, config)?.0) } @@ -46,8 +46,8 @@ mod tests { name: "test".to_string(), }; - let encoded = BincodeCodec::encode(&original).expect("encoding failed"); - let decoded: TestData = BincodeCodec::decode(&encoded).expect("decoding failed"); + let encoded = BincodeCodec.encode(&original).expect("encoding failed"); + let decoded: TestData = BincodeCodec.decode(&encoded).expect("decoding failed"); assert_eq!(original, decoded); } diff --git a/utils/apalis-codec/src/json.rs b/utils/apalis-codec/src/json.rs index 792ed7f5..6bda928c 100644 --- a/utils/apalis-codec/src/json.rs +++ b/utils/apalis-codec/src/json.rs @@ -13,11 +13,11 @@ pub struct JsonCodec { impl Deserialize<'de>> Codec for JsonCodec> { type Compact = Vec; type Error = serde_json::Error; - fn encode(input: &T) -> Result, Self::Error> { + fn encode(&self, input: &T) -> Result, Self::Error> { serde_json::to_vec(input) } - fn decode(compact: &Vec) -> Result { + fn decode(&self, compact: &Vec) -> Result { serde_json::from_slice(compact) } } @@ -25,10 +25,10 @@ impl Deserialize<'de>> Codec for JsonCodec> { impl Deserialize<'de>> Codec for JsonCodec { type Compact = String; type Error = serde_json::Error; - fn encode(input: &T) -> Result { + fn encode(&self, input: &T) -> Result { serde_json::to_string(input) } - fn decode(compact: &String) -> Result { + fn decode(&self, compact: &String) -> Result { serde_json::from_str(compact) } } @@ -36,11 +36,11 @@ impl Deserialize<'de>> Codec for JsonCodec { impl Deserialize<'de>> Codec for JsonCodec { type Compact = Value; type Error = serde_json::Error; - fn encode(input: &T) -> Result { + fn encode(&self, input: &T) -> Result { serde_json::to_value(input) } - fn decode(compact: &Value) -> Result { + fn decode(&self, compact: &Value) -> Result { T::deserialize(compact) } } @@ -61,8 +61,9 @@ mod tests { id: 1, name: "Test".to_string(), }; - let encoded = JsonCodec::>::encode(&original).unwrap(); - let decoded: TestStruct = JsonCodec::>::decode(&encoded).unwrap(); + let codec = JsonCodec::>::default(); + let encoded = codec.encode(&original).unwrap(); + let decoded: TestStruct = codec.decode(&encoded).unwrap(); assert_eq!(original, decoded); } @@ -72,8 +73,9 @@ mod tests { id: 2, name: "Example".to_string(), }; - let encoded = JsonCodec::::encode(&original).unwrap(); - let decoded: TestStruct = JsonCodec::::decode(&encoded).unwrap(); + let codec = JsonCodec::::default(); + let encoded = codec.encode(&original).unwrap(); + let decoded: TestStruct = codec.decode(&encoded).unwrap(); assert_eq!(original, decoded); } @@ -83,8 +85,9 @@ mod tests { id: 3, name: "Sample".to_string(), }; - let encoded = JsonCodec::::encode(&original).unwrap(); - let decoded: TestStruct = JsonCodec::::decode(&encoded).unwrap(); + let codec = JsonCodec::::default(); + let encoded = codec.encode(&original).unwrap(); + let decoded: TestStruct = codec.decode(&encoded).unwrap(); assert_eq!(original, decoded); } } diff --git a/utils/apalis-codec/src/msgpack.rs b/utils/apalis-codec/src/msgpack.rs index d7f6435e..52e0fbb3 100644 --- a/utils/apalis-codec/src/msgpack.rs +++ b/utils/apalis-codec/src/msgpack.rs @@ -19,11 +19,11 @@ pub enum MsgPackCodecError { impl Deserialize<'de>> Codec for MsgPackCodec { type Compact = Vec; type Error = MsgPackCodecError; - fn encode(input: &T) -> Result, Self::Error> { + fn encode(&self, input: &T) -> Result, Self::Error> { Ok(rmp_serde::to_vec(input)?) } - fn decode(compact: &Vec) -> Result { + fn decode(&self, compact: &Vec) -> Result { Ok(rmp_serde::from_slice(compact)?) } } @@ -45,10 +45,10 @@ mod tests { }; // Encode the original struct - let encoded = MsgPackCodec::encode(&original).expect("Encoding failed"); + let encoded = MsgPackCodec.encode(&original).expect("Encoding failed"); // Decode back to struct - let decoded: TestStruct = MsgPackCodec::decode(&encoded).expect("Decoding failed"); + let decoded: TestStruct = MsgPackCodec.decode(&encoded).expect("Decoding failed"); // Assert that the original and decoded structs are equal assert_eq!(original, decoded); diff --git a/utils/apalis-file-storage/src/error.rs b/utils/apalis-file-storage/src/error.rs index c5a2c405..f0192184 100644 --- a/utils/apalis-file-storage/src/error.rs +++ b/utils/apalis-file-storage/src/error.rs @@ -1,7 +1,8 @@ use crate::Adapter; +use std::fmt; /// Error type for [`FileStorage`]. -#[derive(Debug, thiserror::Error)] +#[derive(thiserror::Error)] pub enum FileStorageError { #[error("I/O error: {0}")] Io(#[from] std::io::Error), @@ -21,3 +22,23 @@ pub enum FileStorageError { #[error("Lock would block")] WouldBlockLock, } + +impl fmt::Debug for FileStorageError +where + A: Adapter, + A::Error: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Io(err) => f.debug_tuple("Io").field(err).finish(), + Self::Json(err) => f.debug_tuple("Json").field(err).finish(), + Self::JobNotFound { line_id } => f + .debug_struct("JobNotFound") + .field("line_id", line_id) + .finish(), + Self::Parse(msg) => f.debug_tuple("Parse").field(msg).finish(), + Self::AdapterError(err) => f.debug_tuple("AdapterError").field(err).finish(), + Self::WouldBlockLock => f.write_str("WouldBlockLock"), + } + } +} diff --git a/utils/apalis-file-storage/src/lib.rs b/utils/apalis-file-storage/src/lib.rs index 9263bc42..45a749eb 100644 --- a/utils/apalis-file-storage/src/lib.rs +++ b/utils/apalis-file-storage/src/lib.rs @@ -3,9 +3,8 @@ use apalis_codec::json::JsonCodec; use futures_core::{Stream, stream::BoxStream}; use futures_util::{ - StreamExt, TryStreamExt, + StreamExt, future::{Ready, ready}, - stream, }; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use serde_json::Value; @@ -29,7 +28,7 @@ use crate::error::FileStorageError; use self::util::RawTask; use apalis_core::{ - backend::{Backend, BackendExt, TaskResult, TaskStream, WaitForCompletion, queue::Queue}, + backend::{Backend, TaskResult, WaitForCompletion, queue::Queue}, error::BoxDynError, features_table, task::{ @@ -130,6 +129,7 @@ pub struct FileStorage { read_cursor: usize, sync_policy: SyncPolicy, last_flush: std::time::Instant, + codec: JsonCodec, _args: PhantomData, } @@ -147,6 +147,7 @@ where read_cursor: self.read_cursor, sync_policy: self.sync_policy.clone(), last_flush: self.last_flush, + codec: self.codec.clone(), _args: PhantomData, } } @@ -224,6 +225,7 @@ impl FileStorage { read_cursor: 0, sync_policy: SyncPolicy::Instant, last_flush: std::time::Instant::now(), + codec: JsonCodec::default(), _args: PhantomData, }) } @@ -462,62 +464,60 @@ impl Backend for FileStorage where Args: 'static + Send + Serialize + for<'de> Deserialize<'de> + Unpin, A: Adapter + Unpin + Clone, + A::Error: std::error::Error + Send + Sync + 'static, { type Args = Args; - type IdType = RandomId; + type Id = RandomId; type Error = FileStorageError; type Connection = MetadataStore; - type Stream = TaskStream, Self::Error>; type Layer = AcknowledgeLayer; - type Beat = BoxStream<'static, Result<(), Self::Error>>; - - fn heartbeat(&self, _: &WorkerContext) -> Self::Beat { - stream::once(async { Ok(()) }).boxed() - } + type Codec = JsonCodec; + type Compact = Value; fn middleware(&self) -> Self::Layer { AcknowledgeLayer::new(self.clone()) } - fn poll(self, _worker: &WorkerContext) -> Self::Stream { - (self - .map_ok(|(line_id, mut job)| { - let args = Args::deserialize(&job.args).unwrap(); - job.ctx.insert("line_id", line_id.to_string()).unwrap(); - let mut task = TaskBuilder::new(args).with_metadata(job.ctx); - - if let Some(task_id) = job.task_id { - task = task.task_id(task_id); - } - Some(task.build()) - }) - .boxed()) as _ + fn codec(&self) -> &Self::Codec { + &self.codec } -} - -impl BackendExt for FileStorage -where - Args: 'static + Send + Serialize + for<'de> Deserialize<'de> + Unpin, - A: Adapter + Unpin + Clone, -{ - type Codec = JsonCodec; - type Compact = Value; - - type CompactStream = - TaskStream, FileStorageError>; - - fn get_queue(&self) -> Queue { + fn queue(&self) -> Queue { std::any::type_name::().into() } + fn poll_ready( + &mut self, + cx: &mut Context<'_>, + _: &WorkerContext, + ) -> Poll> { + if self.lock.try_read().is_ok() { + Poll::Ready(Ok(())) + } else { + cx.waker().wake_by_ref(); + Poll::Pending + } + } - fn poll_compact(self, worker: &WorkerContext) -> Self::CompactStream { - self.poll(worker) - .map_ok(|c| { - c.map(|t| { - t.into_builder() - .map(|args| serde_json::to_value(args).expect("to be encodable")) - .build() - }) - }) - .boxed() + fn poll_next( + &mut self, + cx: &mut Context<'_>, + _: &WorkerContext, + ) -> Poll, Self::Error>>> { + self.poll_next_unpin(cx).map_ok(|(line_id, mut job)| { + job.ctx.insert("line_id", line_id.to_string()).unwrap(); + let mut task = TaskBuilder::new(job.args).with_metadata(job.ctx); + + if let Some(task_id) = job.task_id { + task = task.task_id(task_id); + } + task.build() + }) + } + + fn poll_close( + &mut self, + _: &mut Context<'_>, + _: &WorkerContext, + ) -> Poll> { + self.flush()?; + Poll::Ready(Ok(())) } } @@ -716,17 +716,15 @@ where } } -impl WaitForCompletion - for FileStorage +impl WaitForCompletion for FileStorage where Args: Send + DeserializeOwned + 'static + Unpin + Serialize, A: Adapter + Unpin + Sync + Clone, + A::Error: std::error::Error + Send + Sync + 'static, { - type ResultStream = BoxStream<'static, Result, FileStorageError>>; - fn wait_for( - &self, - task_ids: impl IntoIterator>, - ) -> Self::ResultStream { + type ResultStream = + BoxStream<'static, Result, FileStorageError>>; + fn wait_for(&self, task_ids: impl IntoIterator>) -> Self::ResultStream { use futures_util::StreamExt; use std::{collections::HashSet, time::Duration}; @@ -743,7 +741,7 @@ where poll_interval: Duration::from_millis(100), _phantom: std::marker::PhantomData, }; - futures_util::stream::unfold(state, |mut state: PollState| { + futures_util::stream::unfold(state, |mut state: PollState| { async move { if state.pending_tasks.is_empty() { return None; @@ -767,7 +765,7 @@ where if let Some((task_id, result)) = completed_task { state.pending_tasks.remove(&task_id); - let result: Result = serde_json::from_value(result).unwrap(); + let result: Result = serde_json::from_value(result).unwrap(); return Some(( Ok(TaskResult { task_id, @@ -788,8 +786,8 @@ where async fn check_status( &self, - task_ids: impl IntoIterator> + Send, - ) -> Result>, Self::Error> { + task_ids: impl IntoIterator> + Send, + ) -> Result>, Self::Error> { use apalis_core::task::status::Status; use std::collections::HashSet; let task_ids: HashSet<_> = task_ids.into_iter().collect(); @@ -810,7 +808,7 @@ where }); continue; } - let result = match serde_json::from_value::>( + let result = match serde_json::from_value::>( value.result.clone().unwrap(), ) { Ok(result) => TaskResult { diff --git a/utils/apalis-file-storage/src/shared.rs b/utils/apalis-file-storage/src/shared.rs index 326e2e3e..4a5667ab 100644 --- a/utils/apalis-file-storage/src/shared.rs +++ b/utils/apalis-file-storage/src/shared.rs @@ -147,7 +147,7 @@ impl SharedJsonStore { .unwrap(); let res = task - .try_map(|s| { + .try_map_args(|s| { serde_json::to_value(s).map_err(|e| MemoryStorageError::Other(e.into())) }) .map(|t| t.build());