Experiment with addons

Addons allow to execute external programs in some context inside
docspell. Currently it is possible to run them after processing files.
Addons are provided by URLs to zip files.

Author: eikek

build.sbt

@@ -293,6 +293,15 @@ val openapiScalaSettings = Seq(
           field.copy(typeDef =
             TypeDef("DownloadState", Imports("docspell.common.DownloadState"))
           )
+      case "addon-trigger-type" =>
+        field =>
+          field.copy(typeDef =
+            TypeDef("AddonTriggerType", Imports("docspell.addons.AddonTriggerType"))
+          )
+      case "addon-runner-type" =>
+        field =>
+          field
+            .copy(typeDef = TypeDef("RunnerType", Imports("docspell.addons.RunnerType")))
     })
 )
 
@@ -325,6 +334,7 @@ val common = project
     libraryDependencies ++=
       Dependencies.fs2 ++
         Dependencies.circe ++
+        Dependencies.circeGenericExtra ++
         Dependencies.calevCore ++
         Dependencies.calevCirce
   )
@@ -351,7 +361,7 @@ val files = project
   .in(file("modules/files"))
   .disablePlugins(RevolverPlugin)
   .settings(sharedSettings)
-  .withTestSettings
+  .withTestSettingsDependsOn(loggingScribe)
   .settings(
     name := "docspell-files",
     libraryDependencies ++=
@@ -448,6 +458,19 @@ val notificationApi = project
   )
   .dependsOn(common, loggingScribe)
 
+val addonlib = project
+  .in(file("modules/addonlib"))
+  .disablePlugins(RevolverPlugin)
+  .settings(sharedSettings)
+  .withTestSettingsDependsOn(loggingScribe)
+  .settings(
+    libraryDependencies ++=
+      Dependencies.fs2 ++
+        Dependencies.circe ++
+        Dependencies.circeYaml
+  )
+  .dependsOn(common, files, loggingScribe)
+
 val store = project
   .in(file("modules/store"))
   .disablePlugins(RevolverPlugin)
@@ -469,7 +492,16 @@ val store = project
     libraryDependencies ++=
       Dependencies.testContainer.map(_ % Test)
   )
-  .dependsOn(common, query.jvm, totp, files, notificationApi, jsonminiq, loggingScribe)
+  .dependsOn(
+    common,
+    addonlib,
+    query.jvm,
+    totp,
+    files,
+    notificationApi,
+    jsonminiq,
+    loggingScribe
+  )
 
 val notificationImpl = project
   .in(file("modules/notification/impl"))
@@ -647,7 +679,7 @@ val restapi = project
     openapiSpec := (Compile / resourceDirectory).value / "docspell-openapi.yml",
     openapiStaticGen := OpenApiDocGenerator.Redoc
   )
-  .dependsOn(common, query.jvm, notificationApi, jsonminiq)
+  .dependsOn(common, query.jvm, notificationApi, jsonminiq, addonlib)
 
 val joexapi = project
   .in(file("modules/joexapi"))
@@ -667,7 +699,7 @@ val joexapi = project
     openapiSpec := (Compile / resourceDirectory).value / "joex-openapi.yml",
     openapiStaticGen := OpenApiDocGenerator.Redoc
   )
-  .dependsOn(common, loggingScribe)
+  .dependsOn(common, loggingScribe, addonlib)
 
 val backend = project
   .in(file("modules/backend"))
@@ -683,6 +715,7 @@ val backend = project
         Dependencies.emil
   )
   .dependsOn(
+    addonlib,
     store,
     notificationApi,
     joexapi,
@@ -739,7 +772,7 @@ val config = project
       Dependencies.fs2 ++
         Dependencies.pureconfig
   )
-  .dependsOn(common, loggingApi, ftspsql, store)
+  .dependsOn(common, loggingApi, ftspsql, store, addonlib)
 
 // --- Application(s)
 
@@ -946,6 +979,7 @@ val root = project
   )
   .aggregate(
     common,
+    addonlib,
     loggingApi,
     loggingScribe,
     config,

modules/addonlib/src/main/scala/docspell/addons/AddonArchive.scala

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect._
+import cats.syntax.all._
+import fs2.Stream
+import fs2.io.file.{Files, Path}
+
+import docspell.common._
+import docspell.files.Zip
+
+final case class AddonArchive(url: LenientUri, name: String, version: String) {
+  def nameAndVersion: String =
+    s"$name-$version"
+
+  def extractTo[F[_]: Async](
+      reader: UrlReader[F],
+      directory: Path,
+      withSubdir: Boolean = true,
+      glob: Glob = Glob.all
+  ): F[Path] = {
+    val logger = docspell.logging.getLogger[F]
+    val target =
+      if (withSubdir) directory.absolute / nameAndVersion
+      else directory.absolute
+
+    Files[F]
+      .exists(target)
+      .flatMap {
+        case true => target.pure[F]
+        case false =>
+          Files[F].createDirectories(target) *>
+            reader(url)
+              .through(Zip.unzip(8192, glob))
+              .through(Zip.saveTo(logger, target, moveUp = true))
+              .compile
+              .drain
+              .as(target)
+      }
+  }
+
+  /** Read meta either from the given directory or extract the url to find the metadata
+    * file to read
+    */
+  def readMeta[F[_]: Async](
+      urlReader: UrlReader[F],
+      directory: Option[Path] = None
+  ): F[AddonMeta] =
+    directory
+      .map(AddonMeta.findInDirectory[F])
+      .getOrElse(AddonMeta.findInZip(urlReader(url)))
+}
+
+object AddonArchive {
+  def read[F[_]: Async](
+      url: LenientUri,
+      urlReader: UrlReader[F],
+      extractDir: Option[Path] = None
+  ): F[AddonArchive] = {
+    val addon = AddonArchive(url, "", "")
+    addon
+      .readMeta(urlReader, extractDir)
+      .map(m => addon.copy(name = m.meta.name, version = m.meta.version))
+  }
+
+  def dockerAndFlakeExists[F[_]: Async](
+      archive: Either[Path, Stream[F, Byte]]
+  ): F[(Boolean, Boolean)] = {
+    val files = Files[F]
+    def forPath(path: Path): F[(Boolean, Boolean)] =
+      (files.exists(path / "Dockerfile"), files.exists(path / "flake.nix")).tupled
+
+    def forZip(data: Stream[F, Byte]): F[(Boolean, Boolean)] =
+      data
+        .through(Zip.unzip(8192, Glob("Dockerfile|flake.nix")))
+        .collect {
+          case bin if bin.name == "Dockerfile" => (true, false)
+          case bin if bin.name == "flake.nix"  => (false, true)
+        }
+        .compile
+        .fold((false, false))((r, e) => (r._1 || e._1, r._2 || e._2))
+
+    archive.fold(forPath, forZip)
+  }
+}

modules/addonlib/src/main/scala/docspell/addons/AddonExecutionResult.scala

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.Monoid
+import cats.syntax.all._
+
+case class AddonExecutionResult(
+    addonResults: List[AddonResult],
+    pure: Boolean
+) {
+  def addonResult: AddonResult = addonResults.combineAll
+  def isFailure: Boolean = addonResult.isFailure
+  def isSuccess: Boolean = addonResult.isSuccess
+}
+
+object AddonExecutionResult {
+  val empty: AddonExecutionResult =
+    AddonExecutionResult(Nil, false)
+
+  def combine(a: AddonExecutionResult, b: AddonExecutionResult): AddonExecutionResult =
+    AddonExecutionResult(
+      a.addonResults ::: b.addonResults,
+      a.pure && b.pure
+    )
+
+  implicit val executionResultMonoid: Monoid[AddonExecutionResult] =
+    Monoid.instance(empty, combine)
+}

modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.data.Kleisli
+import cats.effect._
+import cats.syntax.all._
+import fs2.Stream
+import fs2.io.file._
+
+import docspell.common.UrlReader
+import docspell.common.exec.Env
+import docspell.logging.Logger
+
+trait AddonExecutor[F[_]] {
+
+  def config: AddonExecutorConfig
+
+  def execute(logger: Logger[F]): AddonExec[F]
+
+  def execute(logger: Logger[F], in: InputEnv): F[AddonExecutionResult] =
+    execute(logger).run(in)
+}
+
+object AddonExecutor {
+
+  def apply[F[_]: Async](
+      cfg: AddonExecutorConfig,
+      urlReader: UrlReader[F]
+  ): AddonExecutor[F] =
+    new AddonExecutor[F] with AddonLoggerExtension {
+      val config = cfg
+
+      def execute(logger: Logger[F]): AddonExec[F] =
+        Kleisli { in =>
+          for {
+            _ <- logger.info(s"About to run ${in.addons.size} addon(s) in ${in.baseDir}")
+            ctx <- prepareDirectory(
+              logger,
+              in.baseDir,
+              in.outputDir,
+              in.cacheDir,
+              in.addons
+            )
+            rs <- ctx.traverse(c => runAddon(logger.withAddon(c), in.env)(c))
+            pure = ctx.foldl(true)((b, c) => b && c.meta.isPure)
+          } yield AddonExecutionResult(rs, pure)
+        }
+
+      private def prepareDirectory(
+          logger: Logger[F],
+          baseDir: Path,
+          outDir: Path,
+          cacheDir: Path,
+          addons: List[AddonRef]
+      ): F[List[Context]] =
+        for {
+          addonsDir <- Directory.create(baseDir / "addons")
+          _ <- Directory.createAll(Context.tempDir(baseDir), outDir, cacheDir)
+          _ <- Context
+            .userInputFile(baseDir)
+            .parent
+            .fold(().pure[F])(Files[F].createDirectories)
+          archives = addons.map(_.archive).distinctBy(_.url)
+          _ <- logger.info(s"Extract ${archives.size} addons to $addonsDir")
+          mkCtxs <- archives.traverse { archive =>
+            for {
+              _ <- logger.debug(s"Extracting $archive")
+              addonDir <- archive.extractTo(urlReader, addonsDir)
+              meta <- AddonMeta.findInDirectory(addonDir)
+              mkCtx = (ref: AddonRef) =>
+                Context(ref, meta, baseDir, addonDir, outDir, cacheDir)
+            } yield archive.url -> mkCtx
+          }
+          ctxFactory = mkCtxs.toMap
+          res = addons.map(ref => ctxFactory(ref.archive.url)(ref))
+        } yield res
+
+      private def runAddon(logger: Logger[F], env: Env)(
+          ctx: Context
+      ): F[AddonResult] =
+        for {
+          _ <- logger.info(s"Executing addon ${ctx.meta.nameAndVersion}")
+          _ <- logger.trace("Storing user input into file")
+          _ <- Stream
+            .emit(ctx.addon.args)
+            .through(fs2.text.utf8.encode)
+            .through(Files[F].writeAll(ctx.userInputFile, Flags.Write))
+            .compile
+            .drain
+
+          runner <- selectRunner(cfg, ctx.meta, ctx.addonDir)
+          result <- runner.run(logger, env, ctx)
+        } yield result
+    }
+
+  def selectRunner[F[_]: Async](
+      cfg: AddonExecutorConfig,
+      meta: AddonMeta,
+      addonDir: Path
+  ): F[AddonRunner[F]] =
+    for {
+      addonRunner <- meta.enabledTypes(Left(addonDir))
+      // intersect on list retains order in first
+      possibleRunner = cfg.runner
+        .intersect(addonRunner)
+        .map(AddonRunner.forType[F](cfg))
+      runner = possibleRunner match {
+        case Nil =>
+          AddonRunner.failWith(
+            s"No runner available for addon config ${meta.runner} and config ${cfg.runner}."
+          )
+        case list =>
+          AddonRunner.firstSuccessful(list)
+      }
+    } yield runner
+}