Merge pull request #1550 from eikek/addons-experiment

Addons experiment

Author: mergify[bot]

build.sbt

@@ -293,6 +293,15 @@ val openapiScalaSettings = Seq(
           field.copy(typeDef =
             TypeDef("DownloadState", Imports("docspell.common.DownloadState"))
           )
+      case "addon-trigger-type" =>
+        field =>
+          field.copy(typeDef =
+            TypeDef("AddonTriggerType", Imports("docspell.addons.AddonTriggerType"))
+          )
+      case "addon-runner-type" =>
+        field =>
+          field
+            .copy(typeDef = TypeDef("RunnerType", Imports("docspell.addons.RunnerType")))
     })
 )
 
@@ -325,6 +334,7 @@ val common = project
     libraryDependencies ++=
       Dependencies.fs2 ++
         Dependencies.circe ++
+        Dependencies.circeGenericExtra ++
         Dependencies.calevCore ++
         Dependencies.calevCirce
   )
@@ -351,7 +361,7 @@ val files = project
   .in(file("modules/files"))
   .disablePlugins(RevolverPlugin)
   .settings(sharedSettings)
-  .withTestSettings
+  .withTestSettingsDependsOn(loggingScribe)
   .settings(
     name := "docspell-files",
     libraryDependencies ++=
@@ -448,6 +458,19 @@ val notificationApi = project
   )
   .dependsOn(common, loggingScribe)
 
+val addonlib = project
+  .in(file("modules/addonlib"))
+  .disablePlugins(RevolverPlugin)
+  .settings(sharedSettings)
+  .withTestSettingsDependsOn(loggingScribe)
+  .settings(
+    libraryDependencies ++=
+      Dependencies.fs2 ++
+        Dependencies.circe ++
+        Dependencies.circeYaml
+  )
+  .dependsOn(common, files, loggingScribe)
+
 val store = project
   .in(file("modules/store"))
   .disablePlugins(RevolverPlugin)
@@ -469,7 +492,16 @@ val store = project
     libraryDependencies ++=
       Dependencies.testContainer.map(_ % Test)
   )
-  .dependsOn(common, query.jvm, totp, files, notificationApi, jsonminiq, loggingScribe)
+  .dependsOn(
+    common,
+    addonlib,
+    query.jvm,
+    totp,
+    files,
+    notificationApi,
+    jsonminiq,
+    loggingScribe
+  )
 
 val notificationImpl = project
   .in(file("modules/notification/impl"))
@@ -647,7 +679,7 @@ val restapi = project
     openapiSpec := (Compile / resourceDirectory).value / "docspell-openapi.yml",
     openapiStaticGen := OpenApiDocGenerator.Redoc
   )
-  .dependsOn(common, query.jvm, notificationApi, jsonminiq)
+  .dependsOn(common, query.jvm, notificationApi, jsonminiq, addonlib)
 
 val joexapi = project
   .in(file("modules/joexapi"))
@@ -667,7 +699,7 @@ val joexapi = project
     openapiSpec := (Compile / resourceDirectory).value / "joex-openapi.yml",
     openapiStaticGen := OpenApiDocGenerator.Redoc
   )
-  .dependsOn(common, loggingScribe)
+  .dependsOn(common, loggingScribe, addonlib)
 
 val backend = project
   .in(file("modules/backend"))
@@ -683,6 +715,7 @@ val backend = project
         Dependencies.emil
   )
   .dependsOn(
+    addonlib,
     store,
     notificationApi,
     joexapi,
@@ -739,7 +772,7 @@ val config = project
       Dependencies.fs2 ++
         Dependencies.pureconfig
   )
-  .dependsOn(common, loggingApi, ftspsql, store)
+  .dependsOn(common, loggingApi, ftspsql, store, addonlib)
 
 // --- Application(s)
 
@@ -946,6 +979,7 @@ val root = project
   )
   .aggregate(
     common,
+    addonlib,
     loggingApi,
     loggingScribe,
     config,

modules/addonlib/src/main/scala/docspell/addons/AddonArchive.scala

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.effect._
+import cats.syntax.all._
+import fs2.Stream
+import fs2.io.file.{Files, Path}
+
+import docspell.common._
+import docspell.files.Zip
+
+final case class AddonArchive(url: LenientUri, name: String, version: String) {
+  def nameAndVersion: String =
+    s"$name-$version"
+
+  def extractTo[F[_]: Async](
+      reader: UrlReader[F],
+      directory: Path,
+      withSubdir: Boolean = true,
+      glob: Glob = Glob.all
+  ): F[Path] = {
+    val logger = docspell.logging.getLogger[F]
+    val target =
+      if (withSubdir) directory.absolute / nameAndVersion
+      else directory.absolute
+
+    Files[F]
+      .exists(target)
+      .flatMap {
+        case true => target.pure[F]
+        case false =>
+          Files[F].createDirectories(target) *>
+            reader(url)
+              .through(Zip.unzip(8192, glob))
+              .through(Zip.saveTo(logger, target, moveUp = true))
+              .compile
+              .drain
+              .as(target)
+      }
+  }
+
+  /** Read meta either from the given directory or extract the url to find the metadata
+    * file to read
+    */
+  def readMeta[F[_]: Async](
+      urlReader: UrlReader[F],
+      directory: Option[Path] = None
+  ): F[AddonMeta] =
+    directory
+      .map(AddonMeta.findInDirectory[F])
+      .getOrElse(AddonMeta.findInZip(urlReader(url)))
+}
+
+object AddonArchive {
+  def read[F[_]: Async](
+      url: LenientUri,
+      urlReader: UrlReader[F],
+      extractDir: Option[Path] = None
+  ): F[AddonArchive] = {
+    val addon = AddonArchive(url, "", "")
+    addon
+      .readMeta(urlReader, extractDir)
+      .map(m => addon.copy(name = m.meta.name, version = m.meta.version))
+  }
+
+  def dockerAndFlakeExists[F[_]: Async](
+      archive: Either[Path, Stream[F, Byte]]
+  ): F[(Boolean, Boolean)] = {
+    val files = Files[F]
+    def forPath(path: Path): F[(Boolean, Boolean)] =
+      (files.exists(path / "Dockerfile"), files.exists(path / "flake.nix")).tupled
+
+    def forZip(data: Stream[F, Byte]): F[(Boolean, Boolean)] =
+      data
+        .through(Zip.unzip(8192, Glob("Dockerfile|flake.nix")))
+        .collect {
+          case bin if bin.name == "Dockerfile" => (true, false)
+          case bin if bin.name == "flake.nix"  => (false, true)
+        }
+        .compile
+        .fold((false, false))((r, e) => (r._1 || e._1, r._2 || e._2))
+
+    archive.fold(forPath, forZip)
+  }
+}

modules/addonlib/src/main/scala/docspell/addons/AddonExecutionResult.scala

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.Monoid
+import cats.syntax.all._
+
+case class AddonExecutionResult(
+    addonResults: List[AddonResult],
+    pure: Boolean
+) {
+  def addonResult: AddonResult = addonResults.combineAll
+  def isFailure: Boolean = addonResult.isFailure
+  def isSuccess: Boolean = addonResult.isSuccess
+}
+
+object AddonExecutionResult {
+  val empty: AddonExecutionResult =
+    AddonExecutionResult(Nil, false)
+
+  def combine(a: AddonExecutionResult, b: AddonExecutionResult): AddonExecutionResult =
+    AddonExecutionResult(
+      a.addonResults ::: b.addonResults,
+      a.pure && b.pure
+    )
+
+  implicit val executionResultMonoid: Monoid[AddonExecutionResult] =
+    Monoid.instance(empty, combine)
+}

modules/addonlib/src/main/scala/docspell/addons/AddonExecutor.scala

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.addons
+
+import cats.data.Kleisli
+import cats.effect._
+import cats.syntax.all._
+import fs2.Stream
+import fs2.io.file._
+
+import docspell.common.UrlReader
+import docspell.common.exec.Env
+import docspell.logging.Logger
+
+trait AddonExecutor[F[_]] {
+
+  def config: AddonExecutorConfig
+
+  def execute(logger: Logger[F]): AddonExec[F]
+
+  def execute(logger: Logger[F], in: InputEnv): F[AddonExecutionResult] =
+    execute(logger).run(in)
+}
+
+object AddonExecutor {
+
+  def apply[F[_]: Async](
+      cfg: AddonExecutorConfig,
+      urlReader: UrlReader[F]
+  ): AddonExecutor[F] =
+    new AddonExecutor[F] with AddonLoggerExtension {
+      val config = cfg
+
+      def execute(logger: Logger[F]): AddonExec[F] =
+        Kleisli { in =>
+          for {
+            _ <- logger.info(s"About to run ${in.addons.size} addon(s) in ${in.baseDir}")
+            ctx <- prepareDirectory(
+              logger,
+              in.baseDir,
+              in.outputDir,
+              in.cacheDir,
+              in.addons
+            )
+            rs <- ctx.traverse(c => runAddon(logger.withAddon(c), in.env)(c))
+            pure = ctx.foldl(true)((b, c) => b && c.meta.isPure)
+          } yield AddonExecutionResult(rs, pure)
+        }
+
+      private def prepareDirectory(
+          logger: Logger[F],
+          baseDir: Path,
+          outDir: Path,
+          cacheDir: Path,
+          addons: List[AddonRef]
+      ): F[List[Context]] =
+        for {
+          addonsDir <- Directory.create(baseDir / "addons")
+          _ <- Directory.createAll(Context.tempDir(baseDir), outDir, cacheDir)
+          _ <- Context
+            .userInputFile(baseDir)
+            .parent
+            .fold(().pure[F])(Files[F].createDirectories)
+          archives = addons.map(_.archive).distinctBy(_.url)
+          _ <- logger.info(s"Extract ${archives.size} addons to $addonsDir")
+          mkCtxs <- archives.traverse { archive =>
+            for {
+              _ <- logger.debug(s"Extracting $archive")
+              addonDir <- archive.extractTo(urlReader, addonsDir)
+              meta <- AddonMeta.findInDirectory(addonDir)
+              mkCtx = (ref: AddonRef) =>
+                Context(ref, meta, baseDir, addonDir, outDir, cacheDir)
+            } yield archive.url -> mkCtx
+          }
+          ctxFactory = mkCtxs.toMap
+          res = addons.map(ref => ctxFactory(ref.archive.url)(ref))
+        } yield res
+
+      private def runAddon(logger: Logger[F], env: Env)(
+          ctx: Context
+      ): F[AddonResult] =
+        for {
+          _ <- logger.info(s"Executing addon ${ctx.meta.nameAndVersion}")
+          _ <- logger.trace("Storing user input into file")
+          _ <- Stream
+            .emit(ctx.addon.args)
+            .through(fs2.text.utf8.encode)
+            .through(Files[F].writeAll(ctx.userInputFile, Flags.Write))
+            .compile
+            .drain
+
+          runner <- selectRunner(cfg, ctx.meta, ctx.addonDir)
+          result <- runner.run(logger, env, ctx)
+        } yield result
+    }
+
+  def selectRunner[F[_]: Async](
+      cfg: AddonExecutorConfig,
+      meta: AddonMeta,
+      addonDir: Path
+  ): F[AddonRunner[F]] =
+    for {
+      addonRunner <- meta.enabledTypes(Left(addonDir))
+      // intersect on list retains order in first
+      possibleRunner = cfg.runner
+        .intersect(addonRunner)
+        .map(AddonRunner.forType[F](cfg))
+      runner = possibleRunner match {
+        case Nil =>
+          AddonRunner.failWith(
+            s"No runner available for addon config ${meta.runner} and config ${cfg.runner}."
+          )
+        case list =>
+          AddonRunner.firstSuccessful(list)
+      }
+    } yield runner
+}