docspell/build.sbt
Eike Kettner bd5066740d Joex depends on backend module
The job executor depends on backend module, since it may control the
application via user tasks. The `ONode` can now be moved from the
store module into the backend module.
2020-05-10 21:03:12 +02:00

561 lines
19 KiB
Scala
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import com.github.eikek.sbt.openapi._
import scala.sys.process._
import com.typesafe.sbt.SbtGit.GitKeys._
import docspell.build._
import microsites.ExtraMdFileConfig
val toolsPackage = taskKey[Seq[File]]("Package the scripts/extension tools")
val elmCompileMode = settingKey[ElmCompileMode]("How to compile elm sources")
// --- Settings
val sharedSettings = Seq(
organization := "com.github.eikek",
scalaVersion := "2.13.2",
scalacOptions ++= Seq(
"-deprecation",
"-encoding", "UTF-8",
"-language:higherKinds",
"-feature",
"-Werror", // fail when there are warnings
"-unchecked",
"-Xlint:_",
"-Wdead-code",
"-Wunused",
"-Wvalue-discard",
"-Wnumeric-widen"
),
LocalRootProject/toolsPackage := {
val v = version.value
val logger = streams.value.log
val dir = (LocalRootProject/baseDirectory).value / "tools"
packageTools(logger, dir, v)
},
scalacOptions in (Compile, console) :=
(scalacOptions.value.filter(o => !o.contains("Xlint")) ++ Seq("-Xlint:_,-unused")),
scalacOptions in (Test, console) :=
(scalacOptions.value.filter(o => !o.contains("Xlint")) ++ Seq("-Xlint:_,-unused"))
)
val testSettings = Seq(
testFrameworks += new TestFramework("minitest.runner.Framework"),
libraryDependencies ++= Dependencies.miniTest ++ Dependencies.logging.map(_ % Test)
)
lazy val noPublish = Seq(
publish := {},
publishLocal := {},
publishArtifact := false
)
val elmSettings = Seq(
elmCompileMode := ElmCompileMode.Debug,
Compile/resourceGenerators += Def.task {
compileElm(streams.value.log
, (Compile/baseDirectory).value
, (Compile/resourceManaged).value
, name.value
, version.value
, elmCompileMode.value)
}.taskValue,
watchSources += Watched.WatchSource(
(Compile/sourceDirectory).value/"elm"
, FileFilter.globFilter("*.elm")
, HiddenFileFilter
)
)
val webjarSettings = Seq(
Compile/resourceGenerators += Def.task {
copyWebjarResources(Seq((sourceDirectory in Compile).value/"webjar")
, (Compile/resourceManaged).value
, name.value
, version.value
, streams.value.log
)
}.taskValue,
watchSources += Watched.WatchSource(
(Compile / sourceDirectory).value/"webjar"
, FileFilter.globFilter("*.js") || FileFilter.globFilter("*.css")
, HiddenFileFilter
)
)
def debianSettings(cfgFile: String) = Seq(
maintainer := "Eike Kettner <eike.kettner@posteo.de>",
packageSummary := description.value,
packageDescription := description.value,
mappings in Universal += {
val conf = (Compile / resourceDirectory).value / "reference.conf"
if (!conf.exists) {
sys.error(s"File $conf not found")
}
conf -> s"conf/$cfgFile.conf"
},
bashScriptExtraDefines += s"""addJava "-Dconfig.file=$${app_home}/../conf/$cfgFile.conf""""
)
val buildInfoSettings = Seq(
buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion, gitHeadCommit, gitHeadCommitDate, gitUncommittedChanges, gitDescribedVersion),
buildInfoOptions += BuildInfoOption.ToJson,
buildInfoOptions += BuildInfoOption.BuildTime
)
val openapiScalaSettings = Seq(
openapiScalaConfig := ScalaConfig().withJson(ScalaJson.circeSemiauto).
addMapping(CustomMapping.forType({
case TypeDef("LocalDateTime", _) =>
TypeDef("Timestamp", Imports("docspell.common.Timestamp"))
})).
addMapping(CustomMapping.forFormatType({
case "ident" => field =>
field.copy(typeDef = TypeDef("Ident", Imports("docspell.common.Ident")))
case "collectivestate" => field =>
field.copy(typeDef = TypeDef("CollectiveState", Imports("docspell.common.CollectiveState")))
case "userstate" => field =>
field.copy(typeDef = TypeDef("UserState", Imports("docspell.common.UserState")))
case "password" => field =>
field.copy(typeDef = TypeDef("Password", Imports("docspell.common.Password")))
case "contactkind" => field =>
field.copy(typeDef = TypeDef("ContactKind", Imports("docspell.common.ContactKind")))
case "direction" => field =>
field.copy(typeDef = TypeDef("Direction", Imports("docspell.common.Direction")))
case "priority" => field =>
field.copy(typeDef = TypeDef("Priority", Imports("docspell.common.Priority")))
case "jobstate" => field =>
field.copy(typeDef = TypeDef("JobState", Imports("docspell.common.JobState")))
case "loglevel" => field =>
field.copy(typeDef = TypeDef("LogLevel", Imports("docspell.common.LogLevel")))
case "mimetype" => field =>
field.copy(typeDef = TypeDef("MimeType", Imports("docspell.common.MimeType")))
case "itemstate" => field =>
field.copy(typeDef = TypeDef("ItemState", Imports("docspell.common.ItemState")))
case "nertag" => field =>
field.copy(typeDef = TypeDef("NerTag", Imports("docspell.common.NerTag")))
case "language" => field =>
field.copy(typeDef = TypeDef("Language", Imports("docspell.common.Language")))
case "calevent" => field =>
field.copy(typeDef = TypeDef("CalEvent", Imports("com.github.eikek.calev.CalEvent",
"com.github.eikek.calev.circe.CalevCirceCodec._")))
}))
)
// --- Modules
// Base module, everything depends on this including restapi and
// joexapi modules. This should aim to have least possible
// dependencies
val common = project.in(file("modules/common")).
disablePlugins(RevolverPlugin).
settings(sharedSettings).
settings(testSettings).
settings(
name := "docspell-common",
libraryDependencies ++=
Dependencies.fs2 ++
Dependencies.circe ++
Dependencies.loggingApi ++
Dependencies.calevCore ++
Dependencies.calevCirce ++
Dependencies.pureconfig.map(_ % "optional")
)
// Some example files for testing
// https://file-examples.com/index.php/sample-documents-download/sample-doc-download/
val files = project.in(file("modules/files")).
disablePlugins(RevolverPlugin).
settings(sharedSettings).
settings(testSettings).
settings(
name := "docspell-files",
libraryDependencies ++=
Dependencies.tika ++
Dependencies.icu4j,
Test / sourceGenerators += Def.task {
val base = (Test/resourceDirectory).value
val files = (base ** (_.isFile)) pair sbt.io.Path.relativeTo(base)
val lines = files.toList.map(_._2).map(s => {
val ident = s.replaceAll("[^a-zA-Z0-9_]+", "_")
ident -> s"""val $ident = createUrl("${s}")"""
})
val content = s"""package docspell.files
object ExampleFiles extends ExampleFilesSupport {
${lines.map(_._2).mkString("\n")}
val all = List(
${lines.map(_._1).mkString(",\n")}
)
}
"""
val target = (Test/sourceManaged).value/"scala"/"ExampleFiles.scala"
IO.createDirectory(target.getParentFile)
IO.write(target, content)
Seq(target)
}.taskValue
).dependsOn(common)
val store = project.in(file("modules/store")).
disablePlugins(RevolverPlugin).
settings(sharedSettings).
settings(testSettings).
settings(
name := "docspell-store",
libraryDependencies ++=
Dependencies.doobie ++
Dependencies.bitpeace ++
Dependencies.tika ++
Dependencies.fs2 ++
Dependencies.databases ++
Dependencies.flyway ++
Dependencies.loggingApi ++
Dependencies.emil ++
Dependencies.emilDoobie ++
Dependencies.calevCore ++
Dependencies.calevFs2
).dependsOn(common)
val extract = project.in(file("modules/extract")).
disablePlugins(RevolverPlugin).
settings(sharedSettings).
settings(testSettings).
settings(
name := "docspell-extract",
libraryDependencies ++=
Dependencies.fs2 ++
Dependencies.twelvemonkeys ++
Dependencies.pdfbox ++
Dependencies.poi ++
Dependencies.commonsIO ++
Dependencies.julOverSlf4j
).dependsOn(common, files % "compile->compile;test->test")
val convert = project.in(file("modules/convert")).
disablePlugins(RevolverPlugin).
settings(sharedSettings).
settings(testSettings).
settings(
name := "docspell-convert",
libraryDependencies ++=
Dependencies.flexmark ++
Dependencies.twelvemonkeys
).dependsOn(common, files % "compile->compile;test->test")
val analysis = project.in(file("modules/analysis")).
disablePlugins(RevolverPlugin).
enablePlugins(NerModelsPlugin).
settings(sharedSettings).
settings(testSettings).
settings(NerModelsPlugin.nerClassifierSettings).
settings(
name := "docspell-analysis",
libraryDependencies ++=
Dependencies.fs2 ++
Dependencies.stanfordNlpCore
).dependsOn(common, files % "test->test")
val restapi = project.in(file("modules/restapi")).
disablePlugins(RevolverPlugin).
enablePlugins(OpenApiSchema).
settings(sharedSettings).
settings(testSettings).
settings(openapiScalaSettings).
settings(
name := "docspell-restapi",
libraryDependencies ++=
Dependencies.circe,
openapiTargetLanguage := Language.Scala,
openapiPackage := Pkg("docspell.restapi.model"),
openapiSpec := (Compile/resourceDirectory).value/"docspell-openapi.yml",
).dependsOn(common)
val joexapi = project.in(file("modules/joexapi")).
disablePlugins(RevolverPlugin).
enablePlugins(OpenApiSchema).
settings(sharedSettings).
settings(testSettings).
settings(openapiScalaSettings).
settings(
name := "docspell-joexapi",
libraryDependencies ++=
Dependencies.circe ++
Dependencies.http4sClient,
openapiTargetLanguage := Language.Scala,
openapiPackage := Pkg("docspell.joexapi.model"),
openapiSpec := (Compile/resourceDirectory).value/"joex-openapi.yml"
).dependsOn(common)
val backend = project.in(file("modules/backend")).
disablePlugins(RevolverPlugin).
settings(sharedSettings).
settings(testSettings).
settings(
name := "docspell-backend",
libraryDependencies ++=
Dependencies.loggingApi ++
Dependencies.fs2 ++
Dependencies.bcrypt ++
Dependencies.http4sClient ++
Dependencies.emil
).dependsOn(store, joexapi)
val webapp = project.in(file("modules/webapp")).
disablePlugins(RevolverPlugin).
enablePlugins(OpenApiSchema).
settings(sharedSettings).
settings(elmSettings).
settings(webjarSettings).
settings(
name := "docspell-webapp",
openapiTargetLanguage := Language.Elm,
openapiPackage := Pkg("Api.Model"),
openapiSpec := (restapi/Compile/resourceDirectory).value/"docspell-openapi.yml",
openapiElmConfig := ElmConfig().withJson(ElmJson.decodePipeline)
)
// --- Application(s)
val joex = project.in(file("modules/joex")).
enablePlugins(BuildInfoPlugin
, JavaServerAppPackaging
, DebianPlugin
, SystemdPlugin).
settings(sharedSettings).
settings(testSettings).
settings(debianSettings("docspell-joex")).
settings(buildInfoSettings).
settings(
name := "docspell-joex",
libraryDependencies ++=
Dependencies.fs2 ++
Dependencies.http4s ++
Dependencies.circe ++
Dependencies.pureconfig ++
Dependencies.emilTnef ++
Dependencies.emilMarkdown ++
Dependencies.emilJsoup ++
Dependencies.jsoup ++
Dependencies.yamusca ++
Dependencies.loggingApi ++
Dependencies.logging.map(_ % Runtime),
addCompilerPlugin(Dependencies.kindProjectorPlugin),
addCompilerPlugin(Dependencies.betterMonadicFor),
buildInfoPackage := "docspell.joex",
reStart/javaOptions ++= Seq(s"-Dconfig.file=${(LocalRootProject/baseDirectory).value/"local"/"dev.conf"}")
).dependsOn(store, backend, extract, convert, analysis, joexapi, restapi)
val restserver = project.in(file("modules/restserver")).
enablePlugins(BuildInfoPlugin
, JavaServerAppPackaging
, DebianPlugin
, SystemdPlugin).
settings(sharedSettings).
settings(testSettings).
settings(debianSettings("docspell-server")).
settings(buildInfoSettings).
settings(
name := "docspell-restserver",
libraryDependencies ++=
Dependencies.http4s ++
Dependencies.circe ++
Dependencies.pureconfig ++
Dependencies.yamusca ++
Dependencies.webjars ++
Dependencies.loggingApi ++
Dependencies.logging.map(_ % Runtime),
addCompilerPlugin(Dependencies.kindProjectorPlugin),
addCompilerPlugin(Dependencies.betterMonadicFor),
buildInfoPackage := "docspell.restserver",
Compile/sourceGenerators += Def.task {
createWebjarSource(Dependencies.webjars, (Compile/sourceManaged).value)
}.taskValue,
Compile/resourceGenerators += Def.task {
copyWebjarResources(Seq((restapi/Compile/resourceDirectory).value/"docspell-openapi.yml")
, (Compile/resourceManaged).value
, name.value
, version.value
, streams.value.log)
}.taskValue,
Compile/unmanagedResourceDirectories ++= Seq((Compile/resourceDirectory).value.getParentFile/"templates"),
reStart/javaOptions ++= Seq(s"-Dconfig.file=${(LocalRootProject/baseDirectory).value/"local"/"dev.conf"}")
).dependsOn(restapi, joexapi, backend, webapp)
// --- Microsite Documentation
val microsite = project.in(file("modules/microsite")).
disablePlugins(RevolverPlugin).
enablePlugins(MicrositesPlugin).
disablePlugins(ReleasePlugin).
settings(sharedSettings).
settings(
name := "docspell-microsite",
publishArtifact := false,
skip in publish := true,
micrositeFooterText := Some(
"""
|<p>&copy; 2020 <a href="https://github.com/eikek/docspell">Docspell, v{{site.version}}</a></p>
|""".stripMargin
),
micrositeName := "Docspell",
micrositeDescription := "Auto-tagging Document Organizer",
micrositeDocumentationUrl := "doc",
micrositeDocumentationLabelDescription := "Documentation",
micrositeFavicons := Seq(microsites.MicrositeFavicon("favicon.png", "96x96")),
micrositeAuthor := "eikek",
micrositeGithubOwner := "eikek",
micrositeGithubRepo := "docspell",
micrositeGitterChannel := false,
micrositeShareOnSocial := false,
micrositeHighlightLanguages ++= Seq("json", "javascript"),
micrositeEditButton := Some(microsites.MicrositeEditButton("Improve this page", "/edit/master/modules/microsite/docs/{{ page.path }}")),
fork in run := true,
micrositeCompilingDocsTool := WithMdoc,
mdocVariables := Map(
"VERSION" -> version.value,
"PVERSION" -> version.value.replace('.', '_')
),
micrositeExtraMdFiles := Map(
file("Changelog.md") -> ExtraMdFileConfig(
"changelog.md",
"docs",
Map("title" -> "Changelog", "permalink" -> "changelog")
)
),
Compile/resourceGenerators += Def.task {
val conf1 = (resourceDirectory in (restserver, Compile)).value / "reference.conf"
val conf2 = (resourceDirectory in (joex, Compile)).value / "reference.conf"
val out1 = resourceManaged.value/"main"/"jekyll"/"_includes"/"server.conf"
val out2 = resourceManaged.value/"main"/"jekyll"/"_includes"/"joex.conf"
streams.value.log.info(s"Copying reference.conf: $conf1 -> $out1, $conf2 -> $out2")
IO.write(out1, "{% raw %}\n")
IO.append(out1, IO.readBytes(conf1))
IO.write(out1, "\n{% endraw %}", append = true)
IO.write(out2, "{% raw %}\n")
IO.append(out2, IO.readBytes(conf2))
IO.write(out2, "\n{% endraw %}", append = true)
val oa1 = (resourceDirectory in (restapi, Compile)).value/"docspell-openapi.yml"
val oaout = resourceManaged.value/"main"/"jekyll"/"openapi"/"docspell-openapi.yml"
IO.copy(Seq(oa1 -> oaout))
Seq(out1, out2, oaout)
}.taskValue,
Compile/resourceGenerators += Def.task {
val staticDoc = (restapi/Compile/openapiStaticDoc).value
val target = resourceManaged.value/"main"/"jekyll"/"openapi"/"docspell-openapi.html"
IO.copy(Seq(staticDoc -> target))
Seq(target)
}.taskValue
)
val root = project.in(file(".")).
settings(sharedSettings).
settings(noPublish).
settings(
name := "docspell-root"
).
aggregate(common
, extract
, convert
, analysis
, files
, store
, joexapi
, joex
, backend
, webapp
, restapi
, restserver)
// --- Helpers
def copyWebjarResources(src: Seq[File], base: File, artifact: String, version: String, logger: Logger): Seq[File] = {
val targetDir = base/"META-INF"/"resources"/"webjars"/artifact/version
logger.info(s"Copy webjar resources from ${src.size} files/directories.")
src.flatMap { dir =>
if (dir.isDirectory) {
val files = (dir ** "*").filter(_.isFile).get pair Path.relativeTo(dir)
files.map { case (f, name) =>
val target = targetDir/name
IO.createDirectories(Seq(target.getParentFile))
IO.copy(Seq(f -> target))
target
}
} else {
val target = targetDir/dir.name
IO.createDirectories(Seq(target.getParentFile))
IO.copy(Seq(dir -> target))
Seq(target)
}
}
}
def compileElm(logger: Logger, wd: File, outBase: File, artifact: String, version: String, mode: ElmCompileMode): Seq[File] = {
logger.info("Compile elm files ...")
val target = outBase/"META-INF"/"resources"/"webjars"/artifact/version/"docspell-app.js"
val cmd = Seq("elm", "make") ++ mode.flags ++ Seq("--output", target.toString)
val proc = Process(cmd ++ Seq(wd/"src"/"main"/"elm"/"Main.elm").map(_.toString), Some(wd))
val out = proc.!!
logger.info(out)
Seq(target)
}
def createWebjarSource(wj: Seq[ModuleID], out: File): Seq[File] = {
val target = out/"Webjars.scala"
val fields = wj.map(m => s"""val ${m.name.toLowerCase.filter(_ != '-')} = "/${m.name}/${m.revision}" """).mkString("\n\n")
val content = s"""package docspell.restserver.webapp
|object Webjars {
|$fields
|}
|""".stripMargin
IO.write(target, content)
Seq(target)
}
def packageTools(logger: Logger, dir: File, version: String): Seq[File] = {
val target = dir/"target"
IO.delete(target)
IO.createDirectory(target)
val archive = target/s"docspell-tools-${version}.zip"
logger.info(s"Packaging tools to $archive ...")
val webext = target/"docspell-firefox-extension.xpi"
val wx = dir/"webextension"
IO.zip(Seq(
wx/"_locales/de/messages.json" -> "_locales/de/messages.json",
wx/"_locales/en/messages.json" -> "_locales/en/messages.json",
wx/"docspell.js" -> "docspell.js",
wx/"icons"/"logo-48.png" -> "icons/logo-48.png",
wx/"icons"/"logo-96.png" -> "icons/logo-96.png",
wx/"manifest.json" -> "manifest.json"
), webext)
IO.zip(Seq(
webext -> s"docspell-tools-${version}/firefox/docspell-extension.xpi",
wx/"native/app_manifest.json" ->s"docspell-tools-${version}/firefox/native/app_manifest.json",
wx/"native/native.py" ->s"docspell-tools-${version}/firefox/native/native.py",
dir/"ds.sh" -> s"docspell-tools-${version}/ds.sh",
dir/"consumedir.sh" -> s"docspell-tools-${version}/consumedir.sh"
), archive)
Seq(archive)
}
// --- aliases
addCommandAlias("make", ";set webapp/elmCompileMode := ElmCompileMode.Production ;root/openapiCodegen ;root/test:compile")
addCommandAlias("make-zip", ";restserver/universal:packageBin ;joex/universal:packageBin")
addCommandAlias("make-deb", ";restserver/debian:packageBin ;joex/debian:packageBin")
addCommandAlias("make-tools", ";root/toolsPackage")
addCommandAlias("make-pkg", ";clean ;make ;make-zip ;make-deb ;make-tools")