mirror of
https://github.com/TheAnachronism/docspell.git
synced 2025-04-08 07:59:32 +00:00
Merge branch 'master' into update/yamusca-circe-0.9.0
This commit is contained in:
commit
3cb293b29c
133
build.sbt
133
build.sbt
@ -9,9 +9,6 @@ val elmCompileMode = settingKey[ElmCompileMode]("How to compile elm sources")
|
|||||||
|
|
||||||
// --- Settings
|
// --- Settings
|
||||||
|
|
||||||
def inTest(d0: Seq[ModuleID], ds: Seq[ModuleID]*) =
|
|
||||||
ds.fold(d0)(_ ++ _).map(_ % Test)
|
|
||||||
|
|
||||||
val scalafixSettings = Seq(
|
val scalafixSettings = Seq(
|
||||||
semanticdbEnabled := true, // enable SemanticDB
|
semanticdbEnabled := true, // enable SemanticDB
|
||||||
semanticdbVersion := scalafixSemanticdb.revision, // "4.4.0"
|
semanticdbVersion := scalafixSemanticdb.revision, // "4.4.0"
|
||||||
@ -58,14 +55,10 @@ val sharedSettings = Seq(
|
|||||||
libraryDependencySchemes ++= Seq(
|
libraryDependencySchemes ++= Seq(
|
||||||
"com.github.eikek" %% "calev-core" % VersionScheme.Always,
|
"com.github.eikek" %% "calev-core" % VersionScheme.Always,
|
||||||
"com.github.eikek" %% "calev-circe" % VersionScheme.Always
|
"com.github.eikek" %% "calev-circe" % VersionScheme.Always
|
||||||
)
|
),
|
||||||
|
addCompilerPlugin(Dependencies.kindProjectorPlugin)
|
||||||
) ++ scalafixSettings
|
) ++ scalafixSettings
|
||||||
|
|
||||||
val testSettingsMUnit = Seq(
|
|
||||||
libraryDependencies ++= inTest(Dependencies.munit, Dependencies.logging),
|
|
||||||
testFrameworks += new TestFramework("munit.Framework")
|
|
||||||
)
|
|
||||||
|
|
||||||
lazy val noPublish = Seq(
|
lazy val noPublish = Seq(
|
||||||
publish := {},
|
publish := {},
|
||||||
publishLocal := {},
|
publishLocal := {},
|
||||||
@ -294,6 +287,20 @@ val openapiScalaSettings = Seq(
|
|||||||
|
|
||||||
// --- Modules
|
// --- Modules
|
||||||
|
|
||||||
|
val loggingApi = project
|
||||||
|
.in(file("modules/logging/api"))
|
||||||
|
.disablePlugins(RevolverPlugin)
|
||||||
|
.settings(sharedSettings)
|
||||||
|
.withTestSettings
|
||||||
|
.settings(
|
||||||
|
name := "docspell-logging-api",
|
||||||
|
libraryDependencies ++=
|
||||||
|
Dependencies.catsEffect ++
|
||||||
|
Dependencies.circeCore ++
|
||||||
|
Dependencies.fs2Core ++
|
||||||
|
Dependencies.sourcecode
|
||||||
|
)
|
||||||
|
|
||||||
// Base module, everything depends on this – including restapi and
|
// Base module, everything depends on this – including restapi and
|
||||||
// joexapi modules. This should aim to have least possible
|
// joexapi modules. This should aim to have least possible
|
||||||
// dependencies
|
// dependencies
|
||||||
@ -301,31 +308,44 @@ val common = project
|
|||||||
.in(file("modules/common"))
|
.in(file("modules/common"))
|
||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettings
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-common",
|
name := "docspell-common",
|
||||||
addCompilerPlugin(Dependencies.kindProjectorPlugin),
|
|
||||||
libraryDependencies ++=
|
libraryDependencies ++=
|
||||||
Dependencies.fs2 ++
|
Dependencies.fs2 ++
|
||||||
Dependencies.circe ++
|
Dependencies.circe ++
|
||||||
Dependencies.loggingApi ++
|
|
||||||
Dependencies.calevCore ++
|
Dependencies.calevCore ++
|
||||||
Dependencies.calevCirce
|
Dependencies.calevCirce
|
||||||
)
|
)
|
||||||
|
.dependsOn(loggingApi)
|
||||||
|
|
||||||
val config = project
|
val config = project
|
||||||
.in(file("modules/config"))
|
.in(file("modules/config"))
|
||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettings
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-config",
|
name := "docspell-config",
|
||||||
addCompilerPlugin(Dependencies.kindProjectorPlugin),
|
|
||||||
libraryDependencies ++=
|
libraryDependencies ++=
|
||||||
Dependencies.fs2 ++
|
Dependencies.fs2 ++
|
||||||
Dependencies.pureconfig
|
Dependencies.pureconfig
|
||||||
)
|
)
|
||||||
.dependsOn(common)
|
.dependsOn(common, loggingApi)
|
||||||
|
|
||||||
|
val loggingScribe = project
|
||||||
|
.in(file("modules/logging/scribe"))
|
||||||
|
.disablePlugins(RevolverPlugin)
|
||||||
|
.settings(sharedSettings)
|
||||||
|
.withTestSettings
|
||||||
|
.settings(
|
||||||
|
name := "docspell-logging-scribe",
|
||||||
|
libraryDependencies ++=
|
||||||
|
Dependencies.scribe ++
|
||||||
|
Dependencies.catsEffect ++
|
||||||
|
Dependencies.circeCore ++
|
||||||
|
Dependencies.fs2Core
|
||||||
|
)
|
||||||
|
.dependsOn(loggingApi)
|
||||||
|
|
||||||
// Some example files for testing
|
// Some example files for testing
|
||||||
// https://file-examples.com/index.php/sample-documents-download/sample-doc-download/
|
// https://file-examples.com/index.php/sample-documents-download/sample-doc-download/
|
||||||
@ -333,7 +353,7 @@ val files = project
|
|||||||
.in(file("modules/files"))
|
.in(file("modules/files"))
|
||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettings
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-files",
|
name := "docspell-files",
|
||||||
libraryDependencies ++=
|
libraryDependencies ++=
|
||||||
@ -372,7 +392,7 @@ val query =
|
|||||||
.in(file("modules/query"))
|
.in(file("modules/query"))
|
||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettings
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-query",
|
name := "docspell-query",
|
||||||
libraryDependencies +=
|
libraryDependencies +=
|
||||||
@ -392,7 +412,7 @@ val totp = project
|
|||||||
.in(file("modules/totp"))
|
.in(file("modules/totp"))
|
||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettings
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-totp",
|
name := "docspell-totp",
|
||||||
libraryDependencies ++=
|
libraryDependencies ++=
|
||||||
@ -406,7 +426,7 @@ val jsonminiq = project
|
|||||||
.in(file("modules/jsonminiq"))
|
.in(file("modules/jsonminiq"))
|
||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettings
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-jsonminiq",
|
name := "docspell-jsonminiq",
|
||||||
libraryDependencies ++=
|
libraryDependencies ++=
|
||||||
@ -419,25 +439,23 @@ val notificationApi = project
|
|||||||
.in(file("modules/notification/api"))
|
.in(file("modules/notification/api"))
|
||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettings
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-notification-api",
|
name := "docspell-notification-api",
|
||||||
addCompilerPlugin(Dependencies.kindProjectorPlugin),
|
|
||||||
libraryDependencies ++=
|
libraryDependencies ++=
|
||||||
Dependencies.fs2 ++
|
Dependencies.fs2 ++
|
||||||
Dependencies.emilCommon ++
|
Dependencies.emilCommon ++
|
||||||
Dependencies.circeGenericExtra
|
Dependencies.circeGenericExtra
|
||||||
)
|
)
|
||||||
.dependsOn(common)
|
.dependsOn(common, loggingScribe)
|
||||||
|
|
||||||
val store = project
|
val store = project
|
||||||
.in(file("modules/store"))
|
.in(file("modules/store"))
|
||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettingsDependsOn(loggingScribe)
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-store",
|
name := "docspell-store",
|
||||||
addCompilerPlugin(Dependencies.kindProjectorPlugin),
|
|
||||||
libraryDependencies ++=
|
libraryDependencies ++=
|
||||||
Dependencies.doobie ++
|
Dependencies.doobie ++
|
||||||
Dependencies.binny ++
|
Dependencies.binny ++
|
||||||
@ -445,7 +463,6 @@ val store = project
|
|||||||
Dependencies.fs2 ++
|
Dependencies.fs2 ++
|
||||||
Dependencies.databases ++
|
Dependencies.databases ++
|
||||||
Dependencies.flyway ++
|
Dependencies.flyway ++
|
||||||
Dependencies.loggingApi ++
|
|
||||||
Dependencies.emil ++
|
Dependencies.emil ++
|
||||||
Dependencies.emilDoobie ++
|
Dependencies.emilDoobie ++
|
||||||
Dependencies.calevCore ++
|
Dependencies.calevCore ++
|
||||||
@ -453,16 +470,15 @@ val store = project
|
|||||||
libraryDependencies ++=
|
libraryDependencies ++=
|
||||||
Dependencies.testContainer.map(_ % Test)
|
Dependencies.testContainer.map(_ % Test)
|
||||||
)
|
)
|
||||||
.dependsOn(common, query.jvm, totp, files, notificationApi, jsonminiq)
|
.dependsOn(common, query.jvm, totp, files, notificationApi, jsonminiq, loggingScribe)
|
||||||
|
|
||||||
val notificationImpl = project
|
val notificationImpl = project
|
||||||
.in(file("modules/notification/impl"))
|
.in(file("modules/notification/impl"))
|
||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettings
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-notification-impl",
|
name := "docspell-notification-impl",
|
||||||
addCompilerPlugin(Dependencies.kindProjectorPlugin),
|
|
||||||
libraryDependencies ++=
|
libraryDependencies ++=
|
||||||
Dependencies.fs2 ++
|
Dependencies.fs2 ++
|
||||||
Dependencies.emil ++
|
Dependencies.emil ++
|
||||||
@ -479,10 +495,9 @@ val pubsubApi = project
|
|||||||
.in(file("modules/pubsub/api"))
|
.in(file("modules/pubsub/api"))
|
||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettings
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-pubsub-api",
|
name := "docspell-pubsub-api",
|
||||||
addCompilerPlugin(Dependencies.kindProjectorPlugin),
|
|
||||||
libraryDependencies ++=
|
libraryDependencies ++=
|
||||||
Dependencies.fs2
|
Dependencies.fs2
|
||||||
)
|
)
|
||||||
@ -492,10 +507,9 @@ val pubsubNaive = project
|
|||||||
.in(file("modules/pubsub/naive"))
|
.in(file("modules/pubsub/naive"))
|
||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettings
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-pubsub-naive",
|
name := "docspell-pubsub-naive",
|
||||||
addCompilerPlugin(Dependencies.kindProjectorPlugin),
|
|
||||||
libraryDependencies ++=
|
libraryDependencies ++=
|
||||||
Dependencies.fs2 ++
|
Dependencies.fs2 ++
|
||||||
Dependencies.http4sCirce ++
|
Dependencies.http4sCirce ++
|
||||||
@ -509,7 +523,7 @@ val extract = project
|
|||||||
.in(file("modules/extract"))
|
.in(file("modules/extract"))
|
||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettingsDependsOn(loggingScribe)
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-extract",
|
name := "docspell-extract",
|
||||||
libraryDependencies ++=
|
libraryDependencies ++=
|
||||||
@ -517,16 +531,15 @@ val extract = project
|
|||||||
Dependencies.twelvemonkeys ++
|
Dependencies.twelvemonkeys ++
|
||||||
Dependencies.pdfbox ++
|
Dependencies.pdfbox ++
|
||||||
Dependencies.poi ++
|
Dependencies.poi ++
|
||||||
Dependencies.commonsIO ++
|
Dependencies.commonsIO
|
||||||
Dependencies.julOverSlf4j
|
|
||||||
)
|
)
|
||||||
.dependsOn(common, files % "compile->compile;test->test")
|
.dependsOn(common, loggingScribe, files % "compile->compile;test->test")
|
||||||
|
|
||||||
val convert = project
|
val convert = project
|
||||||
.in(file("modules/convert"))
|
.in(file("modules/convert"))
|
||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettingsDependsOn(loggingScribe)
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-convert",
|
name := "docspell-convert",
|
||||||
libraryDependencies ++=
|
libraryDependencies ++=
|
||||||
@ -541,7 +554,7 @@ val analysis = project
|
|||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.enablePlugins(NerModelsPlugin)
|
.enablePlugins(NerModelsPlugin)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettingsDependsOn(loggingScribe)
|
||||||
.settings(NerModelsPlugin.nerClassifierSettings)
|
.settings(NerModelsPlugin.nerClassifierSettings)
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-analysis",
|
name := "docspell-analysis",
|
||||||
@ -549,24 +562,24 @@ val analysis = project
|
|||||||
Dependencies.fs2 ++
|
Dependencies.fs2 ++
|
||||||
Dependencies.stanfordNlpCore
|
Dependencies.stanfordNlpCore
|
||||||
)
|
)
|
||||||
.dependsOn(common, files % "test->test")
|
.dependsOn(common, files % "test->test", loggingScribe)
|
||||||
|
|
||||||
val ftsclient = project
|
val ftsclient = project
|
||||||
.in(file("modules/fts-client"))
|
.in(file("modules/fts-client"))
|
||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettings
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-fts-client",
|
name := "docspell-fts-client",
|
||||||
libraryDependencies ++= Seq.empty
|
libraryDependencies ++= Seq.empty
|
||||||
)
|
)
|
||||||
.dependsOn(common)
|
.dependsOn(common, loggingScribe)
|
||||||
|
|
||||||
val ftssolr = project
|
val ftssolr = project
|
||||||
.in(file("modules/fts-solr"))
|
.in(file("modules/fts-solr"))
|
||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettings
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-fts-solr",
|
name := "docspell-fts-solr",
|
||||||
libraryDependencies ++=
|
libraryDependencies ++=
|
||||||
@ -582,7 +595,7 @@ val restapi = project
|
|||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.enablePlugins(OpenApiSchema)
|
.enablePlugins(OpenApiSchema)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettings
|
||||||
.settings(openapiScalaSettings)
|
.settings(openapiScalaSettings)
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-restapi",
|
name := "docspell-restapi",
|
||||||
@ -600,7 +613,7 @@ val joexapi = project
|
|||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.enablePlugins(OpenApiSchema)
|
.enablePlugins(OpenApiSchema)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettings
|
||||||
.settings(openapiScalaSettings)
|
.settings(openapiScalaSettings)
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-joexapi",
|
name := "docspell-joexapi",
|
||||||
@ -613,33 +626,31 @@ val joexapi = project
|
|||||||
openapiSpec := (Compile / resourceDirectory).value / "joex-openapi.yml",
|
openapiSpec := (Compile / resourceDirectory).value / "joex-openapi.yml",
|
||||||
openapiStaticGen := OpenApiDocGenerator.Redoc
|
openapiStaticGen := OpenApiDocGenerator.Redoc
|
||||||
)
|
)
|
||||||
.dependsOn(common)
|
.dependsOn(common, loggingScribe)
|
||||||
|
|
||||||
val backend = project
|
val backend = project
|
||||||
.in(file("modules/backend"))
|
.in(file("modules/backend"))
|
||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettings
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-backend",
|
name := "docspell-backend",
|
||||||
libraryDependencies ++=
|
libraryDependencies ++=
|
||||||
Dependencies.loggingApi ++
|
|
||||||
Dependencies.fs2 ++
|
Dependencies.fs2 ++
|
||||||
Dependencies.bcrypt ++
|
Dependencies.bcrypt ++
|
||||||
Dependencies.http4sClient ++
|
Dependencies.http4sClient ++
|
||||||
Dependencies.emil
|
Dependencies.emil
|
||||||
)
|
)
|
||||||
.dependsOn(store, notificationApi, joexapi, ftsclient, totp, pubsubApi)
|
.dependsOn(store, notificationApi, joexapi, ftsclient, totp, pubsubApi, loggingApi)
|
||||||
|
|
||||||
val oidc = project
|
val oidc = project
|
||||||
.in(file("modules/oidc"))
|
.in(file("modules/oidc"))
|
||||||
.disablePlugins(RevolverPlugin)
|
.disablePlugins(RevolverPlugin)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettings
|
||||||
.settings(
|
.settings(
|
||||||
name := "docspell-oidc",
|
name := "docspell-oidc",
|
||||||
libraryDependencies ++=
|
libraryDependencies ++=
|
||||||
Dependencies.loggingApi ++
|
|
||||||
Dependencies.fs2 ++
|
Dependencies.fs2 ++
|
||||||
Dependencies.http4sClient ++
|
Dependencies.http4sClient ++
|
||||||
Dependencies.http4sCirce ++
|
Dependencies.http4sCirce ++
|
||||||
@ -647,7 +658,7 @@ val oidc = project
|
|||||||
Dependencies.circe ++
|
Dependencies.circe ++
|
||||||
Dependencies.jwtScala
|
Dependencies.jwtScala
|
||||||
)
|
)
|
||||||
.dependsOn(common)
|
.dependsOn(common, loggingScribe)
|
||||||
|
|
||||||
val webapp = project
|
val webapp = project
|
||||||
.in(file("modules/webapp"))
|
.in(file("modules/webapp"))
|
||||||
@ -678,7 +689,7 @@ val joex = project
|
|||||||
ClasspathJarPlugin
|
ClasspathJarPlugin
|
||||||
)
|
)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettings
|
||||||
.settings(debianSettings("docspell-joex"))
|
.settings(debianSettings("docspell-joex"))
|
||||||
.settings(buildInfoSettings)
|
.settings(buildInfoSettings)
|
||||||
.settings(
|
.settings(
|
||||||
@ -698,10 +709,7 @@ val joex = project
|
|||||||
Dependencies.emilMarkdown ++
|
Dependencies.emilMarkdown ++
|
||||||
Dependencies.emilJsoup ++
|
Dependencies.emilJsoup ++
|
||||||
Dependencies.jsoup ++
|
Dependencies.jsoup ++
|
||||||
Dependencies.yamusca ++
|
Dependencies.yamusca,
|
||||||
Dependencies.loggingApi ++
|
|
||||||
Dependencies.logging.map(_ % Runtime),
|
|
||||||
addCompilerPlugin(Dependencies.kindProjectorPlugin),
|
|
||||||
addCompilerPlugin(Dependencies.betterMonadicFor),
|
addCompilerPlugin(Dependencies.betterMonadicFor),
|
||||||
buildInfoPackage := "docspell.joex",
|
buildInfoPackage := "docspell.joex",
|
||||||
reStart / javaOptions ++= Seq(
|
reStart / javaOptions ++= Seq(
|
||||||
@ -713,6 +721,8 @@ val joex = project
|
|||||||
)
|
)
|
||||||
.dependsOn(
|
.dependsOn(
|
||||||
config,
|
config,
|
||||||
|
loggingApi,
|
||||||
|
loggingScribe,
|
||||||
store,
|
store,
|
||||||
backend,
|
backend,
|
||||||
extract,
|
extract,
|
||||||
@ -735,7 +745,7 @@ val restserver = project
|
|||||||
ClasspathJarPlugin
|
ClasspathJarPlugin
|
||||||
)
|
)
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
.settings(testSettingsMUnit)
|
.withTestSettings
|
||||||
.settings(debianSettings("docspell-server"))
|
.settings(debianSettings("docspell-server"))
|
||||||
.settings(buildInfoSettings)
|
.settings(buildInfoSettings)
|
||||||
.settings(
|
.settings(
|
||||||
@ -751,10 +761,7 @@ val restserver = project
|
|||||||
Dependencies.pureconfig ++
|
Dependencies.pureconfig ++
|
||||||
Dependencies.yamusca ++
|
Dependencies.yamusca ++
|
||||||
Dependencies.kittens ++
|
Dependencies.kittens ++
|
||||||
Dependencies.webjars ++
|
Dependencies.webjars,
|
||||||
Dependencies.loggingApi ++
|
|
||||||
Dependencies.logging.map(_ % Runtime),
|
|
||||||
addCompilerPlugin(Dependencies.kindProjectorPlugin),
|
|
||||||
addCompilerPlugin(Dependencies.betterMonadicFor),
|
addCompilerPlugin(Dependencies.betterMonadicFor),
|
||||||
buildInfoPackage := "docspell.restserver",
|
buildInfoPackage := "docspell.restserver",
|
||||||
Compile / sourceGenerators += Def.task {
|
Compile / sourceGenerators += Def.task {
|
||||||
@ -788,6 +795,8 @@ val restserver = project
|
|||||||
)
|
)
|
||||||
.dependsOn(
|
.dependsOn(
|
||||||
config,
|
config,
|
||||||
|
loggingApi,
|
||||||
|
loggingScribe,
|
||||||
restapi,
|
restapi,
|
||||||
joexapi,
|
joexapi,
|
||||||
backend,
|
backend,
|
||||||
@ -869,6 +878,8 @@ val root = project
|
|||||||
)
|
)
|
||||||
.aggregate(
|
.aggregate(
|
||||||
common,
|
common,
|
||||||
|
loggingApi,
|
||||||
|
loggingScribe,
|
||||||
config,
|
config,
|
||||||
extract,
|
extract,
|
||||||
convert,
|
convert,
|
||||||
|
@ -15,8 +15,7 @@ import docspell.analysis.contact.Contact
|
|||||||
import docspell.analysis.date.DateFind
|
import docspell.analysis.date.DateFind
|
||||||
import docspell.analysis.nlp._
|
import docspell.analysis.nlp._
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
|
import docspell.logging.Logger
|
||||||
import org.log4s.getLogger
|
|
||||||
|
|
||||||
trait TextAnalyser[F[_]] {
|
trait TextAnalyser[F[_]] {
|
||||||
|
|
||||||
@ -30,7 +29,6 @@ trait TextAnalyser[F[_]] {
|
|||||||
def classifier: TextClassifier[F]
|
def classifier: TextClassifier[F]
|
||||||
}
|
}
|
||||||
object TextAnalyser {
|
object TextAnalyser {
|
||||||
private[this] val logger = getLogger
|
|
||||||
|
|
||||||
case class Result(labels: Vector[NerLabel], dates: Vector[NerDateLabel]) {
|
case class Result(labels: Vector[NerLabel], dates: Vector[NerDateLabel]) {
|
||||||
|
|
||||||
@ -87,10 +85,11 @@ object TextAnalyser {
|
|||||||
private object Nlp {
|
private object Nlp {
|
||||||
def apply[F[_]: Async](
|
def apply[F[_]: Async](
|
||||||
cfg: TextAnalysisConfig.NlpConfig
|
cfg: TextAnalysisConfig.NlpConfig
|
||||||
): F[Input[F] => F[Vector[NerLabel]]] =
|
): F[Input[F] => F[Vector[NerLabel]]] = {
|
||||||
|
val log = docspell.logging.getLogger[F]
|
||||||
cfg.mode match {
|
cfg.mode match {
|
||||||
case NlpMode.Disabled =>
|
case NlpMode.Disabled =>
|
||||||
Logger.log4s(logger).info("NLP is disabled as defined in config.") *>
|
log.info("NLP is disabled as defined in config.") *>
|
||||||
Applicative[F].pure(_ => Vector.empty[NerLabel].pure[F])
|
Applicative[F].pure(_ => Vector.empty[NerLabel].pure[F])
|
||||||
case _ =>
|
case _ =>
|
||||||
PipelineCache(cfg.clearInterval)(
|
PipelineCache(cfg.clearInterval)(
|
||||||
@ -99,6 +98,7 @@ object TextAnalyser {
|
|||||||
)
|
)
|
||||||
.map(annotate[F])
|
.map(annotate[F])
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
final case class Input[F[_]](
|
final case class Input[F[_]](
|
||||||
key: Ident,
|
key: Ident,
|
||||||
|
@ -17,6 +17,7 @@ import docspell.analysis.classifier.TextClassifier._
|
|||||||
import docspell.analysis.nlp.Properties
|
import docspell.analysis.nlp.Properties
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.common.syntax.FileSyntax._
|
import docspell.common.syntax.FileSyntax._
|
||||||
|
import docspell.logging.Logger
|
||||||
|
|
||||||
import edu.stanford.nlp.classify.ColumnDataClassifier
|
import edu.stanford.nlp.classify.ColumnDataClassifier
|
||||||
|
|
||||||
|
@ -10,7 +10,7 @@ import cats.data.Kleisli
|
|||||||
import fs2.Stream
|
import fs2.Stream
|
||||||
|
|
||||||
import docspell.analysis.classifier.TextClassifier.Data
|
import docspell.analysis.classifier.TextClassifier.Data
|
||||||
import docspell.common._
|
import docspell.logging.Logger
|
||||||
|
|
||||||
trait TextClassifier[F[_]] {
|
trait TextClassifier[F[_]] {
|
||||||
|
|
||||||
|
@ -12,6 +12,7 @@ import cats.{Applicative, FlatMap}
|
|||||||
|
|
||||||
import docspell.analysis.NlpSettings
|
import docspell.analysis.NlpSettings
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
|
import docspell.logging.Logger
|
||||||
|
|
||||||
import edu.stanford.nlp.pipeline.StanfordCoreNLP
|
import edu.stanford.nlp.pipeline.StanfordCoreNLP
|
||||||
|
|
||||||
|
@ -19,14 +19,13 @@ import docspell.common._
|
|||||||
import edu.stanford.nlp.ie.AbstractSequenceClassifier
|
import edu.stanford.nlp.ie.AbstractSequenceClassifier
|
||||||
import edu.stanford.nlp.ie.crf.CRFClassifier
|
import edu.stanford.nlp.ie.crf.CRFClassifier
|
||||||
import edu.stanford.nlp.ling.{CoreAnnotations, CoreLabel}
|
import edu.stanford.nlp.ling.{CoreAnnotations, CoreLabel}
|
||||||
import org.log4s.getLogger
|
|
||||||
|
|
||||||
/** This is only using the CRFClassifier without building an analysis pipeline. The
|
/** This is only using the CRFClassifier without building an analysis pipeline. The
|
||||||
* ner-classifier cannot use results from POS-tagging etc. and is therefore not as good
|
* ner-classifier cannot use results from POS-tagging etc. and is therefore not as good
|
||||||
* as the [[StanfordNerAnnotator]]. But it uses less memory, while still being not bad.
|
* as the [[StanfordNerAnnotator]]. But it uses less memory, while still being not bad.
|
||||||
*/
|
*/
|
||||||
object BasicCRFAnnotator {
|
object BasicCRFAnnotator {
|
||||||
private[this] val logger = getLogger
|
private[this] val logger = docspell.logging.unsafeLogger
|
||||||
|
|
||||||
// assert correct resource names
|
// assert correct resource names
|
||||||
NLPLanguage.all.toList.foreach(classifierResource)
|
NLPLanguage.all.toList.foreach(classifierResource)
|
||||||
|
@ -15,8 +15,6 @@ import cats.implicits._
|
|||||||
import docspell.analysis.NlpSettings
|
import docspell.analysis.NlpSettings
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
|
|
||||||
import org.log4s.getLogger
|
|
||||||
|
|
||||||
/** Creating the StanfordCoreNLP pipeline is quite expensive as it involves IO and
|
/** Creating the StanfordCoreNLP pipeline is quite expensive as it involves IO and
|
||||||
* initializing large objects.
|
* initializing large objects.
|
||||||
*
|
*
|
||||||
@ -31,17 +29,19 @@ trait PipelineCache[F[_]] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
object PipelineCache {
|
object PipelineCache {
|
||||||
private[this] val logger = getLogger
|
private[this] val logger = docspell.logging.unsafeLogger
|
||||||
|
|
||||||
def apply[F[_]: Async](clearInterval: Duration)(
|
def apply[F[_]: Async](clearInterval: Duration)(
|
||||||
creator: NlpSettings => Annotator[F],
|
creator: NlpSettings => Annotator[F],
|
||||||
release: F[Unit]
|
release: F[Unit]
|
||||||
): F[PipelineCache[F]] =
|
): F[PipelineCache[F]] = {
|
||||||
|
val log = docspell.logging.getLogger[F]
|
||||||
for {
|
for {
|
||||||
data <- Ref.of(Map.empty[String, Entry[Annotator[F]]])
|
data <- Ref.of(Map.empty[String, Entry[Annotator[F]]])
|
||||||
cacheClear <- CacheClearing.create(data, clearInterval, release)
|
cacheClear <- CacheClearing.create(data, clearInterval, release)
|
||||||
_ <- Logger.log4s(logger).info("Creating nlp pipeline cache")
|
_ <- log.info("Creating nlp pipeline cache")
|
||||||
} yield new Impl[F](data, creator, cacheClear)
|
} yield new Impl[F](data, creator, cacheClear)
|
||||||
|
}
|
||||||
|
|
||||||
final private class Impl[F[_]: Async](
|
final private class Impl[F[_]: Async](
|
||||||
data: Ref[F, Map[String, Entry[Annotator[F]]]],
|
data: Ref[F, Map[String, Entry[Annotator[F]]]],
|
||||||
@ -116,7 +116,7 @@ object PipelineCache {
|
|||||||
for {
|
for {
|
||||||
counter <- Ref.of(0L)
|
counter <- Ref.of(0L)
|
||||||
cleaning <- Ref.of(None: Option[Fiber[F, Throwable, Unit]])
|
cleaning <- Ref.of(None: Option[Fiber[F, Throwable, Unit]])
|
||||||
log = Logger.log4s(logger)
|
log = docspell.logging.getLogger[F]
|
||||||
result <-
|
result <-
|
||||||
if (interval.millis <= 0)
|
if (interval.millis <= 0)
|
||||||
log
|
log
|
||||||
@ -145,7 +145,7 @@ object PipelineCache {
|
|||||||
release: F[Unit]
|
release: F[Unit]
|
||||||
)(implicit F: Async[F])
|
)(implicit F: Async[F])
|
||||||
extends CacheClearing[F] {
|
extends CacheClearing[F] {
|
||||||
private[this] val log = Logger.log4s[F](logger)
|
private[this] val log = docspell.logging.getLogger[F]
|
||||||
|
|
||||||
def withCache: Resource[F, Unit] =
|
def withCache: Resource[F, Unit] =
|
||||||
Resource.make(counter.update(_ + 1) *> cancelClear)(_ =>
|
Resource.make(counter.update(_ + 1) *> cancelClear)(_ =>
|
||||||
|
@ -14,10 +14,9 @@ import fs2.io.file.Path
|
|||||||
import docspell.common._
|
import docspell.common._
|
||||||
|
|
||||||
import edu.stanford.nlp.pipeline.{CoreDocument, StanfordCoreNLP}
|
import edu.stanford.nlp.pipeline.{CoreDocument, StanfordCoreNLP}
|
||||||
import org.log4s.getLogger
|
|
||||||
|
|
||||||
object StanfordNerAnnotator {
|
object StanfordNerAnnotator {
|
||||||
private[this] val logger = getLogger
|
private[this] val logger = docspell.logging.unsafeLogger
|
||||||
|
|
||||||
/** Runs named entity recognition on the given `text`.
|
/** Runs named entity recognition on the given `text`.
|
||||||
*
|
*
|
||||||
|
@ -17,11 +17,12 @@ import fs2.io.file.Files
|
|||||||
|
|
||||||
import docspell.analysis.classifier.TextClassifier.Data
|
import docspell.analysis.classifier.TextClassifier.Data
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
|
import docspell.logging.TestLoggingConfig
|
||||||
|
|
||||||
import munit._
|
import munit._
|
||||||
|
|
||||||
class StanfordTextClassifierSuite extends FunSuite {
|
class StanfordTextClassifierSuite extends FunSuite with TestLoggingConfig {
|
||||||
val logger = Logger.log4s[IO](org.log4s.getLogger)
|
val logger = docspell.logging.getLogger[IO]
|
||||||
|
|
||||||
test("learn from data") {
|
test("learn from data") {
|
||||||
val cfg = TextClassifierConfig(File.path(Paths.get("target")), NonEmptyList.of(Map()))
|
val cfg = TextClassifierConfig(File.path(Paths.get("target")), NonEmptyList.of(Map()))
|
||||||
|
@ -10,10 +10,11 @@ import docspell.analysis.Env
|
|||||||
import docspell.common.Language.NLPLanguage
|
import docspell.common.Language.NLPLanguage
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.files.TestFiles
|
import docspell.files.TestFiles
|
||||||
|
import docspell.logging.TestLoggingConfig
|
||||||
|
|
||||||
import munit._
|
import munit._
|
||||||
|
|
||||||
class BaseCRFAnnotatorSuite extends FunSuite {
|
class BaseCRFAnnotatorSuite extends FunSuite with TestLoggingConfig {
|
||||||
|
|
||||||
def annotate(language: NLPLanguage): String => Vector[NerLabel] =
|
def annotate(language: NLPLanguage): String => Vector[NerLabel] =
|
||||||
BasicCRFAnnotator.nerAnnotate(BasicCRFAnnotator.Cache.getAnnotator(language))
|
BasicCRFAnnotator.nerAnnotate(BasicCRFAnnotator.Cache.getAnnotator(language))
|
||||||
|
@ -14,11 +14,12 @@ import cats.effect.unsafe.implicits.global
|
|||||||
import docspell.analysis.Env
|
import docspell.analysis.Env
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.files.TestFiles
|
import docspell.files.TestFiles
|
||||||
|
import docspell.logging.TestLoggingConfig
|
||||||
|
|
||||||
import edu.stanford.nlp.pipeline.StanfordCoreNLP
|
import edu.stanford.nlp.pipeline.StanfordCoreNLP
|
||||||
import munit._
|
import munit._
|
||||||
|
|
||||||
class StanfordNerAnnotatorSuite extends FunSuite {
|
class StanfordNerAnnotatorSuite extends FunSuite with TestLoggingConfig {
|
||||||
lazy val germanClassifier =
|
lazy val germanClassifier =
|
||||||
new StanfordCoreNLP(Properties.nerGerman(None, false))
|
new StanfordCoreNLP(Properties.nerGerman(None, false))
|
||||||
lazy val englishClassifier =
|
lazy val englishClassifier =
|
||||||
|
@ -17,7 +17,6 @@ import docspell.store.queries.QLogin
|
|||||||
import docspell.store.records._
|
import docspell.store.records._
|
||||||
import docspell.totp.{OnetimePassword, Totp}
|
import docspell.totp.{OnetimePassword, Totp}
|
||||||
|
|
||||||
import org.log4s.getLogger
|
|
||||||
import org.mindrot.jbcrypt.BCrypt
|
import org.mindrot.jbcrypt.BCrypt
|
||||||
import scodec.bits.ByteVector
|
import scodec.bits.ByteVector
|
||||||
|
|
||||||
@ -41,8 +40,6 @@ trait Login[F[_]] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
object Login {
|
object Login {
|
||||||
private[this] val logger = getLogger
|
|
||||||
|
|
||||||
case class Config(
|
case class Config(
|
||||||
serverSecret: ByteVector,
|
serverSecret: ByteVector,
|
||||||
sessionValid: Duration,
|
sessionValid: Duration,
|
||||||
@ -93,7 +90,7 @@ object Login {
|
|||||||
def apply[F[_]: Async](store: Store[F], totp: Totp): Resource[F, Login[F]] =
|
def apply[F[_]: Async](store: Store[F], totp: Totp): Resource[F, Login[F]] =
|
||||||
Resource.pure[F, Login[F]](new Login[F] {
|
Resource.pure[F, Login[F]](new Login[F] {
|
||||||
|
|
||||||
private val logF = Logger.log4s(logger)
|
private val logF = docspell.logging.getLogger[F]
|
||||||
|
|
||||||
def loginExternal(config: Config)(accountId: AccountId): F[Result] =
|
def loginExternal(config: Config)(accountId: AccountId): F[Result] =
|
||||||
for {
|
for {
|
||||||
@ -124,7 +121,7 @@ object Login {
|
|||||||
case Right(acc) =>
|
case Right(acc) =>
|
||||||
for {
|
for {
|
||||||
data <- store.transact(QLogin.findUser(acc))
|
data <- store.transact(QLogin.findUser(acc))
|
||||||
_ <- Sync[F].delay(logger.trace(s"Account lookup: $data"))
|
_ <- logF.trace(s"Account lookup: $data")
|
||||||
res <-
|
res <-
|
||||||
if (data.exists(check(up.pass))) doLogin(config, acc, up.rememberMe)
|
if (data.exists(check(up.pass))) doLogin(config, acc, up.rememberMe)
|
||||||
else Result.invalidAuth.pure[F]
|
else Result.invalidAuth.pure[F]
|
||||||
|
@ -12,6 +12,7 @@ import cats.effect._
|
|||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.ftsclient.FtsClient
|
import docspell.ftsclient.FtsClient
|
||||||
import docspell.ftsclient.TextData
|
import docspell.ftsclient.TextData
|
||||||
|
import docspell.logging.Logger
|
||||||
import docspell.store.Store
|
import docspell.store.Store
|
||||||
import docspell.store.queries.QAttachment
|
import docspell.store.queries.QAttachment
|
||||||
import docspell.store.queries.QItem
|
import docspell.store.queries.QItem
|
||||||
|
@ -14,6 +14,7 @@ import cats.implicits._
|
|||||||
import docspell.backend.fulltext.CreateIndex
|
import docspell.backend.fulltext.CreateIndex
|
||||||
import docspell.backend.ops.OItem
|
import docspell.backend.ops.OItem
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
|
import docspell.logging.Logger
|
||||||
import docspell.store.Store
|
import docspell.store.Store
|
||||||
import docspell.store.queries.QCustomField
|
import docspell.store.queries.QCustomField
|
||||||
import docspell.store.queries.QCustomField.FieldValue
|
import docspell.store.queries.QCustomField.FieldValue
|
||||||
|
@ -37,11 +37,9 @@ trait OClientSettings[F[_]] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
object OClientSettings {
|
object OClientSettings {
|
||||||
private[this] val logger = org.log4s.getLogger
|
|
||||||
|
|
||||||
def apply[F[_]: Async](store: Store[F]): Resource[F, OClientSettings[F]] =
|
def apply[F[_]: Async](store: Store[F]): Resource[F, OClientSettings[F]] =
|
||||||
Resource.pure[F, OClientSettings[F]](new OClientSettings[F] {
|
Resource.pure[F, OClientSettings[F]](new OClientSettings[F] {
|
||||||
val log = Logger.log4s[F](logger)
|
val log = docspell.logging.getLogger[F]
|
||||||
|
|
||||||
private def getUserId(account: AccountId): OptionT[F, Ident] =
|
private def getUserId(account: AccountId): OptionT[F, Ident] =
|
||||||
OptionT(store.transact(RUser.findByAccount(account))).map(_.uid)
|
OptionT(store.transact(RUser.findByAccount(account))).map(_.uid)
|
||||||
|
@ -31,7 +31,6 @@ import docspell.store.records.RCustomFieldValue
|
|||||||
import docspell.store.records.RItem
|
import docspell.store.records.RItem
|
||||||
|
|
||||||
import doobie._
|
import doobie._
|
||||||
import org.log4s.getLogger
|
|
||||||
|
|
||||||
trait OCustomFields[F[_]] {
|
trait OCustomFields[F[_]] {
|
||||||
|
|
||||||
@ -153,7 +152,7 @@ object OCustomFields {
|
|||||||
): Resource[F, OCustomFields[F]] =
|
): Resource[F, OCustomFields[F]] =
|
||||||
Resource.pure[F, OCustomFields[F]](new OCustomFields[F] {
|
Resource.pure[F, OCustomFields[F]](new OCustomFields[F] {
|
||||||
|
|
||||||
private[this] val logger = Logger.log4s[ConnectionIO](getLogger)
|
private[this] val logger = docspell.logging.getLogger[ConnectionIO]
|
||||||
|
|
||||||
def findAllValues(itemIds: Nel[Ident]): F[List[FieldValue]] =
|
def findAllValues(itemIds: Nel[Ident]): F[List[FieldValue]] =
|
||||||
store.transact(QCustomField.findAllValues(itemIds))
|
store.transact(QCustomField.findAllValues(itemIds))
|
||||||
@ -224,7 +223,7 @@ object OCustomFields {
|
|||||||
.transact(RItem.existsByIdsAndCollective(items, value.collective))
|
.transact(RItem.existsByIdsAndCollective(items, value.collective))
|
||||||
.map(flag => if (flag) Right(()) else Left(SetValueResult.itemNotFound))
|
.map(flag => if (flag) Right(()) else Left(SetValueResult.itemNotFound))
|
||||||
)
|
)
|
||||||
nu <- EitherT.right[SetValueResult](
|
_ <- EitherT.right[SetValueResult](
|
||||||
items
|
items
|
||||||
.traverse(item => store.transact(RCustomField.setValue(field, item, fval)))
|
.traverse(item => store.transact(RCustomField.setValue(field, item, fval)))
|
||||||
.map(_.toList.sum)
|
.map(_.toList.sum)
|
||||||
|
@ -14,7 +14,6 @@ import fs2.Stream
|
|||||||
import docspell.backend.JobFactory
|
import docspell.backend.JobFactory
|
||||||
import docspell.backend.ops.OItemSearch._
|
import docspell.backend.ops.OItemSearch._
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.common.syntax.all._
|
|
||||||
import docspell.ftsclient._
|
import docspell.ftsclient._
|
||||||
import docspell.query.ItemQuery._
|
import docspell.query.ItemQuery._
|
||||||
import docspell.query.ItemQueryDsl._
|
import docspell.query.ItemQueryDsl._
|
||||||
@ -23,8 +22,6 @@ import docspell.store.queue.JobQueue
|
|||||||
import docspell.store.records.RJob
|
import docspell.store.records.RJob
|
||||||
import docspell.store.{Store, qb}
|
import docspell.store.{Store, qb}
|
||||||
|
|
||||||
import org.log4s.getLogger
|
|
||||||
|
|
||||||
trait OFulltext[F[_]] {
|
trait OFulltext[F[_]] {
|
||||||
|
|
||||||
def findItems(maxNoteLen: Int)(
|
def findItems(maxNoteLen: Int)(
|
||||||
@ -59,7 +56,6 @@ trait OFulltext[F[_]] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
object OFulltext {
|
object OFulltext {
|
||||||
private[this] val logger = getLogger
|
|
||||||
|
|
||||||
case class FtsInput(
|
case class FtsInput(
|
||||||
query: String,
|
query: String,
|
||||||
@ -89,16 +85,17 @@ object OFulltext {
|
|||||||
joex: OJoex[F]
|
joex: OJoex[F]
|
||||||
): Resource[F, OFulltext[F]] =
|
): Resource[F, OFulltext[F]] =
|
||||||
Resource.pure[F, OFulltext[F]](new OFulltext[F] {
|
Resource.pure[F, OFulltext[F]](new OFulltext[F] {
|
||||||
|
val logger = docspell.logging.getLogger[F]
|
||||||
def reindexAll: F[Unit] =
|
def reindexAll: F[Unit] =
|
||||||
for {
|
for {
|
||||||
_ <- logger.finfo(s"Re-index all.")
|
_ <- logger.info(s"Re-index all.")
|
||||||
job <- JobFactory.reIndexAll[F]
|
job <- JobFactory.reIndexAll[F]
|
||||||
_ <- queue.insertIfNew(job) *> joex.notifyAllNodes
|
_ <- queue.insertIfNew(job) *> joex.notifyAllNodes
|
||||||
} yield ()
|
} yield ()
|
||||||
|
|
||||||
def reindexCollective(account: AccountId): F[Unit] =
|
def reindexCollective(account: AccountId): F[Unit] =
|
||||||
for {
|
for {
|
||||||
_ <- logger.fdebug(s"Re-index collective: $account")
|
_ <- logger.debug(s"Re-index collective: $account")
|
||||||
exist <- store.transact(
|
exist <- store.transact(
|
||||||
RJob.findNonFinalByTracker(DocspellSystem.migrationTaskTracker)
|
RJob.findNonFinalByTracker(DocspellSystem.migrationTaskTracker)
|
||||||
)
|
)
|
||||||
@ -123,7 +120,7 @@ object OFulltext {
|
|||||||
FtsQuery.HighlightSetting(ftsQ.highlightPre, ftsQ.highlightPost)
|
FtsQuery.HighlightSetting(ftsQ.highlightPre, ftsQ.highlightPost)
|
||||||
)
|
)
|
||||||
for {
|
for {
|
||||||
_ <- logger.ftrace(s"Find index only: ${ftsQ.query}/$batch")
|
_ <- logger.trace(s"Find index only: ${ftsQ.query}/$batch")
|
||||||
folders <- store.transact(QFolder.getMemberFolders(account))
|
folders <- store.transact(QFolder.getMemberFolders(account))
|
||||||
ftsR <- fts.search(fq.withFolders(folders))
|
ftsR <- fts.search(fq.withFolders(folders))
|
||||||
ftsItems = ftsR.results.groupBy(_.itemId)
|
ftsItems = ftsR.results.groupBy(_.itemId)
|
||||||
|
@ -16,6 +16,7 @@ import docspell.backend.fulltext.CreateIndex
|
|||||||
import docspell.backend.item.Merge
|
import docspell.backend.item.Merge
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.ftsclient.FtsClient
|
import docspell.ftsclient.FtsClient
|
||||||
|
import docspell.logging.Logger
|
||||||
import docspell.notification.api.Event
|
import docspell.notification.api.Event
|
||||||
import docspell.store.queries.{QAttachment, QItem, QMoveAttachment}
|
import docspell.store.queries.{QAttachment, QItem, QMoveAttachment}
|
||||||
import docspell.store.queue.JobQueue
|
import docspell.store.queue.JobQueue
|
||||||
@ -23,7 +24,6 @@ import docspell.store.records._
|
|||||||
import docspell.store.{AddResult, Store, UpdateResult}
|
import docspell.store.{AddResult, Store, UpdateResult}
|
||||||
|
|
||||||
import doobie.implicits._
|
import doobie.implicits._
|
||||||
import org.log4s.getLogger
|
|
||||||
|
|
||||||
trait OItem[F[_]] {
|
trait OItem[F[_]] {
|
||||||
|
|
||||||
@ -235,7 +235,7 @@ object OItem {
|
|||||||
otag <- OTag(store)
|
otag <- OTag(store)
|
||||||
oorg <- OOrganization(store)
|
oorg <- OOrganization(store)
|
||||||
oequip <- OEquipment(store)
|
oequip <- OEquipment(store)
|
||||||
logger <- Resource.pure[F, Logger[F]](Logger.log4s(getLogger))
|
logger <- Resource.pure[F, Logger[F]](docspell.logging.getLogger[F])
|
||||||
oitem <- Resource.pure[F, OItem[F]](new OItem[F] {
|
oitem <- Resource.pure[F, OItem[F]](new OItem[F] {
|
||||||
|
|
||||||
def merge(
|
def merge(
|
||||||
|
@ -59,7 +59,7 @@ object OJob {
|
|||||||
pubsub: PubSubT[F]
|
pubsub: PubSubT[F]
|
||||||
): Resource[F, OJob[F]] =
|
): Resource[F, OJob[F]] =
|
||||||
Resource.pure[F, OJob[F]](new OJob[F] {
|
Resource.pure[F, OJob[F]](new OJob[F] {
|
||||||
private[this] val logger = Logger.log4s(org.log4s.getLogger(OJob.getClass))
|
private[this] val logger = docspell.logging.getLogger[F]
|
||||||
|
|
||||||
def queueState(collective: Ident, maxResults: Int): F[CollectiveQueueState] =
|
def queueState(collective: Ident, maxResults: Int): F[CollectiveQueueState] =
|
||||||
store
|
store
|
||||||
|
@ -9,13 +9,10 @@ package docspell.backend.ops
|
|||||||
import cats.effect.{Async, Resource}
|
import cats.effect.{Async, Resource}
|
||||||
import cats.implicits._
|
import cats.implicits._
|
||||||
|
|
||||||
import docspell.common.syntax.all._
|
|
||||||
import docspell.common.{Ident, LenientUri, NodeType}
|
import docspell.common.{Ident, LenientUri, NodeType}
|
||||||
import docspell.store.Store
|
import docspell.store.Store
|
||||||
import docspell.store.records.RNode
|
import docspell.store.records.RNode
|
||||||
|
|
||||||
import org.log4s._
|
|
||||||
|
|
||||||
trait ONode[F[_]] {
|
trait ONode[F[_]] {
|
||||||
|
|
||||||
def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit]
|
def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit]
|
||||||
@ -24,20 +21,19 @@ trait ONode[F[_]] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
object ONode {
|
object ONode {
|
||||||
private[this] val logger = getLogger
|
|
||||||
|
|
||||||
def apply[F[_]: Async](store: Store[F]): Resource[F, ONode[F]] =
|
def apply[F[_]: Async](store: Store[F]): Resource[F, ONode[F]] =
|
||||||
Resource.pure[F, ONode[F]](new ONode[F] {
|
Resource.pure[F, ONode[F]](new ONode[F] {
|
||||||
|
val logger = docspell.logging.getLogger[F]
|
||||||
def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit] =
|
def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit] =
|
||||||
for {
|
for {
|
||||||
node <- RNode(appId, nodeType, uri)
|
node <- RNode(appId, nodeType, uri)
|
||||||
_ <- logger.finfo(s"Registering node ${node.id.id}")
|
_ <- logger.info(s"Registering node ${node.id.id}")
|
||||||
_ <- store.transact(RNode.set(node))
|
_ <- store.transact(RNode.set(node))
|
||||||
} yield ()
|
} yield ()
|
||||||
|
|
||||||
def unregister(appId: Ident): F[Unit] =
|
def unregister(appId: Ident): F[Unit] =
|
||||||
logger.finfo(s"Unregister app ${appId.id}") *>
|
logger.info(s"Unregister app ${appId.id}") *>
|
||||||
store.transact(RNode.delete(appId)).map(_ => ())
|
store.transact(RNode.delete(appId)).map(_ => ())
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -6,9 +6,6 @@
|
|||||||
|
|
||||||
package docspell.backend.ops
|
package docspell.backend.ops
|
||||||
|
|
||||||
import java.io.PrintWriter
|
|
||||||
import java.io.StringWriter
|
|
||||||
|
|
||||||
import cats.data.OptionT
|
import cats.data.OptionT
|
||||||
import cats.data.{NonEmptyList => Nel}
|
import cats.data.{NonEmptyList => Nel}
|
||||||
import cats.effect._
|
import cats.effect._
|
||||||
@ -17,6 +14,7 @@ import cats.implicits._
|
|||||||
import docspell.backend.ops.ONotification.Hook
|
import docspell.backend.ops.ONotification.Hook
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.jsonminiq.JsonMiniQuery
|
import docspell.jsonminiq.JsonMiniQuery
|
||||||
|
import docspell.logging.{Level, LogEvent, Logger}
|
||||||
import docspell.notification.api._
|
import docspell.notification.api._
|
||||||
import docspell.store.AddResult
|
import docspell.store.AddResult
|
||||||
import docspell.store.Store
|
import docspell.store.Store
|
||||||
@ -75,14 +73,13 @@ trait ONotification[F[_]] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
object ONotification {
|
object ONotification {
|
||||||
private[this] val logger = org.log4s.getLogger
|
|
||||||
|
|
||||||
def apply[F[_]: Async](
|
def apply[F[_]: Async](
|
||||||
store: Store[F],
|
store: Store[F],
|
||||||
notMod: NotificationModule[F]
|
notMod: NotificationModule[F]
|
||||||
): Resource[F, ONotification[F]] =
|
): Resource[F, ONotification[F]] =
|
||||||
Resource.pure[F, ONotification[F]](new ONotification[F] {
|
Resource.pure[F, ONotification[F]](new ONotification[F] {
|
||||||
val log = Logger.log4s[F](logger)
|
val log = docspell.logging.getLogger[F]
|
||||||
|
|
||||||
def withUserId[A](
|
def withUserId[A](
|
||||||
account: AccountId
|
account: AccountId
|
||||||
@ -129,9 +126,9 @@ object ONotification {
|
|||||||
.map {
|
.map {
|
||||||
case Right(res) => res
|
case Right(res) => res
|
||||||
case Left(ex) =>
|
case Left(ex) =>
|
||||||
val ps = new StringWriter()
|
val ev =
|
||||||
ex.printStackTrace(new PrintWriter(ps))
|
LogEvent.of(Level.Error, "Failed sending sample event").addError(ex)
|
||||||
SendTestResult(false, Vector(s"${ex.getMessage}\n$ps"))
|
SendTestResult(false, Vector(ev))
|
||||||
}
|
}
|
||||||
|
|
||||||
def listChannels(account: AccountId): F[Vector[Channel]] =
|
def listChannels(account: AccountId): F[Vector[Channel]] =
|
||||||
@ -316,5 +313,5 @@ object ONotification {
|
|||||||
} yield h
|
} yield h
|
||||||
}
|
}
|
||||||
|
|
||||||
final case class SendTestResult(success: Boolean, logMessages: Vector[String])
|
final case class SendTestResult(success: Boolean, logEvents: Vector[LogEvent])
|
||||||
}
|
}
|
||||||
|
@ -152,7 +152,7 @@ object OShare {
|
|||||||
emil: Emil[F]
|
emil: Emil[F]
|
||||||
): OShare[F] =
|
): OShare[F] =
|
||||||
new OShare[F] {
|
new OShare[F] {
|
||||||
private[this] val logger = Logger.log4s[F](org.log4s.getLogger)
|
private[this] val logger = docspell.logging.getLogger[F]
|
||||||
|
|
||||||
def findAll(
|
def findAll(
|
||||||
collective: Ident,
|
collective: Ident,
|
||||||
|
@ -5,6 +5,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
package docspell.backend.ops
|
package docspell.backend.ops
|
||||||
|
|
||||||
import cats.effect._
|
import cats.effect._
|
||||||
import cats.implicits._
|
import cats.implicits._
|
||||||
|
|
||||||
@ -14,8 +15,6 @@ import docspell.store.records.{RTotp, RUser}
|
|||||||
import docspell.store.{AddResult, Store, UpdateResult}
|
import docspell.store.{AddResult, Store, UpdateResult}
|
||||||
import docspell.totp.{Key, OnetimePassword, Totp}
|
import docspell.totp.{Key, OnetimePassword, Totp}
|
||||||
|
|
||||||
import org.log4s.getLogger
|
|
||||||
|
|
||||||
trait OTotp[F[_]] {
|
trait OTotp[F[_]] {
|
||||||
|
|
||||||
/** Return whether TOTP is enabled for this account or not. */
|
/** Return whether TOTP is enabled for this account or not. */
|
||||||
@ -38,8 +37,6 @@ trait OTotp[F[_]] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
object OTotp {
|
object OTotp {
|
||||||
private[this] val logger = getLogger
|
|
||||||
|
|
||||||
sealed trait OtpState {
|
sealed trait OtpState {
|
||||||
def isEnabled: Boolean
|
def isEnabled: Boolean
|
||||||
def isDisabled = !isEnabled
|
def isDisabled = !isEnabled
|
||||||
@ -86,7 +83,7 @@ object OTotp {
|
|||||||
|
|
||||||
def apply[F[_]: Async](store: Store[F], totp: Totp): Resource[F, OTotp[F]] =
|
def apply[F[_]: Async](store: Store[F], totp: Totp): Resource[F, OTotp[F]] =
|
||||||
Resource.pure[F, OTotp[F]](new OTotp[F] {
|
Resource.pure[F, OTotp[F]](new OTotp[F] {
|
||||||
val log = Logger.log4s[F](logger)
|
val log = docspell.logging.getLogger[F]
|
||||||
|
|
||||||
def initialize(accountId: AccountId): F[InitResult] =
|
def initialize(accountId: AccountId): F[InitResult] =
|
||||||
for {
|
for {
|
||||||
|
@ -14,13 +14,10 @@ import fs2.Stream
|
|||||||
|
|
||||||
import docspell.backend.JobFactory
|
import docspell.backend.JobFactory
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.common.syntax.all._
|
|
||||||
import docspell.store.Store
|
import docspell.store.Store
|
||||||
import docspell.store.queue.JobQueue
|
import docspell.store.queue.JobQueue
|
||||||
import docspell.store.records._
|
import docspell.store.records._
|
||||||
|
|
||||||
import org.log4s._
|
|
||||||
|
|
||||||
trait OUpload[F[_]] {
|
trait OUpload[F[_]] {
|
||||||
|
|
||||||
def submit(
|
def submit(
|
||||||
@ -56,8 +53,6 @@ trait OUpload[F[_]] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
object OUpload {
|
object OUpload {
|
||||||
private[this] val logger = getLogger
|
|
||||||
|
|
||||||
case class File[F[_]](
|
case class File[F[_]](
|
||||||
name: Option[String],
|
name: Option[String],
|
||||||
advertisedMime: Option[MimeType],
|
advertisedMime: Option[MimeType],
|
||||||
@ -117,7 +112,7 @@ object OUpload {
|
|||||||
joex: OJoex[F]
|
joex: OJoex[F]
|
||||||
): Resource[F, OUpload[F]] =
|
): Resource[F, OUpload[F]] =
|
||||||
Resource.pure[F, OUpload[F]](new OUpload[F] {
|
Resource.pure[F, OUpload[F]](new OUpload[F] {
|
||||||
|
private[this] val logger = docspell.logging.getLogger[F]
|
||||||
def submit(
|
def submit(
|
||||||
data: OUpload.UploadData[F],
|
data: OUpload.UploadData[F],
|
||||||
account: AccountId,
|
account: AccountId,
|
||||||
@ -155,7 +150,7 @@ object OUpload {
|
|||||||
if (data.multiple) files.map(f => ProcessItemArgs(meta, List(f)))
|
if (data.multiple) files.map(f => ProcessItemArgs(meta, List(f)))
|
||||||
else Vector(ProcessItemArgs(meta, files.toList))
|
else Vector(ProcessItemArgs(meta, files.toList))
|
||||||
jobs <- right(makeJobs(args, account, data.priority, data.tracker))
|
jobs <- right(makeJobs(args, account, data.priority, data.tracker))
|
||||||
_ <- right(logger.fdebug(s"Storing jobs: $jobs"))
|
_ <- right(logger.debug(s"Storing jobs: $jobs"))
|
||||||
res <- right(submitJobs(notifyJoex)(jobs))
|
res <- right(submitJobs(notifyJoex)(jobs))
|
||||||
_ <- right(
|
_ <- right(
|
||||||
store.transact(
|
store.transact(
|
||||||
@ -194,7 +189,7 @@ object OUpload {
|
|||||||
notifyJoex: Boolean
|
notifyJoex: Boolean
|
||||||
)(jobs: Vector[RJob]): F[OUpload.UploadResult] =
|
)(jobs: Vector[RJob]): F[OUpload.UploadResult] =
|
||||||
for {
|
for {
|
||||||
_ <- logger.fdebug(s"Storing jobs: $jobs")
|
_ <- logger.debug(s"Storing jobs: $jobs")
|
||||||
_ <- queue.insertAll(jobs)
|
_ <- queue.insertAll(jobs)
|
||||||
_ <- if (notifyJoex) joex.notifyAllNodes else ().pure[F]
|
_ <- if (notifyJoex) joex.notifyAllNodes else ().pure[F]
|
||||||
} yield UploadResult.Success
|
} yield UploadResult.Success
|
||||||
@ -203,7 +198,7 @@ object OUpload {
|
|||||||
private def saveFile(
|
private def saveFile(
|
||||||
accountId: AccountId
|
accountId: AccountId
|
||||||
)(file: File[F]): F[Option[ProcessItemArgs.File]] =
|
)(file: File[F]): F[Option[ProcessItemArgs.File]] =
|
||||||
logger.finfo(s"Receiving file $file") *>
|
logger.info(s"Receiving file $file") *>
|
||||||
file.data
|
file.data
|
||||||
.through(
|
.through(
|
||||||
store.fileRepo.save(
|
store.fileRepo.save(
|
||||||
|
@ -11,12 +11,10 @@ import cats.implicits._
|
|||||||
|
|
||||||
import docspell.backend.PasswordCrypt
|
import docspell.backend.PasswordCrypt
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.common.syntax.all._
|
|
||||||
import docspell.store.records.{RCollective, RInvitation, RUser}
|
import docspell.store.records.{RCollective, RInvitation, RUser}
|
||||||
import docspell.store.{AddResult, Store}
|
import docspell.store.{AddResult, Store}
|
||||||
|
|
||||||
import doobie.free.connection.ConnectionIO
|
import doobie.free.connection.ConnectionIO
|
||||||
import org.log4s.getLogger
|
|
||||||
|
|
||||||
trait OSignup[F[_]] {
|
trait OSignup[F[_]] {
|
||||||
|
|
||||||
@ -29,10 +27,10 @@ trait OSignup[F[_]] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
object OSignup {
|
object OSignup {
|
||||||
private[this] val logger = getLogger
|
|
||||||
|
|
||||||
def apply[F[_]: Async](store: Store[F]): Resource[F, OSignup[F]] =
|
def apply[F[_]: Async](store: Store[F]): Resource[F, OSignup[F]] =
|
||||||
Resource.pure[F, OSignup[F]](new OSignup[F] {
|
Resource.pure[F, OSignup[F]](new OSignup[F] {
|
||||||
|
private[this] val logger = docspell.logging.getLogger[F]
|
||||||
|
|
||||||
def newInvite(cfg: Config)(password: Password): F[NewInviteResult] =
|
def newInvite(cfg: Config)(password: Password): F[NewInviteResult] =
|
||||||
if (cfg.mode == Config.Mode.Invite)
|
if (cfg.mode == Config.Mode.Invite)
|
||||||
@ -66,7 +64,7 @@ object OSignup {
|
|||||||
_ <-
|
_ <-
|
||||||
if (retryInvite(res))
|
if (retryInvite(res))
|
||||||
logger
|
logger
|
||||||
.fdebug(
|
.debug(
|
||||||
s"Adding account failed ($res). Allow retry with invite."
|
s"Adding account failed ($res). Allow retry with invite."
|
||||||
) *> store
|
) *> store
|
||||||
.transact(
|
.transact(
|
||||||
|
@ -6,6 +6,8 @@
|
|||||||
|
|
||||||
package docspell.common
|
package docspell.common
|
||||||
|
|
||||||
|
import docspell.logging.Level
|
||||||
|
|
||||||
import io.circe.{Decoder, Encoder}
|
import io.circe.{Decoder, Encoder}
|
||||||
|
|
||||||
sealed trait LogLevel { self: Product =>
|
sealed trait LogLevel { self: Product =>
|
||||||
@ -40,6 +42,16 @@ object LogLevel {
|
|||||||
case _ => Left(s"Invalid log-level: $str")
|
case _ => Left(s"Invalid log-level: $str")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def fromLevel(level: Level): LogLevel =
|
||||||
|
level match {
|
||||||
|
case Level.Fatal => LogLevel.Error
|
||||||
|
case Level.Error => LogLevel.Error
|
||||||
|
case Level.Warn => LogLevel.Warn
|
||||||
|
case Level.Info => LogLevel.Info
|
||||||
|
case Level.Debug => LogLevel.Debug
|
||||||
|
case Level.Trace => LogLevel.Debug
|
||||||
|
}
|
||||||
|
|
||||||
def unsafeString(str: String): LogLevel =
|
def unsafeString(str: String): LogLevel =
|
||||||
fromString(str).fold(sys.error, identity)
|
fromString(str).fold(sys.error, identity)
|
||||||
|
|
||||||
|
@ -1,143 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright 2020 Eike K. & Contributors
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
*/
|
|
||||||
|
|
||||||
package docspell.common
|
|
||||||
|
|
||||||
import java.io.{PrintWriter, StringWriter}
|
|
||||||
|
|
||||||
import cats.Applicative
|
|
||||||
import cats.effect.{Ref, Sync}
|
|
||||||
import cats.implicits._
|
|
||||||
import fs2.Stream
|
|
||||||
|
|
||||||
import docspell.common.syntax.all._
|
|
||||||
|
|
||||||
import org.log4s.{Logger => Log4sLogger}
|
|
||||||
|
|
||||||
trait Logger[F[_]] { self =>
|
|
||||||
|
|
||||||
def trace(msg: => String): F[Unit]
|
|
||||||
def debug(msg: => String): F[Unit]
|
|
||||||
def info(msg: => String): F[Unit]
|
|
||||||
def warn(msg: => String): F[Unit]
|
|
||||||
def error(ex: Throwable)(msg: => String): F[Unit]
|
|
||||||
def error(msg: => String): F[Unit]
|
|
||||||
|
|
||||||
final def s: Logger[Stream[F, *]] = new Logger[Stream[F, *]] {
|
|
||||||
def trace(msg: => String): Stream[F, Unit] =
|
|
||||||
Stream.eval(self.trace(msg))
|
|
||||||
|
|
||||||
def debug(msg: => String): Stream[F, Unit] =
|
|
||||||
Stream.eval(self.debug(msg))
|
|
||||||
|
|
||||||
def info(msg: => String): Stream[F, Unit] =
|
|
||||||
Stream.eval(self.info(msg))
|
|
||||||
|
|
||||||
def warn(msg: => String): Stream[F, Unit] =
|
|
||||||
Stream.eval(self.warn(msg))
|
|
||||||
|
|
||||||
def error(msg: => String): Stream[F, Unit] =
|
|
||||||
Stream.eval(self.error(msg))
|
|
||||||
|
|
||||||
def error(ex: Throwable)(msg: => String): Stream[F, Unit] =
|
|
||||||
Stream.eval(self.error(ex)(msg))
|
|
||||||
}
|
|
||||||
def andThen(other: Logger[F])(implicit F: Sync[F]): Logger[F] = {
|
|
||||||
val self = this
|
|
||||||
new Logger[F] {
|
|
||||||
def trace(msg: => String) =
|
|
||||||
self.trace(msg) >> other.trace(msg)
|
|
||||||
|
|
||||||
override def debug(msg: => String) =
|
|
||||||
self.debug(msg) >> other.debug(msg)
|
|
||||||
|
|
||||||
override def info(msg: => String) =
|
|
||||||
self.info(msg) >> other.info(msg)
|
|
||||||
|
|
||||||
override def warn(msg: => String) =
|
|
||||||
self.warn(msg) >> other.warn(msg)
|
|
||||||
|
|
||||||
override def error(ex: Throwable)(msg: => String) =
|
|
||||||
self.error(ex)(msg) >> other.error(ex)(msg)
|
|
||||||
|
|
||||||
override def error(msg: => String) =
|
|
||||||
self.error(msg) >> other.error(msg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
object Logger {
|
|
||||||
|
|
||||||
def off[F[_]: Applicative]: Logger[F] =
|
|
||||||
new Logger[F] {
|
|
||||||
def trace(msg: => String): F[Unit] =
|
|
||||||
Applicative[F].pure(())
|
|
||||||
|
|
||||||
def debug(msg: => String): F[Unit] =
|
|
||||||
Applicative[F].pure(())
|
|
||||||
|
|
||||||
def info(msg: => String): F[Unit] =
|
|
||||||
Applicative[F].pure(())
|
|
||||||
|
|
||||||
def warn(msg: => String): F[Unit] =
|
|
||||||
Applicative[F].pure(())
|
|
||||||
|
|
||||||
def error(ex: Throwable)(msg: => String): F[Unit] =
|
|
||||||
Applicative[F].pure(())
|
|
||||||
|
|
||||||
def error(msg: => String): F[Unit] =
|
|
||||||
Applicative[F].pure(())
|
|
||||||
}
|
|
||||||
|
|
||||||
def log4s[F[_]: Sync](log: Log4sLogger): Logger[F] =
|
|
||||||
new Logger[F] {
|
|
||||||
def trace(msg: => String): F[Unit] =
|
|
||||||
log.ftrace(msg)
|
|
||||||
|
|
||||||
def debug(msg: => String): F[Unit] =
|
|
||||||
log.fdebug(msg)
|
|
||||||
|
|
||||||
def info(msg: => String): F[Unit] =
|
|
||||||
log.finfo(msg)
|
|
||||||
|
|
||||||
def warn(msg: => String): F[Unit] =
|
|
||||||
log.fwarn(msg)
|
|
||||||
|
|
||||||
def error(ex: Throwable)(msg: => String): F[Unit] =
|
|
||||||
log.ferror(ex)(msg)
|
|
||||||
|
|
||||||
def error(msg: => String): F[Unit] =
|
|
||||||
log.ferror(msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
def buffer[F[_]: Sync](): F[(Ref[F, Vector[String]], Logger[F])] =
|
|
||||||
for {
|
|
||||||
buffer <- Ref.of[F, Vector[String]](Vector.empty[String])
|
|
||||||
logger = new Logger[F] {
|
|
||||||
def trace(msg: => String) =
|
|
||||||
buffer.update(_.appended(s"TRACE $msg"))
|
|
||||||
|
|
||||||
def debug(msg: => String) =
|
|
||||||
buffer.update(_.appended(s"DEBUG $msg"))
|
|
||||||
|
|
||||||
def info(msg: => String) =
|
|
||||||
buffer.update(_.appended(s"INFO $msg"))
|
|
||||||
|
|
||||||
def warn(msg: => String) =
|
|
||||||
buffer.update(_.appended(s"WARN $msg"))
|
|
||||||
|
|
||||||
def error(ex: Throwable)(msg: => String) = {
|
|
||||||
val ps = new StringWriter()
|
|
||||||
ex.printStackTrace(new PrintWriter(ps))
|
|
||||||
buffer.update(_.appended(s"ERROR $msg:\n$ps"))
|
|
||||||
}
|
|
||||||
|
|
||||||
def error(msg: => String) =
|
|
||||||
buffer.update(_.appended(s"ERROR $msg"))
|
|
||||||
}
|
|
||||||
} yield (buffer, logger)
|
|
||||||
|
|
||||||
}
|
|
@ -17,6 +17,8 @@ import cats.implicits._
|
|||||||
import fs2.io.file.Path
|
import fs2.io.file.Path
|
||||||
import fs2.{Stream, io, text}
|
import fs2.{Stream, io, text}
|
||||||
|
|
||||||
|
import docspell.logging.Logger
|
||||||
|
|
||||||
object SystemCommand {
|
object SystemCommand {
|
||||||
|
|
||||||
final case class Config(program: String, args: Seq[String], timeout: Duration) {
|
final case class Config(program: String, args: Seq[String], timeout: Duration) {
|
||||||
|
@ -1,42 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright 2020 Eike K. & Contributors
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
*/
|
|
||||||
|
|
||||||
package docspell.common.syntax
|
|
||||||
|
|
||||||
import cats.effect.Sync
|
|
||||||
import fs2.Stream
|
|
||||||
|
|
||||||
import org.log4s.Logger
|
|
||||||
|
|
||||||
trait LoggerSyntax {
|
|
||||||
|
|
||||||
implicit final class LoggerOps(logger: Logger) {
|
|
||||||
|
|
||||||
def ftrace[F[_]: Sync](msg: => String): F[Unit] =
|
|
||||||
Sync[F].delay(logger.trace(msg))
|
|
||||||
|
|
||||||
def fdebug[F[_]: Sync](msg: => String): F[Unit] =
|
|
||||||
Sync[F].delay(logger.debug(msg))
|
|
||||||
|
|
||||||
def sdebug[F[_]: Sync](msg: => String): Stream[F, Nothing] =
|
|
||||||
Stream.eval(fdebug(msg)).drain
|
|
||||||
|
|
||||||
def finfo[F[_]: Sync](msg: => String): F[Unit] =
|
|
||||||
Sync[F].delay(logger.info(msg))
|
|
||||||
|
|
||||||
def sinfo[F[_]: Sync](msg: => String): Stream[F, Nothing] =
|
|
||||||
Stream.eval(finfo(msg)).drain
|
|
||||||
|
|
||||||
def fwarn[F[_]: Sync](msg: => String): F[Unit] =
|
|
||||||
Sync[F].delay(logger.warn(msg))
|
|
||||||
|
|
||||||
def ferror[F[_]: Sync](msg: => String): F[Unit] =
|
|
||||||
Sync[F].delay(logger.error(msg))
|
|
||||||
|
|
||||||
def ferror[F[_]: Sync](ex: Throwable)(msg: => String): F[Unit] =
|
|
||||||
Sync[F].delay(logger.error(ex)(msg))
|
|
||||||
}
|
|
||||||
}
|
|
@ -8,11 +8,6 @@ package docspell.common
|
|||||||
|
|
||||||
package object syntax {
|
package object syntax {
|
||||||
|
|
||||||
object all
|
object all extends EitherSyntax with StreamSyntax with StringSyntax with FileSyntax
|
||||||
extends EitherSyntax
|
|
||||||
with StreamSyntax
|
|
||||||
with StringSyntax
|
|
||||||
with LoggerSyntax
|
|
||||||
with FileSyntax
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -13,7 +13,7 @@ import cats.effect._
|
|||||||
import cats.implicits._
|
import cats.implicits._
|
||||||
import fs2.io.file.{Files, Path}
|
import fs2.io.file.{Files, Path}
|
||||||
|
|
||||||
import docspell.common.Logger
|
import docspell.logging.Logger
|
||||||
|
|
||||||
import pureconfig.{ConfigReader, ConfigSource}
|
import pureconfig.{ConfigReader, ConfigSource}
|
||||||
|
|
||||||
|
@ -13,6 +13,7 @@ import scala.reflect.ClassTag
|
|||||||
import fs2.io.file.Path
|
import fs2.io.file.Path
|
||||||
|
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
|
import docspell.logging.{Level, LogConfig}
|
||||||
|
|
||||||
import com.github.eikek.calev.CalEvent
|
import com.github.eikek.calev.CalEvent
|
||||||
import pureconfig.ConfigReader
|
import pureconfig.ConfigReader
|
||||||
@ -63,6 +64,12 @@ object Implicits {
|
|||||||
implicit val nlpModeReader: ConfigReader[NlpMode] =
|
implicit val nlpModeReader: ConfigReader[NlpMode] =
|
||||||
ConfigReader[String].emap(reason(NlpMode.fromString))
|
ConfigReader[String].emap(reason(NlpMode.fromString))
|
||||||
|
|
||||||
|
implicit val logFormatReader: ConfigReader[LogConfig.Format] =
|
||||||
|
ConfigReader[String].emap(reason(LogConfig.Format.fromString))
|
||||||
|
|
||||||
|
implicit val logLevelReader: ConfigReader[Level] =
|
||||||
|
ConfigReader[String].emap(reason(Level.fromString))
|
||||||
|
|
||||||
def reason[A: ClassTag](
|
def reason[A: ClassTag](
|
||||||
f: String => Either[String, A]
|
f: String => Either[String, A]
|
||||||
): String => Either[FailureReason, A] =
|
): String => Either[FailureReason, A] =
|
||||||
|
@ -17,6 +17,7 @@ import docspell.convert.ConversionResult.Handler
|
|||||||
import docspell.convert.extern._
|
import docspell.convert.extern._
|
||||||
import docspell.convert.flexmark.Markdown
|
import docspell.convert.flexmark.Markdown
|
||||||
import docspell.files.{ImageSize, TikaMimetype}
|
import docspell.files.{ImageSize, TikaMimetype}
|
||||||
|
import docspell.logging.Logger
|
||||||
|
|
||||||
import scodec.bits.ByteVector
|
import scodec.bits.ByteVector
|
||||||
|
|
||||||
@ -46,7 +47,7 @@ object Conversion {
|
|||||||
val allPass = cfg.decryptPdf.passwords ++ additionalPasswords
|
val allPass = cfg.decryptPdf.passwords ++ additionalPasswords
|
||||||
val pdfStream =
|
val pdfStream =
|
||||||
if (cfg.decryptPdf.enabled) {
|
if (cfg.decryptPdf.enabled) {
|
||||||
logger.s
|
logger.stream
|
||||||
.debug(s"Trying to read the PDF using ${allPass.size} passwords")
|
.debug(s"Trying to read the PDF using ${allPass.size} passwords")
|
||||||
.drain ++
|
.drain ++
|
||||||
in.through(RemovePdfEncryption(logger, allPass))
|
in.through(RemovePdfEncryption(logger, allPass))
|
||||||
|
@ -12,6 +12,7 @@ import cats.effect._
|
|||||||
import fs2.{Chunk, Pipe, Stream}
|
import fs2.{Chunk, Pipe, Stream}
|
||||||
|
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
|
import docspell.logging.Logger
|
||||||
|
|
||||||
import org.apache.pdfbox.pdmodel.PDDocument
|
import org.apache.pdfbox.pdmodel.PDDocument
|
||||||
import org.apache.pdfbox.pdmodel.encryption.InvalidPasswordException
|
import org.apache.pdfbox.pdmodel.encryption.InvalidPasswordException
|
||||||
@ -36,7 +37,7 @@ object RemovePdfEncryption {
|
|||||||
.head
|
.head
|
||||||
.flatMap { doc =>
|
.flatMap { doc =>
|
||||||
if (doc.isEncrypted) {
|
if (doc.isEncrypted) {
|
||||||
logger.s.debug("Removing protection/encryption from PDF").drain ++
|
logger.stream.debug("Removing protection/encryption from PDF").drain ++
|
||||||
Stream.eval(Sync[F].delay(doc.setAllSecurityToBeRemoved(true))).drain ++
|
Stream.eval(Sync[F].delay(doc.setAllSecurityToBeRemoved(true))).drain ++
|
||||||
toStream[F](doc)
|
toStream[F](doc)
|
||||||
} else {
|
} else {
|
||||||
@ -44,7 +45,7 @@ object RemovePdfEncryption {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
.ifEmpty(
|
.ifEmpty(
|
||||||
logger.s
|
logger.stream
|
||||||
.info(
|
.info(
|
||||||
s"None of the passwords helped to read the given PDF!"
|
s"None of the passwords helped to read the given PDF!"
|
||||||
)
|
)
|
||||||
@ -64,7 +65,8 @@ object RemovePdfEncryption {
|
|||||||
|
|
||||||
val log =
|
val log =
|
||||||
if (pw.isEmpty) Stream.empty
|
if (pw.isEmpty) Stream.empty
|
||||||
else logger.s.debug(s"Try opening PDF with password: ${pw.pass.take(2)}***").drain
|
else
|
||||||
|
logger.stream.debug(s"Try opening PDF with password: ${pw.pass.take(2)}***").drain
|
||||||
|
|
||||||
in =>
|
in =>
|
||||||
Stream
|
Stream
|
||||||
|
@ -14,6 +14,7 @@ import fs2.{Pipe, Stream}
|
|||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.convert.ConversionResult
|
import docspell.convert.ConversionResult
|
||||||
import docspell.convert.ConversionResult.{Handler, successPdf, successPdfTxt}
|
import docspell.convert.ConversionResult.{Handler, successPdf, successPdfTxt}
|
||||||
|
import docspell.logging.Logger
|
||||||
|
|
||||||
private[extern] object ExternConv {
|
private[extern] object ExternConv {
|
||||||
|
|
||||||
|
@ -13,6 +13,7 @@ import fs2.io.file.Path
|
|||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.convert.ConversionResult
|
import docspell.convert.ConversionResult
|
||||||
import docspell.convert.ConversionResult.Handler
|
import docspell.convert.ConversionResult.Handler
|
||||||
|
import docspell.logging.Logger
|
||||||
|
|
||||||
object OcrMyPdf {
|
object OcrMyPdf {
|
||||||
|
|
||||||
|
@ -13,6 +13,7 @@ import fs2.io.file.Path
|
|||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.convert.ConversionResult
|
import docspell.convert.ConversionResult
|
||||||
import docspell.convert.ConversionResult.Handler
|
import docspell.convert.ConversionResult.Handler
|
||||||
|
import docspell.logging.Logger
|
||||||
|
|
||||||
object Tesseract {
|
object Tesseract {
|
||||||
|
|
||||||
|
@ -13,6 +13,7 @@ import fs2.io.file.Path
|
|||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.convert.ConversionResult
|
import docspell.convert.ConversionResult
|
||||||
import docspell.convert.ConversionResult.Handler
|
import docspell.convert.ConversionResult.Handler
|
||||||
|
import docspell.logging.Logger
|
||||||
|
|
||||||
object Unoconv {
|
object Unoconv {
|
||||||
|
|
||||||
|
@ -16,6 +16,7 @@ import fs2.{Chunk, Stream}
|
|||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.convert.ConversionResult.Handler
|
import docspell.convert.ConversionResult.Handler
|
||||||
import docspell.convert.{ConversionResult, SanitizeHtml}
|
import docspell.convert.{ConversionResult, SanitizeHtml}
|
||||||
|
import docspell.logging.Logger
|
||||||
|
|
||||||
object WkHtmlPdf {
|
object WkHtmlPdf {
|
||||||
|
|
||||||
|
@ -20,12 +20,13 @@ import docspell.convert.extern.OcrMyPdfConfig
|
|||||||
import docspell.convert.extern.{TesseractConfig, UnoconvConfig, WkHtmlPdfConfig}
|
import docspell.convert.extern.{TesseractConfig, UnoconvConfig, WkHtmlPdfConfig}
|
||||||
import docspell.convert.flexmark.MarkdownConfig
|
import docspell.convert.flexmark.MarkdownConfig
|
||||||
import docspell.files.ExampleFiles
|
import docspell.files.ExampleFiles
|
||||||
|
import docspell.logging.TestLoggingConfig
|
||||||
|
|
||||||
import munit._
|
import munit._
|
||||||
|
|
||||||
class ConversionTest extends FunSuite with FileChecks {
|
class ConversionTest extends FunSuite with FileChecks with TestLoggingConfig {
|
||||||
|
|
||||||
val logger = Logger.log4s[IO](org.log4s.getLogger)
|
val logger = docspell.logging.getLogger[IO]
|
||||||
val target = File.path(Paths.get("target"))
|
val target = File.path(Paths.get("target"))
|
||||||
|
|
||||||
val convertConfig = ConvertConfig(
|
val convertConfig = ConvertConfig(
|
||||||
|
@ -11,11 +11,15 @@ import fs2.Stream
|
|||||||
|
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.files.ExampleFiles
|
import docspell.files.ExampleFiles
|
||||||
|
import docspell.logging.{Logger, TestLoggingConfig}
|
||||||
|
|
||||||
import munit.CatsEffectSuite
|
import munit.CatsEffectSuite
|
||||||
|
|
||||||
class RemovePdfEncryptionTest extends CatsEffectSuite with FileChecks {
|
class RemovePdfEncryptionTest
|
||||||
val logger: Logger[IO] = Logger.log4s(org.log4s.getLogger)
|
extends CatsEffectSuite
|
||||||
|
with FileChecks
|
||||||
|
with TestLoggingConfig {
|
||||||
|
val logger: Logger[IO] = docspell.logging.getLogger[IO]
|
||||||
|
|
||||||
private val protectedPdf =
|
private val protectedPdf =
|
||||||
ExampleFiles.secured_protected_test123_pdf.readURL[IO](16 * 1024)
|
ExampleFiles.secured_protected_test123_pdf.readURL[IO](16 * 1024)
|
||||||
|
@ -16,12 +16,13 @@ import fs2.io.file.Path
|
|||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.convert._
|
import docspell.convert._
|
||||||
import docspell.files.ExampleFiles
|
import docspell.files.ExampleFiles
|
||||||
|
import docspell.logging.TestLoggingConfig
|
||||||
|
|
||||||
import munit._
|
import munit._
|
||||||
|
|
||||||
class ExternConvTest extends FunSuite with FileChecks {
|
class ExternConvTest extends FunSuite with FileChecks with TestLoggingConfig {
|
||||||
val utf8 = StandardCharsets.UTF_8
|
val utf8 = StandardCharsets.UTF_8
|
||||||
val logger = Logger.log4s[IO](org.log4s.getLogger)
|
val logger = docspell.logging.getLogger[IO]
|
||||||
val target = File.path(Paths.get("target"))
|
val target = File.path(Paths.get("target"))
|
||||||
|
|
||||||
test("convert html to pdf") {
|
test("convert html to pdf") {
|
||||||
|
@ -18,6 +18,7 @@ import docspell.extract.poi.{PoiExtract, PoiType}
|
|||||||
import docspell.extract.rtf.RtfExtract
|
import docspell.extract.rtf.RtfExtract
|
||||||
import docspell.files.ImageSize
|
import docspell.files.ImageSize
|
||||||
import docspell.files.TikaMimetype
|
import docspell.files.TikaMimetype
|
||||||
|
import docspell.logging.Logger
|
||||||
|
|
||||||
trait Extraction[F[_]] {
|
trait Extraction[F[_]] {
|
||||||
|
|
||||||
|
@ -10,11 +10,12 @@ import cats.effect._
|
|||||||
import cats.implicits._
|
import cats.implicits._
|
||||||
import fs2.Stream
|
import fs2.Stream
|
||||||
|
|
||||||
import docspell.common.{Language, Logger}
|
import docspell.common.Language
|
||||||
import docspell.extract.internal.Text
|
import docspell.extract.internal.Text
|
||||||
import docspell.extract.ocr.{OcrConfig, TextExtract}
|
import docspell.extract.ocr.{OcrConfig, TextExtract}
|
||||||
import docspell.extract.pdfbox.PdfMetaData
|
import docspell.extract.pdfbox.PdfMetaData
|
||||||
import docspell.extract.pdfbox.PdfboxExtract
|
import docspell.extract.pdfbox.PdfboxExtract
|
||||||
|
import docspell.logging.Logger
|
||||||
|
|
||||||
object PdfExtract {
|
object PdfExtract {
|
||||||
final case class Result(txt: Text, meta: Option[PdfMetaData])
|
final case class Result(txt: Text, meta: Option[PdfMetaData])
|
||||||
|
@ -11,6 +11,7 @@ import fs2.Stream
|
|||||||
import fs2.io.file.Path
|
import fs2.io.file.Path
|
||||||
|
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
|
import docspell.logging.Logger
|
||||||
|
|
||||||
object Ocr {
|
object Ocr {
|
||||||
|
|
||||||
|
@ -12,6 +12,7 @@ import fs2.Stream
|
|||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.extract.internal.Text
|
import docspell.extract.internal.Text
|
||||||
import docspell.files._
|
import docspell.files._
|
||||||
|
import docspell.logging.Logger
|
||||||
|
|
||||||
object TextExtract {
|
object TextExtract {
|
||||||
|
|
||||||
|
@ -32,7 +32,7 @@ trait PdfboxPreview[F[_]] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
object PdfboxPreview {
|
object PdfboxPreview {
|
||||||
private[this] val logger = org.log4s.getLogger
|
private[this] val logger = docspell.logging.unsafeLogger
|
||||||
|
|
||||||
def apply[F[_]: Sync](cfg: PreviewConfig): F[PdfboxPreview[F]] =
|
def apply[F[_]: Sync](cfg: PreviewConfig): F[PdfboxPreview[F]] =
|
||||||
Sync[F].pure(new PdfboxPreview[F] {
|
Sync[F].pure(new PdfboxPreview[F] {
|
||||||
|
@ -9,15 +9,15 @@ package docspell.extract.ocr
|
|||||||
import cats.effect.IO
|
import cats.effect.IO
|
||||||
import cats.effect.unsafe.implicits.global
|
import cats.effect.unsafe.implicits.global
|
||||||
|
|
||||||
import docspell.common.Logger
|
|
||||||
import docspell.files.TestFiles
|
import docspell.files.TestFiles
|
||||||
|
import docspell.logging.TestLoggingConfig
|
||||||
|
|
||||||
import munit._
|
import munit._
|
||||||
|
|
||||||
class TextExtractionSuite extends FunSuite {
|
class TextExtractionSuite extends FunSuite with TestLoggingConfig {
|
||||||
import TestFiles._
|
import TestFiles._
|
||||||
|
|
||||||
val logger = Logger.log4s[IO](org.log4s.getLogger)
|
val logger = docspell.logging.getLogger[IO]
|
||||||
|
|
||||||
test("extract english pdf".ignore) {
|
test("extract english pdf".ignore) {
|
||||||
val text = TextExtract
|
val text = TextExtract
|
||||||
|
@ -10,10 +10,11 @@ import cats.effect._
|
|||||||
import cats.effect.unsafe.implicits.global
|
import cats.effect.unsafe.implicits.global
|
||||||
|
|
||||||
import docspell.files.ExampleFiles
|
import docspell.files.ExampleFiles
|
||||||
|
import docspell.logging.TestLoggingConfig
|
||||||
|
|
||||||
import munit._
|
import munit._
|
||||||
|
|
||||||
class OdfExtractTest extends FunSuite {
|
class OdfExtractTest extends FunSuite with TestLoggingConfig {
|
||||||
|
|
||||||
val files = List(
|
val files = List(
|
||||||
ExampleFiles.examples_sample_odt -> 6367,
|
ExampleFiles.examples_sample_odt -> 6367,
|
||||||
|
@ -6,9 +6,11 @@
|
|||||||
|
|
||||||
package docspell.extract.pdfbox
|
package docspell.extract.pdfbox
|
||||||
|
|
||||||
|
import docspell.logging.TestLoggingConfig
|
||||||
|
|
||||||
import munit._
|
import munit._
|
||||||
|
|
||||||
class PdfMetaDataTest extends FunSuite {
|
class PdfMetaDataTest extends FunSuite with TestLoggingConfig {
|
||||||
|
|
||||||
test("split keywords on comma") {
|
test("split keywords on comma") {
|
||||||
val md = PdfMetaData.empty.copy(keywords = Some("a,b, c"))
|
val md = PdfMetaData.empty.copy(keywords = Some("a,b, c"))
|
||||||
|
@ -10,10 +10,11 @@ import cats.effect._
|
|||||||
import cats.effect.unsafe.implicits.global
|
import cats.effect.unsafe.implicits.global
|
||||||
|
|
||||||
import docspell.files.{ExampleFiles, TestFiles}
|
import docspell.files.{ExampleFiles, TestFiles}
|
||||||
|
import docspell.logging.TestLoggingConfig
|
||||||
|
|
||||||
import munit._
|
import munit._
|
||||||
|
|
||||||
class PdfboxExtractTest extends FunSuite {
|
class PdfboxExtractTest extends FunSuite with TestLoggingConfig {
|
||||||
|
|
||||||
val textPDFs = List(
|
val textPDFs = List(
|
||||||
ExampleFiles.letter_de_pdf -> TestFiles.letterDEText,
|
ExampleFiles.letter_de_pdf -> TestFiles.letterDEText,
|
||||||
|
@ -13,10 +13,11 @@ import fs2.io.file.Files
|
|||||||
import fs2.io.file.Path
|
import fs2.io.file.Path
|
||||||
|
|
||||||
import docspell.files.ExampleFiles
|
import docspell.files.ExampleFiles
|
||||||
|
import docspell.logging.TestLoggingConfig
|
||||||
|
|
||||||
import munit._
|
import munit._
|
||||||
|
|
||||||
class PdfboxPreviewTest extends FunSuite {
|
class PdfboxPreviewTest extends FunSuite with TestLoggingConfig {
|
||||||
|
|
||||||
val testPDFs = List(
|
val testPDFs = List(
|
||||||
ExampleFiles.letter_de_pdf -> "7d98be75b239816d6c751b3f3c56118ebf1a4632c43baf35a68a662f9d595ab8",
|
ExampleFiles.letter_de_pdf -> "7d98be75b239816d6c751b3f3c56118ebf1a4632c43baf35a68a662f9d595ab8",
|
||||||
|
@ -11,10 +11,11 @@ import cats.effect.unsafe.implicits.global
|
|||||||
|
|
||||||
import docspell.common.MimeTypeHint
|
import docspell.common.MimeTypeHint
|
||||||
import docspell.files.ExampleFiles
|
import docspell.files.ExampleFiles
|
||||||
|
import docspell.logging.TestLoggingConfig
|
||||||
|
|
||||||
import munit._
|
import munit._
|
||||||
|
|
||||||
class PoiExtractTest extends FunSuite {
|
class PoiExtractTest extends FunSuite with TestLoggingConfig {
|
||||||
|
|
||||||
val officeFiles = List(
|
val officeFiles = List(
|
||||||
ExampleFiles.examples_sample_doc -> 6241,
|
ExampleFiles.examples_sample_doc -> 6241,
|
||||||
|
@ -7,10 +7,11 @@
|
|||||||
package docspell.extract.rtf
|
package docspell.extract.rtf
|
||||||
|
|
||||||
import docspell.files.ExampleFiles
|
import docspell.files.ExampleFiles
|
||||||
|
import docspell.logging.TestLoggingConfig
|
||||||
|
|
||||||
import munit._
|
import munit._
|
||||||
|
|
||||||
class RtfExtractTest extends FunSuite {
|
class RtfExtractTest extends FunSuite with TestLoggingConfig {
|
||||||
|
|
||||||
test("extract text from rtf using java input-stream") {
|
test("extract text from rtf using java input-stream") {
|
||||||
val file = ExampleFiles.examples_sample_rtf
|
val file = ExampleFiles.examples_sample_rtf
|
||||||
|
@ -1,14 +0,0 @@
|
|||||||
<configuration>
|
|
||||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
|
||||||
<withJansi>true</withJansi>
|
|
||||||
|
|
||||||
<encoder>
|
|
||||||
<pattern>level=%-5level thread=%thread logger=%logger{15} message="%replace(%msg){'"', '\\"'}"%n</pattern>
|
|
||||||
</encoder>
|
|
||||||
</appender>
|
|
||||||
|
|
||||||
<logger name="docspell" level="debug" />
|
|
||||||
<root level="error">
|
|
||||||
<appender-ref ref="STDOUT" />
|
|
||||||
</root>
|
|
||||||
</configuration>
|
|
@ -11,8 +11,7 @@ import cats.implicits._
|
|||||||
import fs2.Stream
|
import fs2.Stream
|
||||||
|
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
|
import docspell.logging.Logger
|
||||||
import org.log4s.getLogger
|
|
||||||
|
|
||||||
/** The fts client is the interface for docspell to a fulltext search engine.
|
/** The fts client is the interface for docspell to a fulltext search engine.
|
||||||
*
|
*
|
||||||
@ -127,7 +126,7 @@ object FtsClient {
|
|||||||
|
|
||||||
def none[F[_]: Sync] =
|
def none[F[_]: Sync] =
|
||||||
new FtsClient[F] {
|
new FtsClient[F] {
|
||||||
private[this] val logger = Logger.log4s[F](getLogger)
|
private[this] val logger = docspell.logging.getLogger[F]
|
||||||
|
|
||||||
def initialize: F[List[FtsMigration[F]]] =
|
def initialize: F[List[FtsMigration[F]]] =
|
||||||
Sync[F].pure(Nil)
|
Sync[F].pure(Nil)
|
||||||
|
@ -12,10 +12,10 @@ import fs2.Stream
|
|||||||
|
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.ftsclient._
|
import docspell.ftsclient._
|
||||||
|
import docspell.logging.Logger
|
||||||
|
|
||||||
import org.http4s.client.Client
|
import org.http4s.client.Client
|
||||||
import org.http4s.client.middleware.Logger
|
import org.http4s.client.middleware.{Logger => Http4sLogger}
|
||||||
import org.log4s.getLogger
|
|
||||||
|
|
||||||
final class SolrFtsClient[F[_]: Async](
|
final class SolrFtsClient[F[_]: Async](
|
||||||
solrUpdate: SolrUpdate[F],
|
solrUpdate: SolrUpdate[F],
|
||||||
@ -81,7 +81,6 @@ final class SolrFtsClient[F[_]: Async](
|
|||||||
}
|
}
|
||||||
|
|
||||||
object SolrFtsClient {
|
object SolrFtsClient {
|
||||||
private[this] val logger = getLogger
|
|
||||||
|
|
||||||
def apply[F[_]: Async](
|
def apply[F[_]: Async](
|
||||||
cfg: SolrConfig,
|
cfg: SolrConfig,
|
||||||
@ -100,11 +99,13 @@ object SolrFtsClient {
|
|||||||
private def loggingMiddleware[F[_]: Async](
|
private def loggingMiddleware[F[_]: Async](
|
||||||
cfg: SolrConfig,
|
cfg: SolrConfig,
|
||||||
client: Client[F]
|
client: Client[F]
|
||||||
): Client[F] =
|
): Client[F] = {
|
||||||
Logger(
|
val delegate = docspell.logging.getLogger[F]
|
||||||
|
Http4sLogger(
|
||||||
logHeaders = true,
|
logHeaders = true,
|
||||||
logBody = cfg.logVerbose,
|
logBody = cfg.logVerbose,
|
||||||
logAction = Some((msg: String) => Sync[F].delay(logger.trace(msg)))
|
logAction = Some((msg: String) => delegate.trace(msg))
|
||||||
)(client)
|
)(client)
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -1,14 +0,0 @@
|
|||||||
<configuration>
|
|
||||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
|
||||||
<withJansi>true</withJansi>
|
|
||||||
|
|
||||||
<encoder>
|
|
||||||
<pattern>level=%-5level thread=%thread logger=%logger{15} message="%replace(%msg){'"', '\\"'}"%n</pattern>
|
|
||||||
</encoder>
|
|
||||||
</appender>
|
|
||||||
|
|
||||||
<logger name="docspell" level="debug" />
|
|
||||||
<root level="INFO">
|
|
||||||
<appender-ref ref="STDOUT" />
|
|
||||||
</root>
|
|
||||||
</configuration>
|
|
@ -18,6 +18,17 @@ docspell.joex {
|
|||||||
port = 7878
|
port = 7878
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Configures logging
|
||||||
|
logging {
|
||||||
|
# The format for the log messages. Can be one of:
|
||||||
|
# Json, Logfmt, Fancy or Plain
|
||||||
|
format = "Plain"
|
||||||
|
|
||||||
|
# The minimum level to log. From lowest to highest:
|
||||||
|
# Trace, Debug, Info, Warn, Error
|
||||||
|
minimum-level = "Info"
|
||||||
|
}
|
||||||
|
|
||||||
# The database connection.
|
# The database connection.
|
||||||
#
|
#
|
||||||
# It must be the same connection as the rest server is using.
|
# It must be the same connection as the rest server is using.
|
||||||
|
@ -21,12 +21,14 @@ import docspell.joex.hk.HouseKeepingConfig
|
|||||||
import docspell.joex.routes.InternalHeader
|
import docspell.joex.routes.InternalHeader
|
||||||
import docspell.joex.scheduler.{PeriodicSchedulerConfig, SchedulerConfig}
|
import docspell.joex.scheduler.{PeriodicSchedulerConfig, SchedulerConfig}
|
||||||
import docspell.joex.updatecheck.UpdateCheckConfig
|
import docspell.joex.updatecheck.UpdateCheckConfig
|
||||||
|
import docspell.logging.LogConfig
|
||||||
import docspell.pubsub.naive.PubSubConfig
|
import docspell.pubsub.naive.PubSubConfig
|
||||||
import docspell.store.JdbcConfig
|
import docspell.store.JdbcConfig
|
||||||
|
|
||||||
case class Config(
|
case class Config(
|
||||||
appId: Ident,
|
appId: Ident,
|
||||||
baseUrl: LenientUri,
|
baseUrl: LenientUri,
|
||||||
|
logging: LogConfig,
|
||||||
bind: Config.Bind,
|
bind: Config.Bind,
|
||||||
jdbc: JdbcConfig,
|
jdbc: JdbcConfig,
|
||||||
scheduler: SchedulerConfig,
|
scheduler: SchedulerConfig,
|
||||||
|
@ -8,7 +8,6 @@ package docspell.joex
|
|||||||
|
|
||||||
import cats.effect.Async
|
import cats.effect.Async
|
||||||
|
|
||||||
import docspell.common.Logger
|
|
||||||
import docspell.config.Implicits._
|
import docspell.config.Implicits._
|
||||||
import docspell.config.{ConfigFactory, Validation}
|
import docspell.config.{ConfigFactory, Validation}
|
||||||
import docspell.joex.scheduler.CountingScheme
|
import docspell.joex.scheduler.CountingScheme
|
||||||
@ -23,7 +22,7 @@ object ConfigFile {
|
|||||||
import Implicits._
|
import Implicits._
|
||||||
|
|
||||||
def loadConfig[F[_]: Async](args: List[String]): F[Config] = {
|
def loadConfig[F[_]: Async](args: List[String]): F[Config] = {
|
||||||
val logger = Logger.log4s[F](org.log4s.getLogger)
|
val logger = docspell.logging.getLogger[F]
|
||||||
ConfigFactory
|
ConfigFactory
|
||||||
.default[F, Config](logger, "docspell.joex")(args, validate)
|
.default[F, Config](logger, "docspell.joex")(args, validate)
|
||||||
}
|
}
|
||||||
|
@ -132,10 +132,8 @@ object JoexAppImpl extends MailAddressCodec {
|
|||||||
): Resource[F, JoexApp[F]] =
|
): Resource[F, JoexApp[F]] =
|
||||||
for {
|
for {
|
||||||
pstore <- PeriodicTaskStore.create(store)
|
pstore <- PeriodicTaskStore.create(store)
|
||||||
pubSubT = PubSubT(
|
joexLogger = docspell.logging.getLogger[F](s"joex-${cfg.appId.id}")
|
||||||
pubSub,
|
pubSubT = PubSubT(pubSub, joexLogger)
|
||||||
Logger.log4s(org.log4s.getLogger(s"joex-${cfg.appId.id}"))
|
|
||||||
)
|
|
||||||
javaEmil =
|
javaEmil =
|
||||||
JavaMailEmil(Settings.defaultSettings.copy(debug = cfg.mailDebug))
|
JavaMailEmil(Settings.defaultSettings.copy(debug = cfg.mailDebug))
|
||||||
notificationMod <- Resource.eval(
|
notificationMod <- Resource.eval(
|
||||||
|
@ -9,12 +9,12 @@ package docspell.joex
|
|||||||
import cats.effect._
|
import cats.effect._
|
||||||
|
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
|
import docspell.logging.Logger
|
||||||
import org.log4s.getLogger
|
import docspell.logging.impl.ScribeConfigure
|
||||||
|
|
||||||
object Main extends IOApp {
|
object Main extends IOApp {
|
||||||
|
|
||||||
private val logger: Logger[IO] = Logger.log4s[IO](getLogger)
|
private val logger: Logger[IO] = docspell.logging.getLogger[IO]
|
||||||
|
|
||||||
private val connectEC =
|
private val connectEC =
|
||||||
ThreadFactories.fixed[IO](5, ThreadFactories.ofName("docspell-joex-dbconnect"))
|
ThreadFactories.fixed[IO](5, ThreadFactories.ofName("docspell-joex-dbconnect"))
|
||||||
@ -22,6 +22,7 @@ object Main extends IOApp {
|
|||||||
def run(args: List[String]): IO[ExitCode] =
|
def run(args: List[String]): IO[ExitCode] =
|
||||||
for {
|
for {
|
||||||
cfg <- ConfigFile.loadConfig[IO](args)
|
cfg <- ConfigFile.loadConfig[IO](args)
|
||||||
|
_ <- ScribeConfigure.configure[IO](cfg.logging)
|
||||||
banner = Banner(
|
banner = Banner(
|
||||||
"JOEX",
|
"JOEX",
|
||||||
BuildInfo.version,
|
BuildInfo.version,
|
||||||
|
@ -12,7 +12,6 @@ import cats.implicits._
|
|||||||
import fs2.io.file.Path
|
import fs2.io.file.Path
|
||||||
|
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.common.syntax.all._
|
|
||||||
import docspell.store.Store
|
import docspell.store.Store
|
||||||
import docspell.store.queries.QCollective
|
import docspell.store.queries.QCollective
|
||||||
import docspell.store.records.REquipment
|
import docspell.store.records.REquipment
|
||||||
@ -20,7 +19,6 @@ import docspell.store.records.ROrganization
|
|||||||
import docspell.store.records.RPerson
|
import docspell.store.records.RPerson
|
||||||
|
|
||||||
import io.circe.syntax._
|
import io.circe.syntax._
|
||||||
import org.log4s.getLogger
|
|
||||||
|
|
||||||
/** Maintains a custom regex-ner file per collective for stanford's regexner annotator. */
|
/** Maintains a custom regex-ner file per collective for stanford's regexner annotator. */
|
||||||
trait RegexNerFile[F[_]] {
|
trait RegexNerFile[F[_]] {
|
||||||
@ -30,7 +28,6 @@ trait RegexNerFile[F[_]] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
object RegexNerFile {
|
object RegexNerFile {
|
||||||
private[this] val logger = getLogger
|
|
||||||
|
|
||||||
case class Config(maxEntries: Int, directory: Path, minTime: Duration)
|
case class Config(maxEntries: Int, directory: Path, minTime: Duration)
|
||||||
|
|
||||||
@ -49,6 +46,8 @@ object RegexNerFile {
|
|||||||
writer: Semaphore[F] // TODO allow parallelism per collective
|
writer: Semaphore[F] // TODO allow parallelism per collective
|
||||||
) extends RegexNerFile[F] {
|
) extends RegexNerFile[F] {
|
||||||
|
|
||||||
|
private[this] val logger = docspell.logging.getLogger[F]
|
||||||
|
|
||||||
def makeFile(collective: Ident): F[Option[Path]] =
|
def makeFile(collective: Ident): F[Option[Path]] =
|
||||||
if (cfg.maxEntries > 0) doMakeFile(collective)
|
if (cfg.maxEntries > 0) doMakeFile(collective)
|
||||||
else (None: Option[Path]).pure[F]
|
else (None: Option[Path]).pure[F]
|
||||||
@ -61,7 +60,7 @@ object RegexNerFile {
|
|||||||
case Some(nf) =>
|
case Some(nf) =>
|
||||||
val dur = Duration.between(nf.creation, now)
|
val dur = Duration.between(nf.creation, now)
|
||||||
if (dur > cfg.minTime)
|
if (dur > cfg.minTime)
|
||||||
logger.fdebug(
|
logger.debug(
|
||||||
s"Cache time elapsed ($dur > ${cfg.minTime}). Check for new state."
|
s"Cache time elapsed ($dur > ${cfg.minTime}). Check for new state."
|
||||||
) *> updateFile(
|
) *> updateFile(
|
||||||
collective,
|
collective,
|
||||||
@ -89,12 +88,12 @@ object RegexNerFile {
|
|||||||
case Some(cur) =>
|
case Some(cur) =>
|
||||||
val nerf =
|
val nerf =
|
||||||
if (cur.updated == lup)
|
if (cur.updated == lup)
|
||||||
logger.fdebug(s"No state change detected.") *> updateTimestamp(
|
logger.debug(s"No state change detected.") *> updateTimestamp(
|
||||||
cur,
|
cur,
|
||||||
now
|
now
|
||||||
) *> cur.pure[F]
|
) *> cur.pure[F]
|
||||||
else
|
else
|
||||||
logger.fdebug(
|
logger.debug(
|
||||||
s"There have been state changes for collective '${collective.id}'. Reload NER file."
|
s"There have been state changes for collective '${collective.id}'. Reload NER file."
|
||||||
) *> createFile(lup, collective, now)
|
) *> createFile(lup, collective, now)
|
||||||
nerf.map(_.nerFilePath(cfg.directory).some)
|
nerf.map(_.nerFilePath(cfg.directory).some)
|
||||||
@ -126,7 +125,7 @@ object RegexNerFile {
|
|||||||
writer.permit.use(_ =>
|
writer.permit.use(_ =>
|
||||||
for {
|
for {
|
||||||
jsonFile <- Sync[F].pure(nf.jsonFilePath(cfg.directory))
|
jsonFile <- Sync[F].pure(nf.jsonFilePath(cfg.directory))
|
||||||
_ <- logger.fdebug(
|
_ <- logger.debug(
|
||||||
s"Writing custom NER file for collective '${collective.id}'"
|
s"Writing custom NER file for collective '${collective.id}'"
|
||||||
)
|
)
|
||||||
_ <- jsonFile.parent match {
|
_ <- jsonFile.parent match {
|
||||||
@ -139,7 +138,7 @@ object RegexNerFile {
|
|||||||
)
|
)
|
||||||
|
|
||||||
for {
|
for {
|
||||||
_ <- logger.finfo(s"Generating custom NER file for collective '${collective.id}'")
|
_ <- logger.info(s"Generating custom NER file for collective '${collective.id}'")
|
||||||
names <- store.transact(QCollective.allNames(collective, cfg.maxEntries))
|
names <- store.transact(QCollective.allNames(collective, cfg.maxEntries))
|
||||||
nerFile = NerFile(collective, lastUpdate, now)
|
nerFile = NerFile(collective, lastUpdate, now)
|
||||||
_ <- update(nerFile, NerFile.mkNerConfig(names))
|
_ <- update(nerFile, NerFile.mkNerConfig(names))
|
||||||
|
@ -7,10 +7,10 @@
|
|||||||
package docspell.joex.fts
|
package docspell.joex.fts
|
||||||
|
|
||||||
import docspell.backend.fulltext.CreateIndex
|
import docspell.backend.fulltext.CreateIndex
|
||||||
import docspell.common.Logger
|
|
||||||
import docspell.ftsclient.FtsClient
|
import docspell.ftsclient.FtsClient
|
||||||
import docspell.joex.Config
|
import docspell.joex.Config
|
||||||
import docspell.joex.scheduler.Context
|
import docspell.joex.scheduler.Context
|
||||||
|
import docspell.logging.Logger
|
||||||
import docspell.store.Store
|
import docspell.store.Store
|
||||||
|
|
||||||
case class FtsContext[F[_]](
|
case class FtsContext[F[_]](
|
||||||
|
@ -15,6 +15,7 @@ import docspell.common._
|
|||||||
import docspell.ftsclient._
|
import docspell.ftsclient._
|
||||||
import docspell.joex.Config
|
import docspell.joex.Config
|
||||||
import docspell.joex.scheduler.Context
|
import docspell.joex.scheduler.Context
|
||||||
|
import docspell.logging.Logger
|
||||||
|
|
||||||
object FtsWork {
|
object FtsWork {
|
||||||
import syntax._
|
import syntax._
|
||||||
|
@ -15,6 +15,7 @@ import docspell.backend.fulltext.CreateIndex
|
|||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.ftsclient._
|
import docspell.ftsclient._
|
||||||
import docspell.joex.Config
|
import docspell.joex.Config
|
||||||
|
import docspell.logging.Logger
|
||||||
import docspell.store.Store
|
import docspell.store.Store
|
||||||
|
|
||||||
/** Migrating the index from the previous version to this version.
|
/** Migrating the index from the previous version to this version.
|
||||||
|
@ -11,6 +11,7 @@ import cats.implicits._
|
|||||||
|
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.joex.scheduler.{Context, Task}
|
import docspell.joex.scheduler.{Context, Task}
|
||||||
|
import docspell.logging.Logger
|
||||||
import docspell.store.records._
|
import docspell.store.records._
|
||||||
|
|
||||||
import org.http4s.blaze.client.BlazeClientBuilder
|
import org.http4s.blaze.client.BlazeClientBuilder
|
||||||
|
@ -14,6 +14,7 @@ import fs2.io.file.Path
|
|||||||
|
|
||||||
import docspell.analysis.classifier.{ClassifierModel, TextClassifier}
|
import docspell.analysis.classifier.{ClassifierModel, TextClassifier}
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
|
import docspell.logging.Logger
|
||||||
import docspell.store.Store
|
import docspell.store.Store
|
||||||
import docspell.store.records.RClassifierModel
|
import docspell.store.records.RClassifierModel
|
||||||
|
|
||||||
|
@ -15,6 +15,7 @@ import docspell.backend.ops.OCollective
|
|||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.joex.Config
|
import docspell.joex.Config
|
||||||
import docspell.joex.scheduler._
|
import docspell.joex.scheduler._
|
||||||
|
import docspell.logging.Logger
|
||||||
import docspell.store.records.{RClassifierModel, RClassifierSetting}
|
import docspell.store.records.{RClassifierModel, RClassifierSetting}
|
||||||
|
|
||||||
object LearnClassifierTask {
|
object LearnClassifierTask {
|
||||||
|
@ -13,6 +13,7 @@ import fs2.io.file.Files
|
|||||||
import docspell.analysis.classifier.ClassifierModel
|
import docspell.analysis.classifier.ClassifierModel
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.joex.scheduler._
|
import docspell.joex.scheduler._
|
||||||
|
import docspell.logging.Logger
|
||||||
import docspell.store.Store
|
import docspell.store.Store
|
||||||
import docspell.store.records.RClassifierModel
|
import docspell.store.records.RClassifierModel
|
||||||
|
|
||||||
|
@ -11,6 +11,7 @@ import cats.implicits._
|
|||||||
import fs2.{Pipe, Stream}
|
import fs2.{Pipe, Stream}
|
||||||
|
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
|
import docspell.logging.Logger
|
||||||
import docspell.store.syntax.MimeTypes._
|
import docspell.store.syntax.MimeTypes._
|
||||||
|
|
||||||
import emil.javamail.syntax._
|
import emil.javamail.syntax._
|
||||||
|
@ -12,6 +12,7 @@ import cats.implicits._
|
|||||||
|
|
||||||
import docspell.backend.ops.ONotification
|
import docspell.backend.ops.ONotification
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
|
import docspell.logging.Logger
|
||||||
import docspell.notification.api.ChannelRef
|
import docspell.notification.api.ChannelRef
|
||||||
import docspell.notification.api.Event
|
import docspell.notification.api.Event
|
||||||
import docspell.notification.api.EventContext
|
import docspell.notification.api.EventContext
|
||||||
|
@ -13,6 +13,7 @@ import cats.implicits._
|
|||||||
|
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.joex.scheduler.Task
|
import docspell.joex.scheduler.Task
|
||||||
|
import docspell.logging.Logger
|
||||||
|
|
||||||
/** After candidates have been determined, the set is reduced by doing some cross checks.
|
/** After candidates have been determined, the set is reduced by doing some cross checks.
|
||||||
* For example: if a organization is suggested as correspondent, the correspondent person
|
* For example: if a organization is suggested as correspondent, the correspondent person
|
||||||
|
@ -93,7 +93,7 @@ object ExtractArchive {
|
|||||||
archive: Option[RAttachmentArchive]
|
archive: Option[RAttachmentArchive]
|
||||||
)(ra: RAttachment, pos: Int, mime: MimeType): F[Extracted] =
|
)(ra: RAttachment, pos: Int, mime: MimeType): F[Extracted] =
|
||||||
mime match {
|
mime match {
|
||||||
case MimeType.ZipMatch(_) if ra.name.exists(_.endsWith(".zip")) =>
|
case MimeType.ZipMatch(_) if ra.name.exists(_.toLowerCase.endsWith(".zip")) =>
|
||||||
ctx.logger.info(s"Extracting zip archive ${ra.name.getOrElse("<noname>")}.") *>
|
ctx.logger.info(s"Extracting zip archive ${ra.name.getOrElse("<noname>")}.") *>
|
||||||
extractZip(ctx, archive)(ra, pos)
|
extractZip(ctx, archive)(ra, pos)
|
||||||
.flatMap(cleanupParents(ctx, ra, archive))
|
.flatMap(cleanupParents(ctx, ra, archive))
|
||||||
|
@ -49,7 +49,7 @@ object ReProcessItem {
|
|||||||
|
|
||||||
private def contains[F[_]](ctx: Context[F, Args]): RAttachment => Boolean = {
|
private def contains[F[_]](ctx: Context[F, Args]): RAttachment => Boolean = {
|
||||||
val selection = ctx.args.attachments.toSet
|
val selection = ctx.args.attachments.toSet
|
||||||
if (selection.isEmpty) (_ => true)
|
if (selection.isEmpty) _ => true
|
||||||
else ra => selection.contains(ra.id)
|
else ra => selection.contains(ra.id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1,45 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright 2020 Eike K. & Contributors
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-or-later
|
|
||||||
*/
|
|
||||||
|
|
||||||
package docspell.joex.process
|
|
||||||
|
|
||||||
import cats.effect.Sync
|
|
||||||
import cats.implicits._
|
|
||||||
|
|
||||||
import docspell.common.ProcessItemArgs
|
|
||||||
import docspell.common.syntax.all._
|
|
||||||
import docspell.joex.scheduler.Task
|
|
||||||
|
|
||||||
import org.log4s._
|
|
||||||
|
|
||||||
object TestTasks {
|
|
||||||
private[this] val logger = getLogger
|
|
||||||
|
|
||||||
def success[F[_]]: Task[F, ProcessItemArgs, Unit] =
|
|
||||||
Task(ctx => ctx.logger.info(s"Running task now: ${ctx.args}"))
|
|
||||||
|
|
||||||
def failing[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] =
|
|
||||||
Task { ctx =>
|
|
||||||
ctx.logger
|
|
||||||
.info(s"Failing the task run :(")
|
|
||||||
.map(_ => sys.error("Oh, cannot extract gold from this document"))
|
|
||||||
}
|
|
||||||
|
|
||||||
def longRunning[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] =
|
|
||||||
Task { ctx =>
|
|
||||||
logger.fwarn(s"${Thread.currentThread()} From executing long running task") >>
|
|
||||||
ctx.logger.info(s"${Thread.currentThread()} Running task now: ${ctx.args}") >>
|
|
||||||
sleep(2400) >>
|
|
||||||
ctx.logger.debug("doing things") >>
|
|
||||||
sleep(2400) >>
|
|
||||||
ctx.logger.debug("doing more things") >>
|
|
||||||
sleep(2400) >>
|
|
||||||
ctx.logger.info("doing more things")
|
|
||||||
}
|
|
||||||
|
|
||||||
private def sleep[F[_]: Sync](ms: Long): F[Unit] =
|
|
||||||
Sync[F].delay(Thread.sleep(ms))
|
|
||||||
}
|
|
@ -17,6 +17,7 @@ import docspell.backend.ops.{OJoex, OUpload}
|
|||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.joex.Config
|
import docspell.joex.Config
|
||||||
import docspell.joex.scheduler.{Context, Task}
|
import docspell.joex.scheduler.{Context, Task}
|
||||||
|
import docspell.logging.Logger
|
||||||
import docspell.store.queries.QOrganization
|
import docspell.store.queries.QOrganization
|
||||||
import docspell.store.records._
|
import docspell.store.records._
|
||||||
|
|
||||||
|
@ -11,12 +11,10 @@ import cats.implicits._
|
|||||||
import cats.{Applicative, Functor}
|
import cats.{Applicative, Functor}
|
||||||
|
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.common.syntax.all._
|
import docspell.logging.Logger
|
||||||
import docspell.store.Store
|
import docspell.store.Store
|
||||||
import docspell.store.records.RJob
|
import docspell.store.records.RJob
|
||||||
|
|
||||||
import org.log4s.{Logger => _, _}
|
|
||||||
|
|
||||||
trait Context[F[_], A] { self =>
|
trait Context[F[_], A] { self =>
|
||||||
|
|
||||||
def jobId: Ident
|
def jobId: Ident
|
||||||
@ -42,7 +40,6 @@ trait Context[F[_], A] { self =>
|
|||||||
}
|
}
|
||||||
|
|
||||||
object Context {
|
object Context {
|
||||||
private[this] val log = getLogger
|
|
||||||
|
|
||||||
def create[F[_]: Async, A](
|
def create[F[_]: Async, A](
|
||||||
jobId: Ident,
|
jobId: Ident,
|
||||||
@ -59,13 +56,15 @@ object Context {
|
|||||||
config: SchedulerConfig,
|
config: SchedulerConfig,
|
||||||
logSink: LogSink[F],
|
logSink: LogSink[F],
|
||||||
store: Store[F]
|
store: Store[F]
|
||||||
): F[Context[F, A]] =
|
): F[Context[F, A]] = {
|
||||||
|
val log = docspell.logging.getLogger[F]
|
||||||
for {
|
for {
|
||||||
_ <- log.ftrace("Creating logger for task run")
|
_ <- log.trace("Creating logger for task run")
|
||||||
logger <- QueueLogger(job.id, job.info, config.logBufferSize, logSink)
|
logger <- QueueLogger(job.id, job.info, config.logBufferSize, logSink)
|
||||||
_ <- log.ftrace("Logger created, instantiating context")
|
_ <- log.trace("Logger created, instantiating context")
|
||||||
ctx = create[F, A](job.id, arg, config, logger, store)
|
ctx = create[F, A](job.id, arg, config, logger, store)
|
||||||
} yield ctx
|
} yield ctx
|
||||||
|
}
|
||||||
|
|
||||||
final private class ContextImpl[F[_]: Functor, A](
|
final private class ContextImpl[F[_]: Functor, A](
|
||||||
val args: A,
|
val args: A,
|
||||||
|
@ -11,12 +11,10 @@ import cats.implicits._
|
|||||||
import fs2.Pipe
|
import fs2.Pipe
|
||||||
|
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.common.syntax.all._
|
import docspell.logging
|
||||||
import docspell.store.Store
|
import docspell.store.Store
|
||||||
import docspell.store.records.RJobLog
|
import docspell.store.records.RJobLog
|
||||||
|
|
||||||
import org.log4s.{LogLevel => _, _}
|
|
||||||
|
|
||||||
trait LogSink[F[_]] {
|
trait LogSink[F[_]] {
|
||||||
|
|
||||||
def receive: Pipe[F, LogEvent, Unit]
|
def receive: Pipe[F, LogEvent, Unit]
|
||||||
@ -24,27 +22,31 @@ trait LogSink[F[_]] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
object LogSink {
|
object LogSink {
|
||||||
private[this] val logger = getLogger
|
|
||||||
|
|
||||||
def apply[F[_]](sink: Pipe[F, LogEvent, Unit]): LogSink[F] =
|
def apply[F[_]](sink: Pipe[F, LogEvent, Unit]): LogSink[F] =
|
||||||
new LogSink[F] {
|
new LogSink[F] {
|
||||||
val receive = sink
|
val receive = sink
|
||||||
}
|
}
|
||||||
|
|
||||||
def logInternal[F[_]: Sync](e: LogEvent): F[Unit] =
|
def logInternal[F[_]: Sync](e: LogEvent): F[Unit] = {
|
||||||
|
val logger = docspell.logging.getLogger[F]
|
||||||
|
val addData: logging.LogEvent => logging.LogEvent =
|
||||||
|
_.data("jobId", e.jobId).data("jobInfo", e.jobInfo)
|
||||||
|
|
||||||
e.level match {
|
e.level match {
|
||||||
case LogLevel.Info =>
|
case LogLevel.Info =>
|
||||||
logger.finfo(e.logLine)
|
logger.infoWith(e.logLine)(addData)
|
||||||
case LogLevel.Debug =>
|
case LogLevel.Debug =>
|
||||||
logger.fdebug(e.logLine)
|
logger.debugWith(e.logLine)(addData)
|
||||||
case LogLevel.Warn =>
|
case LogLevel.Warn =>
|
||||||
logger.fwarn(e.logLine)
|
logger.warnWith(e.logLine)(addData)
|
||||||
case LogLevel.Error =>
|
case LogLevel.Error =>
|
||||||
e.ex match {
|
e.ex match {
|
||||||
case Some(exc) =>
|
case Some(exc) =>
|
||||||
logger.ferror(exc)(e.logLine)
|
logger.errorWith(e.logLine)(addData.andThen(_.addError(exc)))
|
||||||
case None =>
|
case None =>
|
||||||
logger.ferror(e.logLine)
|
logger.errorWith(e.logLine)(addData)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -13,13 +13,11 @@ import fs2.concurrent.SignallingRef
|
|||||||
|
|
||||||
import docspell.backend.ops.OJoex
|
import docspell.backend.ops.OJoex
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.common.syntax.all._
|
|
||||||
import docspell.joex.scheduler.PeriodicSchedulerImpl.State
|
import docspell.joex.scheduler.PeriodicSchedulerImpl.State
|
||||||
import docspell.store.queue._
|
import docspell.store.queue._
|
||||||
import docspell.store.records.RPeriodicTask
|
import docspell.store.records.RPeriodicTask
|
||||||
|
|
||||||
import eu.timepit.fs2cron.calev.CalevScheduler
|
import eu.timepit.fs2cron.calev.CalevScheduler
|
||||||
import org.log4s.getLogger
|
|
||||||
|
|
||||||
final class PeriodicSchedulerImpl[F[_]: Async](
|
final class PeriodicSchedulerImpl[F[_]: Async](
|
||||||
val config: PeriodicSchedulerConfig,
|
val config: PeriodicSchedulerConfig,
|
||||||
@ -30,10 +28,10 @@ final class PeriodicSchedulerImpl[F[_]: Async](
|
|||||||
waiter: SignallingRef[F, Boolean],
|
waiter: SignallingRef[F, Boolean],
|
||||||
state: SignallingRef[F, State[F]]
|
state: SignallingRef[F, State[F]]
|
||||||
) extends PeriodicScheduler[F] {
|
) extends PeriodicScheduler[F] {
|
||||||
private[this] val logger = getLogger
|
private[this] val logger = docspell.logging.getLogger[F]
|
||||||
|
|
||||||
def start: Stream[F, Nothing] =
|
def start: Stream[F, Nothing] =
|
||||||
logger.sinfo("Starting periodic scheduler") ++
|
logger.stream.info("Starting periodic scheduler").drain ++
|
||||||
mainLoop
|
mainLoop
|
||||||
|
|
||||||
def shutdown: F[Unit] =
|
def shutdown: F[Unit] =
|
||||||
@ -43,7 +41,7 @@ final class PeriodicSchedulerImpl[F[_]: Async](
|
|||||||
Async[F].start(
|
Async[F].start(
|
||||||
Stream
|
Stream
|
||||||
.awakeEvery[F](config.wakeupPeriod.toScala)
|
.awakeEvery[F](config.wakeupPeriod.toScala)
|
||||||
.evalMap(_ => logger.fdebug("Periodic awake reached") *> notifyChange)
|
.evalMap(_ => logger.debug("Periodic awake reached") *> notifyChange)
|
||||||
.compile
|
.compile
|
||||||
.drain
|
.drain
|
||||||
)
|
)
|
||||||
@ -62,22 +60,22 @@ final class PeriodicSchedulerImpl[F[_]: Async](
|
|||||||
def mainLoop: Stream[F, Nothing] = {
|
def mainLoop: Stream[F, Nothing] = {
|
||||||
val body: F[Boolean] =
|
val body: F[Boolean] =
|
||||||
for {
|
for {
|
||||||
_ <- logger.fdebug(s"Going into main loop")
|
_ <- logger.debug(s"Going into main loop")
|
||||||
now <- Timestamp.current[F]
|
now <- Timestamp.current[F]
|
||||||
_ <- logger.fdebug(s"Looking for next periodic task")
|
_ <- logger.debug(s"Looking for next periodic task")
|
||||||
go <- logThrow("Error getting next task")(
|
go <- logThrow("Error getting next task")(
|
||||||
store
|
store
|
||||||
.takeNext(config.name, None)
|
.takeNext(config.name, None)
|
||||||
.use {
|
.use {
|
||||||
case Marked.Found(pj) =>
|
case Marked.Found(pj) =>
|
||||||
logger
|
logger
|
||||||
.fdebug(s"Found periodic task '${pj.subject}/${pj.timer.asString}'") *>
|
.debug(s"Found periodic task '${pj.subject}/${pj.timer.asString}'") *>
|
||||||
(if (isTriggered(pj, now)) submitJob(pj)
|
(if (isTriggered(pj, now)) submitJob(pj)
|
||||||
else scheduleNotify(pj).map(_ => false))
|
else scheduleNotify(pj).map(_ => false))
|
||||||
case Marked.NotFound =>
|
case Marked.NotFound =>
|
||||||
logger.fdebug("No periodic task found") *> false.pure[F]
|
logger.debug("No periodic task found") *> false.pure[F]
|
||||||
case Marked.NotMarkable =>
|
case Marked.NotMarkable =>
|
||||||
logger.fdebug("Periodic job cannot be marked. Trying again.") *> true
|
logger.debug("Periodic job cannot be marked. Trying again.") *> true
|
||||||
.pure[F]
|
.pure[F]
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
@ -86,7 +84,7 @@ final class PeriodicSchedulerImpl[F[_]: Async](
|
|||||||
Stream
|
Stream
|
||||||
.eval(state.get.map(_.shutdownRequest))
|
.eval(state.get.map(_.shutdownRequest))
|
||||||
.evalTap(
|
.evalTap(
|
||||||
if (_) logger.finfo[F]("Stopping main loop due to shutdown request.")
|
if (_) logger.info("Stopping main loop due to shutdown request.")
|
||||||
else ().pure[F]
|
else ().pure[F]
|
||||||
)
|
)
|
||||||
.flatMap(if (_) Stream.empty else Stream.eval(cancelNotify *> body))
|
.flatMap(if (_) Stream.empty else Stream.eval(cancelNotify *> body))
|
||||||
@ -94,9 +92,9 @@ final class PeriodicSchedulerImpl[F[_]: Async](
|
|||||||
case true =>
|
case true =>
|
||||||
mainLoop
|
mainLoop
|
||||||
case false =>
|
case false =>
|
||||||
logger.sdebug(s"Waiting for notify") ++
|
logger.stream.debug(s"Waiting for notify").drain ++
|
||||||
waiter.discrete.take(2).drain ++
|
waiter.discrete.take(2).drain ++
|
||||||
logger.sdebug(s"Notify signal, going into main loop") ++
|
logger.stream.debug(s"Notify signal, going into main loop").drain ++
|
||||||
mainLoop
|
mainLoop
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -109,12 +107,12 @@ final class PeriodicSchedulerImpl[F[_]: Async](
|
|||||||
.findNonFinalJob(pj.id)
|
.findNonFinalJob(pj.id)
|
||||||
.flatMap {
|
.flatMap {
|
||||||
case Some(job) =>
|
case Some(job) =>
|
||||||
logger.finfo[F](
|
logger.info(
|
||||||
s"There is already a job with non-final state '${job.state}' in the queue"
|
s"There is already a job with non-final state '${job.state}' in the queue"
|
||||||
) *> scheduleNotify(pj) *> false.pure[F]
|
) *> scheduleNotify(pj) *> false.pure[F]
|
||||||
|
|
||||||
case None =>
|
case None =>
|
||||||
logger.finfo[F](s"Submitting job for periodic task '${pj.task.id}'") *>
|
logger.info(s"Submitting job for periodic task '${pj.task.id}'") *>
|
||||||
pj.toJob.flatMap(queue.insert) *> notifyJoex *> true.pure[F]
|
pj.toJob.flatMap(queue.insert) *> notifyJoex *> true.pure[F]
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -125,7 +123,7 @@ final class PeriodicSchedulerImpl[F[_]: Async](
|
|||||||
Timestamp
|
Timestamp
|
||||||
.current[F]
|
.current[F]
|
||||||
.flatMap(now =>
|
.flatMap(now =>
|
||||||
logger.fdebug(
|
logger.debug(
|
||||||
s"Scheduling next notify for timer ${pj.timer.asString} -> ${pj.timer.nextElapse(now.toUtcDateTime)}"
|
s"Scheduling next notify for timer ${pj.timer.asString} -> ${pj.timer.nextElapse(now.toUtcDateTime)}"
|
||||||
)
|
)
|
||||||
) *>
|
) *>
|
||||||
@ -153,13 +151,13 @@ final class PeriodicSchedulerImpl[F[_]: Async](
|
|||||||
private def logError(msg: => String)(fa: F[Unit]): F[Unit] =
|
private def logError(msg: => String)(fa: F[Unit]): F[Unit] =
|
||||||
fa.attempt.flatMap {
|
fa.attempt.flatMap {
|
||||||
case Right(_) => ().pure[F]
|
case Right(_) => ().pure[F]
|
||||||
case Left(ex) => logger.ferror(ex)(msg).map(_ => ())
|
case Left(ex) => logger.error(ex)(msg).map(_ => ())
|
||||||
}
|
}
|
||||||
|
|
||||||
private def logThrow[A](msg: => String)(fa: F[A]): F[A] =
|
private def logThrow[A](msg: => String)(fa: F[A]): F[A] =
|
||||||
fa.attempt.flatMap {
|
fa.attempt.flatMap {
|
||||||
case r @ Right(_) => (r: Either[Throwable, A]).pure[F]
|
case r @ Right(_) => (r: Either[Throwable, A]).pure[F]
|
||||||
case l @ Left(ex) => logger.ferror(ex)(msg).map(_ => (l: Either[Throwable, A]))
|
case l @ Left(ex) => logger.error(ex)(msg).map(_ => (l: Either[Throwable, A]))
|
||||||
}.rethrow
|
}.rethrow
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -12,6 +12,8 @@ import cats.implicits._
|
|||||||
import fs2.Stream
|
import fs2.Stream
|
||||||
|
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
|
import docspell.logging
|
||||||
|
import docspell.logging.{Level, Logger}
|
||||||
|
|
||||||
object QueueLogger {
|
object QueueLogger {
|
||||||
|
|
||||||
@ -21,26 +23,20 @@ object QueueLogger {
|
|||||||
q: Queue[F, LogEvent]
|
q: Queue[F, LogEvent]
|
||||||
): Logger[F] =
|
): Logger[F] =
|
||||||
new Logger[F] {
|
new Logger[F] {
|
||||||
def trace(msg: => String): F[Unit] =
|
|
||||||
LogEvent.create[F](jobId, jobInfo, LogLevel.Debug, msg).flatMap(q.offer)
|
|
||||||
|
|
||||||
def debug(msg: => String): F[Unit] =
|
def log(logEvent: logging.LogEvent) =
|
||||||
LogEvent.create[F](jobId, jobInfo, LogLevel.Debug, msg).flatMap(q.offer)
|
|
||||||
|
|
||||||
def info(msg: => String): F[Unit] =
|
|
||||||
LogEvent.create[F](jobId, jobInfo, LogLevel.Info, msg).flatMap(q.offer)
|
|
||||||
|
|
||||||
def warn(msg: => String): F[Unit] =
|
|
||||||
LogEvent.create[F](jobId, jobInfo, LogLevel.Warn, msg).flatMap(q.offer)
|
|
||||||
|
|
||||||
def error(ex: Throwable)(msg: => String): F[Unit] =
|
|
||||||
LogEvent
|
LogEvent
|
||||||
.create[F](jobId, jobInfo, LogLevel.Error, msg)
|
.create[F](jobId, jobInfo, level2Level(logEvent.level), logEvent.msg())
|
||||||
.map(le => le.copy(ex = Some(ex)))
|
.flatMap { ev =>
|
||||||
.flatMap(q.offer)
|
val event =
|
||||||
|
logEvent.findErrors.headOption
|
||||||
|
.map(ex => ev.copy(ex = Some(ex)))
|
||||||
|
.getOrElse(ev)
|
||||||
|
|
||||||
def error(msg: => String): F[Unit] =
|
q.offer(event)
|
||||||
LogEvent.create[F](jobId, jobInfo, LogLevel.Error, msg).flatMap(q.offer)
|
}
|
||||||
|
|
||||||
|
def asUnsafe = Logger.off
|
||||||
}
|
}
|
||||||
|
|
||||||
def apply[F[_]: Async](
|
def apply[F[_]: Async](
|
||||||
@ -57,4 +53,6 @@ object QueueLogger {
|
|||||||
)
|
)
|
||||||
} yield log
|
} yield log
|
||||||
|
|
||||||
|
private def level2Level(level: Level): LogLevel =
|
||||||
|
LogLevel.fromLevel(level)
|
||||||
}
|
}
|
||||||
|
@ -15,7 +15,6 @@ import fs2.concurrent.SignallingRef
|
|||||||
|
|
||||||
import docspell.backend.msg.JobDone
|
import docspell.backend.msg.JobDone
|
||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.common.syntax.all._
|
|
||||||
import docspell.joex.scheduler.SchedulerImpl._
|
import docspell.joex.scheduler.SchedulerImpl._
|
||||||
import docspell.notification.api.Event
|
import docspell.notification.api.Event
|
||||||
import docspell.notification.api.EventSink
|
import docspell.notification.api.EventSink
|
||||||
@ -26,7 +25,6 @@ import docspell.store.queue.JobQueue
|
|||||||
import docspell.store.records.RJob
|
import docspell.store.records.RJob
|
||||||
|
|
||||||
import io.circe.Json
|
import io.circe.Json
|
||||||
import org.log4s.getLogger
|
|
||||||
|
|
||||||
final class SchedulerImpl[F[_]: Async](
|
final class SchedulerImpl[F[_]: Async](
|
||||||
val config: SchedulerConfig,
|
val config: SchedulerConfig,
|
||||||
@ -41,7 +39,7 @@ final class SchedulerImpl[F[_]: Async](
|
|||||||
permits: Semaphore[F]
|
permits: Semaphore[F]
|
||||||
) extends Scheduler[F] {
|
) extends Scheduler[F] {
|
||||||
|
|
||||||
private[this] val logger = getLogger
|
private[this] val logger = docspell.logging.getLogger[F]
|
||||||
|
|
||||||
/** On startup, get all jobs in state running from this scheduler and put them into
|
/** On startup, get all jobs in state running from this scheduler and put them into
|
||||||
* waiting state, so they get picked up again.
|
* waiting state, so they get picked up again.
|
||||||
@ -53,7 +51,7 @@ final class SchedulerImpl[F[_]: Async](
|
|||||||
Async[F].start(
|
Async[F].start(
|
||||||
Stream
|
Stream
|
||||||
.awakeEvery[F](config.wakeupPeriod.toScala)
|
.awakeEvery[F](config.wakeupPeriod.toScala)
|
||||||
.evalMap(_ => logger.fdebug("Periodic awake reached") *> notifyChange)
|
.evalMap(_ => logger.debug("Periodic awake reached") *> notifyChange)
|
||||||
.compile
|
.compile
|
||||||
.drain
|
.drain
|
||||||
)
|
)
|
||||||
@ -62,7 +60,7 @@ final class SchedulerImpl[F[_]: Async](
|
|||||||
state.get.flatMap(s => QJob.findAll(s.getRunning, store))
|
state.get.flatMap(s => QJob.findAll(s.getRunning, store))
|
||||||
|
|
||||||
def requestCancel(jobId: Ident): F[Boolean] =
|
def requestCancel(jobId: Ident): F[Boolean] =
|
||||||
logger.finfo(s"Scheduler requested to cancel job: ${jobId.id}") *>
|
logger.info(s"Scheduler requested to cancel job: ${jobId.id}") *>
|
||||||
state.get.flatMap(_.cancelRequest(jobId) match {
|
state.get.flatMap(_.cancelRequest(jobId) match {
|
||||||
case Some(ct) => ct.map(_ => true)
|
case Some(ct) => ct.map(_ => true)
|
||||||
case None =>
|
case None =>
|
||||||
@ -74,7 +72,7 @@ final class SchedulerImpl[F[_]: Async](
|
|||||||
)
|
)
|
||||||
} yield true)
|
} yield true)
|
||||||
.getOrElseF(
|
.getOrElseF(
|
||||||
logger.fwarn(s"Job ${jobId.id} not found, cannot cancel.").map(_ => false)
|
logger.warn(s"Job ${jobId.id} not found, cannot cancel.").map(_ => false)
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -90,16 +88,16 @@ final class SchedulerImpl[F[_]: Async](
|
|||||||
|
|
||||||
val wait = Stream
|
val wait = Stream
|
||||||
.eval(runShutdown)
|
.eval(runShutdown)
|
||||||
.evalMap(_ => logger.finfo("Scheduler is shutting down now."))
|
.evalMap(_ => logger.info("Scheduler is shutting down now."))
|
||||||
.flatMap(_ =>
|
.flatMap(_ =>
|
||||||
Stream.eval(state.get) ++ Stream
|
Stream.eval(state.get) ++ Stream
|
||||||
.suspend(state.discrete.takeWhile(_.getRunning.nonEmpty))
|
.suspend(state.discrete.takeWhile(_.getRunning.nonEmpty))
|
||||||
)
|
)
|
||||||
.flatMap { state =>
|
.flatMap { state =>
|
||||||
if (state.getRunning.isEmpty) Stream.eval(logger.finfo("No jobs running."))
|
if (state.getRunning.isEmpty) Stream.eval(logger.info("No jobs running."))
|
||||||
else
|
else
|
||||||
Stream.eval(
|
Stream.eval(
|
||||||
logger.finfo(s"Waiting for ${state.getRunning.size} jobs to finish.")
|
logger.info(s"Waiting for ${state.getRunning.size} jobs to finish.")
|
||||||
) ++
|
) ++
|
||||||
Stream.emit(state)
|
Stream.emit(state)
|
||||||
}
|
}
|
||||||
@ -108,35 +106,35 @@ final class SchedulerImpl[F[_]: Async](
|
|||||||
}
|
}
|
||||||
|
|
||||||
def start: Stream[F, Nothing] =
|
def start: Stream[F, Nothing] =
|
||||||
logger.sinfo("Starting scheduler") ++
|
logger.stream.info("Starting scheduler").drain ++
|
||||||
mainLoop
|
mainLoop
|
||||||
|
|
||||||
def mainLoop: Stream[F, Nothing] = {
|
def mainLoop: Stream[F, Nothing] = {
|
||||||
val body: F[Boolean] =
|
val body: F[Boolean] =
|
||||||
for {
|
for {
|
||||||
_ <- permits.available.flatMap(a =>
|
_ <- permits.available.flatMap(a =>
|
||||||
logger.fdebug(s"Try to acquire permit ($a free)")
|
logger.debug(s"Try to acquire permit ($a free)")
|
||||||
)
|
)
|
||||||
_ <- permits.acquire
|
_ <- permits.acquire
|
||||||
_ <- logger.fdebug("New permit acquired")
|
_ <- logger.debug("New permit acquired")
|
||||||
down <- state.get.map(_.shutdownRequest)
|
down <- state.get.map(_.shutdownRequest)
|
||||||
rjob <-
|
rjob <-
|
||||||
if (down)
|
if (down)
|
||||||
logger.finfo("") *> permits.release *> (None: Option[RJob]).pure[F]
|
logger.info("") *> permits.release *> (None: Option[RJob]).pure[F]
|
||||||
else
|
else
|
||||||
queue.nextJob(
|
queue.nextJob(
|
||||||
group => state.modify(_.nextPrio(group, config.countingScheme)),
|
group => state.modify(_.nextPrio(group, config.countingScheme)),
|
||||||
config.name,
|
config.name,
|
||||||
config.retryDelay
|
config.retryDelay
|
||||||
)
|
)
|
||||||
_ <- logger.fdebug(s"Next job found: ${rjob.map(_.info)}")
|
_ <- logger.debug(s"Next job found: ${rjob.map(_.info)}")
|
||||||
_ <- rjob.map(execute).getOrElse(permits.release)
|
_ <- rjob.map(execute).getOrElse(permits.release)
|
||||||
} yield rjob.isDefined
|
} yield rjob.isDefined
|
||||||
|
|
||||||
Stream
|
Stream
|
||||||
.eval(state.get.map(_.shutdownRequest))
|
.eval(state.get.map(_.shutdownRequest))
|
||||||
.evalTap(
|
.evalTap(
|
||||||
if (_) logger.finfo[F]("Stopping main loop due to shutdown request.")
|
if (_) logger.info("Stopping main loop due to shutdown request.")
|
||||||
else ().pure[F]
|
else ().pure[F]
|
||||||
)
|
)
|
||||||
.flatMap(if (_) Stream.empty else Stream.eval(body))
|
.flatMap(if (_) Stream.empty else Stream.eval(body))
|
||||||
@ -144,9 +142,9 @@ final class SchedulerImpl[F[_]: Async](
|
|||||||
case true =>
|
case true =>
|
||||||
mainLoop
|
mainLoop
|
||||||
case false =>
|
case false =>
|
||||||
logger.sdebug(s"Waiting for notify") ++
|
logger.stream.debug(s"Waiting for notify").drain ++
|
||||||
waiter.discrete.take(2).drain ++
|
waiter.discrete.take(2).drain ++
|
||||||
logger.sdebug(s"Notify signal, going into main loop") ++
|
logger.stream.debug(s"Notify signal, going into main loop").drain ++
|
||||||
mainLoop
|
mainLoop
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -161,17 +159,17 @@ final class SchedulerImpl[F[_]: Async](
|
|||||||
|
|
||||||
task match {
|
task match {
|
||||||
case Left(err) =>
|
case Left(err) =>
|
||||||
logger.ferror(s"Unable to run cancellation task for job ${job.info}: $err")
|
logger.error(s"Unable to run cancellation task for job ${job.info}: $err")
|
||||||
case Right(t) =>
|
case Right(t) =>
|
||||||
for {
|
for {
|
||||||
_ <-
|
_ <-
|
||||||
logger.fdebug(s"Creating context for job ${job.info} to run cancellation $t")
|
logger.debug(s"Creating context for job ${job.info} to run cancellation $t")
|
||||||
ctx <- Context[F, String](job, job.args, config, logSink, store)
|
ctx <- Context[F, String](job, job.args, config, logSink, store)
|
||||||
_ <- t.onCancel.run(ctx)
|
_ <- t.onCancel.run(ctx)
|
||||||
_ <- state.modify(_.markCancelled(job))
|
_ <- state.modify(_.markCancelled(job))
|
||||||
_ <- onFinish(job, Json.Null, JobState.Cancelled)
|
_ <- onFinish(job, Json.Null, JobState.Cancelled)
|
||||||
_ <- ctx.logger.warn("Job has been cancelled.")
|
_ <- ctx.logger.warn("Job has been cancelled.")
|
||||||
_ <- logger.fdebug(s"Job ${job.info} has been cancelled.")
|
_ <- logger.debug(s"Job ${job.info} has been cancelled.")
|
||||||
} yield ()
|
} yield ()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -186,10 +184,10 @@ final class SchedulerImpl[F[_]: Async](
|
|||||||
|
|
||||||
task match {
|
task match {
|
||||||
case Left(err) =>
|
case Left(err) =>
|
||||||
logger.ferror(s"Unable to start a task for job ${job.info}: $err")
|
logger.error(s"Unable to start a task for job ${job.info}: $err")
|
||||||
case Right(t) =>
|
case Right(t) =>
|
||||||
for {
|
for {
|
||||||
_ <- logger.fdebug(s"Creating context for job ${job.info} to run $t")
|
_ <- logger.debug(s"Creating context for job ${job.info} to run $t")
|
||||||
ctx <- Context[F, String](job, job.args, config, logSink, store)
|
ctx <- Context[F, String](job, job.args, config, logSink, store)
|
||||||
jot = wrapTask(job, t.task, ctx)
|
jot = wrapTask(job, t.task, ctx)
|
||||||
tok <- forkRun(job, jot.run(ctx), t.onCancel.run(ctx), ctx)
|
tok <- forkRun(job, jot.run(ctx), t.onCancel.run(ctx), ctx)
|
||||||
@ -200,9 +198,9 @@ final class SchedulerImpl[F[_]: Async](
|
|||||||
|
|
||||||
def onFinish(job: RJob, result: Json, finishState: JobState): F[Unit] =
|
def onFinish(job: RJob, result: Json, finishState: JobState): F[Unit] =
|
||||||
for {
|
for {
|
||||||
_ <- logger.fdebug(s"Job ${job.info} done $finishState. Releasing resources.")
|
_ <- logger.debug(s"Job ${job.info} done $finishState. Releasing resources.")
|
||||||
_ <- permits.release *> permits.available.flatMap(a =>
|
_ <- permits.release *> permits.available.flatMap(a =>
|
||||||
logger.fdebug(s"Permit released ($a free)")
|
logger.debug(s"Permit released ($a free)")
|
||||||
)
|
)
|
||||||
_ <- state.modify(_.removeRunning(job))
|
_ <- state.modify(_.removeRunning(job))
|
||||||
_ <- QJob.setFinalState(job.id, finishState, store)
|
_ <- QJob.setFinalState(job.id, finishState, store)
|
||||||
@ -241,7 +239,7 @@ final class SchedulerImpl[F[_]: Async](
|
|||||||
ctx: Context[F, String]
|
ctx: Context[F, String]
|
||||||
): Task[F, String, Unit] =
|
): Task[F, String, Unit] =
|
||||||
task
|
task
|
||||||
.mapF(fa => onStart(job) *> logger.fdebug("Starting task now") *> fa)
|
.mapF(fa => onStart(job) *> logger.debug("Starting task now") *> fa)
|
||||||
.mapF(_.attempt.flatMap {
|
.mapF(_.attempt.flatMap {
|
||||||
case Right(result) =>
|
case Right(result) =>
|
||||||
logger.info(s"Job execution successful: ${job.info}")
|
logger.info(s"Job execution successful: ${job.info}")
|
||||||
@ -284,11 +282,11 @@ final class SchedulerImpl[F[_]: Async](
|
|||||||
onCancel: F[Unit],
|
onCancel: F[Unit],
|
||||||
ctx: Context[F, String]
|
ctx: Context[F, String]
|
||||||
): F[F[Unit]] =
|
): F[F[Unit]] =
|
||||||
logger.fdebug(s"Forking job ${job.info}") *>
|
logger.debug(s"Forking job ${job.info}") *>
|
||||||
Async[F]
|
Async[F]
|
||||||
.start(code)
|
.start(code)
|
||||||
.map(fiber =>
|
.map(fiber =>
|
||||||
logger.fdebug(s"Cancelling job ${job.info}") *>
|
logger.debug(s"Cancelling job ${job.info}") *>
|
||||||
fiber.cancel *>
|
fiber.cancel *>
|
||||||
onCancel.attempt.map {
|
onCancel.attempt.map {
|
||||||
case Right(_) => ()
|
case Right(_) => ()
|
||||||
@ -299,7 +297,7 @@ final class SchedulerImpl[F[_]: Async](
|
|||||||
state.modify(_.markCancelled(job)) *>
|
state.modify(_.markCancelled(job)) *>
|
||||||
onFinish(job, Json.Null, JobState.Cancelled) *>
|
onFinish(job, Json.Null, JobState.Cancelled) *>
|
||||||
ctx.logger.warn("Job has been cancelled.") *>
|
ctx.logger.warn("Job has been cancelled.") *>
|
||||||
logger.fdebug(s"Job ${job.info} has been cancelled.")
|
logger.debug(s"Job ${job.info} has been cancelled.")
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -11,7 +11,7 @@ import cats.data.Kleisli
|
|||||||
import cats.effect.Sync
|
import cats.effect.Sync
|
||||||
import cats.implicits._
|
import cats.implicits._
|
||||||
|
|
||||||
import docspell.common.Logger
|
import docspell.logging.Logger
|
||||||
|
|
||||||
/** The code that is executed by the scheduler */
|
/** The code that is executed by the scheduler */
|
||||||
trait Task[F[_], A, B] {
|
trait Task[F[_], A, B] {
|
||||||
|
@ -9,7 +9,6 @@ package docspell.joexapi.client
|
|||||||
import cats.effect._
|
import cats.effect._
|
||||||
import cats.implicits._
|
import cats.implicits._
|
||||||
|
|
||||||
import docspell.common.syntax.all._
|
|
||||||
import docspell.common.{Ident, LenientUri}
|
import docspell.common.{Ident, LenientUri}
|
||||||
import docspell.joexapi.model.BasicResult
|
import docspell.joexapi.model.BasicResult
|
||||||
|
|
||||||
@ -17,7 +16,6 @@ import org.http4s.blaze.client.BlazeClientBuilder
|
|||||||
import org.http4s.circe.CirceEntityDecoder
|
import org.http4s.circe.CirceEntityDecoder
|
||||||
import org.http4s.client.Client
|
import org.http4s.client.Client
|
||||||
import org.http4s.{Method, Request, Uri}
|
import org.http4s.{Method, Request, Uri}
|
||||||
import org.log4s.getLogger
|
|
||||||
|
|
||||||
trait JoexClient[F[_]] {
|
trait JoexClient[F[_]] {
|
||||||
|
|
||||||
@ -31,22 +29,21 @@ trait JoexClient[F[_]] {
|
|||||||
|
|
||||||
object JoexClient {
|
object JoexClient {
|
||||||
|
|
||||||
private[this] val logger = getLogger
|
|
||||||
|
|
||||||
def apply[F[_]: Async](client: Client[F]): JoexClient[F] =
|
def apply[F[_]: Async](client: Client[F]): JoexClient[F] =
|
||||||
new JoexClient[F] with CirceEntityDecoder {
|
new JoexClient[F] with CirceEntityDecoder {
|
||||||
|
private[this] val logger = docspell.logging.getLogger[F]
|
||||||
|
|
||||||
def notifyJoex(base: LenientUri): F[BasicResult] = {
|
def notifyJoex(base: LenientUri): F[BasicResult] = {
|
||||||
val notifyUrl = base / "api" / "v1" / "notify"
|
val notifyUrl = base / "api" / "v1" / "notify"
|
||||||
val req = Request[F](Method.POST, uri(notifyUrl))
|
val req = Request[F](Method.POST, uri(notifyUrl))
|
||||||
logger.fdebug(s"Notify joex at ${notifyUrl.asString}") *>
|
logger.debug(s"Notify joex at ${notifyUrl.asString}") *>
|
||||||
client.expect[BasicResult](req)
|
client.expect[BasicResult](req)
|
||||||
}
|
}
|
||||||
|
|
||||||
def notifyJoexIgnoreErrors(base: LenientUri): F[Unit] =
|
def notifyJoexIgnoreErrors(base: LenientUri): F[Unit] =
|
||||||
notifyJoex(base).attempt.map {
|
notifyJoex(base).attempt.flatMap {
|
||||||
case Right(BasicResult(succ, msg)) =>
|
case Right(BasicResult(succ, msg)) =>
|
||||||
if (succ) ()
|
if (succ) ().pure[F]
|
||||||
else
|
else
|
||||||
logger.warn(
|
logger.warn(
|
||||||
s"Notifying Joex instance '${base.asString}' returned with failure: $msg"
|
s"Notifying Joex instance '${base.asString}' returned with failure: $msg"
|
||||||
|
@ -0,0 +1,39 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2020 Eike K. & Contributors
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
*/
|
||||||
|
|
||||||
|
package docspell.logging
|
||||||
|
|
||||||
|
import cats.data.NonEmptyList
|
||||||
|
import cats.syntax.all._
|
||||||
|
import cats.{Applicative, Id}
|
||||||
|
|
||||||
|
final private[logging] class AndThenLogger[F[_]: Applicative](
|
||||||
|
val loggers: NonEmptyList[Logger[F]]
|
||||||
|
) extends Logger[F] {
|
||||||
|
def log(ev: LogEvent): F[Unit] =
|
||||||
|
loggers.traverse(_.log(ev)).as(())
|
||||||
|
|
||||||
|
def asUnsafe: Logger[Id] =
|
||||||
|
new Logger[Id] { self =>
|
||||||
|
def log(ev: LogEvent): Unit =
|
||||||
|
loggers.toList.foreach(_.asUnsafe.log(ev))
|
||||||
|
def asUnsafe = self
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private[logging] object AndThenLogger {
|
||||||
|
def combine[F[_]: Applicative](a: Logger[F], b: Logger[F]): Logger[F] =
|
||||||
|
(a, b) match {
|
||||||
|
case (aa: AndThenLogger[F], bb: AndThenLogger[F]) =>
|
||||||
|
new AndThenLogger[F](aa.loggers ++ bb.loggers.toList)
|
||||||
|
case (aa: AndThenLogger[F], _) =>
|
||||||
|
new AndThenLogger[F](aa.loggers.prepend(b))
|
||||||
|
case (_, bb: AndThenLogger[F]) =>
|
||||||
|
new AndThenLogger[F](bb.loggers.prepend(a))
|
||||||
|
case _ =>
|
||||||
|
new AndThenLogger[F](NonEmptyList.of(a, b))
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,57 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2020 Eike K. & Contributors
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
*/
|
||||||
|
|
||||||
|
package docspell.logging
|
||||||
|
|
||||||
|
import cats.Order
|
||||||
|
import cats.data.NonEmptyList
|
||||||
|
|
||||||
|
import io.circe.{Decoder, Encoder}
|
||||||
|
|
||||||
|
sealed trait Level { self: Product =>
|
||||||
|
val name: String =
|
||||||
|
productPrefix.toUpperCase
|
||||||
|
|
||||||
|
val value: Double
|
||||||
|
}
|
||||||
|
|
||||||
|
object Level {
|
||||||
|
case object Fatal extends Level {
|
||||||
|
val value = 600.0
|
||||||
|
}
|
||||||
|
case object Error extends Level {
|
||||||
|
val value = 500.0
|
||||||
|
}
|
||||||
|
case object Warn extends Level {
|
||||||
|
val value = 400.0
|
||||||
|
}
|
||||||
|
case object Info extends Level {
|
||||||
|
val value = 300.0
|
||||||
|
}
|
||||||
|
case object Debug extends Level {
|
||||||
|
val value = 200.0
|
||||||
|
}
|
||||||
|
case object Trace extends Level {
|
||||||
|
val value = 100.0
|
||||||
|
}
|
||||||
|
|
||||||
|
val all: NonEmptyList[Level] =
|
||||||
|
NonEmptyList.of(Fatal, Error, Warn, Info, Debug, Trace)
|
||||||
|
|
||||||
|
def fromString(str: String): Either[String, Level] = {
|
||||||
|
val s = str.toUpperCase
|
||||||
|
all.find(_.name == s).toRight(s"Invalid level name: $str")
|
||||||
|
}
|
||||||
|
|
||||||
|
implicit val order: Order[Level] =
|
||||||
|
Order.by(_.value)
|
||||||
|
|
||||||
|
implicit val jsonEncoder: Encoder[Level] =
|
||||||
|
Encoder.encodeString.contramap(_.name)
|
||||||
|
|
||||||
|
implicit val jsonDecoder: Decoder[Level] =
|
||||||
|
Decoder.decodeString.emap(fromString)
|
||||||
|
}
|
@ -0,0 +1,45 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2020 Eike K. & Contributors
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
*/
|
||||||
|
|
||||||
|
package docspell.logging
|
||||||
|
|
||||||
|
import cats.data.NonEmptyList
|
||||||
|
|
||||||
|
import io.circe.{Decoder, Encoder}
|
||||||
|
|
||||||
|
final case class LogConfig(minimumLevel: Level, format: LogConfig.Format) {}
|
||||||
|
|
||||||
|
object LogConfig {
|
||||||
|
|
||||||
|
sealed trait Format { self: Product =>
|
||||||
|
def name: String =
|
||||||
|
productPrefix.toLowerCase
|
||||||
|
}
|
||||||
|
object Format {
|
||||||
|
case object Plain extends Format
|
||||||
|
case object Fancy extends Format
|
||||||
|
case object Json extends Format
|
||||||
|
case object Logfmt extends Format
|
||||||
|
|
||||||
|
val all: NonEmptyList[Format] =
|
||||||
|
NonEmptyList.of(Plain, Fancy, Json, Logfmt)
|
||||||
|
|
||||||
|
def fromString(str: String): Either[String, Format] =
|
||||||
|
all.find(_.name.equalsIgnoreCase(str)).toRight(s"Invalid format name: $str")
|
||||||
|
|
||||||
|
implicit val jsonDecoder: Decoder[Format] =
|
||||||
|
Decoder.decodeString.emap(fromString)
|
||||||
|
|
||||||
|
implicit val jsonEncoder: Encoder[Format] =
|
||||||
|
Encoder.encodeString.contramap(_.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
implicit val jsonDecoder: Decoder[LogConfig] =
|
||||||
|
Decoder.forProduct2("minimumLevel", "format")(LogConfig.apply)
|
||||||
|
|
||||||
|
implicit val jsonEncoder: Encoder[LogConfig] =
|
||||||
|
Encoder.forProduct2("minimumLevel", "format")(r => (r.minimumLevel, r.format))
|
||||||
|
}
|
@ -0,0 +1,51 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2020 Eike K. & Contributors
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
*/
|
||||||
|
|
||||||
|
package docspell.logging
|
||||||
|
|
||||||
|
import io.circe.{Encoder, Json}
|
||||||
|
import sourcecode._
|
||||||
|
|
||||||
|
final case class LogEvent(
|
||||||
|
level: Level,
|
||||||
|
msg: () => String,
|
||||||
|
additional: List[() => LogEvent.AdditionalMsg],
|
||||||
|
data: Map[String, () => Json],
|
||||||
|
pkg: Pkg,
|
||||||
|
fileName: FileName,
|
||||||
|
name: Name,
|
||||||
|
line: Line
|
||||||
|
) {
|
||||||
|
|
||||||
|
def asString =
|
||||||
|
s"${level.name} ${name.value}/${fileName}:${line.value} - ${msg()}"
|
||||||
|
|
||||||
|
def data[A: Encoder](key: String, value: => A): LogEvent =
|
||||||
|
copy(data = data.updated(key, () => Encoder[A].apply(value)))
|
||||||
|
|
||||||
|
def addMessage(msg: => String): LogEvent =
|
||||||
|
copy(additional = (() => Left(msg)) :: additional)
|
||||||
|
|
||||||
|
def addError(ex: Throwable): LogEvent =
|
||||||
|
copy(additional = (() => Right(ex)) :: additional)
|
||||||
|
|
||||||
|
def findErrors: List[Throwable] =
|
||||||
|
additional.map(a => a()).collect { case Right(ex) =>
|
||||||
|
ex
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
object LogEvent {
|
||||||
|
|
||||||
|
type AdditionalMsg = Either[String, Throwable]
|
||||||
|
|
||||||
|
def of(l: Level, m: => String)(implicit
|
||||||
|
pkg: Pkg,
|
||||||
|
fileName: FileName,
|
||||||
|
name: Name,
|
||||||
|
line: Line
|
||||||
|
): LogEvent = LogEvent(l, () => m, Nil, Map.empty, pkg, fileName, name, line)
|
||||||
|
}
|
169
modules/logging/api/src/main/scala/docspell/logging/Logger.scala
Normal file
169
modules/logging/api/src/main/scala/docspell/logging/Logger.scala
Normal file
@ -0,0 +1,169 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2020 Eike K. & Contributors
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
*/
|
||||||
|
|
||||||
|
package docspell.logging
|
||||||
|
|
||||||
|
import java.io.PrintStream
|
||||||
|
import java.time.Instant
|
||||||
|
|
||||||
|
import cats.effect.{Ref, Sync}
|
||||||
|
import cats.syntax.applicative._
|
||||||
|
import cats.syntax.functor._
|
||||||
|
import cats.syntax.order._
|
||||||
|
import cats.{Applicative, Id}
|
||||||
|
|
||||||
|
import sourcecode._
|
||||||
|
|
||||||
|
trait Logger[F[_]] extends LoggerExtension[F] {
|
||||||
|
|
||||||
|
def log(ev: LogEvent): F[Unit]
|
||||||
|
|
||||||
|
def asUnsafe: Logger[Id]
|
||||||
|
|
||||||
|
def trace(msg: => String)(implicit
|
||||||
|
pkg: Pkg,
|
||||||
|
fileName: FileName,
|
||||||
|
name: Name,
|
||||||
|
line: Line
|
||||||
|
): F[Unit] =
|
||||||
|
log(LogEvent.of(Level.Trace, msg))
|
||||||
|
|
||||||
|
def traceWith(msg: => String)(modify: LogEvent => LogEvent)(implicit
|
||||||
|
pkg: Pkg,
|
||||||
|
fileName: FileName,
|
||||||
|
name: Name,
|
||||||
|
line: Line
|
||||||
|
): F[Unit] =
|
||||||
|
log(modify(LogEvent.of(Level.Trace, msg)))
|
||||||
|
|
||||||
|
def debug(msg: => String)(implicit
|
||||||
|
pkg: Pkg,
|
||||||
|
fileName: FileName,
|
||||||
|
name: Name,
|
||||||
|
line: Line
|
||||||
|
): F[Unit] =
|
||||||
|
log(LogEvent.of(Level.Debug, msg))
|
||||||
|
|
||||||
|
def debugWith(msg: => String)(modify: LogEvent => LogEvent)(implicit
|
||||||
|
pkg: Pkg,
|
||||||
|
fileName: FileName,
|
||||||
|
name: Name,
|
||||||
|
line: Line
|
||||||
|
): F[Unit] =
|
||||||
|
log(modify(LogEvent.of(Level.Debug, msg)))
|
||||||
|
|
||||||
|
def info(msg: => String)(implicit
|
||||||
|
pkg: Pkg,
|
||||||
|
fileName: FileName,
|
||||||
|
name: Name,
|
||||||
|
line: Line
|
||||||
|
): F[Unit] =
|
||||||
|
log(LogEvent.of(Level.Info, msg))
|
||||||
|
|
||||||
|
def infoWith(msg: => String)(modify: LogEvent => LogEvent)(implicit
|
||||||
|
pkg: Pkg,
|
||||||
|
fileName: FileName,
|
||||||
|
name: Name,
|
||||||
|
line: Line
|
||||||
|
): F[Unit] =
|
||||||
|
log(modify(LogEvent.of(Level.Info, msg)))
|
||||||
|
|
||||||
|
def warn(msg: => String)(implicit
|
||||||
|
pkg: Pkg,
|
||||||
|
fileName: FileName,
|
||||||
|
name: Name,
|
||||||
|
line: Line
|
||||||
|
): F[Unit] =
|
||||||
|
log(LogEvent.of(Level.Warn, msg))
|
||||||
|
|
||||||
|
def warnWith(msg: => String)(modify: LogEvent => LogEvent)(implicit
|
||||||
|
pkg: Pkg,
|
||||||
|
fileName: FileName,
|
||||||
|
name: Name,
|
||||||
|
line: Line
|
||||||
|
): F[Unit] =
|
||||||
|
log(modify(LogEvent.of(Level.Warn, msg)))
|
||||||
|
|
||||||
|
def warn(ex: Throwable)(msg: => String)(implicit
|
||||||
|
pkg: Pkg,
|
||||||
|
fileName: FileName,
|
||||||
|
name: Name,
|
||||||
|
line: Line
|
||||||
|
): F[Unit] =
|
||||||
|
log(LogEvent.of(Level.Warn, msg).addError(ex))
|
||||||
|
|
||||||
|
def error(msg: => String)(implicit
|
||||||
|
pkg: Pkg,
|
||||||
|
fileName: FileName,
|
||||||
|
name: Name,
|
||||||
|
line: Line
|
||||||
|
): F[Unit] =
|
||||||
|
log(LogEvent.of(Level.Error, msg))
|
||||||
|
|
||||||
|
def errorWith(msg: => String)(modify: LogEvent => LogEvent)(implicit
|
||||||
|
pkg: Pkg,
|
||||||
|
fileName: FileName,
|
||||||
|
name: Name,
|
||||||
|
line: Line
|
||||||
|
): F[Unit] =
|
||||||
|
log(modify(LogEvent.of(Level.Error, msg)))
|
||||||
|
|
||||||
|
def error(ex: Throwable)(msg: => String)(implicit
|
||||||
|
pkg: Pkg,
|
||||||
|
fileName: FileName,
|
||||||
|
name: Name,
|
||||||
|
line: Line
|
||||||
|
): F[Unit] =
|
||||||
|
log(LogEvent.of(Level.Error, msg).addError(ex))
|
||||||
|
}
|
||||||
|
|
||||||
|
object Logger {
|
||||||
|
def off: Logger[Id] =
|
||||||
|
new Logger[Id] {
|
||||||
|
def log(ev: LogEvent): Unit = ()
|
||||||
|
def asUnsafe = this
|
||||||
|
}
|
||||||
|
|
||||||
|
def offF[F[_]: Applicative]: Logger[F] =
|
||||||
|
new Logger[F] {
|
||||||
|
def log(ev: LogEvent) = ().pure[F]
|
||||||
|
def asUnsafe = off
|
||||||
|
}
|
||||||
|
|
||||||
|
def buffer[F[_]: Sync](): F[(Ref[F, Vector[LogEvent]], Logger[F])] =
|
||||||
|
for {
|
||||||
|
buffer <- Ref.of[F, Vector[LogEvent]](Vector.empty[LogEvent])
|
||||||
|
logger =
|
||||||
|
new Logger[F] {
|
||||||
|
def log(ev: LogEvent) =
|
||||||
|
buffer.update(_.appended(ev))
|
||||||
|
def asUnsafe = off
|
||||||
|
}
|
||||||
|
} yield (buffer, logger)
|
||||||
|
|
||||||
|
/** Just prints to the given print stream. Useful for testing. */
|
||||||
|
def simple(ps: PrintStream, minimumLevel: Level): Logger[Id] =
|
||||||
|
new Logger[Id] {
|
||||||
|
def log(ev: LogEvent): Unit =
|
||||||
|
if (ev.level >= minimumLevel)
|
||||||
|
ps.println(s"${Instant.now()} [${Thread.currentThread()}] ${ev.asString}")
|
||||||
|
else
|
||||||
|
()
|
||||||
|
|
||||||
|
def asUnsafe = this
|
||||||
|
}
|
||||||
|
|
||||||
|
def simpleF[F[_]: Sync](ps: PrintStream, minimumLevel: Level): Logger[F] =
|
||||||
|
new Logger[F] {
|
||||||
|
def log(ev: LogEvent) =
|
||||||
|
Sync[F].delay(asUnsafe.log(ev))
|
||||||
|
|
||||||
|
val asUnsafe = simple(ps, minimumLevel)
|
||||||
|
}
|
||||||
|
|
||||||
|
def simpleDefault[F[_]: Sync](minimumLevel: Level = Level.Info): Logger[F] =
|
||||||
|
simpleF[F](System.err, minimumLevel)
|
||||||
|
}
|
@ -0,0 +1,27 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2020 Eike K. & Contributors
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
*/
|
||||||
|
|
||||||
|
package docspell.logging
|
||||||
|
|
||||||
|
import cats.Applicative
|
||||||
|
import fs2.Stream
|
||||||
|
|
||||||
|
trait LoggerExtension[F[_]] { self: Logger[F] =>
|
||||||
|
|
||||||
|
def stream: Logger[Stream[F, *]] =
|
||||||
|
new Logger[Stream[F, *]] {
|
||||||
|
def log(ev: LogEvent) =
|
||||||
|
Stream.eval(self.log(ev))
|
||||||
|
|
||||||
|
def asUnsafe = self.asUnsafe
|
||||||
|
}
|
||||||
|
|
||||||
|
def andThen(other: Logger[F])(implicit F: Applicative[F]): Logger[F] =
|
||||||
|
AndThenLogger.combine(self, other)
|
||||||
|
|
||||||
|
def >>(other: Logger[F])(implicit F: Applicative[F]): Logger[F] =
|
||||||
|
AndThenLogger.combine(self, other)
|
||||||
|
}
|
@ -0,0 +1,26 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2020 Eike K. & Contributors
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
*/
|
||||||
|
|
||||||
|
package docspell.logging.impl
|
||||||
|
|
||||||
|
import io.circe.syntax._
|
||||||
|
import scribe._
|
||||||
|
import scribe.output._
|
||||||
|
import scribe.output.format.OutputFormat
|
||||||
|
import scribe.writer._
|
||||||
|
|
||||||
|
final case class JsonWriter(writer: Writer, compact: Boolean = true) extends Writer {
|
||||||
|
override def write[M](
|
||||||
|
record: LogRecord[M],
|
||||||
|
output: LogOutput,
|
||||||
|
outputFormat: OutputFormat
|
||||||
|
): Unit = {
|
||||||
|
val r = Record.fromLogRecord(record)
|
||||||
|
val json = r.asJson
|
||||||
|
val jsonString = if (compact) json.noSpaces else json.spaces2
|
||||||
|
writer.write(record, new TextOutput(jsonString), outputFormat)
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,32 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2020 Eike K. & Contributors
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
*/
|
||||||
|
|
||||||
|
package docspell.logging.impl
|
||||||
|
|
||||||
|
import io.circe.syntax._
|
||||||
|
import scribe._
|
||||||
|
import scribe.output._
|
||||||
|
import scribe.output.format.OutputFormat
|
||||||
|
import scribe.writer._
|
||||||
|
|
||||||
|
// https://brandur.org/logfmt
|
||||||
|
final case class LogfmtWriter(writer: Writer) extends Writer {
|
||||||
|
override def write[M](
|
||||||
|
record: LogRecord[M],
|
||||||
|
output: LogOutput,
|
||||||
|
outputFormat: OutputFormat
|
||||||
|
): Unit = {
|
||||||
|
val r = Record.fromLogRecord(record)
|
||||||
|
val data = r.data
|
||||||
|
.map { case (k, v) =>
|
||||||
|
s"$k=${v.noSpaces}"
|
||||||
|
}
|
||||||
|
.mkString(" ")
|
||||||
|
val logfmtStr =
|
||||||
|
s"""level=${r.level.asJson.noSpaces} levelValue=${r.levelValue} message=${r.message.asJson.noSpaces} fileName=${r.fileName.asJson.noSpaces} className=${r.className.asJson.noSpaces} methodName=${r.methodName.asJson.noSpaces} line=${r.line.asJson.noSpaces} column=${r.column.asJson.noSpaces} $data timestamp=${r.timeStamp} date=${r.date} time=${r.time}"""
|
||||||
|
writer.write(record, new TextOutput(logfmtStr), outputFormat)
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,120 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2020 Eike K. & Contributors
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
*/
|
||||||
|
|
||||||
|
package docspell.logging.impl
|
||||||
|
|
||||||
|
import docspell.logging.impl.Record._
|
||||||
|
|
||||||
|
import io.circe.syntax._
|
||||||
|
import io.circe.{Encoder, Json}
|
||||||
|
import perfolation._
|
||||||
|
import scribe.LogRecord
|
||||||
|
import scribe.data.MDC
|
||||||
|
import scribe.message.Message
|
||||||
|
|
||||||
|
// From: https://github.com/outr/scribe/blob/8e99521e1ee1f0c421629764dd96e4eb193d84bd/json/shared/src/main/scala/scribe/json/JsonWriter.scala
|
||||||
|
// which would introduce jackson and other dependencies. Modified to work with circe.
|
||||||
|
// Original licensed under MIT.
|
||||||
|
|
||||||
|
private[impl] case class Record(
|
||||||
|
level: String,
|
||||||
|
levelValue: Double,
|
||||||
|
message: String,
|
||||||
|
additionalMessages: List[String],
|
||||||
|
fileName: String,
|
||||||
|
className: String,
|
||||||
|
methodName: Option[String],
|
||||||
|
line: Option[Int],
|
||||||
|
column: Option[Int],
|
||||||
|
data: Map[String, Json],
|
||||||
|
traces: List[Trace],
|
||||||
|
timeStamp: Long,
|
||||||
|
date: String,
|
||||||
|
time: String
|
||||||
|
)
|
||||||
|
|
||||||
|
private[impl] object Record {
|
||||||
|
|
||||||
|
def fromLogRecord[M](record: LogRecord[M]): Record = {
|
||||||
|
val l = record.timeStamp
|
||||||
|
val traces = record.additionalMessages.collect {
|
||||||
|
case message: Message[_] if message.value.isInstanceOf[Throwable] =>
|
||||||
|
throwable2Trace(message.value.asInstanceOf[Throwable])
|
||||||
|
}
|
||||||
|
val additionalMessages = record.additionalMessages.map(_.logOutput.plainText)
|
||||||
|
|
||||||
|
Record(
|
||||||
|
level = record.level.name,
|
||||||
|
levelValue = record.levelValue,
|
||||||
|
message = record.logOutput.plainText,
|
||||||
|
additionalMessages = additionalMessages,
|
||||||
|
fileName = record.fileName,
|
||||||
|
className = record.className,
|
||||||
|
methodName = record.methodName,
|
||||||
|
line = record.line,
|
||||||
|
column = record.column,
|
||||||
|
data = (record.data ++ MDC.map).map { case (key, value) =>
|
||||||
|
value() match {
|
||||||
|
case value: Json => key -> value
|
||||||
|
case value: Int => key -> value.asJson
|
||||||
|
case value: Long => key -> value.asJson
|
||||||
|
case value: Double => key -> value.asJson
|
||||||
|
case any => key -> Json.fromString(any.toString)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
traces = traces,
|
||||||
|
timeStamp = l,
|
||||||
|
date = l.t.F,
|
||||||
|
time = s"${l.t.T}.${l.t.L}${l.t.z}"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private def throwable2Trace(throwable: Throwable): Trace = {
|
||||||
|
val elements = throwable.getStackTrace.toList.map { e =>
|
||||||
|
TraceElement(e.getClassName, e.getMethodName, e.getLineNumber)
|
||||||
|
}
|
||||||
|
Trace(
|
||||||
|
throwable.getLocalizedMessage,
|
||||||
|
elements,
|
||||||
|
Option(throwable.getCause).map(throwable2Trace)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
implicit val jsonEncoder: Encoder[Record] =
|
||||||
|
Encoder.forProduct14(
|
||||||
|
"level",
|
||||||
|
"levelValue",
|
||||||
|
"message",
|
||||||
|
"additionalMessages",
|
||||||
|
"fileName",
|
||||||
|
"className",
|
||||||
|
"methodName",
|
||||||
|
"line",
|
||||||
|
"column",
|
||||||
|
"data",
|
||||||
|
"traces",
|
||||||
|
"timestamp",
|
||||||
|
"date",
|
||||||
|
"time"
|
||||||
|
)(r => Record.unapply(r).get)
|
||||||
|
|
||||||
|
case class Trace(message: String, elements: List[TraceElement], cause: Option[Trace])
|
||||||
|
|
||||||
|
object Trace {
|
||||||
|
implicit def jsonEncoder: Encoder[Trace] =
|
||||||
|
Encoder.forProduct3("message", "elements", "cause")(r => Trace.unapply(r).get)
|
||||||
|
|
||||||
|
implicit def openEncoder: Encoder[Option[Trace]] =
|
||||||
|
Encoder.instance(opt => opt.map(jsonEncoder.apply).getOrElse(Json.Null))
|
||||||
|
}
|
||||||
|
|
||||||
|
case class TraceElement(`class`: String, method: String, line: Int)
|
||||||
|
|
||||||
|
object TraceElement {
|
||||||
|
implicit val jsonEncoder: Encoder[TraceElement] =
|
||||||
|
Encoder.forProduct3("class", "method", "line")(r => TraceElement.unapply(r).get)
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,69 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2020 Eike K. & Contributors
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
*/
|
||||||
|
|
||||||
|
package docspell.logging.impl
|
||||||
|
|
||||||
|
import cats.effect.Sync
|
||||||
|
|
||||||
|
import docspell.logging.LogConfig.Format
|
||||||
|
import docspell.logging.{Level, LogConfig}
|
||||||
|
|
||||||
|
import scribe.format.Formatter
|
||||||
|
import scribe.jul.JULHandler
|
||||||
|
|
||||||
|
object ScribeConfigure {
|
||||||
|
private[this] val docspellRootVerbose = "DOCSPELL_ROOT_LOGGER_LEVEL"
|
||||||
|
|
||||||
|
def configure[F[_]: Sync](cfg: LogConfig): F[Unit] =
|
||||||
|
Sync[F].delay {
|
||||||
|
replaceJUL()
|
||||||
|
val docspellLogger = scribe.Logger("docspell")
|
||||||
|
unsafeConfigure(scribe.Logger.root, cfg.copy(minimumLevel = getRootMinimumLevel))
|
||||||
|
unsafeConfigure(docspellLogger, cfg)
|
||||||
|
}
|
||||||
|
|
||||||
|
private[this] def getRootMinimumLevel: Level =
|
||||||
|
Option(System.getenv(docspellRootVerbose))
|
||||||
|
.map(Level.fromString)
|
||||||
|
.flatMap {
|
||||||
|
case Right(level) => Some(level)
|
||||||
|
case Left(err) =>
|
||||||
|
scribe.warn(
|
||||||
|
s"Environment variable '$docspellRootVerbose' has invalid value: $err"
|
||||||
|
)
|
||||||
|
None
|
||||||
|
}
|
||||||
|
.getOrElse(Level.Error)
|
||||||
|
|
||||||
|
def unsafeConfigure(logger: scribe.Logger, cfg: LogConfig): Unit = {
|
||||||
|
val mods = List[scribe.Logger => scribe.Logger](
|
||||||
|
_.clearHandlers(),
|
||||||
|
_.withMinimumLevel(ScribeWrapper.convertLevel(cfg.minimumLevel)),
|
||||||
|
l =>
|
||||||
|
cfg.format match {
|
||||||
|
case Format.Fancy =>
|
||||||
|
l.withHandler(formatter = Formatter.enhanced, writer = StdoutWriter)
|
||||||
|
case Format.Plain =>
|
||||||
|
l.withHandler(formatter = Formatter.classic, writer = StdoutWriter)
|
||||||
|
case Format.Json =>
|
||||||
|
l.withHandler(writer = JsonWriter(StdoutWriter))
|
||||||
|
case Format.Logfmt =>
|
||||||
|
l.withHandler(writer = LogfmtWriter(StdoutWriter))
|
||||||
|
},
|
||||||
|
_.replace()
|
||||||
|
)
|
||||||
|
|
||||||
|
mods.foldLeft(logger)((l, mod) => mod(l))
|
||||||
|
()
|
||||||
|
}
|
||||||
|
|
||||||
|
private def replaceJUL(): Unit = {
|
||||||
|
scribe.Logger.system // just to load effects in Logger singleton
|
||||||
|
val julRoot = java.util.logging.LogManager.getLogManager.getLogger("")
|
||||||
|
julRoot.getHandlers.foreach(julRoot.removeHandler)
|
||||||
|
julRoot.addHandler(JULHandler)
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,52 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2020 Eike K. & Contributors
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
*/
|
||||||
|
|
||||||
|
package docspell.logging.impl
|
||||||
|
|
||||||
|
import cats.Id
|
||||||
|
import cats.effect.Sync
|
||||||
|
|
||||||
|
import docspell.logging.{Level, LogEvent, Logger}
|
||||||
|
|
||||||
|
import scribe.LoggerSupport
|
||||||
|
import scribe.message.{LoggableMessage, Message}
|
||||||
|
|
||||||
|
private[logging] object ScribeWrapper {
|
||||||
|
final class ImplUnsafe(log: scribe.Logger) extends Logger[Id] {
|
||||||
|
override def asUnsafe = this
|
||||||
|
|
||||||
|
override def log(ev: LogEvent): Unit =
|
||||||
|
log.log(convert(ev))
|
||||||
|
}
|
||||||
|
final class Impl[F[_]: Sync](log: scribe.Logger) extends Logger[F] {
|
||||||
|
override def asUnsafe = new ImplUnsafe(log)
|
||||||
|
|
||||||
|
override def log(ev: LogEvent) =
|
||||||
|
Sync[F].delay(log.log(convert(ev)))
|
||||||
|
}
|
||||||
|
|
||||||
|
private[impl] def convertLevel(l: Level): scribe.Level =
|
||||||
|
l match {
|
||||||
|
case Level.Fatal => scribe.Level.Fatal
|
||||||
|
case Level.Error => scribe.Level.Error
|
||||||
|
case Level.Warn => scribe.Level.Warn
|
||||||
|
case Level.Info => scribe.Level.Info
|
||||||
|
case Level.Debug => scribe.Level.Debug
|
||||||
|
case Level.Trace => scribe.Level.Trace
|
||||||
|
}
|
||||||
|
|
||||||
|
private[this] def convert(ev: LogEvent) = {
|
||||||
|
val level = convertLevel(ev.level)
|
||||||
|
val additional: List[LoggableMessage] = ev.additional.map { x =>
|
||||||
|
x() match {
|
||||||
|
case Right(ex) => Message.static(ex)
|
||||||
|
case Left(msg) => Message.static(msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
LoggerSupport(level, ev.msg(), additional, ev.pkg, ev.fileName, ev.name, ev.line)
|
||||||
|
.copy(data = ev.data)
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,56 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2020 Eike K. & Contributors
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
*/
|
||||||
|
|
||||||
|
package docspell.logging.impl
|
||||||
|
|
||||||
|
import scribe._
|
||||||
|
import scribe.output.LogOutput
|
||||||
|
import scribe.output.format.OutputFormat
|
||||||
|
import scribe.writer.Writer
|
||||||
|
|
||||||
|
// From: https://github.com/outr/scribe/blob/8e99521e1ee1f0c421629764dd96e4eb193d84bd/core/shared/src/main/scala/scribe/writer/SystemOutputWriter.scala
|
||||||
|
// Modified to always log to stdout. The original code was logging to stdout and stderr
|
||||||
|
// depending on the log level.
|
||||||
|
// Original code licensed under MIT
|
||||||
|
|
||||||
|
private[impl] object StdoutWriter extends Writer {
|
||||||
|
|
||||||
|
/** If true, will always synchronize writing to the console to avoid interleaved text.
|
||||||
|
* Most native consoles will handle this automatically, but IntelliJ and Eclipse are
|
||||||
|
* notorious about not properly handling this. Defaults to true.
|
||||||
|
*/
|
||||||
|
val synchronizeWriting: Boolean = true
|
||||||
|
|
||||||
|
/** Workaround for some consoles that don't play nicely with asynchronous calls */
|
||||||
|
val alwaysFlush: Boolean = false
|
||||||
|
|
||||||
|
private val stringBuilders = new ThreadLocal[StringBuilder] {
|
||||||
|
override def initialValue(): StringBuilder = new StringBuilder(512)
|
||||||
|
}
|
||||||
|
|
||||||
|
@annotation.nowarn
|
||||||
|
override def write[M](
|
||||||
|
record: LogRecord[M],
|
||||||
|
output: LogOutput,
|
||||||
|
outputFormat: OutputFormat
|
||||||
|
): Unit = {
|
||||||
|
val stream = Logger.system.out
|
||||||
|
val sb = stringBuilders.get()
|
||||||
|
outputFormat.begin(sb.append(_))
|
||||||
|
outputFormat(output, s => sb.append(s))
|
||||||
|
outputFormat.end(sb.append(_))
|
||||||
|
if (synchronizeWriting) {
|
||||||
|
synchronized {
|
||||||
|
stream.println(sb.toString())
|
||||||
|
if (alwaysFlush) stream.flush()
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
stream.println(sb.toString())
|
||||||
|
if (alwaysFlush) stream.flush()
|
||||||
|
}
|
||||||
|
sb.clear()
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,33 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2020 Eike K. & Contributors
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
*/
|
||||||
|
|
||||||
|
package docspell
|
||||||
|
|
||||||
|
import cats.Id
|
||||||
|
import cats.effect._
|
||||||
|
|
||||||
|
import docspell.logging.impl.ScribeWrapper
|
||||||
|
|
||||||
|
import sourcecode.Enclosing
|
||||||
|
|
||||||
|
package object logging {
|
||||||
|
|
||||||
|
def unsafeLogger(name: String): Logger[Id] =
|
||||||
|
new ScribeWrapper.ImplUnsafe(scribe.Logger(name))
|
||||||
|
|
||||||
|
def unsafeLogger(implicit e: Enclosing): Logger[Id] =
|
||||||
|
unsafeLogger(e.value)
|
||||||
|
|
||||||
|
def getLogger[F[_]: Sync](implicit e: Enclosing): Logger[F] =
|
||||||
|
getLogger(e.value)
|
||||||
|
|
||||||
|
def getLogger[F[_]: Sync](name: String): Logger[F] =
|
||||||
|
new ScribeWrapper.Impl[F](scribe.Logger(name))
|
||||||
|
|
||||||
|
def getLogger[F[_]: Sync](clazz: Class[_]): Logger[F] =
|
||||||
|
new ScribeWrapper.Impl[F](scribe.Logger(clazz.getName))
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,26 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2020 Eike K. & Contributors
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
*/
|
||||||
|
|
||||||
|
package docspell.logging
|
||||||
|
|
||||||
|
import docspell.logging.impl.ScribeConfigure
|
||||||
|
|
||||||
|
import munit.Suite
|
||||||
|
|
||||||
|
trait TestLoggingConfig extends Suite {
|
||||||
|
def docspellLogConfig: LogConfig = LogConfig(Level.Warn, LogConfig.Format.Fancy)
|
||||||
|
def rootMinimumLevel: Level = Level.Error
|
||||||
|
|
||||||
|
override def beforeAll(): Unit = {
|
||||||
|
super.beforeAll()
|
||||||
|
val docspellLogger = scribe.Logger("docspell")
|
||||||
|
ScribeConfigure.unsafeConfigure(docspellLogger, docspellLogConfig)
|
||||||
|
val rootCfg = docspellLogConfig.copy(minimumLevel = rootMinimumLevel)
|
||||||
|
ScribeConfigure.unsafeConfigure(scribe.Logger.root, rootCfg)
|
||||||
|
()
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -13,7 +13,7 @@ import cats.effect.std.Queue
|
|||||||
import cats.implicits._
|
import cats.implicits._
|
||||||
import fs2.Stream
|
import fs2.Stream
|
||||||
|
|
||||||
import docspell.common.Logger
|
import docspell.logging.Logger
|
||||||
|
|
||||||
/** Combines a sink and reader to a place where events can be submitted and processed in a
|
/** Combines a sink and reader to a place where events can be submitted and processed in a
|
||||||
* producer-consumer manner.
|
* producer-consumer manner.
|
||||||
@ -21,8 +21,6 @@ import docspell.common.Logger
|
|||||||
trait EventExchange[F[_]] extends EventSink[F] with EventReader[F] {}
|
trait EventExchange[F[_]] extends EventSink[F] with EventReader[F] {}
|
||||||
|
|
||||||
object EventExchange {
|
object EventExchange {
|
||||||
private[this] val logger = org.log4s.getLogger
|
|
||||||
|
|
||||||
def silent[F[_]: Applicative]: EventExchange[F] =
|
def silent[F[_]: Applicative]: EventExchange[F] =
|
||||||
new EventExchange[F] {
|
new EventExchange[F] {
|
||||||
def offer(event: Event): F[Unit] =
|
def offer(event: Event): F[Unit] =
|
||||||
@ -36,7 +34,7 @@ object EventExchange {
|
|||||||
Queue.circularBuffer[F, Event](queueSize).map(q => new Impl(q))
|
Queue.circularBuffer[F, Event](queueSize).map(q => new Impl(q))
|
||||||
|
|
||||||
final class Impl[F[_]: Async](queue: Queue[F, Event]) extends EventExchange[F] {
|
final class Impl[F[_]: Async](queue: Queue[F, Event]) extends EventExchange[F] {
|
||||||
private[this] val log = Logger.log4s[F](logger)
|
private[this] val log: Logger[F] = docspell.logging.getLogger[F]
|
||||||
|
|
||||||
def offer(event: Event): F[Unit] =
|
def offer(event: Event): F[Unit] =
|
||||||
log.debug(s"Pushing event to queue: $event") *>
|
log.debug(s"Pushing event to queue: $event") *>
|
||||||
@ -47,7 +45,7 @@ object EventExchange {
|
|||||||
|
|
||||||
def consume(maxConcurrent: Int)(run: Kleisli[F, Event, Unit]): Stream[F, Nothing] = {
|
def consume(maxConcurrent: Int)(run: Kleisli[F, Event, Unit]): Stream[F, Nothing] = {
|
||||||
val stream = Stream.repeatEval(queue.take).evalMap((logEvent >> run).run)
|
val stream = Stream.repeatEval(queue.take).evalMap((logEvent >> run).run)
|
||||||
log.s.info(s"Starting up $maxConcurrent notification event consumers").drain ++
|
log.stream.info(s"Starting up $maxConcurrent notification event consumers").drain ++
|
||||||
Stream(stream).repeat.take(maxConcurrent.toLong).parJoin(maxConcurrent).drain
|
Stream(stream).repeat.take(maxConcurrent.toLong).parJoin(maxConcurrent).drain
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user