Configure postgres fts backend

build.sbt

@@ -319,19 +319,6 @@ val common = project
   )
   .dependsOn(loggingApi)
 
-val config = project
-  .in(file("modules/config"))
-  .disablePlugins(RevolverPlugin)
-  .settings(sharedSettings)
-  .withTestSettings
-  .settings(
-    name := "docspell-config",
-    libraryDependencies ++=
-      Dependencies.fs2 ++
-        Dependencies.pureconfig
-  )
-  .dependsOn(common, loggingApi)
-
 val loggingScribe = project
   .in(file("modules/logging/scribe"))
   .disablePlugins(RevolverPlugin)
@@ -729,6 +716,20 @@ val webapp = project
   )
   .dependsOn(query.js)
 
+// Config project shared among the two applications only
+val config = project
+  .in(file("modules/config"))
+  .disablePlugins(RevolverPlugin)
+  .settings(sharedSettings)
+  .withTestSettings
+  .settings(
+    name := "docspell-config",
+    libraryDependencies ++=
+      Dependencies.fs2 ++
+        Dependencies.pureconfig
+  )
+  .dependsOn(common, loggingApi, ftspsql, store)
+
 // --- Application(s)
 
 val joex = project

modules/common/src/main/scala/docspell/common/Banner.scala

@@ -14,7 +14,7 @@ case class Banner(
     configFile: Option[String],
     appId: Ident,
     baseUrl: LenientUri,
-    ftsUrl: Option[LenientUri],
+    ftsInfo: Option[String],
     fileStoreConfig: FileStoreConfig
 ) {
 
@@ -35,7 +35,7 @@ case class Banner(
       s"Id:       ${appId.id}",
       s"Base-Url: ${baseUrl.asString}",
       s"Database: ${jdbcUrl.asString}",
-      s"Fts:      ${ftsUrl.map(_.asString).getOrElse("-")}",
+      s"Fts:      ${ftsInfo.getOrElse("-")}",
       s"Config:   ${configFile.getOrElse("")}",
       s"FileRepo: ${fileStoreConfig}",
       ""

modules/config/src/main/scala/docspell/config/FtsType.scala

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.config
+
+import cats.data.NonEmptyList
+
+sealed trait FtsType {
+  def name: String
+}
+
+object FtsType {
+  case object Solr extends FtsType { val name = "solr" }
+  case object PostgreSQL extends FtsType { val name = "postgresql" }
+
+  val all: NonEmptyList[FtsType] =
+    NonEmptyList.of(Solr, PostgreSQL)
+
+  def fromName(str: String): Either[String, FtsType] =
+    all.find(_.name.equalsIgnoreCase(str)).toRight(s"Unknown fts type: $str")
+
+  def unsafeFromName(str: String): FtsType =
+    fromName(str).fold(sys.error, identity)
+}

modules/config/src/main/scala/docspell/config/Implicits.scala

@@ -10,9 +10,11 @@ import java.nio.file.{Path => JPath}
 
 import scala.reflect.ClassTag
 
+import cats.syntax.all._
 import fs2.io.file.Path
 
 import docspell.common._
+import docspell.ftspsql.{PgQueryParser, RankNormalization}
 import docspell.logging.{Level, LogConfig}
 
 import com.github.eikek.calev.CalEvent
@@ -85,11 +87,28 @@ object Implicits {
   implicit val fileStoreTypeReader: ConfigReader[FileStoreType] =
     ConfigReader[String].emap(reason(FileStoreType.fromString))
 
-  def reason[A: ClassTag](
-      f: String => Either[String, A]
-  ): String => Either[FailureReason, A] =
+  implicit val pgQueryParserReader: ConfigReader[PgQueryParser] =
+    ConfigReader[String].emap(reason(PgQueryParser.fromName))
+
+  implicit val pgRankNormalizationReader: ConfigReader[RankNormalization] =
+    ConfigReader[List[Int]].emap(
+      reason(ints => ints.traverse(RankNormalization.byNumber).map(_.reduce(_ && _)))
+    )
+
+  implicit val languageReader: ConfigReader[Language] =
+    ConfigReader[String].emap(reason(Language.fromString))
+
+  implicit def languageMapReader[B: ConfigReader]: ConfigReader[Map[Language, B]] =
+    pureconfig.configurable.genericMapReader[Language, B](reason(Language.fromString))
+
+  implicit val ftsTypeReader: ConfigReader[FtsType] =
+    ConfigReader[String].emap(reason(FtsType.fromName))
+
+  def reason[T, A: ClassTag](
+      f: T => Either[String, A]
+  ): T => Either[FailureReason, A] =
     in =>
       f(in).left.map(str =>
-        CannotConvert(in, implicitly[ClassTag[A]].runtimeClass.toString, str)
+        CannotConvert(in.toString, implicitly[ClassTag[A]].runtimeClass.toString, str)
       )
 }

modules/config/src/main/scala/docspell/config/PgFtsConfig.scala

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.config
+
+import docspell.common._
+import docspell.ftspsql._
+import docspell.store.JdbcConfig
+
+case class PgFtsConfig(
+    useDefaultConnection: Boolean,
+    jdbc: JdbcConfig,
+    pgQueryParser: PgQueryParser,
+    pgRankNormalization: RankNormalization,
+    pgConfig: Map[Language, String]
+) {
+
+  def toPsqlConfig(stdConn: JdbcConfig): PsqlConfig = {
+    val db =
+      if (useDefaultConnection) stdConn
+      else jdbc
+
+    PsqlConfig(
+      db.url,
+      db.user,
+      Password(db.password),
+      pgConfig,
+      pgQueryParser,
+      pgRankNormalization
+    )
+  }
+}
+
+object PgFtsConfig {}

modules/joex/src/main/resources/reference.conf

@@ -697,6 +697,9 @@ Docpell Update Check
     # Currently the SOLR search platform is supported.
     enabled = false
 
+    # Which backend to use, either solr or postgresql
+    backend = "solr"
+
     # Configuration for the SOLR backend.
     solr = {
       # The URL to solr
@@ -713,6 +716,43 @@ Docpell Update Check
       q-op = "OR"
     }
 
+    # Configuration for PostgreSQL backend
+    postgresql = {
+      # Whether to use the default database, only works if it is
+      # postgresql
+      use-default-connection = false
+
+      # The database connection.
+      jdbc {
+        url = "jdbc:postgresql://server:5432/db"
+        user = "pguser"
+        password = ""
+      }
+
+      # A mapping from a language to a postgres text search config. By
+      # default a language is mapped to a predefined config.
+      # PostgreSQL has predefined configs for some languages. This
+      # setting allows to create a custom text search config and
+      # define it here for some or all languages.
+      #
+      # Example:
+      #  { german = "my-german" }
+      #
+      # See https://www.postgresql.org/docs/14/textsearch-tables.html ff.
+      pg-config = {
+      }
+
+      # Define which query parser to use.
+      #
+      # https://www.postgresql.org/docs/14/textsearch-controls.html#TEXTSEARCH-PARSING-QUERIES
+      pg-query-parser = "websearch_to_tsquery"
+
+      # Allows to define a normalization for the ranking.
+      #
+      # https://www.postgresql.org/docs/14/textsearch-controls.html#TEXTSEARCH-RANKING
+      pg-rank-normalization = [ 4 ]
+    }
+
     # Settings for running the index migration tasks
     migration = {
       # Chunk size to use when indexing data from the database. This

modules/joex/src/main/scala/docspell/joex/Config.scala

@@ -13,6 +13,7 @@ import docspell.analysis.TextAnalysisConfig
 import docspell.analysis.classifier.TextClassifierConfig
 import docspell.backend.Config.Files
 import docspell.common._
+import docspell.config.{FtsType, PgFtsConfig}
 import docspell.convert.ConvertConfig
 import docspell.extract.ExtractConfig
 import docspell.ftssolr.SolrConfig
@@ -65,9 +66,25 @@ object Config {
 
   case class FullTextSearch(
       enabled: Boolean,
+      backend: FtsType,
       migration: FullTextSearch.Migration,
-      solr: SolrConfig
-  )
+      solr: SolrConfig,
+      postgresql: PgFtsConfig
+  ) {
+
+    def info: String =
+      if (!enabled) "Disabled."
+      else
+        backend match {
+          case FtsType.Solr =>
+            s"Solr(${solr.url.asString})"
+          case FtsType.PostgreSQL =>
+            if (postgresql.useDefaultConnection)
+              "PostgreSQL(default)"
+            else
+              s"PostgreSQL(${postgresql.jdbc.url.asString})"
+        }
+  }
 
   object FullTextSearch {
 

modules/joex/src/main/scala/docspell/joex/ConfigFile.scala

@@ -9,7 +9,7 @@ package docspell.joex
 import cats.effect.Async
 
 import docspell.config.Implicits._
-import docspell.config.{ConfigFactory, Validation}
+import docspell.config.{ConfigFactory, FtsType, Validation}
 import docspell.scheduler.CountingScheme
 
 import emil.MailAddress
@@ -53,6 +53,14 @@ object ConfigFile {
         cfg => cfg.updateCheck.enabled && cfg.updateCheck.subject.els.isEmpty,
         "No subject given for enabled update check!"
       ),
-      Validation(cfg => cfg.files.validate.map(_ => cfg))
+      Validation(cfg => cfg.files.validate.map(_ => cfg)),
+      Validation.failWhen(
+        cfg =>
+          cfg.fullTextSearch.enabled &&
+            cfg.fullTextSearch.backend == FtsType.PostgreSQL &&
+            cfg.fullTextSearch.postgresql.useDefaultConnection &&
+            !cfg.jdbc.dbmsName.contains("postgresql"),
+        s"PostgreSQL defined fulltext search backend with default-connection, which is not a PostgreSQL connection!"
+      )
     )
 }

modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala

@@ -102,7 +102,8 @@ object JoexAppImpl extends MailAddressCodec {
       termSignal: SignallingRef[F, Boolean],
       store: Store[F],
       httpClient: Client[F],
-      pubSub: PubSub[F]
+      pubSub: PubSub[F],
+      pools: Pools
   ): Resource[F, JoexApp[F]] =
     for {
       joexLogger <- Resource.pure(docspell.logging.getLogger[F](s"joex-${cfg.appId.id}"))
@@ -120,6 +121,7 @@ object JoexAppImpl extends MailAddressCodec {
 
       tasks <- JoexTasks.resource(
         cfg,
+        pools,
         jobStoreModule,
         httpClient,
         pubSubT,

modules/joex/src/main/scala/docspell/joex/JoexServer.scala

@@ -52,7 +52,7 @@ object JoexServer {
         httpClient
       )(Topics.all.map(_.topic))
 
-      joexApp <- JoexAppImpl.create[F](cfg, signal, store, httpClient, pubSub)
+      joexApp <- JoexAppImpl.create[F](cfg, signal, store, httpClient, pubSub, pools)
 
       httpApp = Router(
         "/internal" -> InternalHeader(settings.internalRouteKey) {

modules/joex/src/main/scala/docspell/joex/JoexTasks.scala

@@ -12,8 +12,10 @@ import docspell.analysis.TextAnalyser
 import docspell.backend.fulltext.CreateIndex
 import docspell.backend.ops._
 import docspell.common._
+import docspell.config.FtsType
 import docspell.ftsclient.FtsClient
-import docspell.ftspsql.{PsqlConfig, PsqlFtsClient}
+import docspell.ftspsql.PsqlFtsClient
+import docspell.ftssolr.SolrFtsClient
 import docspell.joex.analysis.RegexNerFile
 import docspell.joex.emptytrash.EmptyTrashTask
 import docspell.joex.filecopy.{FileCopyTask, FileIntegrityCheckTask}
@@ -211,6 +213,7 @@ object JoexTasks {
 
   def resource[F[_]: Async](
       cfg: Config,
+      pools: Pools,
       jobStoreModule: JobStoreModuleBuilder.Module[F],
       httpClient: Client[F],
       pubSub: PubSubT[F],
@@ -221,7 +224,7 @@ object JoexTasks {
       joex <- OJoex(pubSub)
       store = jobStoreModule.store
       upload <- OUpload(store, jobStoreModule.jobs)
-      fts <- createFtsClient(cfg, store)
+      fts <- createFtsClient(cfg, pools, store, httpClient)
       createIndex <- CreateIndex.resource(fts, store)
       itemOps <- OItem(store, fts, createIndex, jobStoreModule.jobs)
       itemSearchOps <- OItemSearch(store)
@@ -250,16 +253,23 @@ object JoexTasks {
 
   private def createFtsClient[F[_]: Async](
       cfg: Config,
-      store: Store[F] /*,
-      client: Client[F] */
+      pools: Pools,
+      store: Store[F],
+      client: Client[F]
   ): Resource[F, FtsClient[F]] =
-    // if (cfg.fullTextSearch.enabled) SolrFtsClient(cfg.fullTextSearch.solr, client)
     if (cfg.fullTextSearch.enabled)
-      Resource.pure[F, FtsClient[F]](
-        new PsqlFtsClient[F](
-          PsqlConfig.defaults(cfg.jdbc.url, cfg.jdbc.user, Password(cfg.jdbc.password)),
-          store.transactor
-        )
-      )
+      cfg.fullTextSearch.backend match {
+        case FtsType.Solr =>
+          SolrFtsClient(cfg.fullTextSearch.solr, client)
+
+        case FtsType.PostgreSQL =>
+          val psqlCfg = cfg.fullTextSearch.postgresql.toPsqlConfig(cfg.jdbc)
+          if (cfg.fullTextSearch.postgresql.useDefaultConnection)
+            Resource.pure[F, FtsClient[F]](
+              new PsqlFtsClient[F](psqlCfg, store.transactor)
+            )
+          else
+            PsqlFtsClient(psqlCfg, pools.connectEC)
+      }
     else Resource.pure[F, FtsClient[F]](FtsClient.none[F])
 }

modules/joex/src/main/scala/docspell/joex/Main.scala

@@ -31,7 +31,7 @@ object Main extends IOApp {
         Option(System.getProperty("config.file")),
         cfg.appId,
         cfg.baseUrl,
-        Some(cfg.fullTextSearch.solr.url).filter(_ => cfg.fullTextSearch.enabled),
+        Some(cfg.fullTextSearch.info).filter(_ => cfg.fullTextSearch.enabled),
         cfg.files.defaultStoreConfig
       )
       _ <- logger.info(s"\n${banner.render("***>")}")

modules/restserver/src/main/resources/reference.conf

@@ -289,6 +289,9 @@ docspell.server {
     # Currently the SOLR search platform is supported.
     enabled = false
 
+    # Which backend to use, either solr or postgresql
+    backend = "solr"
+
     # Configuration for the SOLR backend.
     solr = {
       # The URL to solr
@@ -304,6 +307,43 @@ docspell.server {
       # The default combiner for tokens. One of {AND, OR}.
       q-op = "OR"
     }
+
+    # Configuration for PostgreSQL backend
+    postgresql = {
+      # Whether to use the default database, only works if it is
+      # postgresql
+      use-default-connection = false
+
+      # The database connection.
+      jdbc {
+        url = "jdbc:postgresql://server:5432/db"
+        user = "pguser"
+        password = ""
+      }
+
+      # A mapping from a language to a postgres text search config. By
+      # default a language is mapped to a predefined config.
+      # PostgreSQL has predefined configs for some languages. This
+      # setting allows to create a custom text search config and
+      # define it here for some or all languages.
+      #
+      # Example:
+      #  { german = "my-german" }
+      #
+      # See https://www.postgresql.org/docs/14/textsearch-tables.html ff.
+      pg-config = {
+      }
+
+      # Define which query parser to use.
+      #
+      # https://www.postgresql.org/docs/14/textsearch-controls.html#TEXTSEARCH-PARSING-QUERIES
+      pg-query-parser = "websearch_to_tsquery"
+
+      # Allows to define a normalization for the ranking.
+      #
+      # https://www.postgresql.org/docs/14/textsearch-controls.html#TEXTSEARCH-RANKING
+      pg-rank-normalization = [ 4 ]
+    }
   }
 
   # Configuration for the backend.

modules/restserver/src/main/scala/docspell/restserver/Config.scala

@@ -9,6 +9,7 @@ package docspell.restserver
 import docspell.backend.auth.Login
 import docspell.backend.{Config => BackendConfig}
 import docspell.common._
+import docspell.config.{FtsType, PgFtsConfig}
 import docspell.ftssolr.SolrConfig
 import docspell.logging.LogConfig
 import docspell.oidc.ProviderConfig
@@ -92,7 +93,26 @@ object Config {
     }
   }
 
-  case class FullTextSearch(enabled: Boolean, solr: SolrConfig)
+  case class FullTextSearch(
+      enabled: Boolean,
+      backend: FtsType,
+      solr: SolrConfig,
+      postgresql: PgFtsConfig
+  ) {
+
+    def info: String =
+      if (!enabled) "Disabled."
+      else
+        backend match {
+          case FtsType.Solr =>
+            s"Solr(${solr.url.asString})"
+          case FtsType.PostgreSQL =>
+            if (postgresql.useDefaultConnection)
+              "PostgreSQL(default)"
+            else
+              s"PostgreSQL(${postgresql.jdbc.url.asString})"
+        }
+  }
 
   object FullTextSearch {}
 

modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala

@@ -13,7 +13,7 @@ import cats.effect.Async
 
 import docspell.backend.signup.{Config => SignupConfig}
 import docspell.config.Implicits._
-import docspell.config.{ConfigFactory, Validation}
+import docspell.config.{ConfigFactory, FtsType, Validation}
 import docspell.oidc.{ProviderConfig, SignatureAlgo}
 import docspell.restserver.auth.OpenId
 
@@ -106,4 +106,15 @@ object ConfigFile {
 
   def filesValidate: Validation[Config] =
     Validation(cfg => cfg.backend.files.validate.map(_ => cfg))
+
+  def postgresFtsValidate: Validation[Config] =
+    Validation.failWhen(
+      cfg =>
+        cfg.fullTextSearch.enabled &&
+          cfg.fullTextSearch.backend == FtsType.PostgreSQL &&
+          cfg.fullTextSearch.postgresql.useDefaultConnection &&
+          !cfg.backend.jdbc.dbmsName.contains("postgresql"),
+      s"PostgreSQL defined fulltext search backend with default-connection, which is not a PostgreSQL connection!"
+    )
+
 }

modules/restserver/src/main/scala/docspell/restserver/Main.scala

@@ -28,7 +28,7 @@ object Main extends IOApp {
       Option(System.getProperty("config.file")),
       cfg.appId,
       cfg.baseUrl,
-      Some(cfg.fullTextSearch.solr.url).filter(_ => cfg.fullTextSearch.enabled),
+      Some(cfg.fullTextSearch.info).filter(_ => cfg.fullTextSearch.enabled),
       cfg.backend.files.defaultStoreConfig
     )
     _ <- logger.info(s"\n${banner.render("***>")}")

modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala

@@ -12,9 +12,11 @@ import fs2.concurrent.Topic
 
 import docspell.backend.BackendApp
 import docspell.backend.auth.{AuthToken, ShareToken}
-import docspell.common.Password
+import docspell.common.Pools
+import docspell.config.FtsType
 import docspell.ftsclient.FtsClient
-import docspell.ftspsql.{PsqlConfig, PsqlFtsClient}
+import docspell.ftspsql.PsqlFtsClient
+import docspell.ftssolr.SolrFtsClient
 import docspell.notification.api.NotificationModule
 import docspell.notification.impl.NotificationModuleImpl
 import docspell.oidc.CodeFlowRoutes
@@ -156,6 +158,7 @@ object RestAppImpl {
 
   def create[F[_]: Async](
       cfg: Config,
+      pools: Pools,
       store: Store[F],
       httpClient: Client[F],
       pubSub: PubSub[F],
@@ -164,7 +167,7 @@ object RestAppImpl {
     val logger = docspell.logging.getLogger[F](s"restserver-${cfg.appId.id}")
 
     for {
-      ftsClient <- createFtsClient(cfg, store)
+      ftsClient <- createFtsClient(cfg, pools, store, httpClient)
       pubSubT = PubSubT(pubSub, logger)
       javaEmil = JavaMailEmil(cfg.backend.mailSettings)
       notificationMod <- Resource.eval(
@@ -190,20 +193,24 @@ object RestAppImpl {
 
   private def createFtsClient[F[_]: Async](
       cfg: Config,
-      store: Store[F] /*, client: Client[F] */
+      pools: Pools,
+      store: Store[F],
+      client: Client[F]
   ): Resource[F, FtsClient[F]] =
-    // if (cfg.fullTextSearch.enabled) SolrFtsClient(cfg.fullTextSearch.solr, client)
     if (cfg.fullTextSearch.enabled)
-      Resource.pure[F, FtsClient[F]](
-        new PsqlFtsClient[F](
-          PsqlConfig.defaults(
-            cfg.backend.jdbc.url,
-            cfg.backend.jdbc.user,
-            Password(cfg.backend.jdbc.password)
-          ),
-          store.transactor
-        )
-      )
+      cfg.fullTextSearch.backend match {
+        case FtsType.Solr =>
+          SolrFtsClient(cfg.fullTextSearch.solr, client)
+
+        case FtsType.PostgreSQL =>
+          val psqlCfg = cfg.fullTextSearch.postgresql.toPsqlConfig(cfg.backend.jdbc)
+          if (cfg.fullTextSearch.postgresql.useDefaultConnection)
+            Resource.pure[F, FtsClient[F]](
+              new PsqlFtsClient[F](psqlCfg, store.transactor)
+            )
+          else
+            PsqlFtsClient(psqlCfg, pools.connectEC)
+      }
     else Resource.pure[F, FtsClient[F]](FtsClient.none[F])
 
 }

modules/restserver/src/main/scala/docspell/restserver/RestServer.scala

@@ -88,7 +88,7 @@ object RestServer {
         store,
         httpClient
       )(Topics.all.map(_.topic))
-      restApp <- RestAppImpl.create[F](cfg, store, httpClient, pubSub, wsTopic)
+      restApp <- RestAppImpl.create[F](cfg, pools, store, httpClient, pubSub, wsTopic)
     } yield (restApp, pubSub, setting)
 
   def createHttpApp[F[_]: Async](