From 330fdcdd5bb1acb062501bab0f84b8dbf6253164 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sun, 21 Jun 2020 20:13:33 +0200 Subject: [PATCH] Add rest endpoints to re-create the index --- build.sbt | 6 +- .../scala/docspell/backend/BackendApp.scala | 20 ++++--- .../main/scala/docspell/backend/Config.scala | 5 +- .../docspell/backend/ops/OFulltext.scala | 7 ++- .../src/main/resources/reference.conf | 31 ++++++---- .../scala/docspell/restserver/Config.scala | 9 +-- .../docspell/restserver/RestAppImpl.scala | 11 +++- .../docspell/restserver/RestServer.scala | 6 +- .../routes/FullTextIndexRoutes.scala | 60 +++++++++++++++++++ 9 files changed, 119 insertions(+), 36 deletions(-) create mode 100644 modules/restserver/src/main/scala/docspell/restserver/routes/FullTextIndexRoutes.scala diff --git a/build.sbt b/build.sbt index 56602502..051a767b 100644 --- a/build.sbt +++ b/build.sbt @@ -325,7 +325,7 @@ val backend = project.in(file("modules/backend")). Dependencies.bcrypt ++ Dependencies.http4sClient ++ Dependencies.emil - ).dependsOn(store, joexapi, ftsclient, ftssolr) + ).dependsOn(store, joexapi, ftsclient) val webapp = project.in(file("modules/webapp")). disablePlugins(RevolverPlugin). @@ -374,7 +374,7 @@ val joex = project.in(file("modules/joex")). addCompilerPlugin(Dependencies.betterMonadicFor), buildInfoPackage := "docspell.joex", reStart/javaOptions ++= Seq(s"-Dconfig.file=${(LocalRootProject/baseDirectory).value/"local"/"dev.conf"}") - ).dependsOn(store, backend, extract, convert, analysis, joexapi, restapi) + ).dependsOn(store, backend, extract, convert, analysis, joexapi, restapi, ftssolr) val restserver = project.in(file("modules/restserver")). enablePlugins(BuildInfoPlugin @@ -412,7 +412,7 @@ val restserver = project.in(file("modules/restserver")). }.taskValue, Compile/unmanagedResourceDirectories ++= Seq((Compile/resourceDirectory).value.getParentFile/"templates"), reStart/javaOptions ++= Seq(s"-Dconfig.file=${(LocalRootProject/baseDirectory).value/"local"/"dev.conf"}") - ).dependsOn(restapi, joexapi, backend, webapp) + ).dependsOn(restapi, joexapi, backend, webapp, ftssolr) diff --git a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala index 0cb31808..acb4e08d 100644 --- a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala +++ b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala @@ -1,6 +1,7 @@ package docspell.backend import cats.effect.{Blocker, ConcurrentEffect, ContextShift, Resource} +import org.http4s.client.Client import org.http4s.client.blaze.BlazeClientBuilder import docspell.backend.auth.Login @@ -10,7 +11,7 @@ import docspell.joexapi.client.JoexClient import docspell.store.Store import docspell.store.queue.JobQueue import docspell.store.usertask.UserTaskStore -import docspell.ftssolr.SolrFtsClient +import docspell.ftsclient.FtsClient import scala.concurrent.ExecutionContext import emil.javamail.{JavaMailEmil, Settings} @@ -40,12 +41,11 @@ object BackendApp { def create[F[_]: ConcurrentEffect: ContextShift]( cfg: Config, store: Store[F], - httpClientEc: ExecutionContext, + httpClient: Client[F], + ftsClient: FtsClient[F], blocker: Blocker ): Resource[F, BackendApp[F]] = for { - httpClient <- BlazeClientBuilder[F](httpClientEc).resource - solrFts <- SolrFtsClient(cfg.fullTextSearch.solr, httpClient) utStore <- UserTaskStore(store) queue <- JobQueue(store) loginImpl <- Login[F](store) @@ -59,9 +59,9 @@ object BackendApp { uploadImpl <- OUpload(store, queue, cfg.files, joexImpl) nodeImpl <- ONode(store) jobImpl <- OJob(store, joexImpl) - itemImpl <- OItem(store, solrFts) + itemImpl <- OItem(store, ftsClient) itemSearchImpl <- OItemSearch(store) - fulltextImpl <- OFulltext(itemSearchImpl, solrFts, store, queue) + fulltextImpl <- OFulltext(itemSearchImpl, ftsClient, store, queue, joexImpl) javaEmil = JavaMailEmil(blocker, Settings.defaultSettings.copy(debug = cfg.mailDebug)) mailImpl <- OMail(store, javaEmil) @@ -90,9 +90,11 @@ object BackendApp { connectEC: ExecutionContext, httpClientEc: ExecutionContext, blocker: Blocker - ): Resource[F, BackendApp[F]] = + )(ftsFactory: Client[F] => Resource[F, FtsClient[F]]): Resource[F, BackendApp[F]] = for { - store <- Store.create(cfg.jdbc, connectEC, blocker) - backend <- create(cfg, store, httpClientEc, blocker) + store <- Store.create(cfg.jdbc, connectEC, blocker) + httpClient <- BlazeClientBuilder[F](httpClientEc).resource + ftsClient <- ftsFactory(httpClient) + backend <- create(cfg, store, httpClient, ftsClient, blocker) } yield backend } diff --git a/modules/backend/src/main/scala/docspell/backend/Config.scala b/modules/backend/src/main/scala/docspell/backend/Config.scala index b41e39f0..830363a2 100644 --- a/modules/backend/src/main/scala/docspell/backend/Config.scala +++ b/modules/backend/src/main/scala/docspell/backend/Config.scala @@ -3,19 +3,16 @@ package docspell.backend import docspell.backend.signup.{Config => SignupConfig} import docspell.common._ import docspell.store.JdbcConfig -import docspell.ftssolr.SolrConfig case class Config( mailDebug: Boolean, jdbc: JdbcConfig, signup: SignupConfig, - files: Config.Files, - fullTextSearch: Config.FullTextSearch + files: Config.Files ) {} object Config { case class Files(chunkSize: Int, validMimeTypes: Seq[MimeType]) - case class FullTextSearch(enabled: Boolean, solr: SolrConfig) } diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala index df9feb77..beb1ca82 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala @@ -38,13 +38,14 @@ object OFulltext { itemSearch: OItemSearch[F], fts: FtsClient[F], store: Store[F], - queue: JobQueue[F] + queue: JobQueue[F], + joex: OJoex[F] ): Resource[F, OFulltext[F]] = Resource.pure[F, OFulltext[F]](new OFulltext[F] { def reindexAll: F[Unit] = for { job <- JobFactory.reIndexAll[F] - _ <- queue.insertIfNew(job) + _ <- queue.insertIfNew(job) *> joex.notifyAllNodes } yield () def reindexCollective(account: AccountId): F[Unit] = @@ -55,7 +56,7 @@ object OFulltext { job <- JobFactory.reIndex(account) _ <- if (exist.isDefined) ().pure[F] - else queue.insertIfNew(job) + else queue.insertIfNew(job) *> joex.notifyAllNodes } yield () def findItems(q: Query, ftsQ: String, batch: Batch): F[Vector[ListItem]] = diff --git a/modules/restserver/src/main/resources/reference.conf b/modules/restserver/src/main/resources/reference.conf index 3ac7e9e5..b298fba6 100644 --- a/modules/restserver/src/main/resources/reference.conf +++ b/modules/restserver/src/main/resources/reference.conf @@ -84,8 +84,28 @@ docspell.server { } } - fulltext-search { + # Configuration of the full-text search engine. + full-text-search { + # The full-text search feature can be disabled. It requires an + # additional index server available which needs additional + # memory and disk space. It can be enabled later any time. + # + # Currently the SOLR search platform is supported. enabled = true + + # When re-creating the complete index via a REST call, this key + # is required. If left empty (the default), recreating the index + # is disabled. + # + # Example curl command: + # curl -XPOST http://localhost:7880/api/v1/open/fts/reIndexAll/test123 + recreate-key = "" + + # Configuration for the SOLR backend. + solr = { + url = "http://localhost:8983/solr/docspell_core" + commit-within = 1000 + } } # Configuration for the backend. @@ -147,14 +167,5 @@ docspell.server { # By default all files are allowed. valid-mime-types = [ ] } - - # Configuration of the full-text search engine. - full-text-search { - enabled = true - solr = { - url = "http://localhost:8983/solr/docspell_core" - commit-within = 1000 - } - } } } \ No newline at end of file diff --git a/modules/restserver/src/main/scala/docspell/restserver/Config.scala b/modules/restserver/src/main/scala/docspell/restserver/Config.scala index 84eb39e8..165f6822 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/Config.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/Config.scala @@ -1,9 +1,10 @@ package docspell.restserver import java.net.InetAddress +import docspell.common._ import docspell.backend.auth.Login import docspell.backend.{Config => BackendConfig} -import docspell.common._ +import docspell.ftssolr.SolrConfig case class Config( appName: String, @@ -14,7 +15,7 @@ case class Config( auth: Login.Config, integrationEndpoint: Config.IntegrationEndpoint, maxItemPageSize: Int, - fulltextSearch: Config.FulltextSearch + fullTextSearch: Config.FullTextSearch ) object Config { @@ -52,8 +53,8 @@ object Config { } } - case class FulltextSearch(enabled: Boolean) + case class FullTextSearch(enabled: Boolean, recreateKey: Ident, solr: SolrConfig) - object FulltextSearch {} + object FullTextSearch {} } diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala index b2662cd3..25820e1f 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala @@ -2,8 +2,11 @@ package docspell.restserver import cats.implicits._ import cats.effect._ +import org.http4s.client.Client import docspell.backend.BackendApp import docspell.common.NodeType +import docspell.ftsclient.FtsClient +import docspell.ftssolr.SolrFtsClient import scala.concurrent.ExecutionContext @@ -26,9 +29,15 @@ object RestAppImpl { blocker: Blocker ): Resource[F, RestApp[F]] = for { - backend <- BackendApp(cfg.backend, connectEC, httpClientEc, blocker) + backend <- BackendApp(cfg.backend, connectEC, httpClientEc, blocker)( + createFtsClient[F](cfg) + ) app = new RestAppImpl[F](cfg, backend) appR <- Resource.make(app.init.map(_ => app))(_.shutdown) } yield appR + private def createFtsClient[F[_]: ConcurrentEffect: ContextShift]( + cfg: Config + )(client: Client[F]): Resource[F, FtsClient[F]] = + SolrFtsClient(cfg.fullTextSearch.solr, client) } diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala index e0495298..b8f0a700 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala @@ -78,7 +78,8 @@ object RestServer { "email/sent" -> SentMailRoutes(restApp.backend, token), "usertask/notifydueitems" -> NotifyDueItemsRoutes(cfg, restApp.backend, token), "usertask/scanmailbox" -> ScanMailboxRoutes(restApp.backend, token), - "calevent/check" -> CalEventCheckRoutes() + "calevent/check" -> CalEventCheckRoutes(), + "fts" -> FullTextIndexRoutes.secured(cfg, restApp.backend, token) ) def openRoutes[F[_]: Effect](cfg: Config, restApp: RestApp[F]): HttpRoutes[F] = @@ -87,7 +88,8 @@ object RestServer { "signup" -> RegisterRoutes(restApp.backend, cfg), "upload" -> UploadRoutes.open(restApp.backend, cfg), "checkfile" -> CheckFileRoutes.open(restApp.backend), - "integration" -> IntegrationEndpointRoutes.open(restApp.backend, cfg) + "integration" -> IntegrationEndpointRoutes.open(restApp.backend, cfg), + "fts" -> FullTextIndexRoutes.open(cfg, restApp.backend) ) def redirectTo[F[_]: Effect](path: String): HttpRoutes[F] = { diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/FullTextIndexRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/FullTextIndexRoutes.scala new file mode 100644 index 00000000..9a8d2c7a --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/FullTextIndexRoutes.scala @@ -0,0 +1,60 @@ +package docspell.restserver.routes + +import cats.effect._ +import cats.implicits._ +import cats.data.OptionT +import org.http4s._ +//import org.http4s.circe.CirceEntityDecoder._ +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.dsl.Http4sDsl + +import docspell.common._ +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.restserver.Config +import docspell.restserver.conv.Conversions + +object FullTextIndexRoutes { + + def secured[F[_]: Effect]( + cfg: Config, + backend: BackendApp[F], + user: AuthToken + ): HttpRoutes[F] = + if (!cfg.fullTextSearch.enabled) notFound[F] + else { + val dsl = Http4sDsl[F] + import dsl._ + + HttpRoutes.of { + case POST -> Root / "reIndex" => + for { + res <- backend.fulltext.reindexCollective(user.account).attempt + resp <- + Ok(Conversions.basicResult(res, "Full-text index will be re-created.")) + } yield resp + } + } + + def open[F[_]: Effect](cfg: Config, backend: BackendApp[F]): HttpRoutes[F] = + if (!cfg.fullTextSearch.enabled) notFound[F] + else { + val dsl = Http4sDsl[F] + import dsl._ + + HttpRoutes.of { + case POST -> Root / "reIndexAll" / Ident(id) => + for { + res <- + if (id.nonEmpty && id == cfg.fullTextSearch.recreateKey) + backend.fulltext.reindexAll.attempt + else Left(new Exception("The provided key is invalid.")).pure[F] + resp <- + Ok(Conversions.basicResult(res, "Full-text index will be re-created.")) + } yield resp + } + } + + private def notFound[F[_]: Effect]: HttpRoutes[F] = + HttpRoutes(_ => OptionT.pure(Response.notFound[F])) +}