TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-10-31 09:30:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor full-text migrations and add folder to solr schema

Browse Source
This commit is contained in:
Eike Kettner
2020-07-12 00:30:37 +02:00
parent e387b5513f
commit aeba4ba913
9 changed files with 102 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
modules/fts-client/src/main/scala/docspell/ftsclient/FtsClient.scala
View File

@@ -17,11 +17,12 @@ import org.log4s.getLogger
*/
trait FtsClient[F[_]] {
/** Initialization tasks. This is called exactly once and then never
/** Initialization tasks. This is called exactly once at the very
* beginning when initializing the full-text index and then never
* again (except when re-indexing everything). It may be used to
* setup the database.
*/
def initialize: F[Unit]
def initialize: List[FtsMigration[F]]
/** Run a full-text search. */
def search(q: FtsQuery): F[FtsResult]
@@ -107,8 +108,8 @@ object FtsClient {
new FtsClient[F] {
private[this] val logger = Logger.log4s[F](getLogger)
def initialize: F[Unit] =
logger.info("Full-text search is disabled!")
def initialize: List[FtsMigration[F]] =
Nil
def search(q: FtsQuery): F[FtsResult] =
logger.warn("Full-text search is disabled!") *> FtsResult.empty.pure[F]

24
modules/fts-client/src/main/scala/docspell/ftsclient/FtsMigration.scala Normal file
View File

@@ -0,0 +1,24 @@
package docspell.ftsclient
import docspell.common._
final case class FtsMigration[F[_]](
version: Int,
engine: Ident,
description: String,
task: F[FtsMigration.Result]
)
object FtsMigration {
sealed trait Result
object Result {
case object WorkDone extends Result
case object ReIndexAll extends Result
case object IndexAll extends Result
def workDone: Result = WorkDone
def reIndexAll: Result = ReIndexAll
def indexAll: Result = IndexAll
}
}

1
modules/fts-solr/src/main/scala/docspell/ftssolr/Field.scala
View File

@@ -25,6 +25,7 @@ object Field {
val content_en = Field("content_en")
val itemName = Field("itemName")
val itemNotes = Field("itemNotes")
val folderId = Field("folder")
def contentField(lang: Language): Field =
lang match {

2
modules/fts-solr/src/main/scala/docspell/ftssolr/SolrFtsClient.scala
View File

@@ -17,7 +17,7 @@ final class SolrFtsClient[F[_]: Effect](
solrQuery: SolrQuery[F]
) extends FtsClient[F] {
def initialize: F[Unit] =
def initialize: List[FtsMigration[F]] =
solrSetup.setupSchema
def search(q: FtsQuery): F[FtsResult] =

32
modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala
View File

@@ -4,6 +4,7 @@ import cats.effect._
import cats.implicits._
import docspell.common._
import docspell.ftsclient.FtsMigration
import _root_.io.circe._
import _root_.io.circe.generic.semiauto._
@@ -15,21 +16,48 @@ import org.http4s.client.dsl.Http4sClientDsl
trait SolrSetup[F[_]] {
def setupSchema: F[Unit]
def setupSchema: List[FtsMigration[F]]
}
object SolrSetup {
private val solrEngine = Ident.unsafe("solr")
def apply[F[_]: ConcurrentEffect](cfg: SolrConfig, client: Client[F]): SolrSetup[F] = {
val dsl = new Http4sClientDsl[F] {}
import dsl._
new SolrSetup[F] {
val url = (Uri.unsafeFromString(cfg.url.asString) / "schema")
.withQueryParam("commitWithin", cfg.commitWithin.toString)
def setupSchema: F[Unit] = {
def setupSchema: List[FtsMigration[F]] =
List(
FtsMigration[F](
1,
solrEngine,
"Initialize",
setupCoreSchema.map(_ => FtsMigration.Result.workDone)
),
FtsMigration[F](
3,
solrEngine,
"Add folder field",
addFolderField.map(_ => FtsMigration.Result.workDone)
),
FtsMigration[F](
4,
solrEngine,
"Index all from database",
FtsMigration.Result.indexAll.pure[F]
)
)
def addFolderField: F[Unit] =
addStringField(Field.folderId)
def setupCoreSchema: F[Unit] = {
val cmds0 =
List(
Field.id,

35
modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala
View File

@@ -1,9 +1,8 @@
package docspell.joex.fts
import cats.data.{Kleisli, NonEmptyList}
import cats.effect._
import cats.implicits._
import cats.{ApplicativeError, FlatMap, Semigroup}
import cats.{Applicative, ApplicativeError, FlatMap, Monad, Semigroup}
import docspell.common._
import docspell.ftsclient._
@@ -15,6 +14,19 @@ object FtsWork {
def apply[F[_]](f: FtsContext[F] => F[Unit]): FtsWork[F] =
Kleisli(f)
def allInitializeTasks[F[_]: Monad]: FtsWork[F] =
FtsWork[F](_ => ().pure[F]).tap[FtsContext[F]].flatMap { ctx =>
NonEmptyList.fromList(ctx.fts.initialize.map(fm => from[F](fm.task))) match {
case Some(nel) =>
nel.reduce(semigroup[F])
case None =>
FtsWork[F](_ => ().pure[F])
}
}
def from[F[_]: FlatMap: Applicative](t: F[FtsMigration.Result]): FtsWork[F] =
Kleisli.liftF(t).flatMap(transformResult[F])
def all[F[_]: FlatMap](
m0: FtsWork[F],
mn: FtsWork[F]*
@@ -24,14 +36,25 @@ object FtsWork {
implicit def semigroup[F[_]: FlatMap]: Semigroup[FtsWork[F]] =
Semigroup.instance((mt1, mt2) => mt1.flatMap(_ => mt2))
private def transformResult[F[_]: Applicative: FlatMap](
r: FtsMigration.Result
): FtsWork[F] =
r match {
case FtsMigration.Result.WorkDone =>
Kleisli.pure(())
case FtsMigration.Result.IndexAll =>
insertAll[F](None)
case FtsMigration.Result.ReIndexAll =>
clearIndex[F](None) >> insertAll[F](None)
}
// some tasks
def log[F[_]](f: Logger[F] => F[Unit]): FtsWork[F] =
FtsWork(ctx => f(ctx.logger))
def initialize[F[_]]: FtsWork[F] =
FtsWork(_.fts.initialize)
def clearIndex[F[_]](coll: Option[Ident]): FtsWork[F] =
coll match {
case Some(cid) =>
@@ -40,7 +63,7 @@ object FtsWork {
FtsWork(ctx => ctx.fts.clearAll(ctx.logger))
}
def insertAll[F[_]: Effect](coll: Option[Ident]): FtsWork[F] =
def insertAll[F[_]: FlatMap](coll: Option[Ident]): FtsWork[F] =
FtsWork
.all(
FtsWork(ctx =>

5
modules/joex/src/main/scala/docspell/joex/fts/Migration.scala
View File

@@ -1,6 +1,6 @@
package docspell.joex.fts
import cats.Traverse
import cats.{Applicative, FlatMap, Traverse}
import cats.data.{Kleisli, OptionT}
import cats.effect._
import cats.implicits._
@@ -20,6 +20,9 @@ case class Migration[F[_]](
object Migration {
def from[F[_]: Applicative: FlatMap](fm: FtsMigration[F]): Migration[F] =
Migration(fm.version, fm.engine, fm.description, FtsWork.from(fm.task))
def apply[F[_]: Effect](
cfg: Config.FullTextSearch,
fts: FtsClient[F],

10
modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala
View File

@@ -21,7 +21,7 @@ object MigrationTask {
.flatMap(_ =>
Task(ctx =>
Migration[F](cfg, fts, ctx.store, ctx.logger)
.run(migrationTasks[F])
.run(migrationTasks[F](fts))
)
)
@@ -44,11 +44,7 @@ object MigrationTask {
Some(DocspellSystem.migrationTaskTracker)
)
private val solrEngine = Ident.unsafe("solr")
def migrationTasks[F[_]: Effect]: List[Migration[F]] =
List(
Migration[F](1, solrEngine, "initialize", FtsWork.initialize[F]),
Migration[F](2, solrEngine, "Index all from database", FtsWork.insertAll[F](None))
)
def migrationTasks[F[_]: Effect](fts: FtsClient[F]): List[Migration[F]] =
fts.initialize.map(fm => Migration.from(fm))
}

14
modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala
View File

@@ -21,13 +21,7 @@ object ReIndexTask {
Task
.log[F, Args](_.info(s"Running full-text re-index now"))
.flatMap(_ =>
Task(ctx =>
(clearData[F](ctx.args.collective) ++
FtsWork.log[F](_.info("Inserting data from database")) ++
FtsWork.insertAll[F](
ctx.args.collective
)).forContext(cfg, fts).run(ctx)
)
Task(ctx => clearData[F](ctx.args.collective).forContext(cfg, fts).run(ctx))
)
def onCancel[F[_]: Sync]: Task[F, Args, Unit] =
@@ -41,7 +35,9 @@ object ReIndexTask {
.clearIndex(collective)
.recoverWith(
FtsWork.log[F](_.info("Clearing data failed. Continue re-indexing."))
)
) ++
FtsWork.log[F](_.info("Inserting data from database")) ++
FtsWork.insertAll[F](collective)
case None =>
FtsWork
@@ -50,6 +46,6 @@ object ReIndexTask {
FtsWork.log[F](_.info("Clearing data failed. Continue re-indexing."))
) ++
FtsWork.log[F](_.info("Running index initialize")) ++
FtsWork.initialize[F]
FtsWork.allInitializeTasks[F]
})
}