From b2add008ed22872ec2984eccee64b0329ad13ea2 Mon Sep 17 00:00:00 2001 From: eikek Date: Sun, 20 Mar 2022 13:14:07 +0100 Subject: [PATCH] Pass language when updating fulltext index --- .../backend/fulltext/CreateIndex.scala | 10 ++++-- .../scala/docspell/backend/ops/OItem.scala | 33 +++++++++++++++---- .../scala/docspell/ftsclient/FtsClient.scala | 12 +++++-- .../scala/docspell/ftsclient/TextData.scala | 12 ++++--- .../joex/process/TextExtraction.scala | 5 +-- .../scala/docspell/store/queries/QItem.scala | 8 +++-- .../docspell/store/records/RAttachment.scala | 14 ++++++-- 7 files changed, 72 insertions(+), 22 deletions(-) diff --git a/modules/backend/src/main/scala/docspell/backend/fulltext/CreateIndex.scala b/modules/backend/src/main/scala/docspell/backend/fulltext/CreateIndex.scala index 38c1ea67..bce169e3 100644 --- a/modules/backend/src/main/scala/docspell/backend/fulltext/CreateIndex.scala +++ b/modules/backend/src/main/scala/docspell/backend/fulltext/CreateIndex.scala @@ -8,7 +8,6 @@ package docspell.backend.fulltext import cats.data.NonEmptyList import cats.effect._ - import docspell.common._ import docspell.ftsclient.FtsClient import docspell.ftsclient.TextData @@ -62,7 +61,14 @@ object CreateIndex { val items = store .transact(QItem.allNameAndNotes(collective, itemIds, chunkSize)) .map(nn => - TextData.item(nn.id, nn.collective, nn.folder, Option(nn.name), nn.notes) + TextData.item( + nn.id, + nn.collective, + nn.folder, + Option(nn.name), + nn.notes, + nn.language + ) ) fts.indexData(logger, attachs ++ items) diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala index a6e3b314..391a6593 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala @@ -605,7 +605,14 @@ object OItem { .transact(RItem.updateNotes(item, collective, notes)) ) .flatTap( - onSuccessIgnoreError(fts.updateItemNotes(logger, item, collective, notes)) + onSuccessIgnoreError { + store + .transact(RCollective.findLanguage(collective)) + .map(_.getOrElse(Language.English)) + .flatMap(lang => + fts.updateItemNotes(logger, item, collective, lang, notes) + ) + } ) def setName(item: Ident, name: String, collective: Ident): F[UpdateResult] = @@ -615,7 +622,14 @@ object OItem { .transact(RItem.updateName(item, collective, name)) ) .flatTap( - onSuccessIgnoreError(fts.updateItemName(logger, item, collective, name)) + onSuccessIgnoreError { + store + .transact(RCollective.findLanguage(collective)) + .map(_.getOrElse(Language.English)) + .flatMap(lang => + fts.updateItemName(logger, item, collective, lang, name) + ) + } ) def setNameMultiple( @@ -733,10 +747,17 @@ object OItem { ) .flatTap( onSuccessIgnoreError( - OptionT(store.transact(RAttachment.findItemId(attachId))) - .semiflatMap(itemId => - fts.updateAttachmentName(logger, itemId, attachId, collective, name) - ) + OptionT(store.transact(RAttachment.findItemAndLanguage(attachId))) + .semiflatMap { case (itemId, lang) => + fts.updateAttachmentName( + logger, + itemId, + attachId, + collective, + lang.getOrElse(Language.English), + name + ) + } .fold(())(identity) ) ) diff --git a/modules/fts-client/src/main/scala/docspell/ftsclient/FtsClient.scala b/modules/fts-client/src/main/scala/docspell/ftsclient/FtsClient.scala index 13ee23c3..920ce450 100644 --- a/modules/fts-client/src/main/scala/docspell/ftsclient/FtsClient.scala +++ b/modules/fts-client/src/main/scala/docspell/ftsclient/FtsClient.scala @@ -68,19 +68,24 @@ trait FtsClient[F[_]] { logger: Logger[F], itemId: Ident, collective: Ident, + language: Language, name: String ): F[Unit] = - updateIndex(logger, TextData.item(itemId, collective, None, Some(name), None)) + updateIndex( + logger, + TextData.item(itemId, collective, None, Some(name), None, language) + ) def updateItemNotes( logger: Logger[F], itemId: Ident, collective: Ident, + language: Language, notes: Option[String] ): F[Unit] = updateIndex( logger, - TextData.item(itemId, collective, None, None, Some(notes.getOrElse(""))) + TextData.item(itemId, collective, None, None, Some(notes.getOrElse("")), language) ) def updateAttachmentName( @@ -88,6 +93,7 @@ trait FtsClient[F[_]] { itemId: Ident, attachId: Ident, collective: Ident, + language: Language, name: Option[String] ): F[Unit] = updateIndex( @@ -97,7 +103,7 @@ trait FtsClient[F[_]] { attachId, collective, None, - Language.English, + language, Some(name.getOrElse("")), None ) diff --git a/modules/fts-client/src/main/scala/docspell/ftsclient/TextData.scala b/modules/fts-client/src/main/scala/docspell/ftsclient/TextData.scala index 8d71e17a..90da688a 100644 --- a/modules/fts-client/src/main/scala/docspell/ftsclient/TextData.scala +++ b/modules/fts-client/src/main/scala/docspell/ftsclient/TextData.scala @@ -18,6 +18,8 @@ sealed trait TextData { def folder: Option[Ident] + def language: Language + final def fold[A](f: TextData.Attachment => A, g: TextData.Item => A): A = this match { case a: TextData.Attachment => f(a) @@ -32,7 +34,7 @@ object TextData { attachId: Ident, collective: Ident, folder: Option[Ident], - lang: Language, + language: Language, name: Option[String], text: Option[String] ) extends TextData { @@ -57,7 +59,8 @@ object TextData { collective: Ident, folder: Option[Ident], name: Option[String], - notes: Option[String] + notes: Option[String], + language: Language ) extends TextData { val id = Ident.unsafe("item") / item @@ -69,8 +72,9 @@ object TextData { collective: Ident, folder: Option[Ident], name: Option[String], - notes: Option[String] + notes: Option[String], + lang: Language ): TextData = - Item(item, collective, folder, name, notes) + Item(item, collective, folder, name, notes, lang) } diff --git a/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala b/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala index ee6b8939..52ce8601 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala @@ -47,9 +47,10 @@ object TextExtraction { ctx.args.meta.collective, ctx.args.meta.folderId, item.item.name.some, - None + None, + ctx.args.meta.language ) - _ <- fts.indexData(ctx.logger, (idxItem +: txt.map(_.td)).toSeq: _*) + _ <- fts.indexData(ctx.logger, (idxItem +: txt.map(_.td)): _*) dur <- start extractedTags = txt.flatMap(_.tags).distinct.toList _ <- ctx.logger.info(s"Text extraction finished in ${dur.formatExact}.") diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index 41641538..5e9dacb4 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -576,7 +576,8 @@ object QItem { collective: Ident, folder: Option[Ident], name: String, - notes: Option[String] + notes: Option[String], + language: Language ) def allNameAndNotes( coll: Option[Ident], @@ -584,10 +585,11 @@ object QItem { chunkSize: Int ): Stream[ConnectionIO, NameAndNotes] = { val i = RItem.as("i") + val c = RCollective.as("c") Select( - select(i.id, i.cid, i.folder, i.name, i.notes), - from(i) + select(i.id, i.cid, i.folder, i.name, i.notes, c.language), + from(i).innerJoin(c, c.id === i.cid) ).where( i.state.in(ItemState.validStates) &&? itemIds.map(ids => i.id.in(ids)) &&? diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala index bf99a01b..b1a9dcc9 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala @@ -236,8 +236,18 @@ object RAttachment { n3 <- DML.delete(T, T.id === attachId) } yield n0 + n1 + n2 + n3 - def findItemId(attachId: Ident): ConnectionIO[Option[Ident]] = - Select(T.itemId.s, from(T), T.id === attachId).build.query[Ident].option + def findItemAndLanguage( + attachId: Ident + ): ConnectionIO[Option[(Ident, Option[Language])]] = { + val a = RAttachment.as("a") + val m = RAttachmentMeta.as("m") + + Select( + select(a.itemId, m.language), + from(a).leftJoin(m, m.id === a.id), + a.id === attachId + ).build.query[(Ident, Option[Language])].option + } def findAll( coll: Option[Ident],