TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-04-02 09:05:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Pass language when updating fulltext index

Browse Source
This commit is contained in:
eikek 2022-03-20 13:14:07 +01:00
parent f2a2d15e7f
commit b2add008ed
7 changed files with 72 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
modules/backend/src/main/scala/docspell/backend/fulltext/CreateIndex.scala
View File

@ -8,7 +8,6 @@ package docspell.backend.fulltext
import cats.data.NonEmptyList
import cats.effect._
import docspell.common._
import docspell.ftsclient.FtsClient
import docspell.ftsclient.TextData
@ -62,7 +61,14 @@ object CreateIndex {
val items = store
.transact(QItem.allNameAndNotes(collective, itemIds, chunkSize))
.map(nn =>
TextData.item(nn.id, nn.collective, nn.folder, Option(nn.name), nn.notes)
TextData.item(
nn.id,
nn.collective,
nn.folder,
Option(nn.name),
nn.notes,
nn.language
)
)
fts.indexData(logger, attachs ++ items)

33
modules/backend/src/main/scala/docspell/backend/ops/OItem.scala
View File

@ -605,7 +605,14 @@ object OItem {
.transact(RItem.updateNotes(item, collective, notes))
)
.flatTap(
onSuccessIgnoreError(fts.updateItemNotes(logger, item, collective, notes))
onSuccessIgnoreError {
store
.transact(RCollective.findLanguage(collective))
.map(_.getOrElse(Language.English))
.flatMap(lang =>
fts.updateItemNotes(logger, item, collective, lang, notes)
)
}
)
def setName(item: Ident, name: String, collective: Ident): F[UpdateResult] =
@ -615,7 +622,14 @@ object OItem {
.transact(RItem.updateName(item, collective, name))
)
.flatTap(
onSuccessIgnoreError(fts.updateItemName(logger, item, collective, name))
onSuccessIgnoreError {
store
.transact(RCollective.findLanguage(collective))
.map(_.getOrElse(Language.English))
.flatMap(lang =>
fts.updateItemName(logger, item, collective, lang, name)
)
}
)
def setNameMultiple(
@ -733,10 +747,17 @@ object OItem {
)
.flatTap(
onSuccessIgnoreError(
OptionT(store.transact(RAttachment.findItemId(attachId)))
.semiflatMap(itemId =>
fts.updateAttachmentName(logger, itemId, attachId, collective, name)
)
OptionT(store.transact(RAttachment.findItemAndLanguage(attachId)))
.semiflatMap { case (itemId, lang) =>
fts.updateAttachmentName(
logger,
itemId,
attachId,
collective,
lang.getOrElse(Language.English),
name
)
}
.fold(())(identity)
)
)

12
modules/fts-client/src/main/scala/docspell/ftsclient/FtsClient.scala
View File

@ -68,19 +68,24 @@ trait FtsClient[F[_]] {
logger: Logger[F],
itemId: Ident,
collective: Ident,
language: Language,
name: String
): F[Unit] =
updateIndex(logger, TextData.item(itemId, collective, None, Some(name), None))
updateIndex(
logger,
TextData.item(itemId, collective, None, Some(name), None, language)
)
def updateItemNotes(
logger: Logger[F],
itemId: Ident,
collective: Ident,
language: Language,
notes: Option[String]
): F[Unit] =
updateIndex(
logger,
TextData.item(itemId, collective, None, None, Some(notes.getOrElse("")))
TextData.item(itemId, collective, None, None, Some(notes.getOrElse("")), language)
)
def updateAttachmentName(
@ -88,6 +93,7 @@ trait FtsClient[F[_]] {
itemId: Ident,
attachId: Ident,
collective: Ident,
language: Language,
name: Option[String]
): F[Unit] =
updateIndex(
@ -97,7 +103,7 @@ trait FtsClient[F[_]] {
attachId,
collective,
None,
Language.English,
language,
Some(name.getOrElse("")),
None
)

12
modules/fts-client/src/main/scala/docspell/ftsclient/TextData.scala
View File

@ -18,6 +18,8 @@ sealed trait TextData {
def folder: Option[Ident]
def language: Language
final def fold[A](f: TextData.Attachment => A, g: TextData.Item => A): A =
this match {
case a: TextData.Attachment => f(a)
@ -32,7 +34,7 @@ object TextData {
attachId: Ident,
collective: Ident,
folder: Option[Ident],
lang: Language,
language: Language,
name: Option[String],
text: Option[String]
) extends TextData {
@ -57,7 +59,8 @@ object TextData {
collective: Ident,
folder: Option[Ident],
name: Option[String],
notes: Option[String]
notes: Option[String],
language: Language
) extends TextData {
val id = Ident.unsafe("item") / item
@ -69,8 +72,9 @@ object TextData {
collective: Ident,
folder: Option[Ident],
name: Option[String],
notes: Option[String]
notes: Option[String],
lang: Language
): TextData =
Item(item, collective, folder, name, notes)
Item(item, collective, folder, name, notes, lang)
}

5
modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala
View File

@ -47,9 +47,10 @@ object TextExtraction {
ctx.args.meta.collective,
ctx.args.meta.folderId,
item.item.name.some,
None
None,
ctx.args.meta.language
)
_ <- fts.indexData(ctx.logger, (idxItem +: txt.map(_.td)).toSeq: _*)
_ <- fts.indexData(ctx.logger, (idxItem +: txt.map(_.td)): _*)
dur <- start
extractedTags = txt.flatMap(_.tags).distinct.toList
_ <- ctx.logger.info(s"Text extraction finished in ${dur.formatExact}.")

8
modules/store/src/main/scala/docspell/store/queries/QItem.scala
View File

@ -576,7 +576,8 @@ object QItem {
collective: Ident,
folder: Option[Ident],
name: String,
notes: Option[String]
notes: Option[String],
language: Language
)
def allNameAndNotes(
coll: Option[Ident],
@ -584,10 +585,11 @@ object QItem {
chunkSize: Int
): Stream[ConnectionIO, NameAndNotes] = {
val i = RItem.as("i")
val c = RCollective.as("c")
Select(
select(i.id, i.cid, i.folder, i.name, i.notes),
from(i)
select(i.id, i.cid, i.folder, i.name, i.notes, c.language),
from(i).innerJoin(c, c.id === i.cid)
).where(
i.state.in(ItemState.validStates) &&?
itemIds.map(ids => i.id.in(ids)) &&?

14
modules/store/src/main/scala/docspell/store/records/RAttachment.scala
View File

@ -236,8 +236,18 @@ object RAttachment {
n3 <- DML.delete(T, T.id === attachId)
} yield n0 + n1 + n2 + n3
def findItemId(attachId: Ident): ConnectionIO[Option[Ident]] =
Select(T.itemId.s, from(T), T.id === attachId).build.query[Ident].option
def findItemAndLanguage(
attachId: Ident
): ConnectionIO[Option[(Ident, Option[Language])]] = {
val a = RAttachment.as("a")
val m = RAttachmentMeta.as("m")
Select(
select(a.itemId, m.language),
from(a).leftJoin(m, m.id === a.id),
a.id === attachId
).build.query[(Ident, Option[Language])].option
}
def findAll(
coll: Option[Ident],