TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-06-04 14:15:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Only learn from confirmed items

Browse Source 
Text classification should only learn from confirmed items. Log if
classification is disabled when processing an item.
This commit is contained in:
Eike Kettner 2020-09-07 13:04:40 +02:00
parent cb1a9e0699
commit 76ccfb8a81
4 changed files with 24 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
modules/common/src/main/scala/docspell/common/ItemState.scala
View File

@ -23,6 +23,11 @@ object ItemState {
case object Created extends ItemState case object Created extends ItemState
case object Confirmed extends ItemState case object Confirmed extends ItemState
def premature: ItemState = Premature
def processing: ItemState = Processing
def created: ItemState = Created
def confirmed: ItemState = Confirmed
def fromString(str: String): Either[String, ItemState] = def fromString(str: String): Either[String, ItemState] =
str.toLowerCase match { str.toLowerCase match {
case "premature" => Right(Premature) case "premature" => Right(Premature)

3
modules/joex/src/main/scala/docspell/joex/analysis/RegexNerFile.scala
View File

@ -157,8 +157,9 @@ object RegexNerFile {
.reduce(_ ++ fr"UNION ALL" ++ _) .reduce(_ ++ fr"UNION ALL" ++ _)
selectSimple(fr"MAX(t)", fr"(" ++ sql ++ fr") as x", Fragment.empty) selectSimple(fr"MAX(t)", fr"(" ++ sql ++ fr") as x", Fragment.empty)
.query[Timestamp] .query[Option[Timestamp]]
.option .option
.map(_.flatten)
} }
} }
} }

15
modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala
View File

@ -98,11 +98,14 @@ object TextAnalysis {
ctx: Context[F, Args], ctx: Context[F, Args],
cfg: Config.TextAnalysis cfg: Config.TextAnalysis
): OptionT[F, Ident] = ): OptionT[F, Ident] =
if (cfg.classification.enabled) (if (cfg.classification.enabled)
OptionT(ctx.store.transact(RClassifierSetting.findById(ctx.args.meta.collective))) OptionT(ctx.store.transact(RClassifierSetting.findById(ctx.args.meta.collective)))
.filter(_.enabled) .filter(_.enabled)
.mapFilter(_.fileId) .mapFilter(_.fileId)
else else
OptionT.none OptionT.none[F, Ident]).orElse(
OptionT.liftF(ctx.logger.info("Classification is disabled.")) *> OptionT
.none[F, Ident]
)
} }

10
modules/store/src/main/scala/docspell/store/queries/QItem.scala
View File

@ -621,8 +621,14 @@ object QItem {
collective: Ident, collective: Ident,
chunkSize: Int chunkSize: Int
): Stream[ConnectionIO, Ident] = { ): Stream[ConnectionIO, Ident] = {
val cols = Seq(RItem.Columns.id) val cols = Seq(RItem.Columns.id)
(selectSimple(cols, RItem.table, RItem.Columns.cid.is(collective)) ++ val iColl = RItem.Columns.cid
val iState = RItem.Columns.state
(selectSimple(
cols,
RItem.table,
and(iColl.is(collective), iState.is(ItemState.confirmed))
) ++
orderBy(RItem.Columns.created.desc)) orderBy(RItem.Columns.created.desc))
.query[Ident] .query[Ident]
.streamWithChunkSize(chunkSize) .streamWithChunkSize(chunkSize)