TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-04-04 18:39:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move blocker into constructor of text analyser

Browse Source
This commit is contained in:
Eike Kettner 2021-01-13 21:26:44 +01:00
parent ffbec3502f
commit f02f15e5bd
4 changed files with 8 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
modules/analysis/src/main/scala/docspell/analysis/TextAnalyser.scala
View File

@ -21,7 +21,7 @@ trait TextAnalyser[F[_]] {
text: String
): F[TextAnalyser.Result]
def classifier(blocker: Blocker)(implicit CS: ContextShift[F]): TextClassifier[F]
def classifier: TextClassifier[F]
}
object TextAnalyser {
@ -31,8 +31,9 @@ object TextAnalyser {
labels ++ dates.map(dl => dl.label.copy(label = dl.date.toString))
}
def create[F[_]: Concurrent: Timer](
cfg: TextAnalysisConfig
def create[F[_]: Concurrent: Timer: ContextShift](
cfg: TextAnalysisConfig,
blocker: Blocker
): Resource[F, TextAnalyser[F]] =
Resource
.liftF(PipelineCache[F](cfg.clearStanfordPipelineInterval))
@ -53,9 +54,7 @@ object TextAnalyser {
spans = NerLabelSpan.build(list)
} yield Result(spans ++ list, dates)
def classifier(blocker: Blocker)(implicit
CS: ContextShift[F]
): TextClassifier[F] =
def classifier: TextClassifier[F] =
new StanfordTextClassifier[F](cfg.classifier, blocker)
private def textLimit(logger: Logger[F], text: String): F[String] =

2
modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala
View File

@ -97,7 +97,7 @@ object JoexAppImpl {
upload <- OUpload(store, queue, cfg.files, joex)
fts <- createFtsClient(cfg)(httpClient)
itemOps <- OItem(store, fts, queue, joex)
analyser <- TextAnalyser.create[F](cfg.textAnalysis.textAnalysisConfig)
analyser <- TextAnalyser.create[F](cfg.textAnalysis.textAnalysisConfig, blocker)
regexNer <- RegexNerFile(cfg.textAnalysis.regexNerFileConfig, blocker, store)
javaEmil =
JavaMailEmil(blocker, Settings.defaultSettings.copy(debug = cfg.mailDebug))

3
modules/joex/src/main/scala/docspell/joex/learn/LearnClassifierTask.scala
View File

@ -41,8 +41,7 @@ object LearnClassifierTask {
sett.category.getOrElse("")
)
_ <- OptionT.liftF(
analyser
.classifier(blocker)
analyser.classifier
.trainClassifier[Unit](ctx.logger, data)(Kleisli(handleModel(ctx, blocker)))
)
} yield ())

2
modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala
View File

@ -44,7 +44,7 @@ object TextAnalysis {
e <- s
_ <- ctx.logger.info(s"Text-Analysis finished in ${e.formatExact}")
v = t.toVector
tag <- predictTag(ctx, cfg, item.metas, analyser.classifier(ctx.blocker)).value
tag <- predictTag(ctx, cfg, item.metas, analyser.classifier).value
} yield item
.copy(metas = v.map(_._1), dateLabels = v.map(_._2))
.appendTags(tag.toSeq)