TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-06-22 18:38:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Upgrade code base to CE3

Browse Source
This commit is contained in:
eikek
2021-06-21 21:33:54 +02:00
parent 903ec26e54
commit bd791b4593
146 changed files with 638 additions and 758 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
modules/analysis/src/test/scala/docspell/analysis/classifier/StanfordTextClassifierSuite.scala
View File

@ -2,12 +2,12 @@ package docspell.analysis.classifier
import java.nio.file.Paths
import scala.concurrent.ExecutionContext
import cats.data.Kleisli
import cats.data.NonEmptyList
import cats.effect._
import cats.effect.unsafe.implicits.global
import fs2.Stream
import fs2.io.file.Files
import docspell.analysis.classifier.TextClassifier.Data
import docspell.common._
@ -17,8 +17,6 @@ import munit._
class StanfordTextClassifierSuite extends FunSuite {
val logger = Logger.log4s[IO](org.log4s.getLogger)
implicit val CS = IO.contextShift(ExecutionContext.global)
test("learn from data") {
val cfg = TextClassifierConfig(Paths.get("target"), NonEmptyList.of(Map()))
@ -38,34 +36,30 @@ class StanfordTextClassifierSuite extends FunSuite {
})
.covary[IO]
val modelExists =
Blocker[IO].use { blocker =>
val classifier = new StanfordTextClassifier[IO](cfg, blocker)
classifier.trainClassifier[Boolean](logger, data)(
Kleisli(result => File.existsNonEmpty[IO](result.model))
)
}
val modelExists = {
val classifier = new StanfordTextClassifier[IO](cfg)
classifier.trainClassifier[Boolean](logger, data)(
Kleisli(result => File.existsNonEmpty[IO](result.model))
)
}
assertEquals(modelExists.unsafeRunSync(), true)
}
test("run classifier") {
val cfg = TextClassifierConfig(Paths.get("target"), NonEmptyList.of(Map()))
val things = for {
dir <- File.withTempDir[IO](Paths.get("target"), "testcls")
blocker <- Blocker[IO]
} yield (dir, blocker)
val cfg = TextClassifierConfig(Paths.get("target"), NonEmptyList.of(Map()))
val things = File.withTempDir[IO](Paths.get("target"), "testcls")
things
.use { case (dir, blocker) =>
val classifier = new StanfordTextClassifier[IO](cfg, blocker)
.use { dir =>
val classifier = new StanfordTextClassifier[IO](cfg)
val modelFile = dir.resolve("test.ser.gz")
for {
_ <-
LenientUri
.fromJava(getClass.getResource("/test.ser.gz"))
.readURL[IO](4096, blocker)
.through(fs2.io.file.writeAll(modelFile, blocker))
.readURL[IO](4096)
.through(Files[IO].writeAll(modelFile))
.compile
.drain
model = ClassifierModel(modelFile)

1
modules/analysis/src/test/scala/docspell/analysis/nlp/StanfordNerAnnotatorSuite.scala
View File

@ -3,6 +3,7 @@ package docspell.analysis.nlp
import java.nio.file.Paths
import cats.effect.IO
import cats.effect.unsafe.implicits.global
import docspell.analysis.Env
import docspell.common._