TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-06-22 02:18:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adopt deprecated APIs from fs2; use fs2.Path

Browse Source
This commit is contained in:
eikek
2021-08-07 12:20:38 +02:00
parent f92aeb6a0f
commit 1901fe1a8c
41 changed files with 124 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
modules/analysis/src/main/scala/docspell/analysis/NlpSettings.scala
View File

@ -6,7 +6,7 @@
package docspell.analysis
import java.nio.file.Path
import fs2.io.file.Path
import docspell.common._

2
modules/analysis/src/main/scala/docspell/analysis/classifier/ClassifierModel.scala
View File

@ -6,6 +6,6 @@
package docspell.analysis.classifier
import java.nio.file.Path
import fs2.io.file.Path
case class ClassifierModel(model: Path)

18
modules/analysis/src/main/scala/docspell/analysis/classifier/StanfordTextClassifier.scala
View File

@ -6,13 +6,11 @@
package docspell.analysis.classifier
import java.nio.file.Path
import cats.effect.Ref
import cats.effect._
import cats.implicits._
import fs2.Stream
import fs2.io.file.Files
import fs2.io.file.{Files, Path}
import docspell.analysis.classifier
import docspell.analysis.classifier.TextClassifier._
@ -51,7 +49,7 @@ final class StanfordTextClassifier[F[_]: Async](cfg: TextClassifierConfig)
case Some(text) =>
Sync[F].delay {
val cls = ColumnDataClassifier.getClassifier(
model.model.normalize().toAbsolutePath.toString
model.model.normalize.absolute.toString
)
val cat = cls.classOf(cls.makeDatumFromLine("\t\t" + normalisedText(text)))
Option(cat)
@ -71,8 +69,8 @@ final class StanfordTextClassifier[F[_]: Async](cfg: TextClassifierConfig)
_ <- logger.debug(s"Training classifier from $props")
res <- Sync[F].delay {
val cdc = new ColumnDataClassifier(Properties.fromMap(amendProps(in, props)))
cdc.trainClassifier(in.train.toString())
val score = cdc.testClassifier(in.test.toString())
cdc.trainClassifier(in.train.toString)
val score = cdc.testClassifier(in.test.toString)
TrainResult(score.first(), classifier.ClassifierModel(in.modelFile))
}
_ <- logger.debug(s"Trained with result $res")
@ -88,7 +86,7 @@ final class StanfordTextClassifier[F[_]: Async](cfg: TextClassifierConfig)
val fileLines =
File
.readAll[F](in.file, 4096)
.through(fs2.text.utf8Decode)
.through(fs2.text.utf8.decode)
.through(fs2.text.lines)
for {
@ -99,7 +97,7 @@ final class StanfordTextClassifier[F[_]: Async](cfg: TextClassifierConfig)
fileLines
.take(nTest)
.intersperse("\n")
.through(fs2.text.utf8Encode)
.through(fs2.text.utf8.encode)
.through(Files[F].writeAll(td.test))
.compile
.drain
@ -107,7 +105,7 @@ final class StanfordTextClassifier[F[_]: Async](cfg: TextClassifierConfig)
fileLines
.drop(nTest)
.intersperse("\n")
.through(fs2.text.utf8Encode)
.through(fs2.text.utf8.encode)
.through(Files[F].writeAll(td.train))
.compile
.drain
@ -124,7 +122,7 @@ final class StanfordTextClassifier[F[_]: Async](cfg: TextClassifierConfig)
.map(d => s"${d.cls}\t${fixRef(d.ref)}\t${normalisedText(d.text)}")
.evalTap(_ => counter.update(_ + 1))
.intersperse("\r\n")
.through(fs2.text.utf8Encode)
.through(fs2.text.utf8.encode)
.through(Files[F].writeAll(target))
.compile
.drain

3
modules/analysis/src/main/scala/docspell/analysis/classifier/TextClassifierConfig.scala
View File

@ -6,9 +6,8 @@
package docspell.analysis.classifier
import java.nio.file.Path
import cats.data.NonEmptyList
import fs2.io.file.Path
case class TextClassifierConfig(
workingDir: Path,

3
modules/analysis/src/main/scala/docspell/analysis/nlp/Properties.scala
View File

@ -6,9 +6,10 @@
package docspell.analysis.nlp
import java.nio.file.Path
import java.util.{Properties => JProps}
import fs2.io.file.Path
import docspell.analysis.nlp.Properties.Implicits._
import docspell.common._
import docspell.common.syntax.FileSyntax._

3
modules/analysis/src/main/scala/docspell/analysis/nlp/StanfordNerAnnotator.scala
View File

@ -6,11 +6,10 @@
package docspell.analysis.nlp
import java.nio.file.Path
import scala.jdk.CollectionConverters._
import cats.effect._
import fs2.io.file.Path
import docspell.common._

2
modules/analysis/src/main/scala/docspell/analysis/nlp/StanfordNerSettings.scala
View File

@ -6,7 +6,7 @@
package docspell.analysis.nlp
import java.nio.file.Path
import fs2.io.file.Path
import docspell.analysis.NlpSettings
import docspell.common.Language.NLPLanguage

6
modules/analysis/src/test/scala/docspell/analysis/classifier/StanfordTextClassifierSuite.scala
View File

@ -24,7 +24,7 @@ class StanfordTextClassifierSuite extends FunSuite {
val logger = Logger.log4s[IO](org.log4s.getLogger)
test("learn from data") {
val cfg = TextClassifierConfig(Paths.get("target"), NonEmptyList.of(Map()))
val cfg = TextClassifierConfig(File.path(Paths.get("target")), NonEmptyList.of(Map()))
val data =
Stream
@ -52,8 +52,8 @@ class StanfordTextClassifierSuite extends FunSuite {
}
test("run classifier") {
val cfg = TextClassifierConfig(Paths.get("target"), NonEmptyList.of(Map()))
val things = File.withTempDir[IO](Paths.get("target"), "testcls")
val cfg = TextClassifierConfig(File.path(Paths.get("target")), NonEmptyList.of(Map()))
val things = File.withTempDir[IO](File.path(Paths.get("target")), "testcls")
things
.use { dir =>

3
modules/analysis/src/test/scala/docspell/analysis/nlp/StanfordNerAnnotatorSuite.scala
View File

@ -13,7 +13,6 @@ import cats.effect.unsafe.implicits.global
import docspell.analysis.Env
import docspell.common._
import docspell.common.syntax.FileSyntax._
import docspell.files.TestFiles
import edu.stanford.nlp.pipeline.StanfordCoreNLP
@ -100,7 +99,7 @@ class StanfordNerAnnotatorSuite extends FunSuite {
|""".stripMargin
File
.withTempDir[IO](Paths.get("target"), "test-regex-ner")
.withTempDir[IO](File.path(Paths.get("target")), "test-regex-ner")
.use { dir =>
for {
out <- File.writeString[IO](dir / "regex.txt", regexNerContent)