Rename example files

build.sbt

@@ -150,8 +150,34 @@ val exampleFiles = project.in(file("modules/example-files")).
   settings(sharedSettings).
   settings(testSettings).
   settings(
-    name := "docspell-examplefiles"
+    name := "docspell-examplefiles",
-  )
+    libraryDependencies ++=
+      Dependencies.tika,
+    Test / sourceGenerators += Def.task {
+      val base = (Test/resourceDirectory).value
+      val files = (base ** (_.isFile)) pair sbt.io.Path.relativeTo(base)
+      val lines = files.toList.map(_._2).map(s => {
+        val ident = s.replaceAll("[^a-zA-Z0-9_]+", "_")
+        ident -> s"""val $ident = createUrl("${s}")"""
+      })
+      val content = s"""package docspell.examplefiles
+
+object ExampleFiles extends ExampleFilesSupport {
+
+${lines.map(_._2).mkString("\n")}
+
+val all = List(
+${lines.map(_._1).mkString(",\n")}
+)
+
+}
+"""
+      val target = (Test/sourceManaged).value/"scala"/"ExampleFiles.scala"
+      IO.createDirectory(target.getParentFile)
+      IO.write(target, content)
+      Seq(target)
+    }.taskValue
+  ).dependsOn(common)
 
 val store = project.in(file("modules/store")).
   disablePlugins(RevolverPlugin).
@@ -180,9 +206,7 @@ val text = project.in(file("modules/text")).
     name := "docspell-text",
     libraryDependencies ++=
       Dependencies.fs2 ++
-      Dependencies.tika ++
+      Dependencies.stanfordNlpCore 
-      Dependencies.stanfordNlpCore ++
-      Dependencies.poi
   ).dependsOn(common, exampleFiles % "compile->compile;test->test")
 
 val convert = project.in(file("modules/convert")).

modules/common/src/main/scala/docspell/common/LenientUri.scala

@@ -65,6 +65,11 @@ case class LenientUri(
         fs2.io.readInputStream(Sync[F].delay(url.openStream()), chunkSize, blocker, true)
       )
 
+  def readText[F[_]: Sync: ContextShift](chunkSize: Int, blocker: Blocker): F[String] =
+    readURL[F](chunkSize, blocker).
+      through(fs2.text.utf8Decode).
+      compile.foldMonoid
+
   def host: Option[String] =
     authority.map(a =>
       a.indexOf(':') match {

modules/example-files/src/test/resources/letter-de-source.txt

@@ -0,0 +1,30 @@
+Max Mustermann
+
+Lilienweg 21
+
+12345 Nebendorf
+
+E-Mail: max.muster@gmail.com
+
+Max Mustermann, Lilienweg 21, 12345 Nebendorf
+
+EasyCare AG
+Abteilung Buchhaltung
+Ackerweg 12
+
+12346 Ulmen
+
+Nebendorf, 3. September 2019
+Sehr geehrte Damen und Herren,
+
+hiermit kündige ich meine Mitgliedschaft in der Kranken- und Pflegeversicherung zum
+nächstmöglichen Termin.
+
+Bitte senden Sie mir innerhalb der gesetzlichen Frist von 14 Tagen eine Kündigungsbe-
+stätigung zu.
+
+Vielen Dank im Vorraus!
+
+Mit freundlichen Grüßen
+
+Max Mustermann

modules/example-files/src/test/resources/letter-en-source.txt

@@ -0,0 +1,38 @@
+Derek Jeter
+
+123 Elm Ave.
+
+Treesville, ON MI1N 2P3
+November 7, 2016
+
+Derek Jeter, 123 Elm Ave., Treesville, ON M1N 2P3, November 7, 2016
+
+Mr. M. Leat
+
+Chief of Syrup Production
+Old Sticky Pancake Company
+456 Maple Lane
+
+Forest, ON 7TW8 9Y0
+
+Hemptown, September 3, 2019
+Dear Mr. Leaf,
+
+Let me begin by thanking you for your past contributions to our Little League baseball
+team. Your sponsorship aided in the purchase of ten full uniforms and several pieces of
+baseball equipment for last year’s season.
+
+Next month, our company is planning an employee appreciation pancake breakfast hon-
+oring retired employees for their past years of service and present employees for their
+loyalty and dedication in spite of the current difficult economic conditions.
+
+We would like to place an order with your company for 25 pounds of pancake mix and
+five gallons of maple syrup. We hope you will be able to provide these products in the
+bulk quantities we require.
+
+As you are a committed corporate sponsor and long-time associate, we hope that you
+will be able to join us for breakfast on December 12, 2016.
+
+Respectfully yours,
+
+Derek Jeter

modules/example-files/src/test/scala/docspell/examplefiles/ExampleFiles.scala

@@ -1,5 +0,0 @@
-package docspell.examplefiles
-
-final class ExampleFiles {
-
-}

modules/example-files/src/test/scala/docspell/examplefiles/ExampleFilesSupport.scala

@@ -0,0 +1,14 @@
+package docspell.examplefiles
+
+import docspell.common._
+
+trait ExampleFilesSupport {
+
+  def createUrl(resource: String): LenientUri =
+    Option(getClass.getResource("/" + resource)) match {
+      case Some(u) => LenientUri.fromJava(u)
+      case None => sys.error(s"Resource '$resource' not found")
+    }
+
+
+}

modules/text/src/test/scala/docspell/text/TestFiles.scala

@@ -1,8 +1,8 @@
 package docspell.text
 
-import cats.effect.{Blocker, IO}
-import docspell.common.LenientUri
 import fs2.Stream
+import cats.effect.{Blocker, IO}
+import docspell.examplefiles._
 
 import scala.concurrent.ExecutionContext
 
@@ -11,86 +11,20 @@ object TestFiles {
   implicit val CS = IO.contextShift(ExecutionContext.global)
 
   val letterSourceDE: Stream[IO, Byte] =
-    LenientUri
+    ExampleFiles.letter_de_source_pdf
-      .fromJava(getClass.getResource("/letter-de-source.pdf"))
       .readURL[IO](16 * 1024, blocker)
 
   val letterSourceEN: Stream[IO, Byte] =
-    LenientUri
+    ExampleFiles.letter_en_source_pdf
-      .fromJava(getClass.getResource("/letter-en-source.pdf"))
       .readURL[IO](16 * 1024, blocker)
 
-  val letterDEText =
+  lazy val letterDEText =
-    """Max Mustermann
+    ExampleFiles.letter_de_source_txt
-               |
+      .readText[IO](16 * 1024, blocker)
-               |Lilienweg 21
+      .unsafeRunSync
-               |
-               |12345 Nebendorf
-               |
-               |E-Mail: max.muster@gmail.com
-               |
-               |Max Mustermann, Lilienweg 21, 12345 Nebendorf
-               |
-               |EasyCare AG
-               |Abteilung Buchhaltung
-               |Ackerweg 12
-               |
-               |12346 Ulmen
-               |
-               |Nebendorf, 3. September 2019
-               |Sehr geehrte Damen und Herren,
-               |
-               |hiermit kündige ich meine Mitgliedschaft in der Kranken- und Pflegeversicherung zum
-               |nächstmöglichen Termin.
-               |
-               |Bitte senden Sie mir innerhalb der gesetzlichen Frist von 14 Tagen eine Kündigungsbe-
-               |stätigung zu.
-               |
-               |Vielen Dank im Vorraus!
-               |
-               |Mit freundlichen Grüßen
-               |
-               |Max Mustermann
-               |""".stripMargin.trim
 
-  val letterENText =
+  lazy val letterENText =
-    """Derek Jeter
+    ExampleFiles.letter_en_source_txt
-                       |
+      .readText[IO](16 * 1024, blocker)
-                       |123 Elm Ave.
+      .unsafeRunSync
-                       |
-                       |Treesville, ON MI1N 2P3
-                       |November 7, 2016
-                       |
-                       |Derek Jeter, 123 Elm Ave., Treesville, ON M1N 2P3, November 7, 2016
-                       |
-                       |Mr. M. Leat
-                       |
-                       |Chief of Syrup Production
-                       |Old Sticky Pancake Company
-                       |456 Maple Lane
-                       |
-                       |Forest, ON 7TW8 9Y0
-                       |
-                       |Hemptown, September 3, 2019
-                       |Dear Mr. Leaf,
-                       |
-                       |Let me begin by thanking you for your past contributions to our Little League baseball
-                       |team. Your sponsorship aided in the purchase of ten full uniforms and several pieces of
-                       |baseball equipment for last year’s season.
-                       |
-                       |Next month, our company is planning an employee appreciation pancake breakfast hon-
-                       |oring retired employees for their past years of service and present employees for their
-                       |loyalty and dedication in spite of the current difficult economic conditions.
-                       |
-                       |We would like to place an order with your company for 25 pounds of pancake mix and
-                       |five gallons of maple syrup. We hope you will be able to provide these products in the
-                       |bulk quantities we require.
-                       |
-                       |As you are a committed corporate sponsor and long-time associate, we hope that you
-                       |will be able to join us for breakfast on December 12, 2016.
-                       |
-                       |Respectfully yours,
-                       |
-                       |Derek Jeter
-                       |""".stripMargin.trim
 }