TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-03-28 17:55:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move mimetype detection to docspell-files

Browse Source
This commit is contained in:
Eike Kettner 2020-02-14 11:24:05 +01:00
parent 5c3d2b2e28
commit 1309c8b7fa
4 changed files with 19 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/text/src/main/scala/docspell/text/ocr/MimeTypeHint.scala → modules/common/src/main/scala/docspell/common/MimeTypeHint.scala
View File

@ -1,4 +1,4 @@
package docspell.text.ocr
package docspell.common
case class MimeTypeHint(filename: Option[String], advertised: Option[String]) {}

8
modules/text/src/main/scala/docspell/text/ocr/TikaMimetype.scala → modules/files/src/main/scala/docspell/files/TikaMimetype.scala
View File

@ -1,8 +1,8 @@
package docspell.text.ocr
package docspell.files
import cats.implicits._
import cats.effect.Sync
import docspell.common.MimeType
import docspell.common._
import fs2.Stream
import org.apache.tika.config.TikaConfig
import org.apache.tika.metadata.{HttpHeaders, Metadata, TikaMetadataKeys}
@ -35,7 +35,7 @@ object TikaMimetype {
private def fromBytes(bv: Array[Byte], hint: MimeTypeHint): MimeType =
convert(tika.detect(new java.io.ByteArrayInputStream(bv), makeMetadata(hint)))
def detect[F[_]: Sync](data: Stream[F, Byte]): F[MimeType] =
data.take(1024).compile.toVector.map(bytes => fromBytes(bytes.toArray, MimeTypeHint.none))
def detect[F[_]: Sync](data: Stream[F, Byte], hint: MimeTypeHint): F[MimeType] =
data.take(64).compile.toVector.map(bytes => fromBytes(bytes.toArray, hint))
}

5
modules/text/src/main/scala/docspell/text/ocr/TextExtract.scala
View File

@ -1,7 +1,8 @@
package docspell.text.ocr
import cats.effect.{Blocker, ContextShift, Sync}
import docspell.common.MimeType
import docspell.common._
import docspell.files._
import fs2.Stream
object TextExtract {
@ -21,7 +22,7 @@ object TextExtract {
config: Config
): Stream[F, String] =
Stream
.eval(TikaMimetype.detect(in))
.eval(TikaMimetype.detect(in, MimeTypeHint.none))
.flatMap({
case mt if !config.isAllowed(mt) =>
raiseError(s"File `$mt` not allowed")

11
modules/text/src/test/scala/docspell/text/ocr/TextExtractionSuite.scala
View File

@ -1,6 +1,8 @@
package docspell.text.ocr
import cats.effect.IO
import docspell.common._
import docspell.files._
import docspell.text.TestFiles
import minitest.SimpleTestSuite
@ -28,4 +30,13 @@ object TextExtractionSuite extends SimpleTestSuite {
assertEquals(extract.trim, expect.trim)
}
test("find mimetypes") {
docspell.examplefiles.ExampleFiles.
all.foreach { url =>
TikaMimetype.detect(url.readURL[IO](8192, blocker), MimeTypeHint.none).
map(mt => println(url.asString + ": " + mt.asString)).
unsafeRunSync
}
}
}