Merge pull request #1086 from eikek/fixup/improvements

Fixup/improvements

build.sbt

@@ -381,7 +381,7 @@ val store = project
     libraryDependencies ++=
       Dependencies.testContainer.map(_ % Test)
   )
-  .dependsOn(common, query.jvm, totp)
+  .dependsOn(common, query.jvm, totp, files)
 
 val extract = project
   .in(file("modules/extract"))

modules/backend/src/main/scala/docspell/backend/ops/OTotp.scala

@@ -18,13 +18,23 @@ import org.log4s.getLogger
 
 trait OTotp[F[_]] {
 
+  /** Return whether TOTP is enabled for this account or not. */
   def state(accountId: AccountId): F[OtpState]
 
+  /** Initializes TOTP by generating a secret and storing it in the database. TOTP is
+    * still disabled, it must be confirmed in order to be active.
+    */
   def initialize(accountId: AccountId): F[InitResult]
 
+  /** Confirms and finishes initialization. TOTP is active after this for the given
+    * account.
+    */
   def confirmInit(accountId: AccountId, otp: OnetimePassword): F[ConfirmResult]
 
-  def disable(accountId: AccountId): F[UpdateResult]
+  /** Disables TOTP and removes the shared secret. If a otp is specified, it must be
+    * valid.
+    */
+  def disable(accountId: AccountId, otp: Option[OnetimePassword]): F[UpdateResult]
 }
 
 object OTotp {
@@ -133,8 +143,31 @@ object OTotp {
           }
         } yield res
 
-      def disable(accountId: AccountId): F[UpdateResult] =
+      def disable(accountId: AccountId, otp: Option[OnetimePassword]): F[UpdateResult] =
+        otp match {
+          case Some(pw) =>
+            for {
+              _ <- log.info(s"Validating TOTP, because it is requested to disable it.")
+              key <- store.transact(RTotp.findEnabledByLogin(accountId, true))
+              now <- Timestamp.current[F]
+              res <- key match {
+                case None =>
+                  UpdateResult.failure(new Exception("TOTP not enabled.")).pure[F]
+                case Some(r) =>
+                  val check = totp.checkPassword(r.secret, pw, now.value)
+                  if (check)
+                    UpdateResult.fromUpdate(
+                      store.transact(RTotp.setEnabled(accountId, false))
+                    )
+                  else
+                    log.info(s"TOTP code was invalid. Not disabling it.") *> UpdateResult
+                      .failure(new Exception("Code invalid!"))
+                      .pure[F]
+              }
+            } yield res
+          case None =>
             UpdateResult.fromUpdate(store.transact(RTotp.setEnabled(accountId, false)))
+        }
 
       def state(accountId: AccountId): F[OtpState] =
         for {

modules/common/src/main/scala/docspell/common/MimeType.scala

@@ -9,6 +9,8 @@ package docspell.common
 import java.nio.charset.Charset
 import java.nio.charset.StandardCharsets
 
+import scala.util.Try
+
 import cats.data.NonEmptyList
 
 import docspell.common.syntax.all._
@@ -16,33 +18,31 @@ import docspell.common.syntax.all._
 import io.circe.{Decoder, Encoder}
 
 /** A MIME Type impl with just enough features for the use here. */
-case class MimeType(primary: String, sub: String, params: Map[String, String]) {
+case class MimeType(primary: String, sub: String, charset: Option[Charset]) {
-  def withParam(name: String, value: String): MimeType =
-    copy(params = params.updated(name, value))
 
   def withCharset(cs: Charset): MimeType =
-    withParam("charset", cs.name())
+    copy(charset = Some(cs))
 
   def withUtf8Charset: MimeType =
     withCharset(StandardCharsets.UTF_8)
 
-  def resolveCharset: Option[Charset] =
+  def withCharsetName(csName: String): MimeType =
-    params.get("charset").flatMap { cs =>
+    if (Try(Charset.isSupported(csName)).getOrElse(false))
-      if (Charset.isSupported(cs)) Some(Charset.forName(cs))
+      withCharset(Charset.forName(csName))
-      else None
+    else this
-    }
 
   def charsetOrUtf8: Charset =
-    resolveCharset.getOrElse(StandardCharsets.UTF_8)
+    charset.getOrElse(StandardCharsets.UTF_8)
 
   def baseType: MimeType =
-    if (params.isEmpty) this else copy(params = Map.empty)
+    if (charset.isEmpty) this else copy(charset = None)
 
   def asString: String =
-    if (params.isEmpty) s"$primary/$sub"
+    charset match {
-    else {
+      case Some(cs) =>
-      val parameters = params.toList.map(t => s"""${t._1}="${t._2}"""").mkString(";")
+        s"$primary/$sub; charset=\"${cs.name()}\""
-      s"$primary/$sub; $parameters"
+      case None =>
+        s"$primary/$sub"
     }
 
   def matches(other: MimeType): Boolean =
@@ -53,46 +53,16 @@ case class MimeType(primary: String, sub: String, params: Map[String, String]) {
 object MimeType {
 
   def application(sub: String): MimeType =
-    MimeType("application", sub, Map.empty)
+    MimeType("application", sub, None)
 
   def text(sub: String): MimeType =
-    MimeType("text", sub, Map.empty)
+    MimeType("text", sub, None)
 
   def image(sub: String): MimeType =
-    MimeType("image", sub, Map.empty)
+    MimeType("image", sub, None)
 
-  def parse(str: String): Either[String, MimeType] = {
+  def parse(str: String): Either[String, MimeType] =
-    def parsePrimary: Either[String, (String, String)] =
+    Parser.parse(str)
-      str.indexOf('/') match {
-        case -1 => Left(s"Invalid mediatype: $str")
-        case n  => Right(str.take(n) -> str.drop(n + 1))
-      }
-
-    def parseSub(s: String): Either[String, (String, String)] =
-      s.indexOf(';') match {
-        case -1 => Right((s, ""))
-        case n  => Right((s.take(n), s.drop(n)))
-      }
-
-    def parseParams(s: String): Map[String, String] =
-      s.split(';')
-        .map(_.trim)
-        .filter(_.nonEmpty)
-        .toList
-        .flatMap(p =>
-          p.split("=", 2).toList match {
-            case a :: b :: Nil => Some((a, b))
-            case _             => None
-          }
-        )
-        .toMap
-
-    for {
-      pt <- parsePrimary
-      st <- parseSub(pt._2)
-      pa = parseParams(st._2)
-    } yield MimeType(pt._1, st._1, pa)
-  }
 
   def unsafe(str: String): MimeType =
     parse(str).throwLeft
@@ -105,8 +75,9 @@ object MimeType {
   val tiff = image("tiff")
   val html = text("html")
   val plain = text("plain")
+  val json = application("json")
   val emls = NonEmptyList.of(
-    MimeType("message", "rfc822", Map.empty),
+    MimeType("message", "rfc822", None),
     application("mbox")
   )
 
@@ -158,4 +129,88 @@ object MimeType {
 
   implicit val jsonDecoder: Decoder[MimeType] =
     Decoder.decodeString.emap(parse)
+
+  private object Parser {
+    def parse(s: String): Either[String, MimeType] =
+      mimeType(s).map(_._1)
+
+    type Result[A] = Either[String, (A, String)]
+    type P[A] = String => Result[A]
+
+    private[this] val tokenExtraChars = "+-$%*._~".toSet
+
+    private def seq[A, B, C](pa: P[A], pb: P[B])(f: (A, B) => C): P[C] =
+      in =>
+        pa(in) match {
+          case Right((a, resta)) =>
+            pb(resta) match {
+              case Right((b, restb)) =>
+                Right((f(a, b), restb))
+              case left =>
+                left.asInstanceOf[Result[C]]
+            }
+          case left =>
+            left.asInstanceOf[Result[C]]
+        }
+
+    private def takeWhile(p: Char => Boolean): P[String] =
+      in => {
+        val (prefix, suffix) = in.span(p)
+        Right((prefix.trim, suffix.drop(1).trim))
+      }
+
+    private def check[A](p: P[A], test: A => Boolean, err: => String): P[A] =
+      in =>
+        p(in) match {
+          case r @ Right((a, _)) =>
+            if (test(a)) r else Left(err)
+          case left =>
+            left
+        }
+
+    //https://datatracker.ietf.org/doc/html/rfc7230#section-3.2.6
+    private def isToken(s: String): Boolean =
+      s.nonEmpty && s.forall(c => c.isLetterOrDigit || tokenExtraChars.contains(c))
+
+    private val baseType: P[MimeType] = {
+      val primary = check(
+        takeWhile(_ != '/'),
+        isToken,
+        "Primary type must be non-empty and contain valid characters"
+      )
+      val sub = check(
+        takeWhile(_ != ';'),
+        isToken,
+        "Subtype must be non-empty and contain valid characters"
+      )
+      seq(primary, sub)((p, s) => MimeType(p.toLowerCase, s.toLowerCase, None))
+    }
+
+    //https://datatracker.ietf.org/doc/html/rfc2046#section-4.1.2
+    private val charset: P[Option[Charset]] = in =>
+      in.trim.toLowerCase.indexOf("charset=") match {
+        case -1 => Right((None, in))
+        case n =>
+          val csValueStart = in.substring(n + "charset=".length).trim
+          val csName = csValueStart.indexOf(';') match {
+            case -1 => unquote(csValueStart).trim
+            case n  => unquote(csValueStart.substring(0, n)).trim
+          }
+          if (Charset.isSupported(csName)) Right((Some(Charset.forName(csName)), ""))
+          else Right((None, ""))
+      }
+
+    private val mimeType =
+      seq(baseType, charset)((bt, cs) => bt.copy(charset = cs))
+
+    private def unquote(s: String): String = {
+      val len = s.length
+      if (len == 0 || len == 1) s
+      else {
+        if (s.charAt(0) == '"' && s.charAt(len - 1) == '"')
+          unquote(s.substring(1, len - 1))
+        else s
+      }
+    }
+  }
 }

modules/common/src/main/scala/docspell/common/syntax/EitherSyntax.scala

@@ -10,10 +10,7 @@ trait EitherSyntax {
 
   implicit final class LeftStringEitherOps[A](e: Either[String, A]) {
     def throwLeft: A =
-      e match {
+      e.fold(sys.error, identity)
-        case Right(a)  => a
-        case Left(err) => sys.error(err)
-      }
   }
 
   implicit final class ThrowableLeftEitherOps[A](e: Either[Throwable, A]) {

modules/common/src/main/scala/docspell/common/syntax/StreamSyntax.scala

@@ -14,24 +14,14 @@ import io.circe._
 import io.circe.parser._
 
 trait StreamSyntax {
-
   implicit class StringStreamOps[F[_]](s: Stream[F, String]) {
-
     def parseJsonAs[A](implicit d: Decoder[A], F: Sync[F]): F[Either[Throwable, A]] =
-      s.fold("")(_ + _)
+      s.compile.string
-        .compile
+        .map(str =>
-        .last
-        .map(optStr =>
           for {
-            str <-
-              optStr
-                .map(_.trim)
-                .toRight(new Exception("Empty string cannot be parsed into a value"))
             json <- parse(str).leftMap(_.underlying)
             value <- json.as[A]
           } yield value
         )
-
   }
-
 }

modules/common/src/test/scala/docspell/common/MimeTypeTest.scala

@@ -0,0 +1,128 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common
+
+import java.nio.charset.{Charset, StandardCharsets}
+
+import scala.jdk.CollectionConverters._
+
+import munit.ScalaCheckSuite
+import org.scalacheck.Gen
+import org.scalacheck.Prop.forAll
+
+class MimeTypeTest extends ScalaCheckSuite {
+
+  test("asString") {
+    assertEquals(MimeType.html.asString, "text/html")
+    assertEquals(
+      MimeType.html.withCharset(StandardCharsets.ISO_8859_1).asString,
+      "text/html; charset=\"ISO-8859-1\""
+    )
+    assertEquals(
+      MimeType.html.withUtf8Charset.asString,
+      "text/html; charset=\"UTF-8\""
+    )
+  }
+
+  test("parse without params") {
+    assertEquals(MimeType.unsafe("application/pdf"), MimeType.pdf)
+    assertEquals(MimeType.unsafe("image/jpeg"), MimeType.jpeg)
+
+    assertEquals(MimeType.unsafe("image/jpeg "), MimeType.jpeg)
+    assertEquals(MimeType.unsafe("  image/jpeg "), MimeType.jpeg)
+    assertEquals(MimeType.unsafe("  image / jpeg "), MimeType.jpeg)
+
+    assertEquals(
+      MimeType.unsafe("application/xml+html"),
+      MimeType.application("xml+html")
+    )
+    assertEquals(
+      MimeType.unsafe(
+        "application/vnd.openxmlformats-officedocument.presentationml.viewProps+xml"
+      ),
+      MimeType.application(
+        "vnd.openxmlformats-officedocument.presentationml.viewprops+xml"
+      )
+    )
+    assertEquals(
+      MimeType.unsafe("application/vnd.powerbuilder75-s"),
+      MimeType.application("vnd.powerbuilder75-s")
+    )
+  }
+
+  test("parse with charset") {
+    assertEquals(
+      MimeType.unsafe("text/plain; charset=UTF-8"),
+      MimeType.plain.withUtf8Charset
+    )
+    assertEquals(
+      MimeType.unsafe("text/plain; CHARSET=UTF-8"),
+      MimeType.plain.withUtf8Charset
+    )
+    assertEquals(
+      MimeType.unsafe("text/plain; CharSet=UTF-8"),
+      MimeType.plain.withUtf8Charset
+    )
+    assertEquals(
+      MimeType.unsafe("text/html; charset=\"ISO-8859-1\""),
+      MimeType.html.withCharset(StandardCharsets.ISO_8859_1)
+    )
+  }
+
+  test("parse with charset and more params") {
+    assertEquals(
+      MimeType.unsafe("text/plain; charset=UTF-8; action=test"),
+      MimeType.plain.withUtf8Charset
+    )
+    assertEquals(
+      MimeType.unsafe("text/plain; run=\"2\"; charset=UTF-8; action=test"),
+      MimeType.plain.withUtf8Charset
+    )
+  }
+
+  test("parse without charset but params") {
+    assertEquals(MimeType.unsafe("image/jpeg; action=urn:2"), MimeType.jpeg)
+  }
+
+  test("parse some stranger values") {
+    assertEquals(
+      MimeType.unsafe("text/plain; charset=\"\"ISO-8859-1\"\""),
+      MimeType.plain.withCharset(StandardCharsets.ISO_8859_1)
+    )
+    assertEquals(
+      MimeType.unsafe("text/plain; charset=\"\" ISO-8859-1 \"\""),
+      MimeType.plain.withCharset(StandardCharsets.ISO_8859_1)
+    )
+  }
+
+  test("parse invalid mime types") {
+    assert(MimeType.parse("").isLeft)
+    assert(MimeType.parse("_ _/plain").isLeft)
+    assert(MimeType.parse("/").isLeft)
+    assert(MimeType.parse("()").isLeft)
+  }
+
+  property("read own asString") {
+    forAll(MimeTypeTest.mimeType) { mt: MimeType =>
+      assertEquals(MimeType.unsafe(mt.asString), mt)
+    }
+  }
+}
+
+object MimeTypeTest {
+  val someTypes = List(
+    MimeType.plain,
+    MimeType.html
+  ) ++ MimeType.emls.toList
+
+  val mimeType =
+    for {
+      base <- Gen.atLeastOne(someTypes)
+      cs <- Gen.someOf(Charset.availableCharsets().values().asScala)
+    } yield base.head.copy(charset = cs.headOption)
+
+}

modules/files/src/main/scala/docspell/files/TikaMimetype.scala

@@ -24,6 +24,7 @@ import org.apache.tika.config.TikaConfig
 import org.apache.tika.metadata.{HttpHeaders, Metadata, TikaCoreProperties}
 import org.apache.tika.mime.MediaType
 import org.apache.tika.parser.txt.Icu4jEncodingDetector
+import scodec.bits.ByteVector
 
 object TikaMimetype {
   private val tika = new TikaConfig().getDetector
@@ -31,10 +32,10 @@ object TikaMimetype {
   private def convert(mt: MediaType): MimeType =
     Option(mt) match {
       case Some(_) =>
-        val params = mt.getParameters.asScala.toMap
+        val cs = mt.getParameters.asScala.toMap.get("charset").getOrElse("unknown")
         val primary = mt.getType
         val sub = mt.getSubtype
-        normalize(MimeType(primary, sub, params))
+        normalize(MimeType(primary, sub, None).withCharsetName(cs))
       case None =>
         MimeType.octetStream
     }
@@ -48,8 +49,8 @@ object TikaMimetype {
 
   private def normalize(in: MimeType): MimeType =
     in match {
-      case MimeType(_, sub, p) if sub contains "xhtml" =>
+      case MimeType(_, sub, cs) if sub contains "xhtml" =>
-        MimeType.html.copy(params = p)
+        MimeType.html.copy(charset = cs)
       case _ => in
     }
 
@@ -83,10 +84,13 @@ object TikaMimetype {
   def detect[F[_]: Sync](data: Stream[F, Byte], hint: MimeTypeHint): F[MimeType] =
     data.take(64).compile.toVector.map(bytes => fromBytes(bytes.toArray, hint))
 
+  def detect(data: ByteVector, hint: MimeTypeHint): MimeType =
+    fromBytes(data.toArray, hint)
+
   def resolve[F[_]: Sync](dt: DataType, data: Stream[F, Byte]): F[MimeType] =
     dt match {
       case DataType.Exact(mt) =>
-        mt.resolveCharset match {
+        mt.charset match {
           case None if mt.primary == "text" =>
             detectCharset[F](data, MimeTypeHint.advertised(mt))
               .map {

modules/files/src/test/scala/docspell/files/TikaMimetypeTest.scala

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.files
+
+import docspell.common.{MimeType, MimeTypeHint}
+
+import munit.FunSuite
+import scodec.bits.ByteVector
+
+class TikaMimetypeTest extends FunSuite {
+
+  private def detect(bv: ByteVector, hint: MimeTypeHint): MimeType =
+    TikaMimetype.detect(bv, hint)
+
+  test("detect text/plain") {
+    val mt = detect(ByteVector.view("hello world".getBytes), MimeTypeHint.none)
+    assertEquals(mt.baseType, MimeType.plain)
+  }
+
+  test("detect image/jpeg") {
+    val mt = detect(
+      ByteVector.fromValidBase64("/9j/4AAQSkZJRgABAgAAZABkAAA="),
+      MimeTypeHint.none
+    )
+    assertEquals(mt, MimeType.jpeg)
+  }
+
+  test("detect image/png") {
+    val mt = detect(
+      ByteVector.fromValidBase64("iVBORw0KGgoAAAANSUhEUgAAA2I="),
+      MimeTypeHint.none
+    )
+    assertEquals(mt, MimeType.png)
+  }
+
+  test("detect application/json") {
+    val mt =
+      detect(
+        ByteVector.view("""{"name":"me"}""".getBytes),
+        MimeTypeHint.filename("me.json")
+      )
+    assertEquals(mt, MimeType.json)
+  }
+
+  test("detect application/json") {
+    val mt = detect(
+      ByteVector.view("""{"name":"me"}""".getBytes),
+      MimeTypeHint.advertised("application/json")
+    )
+    assertEquals(mt, MimeType.json)
+  }
+
+  test("detect image/jpeg wrong advertised") {
+    val mt = detect(
+      ByteVector.fromValidBase64("/9j/4AAQSkZJRgABAgAAZABkAAA="),
+      MimeTypeHint.advertised("image/png")
+    )
+    assertEquals(mt, MimeType.jpeg)
+  }
+
+  test("just filename") {
+    assertEquals(
+      detect(ByteVector.empty, MimeTypeHint.filename("doc.pdf")),
+      MimeType.pdf
+    )
+  }
+
+}

modules/oidc/src/main/scala/docspell/oidc/OpenidConnect.scala

@@ -1,16 +0,0 @@
-/*
- * Copyright 2020 Eike K. & Contributors
- *
- * SPDX-License-Identifier: AGPL-3.0-or-later
- */
-
-package docspell.oidc
-
-import org.http4s.HttpRoutes
-import org.http4s.client.Client
-
-object OpenidConnect {
-
-  def codeFlow[F[_]](client: Client[F]): HttpRoutes[F] =
-    ???
-}

modules/restapi/src/main/resources/docspell-openapi.yml

@@ -1442,11 +1442,18 @@ paths:
       summary: Disables two factor authentication.
       description: |
         Disables two factor authentication for the current user. If
-        the user has no two factor authentication enabled, this
+        the user has no two factor authentication enabled, an error is
-        returns success, too.
+        returned.
+
+        It requires to specify a valid otp.
 
         After this completes successfully, two factor auth can be
         enabled again by initializing it anew.
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/OtpConfirm"
       responses:
         200:
           description: Ok

modules/restserver/src/main/resources/reference.conf

@@ -66,31 +66,31 @@ docspell.server {
   #
   # Multiple authentication providers can be defined. Each is
   # configured in the array below. The `provider` block gives all
-  # details necessary to authenticate agains an external OIDC or OAuth
+  # details necessary to authenticate against an external OIDC or
-  # provider. This requires at least two URLs for OIDC and three for
+  # OAuth provider. This requires at least two URLs for OIDC and three
-  # OAuth2. The `user-url` is only required for OIDC, if the account
+  # for OAuth2. When using OIDC, the `user-url` is only required if
-  # data is to be retrieved from the user-info endpoint and not from
+  # the account data is to be retrieved from the user-info endpoint
-  # the JWT token. The access token is then used to authenticate at
+  # and not from the JWT token. For the request to the `user-url`, the
-  # the provider to obtain user info. Thus, it doesn't need to be
+  # access token is then used to authenticate at the provider. Thus,
-  # validated here and therefore no `sign-key` setting is needed.
+  # it doesn't need to be validated here and therefore no `sign-key`
-  # However, if you want to extract the account information from the
+  # setting is needed. However, if you want to extract the account
-  # access token, it must be validated here and therefore the correct
+  # information from the access token, it must be validated here and
-  # signature key and algorithm must be provided. This would save
+  # therefore the correct signature key and algorithm must be
-  # another request. If the `sign-key` is left empty, the `user-url`
+  # provided. If the `sign-key` is left empty, the `user-url` is used
-  # is used and must be specified. If the `sign-key` is _not_ empty,
+  # and must be specified. If the `sign-key` is _not_ empty, the
-  # the response from the authentication provider is validated using
+  # response from the authentication provider is validated using this
-  # this key.
+  # key.
   #
   # After successful authentication, docspell needs to create the
   # account. For this a username and collective name is required. The
-  # username is defined by the `user-key` setting. The `user-key` is
+  # account name is defined by the `user-key` and `collective-key`
-  # used to search the JSON structure, that is obtained from the JWT
+  # setting. The `user-key` is used to search the JSON structure, that
-  # token or the user-info endpoint, for the login name to use. It
+  # is obtained from the JWT token or the user-info endpoint, for the
-  # traverses the JSON structure recursively, until it finds an object
+  # login name to use. It traverses the JSON structure recursively,
-  # with that key. The first value is used.
+  # until it finds an object with that key. The first value is used.
   #
-  # There are the following ways to specify how to retrieve the full
+  # The `collective-key` can be used in multiple ways and both can
-  # account id depending on the value of `collective-key`:
+  # work together to retrieve the full account id:
   #
   # - If it starts with `fixed:`, like "fixed:collective", the name
   #   after the `fixed:` prefix is used as collective as is. So all
@@ -100,15 +100,15 @@ docspell.server {
   #   value after the prefix is used to search the JSON response for
   #   an object with this key, just like it works with the `user-key`.
   #
-  # - If it starts with `account:`, like "account:ds-account", it
+  # - If it starts with `account:`, like "account:demo", it works the
-  #   works the same as `lookup:` only that the value is interpreted
+  #   same as `lookup:` only that the value is interpreted as the full
-  #   as the full account name of form `collective/login`. The
+  #   account name of form `collective/login`. The `user-key` value is
-  #   `user-key` value is ignored in this case.
+  #   ignored in this case.
   #
   # If these values cannot be obtained from the response, docspell
-  # fails the authentication by denying access. It is then assumed
+  # fails the authentication. It is then assumed that the successfully
-  # that the successfully authenticated user has not enough
+  # authenticated user at the OP has not enough permissions to access
-  # permissions to access docspell.
+  # docspell.
   #
   # Below are examples for OpenID Connect (keycloak) and OAuth2
   # (github).

modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala

@@ -695,7 +695,7 @@ trait Conversions {
       case UpdateResult.Success  => BasicResult(true, successMsg)
       case UpdateResult.NotFound => BasicResult(false, "Not found")
       case UpdateResult.Failure(ex) =>
-        BasicResult(false, s"Internal error: ${ex.getMessage}")
+        BasicResult(false, s"Error: ${ex.getMessage}")
     }
 
   def basicResult(ur: OUpload.UploadResult): BasicResult =
@@ -730,8 +730,8 @@ trait Conversions {
     MimeType(
       header.mediaType.mainType,
       header.mediaType.subType,
-      header.mediaType.extensions
+      None
-    )
+    ).withCharsetName(header.mediaType.extensions.get("charset").getOrElse("unknown"))
 }
 
 object Conversions extends Conversions {

modules/restserver/src/main/scala/docspell/restserver/routes/TotpRoutes.scala

@@ -68,9 +68,13 @@ object TotpRoutes {
           }
         } yield resp
 
-      case POST -> Root / "disable" =>
+      case req @ POST -> Root / "disable" =>
         for {
-          result <- backend.totp.disable(user.account)
+          data <- req.as[OtpConfirm]
+          result <- backend.totp.disable(
+            user.account,
+            OnetimePassword(data.otp.pass).some
+          )
           resp <- Ok(Conversions.basicResult(result, "TOTP setup disabled."))
         } yield resp
     }
@@ -83,7 +87,7 @@ object TotpRoutes {
     HttpRoutes.of { case req @ POST -> Root / "resetOTP" =>
       for {
         data <- req.as[ResetPassword]
-        result <- backend.totp.disable(data.account)
+        result <- backend.totp.disable(data.account, None)
         resp <- Ok(Conversions.basicResult(result, "TOTP setup disabled."))
       } yield resp
     }

modules/restserver/src/main/scala/docspell/restserver/webapp/TemplateRoutes.scala

@@ -11,7 +11,7 @@ import java.util.concurrent.atomic.AtomicReference
 
 import cats.effect._
 import cats.implicits._
-import fs2.{Stream, text}
+import fs2.text
 
 import docspell.restserver.{BuildInfo, Config}
 
@@ -20,6 +20,7 @@ import org.http4s.HttpRoutes
 import org.http4s._
 import org.http4s.dsl.Http4sDsl
 import org.http4s.headers._
+import org.http4s.implicits._
 import org.log4s._
 import yamusca.implicits._
 import yamusca.imports._
@@ -27,8 +28,8 @@ import yamusca.imports._
 object TemplateRoutes {
   private[this] val logger = getLogger
 
-  val `text/html` = new MediaType("text", "html")
+  private val textHtml = mediaType"text/html"
-  val `application/javascript` = new MediaType("application", "javascript")
+  private val appJavascript = mediaType"application/javascript"
 
   trait InnerRoutes[F[_]] {
     def doc: HttpRoutes[F]
@@ -52,7 +53,7 @@ object TemplateRoutes {
             templ <- docTemplate
             resp <- Ok(
               DocData().render(templ),
-              `Content-Type`(`text/html`, Charset.`UTF-8`)
+              `Content-Type`(textHtml, Charset.`UTF-8`)
             )
           } yield resp
         }
@@ -62,7 +63,7 @@ object TemplateRoutes {
             templ <- indexTemplate
             resp <- Ok(
               IndexData(cfg).render(templ),
-              `Content-Type`(`text/html`, Charset.`UTF-8`)
+              `Content-Type`(textHtml, Charset.`UTF-8`)
             )
           } yield resp
         }
@@ -73,7 +74,7 @@ object TemplateRoutes {
             templ <- swTemplate
             resp <- Ok(
               IndexData(cfg).render(templ),
-              `Content-Type`(`application/javascript`, Charset.`UTF-8`)
+              `Content-Type`(appJavascript, Charset.`UTF-8`)
             )
           } yield resp
         }
@@ -89,23 +90,17 @@ object TemplateRoutes {
     }
 
   def loadUrl[F[_]: Sync](url: URL): F[String] =
-    Stream
+    fs2.io
-      .bracket(Sync[F].delay(url.openStream))(in => Sync[F].delay(in.close()))
+      .readInputStream(Sync[F].delay(url.openStream()), 64 * 1024)
-      .flatMap(in => fs2.io.readInputStream(in.pure[F], 64 * 1024, false))
       .through(text.utf8.decode)
       .compile
-      .fold("")(_ + _)
+      .string
 
   def parseTemplate[F[_]: Sync](str: String): F[Template] =
-    Sync[F].delay {
+    Sync[F].pure(mustache.parse(str).leftMap(err => new Exception(err._2))).rethrow
-      mustache.parse(str) match {
-        case Right(t)       => t
-        case Left((_, err)) => sys.error(err)
-      }
-    }
 
   def loadTemplate[F[_]: Sync](url: URL): F[Template] =
-    loadUrl[F](url).flatMap(s => parseTemplate(s)).map { t =>
+    loadUrl[F](url).flatMap(parseTemplate[F]).map { t =>
       logger.info(s"Compiled template $url")
       t
     }

modules/store/src/main/scala/docspell/store/file/FileStore.scala

@@ -13,12 +13,13 @@ import cats.effect._
 import fs2.{Pipe, Stream}
 
 import docspell.common._
+import docspell.files.TikaMimetype
 import docspell.store.records.RFileMeta
 
 import binny._
 import binny.jdbc.{GenericJdbcStore, JdbcStoreConfig}
-import binny.tika.TikaContentTypeDetect
 import doobie._
+import scodec.bits.ByteVector
 
 trait FileStore[F[_]] {
 
@@ -42,8 +43,9 @@ object FileStore {
       chunkSize: Int
   ): FileStore[F] = {
     val attrStore = new AttributeStore[F](xa)
-    val cfg = JdbcStoreConfig("filechunk", chunkSize, TikaContentTypeDetect.default)
+    val cfg = JdbcStoreConfig("filechunk", chunkSize, TikaContentTypeDetect)
-    val binStore = GenericJdbcStore[F](ds, Log4sLogger[F](logger), cfg, attrStore)
+    val log = Logger.log4s[F](logger)
+    val binStore = GenericJdbcStore[F](ds, LoggerAdapter(log), cfg, attrStore)
     new Impl[F](binStore, attrStore)
   }
 
@@ -66,27 +68,24 @@ object FileStore {
         .andThen(_.map(bid => Ident.unsafe(bid.id)))
   }
 
-  private object Log4sLogger {
+  private object LoggerAdapter {
-
+    def apply[F[_]](log: Logger[F]): binny.util.Logger[F] =
-    def apply[F[_]: Sync](log: org.log4s.Logger): binny.util.Logger[F] =
       new binny.util.Logger[F] {
-        override def trace(msg: => String): F[Unit] =
+        override def trace(msg: => String): F[Unit] = log.trace(msg)
-          Sync[F].delay(log.trace(msg))
+        override def debug(msg: => String): F[Unit] = log.debug(msg)
-
+        override def info(msg: => String): F[Unit] = log.info(msg)
-        override def debug(msg: => String): F[Unit] =
+        override def warn(msg: => String): F[Unit] = log.warn(msg)
-          Sync[F].delay(log.debug(msg))
+        override def error(msg: => String): F[Unit] = log.error(msg)
-
+        override def error(ex: Throwable)(msg: => String): F[Unit] = log.error(ex)(msg)
-        override def info(msg: => String): F[Unit] =
-          Sync[F].delay(log.info(msg))
-
-        override def warn(msg: => String): F[Unit] =
-          Sync[F].delay(log.warn(msg))
-
-        override def error(msg: => String): F[Unit] =
-          Sync[F].delay(log.error(msg))
-
-        override def error(ex: Throwable)(msg: => String): F[Unit] =
-          Sync[F].delay(log.error(ex)(msg))
       }
   }
+
+  private object TikaContentTypeDetect extends ContentTypeDetect {
+    override def detect(data: ByteVector, hint: Hint): SimpleContentType =
+      SimpleContentType(
+        TikaMimetype
+          .detect(data, MimeTypeHint(hint.filename, hint.advertisedType))
+          .asString
+      )
+  }
 }

modules/store/src/main/scala/docspell/store/syntax/MimeTypes.scala

@@ -12,11 +12,16 @@ object MimeTypes {
 
   implicit final class EmilMimeTypeOps(emt: emil.MimeType) {
     def toLocal: MimeType =
-      MimeType(emt.primary, emt.sub, emt.params)
+      MimeType(emt.primary, emt.sub, None)
+        .withCharsetName(emt.params.get("charset").getOrElse("unknown"))
   }
 
   implicit final class DocspellMimeTypeOps(mt: MimeType) {
     def toEmil: emil.MimeType =
-      emil.MimeType(mt.primary, mt.sub, mt.params)
+      emil.MimeType(
+        mt.primary,
+        mt.sub,
+        mt.charset.map(cs => Map("charset" -> cs.name())).getOrElse(Map.empty)
+      )
   }
 }

modules/webapp/src/main/elm/Api.elm

@@ -2195,12 +2195,12 @@ confirmOtp flags confirm receive =
         }
 
 
-disableOtp : Flags -> (Result Http.Error BasicResult -> msg) -> Cmd msg
+disableOtp : Flags -> OtpConfirm -> (Result Http.Error BasicResult -> msg) -> Cmd msg
-disableOtp flags receive =
+disableOtp flags otp receive =
     Http2.authPost
         { url = flags.config.baseUrl ++ "/api/v1/sec/user/otp/disable"
         , account = getAccount flags
-        , body = Http.emptyBody
+        , body = Http.jsonBody (Api.Model.OtpConfirm.encode otp)
         , expect = Http.expectJson receive Api.Model.BasicResult.decoder
         }
 

modules/webapp/src/main/elm/Comp/OtpSetup.elm

@@ -58,8 +58,8 @@ initDisabledModel =
 type alias EnabledModel =
     { created : Int
     , loading : Bool
-    , confirmText : String
+    , confirmCode : String
-    , confirmTextWrong : Bool
+    , serverErrorMsg : String
     }
 
 
@@ -67,8 +67,8 @@ initEnabledModel : Int -> EnabledModel
 initEnabledModel created =
     { created = created
     , loading = False
-    , confirmText = ""
+    , confirmCode = ""
-    , confirmTextWrong = False
+    , serverErrorMsg = ""
     }
 
 
@@ -85,7 +85,7 @@ type Msg
     | SecretMsg Comp.PasswordInput.Msg
     | Confirm
     | ConfirmResp (Result Http.Error BasicResult)
-    | SetDisableConfirmText String
+    | SetDisableConfirmCode String
     | Disable
     | DisableResp (Result Http.Error BasicResult)
 
@@ -178,10 +178,10 @@ update flags msg model =
         ConfirmResp (Err err) ->
             ( ConfirmError err, Cmd.none )
 
-        SetDisableConfirmText str ->
+        SetDisableConfirmCode str ->
             case model of
                 StateEnabled m ->
-                    ( StateEnabled { m | confirmText = str }, Cmd.none )
+                    ( StateEnabled { m | confirmCode = str }, Cmd.none )
 
                 _ ->
                     ( model, Cmd.none )
@@ -189,14 +189,10 @@ update flags msg model =
         Disable ->
             case model of
                 StateEnabled m ->
-                    if String.toLower m.confirmText == "ok" then
+                    ( StateEnabled { m | loading = True }
-                        ( StateEnabled { m | confirmTextWrong = False, loading = True }
+                    , Api.disableOtp flags (OtpConfirm m.confirmCode) DisableResp
-                        , Api.disableOtp flags DisableResp
                     )
 
-                    else
-                        ( StateEnabled { m | confirmTextWrong = True }, Cmd.none )
-
                 _ ->
                     ( model, Cmd.none )
 
@@ -205,6 +201,11 @@ update flags msg model =
                 init flags
 
             else
+                case model of
+                    StateEnabled m ->
+                        ( StateEnabled { m | serverErrorMsg = result.message, loading = False }, Cmd.none )
+
+                    _ ->
                         ( model, Cmd.none )
 
         DisableResp (Err err) ->
@@ -253,14 +254,15 @@ viewEnabled texts model =
         , p []
             [ text texts.revert2FAText
             ]
-        , div [ class "flex flex-col items-center mt-6" ]
+        , div [ class "flex flex-col mt-6" ]
             [ div [ class "flex flex-row max-w-md" ]
                 [ input
                     [ type_ "text"
-                    , value model.confirmText
+                    , value model.confirmCode
-                    , onInput SetDisableConfirmText
+                    , onInput SetDisableConfirmCode
                     , class S.textInput
-                    , class "rounded-r-none"
+                    , class "rounded-r-none pl-2 pr-10 py-2 rounded-lg max-w-xs text-center font-mono"
+                    , placeholder "123456"
                     ]
                     []
                 , B.genericButton
@@ -281,9 +283,9 @@ viewEnabled texts model =
             , div
                 [ class S.errorMessage
                 , class "my-2"
-                , classList [ ( "hidden", not model.confirmTextWrong ) ]
+                , classList [ ( "hidden", model.serverErrorMsg == "" ) ]
                 ]
-                [ text texts.disableConfirmErrorMsg
+                [ text texts.codeInvalid
                 ]
             , Markdown.toHtml [ class "mt-2" ] texts.disableConfirmBoxInfo
             ]
@@ -367,7 +369,7 @@ viewDisabled texts model =
                             , class S.errorMessage
                             , class "mt-2"
                             ]
-                            [ text texts.setupCodeInvalid ]
+                            [ text texts.codeInvalid ]
                         , div [ class "mt-6" ]
                             [ p [] [ text texts.ifNotQRCode ]
                             , div [ class "max-w-md mx-auto mt-4" ]

modules/webapp/src/main/elm/Messages/Comp/OtpSetup.elm

@@ -29,14 +29,13 @@ type alias Texts =
     , twoFaActiveSince : String
     , revert2FAText : String
     , disableButton : String
-    , disableConfirmErrorMsg : String
     , disableConfirmBoxInfo : String
     , setupTwoFactorAuth : String
     , setupTwoFactorAuthInfo : String
     , activateButton : String
     , setupConfirmLabel : String
     , scanQRCode : String
-    , setupCodeInvalid : String
+    , codeInvalid : String
     , ifNotQRCode : String
     , reloadToTryAgain : String
     , twoFactorNowActive : String
@@ -57,14 +56,13 @@ gb =
     , twoFaActiveSince = "Two Factor Authentication is active since "
     , revert2FAText = "If you really want to revert back to password-only authentication, you can do this here. You can run the setup any time to enable the second factor again."
     , disableButton = "Disable 2FA"
-    , disableConfirmErrorMsg = "Please type OK if you really want to disable this!"
+    , disableConfirmBoxInfo = "Enter a TOTP code and click the button to disable 2FA."
-    , disableConfirmBoxInfo = "Type `OK` into the text box and click the button to disable 2FA."
     , setupTwoFactorAuth = "Setup Two Factor Authentication"
     , setupTwoFactorAuthInfo = "You can setup a second factor for authentication using a one-time password. When clicking the button a secret is generated that you can load into an app on your mobile device. The app then provides a 6 digit code that you need to pass in the field in order to confirm and finalize the setup."
     , activateButton = "Activate two-factor authentication"
     , setupConfirmLabel = "Confirm"
     , scanQRCode = "Scan this QR code with your device and enter the 6 digit code:"
-    , setupCodeInvalid = "The confirmation code was invalid!"
+    , codeInvalid = "The code was invalid!"
     , ifNotQRCode = "If you cannot use the qr code, enter this secret:"
     , reloadToTryAgain = "If you want to try again, reload the page."
     , twoFactorNowActive = "Two Factor Authentication is now active!"
@@ -85,14 +83,13 @@ de =
     , twoFaActiveSince = "Die Zwei-Faktor-Authentifizierung ist aktiv seit "
     , revert2FAText = "Die Zwei-Faktor-Authentifizierung kann hier wieder deaktiviert werden. Danach kann die Einrichtung wieder von neuem gestartet werden, um 2FA wieder zu aktivieren."
     , disableButton = "Deaktiviere 2FA"
-    , disableConfirmErrorMsg = "Bitte tippe OK ein, um die Zwei-Faktor-Authentifizierung zu deaktivieren."
     , disableConfirmBoxInfo = "Tippe `OK` in das Feld und klicke, um die Zwei-Faktor-Authentifizierung zu deaktivieren."
     , setupTwoFactorAuth = "Zwei-Faktor-Authentifizierung einrichten"
     , setupTwoFactorAuthInfo = "Ein zweiter Faktor zur Authentifizierung mittels eines Einmalkennworts kann eingerichtet werden. Beim Klicken des Button wird ein Schlüssel generiert, der an eine Authentifizierungs-App eines mobilen Gerätes übetragen werden kann. Danach präsentiert die App ein 6-stelliges Kennwort, welches zur Bestätigung und zum Abschluss angegeben werden muss."
     , activateButton = "Zwei-Faktor-Authentifizierung aktivieren"
     , setupConfirmLabel = "Bestätigung"
     , scanQRCode = "Scanne den QR Code mit der Authentifizierungs-App und gebe den 6-stelligen Code ein:"
-    , setupCodeInvalid = "Der Code war ungültig!"
+    , codeInvalid = "Der Code war ungültig!"
     , ifNotQRCode = "Wenn der QR-Code nicht möglich ist, kann der Schlüssel manuell eingegeben werden:"
     , reloadToTryAgain = "Um es noch einmal zu versuchen, bitte die Seite neu laden."
     , twoFactorNowActive = "Die Zwei-Faktor-Authentifizierung ist nun aktiv!"

modules/webapp/src/main/elm/Messages/Page/UserSettings.elm

@@ -84,7 +84,7 @@ gb =
             or to just leave it there. In the latter case you should
             adjust the schedule to avoid reading over the same mails
             again."""
-    , otpMenu = "Two Factor"
+    , otpMenu = "Two Factor Authentication"
     }
 
 

project/Dependencies.scala

@@ -275,8 +275,7 @@ object Dependencies {
 
   val binny = Seq(
     "com.github.eikek" %% "binny-core" % BinnyVersion,
-    "com.github.eikek" %% "binny-jdbc" % BinnyVersion,
+    "com.github.eikek" %% "binny-jdbc" % BinnyVersion
-    "com.github.eikek" %% "binny-tika-detect" % BinnyVersion
   )
 
   // https://github.com/flyway/flyway

website/site/content/docs/configure/_index.md

@@ -342,7 +342,7 @@ The `session-valid` determines how long a token is valid. This can be
 just some minutes, the web application obtains new ones
 periodically. So a rather short time is recommended.
 
-### OpenID Connect / OAuth2
+## OpenID Connect / OAuth2
 
 You can integrate Docspell into your SSO solution via [OpenID
 Connect](https://openid.net/connect/) (OIDC). This requires to set up

website/site/content/docs/features/_index.md

@@ -36,6 +36,8 @@ description = "A list of features and limitations."
 - [OpenID Connect](@/docs/configure/_index.md#openid-connect-oauth2)
   support allows Docspell to integrate into your SSO setup, for
   example with keycloak.
+- Two-Factor Authentication using [TOTP](@/docs/webapp/totp.md) built
+  in
 - mobile-friendly Web-UI with dark and light theme
 - [Create anonymous
   “upload-urls”](@/docs/webapp/uploading.md#anonymous-upload) to
@@ -76,8 +78,6 @@ description = "A list of features and limitations."
     link and send the file to docspell
   - [SMTP Gateway](@/docs/tools/smtpgateway.md): Setup a SMTP server
     that delivers mails directly to docspell.
-  - [Paperless Import](@/docs/tools/paperless-import.md) for importing
-    your data from paperless
 - License: AGPLv3
 
 

website/site/content/docs/webapp/totp.md

@@ -0,0 +1,79 @@
++++
+title = "Two-Factor Authentication"
+weight = 110
+[extra]
+mktoc = true
++++
+
+Docspell has built-in support for two-factor (2FA) authentication
+using
+[TOTP](https://en.wikipedia.org/wiki/Time-based_One-Time_Password)s.
+For anything more, consider a dedicated account management tool and
+[OpenID Connect](@/docs/configure/_index.md#openid-connect-oauth2).
+
+# Setup
+
+A user can enable a TOTP as a second factor in their user settings. It
+is required to have some external device to hold the shared secret. A
+popular way is using your phone. Some Android apps are for example
+[Aegis](https://f-droid.org/en/packages/com.beemdevelopment.aegis/) or
+[andOTP](https://f-droid.org/en/packages/org.shadowice.flocke.andotp/);
+and there are others as well.
+
+In user settings, go to _Two Factor Authentication_ and click on
+_Activate two-factor authentication_. This then shows you a QR code:
+
+{{ figure(file="totp-01.png") }}
+
+Open the app (or whatever you use) and scan the QR code. A new account
+is created and a 6-digit code will be shown to you. Enter this code in
+the box below to confirm.
+
+If you cannot scan the QR code, click on the "eye icon" to reveal the
+secret that you then need to type/copy. This secret will never be
+shown again. Should you loose it (or your device where it is saved),
+you cannot log in anymore. See below for how to get into your account
+in this case.
+
+Once you typed in the code, the 2FA is enabled.
+
+{{ figure(file="totp-02.png") }}
+
+When you now login, a second login form will be shown where you must
+now enter a one time password from the device.
+
+# Remove 2FA
+
+If you go to this page with 2FA enabled (refresh the page after
+finishing the setup), you can disable it. The secret will be removed
+from the database.
+
+It shows a form that allows you to disable 2FA again, but requires you
+to enter a one time password.
+
+{{ figure(file="totp-03.png") }}
+
+If you have successfully disabled 2FA, you'll see the first screen
+where you can activate 2FA. You can remove the account from your
+device. Should you want to go back to 2FA, you need to go through the
+setup again and create a new secret.
+
+# When secret is lost
+
+Should you loose your device where the secret is stored, you cannot
+log into docspell anymore. In this case you can use the [command line
+client](@/docs/tools/cli.md) to execute an admin command that removes
+2FA for a given user.
+
+For this to work, you need to [enable the admin
+endpoint](@/docs/configure/_index.md#admin-endpoint). Then execute the
+`disable-2fa` admin command and specify the complete account.
+
+```
+$ dsc admin -a test123 disable-2fa --account demo
+┌─────────┬──────────────────────┐
+│ success │ message              │
+├─────────┼──────────────────────┤
+│ true    │ TOTP setup disabled. │
+└─────────┴──────────────────────┘
+```