diff --git a/build.sbt b/build.sbt index 09001f92..bd15be3f 100644 --- a/build.sbt +++ b/build.sbt @@ -381,7 +381,7 @@ val store = project libraryDependencies ++= Dependencies.testContainer.map(_ % Test) ) - .dependsOn(common, query.jvm, totp) + .dependsOn(common, query.jvm, totp, files) val extract = project .in(file("modules/extract")) diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OTotp.scala b/modules/backend/src/main/scala/docspell/backend/ops/OTotp.scala index e0842b94..a872fa53 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OTotp.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OTotp.scala @@ -18,13 +18,23 @@ import org.log4s.getLogger trait OTotp[F[_]] { + /** Return whether TOTP is enabled for this account or not. */ def state(accountId: AccountId): F[OtpState] + /** Initializes TOTP by generating a secret and storing it in the database. TOTP is + * still disabled, it must be confirmed in order to be active. + */ def initialize(accountId: AccountId): F[InitResult] + /** Confirms and finishes initialization. TOTP is active after this for the given + * account. + */ def confirmInit(accountId: AccountId, otp: OnetimePassword): F[ConfirmResult] - def disable(accountId: AccountId): F[UpdateResult] + /** Disables TOTP and removes the shared secret. If a otp is specified, it must be + * valid. + */ + def disable(accountId: AccountId, otp: Option[OnetimePassword]): F[UpdateResult] } object OTotp { @@ -133,8 +143,31 @@ object OTotp { } } yield res - def disable(accountId: AccountId): F[UpdateResult] = - UpdateResult.fromUpdate(store.transact(RTotp.setEnabled(accountId, false))) + def disable(accountId: AccountId, otp: Option[OnetimePassword]): F[UpdateResult] = + otp match { + case Some(pw) => + for { + _ <- log.info(s"Validating TOTP, because it is requested to disable it.") + key <- store.transact(RTotp.findEnabledByLogin(accountId, true)) + now <- Timestamp.current[F] + res <- key match { + case None => + UpdateResult.failure(new Exception("TOTP not enabled.")).pure[F] + case Some(r) => + val check = totp.checkPassword(r.secret, pw, now.value) + if (check) + UpdateResult.fromUpdate( + store.transact(RTotp.setEnabled(accountId, false)) + ) + else + log.info(s"TOTP code was invalid. Not disabling it.") *> UpdateResult + .failure(new Exception("Code invalid!")) + .pure[F] + } + } yield res + case None => + UpdateResult.fromUpdate(store.transact(RTotp.setEnabled(accountId, false))) + } def state(accountId: AccountId): F[OtpState] = for { diff --git a/modules/common/src/main/scala/docspell/common/MimeType.scala b/modules/common/src/main/scala/docspell/common/MimeType.scala index c8d76611..61fc7494 100644 --- a/modules/common/src/main/scala/docspell/common/MimeType.scala +++ b/modules/common/src/main/scala/docspell/common/MimeType.scala @@ -9,6 +9,8 @@ package docspell.common import java.nio.charset.Charset import java.nio.charset.StandardCharsets +import scala.util.Try + import cats.data.NonEmptyList import docspell.common.syntax.all._ @@ -16,33 +18,31 @@ import docspell.common.syntax.all._ import io.circe.{Decoder, Encoder} /** A MIME Type impl with just enough features for the use here. */ -case class MimeType(primary: String, sub: String, params: Map[String, String]) { - def withParam(name: String, value: String): MimeType = - copy(params = params.updated(name, value)) +case class MimeType(primary: String, sub: String, charset: Option[Charset]) { def withCharset(cs: Charset): MimeType = - withParam("charset", cs.name()) + copy(charset = Some(cs)) def withUtf8Charset: MimeType = withCharset(StandardCharsets.UTF_8) - def resolveCharset: Option[Charset] = - params.get("charset").flatMap { cs => - if (Charset.isSupported(cs)) Some(Charset.forName(cs)) - else None - } + def withCharsetName(csName: String): MimeType = + if (Try(Charset.isSupported(csName)).getOrElse(false)) + withCharset(Charset.forName(csName)) + else this def charsetOrUtf8: Charset = - resolveCharset.getOrElse(StandardCharsets.UTF_8) + charset.getOrElse(StandardCharsets.UTF_8) def baseType: MimeType = - if (params.isEmpty) this else copy(params = Map.empty) + if (charset.isEmpty) this else copy(charset = None) def asString: String = - if (params.isEmpty) s"$primary/$sub" - else { - val parameters = params.toList.map(t => s"""${t._1}="${t._2}"""").mkString(";") - s"$primary/$sub; $parameters" + charset match { + case Some(cs) => + s"$primary/$sub; charset=\"${cs.name()}\"" + case None => + s"$primary/$sub" } def matches(other: MimeType): Boolean = @@ -53,46 +53,16 @@ case class MimeType(primary: String, sub: String, params: Map[String, String]) { object MimeType { def application(sub: String): MimeType = - MimeType("application", sub, Map.empty) + MimeType("application", sub, None) def text(sub: String): MimeType = - MimeType("text", sub, Map.empty) + MimeType("text", sub, None) def image(sub: String): MimeType = - MimeType("image", sub, Map.empty) + MimeType("image", sub, None) - def parse(str: String): Either[String, MimeType] = { - def parsePrimary: Either[String, (String, String)] = - str.indexOf('/') match { - case -1 => Left(s"Invalid mediatype: $str") - case n => Right(str.take(n) -> str.drop(n + 1)) - } - - def parseSub(s: String): Either[String, (String, String)] = - s.indexOf(';') match { - case -1 => Right((s, "")) - case n => Right((s.take(n), s.drop(n))) - } - - def parseParams(s: String): Map[String, String] = - s.split(';') - .map(_.trim) - .filter(_.nonEmpty) - .toList - .flatMap(p => - p.split("=", 2).toList match { - case a :: b :: Nil => Some((a, b)) - case _ => None - } - ) - .toMap - - for { - pt <- parsePrimary - st <- parseSub(pt._2) - pa = parseParams(st._2) - } yield MimeType(pt._1, st._1, pa) - } + def parse(str: String): Either[String, MimeType] = + Parser.parse(str) def unsafe(str: String): MimeType = parse(str).throwLeft @@ -105,8 +75,9 @@ object MimeType { val tiff = image("tiff") val html = text("html") val plain = text("plain") + val json = application("json") val emls = NonEmptyList.of( - MimeType("message", "rfc822", Map.empty), + MimeType("message", "rfc822", None), application("mbox") ) @@ -158,4 +129,88 @@ object MimeType { implicit val jsonDecoder: Decoder[MimeType] = Decoder.decodeString.emap(parse) + + private object Parser { + def parse(s: String): Either[String, MimeType] = + mimeType(s).map(_._1) + + type Result[A] = Either[String, (A, String)] + type P[A] = String => Result[A] + + private[this] val tokenExtraChars = "+-$%*._~".toSet + + private def seq[A, B, C](pa: P[A], pb: P[B])(f: (A, B) => C): P[C] = + in => + pa(in) match { + case Right((a, resta)) => + pb(resta) match { + case Right((b, restb)) => + Right((f(a, b), restb)) + case left => + left.asInstanceOf[Result[C]] + } + case left => + left.asInstanceOf[Result[C]] + } + + private def takeWhile(p: Char => Boolean): P[String] = + in => { + val (prefix, suffix) = in.span(p) + Right((prefix.trim, suffix.drop(1).trim)) + } + + private def check[A](p: P[A], test: A => Boolean, err: => String): P[A] = + in => + p(in) match { + case r @ Right((a, _)) => + if (test(a)) r else Left(err) + case left => + left + } + + //https://datatracker.ietf.org/doc/html/rfc7230#section-3.2.6 + private def isToken(s: String): Boolean = + s.nonEmpty && s.forall(c => c.isLetterOrDigit || tokenExtraChars.contains(c)) + + private val baseType: P[MimeType] = { + val primary = check( + takeWhile(_ != '/'), + isToken, + "Primary type must be non-empty and contain valid characters" + ) + val sub = check( + takeWhile(_ != ';'), + isToken, + "Subtype must be non-empty and contain valid characters" + ) + seq(primary, sub)((p, s) => MimeType(p.toLowerCase, s.toLowerCase, None)) + } + + //https://datatracker.ietf.org/doc/html/rfc2046#section-4.1.2 + private val charset: P[Option[Charset]] = in => + in.trim.toLowerCase.indexOf("charset=") match { + case -1 => Right((None, in)) + case n => + val csValueStart = in.substring(n + "charset=".length).trim + val csName = csValueStart.indexOf(';') match { + case -1 => unquote(csValueStart).trim + case n => unquote(csValueStart.substring(0, n)).trim + } + if (Charset.isSupported(csName)) Right((Some(Charset.forName(csName)), "")) + else Right((None, "")) + } + + private val mimeType = + seq(baseType, charset)((bt, cs) => bt.copy(charset = cs)) + + private def unquote(s: String): String = { + val len = s.length + if (len == 0 || len == 1) s + else { + if (s.charAt(0) == '"' && s.charAt(len - 1) == '"') + unquote(s.substring(1, len - 1)) + else s + } + } + } } diff --git a/modules/common/src/main/scala/docspell/common/syntax/EitherSyntax.scala b/modules/common/src/main/scala/docspell/common/syntax/EitherSyntax.scala index 0457e232..5415353b 100644 --- a/modules/common/src/main/scala/docspell/common/syntax/EitherSyntax.scala +++ b/modules/common/src/main/scala/docspell/common/syntax/EitherSyntax.scala @@ -10,10 +10,7 @@ trait EitherSyntax { implicit final class LeftStringEitherOps[A](e: Either[String, A]) { def throwLeft: A = - e match { - case Right(a) => a - case Left(err) => sys.error(err) - } + e.fold(sys.error, identity) } implicit final class ThrowableLeftEitherOps[A](e: Either[Throwable, A]) { diff --git a/modules/common/src/main/scala/docspell/common/syntax/StreamSyntax.scala b/modules/common/src/main/scala/docspell/common/syntax/StreamSyntax.scala index 7e05fb17..8e1a4d48 100644 --- a/modules/common/src/main/scala/docspell/common/syntax/StreamSyntax.scala +++ b/modules/common/src/main/scala/docspell/common/syntax/StreamSyntax.scala @@ -14,24 +14,14 @@ import io.circe._ import io.circe.parser._ trait StreamSyntax { - implicit class StringStreamOps[F[_]](s: Stream[F, String]) { - def parseJsonAs[A](implicit d: Decoder[A], F: Sync[F]): F[Either[Throwable, A]] = - s.fold("")(_ + _) - .compile - .last - .map(optStr => + s.compile.string + .map(str => for { - str <- - optStr - .map(_.trim) - .toRight(new Exception("Empty string cannot be parsed into a value")) json <- parse(str).leftMap(_.underlying) value <- json.as[A] } yield value ) - } - } diff --git a/modules/common/src/test/scala/docspell/common/MimeTypeTest.scala b/modules/common/src/test/scala/docspell/common/MimeTypeTest.scala new file mode 100644 index 00000000..2f13c1dc --- /dev/null +++ b/modules/common/src/test/scala/docspell/common/MimeTypeTest.scala @@ -0,0 +1,128 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import java.nio.charset.{Charset, StandardCharsets} + +import scala.jdk.CollectionConverters._ + +import munit.ScalaCheckSuite +import org.scalacheck.Gen +import org.scalacheck.Prop.forAll + +class MimeTypeTest extends ScalaCheckSuite { + + test("asString") { + assertEquals(MimeType.html.asString, "text/html") + assertEquals( + MimeType.html.withCharset(StandardCharsets.ISO_8859_1).asString, + "text/html; charset=\"ISO-8859-1\"" + ) + assertEquals( + MimeType.html.withUtf8Charset.asString, + "text/html; charset=\"UTF-8\"" + ) + } + + test("parse without params") { + assertEquals(MimeType.unsafe("application/pdf"), MimeType.pdf) + assertEquals(MimeType.unsafe("image/jpeg"), MimeType.jpeg) + + assertEquals(MimeType.unsafe("image/jpeg "), MimeType.jpeg) + assertEquals(MimeType.unsafe(" image/jpeg "), MimeType.jpeg) + assertEquals(MimeType.unsafe(" image / jpeg "), MimeType.jpeg) + + assertEquals( + MimeType.unsafe("application/xml+html"), + MimeType.application("xml+html") + ) + assertEquals( + MimeType.unsafe( + "application/vnd.openxmlformats-officedocument.presentationml.viewProps+xml" + ), + MimeType.application( + "vnd.openxmlformats-officedocument.presentationml.viewprops+xml" + ) + ) + assertEquals( + MimeType.unsafe("application/vnd.powerbuilder75-s"), + MimeType.application("vnd.powerbuilder75-s") + ) + } + + test("parse with charset") { + assertEquals( + MimeType.unsafe("text/plain; charset=UTF-8"), + MimeType.plain.withUtf8Charset + ) + assertEquals( + MimeType.unsafe("text/plain; CHARSET=UTF-8"), + MimeType.plain.withUtf8Charset + ) + assertEquals( + MimeType.unsafe("text/plain; CharSet=UTF-8"), + MimeType.plain.withUtf8Charset + ) + assertEquals( + MimeType.unsafe("text/html; charset=\"ISO-8859-1\""), + MimeType.html.withCharset(StandardCharsets.ISO_8859_1) + ) + } + + test("parse with charset and more params") { + assertEquals( + MimeType.unsafe("text/plain; charset=UTF-8; action=test"), + MimeType.plain.withUtf8Charset + ) + assertEquals( + MimeType.unsafe("text/plain; run=\"2\"; charset=UTF-8; action=test"), + MimeType.plain.withUtf8Charset + ) + } + + test("parse without charset but params") { + assertEquals(MimeType.unsafe("image/jpeg; action=urn:2"), MimeType.jpeg) + } + + test("parse some stranger values") { + assertEquals( + MimeType.unsafe("text/plain; charset=\"\"ISO-8859-1\"\""), + MimeType.plain.withCharset(StandardCharsets.ISO_8859_1) + ) + assertEquals( + MimeType.unsafe("text/plain; charset=\"\" ISO-8859-1 \"\""), + MimeType.plain.withCharset(StandardCharsets.ISO_8859_1) + ) + } + + test("parse invalid mime types") { + assert(MimeType.parse("").isLeft) + assert(MimeType.parse("_ _/plain").isLeft) + assert(MimeType.parse("/").isLeft) + assert(MimeType.parse("()").isLeft) + } + + property("read own asString") { + forAll(MimeTypeTest.mimeType) { mt: MimeType => + assertEquals(MimeType.unsafe(mt.asString), mt) + } + } +} + +object MimeTypeTest { + val someTypes = List( + MimeType.plain, + MimeType.html + ) ++ MimeType.emls.toList + + val mimeType = + for { + base <- Gen.atLeastOne(someTypes) + cs <- Gen.someOf(Charset.availableCharsets().values().asScala) + } yield base.head.copy(charset = cs.headOption) + +} diff --git a/modules/files/src/main/scala/docspell/files/TikaMimetype.scala b/modules/files/src/main/scala/docspell/files/TikaMimetype.scala index 7c0be3e5..74eedb60 100644 --- a/modules/files/src/main/scala/docspell/files/TikaMimetype.scala +++ b/modules/files/src/main/scala/docspell/files/TikaMimetype.scala @@ -24,6 +24,7 @@ import org.apache.tika.config.TikaConfig import org.apache.tika.metadata.{HttpHeaders, Metadata, TikaCoreProperties} import org.apache.tika.mime.MediaType import org.apache.tika.parser.txt.Icu4jEncodingDetector +import scodec.bits.ByteVector object TikaMimetype { private val tika = new TikaConfig().getDetector @@ -31,10 +32,10 @@ object TikaMimetype { private def convert(mt: MediaType): MimeType = Option(mt) match { case Some(_) => - val params = mt.getParameters.asScala.toMap + val cs = mt.getParameters.asScala.toMap.get("charset").getOrElse("unknown") val primary = mt.getType val sub = mt.getSubtype - normalize(MimeType(primary, sub, params)) + normalize(MimeType(primary, sub, None).withCharsetName(cs)) case None => MimeType.octetStream } @@ -48,8 +49,8 @@ object TikaMimetype { private def normalize(in: MimeType): MimeType = in match { - case MimeType(_, sub, p) if sub contains "xhtml" => - MimeType.html.copy(params = p) + case MimeType(_, sub, cs) if sub contains "xhtml" => + MimeType.html.copy(charset = cs) case _ => in } @@ -83,10 +84,13 @@ object TikaMimetype { def detect[F[_]: Sync](data: Stream[F, Byte], hint: MimeTypeHint): F[MimeType] = data.take(64).compile.toVector.map(bytes => fromBytes(bytes.toArray, hint)) + def detect(data: ByteVector, hint: MimeTypeHint): MimeType = + fromBytes(data.toArray, hint) + def resolve[F[_]: Sync](dt: DataType, data: Stream[F, Byte]): F[MimeType] = dt match { case DataType.Exact(mt) => - mt.resolveCharset match { + mt.charset match { case None if mt.primary == "text" => detectCharset[F](data, MimeTypeHint.advertised(mt)) .map { diff --git a/modules/files/src/test/scala/docspell/files/TikaMimetypeTest.scala b/modules/files/src/test/scala/docspell/files/TikaMimetypeTest.scala new file mode 100644 index 00000000..1cc10359 --- /dev/null +++ b/modules/files/src/test/scala/docspell/files/TikaMimetypeTest.scala @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.files + +import docspell.common.{MimeType, MimeTypeHint} + +import munit.FunSuite +import scodec.bits.ByteVector + +class TikaMimetypeTest extends FunSuite { + + private def detect(bv: ByteVector, hint: MimeTypeHint): MimeType = + TikaMimetype.detect(bv, hint) + + test("detect text/plain") { + val mt = detect(ByteVector.view("hello world".getBytes), MimeTypeHint.none) + assertEquals(mt.baseType, MimeType.plain) + } + + test("detect image/jpeg") { + val mt = detect( + ByteVector.fromValidBase64("/9j/4AAQSkZJRgABAgAAZABkAAA="), + MimeTypeHint.none + ) + assertEquals(mt, MimeType.jpeg) + } + + test("detect image/png") { + val mt = detect( + ByteVector.fromValidBase64("iVBORw0KGgoAAAANSUhEUgAAA2I="), + MimeTypeHint.none + ) + assertEquals(mt, MimeType.png) + } + + test("detect application/json") { + val mt = + detect( + ByteVector.view("""{"name":"me"}""".getBytes), + MimeTypeHint.filename("me.json") + ) + assertEquals(mt, MimeType.json) + } + + test("detect application/json") { + val mt = detect( + ByteVector.view("""{"name":"me"}""".getBytes), + MimeTypeHint.advertised("application/json") + ) + assertEquals(mt, MimeType.json) + } + + test("detect image/jpeg wrong advertised") { + val mt = detect( + ByteVector.fromValidBase64("/9j/4AAQSkZJRgABAgAAZABkAAA="), + MimeTypeHint.advertised("image/png") + ) + assertEquals(mt, MimeType.jpeg) + } + + test("just filename") { + assertEquals( + detect(ByteVector.empty, MimeTypeHint.filename("doc.pdf")), + MimeType.pdf + ) + } + +} diff --git a/modules/oidc/src/main/scala/docspell/oidc/OpenidConnect.scala b/modules/oidc/src/main/scala/docspell/oidc/OpenidConnect.scala deleted file mode 100644 index 5216b5b9..00000000 --- a/modules/oidc/src/main/scala/docspell/oidc/OpenidConnect.scala +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.oidc - -import org.http4s.HttpRoutes -import org.http4s.client.Client - -object OpenidConnect { - - def codeFlow[F[_]](client: Client[F]): HttpRoutes[F] = - ??? -} diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 34e60298..3ebfaa27 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -1442,11 +1442,18 @@ paths: summary: Disables two factor authentication. description: | Disables two factor authentication for the current user. If - the user has no two factor authentication enabled, this - returns success, too. + the user has no two factor authentication enabled, an error is + returned. + + It requires to specify a valid otp. After this completes successfully, two factor auth can be enabled again by initializing it anew. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/OtpConfirm" responses: 200: description: Ok diff --git a/modules/restserver/src/main/resources/reference.conf b/modules/restserver/src/main/resources/reference.conf index 1e541b47..29b63dc1 100644 --- a/modules/restserver/src/main/resources/reference.conf +++ b/modules/restserver/src/main/resources/reference.conf @@ -66,31 +66,31 @@ docspell.server { # # Multiple authentication providers can be defined. Each is # configured in the array below. The `provider` block gives all - # details necessary to authenticate agains an external OIDC or OAuth - # provider. This requires at least two URLs for OIDC and three for - # OAuth2. The `user-url` is only required for OIDC, if the account - # data is to be retrieved from the user-info endpoint and not from - # the JWT token. The access token is then used to authenticate at - # the provider to obtain user info. Thus, it doesn't need to be - # validated here and therefore no `sign-key` setting is needed. - # However, if you want to extract the account information from the - # access token, it must be validated here and therefore the correct - # signature key and algorithm must be provided. This would save - # another request. If the `sign-key` is left empty, the `user-url` - # is used and must be specified. If the `sign-key` is _not_ empty, - # the response from the authentication provider is validated using - # this key. + # details necessary to authenticate against an external OIDC or + # OAuth provider. This requires at least two URLs for OIDC and three + # for OAuth2. When using OIDC, the `user-url` is only required if + # the account data is to be retrieved from the user-info endpoint + # and not from the JWT token. For the request to the `user-url`, the + # access token is then used to authenticate at the provider. Thus, + # it doesn't need to be validated here and therefore no `sign-key` + # setting is needed. However, if you want to extract the account + # information from the access token, it must be validated here and + # therefore the correct signature key and algorithm must be + # provided. If the `sign-key` is left empty, the `user-url` is used + # and must be specified. If the `sign-key` is _not_ empty, the + # response from the authentication provider is validated using this + # key. # # After successful authentication, docspell needs to create the # account. For this a username and collective name is required. The - # username is defined by the `user-key` setting. The `user-key` is - # used to search the JSON structure, that is obtained from the JWT - # token or the user-info endpoint, for the login name to use. It - # traverses the JSON structure recursively, until it finds an object - # with that key. The first value is used. + # account name is defined by the `user-key` and `collective-key` + # setting. The `user-key` is used to search the JSON structure, that + # is obtained from the JWT token or the user-info endpoint, for the + # login name to use. It traverses the JSON structure recursively, + # until it finds an object with that key. The first value is used. # - # There are the following ways to specify how to retrieve the full - # account id depending on the value of `collective-key`: + # The `collective-key` can be used in multiple ways and both can + # work together to retrieve the full account id: # # - If it starts with `fixed:`, like "fixed:collective", the name # after the `fixed:` prefix is used as collective as is. So all @@ -100,15 +100,15 @@ docspell.server { # value after the prefix is used to search the JSON response for # an object with this key, just like it works with the `user-key`. # - # - If it starts with `account:`, like "account:ds-account", it - # works the same as `lookup:` only that the value is interpreted - # as the full account name of form `collective/login`. The - # `user-key` value is ignored in this case. + # - If it starts with `account:`, like "account:demo", it works the + # same as `lookup:` only that the value is interpreted as the full + # account name of form `collective/login`. The `user-key` value is + # ignored in this case. # # If these values cannot be obtained from the response, docspell - # fails the authentication by denying access. It is then assumed - # that the successfully authenticated user has not enough - # permissions to access docspell. + # fails the authentication. It is then assumed that the successfully + # authenticated user at the OP has not enough permissions to access + # docspell. # # Below are examples for OpenID Connect (keycloak) and OAuth2 # (github). diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala index 48ea6526..7cc03c6b 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -695,7 +695,7 @@ trait Conversions { case UpdateResult.Success => BasicResult(true, successMsg) case UpdateResult.NotFound => BasicResult(false, "Not found") case UpdateResult.Failure(ex) => - BasicResult(false, s"Internal error: ${ex.getMessage}") + BasicResult(false, s"Error: ${ex.getMessage}") } def basicResult(ur: OUpload.UploadResult): BasicResult = @@ -730,8 +730,8 @@ trait Conversions { MimeType( header.mediaType.mainType, header.mediaType.subType, - header.mediaType.extensions - ) + None + ).withCharsetName(header.mediaType.extensions.get("charset").getOrElse("unknown")) } object Conversions extends Conversions { diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/TotpRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/TotpRoutes.scala index 255ce2c2..9b1838eb 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/TotpRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/TotpRoutes.scala @@ -68,9 +68,13 @@ object TotpRoutes { } } yield resp - case POST -> Root / "disable" => + case req @ POST -> Root / "disable" => for { - result <- backend.totp.disable(user.account) + data <- req.as[OtpConfirm] + result <- backend.totp.disable( + user.account, + OnetimePassword(data.otp.pass).some + ) resp <- Ok(Conversions.basicResult(result, "TOTP setup disabled.")) } yield resp } @@ -83,7 +87,7 @@ object TotpRoutes { HttpRoutes.of { case req @ POST -> Root / "resetOTP" => for { data <- req.as[ResetPassword] - result <- backend.totp.disable(data.account) + result <- backend.totp.disable(data.account, None) resp <- Ok(Conversions.basicResult(result, "TOTP setup disabled.")) } yield resp } diff --git a/modules/restserver/src/main/scala/docspell/restserver/webapp/TemplateRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/webapp/TemplateRoutes.scala index 1e0021c3..43ed9a5b 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/webapp/TemplateRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/webapp/TemplateRoutes.scala @@ -11,7 +11,7 @@ import java.util.concurrent.atomic.AtomicReference import cats.effect._ import cats.implicits._ -import fs2.{Stream, text} +import fs2.text import docspell.restserver.{BuildInfo, Config} @@ -20,6 +20,7 @@ import org.http4s.HttpRoutes import org.http4s._ import org.http4s.dsl.Http4sDsl import org.http4s.headers._ +import org.http4s.implicits._ import org.log4s._ import yamusca.implicits._ import yamusca.imports._ @@ -27,8 +28,8 @@ import yamusca.imports._ object TemplateRoutes { private[this] val logger = getLogger - val `text/html` = new MediaType("text", "html") - val `application/javascript` = new MediaType("application", "javascript") + private val textHtml = mediaType"text/html" + private val appJavascript = mediaType"application/javascript" trait InnerRoutes[F[_]] { def doc: HttpRoutes[F] @@ -52,7 +53,7 @@ object TemplateRoutes { templ <- docTemplate resp <- Ok( DocData().render(templ), - `Content-Type`(`text/html`, Charset.`UTF-8`) + `Content-Type`(textHtml, Charset.`UTF-8`) ) } yield resp } @@ -62,7 +63,7 @@ object TemplateRoutes { templ <- indexTemplate resp <- Ok( IndexData(cfg).render(templ), - `Content-Type`(`text/html`, Charset.`UTF-8`) + `Content-Type`(textHtml, Charset.`UTF-8`) ) } yield resp } @@ -73,7 +74,7 @@ object TemplateRoutes { templ <- swTemplate resp <- Ok( IndexData(cfg).render(templ), - `Content-Type`(`application/javascript`, Charset.`UTF-8`) + `Content-Type`(appJavascript, Charset.`UTF-8`) ) } yield resp } @@ -89,23 +90,17 @@ object TemplateRoutes { } def loadUrl[F[_]: Sync](url: URL): F[String] = - Stream - .bracket(Sync[F].delay(url.openStream))(in => Sync[F].delay(in.close())) - .flatMap(in => fs2.io.readInputStream(in.pure[F], 64 * 1024, false)) + fs2.io + .readInputStream(Sync[F].delay(url.openStream()), 64 * 1024) .through(text.utf8.decode) .compile - .fold("")(_ + _) + .string def parseTemplate[F[_]: Sync](str: String): F[Template] = - Sync[F].delay { - mustache.parse(str) match { - case Right(t) => t - case Left((_, err)) => sys.error(err) - } - } + Sync[F].pure(mustache.parse(str).leftMap(err => new Exception(err._2))).rethrow def loadTemplate[F[_]: Sync](url: URL): F[Template] = - loadUrl[F](url).flatMap(s => parseTemplate(s)).map { t => + loadUrl[F](url).flatMap(parseTemplate[F]).map { t => logger.info(s"Compiled template $url") t } diff --git a/modules/store/src/main/scala/docspell/store/file/FileStore.scala b/modules/store/src/main/scala/docspell/store/file/FileStore.scala index 295d4bdf..3afd4216 100644 --- a/modules/store/src/main/scala/docspell/store/file/FileStore.scala +++ b/modules/store/src/main/scala/docspell/store/file/FileStore.scala @@ -13,12 +13,13 @@ import cats.effect._ import fs2.{Pipe, Stream} import docspell.common._ +import docspell.files.TikaMimetype import docspell.store.records.RFileMeta import binny._ import binny.jdbc.{GenericJdbcStore, JdbcStoreConfig} -import binny.tika.TikaContentTypeDetect import doobie._ +import scodec.bits.ByteVector trait FileStore[F[_]] { @@ -42,8 +43,9 @@ object FileStore { chunkSize: Int ): FileStore[F] = { val attrStore = new AttributeStore[F](xa) - val cfg = JdbcStoreConfig("filechunk", chunkSize, TikaContentTypeDetect.default) - val binStore = GenericJdbcStore[F](ds, Log4sLogger[F](logger), cfg, attrStore) + val cfg = JdbcStoreConfig("filechunk", chunkSize, TikaContentTypeDetect) + val log = Logger.log4s[F](logger) + val binStore = GenericJdbcStore[F](ds, LoggerAdapter(log), cfg, attrStore) new Impl[F](binStore, attrStore) } @@ -66,27 +68,24 @@ object FileStore { .andThen(_.map(bid => Ident.unsafe(bid.id))) } - private object Log4sLogger { - - def apply[F[_]: Sync](log: org.log4s.Logger): binny.util.Logger[F] = + private object LoggerAdapter { + def apply[F[_]](log: Logger[F]): binny.util.Logger[F] = new binny.util.Logger[F] { - override def trace(msg: => String): F[Unit] = - Sync[F].delay(log.trace(msg)) - - override def debug(msg: => String): F[Unit] = - Sync[F].delay(log.debug(msg)) - - override def info(msg: => String): F[Unit] = - Sync[F].delay(log.info(msg)) - - override def warn(msg: => String): F[Unit] = - Sync[F].delay(log.warn(msg)) - - override def error(msg: => String): F[Unit] = - Sync[F].delay(log.error(msg)) - - override def error(ex: Throwable)(msg: => String): F[Unit] = - Sync[F].delay(log.error(ex)(msg)) + override def trace(msg: => String): F[Unit] = log.trace(msg) + override def debug(msg: => String): F[Unit] = log.debug(msg) + override def info(msg: => String): F[Unit] = log.info(msg) + override def warn(msg: => String): F[Unit] = log.warn(msg) + override def error(msg: => String): F[Unit] = log.error(msg) + override def error(ex: Throwable)(msg: => String): F[Unit] = log.error(ex)(msg) } } + + private object TikaContentTypeDetect extends ContentTypeDetect { + override def detect(data: ByteVector, hint: Hint): SimpleContentType = + SimpleContentType( + TikaMimetype + .detect(data, MimeTypeHint(hint.filename, hint.advertisedType)) + .asString + ) + } } diff --git a/modules/store/src/main/scala/docspell/store/syntax/MimeTypes.scala b/modules/store/src/main/scala/docspell/store/syntax/MimeTypes.scala index 40efd836..fa55fd99 100644 --- a/modules/store/src/main/scala/docspell/store/syntax/MimeTypes.scala +++ b/modules/store/src/main/scala/docspell/store/syntax/MimeTypes.scala @@ -12,11 +12,16 @@ object MimeTypes { implicit final class EmilMimeTypeOps(emt: emil.MimeType) { def toLocal: MimeType = - MimeType(emt.primary, emt.sub, emt.params) + MimeType(emt.primary, emt.sub, None) + .withCharsetName(emt.params.get("charset").getOrElse("unknown")) } implicit final class DocspellMimeTypeOps(mt: MimeType) { def toEmil: emil.MimeType = - emil.MimeType(mt.primary, mt.sub, mt.params) + emil.MimeType( + mt.primary, + mt.sub, + mt.charset.map(cs => Map("charset" -> cs.name())).getOrElse(Map.empty) + ) } } diff --git a/modules/webapp/src/main/elm/Api.elm b/modules/webapp/src/main/elm/Api.elm index 531125a6..5ae8945d 100644 --- a/modules/webapp/src/main/elm/Api.elm +++ b/modules/webapp/src/main/elm/Api.elm @@ -2195,12 +2195,12 @@ confirmOtp flags confirm receive = } -disableOtp : Flags -> (Result Http.Error BasicResult -> msg) -> Cmd msg -disableOtp flags receive = +disableOtp : Flags -> OtpConfirm -> (Result Http.Error BasicResult -> msg) -> Cmd msg +disableOtp flags otp receive = Http2.authPost { url = flags.config.baseUrl ++ "/api/v1/sec/user/otp/disable" , account = getAccount flags - , body = Http.emptyBody + , body = Http.jsonBody (Api.Model.OtpConfirm.encode otp) , expect = Http.expectJson receive Api.Model.BasicResult.decoder } diff --git a/modules/webapp/src/main/elm/Comp/OtpSetup.elm b/modules/webapp/src/main/elm/Comp/OtpSetup.elm index b453b096..46db0f75 100644 --- a/modules/webapp/src/main/elm/Comp/OtpSetup.elm +++ b/modules/webapp/src/main/elm/Comp/OtpSetup.elm @@ -58,8 +58,8 @@ initDisabledModel = type alias EnabledModel = { created : Int , loading : Bool - , confirmText : String - , confirmTextWrong : Bool + , confirmCode : String + , serverErrorMsg : String } @@ -67,8 +67,8 @@ initEnabledModel : Int -> EnabledModel initEnabledModel created = { created = created , loading = False - , confirmText = "" - , confirmTextWrong = False + , confirmCode = "" + , serverErrorMsg = "" } @@ -85,7 +85,7 @@ type Msg | SecretMsg Comp.PasswordInput.Msg | Confirm | ConfirmResp (Result Http.Error BasicResult) - | SetDisableConfirmText String + | SetDisableConfirmCode String | Disable | DisableResp (Result Http.Error BasicResult) @@ -178,10 +178,10 @@ update flags msg model = ConfirmResp (Err err) -> ( ConfirmError err, Cmd.none ) - SetDisableConfirmText str -> + SetDisableConfirmCode str -> case model of StateEnabled m -> - ( StateEnabled { m | confirmText = str }, Cmd.none ) + ( StateEnabled { m | confirmCode = str }, Cmd.none ) _ -> ( model, Cmd.none ) @@ -189,13 +189,9 @@ update flags msg model = Disable -> case model of StateEnabled m -> - if String.toLower m.confirmText == "ok" then - ( StateEnabled { m | confirmTextWrong = False, loading = True } - , Api.disableOtp flags DisableResp - ) - - else - ( StateEnabled { m | confirmTextWrong = True }, Cmd.none ) + ( StateEnabled { m | loading = True } + , Api.disableOtp flags (OtpConfirm m.confirmCode) DisableResp + ) _ -> ( model, Cmd.none ) @@ -205,7 +201,12 @@ update flags msg model = init flags else - ( model, Cmd.none ) + case model of + StateEnabled m -> + ( StateEnabled { m | serverErrorMsg = result.message, loading = False }, Cmd.none ) + + _ -> + ( model, Cmd.none ) DisableResp (Err err) -> ( DisableError err, Cmd.none ) @@ -253,14 +254,15 @@ viewEnabled texts model = , p [] [ text texts.revert2FAText ] - , div [ class "flex flex-col items-center mt-6" ] + , div [ class "flex flex-col mt-6" ] [ div [ class "flex flex-row max-w-md" ] [ input [ type_ "text" - , value model.confirmText - , onInput SetDisableConfirmText + , value model.confirmCode + , onInput SetDisableConfirmCode , class S.textInput - , class "rounded-r-none" + , class "rounded-r-none pl-2 pr-10 py-2 rounded-lg max-w-xs text-center font-mono" + , placeholder "123456" ] [] , B.genericButton @@ -281,9 +283,9 @@ viewEnabled texts model = , div [ class S.errorMessage , class "my-2" - , classList [ ( "hidden", not model.confirmTextWrong ) ] + , classList [ ( "hidden", model.serverErrorMsg == "" ) ] ] - [ text texts.disableConfirmErrorMsg + [ text texts.codeInvalid ] , Markdown.toHtml [ class "mt-2" ] texts.disableConfirmBoxInfo ] @@ -367,7 +369,7 @@ viewDisabled texts model = , class S.errorMessage , class "mt-2" ] - [ text texts.setupCodeInvalid ] + [ text texts.codeInvalid ] , div [ class "mt-6" ] [ p [] [ text texts.ifNotQRCode ] , div [ class "max-w-md mx-auto mt-4" ] diff --git a/modules/webapp/src/main/elm/Messages/Comp/OtpSetup.elm b/modules/webapp/src/main/elm/Messages/Comp/OtpSetup.elm index 2378301f..3c4112d8 100644 --- a/modules/webapp/src/main/elm/Messages/Comp/OtpSetup.elm +++ b/modules/webapp/src/main/elm/Messages/Comp/OtpSetup.elm @@ -29,14 +29,13 @@ type alias Texts = , twoFaActiveSince : String , revert2FAText : String , disableButton : String - , disableConfirmErrorMsg : String , disableConfirmBoxInfo : String , setupTwoFactorAuth : String , setupTwoFactorAuthInfo : String , activateButton : String , setupConfirmLabel : String , scanQRCode : String - , setupCodeInvalid : String + , codeInvalid : String , ifNotQRCode : String , reloadToTryAgain : String , twoFactorNowActive : String @@ -57,14 +56,13 @@ gb = , twoFaActiveSince = "Two Factor Authentication is active since " , revert2FAText = "If you really want to revert back to password-only authentication, you can do this here. You can run the setup any time to enable the second factor again." , disableButton = "Disable 2FA" - , disableConfirmErrorMsg = "Please type OK if you really want to disable this!" - , disableConfirmBoxInfo = "Type `OK` into the text box and click the button to disable 2FA." + , disableConfirmBoxInfo = "Enter a TOTP code and click the button to disable 2FA." , setupTwoFactorAuth = "Setup Two Factor Authentication" , setupTwoFactorAuthInfo = "You can setup a second factor for authentication using a one-time password. When clicking the button a secret is generated that you can load into an app on your mobile device. The app then provides a 6 digit code that you need to pass in the field in order to confirm and finalize the setup." , activateButton = "Activate two-factor authentication" , setupConfirmLabel = "Confirm" , scanQRCode = "Scan this QR code with your device and enter the 6 digit code:" - , setupCodeInvalid = "The confirmation code was invalid!" + , codeInvalid = "The code was invalid!" , ifNotQRCode = "If you cannot use the qr code, enter this secret:" , reloadToTryAgain = "If you want to try again, reload the page." , twoFactorNowActive = "Two Factor Authentication is now active!" @@ -85,14 +83,13 @@ de = , twoFaActiveSince = "Die Zwei-Faktor-Authentifizierung ist aktiv seit " , revert2FAText = "Die Zwei-Faktor-Authentifizierung kann hier wieder deaktiviert werden. Danach kann die Einrichtung wieder von neuem gestartet werden, um 2FA wieder zu aktivieren." , disableButton = "Deaktiviere 2FA" - , disableConfirmErrorMsg = "Bitte tippe OK ein, um die Zwei-Faktor-Authentifizierung zu deaktivieren." , disableConfirmBoxInfo = "Tippe `OK` in das Feld und klicke, um die Zwei-Faktor-Authentifizierung zu deaktivieren." , setupTwoFactorAuth = "Zwei-Faktor-Authentifizierung einrichten" , setupTwoFactorAuthInfo = "Ein zweiter Faktor zur Authentifizierung mittels eines Einmalkennworts kann eingerichtet werden. Beim Klicken des Button wird ein Schlüssel generiert, der an eine Authentifizierungs-App eines mobilen Gerätes übetragen werden kann. Danach präsentiert die App ein 6-stelliges Kennwort, welches zur Bestätigung und zum Abschluss angegeben werden muss." , activateButton = "Zwei-Faktor-Authentifizierung aktivieren" , setupConfirmLabel = "Bestätigung" , scanQRCode = "Scanne den QR Code mit der Authentifizierungs-App und gebe den 6-stelligen Code ein:" - , setupCodeInvalid = "Der Code war ungültig!" + , codeInvalid = "Der Code war ungültig!" , ifNotQRCode = "Wenn der QR-Code nicht möglich ist, kann der Schlüssel manuell eingegeben werden:" , reloadToTryAgain = "Um es noch einmal zu versuchen, bitte die Seite neu laden." , twoFactorNowActive = "Die Zwei-Faktor-Authentifizierung ist nun aktiv!" diff --git a/modules/webapp/src/main/elm/Messages/Page/UserSettings.elm b/modules/webapp/src/main/elm/Messages/Page/UserSettings.elm index df492a0b..33531d08 100644 --- a/modules/webapp/src/main/elm/Messages/Page/UserSettings.elm +++ b/modules/webapp/src/main/elm/Messages/Page/UserSettings.elm @@ -84,7 +84,7 @@ gb = or to just leave it there. In the latter case you should adjust the schedule to avoid reading over the same mails again.""" - , otpMenu = "Two Factor" + , otpMenu = "Two Factor Authentication" } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 3494f845..a9748e69 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -275,8 +275,7 @@ object Dependencies { val binny = Seq( "com.github.eikek" %% "binny-core" % BinnyVersion, - "com.github.eikek" %% "binny-jdbc" % BinnyVersion, - "com.github.eikek" %% "binny-tika-detect" % BinnyVersion + "com.github.eikek" %% "binny-jdbc" % BinnyVersion ) // https://github.com/flyway/flyway diff --git a/website/site/content/docs/configure/_index.md b/website/site/content/docs/configure/_index.md index 9651f81a..7ebb0950 100644 --- a/website/site/content/docs/configure/_index.md +++ b/website/site/content/docs/configure/_index.md @@ -342,7 +342,7 @@ The `session-valid` determines how long a token is valid. This can be just some minutes, the web application obtains new ones periodically. So a rather short time is recommended. -### OpenID Connect / OAuth2 +## OpenID Connect / OAuth2 You can integrate Docspell into your SSO solution via [OpenID Connect](https://openid.net/connect/) (OIDC). This requires to set up diff --git a/website/site/content/docs/features/_index.md b/website/site/content/docs/features/_index.md index a54920b3..b5855066 100644 --- a/website/site/content/docs/features/_index.md +++ b/website/site/content/docs/features/_index.md @@ -36,6 +36,8 @@ description = "A list of features and limitations." - [OpenID Connect](@/docs/configure/_index.md#openid-connect-oauth2) support allows Docspell to integrate into your SSO setup, for example with keycloak. +- Two-Factor Authentication using [TOTP](@/docs/webapp/totp.md) built + in - mobile-friendly Web-UI with dark and light theme - [Create anonymous “upload-urls”](@/docs/webapp/uploading.md#anonymous-upload) to @@ -76,8 +78,6 @@ description = "A list of features and limitations." link and send the file to docspell - [SMTP Gateway](@/docs/tools/smtpgateway.md): Setup a SMTP server that delivers mails directly to docspell. - - [Paperless Import](@/docs/tools/paperless-import.md) for importing - your data from paperless - License: AGPLv3 diff --git a/website/site/content/docs/webapp/totp-01.png b/website/site/content/docs/webapp/totp-01.png new file mode 100644 index 00000000..6e45101d Binary files /dev/null and b/website/site/content/docs/webapp/totp-01.png differ diff --git a/website/site/content/docs/webapp/totp-02.png b/website/site/content/docs/webapp/totp-02.png new file mode 100644 index 00000000..133ace34 Binary files /dev/null and b/website/site/content/docs/webapp/totp-02.png differ diff --git a/website/site/content/docs/webapp/totp-03.png b/website/site/content/docs/webapp/totp-03.png new file mode 100644 index 00000000..a0a022ac Binary files /dev/null and b/website/site/content/docs/webapp/totp-03.png differ diff --git a/website/site/content/docs/webapp/totp.md b/website/site/content/docs/webapp/totp.md new file mode 100644 index 00000000..8e34e64f --- /dev/null +++ b/website/site/content/docs/webapp/totp.md @@ -0,0 +1,79 @@ ++++ +title = "Two-Factor Authentication" +weight = 110 +[extra] +mktoc = true ++++ + +Docspell has built-in support for two-factor (2FA) authentication +using +[TOTP](https://en.wikipedia.org/wiki/Time-based_One-Time_Password)s. +For anything more, consider a dedicated account management tool and +[OpenID Connect](@/docs/configure/_index.md#openid-connect-oauth2). + +# Setup + +A user can enable a TOTP as a second factor in their user settings. It +is required to have some external device to hold the shared secret. A +popular way is using your phone. Some Android apps are for example +[Aegis](https://f-droid.org/en/packages/com.beemdevelopment.aegis/) or +[andOTP](https://f-droid.org/en/packages/org.shadowice.flocke.andotp/); +and there are others as well. + +In user settings, go to _Two Factor Authentication_ and click on +_Activate two-factor authentication_. This then shows you a QR code: + +{{ figure(file="totp-01.png") }} + +Open the app (or whatever you use) and scan the QR code. A new account +is created and a 6-digit code will be shown to you. Enter this code in +the box below to confirm. + +If you cannot scan the QR code, click on the "eye icon" to reveal the +secret that you then need to type/copy. This secret will never be +shown again. Should you loose it (or your device where it is saved), +you cannot log in anymore. See below for how to get into your account +in this case. + +Once you typed in the code, the 2FA is enabled. + +{{ figure(file="totp-02.png") }} + +When you now login, a second login form will be shown where you must +now enter a one time password from the device. + +# Remove 2FA + +If you go to this page with 2FA enabled (refresh the page after +finishing the setup), you can disable it. The secret will be removed +from the database. + +It shows a form that allows you to disable 2FA again, but requires you +to enter a one time password. + +{{ figure(file="totp-03.png") }} + +If you have successfully disabled 2FA, you'll see the first screen +where you can activate 2FA. You can remove the account from your +device. Should you want to go back to 2FA, you need to go through the +setup again and create a new secret. + +# When secret is lost + +Should you loose your device where the secret is stored, you cannot +log into docspell anymore. In this case you can use the [command line +client](@/docs/tools/cli.md) to execute an admin command that removes +2FA for a given user. + +For this to work, you need to [enable the admin +endpoint](@/docs/configure/_index.md#admin-endpoint). Then execute the +`disable-2fa` admin command and specify the complete account. + +``` +$ dsc admin -a test123 disable-2fa --account demo +┌─────────┬──────────────────────┐ +│ success │ message │ +├─────────┼──────────────────────┤ +│ true │ TOTP setup disabled. │ +└─────────┴──────────────────────┘ +```