Initial version.

Features:

- Upload PDF files let them analyze

- Manage meta data and items

- See processing in webapp

modules/common/src/main/scala/docspell/common/AccountId.scala

@@ -0,0 +1,35 @@
+package docspell.common
+
+case class AccountId(collective: Ident, user: Ident) {
+
+  def asString =
+    s"${collective.id}/${user.id}"
+}
+
+object AccountId {
+  private[this] val sepearatorChars: String = "/\\:"
+
+  def parse(str: String): Either[String, AccountId] = {
+    val input = str.replaceAll("\\s+", "").trim
+    val invalid: Either[String, AccountId] =
+      Left(s"Cannot parse account id: $str")
+
+    def parse0(sep: Char): Either[String, AccountId] =
+      input.indexOf(sep.toInt) match {
+        case n if n > 0 && input.length > 2 =>
+          val coll = input.substring(0, n)
+          val user = input.substring(n + 1)
+          Ident.fromString(coll).
+            flatMap(collId => Ident.fromString(user).
+              map(userId => AccountId(collId, userId)))
+        case _ =>
+          invalid
+      }
+
+    val separated = sepearatorChars.foldRight(invalid) { (c, v) =>
+      v.orElse(parse0(c))
+    }
+
+    separated.orElse(Ident.fromString(str).map(id => AccountId(id, id)))
+  }
+}

modules/common/src/main/scala/docspell/common/Banner.scala

@@ -0,0 +1,34 @@
+package docspell.common
+
+case class Banner( component: String
+                 , version: String
+                 , gitHash: Option[String]
+                 , jdbcUrl: LenientUri
+                 , configFile: Option[String]
+                 , appId: Ident
+                 , baseUrl: LenientUri) {
+
+  private val banner =
+    """______                          _ _
+      ||  _  \                        | | |
+      || | | |___   ___ ___ _ __   ___| | |
+      || | | / _ \ / __/ __| '_ \ / _ \ | |
+      || |/ / (_) | (__\__ \ |_) |  __/ | |
+      ||___/ \___/ \___|___/ .__/ \___|_|_|
+      |                    | |
+      |""".stripMargin +
+    s"""                    |_| v$version (#${gitHash.map(_.take(8)).getOrElse("")})"""
+
+  def render(prefix: String): String = {
+    val text = banner.split('\n').toList ++ List(
+      s"<< $component >>"
+      , s"Id:       ${appId.id}"
+      , s"Base-Url: ${baseUrl.asString}"
+      , s"Database: ${jdbcUrl.asString}"
+      , s"Config:   ${configFile.getOrElse("")}"
+      , ""
+    )
+
+    text.map(line => s"$prefix  $line").mkString("\n")
+  }
+}

modules/common/src/main/scala/docspell/common/BaseJsonCodecs.scala

@@ -0,0 +1,16 @@
+package docspell.common
+
+import java.time.Instant
+
+import io.circe._
+
+object BaseJsonCodecs {
+
+  implicit val encodeInstantEpoch: Encoder[Instant] =
+    Encoder.encodeJavaLong.contramap(_.toEpochMilli)
+
+  implicit val decodeInstantEpoch: Decoder[Instant] =
+    Decoder.decodeLong.map(Instant.ofEpochMilli)
+
+
+}

modules/common/src/main/scala/docspell/common/CollectiveState.scala

@@ -0,0 +1,52 @@
+package docspell.common
+
+import io.circe.{Decoder, Encoder}
+
+sealed trait CollectiveState
+object CollectiveState {
+  val all = List(Active, ReadOnly, Closed, Blocked)
+
+  /** A normal active collective */
+  case object Active extends CollectiveState
+
+  /** A collective may be readonly in cases it is implicitly closed
+    * (e.g. no payment). Users can still see there data and
+    * download, but have no write access. */
+  case object ReadOnly extends CollectiveState
+
+  /** A collective that has been explicitely closed. */
+  case object Closed extends CollectiveState
+
+  /** A collective blocked by a super user, usually some emergency
+    * action. */
+  case object Blocked extends CollectiveState
+
+
+  def fromString(s: String): Either[String, CollectiveState] =
+    s.toLowerCase match {
+      case "active" => Right(Active)
+      case "readonly" => Right(ReadOnly)
+      case "closed" => Right(Closed)
+      case "blocked" => Right(Blocked)
+      case _ => Left(s"Unknown state: $s")
+    }
+
+  def unsafe(str: String): CollectiveState =
+    fromString(str).fold(sys.error, identity)
+
+  def asString(state: CollectiveState): String = state match {
+    case Active => "active"
+    case Blocked => "blocked"
+    case Closed => "closed"
+    case ReadOnly => "readonly"
+  }
+
+
+
+  implicit val collectiveStateEncoder: Encoder[CollectiveState] =
+    Encoder.encodeString.contramap(CollectiveState.asString)
+
+  implicit val collectiveStateDecoder: Decoder[CollectiveState] =
+    Decoder.decodeString.emap(CollectiveState.fromString)
+
+}

modules/common/src/main/scala/docspell/common/ContactKind.scala

@@ -0,0 +1,44 @@
+package docspell.common
+
+import io.circe.{Decoder, Encoder}
+
+sealed trait ContactKind { self: Product =>
+
+  def asString: String = self.productPrefix
+}
+
+object ContactKind {
+  val all = List()
+
+  case object Phone extends ContactKind
+  case object Mobile extends ContactKind
+  case object Fax extends ContactKind
+  case object Email extends ContactKind
+  case object Docspell extends ContactKind
+  case object Website extends ContactKind
+
+  def fromString(s: String): Either[String, ContactKind] =
+    s.toLowerCase match {
+      case "phone" => Right(Phone)
+      case "mobile" => Right(Mobile)
+      case "fax" => Right(Fax)
+      case "email" => Right(Email)
+      case "docspell" => Right(Docspell)
+      case "website" => Right(Website)
+      case _ => Left(s"Not a state value: $s")
+    }
+
+  def unsafe(str: String): ContactKind =
+    fromString(str).fold(sys.error, identity)
+
+  def asString(s: ContactKind): String =
+    s.asString.toLowerCase
+
+
+  implicit val contactKindEncoder: Encoder[ContactKind] =
+    Encoder.encodeString.contramap(_.asString)
+
+  implicit val contactKindDecoder: Decoder[ContactKind] =
+    Decoder.decodeString.emap(ContactKind.fromString)
+
+}

modules/common/src/main/scala/docspell/common/Direction.scala

@@ -0,0 +1,42 @@
+package docspell.common
+
+import io.circe.{Decoder, Encoder}
+
+sealed trait Direction {
+  self: Product =>
+
+  def name: String =
+    productPrefix.toLowerCase
+}
+
+object Direction {
+
+  case object Incoming extends Direction
+  case object Outgoing extends Direction
+
+  def incoming: Direction = Incoming
+  def outgoing: Direction = Outgoing
+
+  def parse(str: String): Either[String, Direction] =
+    str.toLowerCase match {
+      case "incoming" => Right(Incoming)
+      case "outgoing" => Right(Outgoing)
+      case _ => Left(s"No direction: $str")
+    }
+
+  def unsafe(str: String): Direction =
+    parse(str).fold(sys.error, identity)
+
+  def isIncoming(dir: Direction): Boolean =
+    dir == Direction.Incoming
+
+  def isOutgoing(dir: Direction): Boolean =
+    dir == Direction.Outgoing
+
+  implicit val directionEncoder: Encoder[Direction] =
+    Encoder.encodeString.contramap(_.name)
+
+  implicit val directionDecoder: Decoder[Direction] =
+    Decoder.decodeString.emap(Direction.parse)
+
+}

modules/common/src/main/scala/docspell/common/Duration.scala

@@ -0,0 +1,54 @@
+package docspell.common
+
+import cats.implicits._
+import scala.concurrent.duration.{FiniteDuration, Duration => SDur}
+import java.time.{Duration => JDur}
+import java.util.concurrent.TimeUnit
+
+import cats.effect.Sync
+
+case class Duration(nanos: Long) {
+
+  def millis: Long = nanos / 1000000
+
+  def seconds: Long = millis / 1000
+
+  def toScala: FiniteDuration =
+    FiniteDuration(nanos, TimeUnit.NANOSECONDS)
+
+  def toJava: JDur =
+    JDur.ofNanos(nanos)
+
+  def formatExact: String =
+    s"$millis ms"
+}
+
+object Duration {
+
+  def apply(d: SDur): Duration =
+    Duration(d.toNanos)
+
+  def apply(d: JDur): Duration =
+    Duration(d.toNanos)
+
+  def seconds(n: Long): Duration =
+    apply(JDur.ofSeconds(n))
+
+  def millis(n: Long): Duration =
+    apply(JDur.ofMillis(n))
+
+  def minutes(n: Long): Duration =
+    apply(JDur.ofMinutes(n))
+
+  def hours(n: Long): Duration =
+    apply(JDur.ofHours(n))
+
+  def nanos(n: Long): Duration =
+    Duration(n)
+
+  def stopTime[F[_]: Sync]: F[F[Duration]] =
+    for {
+      now <- Timestamp.current[F]
+      end  = Timestamp.current[F]
+    } yield end.map(e => Duration.millis(e.toMillis - now.toMillis))
+}

modules/common/src/main/scala/docspell/common/IdRef.scala

@@ -0,0 +1,16 @@
+package docspell.common
+
+import io.circe._
+import io.circe.generic.semiauto._
+
+case class IdRef(id: Ident, name: String) {
+
+}
+
+object IdRef {
+
+  implicit val jsonEncoder: Encoder[IdRef] =
+    deriveEncoder[IdRef]
+  implicit val jsonDecoder: Decoder[IdRef] =
+    deriveDecoder[IdRef]
+}

modules/common/src/main/scala/docspell/common/Ident.scala

@@ -0,0 +1,57 @@
+package docspell.common
+
+import java.security.SecureRandom
+import java.util.UUID
+
+import cats.effect.Sync
+import io.circe.{Decoder, Encoder}
+import scodec.bits.ByteVector
+
+case class Ident(id: String) {
+}
+
+object Ident {
+
+  val chars: Set[Char] = (('A' to 'Z') ++ ('a' to 'z') ++ ('0' to '9') ++ "-_").toSet
+
+  def randomUUID[F[_]: Sync]: F[Ident] =
+    Sync[F].delay(unsafe(UUID.randomUUID.toString))
+
+  def randomId[F[_]: Sync]: F[Ident] = Sync[F].delay {
+    val random = new SecureRandom()
+    val buffer = new Array[Byte](32)
+    random.nextBytes(buffer)
+    unsafe(ByteVector.view(buffer).toBase58.grouped(11).mkString("-"))
+  }
+
+  def apply(str: String): Either[String, Ident] =
+    fromString(str)
+
+  def fromString(s: String): Either[String, Ident] =
+    if (s.forall(chars.contains)) Right(new Ident(s))
+    else Left(s"Invalid identifier: $s. Allowed chars: ${chars.mkString}")
+
+  def fromBytes(bytes: ByteVector): Ident =
+    unsafe(bytes.toBase58)
+
+  def fromByteArray(bytes: Array[Byte]): Ident =
+    fromBytes(ByteVector.view(bytes))
+
+  def unsafe(s: String): Ident =
+    fromString(s) match {
+      case Right(id) => id
+      case Left(err) => sys.error(err)
+    }
+
+  def unapply(arg: String): Option[Ident] =
+    fromString(arg).toOption
+
+
+
+  implicit val encodeIdent: Encoder[Ident] =
+    Encoder.encodeString.contramap(_.id)
+
+  implicit val decodeIdent: Decoder[Ident] =
+    Decoder.decodeString.emap(Ident.fromString)
+
+}

modules/common/src/main/scala/docspell/common/ItemState.scala

@@ -0,0 +1,35 @@
+package docspell.common
+
+import io.circe.{Decoder, Encoder}
+
+sealed trait ItemState { self: Product =>
+
+  final def name: String =
+    productPrefix.toLowerCase
+}
+
+object ItemState {
+
+  case object Premature extends ItemState
+  case object Processing extends ItemState
+  case object Created extends ItemState
+  case object Confirmed extends ItemState
+
+  def fromString(str: String): Either[String, ItemState] =
+    str.toLowerCase match {
+      case "premature" => Right(Premature)
+      case "processing" => Right(Processing)
+      case "created" => Right(Created)
+      case "confirmed" => Right(Confirmed)
+      case _ => Left(s"Invalid item state: $str")
+    }
+
+  def unsafe(str: String): ItemState =
+    fromString(str).fold(sys.error, identity)
+
+  implicit val jsonDecoder: Decoder[ItemState] =
+    Decoder.decodeString.emap(fromString)
+  implicit val jsonEncoder: Encoder[ItemState] =
+    Encoder.encodeString.contramap(_.name)
+}
+

modules/common/src/main/scala/docspell/common/JobState.scala

@@ -0,0 +1,70 @@
+package docspell.common
+
+import io.circe.{Decoder, Encoder}
+
+sealed trait JobState { self: Product =>
+  def name: String =
+    productPrefix.toLowerCase
+}
+
+object JobState {
+
+  /** Waiting for being executed. */
+  case object Waiting extends JobState {
+  }
+
+  /** A scheduler has picked up this job and will pass it to the next
+    * free slot. */
+  case object Scheduled extends JobState {
+  }
+
+  /** Is currently executing */
+  case object Running extends JobState {
+  }
+
+  /** Finished with failure and is being retried. */
+  case object Stuck extends JobState {
+  }
+
+  /** Finished finally with a failure */
+  case object Failed extends JobState {
+  }
+
+  /** Finished by cancellation. */
+  case object Cancelled extends JobState {
+  }
+
+  /** Finished with success */
+  case object Success extends JobState {
+  }
+
+  val all: Set[JobState] = Set(Waiting, Scheduled, Running, Stuck, Failed, Cancelled, Success)
+  val queued: Set[JobState] = Set(Waiting, Scheduled, Stuck)
+  val done: Set[JobState] = Set(Failed, Cancelled, Success)
+
+  def parse(str: String): Either[String, JobState] =
+    str.toLowerCase match {
+      case "waiting" => Right(Waiting)
+      case "scheduled" => Right(Scheduled)
+      case "running" => Right(Running)
+      case "stuck" => Right(Stuck)
+      case "failed" => Right(Failed)
+      case "cancelled" => Right(Cancelled)
+      case "success" => Right(Success)
+      case _ => Left(s"Not a job state: $str")
+    }
+
+  def unsafe(str: String): JobState =
+    parse(str).fold(sys.error, identity)
+
+  def asString(state: JobState): String =
+    state.name
+
+
+  implicit val jobStateEncoder: Encoder[JobState] =
+    Encoder.encodeString.contramap(_.name)
+
+  implicit val jobStateDecoder: Decoder[JobState] =
+    Decoder.decodeString.emap(JobState.parse)
+
+}

modules/common/src/main/scala/docspell/common/Language.scala

@@ -0,0 +1,46 @@
+package docspell.common
+
+import io.circe.{Decoder, Encoder}
+
+sealed trait Language { self: Product =>
+
+  final def name: String =
+    productPrefix.toLowerCase
+
+  def iso2: String
+
+  def iso3: String
+
+  private[common] def allNames =
+    Set(name, iso3, iso2)
+}
+
+object Language {
+
+  case object German extends Language {
+    val iso2 = "de"
+    val iso3 = "deu"
+  }
+
+  case object English extends Language {
+    val iso2 = "en"
+    val iso3 = "eng"
+  }
+
+  val all: List[Language] = List(German, English)
+
+  def fromString(str: String): Either[String, Language] = {
+    val lang = str.toLowerCase
+    all.find(_.allNames.contains(lang)).
+      toRight(s"Unsupported or invalid language: $str")
+  }
+
+  def unsafe(str: String): Language =
+    fromString(str).fold(sys.error, identity)
+
+
+  implicit val jsonDecoder: Decoder[Language] =
+    Decoder.decodeString.emap(fromString)
+  implicit val jsonEncoder: Encoder[Language] =
+    Encoder.encodeString.contramap(_.iso3)
+}

modules/common/src/main/scala/docspell/common/LenientUri.scala

@@ -0,0 +1,186 @@
+package docspell.common
+
+import java.net.URL
+
+import fs2.Stream
+import cats.implicits._
+import cats.data.NonEmptyList
+import cats.effect.{Blocker, ContextShift, Sync}
+import docspell.common.LenientUri.Path
+import io.circe.{Decoder, Encoder}
+
+/** A URI.
+  *
+  * It is not compliant to rfc3986, but covers most use cases in a convenient way.
+  */
+case class LenientUri(scheme: NonEmptyList[String]
+  , authority: Option[String]
+  , path: LenientUri.Path
+  , query: Option[String]
+  , fragment: Option[String]) {
+
+  def /(segment: String): LenientUri =
+    copy(path = path / segment)
+
+  def ++ (np: Path): LenientUri =
+    copy(path = np.segments.foldLeft(path)(_ / _))
+
+  def ++ (np: String): LenientUri = {
+    val rel = LenientUri.stripLeading(np, '/')
+    ++(LenientUri.unsafe(s"a:$rel").path)
+  }
+
+  def toJavaUrl: Either[String, URL] =
+    Either.catchNonFatal(new URL(asString)).left.map(_.getMessage)
+
+  def readURL[F[_]: Sync : ContextShift](chunkSize: Int, blocker: Blocker): Stream[F, Byte] =
+    Stream.emit(Either.catchNonFatal(new URL(asString))).
+      covary[F].
+      rethrow.
+      flatMap(url => fs2.io.readInputStream(Sync[F].delay(url.openStream()), chunkSize, blocker, true))
+
+  def asString: String = {
+    val schemePart = scheme.toList.mkString(":")
+    val authPart = authority.map(a => s"//$a").getOrElse("")
+    val pathPart = path.asString
+    val queryPart = query.map(q => s"?$q").getOrElse("")
+    val fragPart = fragment.map(f => s"#$f").getOrElse("")
+    s"$schemePart:$authPart$pathPart$queryPart$fragPart"
+  }
+}
+
+object LenientUri {
+
+  sealed trait Path {
+    def segments: List[String]
+    def isRoot: Boolean
+    def isEmpty: Boolean
+    def /(segment: String): Path
+    def asString: String
+  }
+  case object RootPath extends Path {
+    val segments = Nil
+    val isRoot = true
+    val isEmpty = false
+    def /(seg: String): Path =
+      NonEmptyPath(NonEmptyList.of(seg))
+    def asString = "/"
+  }
+  case object EmptyPath extends Path {
+    val segments = Nil
+    val isRoot = false
+    val isEmpty = true
+    def /(seg: String): Path =
+      NonEmptyPath(NonEmptyList.of(seg))
+    def asString = ""
+  }
+  case class NonEmptyPath(segs: NonEmptyList[String]) extends Path {
+    def segments = segs.toList
+    val isEmpty = false
+    val isRoot = false
+    def /(seg: String): Path =
+      copy(segs = segs.append(seg))
+    def asString = segs.head match {
+      case "." => segments.map(percentEncode).mkString("/")
+      case ".." => segments.map(percentEncode).mkString("/")
+      case _ => "/" + segments.map(percentEncode).mkString("/")
+    }
+  }
+
+  def unsafe(str: String): LenientUri =
+    parse(str).fold(sys.error, identity)
+
+  def fromJava(u: URL): LenientUri =
+    unsafe(u.toExternalForm)
+
+  def parse(str: String): Either[String, LenientUri] = {
+    def makePath(str: String): Path = str.trim match {
+      case "/" => RootPath
+      case "" => EmptyPath
+      case _ => NonEmptyList.fromList(stripLeading(str, '/').split('/').toList.map(percentDecode)) match {
+        case Some(nl) => NonEmptyPath(nl)
+        case None => sys.error(s"Invalid url: $str")
+      }
+    }
+
+    def makeNonEmpty(str: String): Option[String] =
+      Option(str).filter(_.nonEmpty)
+    def makeScheme(s: String): Option[NonEmptyList[String]] =
+      NonEmptyList.fromList(s.split(':').toList.filter(_.nonEmpty).map(_.toLowerCase))
+
+    def splitPathQF(pqf: String): (Path, Option[String], Option[String]) =
+      pqf.indexOf('?') match {
+        case -1 =>
+          pqf.indexOf('#') match {
+            case -1 =>
+              (makePath(pqf), None, None)
+            case n =>
+              (makePath(pqf.substring(0, n)), None, makeNonEmpty(pqf.substring(n + 1)))
+          }
+        case n =>
+          pqf.indexOf('#', n) match {
+            case -1 =>
+              (makePath(pqf.substring(0, n)), makeNonEmpty(pqf.substring(n+1)), None)
+            case k =>
+              (makePath(pqf.substring(0, n)), makeNonEmpty(pqf.substring(n+1, k)), makeNonEmpty(pqf.substring(k+1)))
+          }
+      }
+
+    str.split("//", 2) match {
+      case Array(p0, p1) =>
+        // scheme:scheme:authority/path
+        val scheme = makeScheme(p0)
+        val (auth, pathQF) = p1.indexOf('/') match {
+          case -1 => (Some(p1), "")
+          case n => (Some(p1.substring(0, n)), p1.substring(n))
+        }
+        val (path, query, frag) = splitPathQF(pathQF)
+        scheme match {
+          case None =>
+            Left(s"No scheme found: $str")
+          case Some(nl) =>
+            Right(LenientUri(nl, auth, path, query, frag))
+        }
+      case Array(p0) =>
+        // scheme:scheme:path
+        p0.lastIndexOf(':') match {
+          case -1 =>
+            Left(s"No scheme found: $str")
+          case n =>
+            val scheme = makeScheme(p0.substring(0, n))
+            val (path, query, frag) = splitPathQF(p0.substring(n + 1))
+            scheme match {
+              case None =>
+                Left(s"No scheme found: $str")
+              case Some(nl) =>
+                Right(LenientUri(nl, None, path, query, frag))
+            }
+        }
+    }
+  }
+
+  private[this] val delims: Set[Char] = ":/?#[]@".toSet
+
+  private def percentEncode(s: String): String =
+    s.flatMap(c => if (delims.contains(c)) s"%${c.toInt.toHexString}" else c.toString)
+
+  private def percentDecode(s: String): String =
+    if (!s.contains("%")) s
+    else s.foldLeft(("", "")) { case ((acc, res), c) =>
+        if (acc.length == 2) ("", res :+ Integer.parseInt(acc.drop(1) :+ c, 16).toChar)
+        else if (acc.startsWith("%")) (acc :+ c, res)
+        else if (c == '%') ("%", res)
+        else (acc, res :+ c)
+    }._2
+
+  private def stripLeading(s: String, c: Char): String =
+    if (s.length > 0 && s.charAt(0) == c) s.substring(1)
+    else s
+
+
+  implicit val encodeLenientUri: Encoder[LenientUri] =
+    Encoder.encodeString.contramap(_.asString)
+
+  implicit val decodeLenientUri: Decoder[LenientUri] =
+    Decoder.decodeString.emap(LenientUri.parse)
+}

modules/common/src/main/scala/docspell/common/LogLevel.scala

@@ -0,0 +1,44 @@
+package docspell.common
+
+import io.circe.{Decoder, Encoder}
+
+sealed trait LogLevel { self: Product =>
+  def toInt: Int
+  final def name: String =
+    productPrefix.toLowerCase
+}
+
+object LogLevel {
+
+  case object Debug extends LogLevel { val toInt = 0 }
+  case object Info extends LogLevel { val toInt = 1 }
+  case object Warn extends LogLevel { val toInt = 2 }
+  case object Error extends LogLevel { val toInt = 3 }
+
+  def fromInt(n: Int): LogLevel =
+    n match {
+      case 0 => Debug
+      case 1 => Info
+      case 2 => Warn
+      case 3 => Error
+      case _ => Debug
+    }
+
+  def fromString(str: String): Either[String, LogLevel] =
+    str.toLowerCase match {
+      case "debug" => Right(Debug)
+      case "info" => Right(Info)
+      case "warn" => Right(Warn)
+      case "warning" => Right(Warn)
+      case "error" => Right(Error)
+      case _ => Left(s"Invalid log-level: $str")
+    }
+
+  def unsafeString(str: String): LogLevel =
+    fromString(str).fold(sys.error, identity)
+
+  implicit val jsonDecoder: Decoder[LogLevel] =
+    Decoder.decodeString.emap(fromString)
+  implicit val jsonEncoder: Encoder[LogLevel] =
+    Encoder.encodeString.contramap(_.name)
+}

modules/common/src/main/scala/docspell/common/MetaProposal.scala

@@ -0,0 +1,44 @@
+package docspell.common
+
+import cats.data.NonEmptyList
+import docspell.common.MetaProposal.Candidate
+import io.circe._
+import io.circe.generic.semiauto._
+
+case class MetaProposal(proposalType: MetaProposalType, values: NonEmptyList[Candidate]) {
+
+  def addIdRef(refs: Seq[Candidate]): MetaProposal =
+    copy(values = MetaProposal.flatten(values ++ refs.toList))
+
+  def isSingleValue: Boolean =
+    values.tail.isEmpty
+
+  def isMultiValue: Boolean =
+    !isSingleValue
+
+  def size: Int =
+    values.size
+}
+
+object MetaProposal {
+
+  case class Candidate(ref: IdRef, origin: Set[NerLabel])
+  object Candidate {
+    implicit val jsonEncoder: Encoder[Candidate] =
+      deriveEncoder[Candidate]
+    implicit val jsonDecoder: Decoder[Candidate] =
+      deriveDecoder[Candidate]
+  }
+
+  def flatten(s: NonEmptyList[Candidate]): NonEmptyList[Candidate] = {
+    def append(list: List[Candidate]): Candidate =
+      list.reduce((l0, l1) => l0.copy(origin = l0.origin ++ l1.origin))
+    val grouped = s.toList.groupBy(_.ref.id)
+    NonEmptyList.fromListUnsafe(grouped.values.toList.map(append))
+  }
+
+  implicit val jsonDecoder: Decoder[MetaProposal] =
+    deriveDecoder[MetaProposal]
+  implicit val jsonEncoder: Encoder[MetaProposal] =
+    deriveEncoder[MetaProposal]
+}

modules/common/src/main/scala/docspell/common/MetaProposalList.scala

@@ -0,0 +1,82 @@
+package docspell.common
+
+import cats.data.NonEmptyList
+import cats.kernel.Monoid
+import docspell.common.MetaProposal.Candidate
+import io.circe._
+import io.circe.generic.semiauto._
+
+case class MetaProposalList private (proposals: List[MetaProposal]) {
+
+  def isEmpty: Boolean = proposals.isEmpty
+  def nonEmpty: Boolean = proposals.nonEmpty
+
+  def hasResults(mt: MetaProposalType, mts: MetaProposalType*): Boolean = {
+    (mts :+ mt).map(mtp => proposals.exists(_.proposalType == mtp)).
+      reduce(_ && _)
+  }
+
+  def hasResultsAll: Boolean =
+    proposals.map(_.proposalType).toSet == MetaProposalType.all.toSet
+
+  def getTypes: Set[MetaProposalType] =
+    proposals.foldLeft(Set.empty[MetaProposalType])(_ + _.proposalType)
+
+  def fillEmptyFrom(ml: MetaProposalList): MetaProposalList = {
+    val list = ml.proposals.foldLeft(proposals){ (mine, mp) =>
+      if (hasResults(mp.proposalType)) mine
+      else mp :: mine
+    }
+    new MetaProposalList(list)
+  }
+
+  def find(mpt: MetaProposalType): Option[MetaProposal] =
+    proposals.find(_.proposalType == mpt)
+
+}
+
+object MetaProposalList {
+  val empty = MetaProposalList(Nil)
+
+  def apply(lmp: List[MetaProposal]): MetaProposalList =
+    flatten(lmp.map(m => new MetaProposalList(List(m))))
+
+  def of(mps: MetaProposal*): MetaProposalList =
+    flatten(mps.toList.map(mp => MetaProposalList(List(mp))))
+
+  def from(mt: MetaProposalType, label: NerLabel)(refs: Seq[IdRef]): MetaProposalList =
+    fromSeq1(mt, refs.map(ref => Candidate(ref, Set(label))))
+
+  def fromSeq1(mt: MetaProposalType, refs: Seq[Candidate]): MetaProposalList =
+    NonEmptyList.fromList(refs.toList).
+      map(nl => MetaProposalList.of(MetaProposal(mt, nl))).
+      getOrElse(empty)
+
+  def fromMap(m: Map[MetaProposalType, MetaProposal]): MetaProposalList = {
+    new MetaProposalList(m.toList.map({ case (k, v) => v.copy(proposalType = k) }))
+  }
+
+  def flatten(ml: Seq[MetaProposalList]): MetaProposalList = {
+    val init: Map[MetaProposalType, MetaProposal] = Map.empty
+
+    def updateMap(map: Map[MetaProposalType, MetaProposal], mp: MetaProposal): Map[MetaProposalType, MetaProposal] =
+      map.get(mp.proposalType) match {
+        case Some(mp0) => map.updated(mp.proposalType, mp0.addIdRef(mp.values.toList))
+        case None => map.updated(mp.proposalType, mp)
+      }
+
+    val merged = ml.foldLeft(init) { (map, el) =>
+      el.proposals.foldLeft(map)(updateMap)
+    }
+
+    fromMap(merged)
+  }
+
+  implicit val jsonEncoder: Encoder[MetaProposalList] =
+    deriveEncoder[MetaProposalList]
+  implicit val jsonDecoder: Decoder[MetaProposalList] =
+    deriveDecoder[MetaProposalList]
+
+  implicit val metaProposalListMonoid: Monoid[MetaProposalList] =
+    Monoid.instance(empty, (m0, m1) => flatten(Seq(m0, m1)))
+}

modules/common/src/main/scala/docspell/common/MetaProposalType.scala

@@ -0,0 +1,41 @@
+package docspell.common
+
+import io.circe._
+
+sealed trait MetaProposalType { self: Product =>
+
+  final def name: String =
+    productPrefix.toLowerCase
+}
+
+object MetaProposalType {
+
+  case object CorrOrg extends MetaProposalType
+  case object CorrPerson extends MetaProposalType
+  case object ConcPerson extends MetaProposalType
+  case object ConcEquip extends MetaProposalType
+  case object DocDate extends MetaProposalType
+  case object DueDate extends MetaProposalType
+
+  val all: List[MetaProposalType] =
+    List(CorrOrg, CorrPerson, ConcPerson, ConcEquip)
+
+  def fromString(str: String): Either[String, MetaProposalType] =
+    str.toLowerCase match {
+      case "corrorg" => Right(CorrOrg)
+      case "corrperson" => Right(CorrPerson)
+      case "concperson" => Right(ConcPerson)
+      case "concequip" => Right(ConcEquip)
+      case "docdate" => Right(DocDate)
+      case "duedate" => Right(DueDate)
+      case _ => Left(s"Invalid item-proposal-type: $str")
+    }
+
+  def unsafe(str: String): MetaProposalType =
+    fromString(str).fold(sys.error, identity)
+
+  implicit val jsonDecoder: Decoder[MetaProposalType] =
+    Decoder.decodeString.emap(fromString)
+  implicit val jsonEncoder: Encoder[MetaProposalType] =
+    Encoder.encodeString.contramap(_.name)
+}

modules/common/src/main/scala/docspell/common/MimeType.scala

@@ -0,0 +1,62 @@
+package docspell.common
+
+import docspell.common.syntax.all._
+import io.circe.{Decoder, Encoder}
+
+/** A MIME Type impl with just enough features for the use here.
+  */
+case class MimeType(primary: String, sub: String) {
+
+  def asString: String =
+    s"$primary/$sub"
+
+  def matches(other: MimeType): Boolean =
+    primary == other.primary &&
+      (sub == other.sub || sub == "*" )
+}
+
+object MimeType {
+
+  def application(sub: String): MimeType =
+    MimeType("application", partFromString(sub).throwLeft)
+
+  def text(sub: String): MimeType =
+    MimeType("text", partFromString(sub).throwLeft)
+
+  def image(sub: String): MimeType =
+    MimeType("image", partFromString(sub).throwLeft)
+
+  private[this] val validChars: Set[Char] = (('A' to 'Z') ++ ('a' to 'z') ++ ('0' to '9') ++ "*-").toSet
+
+  def parse(str: String): Either[String, MimeType] = {
+    str.indexOf('/') match {
+      case -1 => Left(s"Invalid MIME type: $str")
+      case n =>
+        for {
+          prim <- partFromString(str.substring(0, n))
+          sub  <- partFromString(str.substring(n + 1))
+        } yield MimeType(prim.toLowerCase, sub.toLowerCase)
+    }
+  }
+
+  def unsafe(str: String): MimeType =
+    parse(str).throwLeft
+
+  private def partFromString(s: String): Either[String, String] =
+    if (s.forall(validChars.contains)) Right(s)
+    else Left(s"Invalid identifier: $s. Allowed chars: ${validChars.mkString}")
+
+  val octetStream = application("octet-stream")
+  val pdf = application("pdf")
+  val png = image("png")
+  val jpeg = image("jpeg")
+  val tiff = image("tiff")
+  val html = text("html")
+  val plain = text("plain")
+
+  implicit val jsonEncoder: Encoder[MimeType] =
+    Encoder.encodeString.contramap(_.asString)
+
+  implicit val jsonDecoder: Decoder[MimeType] =
+    Decoder.decodeString.emap(parse)
+}

modules/common/src/main/scala/docspell/common/NerDateLabel.scala

@@ -0,0 +1,7 @@
+package docspell.common
+
+import java.time.LocalDate
+
+case class NerDateLabel(date: LocalDate, label: NerLabel) {
+
+}

modules/common/src/main/scala/docspell/common/NerLabel.scala

@@ -0,0 +1,13 @@
+package docspell.common
+
+import io.circe.generic.semiauto._
+import io.circe.{Decoder, Encoder}
+
+case class NerLabel(label: String, tag: NerTag, startPosition: Int, endPosition: Int) {
+
+}
+
+object NerLabel {
+  implicit val jsonEncoder: Encoder[NerLabel] = deriveEncoder[NerLabel]
+  implicit val jsonDecoder: Decoder[NerLabel] = deriveDecoder[NerLabel]
+}

modules/common/src/main/scala/docspell/common/NerTag.scala

@@ -0,0 +1,43 @@
+package docspell.common
+
+import io.circe.{Decoder, Encoder}
+
+sealed trait NerTag { self: Product =>
+
+  final def name: String =
+    productPrefix.toLowerCase
+}
+
+object NerTag {
+
+  case object Organization extends NerTag
+  case object Person extends NerTag
+  case object Location extends NerTag
+  case object Misc extends NerTag
+  case object Email extends NerTag
+  case object Website extends NerTag
+  case object Date extends NerTag
+
+  val all: List[NerTag] = List(Organization, Person, Location)
+
+  def fromString(str: String): Either[String, NerTag] =
+    str.toLowerCase match {
+      case "organization" => Right(Organization)
+      case "person" => Right(Person)
+      case "location" => Right(Location)
+      case "misc" => Right(Misc)
+      case "email" => Right(Email)
+      case "website" => Right(Website)
+      case "date" => Right(Date)
+      case _ => Left(s"Invalid ner tag: $str")
+    }
+
+  def unsafe(str: String): NerTag =
+    fromString(str).fold(sys.error, identity)
+
+
+  implicit val jsonDecoder: Decoder[NerTag] =
+    Decoder.decodeString.emap(fromString)
+  implicit val jsonEncoder: Encoder[NerTag] =
+    Encoder.encodeString.contramap(_.name)
+}

modules/common/src/main/scala/docspell/common/NodeType.scala

@@ -0,0 +1,25 @@
+package docspell.common
+
+sealed trait NodeType { self: Product =>
+
+  final def name: String =
+    self.productPrefix.toLowerCase
+
+}
+
+object NodeType {
+
+  case object Restserver extends NodeType
+  case object Joex extends NodeType
+
+  def fromString(str: String): Either[String, NodeType] =
+    str.toLowerCase match {
+      case "restserver" => Right(Restserver)
+      case "joex" => Right(Joex)
+      case _ => Left(s"Invalid node type: $str")
+    }
+
+  def unsafe(str: String): NodeType =
+    fromString(str).fold(sys.error, identity)
+
+}

modules/common/src/main/scala/docspell/common/Password.scala

@@ -0,0 +1,27 @@
+package docspell.common
+
+import io.circe.{Decoder, Encoder}
+
+final class Password(val pass: String) extends AnyVal {
+
+  def isEmpty: Boolean= pass.isEmpty
+
+  override def toString: String =
+    if (pass.isEmpty) "<empty>" else "***"
+
+}
+
+object Password {
+
+  val empty = Password("")
+
+  def apply(pass: String): Password =
+    new Password(pass)
+
+  implicit val passwordEncoder: Encoder[Password] =
+    Encoder.encodeString.contramap(_.pass)
+
+  implicit val passwordDecoder: Decoder[Password] =
+    Decoder.decodeString.map(Password(_))
+
+}

modules/common/src/main/scala/docspell/common/Priority.scala

@@ -0,0 +1,48 @@
+package docspell.common
+
+import cats.implicits._
+import cats.Order
+import io.circe.{Decoder, Encoder}
+
+sealed trait Priority { self: Product =>
+
+  final def name: String =
+    productPrefix.toLowerCase
+}
+
+object Priority {
+
+  case object High extends Priority
+
+  case object Low extends Priority
+
+
+  def fromString(str: String): Either[String, Priority] =
+    str.toLowerCase match {
+      case "high" => Right(High)
+      case "low" => Right(Low)
+      case _ => Left(s"Invalid priority: $str")
+    }
+
+  def unsafe(str: String): Priority =
+    fromString(str).fold(sys.error, identity)
+
+
+  def fromInt(n: Int): Priority =
+    if (n <= toInt(Low)) Low
+    else High
+
+  def toInt(p: Priority): Int =
+    p match {
+      case Low => 0
+      case High => 10
+    }
+
+  implicit val priorityOrder: Order[Priority] =
+    Order.by[Priority, Int](toInt)
+
+  implicit val jsonEncoder: Encoder[Priority] =
+    Encoder.encodeString.contramap(_.name)
+  implicit val jsonDecoder: Decoder[Priority] =
+    Decoder.decodeString.emap(fromString)
+}

modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala

@@ -0,0 +1,47 @@
+package docspell.common
+
+import io.circe._, io.circe.generic.semiauto._
+import docspell.common.syntax.all._
+import ProcessItemArgs._
+
+case class ProcessItemArgs(meta: ProcessMeta, files: List[File]) {
+
+  def makeSubject: String = {
+    files.flatMap(_.name) match {
+      case Nil => s"${meta.sourceAbbrev}: No files"
+      case n :: Nil => n
+      case n1 :: n2 :: Nil => s"$n1, $n2"
+      case more => s"${files.size} files from ${meta.sourceAbbrev}"
+    }
+  }
+
+}
+
+object ProcessItemArgs {
+
+  val taskName = Ident.unsafe("process-item")
+
+  case class ProcessMeta( collective: Ident
+                        , language: Language
+                        , direction: Option[Direction]
+                        , sourceAbbrev: String
+                        , validFileTypes: Seq[MimeType])
+
+  object ProcessMeta {
+    implicit val jsonEncoder: Encoder[ProcessMeta] = deriveEncoder[ProcessMeta]
+    implicit val jsonDecoder: Decoder[ProcessMeta] = deriveDecoder[ProcessMeta]
+  }
+
+  case class File(name: Option[String], fileMetaId: Ident)
+  object File {
+    implicit val jsonEncoder: Encoder[File] = deriveEncoder[File]
+    implicit val jsonDecoder: Decoder[File] = deriveDecoder[File]
+  }
+
+  implicit val jsonEncoder: Encoder[ProcessItemArgs] = deriveEncoder[ProcessItemArgs]
+  implicit val jsonDecoder: Decoder[ProcessItemArgs] = deriveDecoder[ProcessItemArgs]
+
+  def parse(str: String): Either[Throwable, ProcessItemArgs] =
+    str.parseJsonAs[ProcessItemArgs]
+
+}

modules/common/src/main/scala/docspell/common/ThreadFactories.scala

@@ -0,0 +1,20 @@
+package docspell.common
+
+import java.util.concurrent.atomic.AtomicLong
+import java.util.concurrent.{Executors, ThreadFactory}
+
+object ThreadFactories {
+
+  def ofName(prefix: String): ThreadFactory =
+    new ThreadFactory {
+
+      val counter = new AtomicLong(0)
+
+      override def newThread(r: Runnable): Thread = {
+        val t = Executors.defaultThreadFactory().newThread(r)
+        t.setName(s"$prefix-${counter.getAndIncrement()}")
+        t
+      }
+    }
+
+}

modules/common/src/main/scala/docspell/common/Timestamp.scala

@@ -0,0 +1,41 @@
+package docspell.common
+
+import java.time.{Instant, LocalDate, ZoneId}
+
+import cats.effect.Sync
+import io.circe.{Decoder, Encoder}
+
+case class Timestamp(value: Instant) {
+
+  def toMillis: Long = value.toEpochMilli
+
+  def toSeconds: Long = value.toEpochMilli / 1000L
+
+  def minus(d: Duration): Timestamp =
+    Timestamp(value.minusNanos(d.nanos))
+
+  def minusHours(n: Long): Timestamp =
+    Timestamp(value.minusSeconds(n * 60 * 60))
+
+  def toDate: LocalDate =
+    value.atZone(ZoneId.of("UTC")).toLocalDate
+
+  def asString: String = value.toString
+}
+
+object Timestamp {
+
+  val Epoch = Timestamp(Instant.EPOCH)
+
+  def current[F[_]: Sync]: F[Timestamp] =
+    Sync[F].delay(Timestamp(Instant.now))
+
+
+
+  implicit  val encodeTimestamp: Encoder[Timestamp] =
+    BaseJsonCodecs.encodeInstantEpoch.contramap(_.value)
+
+  implicit val decodeTimestamp: Decoder[Timestamp] =
+    BaseJsonCodecs.decodeInstantEpoch.map(Timestamp(_))
+
+}

modules/common/src/main/scala/docspell/common/UserState.scala

@@ -0,0 +1,37 @@
+package docspell.common
+
+import io.circe.{Decoder, Encoder}
+
+sealed trait UserState
+object UserState {
+  val all = List(Active, Disabled)
+
+  /** An active or enabled user. */
+  case object Active extends UserState
+
+  /** The user is blocked by an admin. */
+  case object Disabled extends UserState
+
+
+  def fromString(s: String): Either[String, UserState] =
+    s.toLowerCase match {
+      case "active" => Right(Active)
+      case "disabled" => Right(Disabled)
+      case _ => Left(s"Not a state value: $s")
+    }
+
+  def unsafe(str: String): UserState =
+    fromString(str).fold(sys.error, identity)
+
+  def asString(s: UserState): String = s match {
+    case Active => "active"
+    case Disabled => "disabled"
+  }
+
+  implicit val userStateEncoder: Encoder[UserState] =
+    Encoder.encodeString.contramap(UserState.asString)
+
+  implicit val userStateDecoder: Decoder[UserState] =
+    Decoder.decodeString.emap(UserState.fromString)
+
+}

modules/common/src/main/scala/docspell/common/pureconfig/Implicits.scala

@@ -0,0 +1,35 @@
+package docspell.common.pureconfig
+
+import docspell.common._
+import _root_.pureconfig._
+import _root_.pureconfig.error.{CannotConvert, FailureReason}
+import scodec.bits.ByteVector
+
+import scala.reflect.ClassTag
+
+object Implicits {
+  implicit val lenientUriReader: ConfigReader[LenientUri] =
+    ConfigReader[String].emap(reason(LenientUri.parse))
+
+  implicit val durationReader: ConfigReader[Duration] =
+    ConfigReader[scala.concurrent.duration.Duration].map(sd => Duration(sd))
+
+  implicit val passwordReader: ConfigReader[Password] =
+    ConfigReader[String].map(Password(_))
+
+  implicit val mimeTypeReader: ConfigReader[MimeType] =
+    ConfigReader[String].emap(reason(MimeType.parse))
+
+  implicit val identReader: ConfigReader[Ident] =
+    ConfigReader[String].emap(reason(Ident.fromString))
+
+  implicit val byteVectorReader: ConfigReader[ByteVector] =
+    ConfigReader[String].emap(reason(str => {
+      if (str.startsWith("hex:")) ByteVector.fromHex(str.drop(4)).toRight("Invalid hex value.")
+      else if (str.startsWith("b64:")) ByteVector.fromBase64(str.drop(4)).toRight("Invalid Base64 string.")
+      else ByteVector.fromHex(str).toRight("Invalid hex value.")
+    }))
+
+  def reason[A: ClassTag](f: String => Either[String, A]): String => Either[FailureReason, A] =
+    in => f(in).left.map(str => CannotConvert(in, implicitly[ClassTag[A]].runtimeClass.toString, str))
+}

modules/common/src/main/scala/docspell/common/syntax/EitherSyntax.scala

@@ -0,0 +1,21 @@
+package docspell.common.syntax
+
+trait EitherSyntax {
+
+  implicit final class LeftStringEitherOps[A](e: Either[String, A]) {
+    def throwLeft: A = e match {
+      case Right(a) => a
+      case Left(err) => sys.error(err)
+    }
+  }
+
+  implicit final class ThrowableLeftEitherOps[A](e: Either[Throwable, A]) {
+    def throwLeft: A = e match {
+      case Right(a) => a
+      case Left(err) => throw err
+    }
+  }
+
+}
+
+object EitherSyntax extends EitherSyntax

modules/common/src/main/scala/docspell/common/syntax/LoggerSyntax.scala

@@ -0,0 +1,35 @@
+package docspell.common.syntax
+
+import cats.effect.Sync
+import fs2.Stream
+import org.log4s.Logger
+
+trait LoggerSyntax {
+
+  implicit final class LoggerOps(logger: Logger) {
+
+    def ftrace[F[_]: Sync](msg: => String): F[Unit] =
+      Sync[F].delay(logger.trace(msg))
+
+    def fdebug[F[_]: Sync](msg: => String): F[Unit] =
+      Sync[F].delay(logger.debug(msg))
+
+    def sdebug[F[_]: Sync](msg: => String): Stream[F, Nothing] =
+      Stream.eval(fdebug(msg)).drain
+
+    def finfo[F[_]: Sync](msg: => String): F[Unit] =
+      Sync[F].delay(logger.info(msg))
+
+    def sinfo[F[_]: Sync](msg: => String): Stream[F, Nothing] =
+      Stream.eval(finfo(msg)).drain
+
+    def fwarn[F[_]: Sync](msg: => String): F[Unit] =
+      Sync[F].delay(logger.warn(msg))
+
+    def ferror[F[_]: Sync](msg: => String): F[Unit] =
+      Sync[F].delay(logger.error(msg))
+
+    def ferror[F[_]: Sync](ex: Throwable)(msg: => String): F[Unit] =
+      Sync[F].delay(logger.error(ex)(msg))
+  }
+}

modules/common/src/main/scala/docspell/common/syntax/StreamSyntax.scala

@@ -0,0 +1,24 @@
+package docspell.common.syntax
+
+import cats.effect.Sync
+import fs2.Stream
+import cats.implicits._
+import io.circe._
+import io.circe.parser._
+
+trait StreamSyntax {
+
+  implicit class StringStreamOps[F[_]](s: Stream[F, String]) {
+
+    def parseJsonAs[A](implicit d: Decoder[A], F: Sync[F]): F[Either[Throwable, A]] =
+      s.fold("")(_ + _).
+        compile.last.
+        map(optStr => for {
+          str   <- optStr.map(_.trim).toRight(new Exception("Empty string cannot be parsed into a value"))
+          json  <- parse(str).leftMap(_.underlying)
+          value <- json.as[A]
+        }  yield value)
+
+  }
+
+}

modules/common/src/main/scala/docspell/common/syntax/StringSyntax.scala

@@ -0,0 +1,21 @@
+package docspell.common.syntax
+
+import cats.implicits._
+import io.circe.Decoder
+import io.circe.parser._
+
+trait StringSyntax {
+
+  implicit class EvenMoreStringOps(s: String) {
+
+    def asNonBlank: Option[String] =
+      Option(s).filter(_.trim.nonEmpty)
+
+    def parseJsonAs[A](implicit d: Decoder[A]): Either[Throwable, A] =
+      for {
+        json  <- parse(s).leftMap(_.underlying)
+        value <- json.as[A]
+      } yield value
+  }
+
+}

modules/common/src/main/scala/docspell/common/syntax/package.scala

@@ -0,0 +1,10 @@
+package docspell.common
+
+package object syntax {
+
+  object all extends EitherSyntax
+    with StreamSyntax
+    with StringSyntax
+    with LoggerSyntax
+
+}