Reformat with scalafmt 3.0.0

This commit is contained in:
Scala Steward 2021-08-19 08:50:30 +02:00
parent 5a2a0295ef
commit e4fecefaea
No known key found for this signature in database
GPG Key ID: 96BDF10FFAB8B6A6
127 changed files with 558 additions and 658 deletions

View File

@ -157,10 +157,10 @@ val buildInfoSettings = Seq(
val openapiScalaSettings = Seq( val openapiScalaSettings = Seq(
openapiScalaConfig := ScalaConfig() openapiScalaConfig := ScalaConfig()
.withJson(ScalaJson.circeSemiauto) .withJson(ScalaJson.circeSemiauto)
.addMapping(CustomMapping.forType({ case TypeDef("LocalDateTime", _) => .addMapping(CustomMapping.forType { case TypeDef("LocalDateTime", _) =>
TypeDef("Timestamp", Imports("docspell.common.Timestamp")) TypeDef("Timestamp", Imports("docspell.common.Timestamp"))
})) })
.addMapping(CustomMapping.forFormatType({ .addMapping(CustomMapping.forFormatType {
case "ident" => case "ident" =>
field => field.copy(typeDef = TypeDef("Ident", Imports("docspell.common.Ident"))) field => field.copy(typeDef = TypeDef("Ident", Imports("docspell.common.Ident")))
case "accountid" => case "accountid" =>
@ -246,7 +246,7 @@ val openapiScalaSettings = Seq(
field => field =>
field field
.copy(typeDef = TypeDef("Duration", Imports("docspell.common.Duration"))) .copy(typeDef = TypeDef("Duration", Imports("docspell.common.Duration")))
})) })
) )
// --- Modules // --- Modules
@ -287,7 +287,7 @@ val files = project
val files = (base ** (_.isFile)).pair(sbt.io.Path.relativeTo(base)) val files = (base ** (_.isFile)).pair(sbt.io.Path.relativeTo(base))
val lines = files.toList.map(_._2).map { s => val lines = files.toList.map(_._2).map { s =>
val ident = s.replaceAll("[^a-zA-Z0-9_]+", "_") val ident = s.replaceAll("[^a-zA-Z0-9_]+", "_")
ident -> s"""val $ident = createUrl("${s}")""" ident -> s"""val $ident = createUrl("$s")"""
} }
val content = s"""package docspell.files val content = s"""package docspell.files
@ -759,7 +759,7 @@ def packageTools(logger: Logger, dir: File, version: String): Seq[File] = {
val target = dir / "target" val target = dir / "target"
IO.delete(target) IO.delete(target)
IO.createDirectory(target) IO.createDirectory(target)
val archive = target / s"docspell-tools-${version}.zip" val archive = target / s"docspell-tools-$version.zip"
logger.info(s"Packaging tools to $archive ...") logger.info(s"Packaging tools to $archive ...")
val webext = target / "docspell-firefox-extension.xpi" val webext = target / "docspell-firefox-extension.xpi"
val wx = dir / "webextension" val wx = dir / "webextension"
@ -782,13 +782,13 @@ def packageTools(logger: Logger, dir: File, version: String): Seq[File] = {
(dir ** "*") (dir ** "*")
.filter(f => !excludes.exists(p => f.absolutePath.startsWith(p.absolutePath))) .filter(f => !excludes.exists(p => f.absolutePath.startsWith(p.absolutePath)))
.pair(sbt.io.Path.relativeTo(dir)) .pair(sbt.io.Path.relativeTo(dir))
.map({ case (f, name) => (f, s"docspell-tools-${version}/$name") }) .map { case (f, name) => (f, s"docspell-tools-$version/$name") }
IO.zip( IO.zip(
Seq( Seq(
webext -> s"docspell-tools-${version}/firefox/docspell-extension.xpi", webext -> s"docspell-tools-$version/firefox/docspell-extension.xpi",
wx / "native/app_manifest.json" -> s"docspell-tools-${version}/firefox/native/app_manifest.json", wx / "native/app_manifest.json" -> s"docspell-tools-$version/firefox/native/app_manifest.json",
wx / "native/native.py" -> s"docspell-tools-${version}/firefox/native/native.py" wx / "native/native.py" -> s"docspell-tools-$version/firefox/native/native.py"
) ++ files, ) ++ files,
archive, archive,
None None

View File

@ -155,10 +155,8 @@ final class StanfordTextClassifier[F[_]: Async](cfg: TextClassifierConfig)
case class TrainResult(score: Double, model: ClassifierModel) case class TrainResult(score: Double, model: ClassifierModel)
def prepend(pre: String, data: Map[String, String]): Map[String, String] = def prepend(pre: String, data: Map[String, String]): Map[String, String] =
data.toList data.toList.map { case (k, v) =>
.map({ case (k, v) =>
if (k.startsWith(pre)) (k, v) if (k.startsWith(pre)) (k, v)
else (pre + k, v) else (pre + k, v)
}) }.toMap
.toMap
} }

View File

@ -32,7 +32,7 @@ object Domain {
Tld Tld
.findTld(str) .findTld(str)
.map(tld => (str.dropRight(tld.length), tld)) .map(tld => (str.dropRight(tld.length), tld))
.map({ case (names, tld) => .map { case (names, tld) =>
names.split('.').toList match { names.split('.').toList match {
case Nil => Left(s"Not a domain: $str") case Nil => Left(s"Not a domain: $str")
case segs case segs
@ -43,7 +43,7 @@ object Domain {
Right(Domain(NonEmptyList.fromListUnsafe(segs), tld)) Right(Domain(NonEmptyList.fromListUnsafe(segs), tld))
case _ => Left(s"Not a domain: $str") case _ => Left(s"Not a domain: $str")
} }
}) }
.getOrElse(Left(s"Not a domain $str")) .getOrElse(Left(s"Not a domain $str"))
def isDomain(str: String): Boolean = def isDomain(str: String): Boolean =

View File

@ -160,11 +160,11 @@ object DateFind {
Reader(words => Nel.of(reader, more: _*).map(_.read(words)).reduce) Reader(words => Nel.of(reader, more: _*).map(_.read(words)).reduce)
def readFirst[A](f: Word => Option[A]): Reader[A] = def readFirst[A](f: Word => Option[A]): Reader[A] =
Reader({ Reader {
case Nil => Result.Failure case Nil => Result.Failure
case a :: as => case a :: as =>
f(a).map(value => Result.Success(value, as)).getOrElse(Result.Failure) f(a).map(value => Result.Success(value, as)).getOrElse(Result.Failure)
}) }
} }
sealed trait Result[+A] { sealed trait Result[+A] {

View File

@ -15,7 +15,7 @@ object MonthName {
private def merge(n0: List[List[String]], ns: List[List[String]]*): List[List[String]] = private def merge(n0: List[List[String]], ns: List[List[String]]*): List[List[String]] =
ns.foldLeft(n0) { (res, el) => ns.foldLeft(n0) { (res, el) =>
res.zip(el).map({ case (a, b) => a ++ b }) res.zip(el).map { case (a, b) => a ++ b }
} }
private def forLang(lang: Language): List[List[String]] = private def forLang(lang: Language): List[List[String]] =

View File

@ -39,8 +39,8 @@ object Annotator {
* - full: the complete stanford pipeline is used * - full: the complete stanford pipeline is used
* - basic: only the ner classifier is used * - basic: only the ner classifier is used
* *
* Additionally, if there is a regexNer-file specified, the regexner annotator is * Additionally, if there is a regexNer-file specified, the regexner annotator is also
* also run. In case the full pipeline is used, this is already included. * run. In case the full pipeline is used, this is already included.
*/ */
def apply[F[_]: Sync](mode: NlpMode)(settings: NlpSettings): Annotator[F] = def apply[F[_]: Sync](mode: NlpMode)(settings: NlpSettings): Annotator[F] =
mode match { mode match {

View File

@ -21,10 +21,9 @@ import edu.stanford.nlp.ie.crf.CRFClassifier
import edu.stanford.nlp.ling.{CoreAnnotations, CoreLabel} import edu.stanford.nlp.ling.{CoreAnnotations, CoreLabel}
import org.log4s.getLogger import org.log4s.getLogger
/** This is only using the CRFClassifier without building an analysis /** This is only using the CRFClassifier without building an analysis pipeline. The
* pipeline. The ner-classifier cannot use results from POS-tagging * ner-classifier cannot use results from POS-tagging etc. and is therefore not as good
* etc. and is therefore not as good as the [[StanfordNerAnnotator]]. * as the [[StanfordNerAnnotator]]. But it uses less memory, while still being not bad.
* But it uses less memory, while still being not bad.
*/ */
object BasicCRFAnnotator { object BasicCRFAnnotator {
private[this] val logger = getLogger private[this] val logger = getLogger

View File

@ -17,8 +17,8 @@ import docspell.common._
import org.log4s.getLogger import org.log4s.getLogger
/** Creating the StanfordCoreNLP pipeline is quite expensive as it /** Creating the StanfordCoreNLP pipeline is quite expensive as it involves IO and
* involves IO and initializing large objects. * initializing large objects.
* *
* Therefore, the instances are cached, because they are thread-safe. * Therefore, the instances are cached, because they are thread-safe.
* *

View File

@ -22,13 +22,11 @@ object StanfordNerAnnotator {
/** Runs named entity recognition on the given `text`. /** Runs named entity recognition on the given `text`.
* *
* This uses the classifier pipeline from stanford-nlp, see * This uses the classifier pipeline from stanford-nlp, see
* https://nlp.stanford.edu/software/CRF-NER.html. Creating these * https://nlp.stanford.edu/software/CRF-NER.html. Creating these classifiers is quite
* classifiers is quite expensive, it involves loading large model * expensive, it involves loading large model files. The classifiers are thread-safe
* files. The classifiers are thread-safe and so they are cached. * and so they are cached. The `cacheKey` defines the "slot" where classifiers are
* The `cacheKey` defines the "slot" where classifiers are stored * stored and retrieved. If for a given `cacheKey` the `settings` change, a new
* and retrieved. If for a given `cacheKey` the `settings` change, * classifier must be created. It will then replace the previous one.
* a new classifier must be created. It will then replace the
* previous one.
*/ */
def nerAnnotate(nerClassifier: StanfordCoreNLP, text: String): Vector[NerLabel] = { def nerAnnotate(nerClassifier: StanfordCoreNLP, text: String): Vector[NerLabel] = {
val doc = new CoreDocument(text) val doc = new CoreDocument(text)

View File

@ -17,18 +17,16 @@ object StanfordNerSettings {
/** Settings for configuring the stanford NER pipeline. /** Settings for configuring the stanford NER pipeline.
* *
* The language is mandatory, only the provided ones are supported. * The language is mandatory, only the provided ones are supported. The `highRecall`
* The `highRecall` only applies for non-English languages. For * only applies for non-English languages. For non-English languages the english
* non-English languages the english classifier is run as second * classifier is run as second classifier and if `highRecall` is true, then it will be
* classifier and if `highRecall` is true, then it will be used to * used to tag untagged tokens. This may lead to a lot of false positives, but since
* tag untagged tokens. This may lead to a lot of false positives, * English is omnipresent in other languages, too it depends on the use case for
* but since English is omnipresent in other languages, too it * whether this is useful or not.
* depends on the use case for whether this is useful or not.
* *
* The `regexNer` allows to specify a text file as described here: * The `regexNer` allows to specify a text file as described here:
* https://nlp.stanford.edu/software/regexner.html. This will be used * https://nlp.stanford.edu/software/regexner.html. This will be used as a last step to
* as a last step to tag untagged tokens using the provided list of * tag untagged tokens using the provided list of regexps.
* regexps.
*/ */
case class Full( case class Full(
lang: NLPLanguage, lang: NLPLanguage,
@ -36,7 +34,8 @@ object StanfordNerSettings {
regexNer: Option[Path] regexNer: Option[Path]
) extends StanfordNerSettings ) extends StanfordNerSettings
/** Not all languages are supported with predefined statistical models. This allows to provide regexps only. /** Not all languages are supported with predefined statistical models. This allows to
* provide regexps only.
*/ */
case class RegexOnly(regexNerFile: Path) extends StanfordNerSettings case class RegexOnly(regexNerFile: Path) extends StanfordNerSettings

View File

@ -37,9 +37,9 @@ class StanfordTextClassifierSuite extends FunSuite {
.repeat .repeat
.take(10) .take(10)
) )
.flatMap({ case (a, b) => .flatMap { case (a, b) =>
Stream.emits(Seq(a, b)) Stream.emits(Seq(a, b))
}) }
.covary[IO] .covary[IO]
val modelExists = { val modelExists = {

View File

@ -41,6 +41,6 @@ private[auth] object TokenUtil {
def constTimeEq(s1: String, s2: String): Boolean = def constTimeEq(s1: String, s2: String): Boolean =
s1.zip(s2) s1.zip(s2)
.foldLeft(true)({ case (r, (c1, c2)) => r & c1 == c2 }) & s1.length == s2.length .foldLeft(true) { case (r, (c1, c2)) => r & c1 == c2 } & s1.length == s2.length
} }

View File

@ -18,8 +18,8 @@ import docspell.store.queries.QItem
trait CreateIndex[F[_]] { trait CreateIndex[F[_]] {
/** Low-level function to re-index data. It is not submitted as a job, /** Low-level function to re-index data. It is not submitted as a job, but invoked on
* but invoked on the current machine. * the current machine.
*/ */
def reIndexData( def reIndexData(
logger: Logger[F], logger: Logger[F],

View File

@ -84,9 +84,9 @@ object Merge {
nextPos <- store.transact(RAttachment.nextPosition(target)) nextPos <- store.transact(RAttachment.nextPosition(target))
attachs <- store.transact(items.tail.traverse(id => RAttachment.findByItem(id))) attachs <- store.transact(items.tail.traverse(id => RAttachment.findByItem(id)))
attachFlat = attachs.flatMap(_.toList) attachFlat = attachs.flatMap(_.toList)
n <- attachFlat.zipWithIndex.traverse({ case (a, idx) => n <- attachFlat.zipWithIndex.traverse { case (a, idx) =>
store.transact(RAttachment.updateItemId(a.id, target, nextPos + idx)) store.transact(RAttachment.updateItemId(a.id, target, nextPos + idx))
}) }
} yield n.sum } yield n.sum
} }

View File

@ -63,8 +63,8 @@ trait OCollective[F[_]] {
def startEmptyTrash(args: EmptyTrashArgs): F[Unit] def startEmptyTrash(args: EmptyTrashArgs): F[Unit]
/** Submits a task that (re)generates the preview images for all /** Submits a task that (re)generates the preview images for all attachments of the
* attachments of the given collective. * given collective.
*/ */
def generatePreviews( def generatePreviews(
storeMode: MakePreviewArgs.StoreMode, storeMode: MakePreviewArgs.StoreMode,

View File

@ -23,9 +23,8 @@ trait OFolder[F[_]] {
def findById(id: Ident, account: AccountId): F[Option[OFolder.FolderDetail]] def findById(id: Ident, account: AccountId): F[Option[OFolder.FolderDetail]]
/** Adds a new folder. If `login` is non-empty, the `folder.user` /** Adds a new folder. If `login` is non-empty, the `folder.user` property is ignored
* property is ignored and the user-id is determined by the given * and the user-id is determined by the given login name.
* login name.
*/ */
def add(folder: RFolder, login: Option[Ident]): F[AddResult] def add(folder: RFolder, login: Option[Ident]): F[AddResult]

View File

@ -49,13 +49,12 @@ trait OFulltext[F[_]] {
def findIndexOnlySummary(account: AccountId, fts: OFulltext.FtsInput): F[SearchSummary] def findIndexOnlySummary(account: AccountId, fts: OFulltext.FtsInput): F[SearchSummary]
def findItemsSummary(q: Query, fts: OFulltext.FtsInput): F[SearchSummary] def findItemsSummary(q: Query, fts: OFulltext.FtsInput): F[SearchSummary]
/** Clears the full-text index completely and launches a task that /** Clears the full-text index completely and launches a task that indexes all data.
* indexes all data.
*/ */
def reindexAll: F[Unit] def reindexAll: F[Unit]
/** Clears the full-text index for the given collective and starts a /** Clears the full-text index for the given collective and starts a task indexing all
* task indexing all their data. * their data.
*/ */
def reindexCollective(account: AccountId): F[Unit] def reindexCollective(account: AccountId): F[Unit]
} }
@ -125,7 +124,7 @@ object OFulltext {
FtsQuery.HighlightSetting(ftsQ.highlightPre, ftsQ.highlightPost) FtsQuery.HighlightSetting(ftsQ.highlightPre, ftsQ.highlightPost)
) )
for { for {
_ <- logger.ftrace(s"Find index only: ${ftsQ.query}/${batch}") _ <- logger.ftrace(s"Find index only: ${ftsQ.query}/$batch")
folders <- store.transact(QFolder.getMemberFolders(account)) folders <- store.transact(QFolder.getMemberFolders(account))
ftsR <- fts.search(fq.withFolders(folders)) ftsR <- fts.search(fq.withFolders(folders))
ftsItems = ftsR.results.groupBy(_.itemId) ftsItems = ftsR.results.groupBy(_.itemId)
@ -154,7 +153,7 @@ object OFulltext {
res = res =
itemsWithTags itemsWithTags
.collect(convertFtsData(ftsR, ftsItems)) .collect(convertFtsData(ftsR, ftsItems))
.map({ case (li, fd) => FtsItemWithTags(li, fd) }) .map { case (li, fd) => FtsItemWithTags(li, fd) }
} yield res } yield res
} }
@ -203,7 +202,7 @@ object OFulltext {
) )
.drop(batch.offset.toLong) .drop(batch.offset.toLong)
.take(batch.limit.toLong) .take(batch.limit.toLong)
.map({ case (li, fd) => FtsItem(li, fd) }) .map { case (li, fd) => FtsItem(li, fd) }
.compile .compile
.toVector .toVector
@ -221,7 +220,7 @@ object OFulltext {
) )
.drop(batch.offset.toLong) .drop(batch.offset.toLong)
.take(batch.limit.toLong) .take(batch.limit.toLong)
.map({ case (li, fd) => FtsItemWithTags(li, fd) }) .map { case (li, fd) => FtsItemWithTags(li, fd) }
.compile .compile
.toVector .toVector

View File

@ -28,9 +28,8 @@ trait OItem[F[_]] {
/** Sets the given tags (removing all existing ones). */ /** Sets the given tags (removing all existing ones). */
def setTags(item: Ident, tagIds: List[String], collective: Ident): F[UpdateResult] def setTags(item: Ident, tagIds: List[String], collective: Ident): F[UpdateResult]
/** Sets tags for multiple items. The tags of the items will be /** Sets tags for multiple items. The tags of the items will be replaced with the given
* replaced with the given ones. Same as `setTags` but for multiple * ones. Same as `setTags` but for multiple items.
* items.
*/ */
def setTagsMultipleItems( def setTagsMultipleItems(
items: NonEmptyList[Ident], items: NonEmptyList[Ident],
@ -41,8 +40,8 @@ trait OItem[F[_]] {
/** Create a new tag and add it to the item. */ /** Create a new tag and add it to the item. */
def addNewTag(item: Ident, tag: RTag): F[AddResult] def addNewTag(item: Ident, tag: RTag): F[AddResult]
/** Apply all tags to the given item. Tags must exist, but can be IDs /** Apply all tags to the given item. Tags must exist, but can be IDs or names. Existing
* or names. Existing tags on the item are left unchanged. * tags on the item are left unchanged.
*/ */
def linkTags(item: Ident, tags: List[String], collective: Ident): F[UpdateResult] def linkTags(item: Ident, tags: List[String], collective: Ident): F[UpdateResult]
@ -163,10 +162,9 @@ trait OItem[F[_]] {
collective: Ident collective: Ident
): F[UpdateResult] ): F[UpdateResult]
/** Submits the item for re-processing. The list of attachment ids can /** Submits the item for re-processing. The list of attachment ids can be used to only
* be used to only re-process a subset of the item's attachments. * re-process a subset of the item's attachments. If this list is empty, all
* If this list is empty, all attachments are reprocessed. This * attachments are reprocessed. This call only submits the job into the queue.
* call only submits the job into the queue.
*/ */
def reprocess( def reprocess(
item: Ident, item: Ident,
@ -181,9 +179,8 @@ trait OItem[F[_]] {
notifyJoex: Boolean notifyJoex: Boolean
): F[UpdateResult] ): F[UpdateResult]
/** Submits a task that finds all non-converted pdfs and triggers /** Submits a task that finds all non-converted pdfs and triggers converting them using
* converting them using ocrmypdf. Each file is converted by a * ocrmypdf. Each file is converted by a separate task.
* separate task.
*/ */
def convertAllPdf( def convertAllPdf(
collective: Option[Ident], collective: Option[Ident],
@ -191,8 +188,7 @@ trait OItem[F[_]] {
notifyJoex: Boolean notifyJoex: Boolean
): F[UpdateResult] ): F[UpdateResult]
/** Submits a task that (re)generates the preview image for an /** Submits a task that (re)generates the preview image for an attachment.
* attachment.
*/ */
def generatePreview( def generatePreview(
args: MakePreviewArgs, args: MakePreviewArgs,
@ -200,8 +196,7 @@ trait OItem[F[_]] {
notifyJoex: Boolean notifyJoex: Boolean
): F[UpdateResult] ): F[UpdateResult]
/** Submits a task that (re)generates the preview images for all /** Submits a task that (re)generates the preview images for all attachments.
* attachments.
*/ */
def generateAllPreviews( def generateAllPreviews(
storeMode: MakePreviewArgs.StoreMode, storeMode: MakePreviewArgs.StoreMode,

View File

@ -183,7 +183,7 @@ object OItemSearch {
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] = def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] =
store store
.transact(RAttachment.findByIdAndCollective(id, collective)) .transact(RAttachment.findByIdAndCollective(id, collective))
.flatMap({ .flatMap {
case Some(ra) => case Some(ra) =>
makeBinaryData(ra.fileId) { m => makeBinaryData(ra.fileId) { m =>
AttachmentData[F]( AttachmentData[F](
@ -195,7 +195,7 @@ object OItemSearch {
case None => case None =>
(None: Option[AttachmentData[F]]).pure[F] (None: Option[AttachmentData[F]]).pure[F]
}) }
def findAttachmentSource( def findAttachmentSource(
id: Ident, id: Ident,
@ -203,7 +203,7 @@ object OItemSearch {
): F[Option[AttachmentSourceData[F]]] = ): F[Option[AttachmentSourceData[F]]] =
store store
.transact(RAttachmentSource.findByIdAndCollective(id, collective)) .transact(RAttachmentSource.findByIdAndCollective(id, collective))
.flatMap({ .flatMap {
case Some(ra) => case Some(ra) =>
makeBinaryData(ra.fileId) { m => makeBinaryData(ra.fileId) { m =>
AttachmentSourceData[F]( AttachmentSourceData[F](
@ -215,7 +215,7 @@ object OItemSearch {
case None => case None =>
(None: Option[AttachmentSourceData[F]]).pure[F] (None: Option[AttachmentSourceData[F]]).pure[F]
}) }
def findAttachmentPreview( def findAttachmentPreview(
id: Ident, id: Ident,
@ -223,7 +223,7 @@ object OItemSearch {
): F[Option[AttachmentPreviewData[F]]] = ): F[Option[AttachmentPreviewData[F]]] =
store store
.transact(RAttachmentPreview.findByIdAndCollective(id, collective)) .transact(RAttachmentPreview.findByIdAndCollective(id, collective))
.flatMap({ .flatMap {
case Some(ra) => case Some(ra) =>
makeBinaryData(ra.fileId) { m => makeBinaryData(ra.fileId) { m =>
AttachmentPreviewData[F]( AttachmentPreviewData[F](
@ -235,7 +235,7 @@ object OItemSearch {
case None => case None =>
(None: Option[AttachmentPreviewData[F]]).pure[F] (None: Option[AttachmentPreviewData[F]]).pure[F]
}) }
def findItemPreview( def findItemPreview(
item: Ident, item: Ident,
@ -243,7 +243,7 @@ object OItemSearch {
): F[Option[AttachmentPreviewData[F]]] = ): F[Option[AttachmentPreviewData[F]]] =
store store
.transact(RAttachmentPreview.findByItemAndCollective(item, collective)) .transact(RAttachmentPreview.findByItemAndCollective(item, collective))
.flatMap({ .flatMap {
case Some(ra) => case Some(ra) =>
makeBinaryData(ra.fileId) { m => makeBinaryData(ra.fileId) { m =>
AttachmentPreviewData[F]( AttachmentPreviewData[F](
@ -255,7 +255,7 @@ object OItemSearch {
case None => case None =>
(None: Option[AttachmentPreviewData[F]]).pure[F] (None: Option[AttachmentPreviewData[F]]).pure[F]
}) }
def findAttachmentArchive( def findAttachmentArchive(
id: Ident, id: Ident,
@ -263,7 +263,7 @@ object OItemSearch {
): F[Option[AttachmentArchiveData[F]]] = ): F[Option[AttachmentArchiveData[F]]] =
store store
.transact(RAttachmentArchive.findByIdAndCollective(id, collective)) .transact(RAttachmentArchive.findByIdAndCollective(id, collective))
.flatMap({ .flatMap {
case Some(ra) => case Some(ra) =>
makeBinaryData(ra.fileId) { m => makeBinaryData(ra.fileId) { m =>
AttachmentArchiveData[F]( AttachmentArchiveData[F](
@ -275,7 +275,7 @@ object OItemSearch {
case None => case None =>
(None: Option[AttachmentArchiveData[F]]).pure[F] (None: Option[AttachmentArchiveData[F]]).pure[F]
}) }
private def makeBinaryData[A](fileId: Ident)(f: FileMeta => A): F[Option[A]] = private def makeBinaryData[A](fileId: Ident)(f: FileMeta => A): F[Option[A]] =
store.bitpeace store.bitpeace

View File

@ -64,14 +64,14 @@ object OOrganization {
): F[Vector[OrgAndContacts]] = ): F[Vector[OrgAndContacts]] =
store store
.transact(QOrganization.findOrgAndContact(account.collective, query, _.name)) .transact(QOrganization.findOrgAndContact(account.collective, query, _.name))
.map({ case (org, cont) => OrgAndContacts(org, cont) }) .map { case (org, cont) => OrgAndContacts(org, cont) }
.compile .compile
.toVector .toVector
def findOrg(account: AccountId, orgId: Ident): F[Option[OrgAndContacts]] = def findOrg(account: AccountId, orgId: Ident): F[Option[OrgAndContacts]] =
store store
.transact(QOrganization.getOrgAndContact(account.collective, orgId)) .transact(QOrganization.getOrgAndContact(account.collective, orgId))
.map(_.map({ case (org, cont) => OrgAndContacts(org, cont) })) .map(_.map { case (org, cont) => OrgAndContacts(org, cont) })
def findAllOrgRefs( def findAllOrgRefs(
account: AccountId, account: AccountId,
@ -91,14 +91,14 @@ object OOrganization {
): F[Vector[PersonAndContacts]] = ): F[Vector[PersonAndContacts]] =
store store
.transact(QOrganization.findPersonAndContact(account.collective, query, _.name)) .transact(QOrganization.findPersonAndContact(account.collective, query, _.name))
.map({ case (person, org, cont) => PersonAndContacts(person, org, cont) }) .map { case (person, org, cont) => PersonAndContacts(person, org, cont) }
.compile .compile
.toVector .toVector
def findPerson(account: AccountId, persId: Ident): F[Option[PersonAndContacts]] = def findPerson(account: AccountId, persId: Ident): F[Option[PersonAndContacts]] =
store store
.transact(QOrganization.getPersonAndContact(account.collective, persId)) .transact(QOrganization.getPersonAndContact(account.collective, persId))
.map(_.map({ case (pers, org, cont) => PersonAndContacts(pers, org, cont) })) .map(_.map { case (pers, org, cont) => PersonAndContacts(pers, org, cont) })
def findAllPersonRefs( def findAllPersonRefs(
account: AccountId, account: AccountId,

View File

@ -19,31 +19,27 @@ import docspell.store.queries.SearchSummary
import org.log4s.getLogger import org.log4s.getLogger
/** A "porcelain" api on top of OFulltext and OItemSearch. This takes /** A "porcelain" api on top of OFulltext and OItemSearch. This takes care of restricting
* care of restricting the items to a subset, e.g. only items that * the items to a subset, e.g. only items that have a "valid" state.
* have a "valid" state.
*/ */
trait OSimpleSearch[F[_]] { trait OSimpleSearch[F[_]] {
/** Search for items using the given query and optional fulltext /** Search for items using the given query and optional fulltext search.
* search.
* *
* When using fulltext search only (the query is empty), only the * When using fulltext search only (the query is empty), only the index is searched. It
* index is searched. It is assumed that the index doesn't contain * is assumed that the index doesn't contain "invalid" items. When using a query, then
* "invalid" items. When using a query, then a condition to select * a condition to select only valid items is added to it.
* only valid items is added to it.
*/ */
def search(settings: Settings)(q: Query, fulltextQuery: Option[String]): F[Items] def search(settings: Settings)(q: Query, fulltextQuery: Option[String]): F[Items]
/** Using the same arguments as in `search`, this returns a summary /** Using the same arguments as in `search`, this returns a summary and not the results.
* and not the results.
*/ */
def searchSummary( def searchSummary(
settings: StatsSettings settings: StatsSettings
)(q: Query, fulltextQuery: Option[String]): F[SearchSummary] )(q: Query, fulltextQuery: Option[String]): F[SearchSummary]
/** Calls `search` by parsing the given query string into a query that /** Calls `search` by parsing the given query string into a query that is then amended
* is then amended wtih the given `fix` query. * wtih the given `fix` query.
*/ */
final def searchByString( final def searchByString(
settings: Settings settings: Settings
@ -52,8 +48,7 @@ trait OSimpleSearch[F[_]] {
): F[StringSearchResult[Items]] = ): F[StringSearchResult[Items]] =
OSimpleSearch.applySearch[F, Items](fix, q)((iq, fts) => search(settings)(iq, fts)) OSimpleSearch.applySearch[F, Items](fix, q)((iq, fts) => search(settings)(iq, fts))
/** Same as `searchByString` but returning a summary instead of the /** Same as `searchByString` but returning a summary instead of the results.
* results.
*/ */
final def searchSummaryByString( final def searchSummaryByString(
settings: StatsSettings settings: StatsSettings
@ -190,8 +185,8 @@ object OSimpleSearch {
} }
} }
/** Calls `run` with one of the success results when extracting the /** Calls `run` with one of the success results when extracting the fulltext search node
* fulltext search node from the query. * from the query.
*/ */
private def runQuery[F[_]: Applicative, A]( private def runQuery[F[_]: Applicative, A](
itemQuery: Option[ItemQuery] itemQuery: Option[ItemQuery]
@ -211,10 +206,9 @@ object OSimpleSearch {
final class Impl[F[_]: Sync](fts: OFulltext[F], is: OItemSearch[F]) final class Impl[F[_]: Sync](fts: OFulltext[F], is: OItemSearch[F])
extends OSimpleSearch[F] { extends OSimpleSearch[F] {
/** Implements searching like this: it exploits the fact that teh /** Implements searching like this: it exploits the fact that teh fulltext index only
* fulltext index only contains valid items. When searching via * contains valid items. When searching via sql the query expression selecting only
* sql the query expression selecting only valid items is added * valid items is added here.
* here.
*/ */
def search( def search(
settings: Settings settings: Settings

View File

@ -31,10 +31,9 @@ trait OUpload[F[_]] {
itemId: Option[Ident] itemId: Option[Ident]
): F[OUpload.UploadResult] ): F[OUpload.UploadResult]
/** Submit files via a given source identifier. The source is looked /** Submit files via a given source identifier. The source is looked up to identify the
* up to identify the collective the files belong to. Metadata * collective the files belong to. Metadata defined in the source is used as a fallback
* defined in the source is used as a fallback to those specified * to those specified here (in UploadData).
* here (in UploadData).
*/ */
def submit( def submit(
data: OUpload.UploadData[F], data: OUpload.UploadData[F],
@ -103,8 +102,7 @@ object OUpload {
def noSource: UploadResult = NoSource def noSource: UploadResult = NoSource
/** When adding files to an item, no item was found using the given /** When adding files to an item, no item was found using the given item-id.
* item-id.
*/ */
case object NoItem extends UploadResult case object NoItem extends UploadResult

View File

@ -37,8 +37,7 @@ trait OUserTask[F[_]] {
task: UserTask[ScanMailboxArgs] task: UserTask[ScanMailboxArgs]
): F[Unit] ): F[Unit]
/** Return the settings for all the notify-due-items task of the /** Return the settings for all the notify-due-items task of the current user.
* current user.
*/ */
def getNotifyDueItems(scope: UserTaskScope): Stream[F, UserTask[NotifyDueItemsArgs]] def getNotifyDueItems(scope: UserTaskScope): Stream[F, UserTask[NotifyDueItemsArgs]]
@ -59,9 +58,8 @@ trait OUserTask[F[_]] {
/** Removes a user task with the given id. */ /** Removes a user task with the given id. */
def deleteTask(scope: UserTaskScope, id: Ident): F[Unit] def deleteTask(scope: UserTaskScope, id: Ident): F[Unit]
/** Discards the schedule and immediately submits the task to the job /** Discards the schedule and immediately submits the task to the job executor's queue.
* executor's queue. It will not update the corresponding periodic * It will not update the corresponding periodic task.
* task.
*/ */
def executeNow[A](scope: UserTaskScope, subject: Option[String], task: UserTask[A])( def executeNow[A](scope: UserTaskScope, subject: Option[String], task: UserTask[A])(
implicit E: Encoder[A] implicit E: Encoder[A]

View File

@ -16,8 +16,7 @@ object SendResult {
*/ */
case class Success(id: Ident) extends SendResult case class Success(id: Ident) extends SendResult
/** There was a failure sending the mail. The mail is then not saved /** There was a failure sending the mail. The mail is then not saved to db.
* to db.
*/ */
case class SendFailure(ex: Throwable) extends SendResult case class SendFailure(ex: Throwable) extends SendResult
@ -25,8 +24,7 @@ object SendResult {
*/ */
case class StoreFailure(ex: Throwable) extends SendResult case class StoreFailure(ex: Throwable) extends SendResult
/** Something could not be found required for sending (mail configs, /** Something could not be found required for sending (mail configs, items etc).
* items etc).
*/ */
case object NotFound extends SendResult case object NotFound extends SendResult
} }

View File

@ -9,12 +9,11 @@ package docspell.common
import io.circe.generic.semiauto._ import io.circe.generic.semiauto._
import io.circe.{Decoder, Encoder} import io.circe.{Decoder, Encoder}
/** Arguments for the `AllPreviewsTask` that submits tasks to /** Arguments for the `AllPreviewsTask` that submits tasks to generates a preview image
* generates a preview image for attachments. * for attachments.
* *
* It can replace the current preview image or only generate one, if * It can replace the current preview image or only generate one, if it is missing. If no
* it is missing. If no collective is specified, it considers all * collective is specified, it considers all attachments.
* attachments.
*/ */
case class AllPreviewsArgs( case class AllPreviewsArgs(
collective: Option[Ident], collective: Option[Ident],

View File

@ -15,17 +15,15 @@ object CollectiveState {
/** A normal active collective */ /** A normal active collective */
case object Active extends CollectiveState case object Active extends CollectiveState
/** A collective may be readonly in cases it is implicitly closed /** A collective may be readonly in cases it is implicitly closed (e.g. no payment).
* (e.g. no payment). Users can still see there data and * Users can still see there data and download, but have no write access.
* download, but have no write access.
*/ */
case object ReadOnly extends CollectiveState case object ReadOnly extends CollectiveState
/** A collective that has been explicitely closed. */ /** A collective that has been explicitely closed. */
case object Closed extends CollectiveState case object Closed extends CollectiveState
/** A collective blocked by a super user, usually some emergency /** A collective blocked by a super user, usually some emergency action.
* action.
*/ */
case object Blocked extends CollectiveState case object Blocked extends CollectiveState

View File

@ -9,14 +9,12 @@ package docspell.common
import io.circe._ import io.circe._
import io.circe.generic.semiauto._ import io.circe.generic.semiauto._
/** Arguments for the task that finds all pdf files that have not been /** Arguments for the task that finds all pdf files that have not been converted and
* converted and submits for each a job that will convert the file * submits for each a job that will convert the file using ocrmypdf.
* using ocrmypdf.
* *
* If the `collective` argument is present, then this task and the * If the `collective` argument is present, then this task and the ones that are
* ones that are submitted by this task run in the realm of the * submitted by this task run in the realm of the collective (and only their files are
* collective (and only their files are considered). If it is empty, * considered). If it is empty, it is a system task and all files are considered.
* it is a system task and all files are considered.
*/ */
case class ConvertAllPdfArgs(collective: Option[Ident]) case class ConvertAllPdfArgs(collective: Option[Ident])

View File

@ -14,8 +14,8 @@ import io.circe.generic.semiauto._
/** Arguments to the empty-trash task. /** Arguments to the empty-trash task.
* *
* This task is run periodically to really delete all soft-deleted * This task is run periodically to really delete all soft-deleted items. These are items
* items. These are items with state `ItemState.Deleted`. * with state `ItemState.Deleted`.
*/ */
case class EmptyTrashArgs( case class EmptyTrashArgs(
collective: Ident, collective: Ident,

View File

@ -14,8 +14,8 @@ case class FileName private (name: String) {
case n => (name.take(n), Some(name.drop(n + 1))) case n => (name.take(n), Some(name.drop(n + 1)))
} }
/** Returns the name part without the extension. If there is no /** Returns the name part without the extension. If there is no extension, it is the
* extension, it is the same as fullname. * same as fullname.
*/ */
def baseName: String = def baseName: String =
base base
@ -27,20 +27,20 @@ case class FileName private (name: String) {
def fullName: String = def fullName: String =
name name
/** Creates a new name where part is spliced into the name before the /** Creates a new name where part is spliced into the name before the extension,
* extension, separated by separator. * separated by separator.
*/ */
def withPart(part: String, sep: Char): FileName = def withPart(part: String, sep: Char): FileName =
if (part.isEmpty()) this if (part.isEmpty()) this
else else
ext ext
.map(e => new FileName(s"${base}${sep}${part}.${e}")) .map(e => new FileName(s"$base$sep$part.$e"))
.getOrElse(new FileName(s"${base}${sep}${part}")) .getOrElse(new FileName(s"$base$sep$part"))
/** Create a new name using the given extension. */ /** Create a new name using the given extension. */
def withExtension(newExt: String): FileName = def withExtension(newExt: String): FileName =
if (newExt.isEmpty()) new FileName(base) if (newExt.isEmpty()) new FileName(base)
else new FileName(s"${base}.${newExt}") else new FileName(s"$base.$newExt")
} }
object FileName { object FileName {

View File

@ -16,14 +16,11 @@ trait Glob {
/** Matches the input string against this glob. */ /** Matches the input string against this glob. */
def matches(caseSensitive: Boolean)(in: String): Boolean def matches(caseSensitive: Boolean)(in: String): Boolean
/** If this glob consists of multiple segments, it is the same as /** If this glob consists of multiple segments, it is the same as `matches`. If it is
* `matches`. If it is only a single segment, it is matched against * only a single segment, it is matched against the last segment of the input string
* the last segment of the input string that is assumed to be a * that is assumed to be a pathname separated by slash.
* pathname separated by slash.
* *
* Example: * Example: test.* <> "/a/b/test.txt" => true /test.* <> "/a/b/test.txt" => false
* test.* <> "/a/b/test.txt" => true
* /test.* <> "/a/b/test.txt" => false
*/ */
def matchFilenameOrPath(in: String): Boolean def matchFilenameOrPath(in: String): Boolean

View File

@ -20,8 +20,7 @@ object JobState {
/** Waiting for being executed. */ /** Waiting for being executed. */
case object Waiting extends JobState {} case object Waiting extends JobState {}
/** A scheduler has picked up this job and will pass it to the next /** A scheduler has picked up this job and will pass it to the next free slot.
* free slot.
*/ */
case object Scheduled extends JobState {} case object Scheduled extends JobState {}

View File

@ -13,9 +13,9 @@ import io.circe.generic.semiauto._
/** Arguments to the classify-item task. /** Arguments to the classify-item task.
* *
* This task is run periodically and learns from existing documents * This task is run periodically and learns from existing documents to create a model for
* to create a model for predicting tags of new documents. The user * predicting tags of new documents. The user must give a tag category as a subset of
* must give a tag category as a subset of possible tags.. * possible tags..
*/ */
case class LearnClassifierArgs( case class LearnClassifierArgs(
collective: Ident collective: Ident

View File

@ -9,9 +9,8 @@ package docspell.common
import io.circe.generic.semiauto._ import io.circe.generic.semiauto._
import io.circe.{Decoder, Encoder} import io.circe.{Decoder, Encoder}
/** Arguments for the `MakePageCountTask` that reads the number of /** Arguments for the `MakePageCountTask` that reads the number of pages for an attachment
* pages for an attachment and stores it into the meta data of the * and stores it into the meta data of the attachment.
* attachment.
*/ */
case class MakePageCountArgs( case class MakePageCountArgs(
attachment: Ident attachment: Ident

View File

@ -9,11 +9,9 @@ package docspell.common
import io.circe.generic.semiauto._ import io.circe.generic.semiauto._
import io.circe.{Decoder, Encoder} import io.circe.{Decoder, Encoder}
/** Arguments for the `MakePreviewTask` that generates a preview image /** Arguments for the `MakePreviewTask` that generates a preview image for an attachment.
* for an attachment.
* *
* It can replace the current preview image or only generate one, if * It can replace the current preview image or only generate one, if it is missing.
* it is missing.
*/ */
case class MakePreviewArgs( case class MakePreviewArgs(
attachment: Ident, attachment: Ident,

View File

@ -20,14 +20,12 @@ import io.circe.generic.semiauto._
/** A proposed meta data to an item. /** A proposed meta data to an item.
* *
* There is only one value for each proposal type. The list of * There is only one value for each proposal type. The list of candidates is meant to be
* candidates is meant to be ordered from the best match to the * ordered from the best match to the lowest match.
* lowest match.
* *
* The candidate is already "resolved" against the database and * The candidate is already "resolved" against the database and contains a valid record
* contains a valid record (with its ID and a human readable name). * (with its ID and a human readable name). Additionally it carries a set of "labels"
* Additionally it carries a set of "labels" (which may be empty) * (which may be empty) that are the source of this candidate.
* that are the source of this candidate.
*/ */
case class MetaProposal(proposalType: MetaProposalType, values: NonEmptyList[Candidate]) { case class MetaProposal(proposalType: MetaProposalType, values: NonEmptyList[Candidate]) {
@ -96,8 +94,8 @@ object MetaProposal {
} }
} }
/** Merges candidates with same `IdRef` values and concatenates their /** Merges candidates with same `IdRef` values and concatenates their respective labels.
* respective labels. The candidate order is preserved. * The candidate order is preserved.
*/ */
def flatten(s: NonEmptyList[Candidate]): NonEmptyList[Candidate] = { def flatten(s: NonEmptyList[Candidate]): NonEmptyList[Candidate] = {
def mergeInto( def mergeInto(

View File

@ -91,13 +91,12 @@ object MetaProposalList {
.getOrElse(empty) .getOrElse(empty)
def fromMap(m: Map[MetaProposalType, MetaProposal]): MetaProposalList = def fromMap(m: Map[MetaProposalType, MetaProposal]): MetaProposalList =
new MetaProposalList(m.toList.map({ case (k, v) => v.copy(proposalType = k) })) new MetaProposalList(m.toList.map { case (k, v) => v.copy(proposalType = k) })
/** Flattens the given list of meta-proposals into a single list, /** Flattens the given list of meta-proposals into a single list, where each
* where each meta-proposal type exists at most once. Candidates to * meta-proposal type exists at most once. Candidates to equal proposal-types are
* equal proposal-types are merged together. The candidate's order * merged together. The candidate's order is preserved and candidates of proposals are
* is preserved and candidates of proposals are appended as given * appended as given by the order of the given `seq'.
* by the order of the given `seq'.
*/ */
def flatten(ml: Seq[MetaProposalList]): MetaProposalList = def flatten(ml: Seq[MetaProposalList]): MetaProposalList =
flatten0( flatten0(

View File

@ -13,11 +13,10 @@ import io.circe.generic.semiauto._
/** Arguments to the notification task. /** Arguments to the notification task.
* *
* This tasks queries items with a due date and informs the user via * This tasks queries items with a due date and informs the user via mail.
* mail.
* *
* If the structure changes, there must be some database migration to * If the structure changes, there must be some database migration to update or remove
* update or remove the json data of the corresponding task. * the json data of the corresponding task.
*/ */
case class NotifyDueItemsArgs( case class NotifyDueItemsArgs(
account: AccountId, account: AccountId,

View File

@ -14,11 +14,11 @@ import io.circe.generic.semiauto._
/** Arguments to the process-item task. /** Arguments to the process-item task.
* *
* This task is run for each new file to create a new item from it or * This task is run for each new file to create a new item from it or to add this file as
* to add this file as an attachment to an existing item. * an attachment to an existing item.
* *
* If the `itemId' is set to some value, the item is tried to load to * If the `itemId' is set to some value, the item is tried to load to ammend with the
* ammend with the given files. Otherwise a new item is created. * given files. Otherwise a new item is created.
* *
* It is also re-used by the 'ReProcessItem' task. * It is also re-used by the 'ReProcessItem' task.
*/ */

View File

@ -11,10 +11,9 @@ import io.circe.{Decoder, Encoder}
/** Arguments when re-processing an item. /** Arguments when re-processing an item.
* *
* The `itemId` must exist and point to some item. If the attachment * The `itemId` must exist and point to some item. If the attachment list is non-empty,
* list is non-empty, only those attachments are re-processed. They * only those attachments are re-processed. They must belong to the given item. If the
* must belong to the given item. If the list is empty, then all * list is empty, then all attachments are re-processed.
* attachments are re-processed.
*/ */
case class ReProcessItemArgs(itemId: Ident, attachments: List[Ident]) case class ReProcessItemArgs(itemId: Ident, attachments: List[Ident])

View File

@ -13,11 +13,10 @@ import io.circe.generic.semiauto._
/** Arguments to the poll-mailbox task. /** Arguments to the poll-mailbox task.
* *
* This tasks queries user mailboxes and pushes found mails into * This tasks queries user mailboxes and pushes found mails into docspell for processing.
* docspell for processing.
* *
* If the structure changes, there must be some database migration to * If the structure changes, there must be some database migration to update or remove
* update or remove the json data of the corresponding task. * the json data of the corresponding task.
*/ */
case class ScanMailboxArgs( case class ScanMailboxArgs(
// the docspell user account // the docspell user account

View File

@ -19,10 +19,9 @@ sealed trait ConversionResult[F[_]] {
object ConversionResult { object ConversionResult {
/** The conversion is done by external tools that write files to the /** The conversion is done by external tools that write files to the file system. These
* file system. These are temporary files and they will be deleted * are temporary files and they will be deleted once the process finishes. This handler
* once the process finishes. This handler is used to do something * is used to do something relevant with the resulting files.
* relevant with the resulting files.
*/ */
type Handler[F[_], A] = Kleisli[F, ConversionResult[F], A] type Handler[F[_], A] = Kleisli[F, ConversionResult[F], A]

View File

@ -12,11 +12,10 @@ import scodec.bits.ByteVector
@FunctionalInterface @FunctionalInterface
trait SanitizeHtml { trait SanitizeHtml {
/** The given `bytes' are html which can be modified to strip out /** The given `bytes' are html which can be modified to strip out unwanted content.
* unwanted content.
* *
* The result should use the same character encoding as the given * The result should use the same character encoding as the given charset implies, or
* charset implies, or utf8 if not specified. * utf8 if not specified.
*/ */
def apply(bytes: ByteVector, charset: Option[Charset]): ByteVector def apply(bytes: ByteVector, charset: Option[Charset]): ByteVector

View File

@ -132,7 +132,7 @@ private[extern] object ExternConv {
): Pipe[F, Byte, Unit] = ): Pipe[F, Byte, Unit] =
in => in =>
Stream Stream
.eval(logger.debug(s"Storing input to file ${inFile} for running $name")) .eval(logger.debug(s"Storing input to file $inFile for running $name"))
.drain ++ .drain ++
Stream.eval(storeFile(in, inFile)) Stream.eval(storeFile(in, inFile))

View File

@ -150,12 +150,12 @@ class ConversionTest extends FunSuite with FileChecks {
conversion conversion
.use { conv => .use { conv =>
def check: Handler[IO, Unit] = def check: Handler[IO, Unit] =
Kleisli({ Kleisli {
case ConversionResult.InputMalformed(_, _) => case ConversionResult.InputMalformed(_, _) =>
().pure[IO] ().pure[IO]
case cr => case cr =>
IO.raiseError(new Exception(s"Unexpected result: $cr")) IO.raiseError(new Exception(s"Unexpected result: $cr"))
}) }
runConversion(bombs, _ => check, conv).compile.drain runConversion(bombs, _ => check, conv).compile.drain
} }
@ -171,12 +171,12 @@ class ConversionTest extends FunSuite with FileChecks {
.emits(uris) .emits(uris)
.covary[IO] .covary[IO]
.zipWithIndex .zipWithIndex
.evalMap({ case (uri, index) => .evalMap { case (uri, index) =>
val load = uri.readURL[IO](8192) val load = uri.readURL[IO](8192)
val dataType = DataType.filename(uri.path.segments.last) val dataType = DataType.filename(uri.path.segments.last)
logger.info(s"Processing file ${uri.path.asString}") *> logger.info(s"Processing file ${uri.path.asString}") *>
conv.toPDF(dataType, Language.German, handler(index))(load) conv.toPDF(dataType, Language.German, handler(index))(load)
}) }
def commandsExist: Boolean = def commandsExist: Boolean =
commandExists(convertConfig.unoconv.command.program) && commandExists(convertConfig.unoconv.command.program) &&

View File

@ -48,7 +48,7 @@ trait FileChecks {
storePdfTxtHandler(file, file.resolveSibling("unexpected.txt")).map(_._1) storePdfTxtHandler(file, file.resolveSibling("unexpected.txt")).map(_._1)
def storePdfTxtHandler(filePdf: Path, fileTxt: Path): Handler[IO, (Path, Path)] = def storePdfTxtHandler(filePdf: Path, fileTxt: Path): Handler[IO, (Path, Path)] =
Kleisli({ Kleisli {
case ConversionResult.SuccessPdfTxt(pdf, txt) => case ConversionResult.SuccessPdfTxt(pdf, txt) =>
for { for {
pout <- pdf.through(storeFile(filePdf)).compile.lastOrError pout <- pdf.through(storeFile(filePdf)).compile.lastOrError
@ -64,7 +64,7 @@ trait FileChecks {
case cr => case cr =>
throw new Exception(s"Unexpected result: $cr") throw new Exception(s"Unexpected result: $cr")
}) }
def commandExists(cmd: String): Boolean = def commandExists(cmd: String): Boolean =
Runtime.getRuntime.exec(Array("which", cmd)).waitFor() == 0 Runtime.getRuntime.exec(Array("which", cmd)).waitFor() == 0

View File

@ -62,8 +62,8 @@ object Ocr {
): Stream[F, String] = ): Stream[F, String] =
runTesseractFile(img, logger, lang, config) runTesseractFile(img, logger, lang, config)
/** Run ghostscript to extract all pdf pages into tiff files. The /** Run ghostscript to extract all pdf pages into tiff files. The files are stored to a
* files are stored to a temporary location on disk and returned. * temporary location on disk and returned.
*/ */
private[extract] def runGhostscript[F[_]: Async]( private[extract] def runGhostscript[F[_]: Async](
pdf: Stream[F, Byte], pdf: Stream[F, Byte],
@ -88,8 +88,8 @@ object Ocr {
.flatMap(_ => File.listFiles(pathEndsWith(".tif"), wd)) .flatMap(_ => File.listFiles(pathEndsWith(".tif"), wd))
} }
/** Run ghostscript to extract all pdf pages into tiff files. The /** Run ghostscript to extract all pdf pages into tiff files. The files are stored to a
* files are stored to a temporary location on disk and returned. * temporary location on disk and returned.
*/ */
private[extract] def runGhostscriptFile[F[_]: Async]( private[extract] def runGhostscriptFile[F[_]: Async](
pdf: Path, pdf: Path,
@ -111,8 +111,8 @@ object Ocr {
private def pathEndsWith(ext: String): Path => Boolean = private def pathEndsWith(ext: String): Path => Boolean =
p => p.fileName.toString.endsWith(ext) p => p.fileName.toString.endsWith(ext)
/** Run unpaper to optimize the image for ocr. The /** Run unpaper to optimize the image for ocr. The files are stored to a temporary
* files are stored to a temporary location on disk and returned. * location on disk and returned.
*/ */
private[extract] def runUnpaperFile[F[_]: Async]( private[extract] def runUnpaperFile[F[_]: Async](
img: Path, img: Path,
@ -139,8 +139,7 @@ object Ocr {
} }
} }
/** Run tesseract on the given image file and return the extracted /** Run tesseract on the given image file and return the extracted text.
* text.
*/ */
private[extract] def runTesseractFile[F[_]: Async]( private[extract] def runTesseractFile[F[_]: Async](
img: Path, img: Path,
@ -160,8 +159,7 @@ object Ocr {
.map(_.stdout) .map(_.stdout)
} }
/** Run tesseract on the given image file and return the extracted /** Run tesseract on the given image file and return the extracted text.
* text.
*/ */
private[extract] def runTesseractStdin[F[_]: Async]( private[extract] def runTesseractStdin[F[_]: Async](
img: Stream[F, Byte], img: Stream[F, Byte],

View File

@ -31,7 +31,7 @@ object TextExtract {
): Stream[F, Text] = ): Stream[F, Text] =
Stream Stream
.eval(TikaMimetype.detect(in, MimeTypeHint.none)) .eval(TikaMimetype.detect(in, MimeTypeHint.none))
.flatMap({ .flatMap {
case MimeType.pdf => case MimeType.pdf =>
Stream.eval(Ocr.extractPdf(in, logger, lang, config)).unNoneTerminate Stream.eval(Ocr.extractPdf(in, logger, lang, config)).unNoneTerminate
@ -40,7 +40,7 @@ object TextExtract {
case mt => case mt =>
raiseError(s"File `$mt` not supported") raiseError(s"File `$mt` not supported")
}) }
.map(Text.apply) .map(Text.apply)
private def raiseError[F[_]: Sync](msg: String): Stream[F, Nothing] = private def raiseError[F[_]: Sync](msg: String): Stream[F, Nothing] =

View File

@ -49,17 +49,13 @@ object PoiExtract {
case PoiType.docx => case PoiType.docx =>
getDocx(data) getDocx(data)
case PoiType.msoffice => case PoiType.msoffice =>
EitherT(getDoc[F](data)) EitherT(getDoc[F](data)).recoverWith { case _ =>
.recoverWith({ case _ =>
EitherT(getXls[F](data)) EitherT(getXls[F](data))
}) }.value
.value
case PoiType.ooxml => case PoiType.ooxml =>
EitherT(getDocx[F](data)) EitherT(getDocx[F](data)).recoverWith { case _ =>
.recoverWith({ case _ =>
EitherT(getXlsx[F](data)) EitherT(getXlsx[F](data))
}) }.value
.value
case mt => case mt =>
Sync[F].pure(Left(new Exception(s"Unsupported content: ${mt.asString}"))) Sync[F].pure(Left(new Exception(s"Unsupported content: ${mt.asString}")))
} }

View File

@ -20,20 +20,17 @@ import fs2.io.file.Path
object ImageSize { object ImageSize {
/** Return the image size from its header without reading /** Return the image size from its header without reading the whole image into memory.
* the whole image into memory.
*/ */
def get(file: Path): Option[Dimension] = def get(file: Path): Option[Dimension] =
Using(new FileImageInputStream(file.toNioPath.toFile))(getDimension).toOption.flatten Using(new FileImageInputStream(file.toNioPath.toFile))(getDimension).toOption.flatten
/** Return the image size from its header without reading /** Return the image size from its header without reading the whole image into memory.
* the whole image into memory.
*/ */
def get(in: InputStream): Option[Dimension] = def get(in: InputStream): Option[Dimension] =
Option(ImageIO.createImageInputStream(in)).flatMap(getDimension) Option(ImageIO.createImageInputStream(in)).flatMap(getDimension)
/** Return the image size from its header without reading /** Return the image size from its header without reading the whole image into memory.
* the whole image into memory.
*/ */
def get[F[_]: Sync](data: Stream[F, Byte]): F[Option[Dimension]] = def get[F[_]: Sync](data: Stream[F, Byte]): F[Option[Dimension]] =
data.take(768).compile.to(Array).map { ar => data.take(768).compile.to(Array).map { ar =>

View File

@ -14,32 +14,28 @@ import docspell.common._
import org.log4s.getLogger import org.log4s.getLogger
/** The fts client is the interface for docspell to a fulltext search /** The fts client is the interface for docspell to a fulltext search engine.
* engine.
* *
* It defines all operations required for integration into docspell. * It defines all operations required for integration into docspell. It uses data
* It uses data structures from docspell. Implementation modules need * structures from docspell. Implementation modules need to translate it to the engine
* to translate it to the engine that provides the features. * that provides the features.
*/ */
trait FtsClient[F[_]] { trait FtsClient[F[_]] {
/** Initialization tasks. This can be used to setup the fulltext /** Initialization tasks. This can be used to setup the fulltext search engine. The
* search engine. The implementation is expected to keep track of * implementation is expected to keep track of run migrations, so that running these is
* run migrations, so that running these is idempotent. For * idempotent. For example, it may be run on each application start.
* example, it may be run on each application start.
* *
* Initialization may involve re-indexing all data, therefore it * Initialization may involve re-indexing all data, therefore it must run outside the
* must run outside the scope of this client. The migration may * scope of this client. The migration may include a task that applies any work and/or
* include a task that applies any work and/or it can return a * it can return a result indicating that after this task a re-index is necessary.
* result indicating that after this task a re-index is necessary.
*/ */
def initialize: F[List[FtsMigration[F]]] def initialize: F[List[FtsMigration[F]]]
/** A list of initialization tasks that can be run when re-creating /** A list of initialization tasks that can be run when re-creating the index.
* the index.
* *
* This is not run on startup, but only when required, for example * This is not run on startup, but only when required, for example when re-creating the
* when re-creating the entire index. * entire index.
*/ */
def initializeNew: List[FtsMigration[F]] def initializeNew: List[FtsMigration[F]]
@ -53,18 +49,16 @@ trait FtsClient[F[_]] {
else Stream.emit(result) ++ searchAll(q.nextPage) else Stream.emit(result) ++ searchAll(q.nextPage)
} }
/** Push all data to the index. Data with same `id' is replaced. /** Push all data to the index. Data with same `id' is replaced. Values that are `None'
* Values that are `None' are removed from the index (or set to an * are removed from the index (or set to an empty string).
* empty string).
*/ */
def indexData(logger: Logger[F], data: Stream[F, TextData]): F[Unit] def indexData(logger: Logger[F], data: Stream[F, TextData]): F[Unit]
def indexData(logger: Logger[F], data: TextData*): F[Unit] = def indexData(logger: Logger[F], data: TextData*): F[Unit] =
indexData(logger, Stream.emits(data)) indexData(logger, Stream.emits(data))
/** Push all data to the index, but only update existing entries. No /** Push all data to the index, but only update existing entries. No new entries are
* new entries are created and values that are given as `None' are * created and values that are given as `None' are skipped.
* skipped.
*/ */
def updateIndex(logger: Logger[F], data: Stream[F, TextData]): F[Unit] def updateIndex(logger: Logger[F], data: Stream[F, TextData]): F[Unit]

View File

@ -10,16 +10,14 @@ import docspell.common._
/** A fulltext query. /** A fulltext query.
* *
* The query itself is a raw string. Each implementation may * The query itself is a raw string. Each implementation may interpret it according to
* interpret it according to the system in use. * the system in use.
* *
* Searches must only look for given collective and in the given list * Searches must only look for given collective and in the given list of item ids, if it
* of item ids, if it is non-empty. If the item set is empty, then * is non-empty. If the item set is empty, then don't restrict the result in this way.
* don't restrict the result in this way.
* *
* The set of folders must be used to restrict the results only to * The set of folders must be used to restrict the results only to items that have one of
* items that have one of the folders set or no folder set. If the * the folders set or no folder set. If the set is empty, the restriction does not apply.
* set is empty, the restriction does not apply.
*/ */
final case class FtsQuery( final case class FtsQuery(
q: String, q: String,

View File

@ -22,10 +22,9 @@ trait JoexApp[F[_]] {
/** Shuts down the job executor. /** Shuts down the job executor.
* *
* It will immediately stop taking new jobs, waiting for currently * It will immediately stop taking new jobs, waiting for currently running jobs to
* running jobs to complete normally (i.e. running jobs are not * complete normally (i.e. running jobs are not canceled). After this completed, the
* canceled). After this completed, the webserver stops and the * webserver stops and the main loop will exit.
* main loop will exit.
*/ */
def initShutdown: F[Unit] def initShutdown: F[Unit]
} }

View File

@ -97,7 +97,7 @@ object NerFile {
private def sanitizeRegex(str: String): String = private def sanitizeRegex(str: String): String =
str.trim.toLowerCase.foldLeft("") { (res, ch) => str.trim.toLowerCase.foldLeft("") { (res, ch) =>
if (invalidChars.contains(ch)) s"${res}\\$ch" if (invalidChars.contains(ch)) s"$res\\$ch"
else s"$res$ch" else s"$res$ch"
} }
} }

View File

@ -22,8 +22,7 @@ import docspell.store.records.RPerson
import io.circe.syntax._ import io.circe.syntax._
import org.log4s.getLogger import org.log4s.getLogger
/** Maintains a custom regex-ner file per collective for stanford's /** Maintains a custom regex-ner file per collective for stanford's regexner annotator.
* regexner annotator.
*/ */
trait RegexNerFile[F[_]] { trait RegexNerFile[F[_]] {
@ -64,7 +63,7 @@ object RegexNerFile {
val dur = Duration.between(nf.creation, now) val dur = Duration.between(nf.creation, now)
if (dur > cfg.minTime) if (dur > cfg.minTime)
logger.fdebug( logger.fdebug(
s"Cache time elapsed (${dur} > ${cfg.minTime}). Check for new state." s"Cache time elapsed ($dur > ${cfg.minTime}). Check for new state."
) *> updateFile( ) *> updateFile(
collective, collective,
now, now,

View File

@ -52,7 +52,7 @@ object EmptyTrashTask {
s"Starting removing all soft-deleted items older than ${maxDate.asString}" s"Starting removing all soft-deleted items older than ${maxDate.asString}"
) )
nDeleted <- deleteAll(ctx.args, maxDate, itemOps, itemSearchOps, ctx) nDeleted <- deleteAll(ctx.args, maxDate, itemOps, itemSearchOps, ctx)
_ <- ctx.logger.info(s"Finished deleting ${nDeleted} items") _ <- ctx.logger.info(s"Finished deleting $nDeleted items")
} yield () } yield ()
} }

View File

@ -101,7 +101,7 @@ object FtsWork {
def recoverWith( def recoverWith(
other: FtsWork[F] other: FtsWork[F]
)(implicit ev: ApplicativeError[F, Throwable]): FtsWork[F] = )(implicit ev: ApplicativeError[F, Throwable]): FtsWork[F] =
Kleisli(ctx => mt.run(ctx).onError({ case _ => other.run(ctx) })) Kleisli(ctx => mt.run(ctx).onError { case _ => other.run(ctx) })
def forContext( def forContext(
cfg: Config.FullTextSearch, cfg: Config.FullTextSearch,

View File

@ -19,9 +19,8 @@ import docspell.store.Store
/** Migrating the index from the previous version to this version. /** Migrating the index from the previous version to this version.
* *
* The migration asks the fulltext search client for a list of * The migration asks the fulltext search client for a list of migration tasks to run. It
* migration tasks to run. It may be empty when there is no migration * may be empty when there is no migration required.
* required.
*/ */
case class Migration[F[_]]( case class Migration[F[_]](
version: Int, version: Int,

View File

@ -10,8 +10,7 @@ import cats.data.Kleisli
package object fts { package object fts {
/** Some work that must be done to advance the schema of the fulltext /** Some work that must be done to advance the schema of the fulltext index.
* index.
*/ */
type FtsWork[F[_]] = Kleisli[F, FtsContext[F], Unit] type FtsWork[F[_]] = Kleisli[F, FtsContext[F], Unit]

View File

@ -23,7 +23,7 @@ object ClassifierName {
private val categoryPrefix = "tagcategory-" private val categoryPrefix = "tagcategory-"
def tagCategory(cat: String): ClassifierName = def tagCategory(cat: String): ClassifierName =
apply(s"${categoryPrefix}${cat}") apply(s"$categoryPrefix$cat")
val concernedPerson: ClassifierName = val concernedPerson: ClassifierName =
apply("concernedperson") apply("concernedperson")
@ -56,7 +56,7 @@ object ClassifierName {
def findOrphanTagModels[F[_]](coll: Ident): ConnectionIO[List[RClassifierModel]] = def findOrphanTagModels[F[_]](coll: Ident): ConnectionIO[List[RClassifierModel]] =
for { for {
cats <- RClassifierSetting.getActiveCategories(coll) cats <- RClassifierSetting.getActiveCategories(coll)
allModels = RClassifierModel.findAllByQuery(coll, s"${categoryPrefix}%") allModels = RClassifierModel.findAllByQuery(coll, s"$categoryPrefix%")
result <- NonEmptyList.fromList(cats) match { result <- NonEmptyList.fromList(cats) match {
case Some(nel) => case Some(nel) =>
allModels.flatMap(all => allModels.flatMap(all =>

View File

@ -47,7 +47,7 @@ object Classify {
.flatMap(_ => classifier.classify(logger, ClassifierModel(modelFile), text)) .flatMap(_ => classifier.classify(logger, ClassifierModel(modelFile), text))
}).filter(_ != LearnClassifierTask.noClass) }).filter(_ != LearnClassifierTask.noClass)
.flatTapNone(logger.debug("Guessed: <none>")) .flatTapNone(logger.debug("Guessed: <none>"))
_ <- OptionT.liftF(logger.debug(s"Guessed: ${cls}")) _ <- OptionT.liftF(logger.debug(s"Guessed: $cls"))
} yield cls).value } yield cls).value
} }

View File

@ -26,9 +26,8 @@ import bitpeace.RangeDef
import io.circe.generic.semiauto._ import io.circe.generic.semiauto._
import io.circe.{Decoder, Encoder} import io.circe.{Decoder, Encoder}
/** Converts the given attachment file using ocrmypdf if it is a pdf /** Converts the given attachment file using ocrmypdf if it is a pdf and has not already
* and has not already been converted (the source file is the same as * been converted (the source file is the same as in the attachment).
* in the attachment).
*/ */
object PdfConvTask { object PdfConvTask {
case class Args(attachId: Ident) case class Args(attachId: Ident)
@ -100,7 +99,7 @@ object PdfConvTask {
.through(bp.fetchData2(RangeDef.all)) .through(bp.fetchData2(RangeDef.all))
val storeResult: ConversionResult.Handler[F, Unit] = val storeResult: ConversionResult.Handler[F, Unit] =
Kleisli({ Kleisli {
case ConversionResult.SuccessPdf(file) => case ConversionResult.SuccessPdf(file) =>
storeToAttachment(ctx, in, file) storeToAttachment(ctx, in, file)
@ -109,15 +108,15 @@ object PdfConvTask {
case ConversionResult.UnsupportedFormat(mime) => case ConversionResult.UnsupportedFormat(mime) =>
ctx.logger.warn( ctx.logger.warn(
s"Unable to convert '${mime}' file ${ctx.args}: unsupported format." s"Unable to convert '$mime' file ${ctx.args}: unsupported format."
) )
case ConversionResult.InputMalformed(mime, reason) => case ConversionResult.InputMalformed(mime, reason) =>
ctx.logger.warn(s"Unable to convert '${mime}' file ${ctx.args}: $reason") ctx.logger.warn(s"Unable to convert '$mime' file ${ctx.args}: $reason")
case ConversionResult.Failure(ex) => case ConversionResult.Failure(ex) =>
Sync[F].raiseError(ex) Sync[F].raiseError(ex)
}) }
def ocrMyPdf(lang: Language): F[Unit] = def ocrMyPdf(lang: Language): F[Unit] =
OcrMyPdf.toPDF[F, Unit]( OcrMyPdf.toPDF[F, Unit](

View File

@ -22,9 +22,8 @@ import docspell.store.syntax.MimeTypes._
import bitpeace.{Mimetype, RangeDef} import bitpeace.{Mimetype, RangeDef}
/** Goes through all attachments that must be already converted into a /** Goes through all attachments that must be already converted into a pdf. If it is a
* pdf. If it is a pdf, the number of pages are retrieved and stored * pdf, the number of pages are retrieved and stored in the attachment metadata.
* in the attachment metadata.
*/ */
object AttachmentPageCount { object AttachmentPageCount {

View File

@ -24,9 +24,9 @@ import docspell.store.syntax.MimeTypes._
import bitpeace.{Mimetype, MimetypeHint, RangeDef} import bitpeace.{Mimetype, MimetypeHint, RangeDef}
/** Goes through all attachments that must be already converted into a /** Goes through all attachments that must be already converted into a pdf. If it is a
* pdf. If it is a pdf, the first page is converted into a small * pdf, the first page is converted into a small preview png image and linked to the
* preview png image and linked to the attachment. * attachment.
*/ */
object AttachmentPreview { object AttachmentPreview {

View File

@ -23,19 +23,16 @@ import docspell.store.syntax.MimeTypes._
import bitpeace.{Mimetype, MimetypeHint, RangeDef} import bitpeace.{Mimetype, MimetypeHint, RangeDef}
/** Goes through all attachments and creates a PDF version of it where /** Goes through all attachments and creates a PDF version of it where supported.
* supported.
* *
* The `attachment` record is updated with the PDF version while the * The `attachment` record is updated with the PDF version while the original file has
* original file has been stored in the `attachment_source` record. * been stored in the `attachment_source` record.
* *
* If pdf conversion is not possible or if the input is already a * If pdf conversion is not possible or if the input is already a pdf, both files are
* pdf, both files are identical. That is, the `file_id`s point to * identical. That is, the `file_id`s point to the same file. Since the name of an
* the same file. Since the name of an attachment may be changed by * attachment may be changed by the user, the `attachment_origin` record keeps that, too.
* the user, the `attachment_origin` record keeps that, too.
* *
* This step assumes an existing premature item, it traverses its * This step assumes an existing premature item, it traverses its attachments.
* attachments.
*/ */
object ConvertPdf { object ConvertPdf {
@ -104,7 +101,7 @@ object ConvertPdf {
ra: RAttachment, ra: RAttachment,
item: ItemData item: ItemData
): Handler[F, (RAttachment, Option[RAttachmentMeta])] = ): Handler[F, (RAttachment, Option[RAttachmentMeta])] =
Kleisli({ Kleisli {
case ConversionResult.SuccessPdf(pdf) => case ConversionResult.SuccessPdf(pdf) =>
ctx.logger.info(s"Conversion to pdf successful. Saving file.") *> ctx.logger.info(s"Conversion to pdf successful. Saving file.") *>
storePDF(ctx, cfg, ra, pdf) storePDF(ctx, cfg, ra, pdf)
@ -142,7 +139,7 @@ object ConvertPdf {
ctx.logger ctx.logger
.error(s"PDF conversion failed: ${ex.getMessage}. Go without PDF file") *> .error(s"PDF conversion failed: ${ex.getMessage}. Go without PDF file") *>
(ra, None: Option[RAttachmentMeta]).pure[F] (ra, None: Option[RAttachmentMeta]).pure[F]
}) }
private def storePDF[F[_]: Sync]( private def storePDF[F[_]: Sync](
ctx: Context[F, ProcessItemArgs], ctx: Context[F, ProcessItemArgs],
@ -196,7 +193,7 @@ object ConvertPdf {
case Right(_) => ().pure[F] case Right(_) => ().pure[F]
case Left(ex) => case Left(ex) =>
ctx.logger ctx.logger
.error(ex)(s"Cannot delete previous attachment file: ${raPrev}") .error(ex)(s"Cannot delete previous attachment file: $raPrev")
} }
} yield () } yield ()

View File

@ -45,9 +45,9 @@ object CreateItem {
Stream Stream
.emits(ctx.args.files) .emits(ctx.args.files)
.flatMap(f => ctx.store.bitpeace.get(f.fileMetaId.id).map(fm => (f, fm))) .flatMap(f => ctx.store.bitpeace.get(f.fileMetaId.id).map(fm => (f, fm)))
.collect({ case (f, Some(fm)) if isValidFile(fm) => f }) .collect { case (f, Some(fm)) if isValidFile(fm) => f }
.zipWithIndex .zipWithIndex
.evalMap({ case (f, index) => .evalMap { case (f, index) =>
Ident Ident
.randomId[F] .randomId[F]
.map(id => .map(id =>
@ -60,7 +60,7 @@ object CreateItem {
f.name f.name
) )
) )
}) }
} }
.compile .compile
.toVector .toVector
@ -152,7 +152,7 @@ object CreateItem {
.transact(RAttachment.findByItemCollectiveSource(ri.id, ri.cid, fids)) .transact(RAttachment.findByItemCollectiveSource(ri.id, ri.cid, fids))
.flatTap(ats => .flatTap(ats =>
ctx.logger.debug( ctx.logger.debug(
s"Found ${ats.size} attachments. Use only those from task args: ${fileMetaIds}" s"Found ${ats.size} attachments. Use only those from task args: $fileMetaIds"
) )
) )
) )

View File

@ -14,12 +14,10 @@ import cats.implicits._
import docspell.common._ import docspell.common._
import docspell.joex.scheduler.Task import docspell.joex.scheduler.Task
/** After candidates have been determined, the set is reduced by doing /** After candidates have been determined, the set is reduced by doing some cross checks.
* some cross checks. For example: if a organization is suggested as * For example: if a organization is suggested as correspondent, the correspondent person
* correspondent, the correspondent person must be linked to that * must be linked to that organization. So this *removes all* person candidates that are
* organization. So this *removes all* person candidates that are not * not linked to the first organization candidate (which will be linked to the item).
* linked to the first organization candidate (which will be linked
* to the item).
*/ */
object CrossCheckProposals { object CrossCheckProposals {

View File

@ -52,7 +52,7 @@ object DuplicateCheck {
val fname = ctx.args.files.find(_.fileMetaId.id == fd.fm.id).flatMap(_.name) val fname = ctx.args.files.find(_.fileMetaId.id == fd.fm.id).flatMap(_.name)
if (fd.exists) if (fd.exists)
ctx.logger ctx.logger
.info(s"Deleting duplicate file ${fname}!") *> ctx.store.bitpeace .info(s"Deleting duplicate file $fname!") *> ctx.store.bitpeace
.delete(fd.fm.id) .delete(fd.fm.id)
.compile .compile
.drain .drain

View File

@ -15,8 +15,7 @@ import docspell.common._
import docspell.joex.scheduler.{Context, Task} import docspell.joex.scheduler.{Context, Task}
import docspell.store.records.{RAttachmentMeta, RPerson} import docspell.store.records.{RAttachmentMeta, RPerson}
/** Calculate weights for candidates that adds the most likely /** Calculate weights for candidates that adds the most likely candidate a lower number.
* candidate a lower number.
*/ */
object EvalProposals { object EvalProposals {

View File

@ -25,16 +25,13 @@ import docspell.store.syntax.MimeTypes._
import bitpeace.{Mimetype, MimetypeHint, RangeDef} import bitpeace.{Mimetype, MimetypeHint, RangeDef}
import emil.Mail import emil.Mail
/** Goes through all attachments and extracts archive files, like zip /** Goes through all attachments and extracts archive files, like zip files. The process
* files. The process is recursive, until all archives have been * is recursive, until all archives have been extracted.
* extracted.
* *
* The archive file is stored as a `attachment_archive` record that * The archive file is stored as a `attachment_archive` record that references all its
* references all its elements. If there are inner archive, only the * elements. If there are inner archive, only the outer archive file is preserved.
* outer archive file is preserved.
* *
* This step assumes an existing premature item, it traverses its * This step assumes an existing premature item, it traverses its attachments.
* attachments.
*/ */
object ExtractArchive { object ExtractArchive {
@ -78,11 +75,10 @@ object ExtractArchive {
) )
} }
/** After all files have been extracted, the `extract' contains the /** After all files have been extracted, the `extract' contains the whole (combined)
* whole (combined) result. This fixes positions of the attachments * result. This fixes positions of the attachments such that the elements of an archive
* such that the elements of an archive are "spliced" into the * are "spliced" into the attachment list at the position of the archive. If there is
* attachment list at the position of the archive. If there is no * no archive, positions don't need to be fixed.
* archive, positions don't need to be fixed.
*/ */
private def fixPositions(extract: Extracted): Extracted = private def fixPositions(extract: Extracted): Extracted =
if (extract.archives.isEmpty) extract if (extract.archives.isEmpty) extract
@ -267,7 +263,7 @@ object ExtractArchive {
val sorted = nel.sorted val sorted = nel.sorted
val offset = sorted.head.first val offset = sorted.head.first
val pos = val pos =
sorted.zipWithIndex.map({ case (p, i) => p.id -> (i + offset) }).toList.toMap sorted.zipWithIndex.map { case (p, i) => p.id -> (i + offset) }.toList.toMap
val nf = val nf =
files.map(f => pos.get(f.id).map(n => f.copy(position = n)).getOrElse(f)) files.map(f => pos.get(f.id).map(n => f.copy(position = n)).getOrElse(f))
copy(files = nf) copy(files = nf)

View File

@ -19,8 +19,8 @@ import docspell.joex.Config
import docspell.joex.scheduler.{Context, Task} import docspell.joex.scheduler.{Context, Task}
import docspell.store.records._ import docspell.store.records._
/** Super simple approach to find corresponding meta data to an item /** Super simple approach to find corresponding meta data to an item by looking up values
* by looking up values from NER in the users address book. * from NER in the users address book.
*/ */
object FindProposal { object FindProposal {
type Args = ProcessItemArgs type Args = ProcessItemArgs

View File

@ -12,18 +12,23 @@ import docspell.store.records.{RAttachment, RAttachmentMeta, RItem}
/** Data that is carried across all processing tasks. /** Data that is carried across all processing tasks.
* *
* @param item the stored item record * @param item
* @param attachments the attachments belonging to the item * the stored item record
* @param metas the meta data to each attachment; depending on the * @param attachments
* state of processing, this may be empty * the attachments belonging to the item
* @param dateLabels a separate list of found dates * @param metas
* @param originFile a mapping from an attachment id to a filemeta-id * the meta data to each attachment; depending on the state of processing, this may be
* containng the source or origin file * empty
* @param givenMeta meta data to this item that was not "guessed" * @param dateLabels
* from an attachment but given and thus is always correct * a separate list of found dates
* @param classifyProposals these are proposals that were obtained by * @param originFile
* a trained classifier. There are no ner-tags, it will only provide a * a mapping from an attachment id to a filemeta-id containng the source or origin file
* single label * @param givenMeta
* meta data to this item that was not "guessed" from an attachment but given and thus
* is always correct
* @param classifyProposals
* these are proposals that were obtained by a trained classifier. There are no
* ner-tags, it will only provide a single label
*/ */
case class ItemData( case class ItemData(
item: RItem, item: RItem,
@ -39,9 +44,8 @@ case class ItemData(
classifyTags: List[String] classifyTags: List[String]
) { ) {
/** sort by weight; order of equal weights is not important, just /** sort by weight; order of equal weights is not important, just choose one others are
* choose one others are then suggestions * then suggestions doc-date is only set when given explicitely, not from "guessing"
* doc-date is only set when given explicitely, not from "guessing"
*/ */
def finalProposals: MetaProposalList = def finalProposals: MetaProposalList =
MetaProposalList MetaProposalList

View File

@ -77,7 +77,7 @@ object ItemHandler {
)(data: ItemData): Task[F, Args, ItemData] = )(data: ItemData): Task[F, Args, ItemData] =
isLastRetry[F].flatMap { isLastRetry[F].flatMap {
case true => case true =>
ProcessItem[F](cfg, itemOps, fts, analyser, regexNer)(data).attempt.flatMap({ ProcessItem[F](cfg, itemOps, fts, analyser, regexNer)(data).attempt.flatMap {
case Right(d) => case Right(d) =>
Task.pure(d) Task.pure(d)
case Left(ex) => case Left(ex) =>
@ -85,7 +85,7 @@ object ItemHandler {
"Processing failed on last retry. Creating item but without proposals." "Processing failed on last retry. Creating item but without proposals."
).flatMap(_ => itemStateTask(ItemState.Created)(data)) ).flatMap(_ => itemStateTask(ItemState.Created)(data))
.andThen(_ => Sync[F].raiseError(ex)) .andThen(_ => Sync[F].raiseError(ex))
}) }
case false => case false =>
ProcessItem[F](cfg, itemOps, fts, analyser, regexNer)(data) ProcessItem[F](cfg, itemOps, fts, analyser, regexNer)(data)
.flatMap(itemStateTask(ItemState.Created)) .flatMap(itemStateTask(ItemState.Created))

View File

@ -57,7 +57,7 @@ object LinkProposal {
case Some(a) => case Some(a) =>
val ids = a.values.map(_.ref.id.id) val ids = a.values.map(_.ref.id.id)
ctx.logger.info( ctx.logger.info(
s"Found many (${a.size}, ${ids}) candidates for ${a.proposalType}. Setting first." s"Found many (${a.size}, $ids) candidates for ${a.proposalType}. Setting first."
) *> ) *>
setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).map(_ => setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).map(_ =>
Result.multiple(mpt) Result.multiple(mpt)

View File

@ -149,14 +149,14 @@ object ReProcessItem {
isLastRetry[F].flatMap { isLastRetry[F].flatMap {
case true => case true =>
processFiles[F](cfg, fts, itemOps, analyser, regexNer, data).attempt processFiles[F](cfg, fts, itemOps, analyser, regexNer, data).attempt
.flatMap({ .flatMap {
case Right(d) => case Right(d) =>
Task.pure(d) Task.pure(d)
case Left(ex) => case Left(ex) =>
logWarn[F]( logWarn[F](
"Processing failed on last retry." "Processing failed on last retry."
).andThen(_ => Sync[F].raiseError(ex)) ).andThen(_ => Sync[F].raiseError(ex))
}) }
case false => case false =>
processFiles[F](cfg, fts, itemOps, analyser, regexNer, data) processFiles[F](cfg, fts, itemOps, analyser, regexNer, data)
} }

View File

@ -67,7 +67,7 @@ object SetGivenData {
val tags = val tags =
(ctx.args.meta.tags.getOrElse(Nil) ++ data.tags ++ data.classifyTags).distinct (ctx.args.meta.tags.getOrElse(Nil) ++ data.tags ++ data.classifyTags).distinct
for { for {
_ <- ctx.logger.info(s"Set tags from given data: ${tags}") _ <- ctx.logger.info(s"Set tags from given data: $tags")
e <- ops.linkTags(itemId, tags, collective).attempt e <- ops.linkTags(itemId, tags, collective).attempt
_ <- e.fold( _ <- e.fold(
ex => ctx.logger.warn(s"Error setting tags: ${ex.getMessage}"), ex => ctx.logger.warn(s"Error setting tags: ${ex.getMessage}"),

View File

@ -158,7 +158,7 @@ object TextExtraction {
val extr = Extraction.create[F](ctx.logger, cfg) val extr = Extraction.create[F](ctx.logger, cfg)
extractText[F](ctx, extr, lang)(id) extractText[F](ctx, extr, lang)(id)
.flatMap({ .flatMap {
case res @ ExtractResult.Success(_, _) => case res @ ExtractResult.Success(_, _) =>
res.some.pure[F] res.some.pure[F]
@ -173,15 +173,14 @@ object TextExtraction {
ctx.logger ctx.logger
.warn(s"Cannot extract text: ${ex.getMessage}. Try with converted file") .warn(s"Cannot extract text: ${ex.getMessage}. Try with converted file")
.flatMap(_ => extractTextFallback[F](ctx, cfg, ra, lang)(rest)) .flatMap(_ => extractTextFallback[F](ctx, cfg, ra, lang)(rest))
}) }
} }
/** Returns the fileIds to extract text from. First, the source file /** Returns the fileIds to extract text from. First, the source file is tried. If that
* is tried. If that fails, the converted file is tried. * fails, the converted file is tried.
* *
* If the source file is a PDF, then use the converted file. This * If the source file is a PDF, then use the converted file. This may then already
* may then already contain the text if ocrmypdf is enabled. If it * contain the text if ocrmypdf is enabled. If it is disabled, both files are the same.
* is disabled, both files are the same.
*/ */
private def filesToExtract[F[_]: Sync](ctx: Context[F, _])( private def filesToExtract[F[_]: Sync](ctx: Context[F, _])(
item: ItemData, item: ItemData,

View File

@ -50,7 +50,10 @@ object JoexRoutes {
for { for {
optJob <- app.scheduler.getRunning.map(_.find(_.id == id)) optJob <- app.scheduler.getRunning.map(_.find(_.id == id))
optLog <- optJob.traverse(j => app.findLogs(j.id)) optLog <- optJob.traverse(j => app.findLogs(j.id))
jAndL = for { job <- optJob; log <- optLog } yield mkJobLog(job, log) jAndL = for {
job <- optJob
log <- optLog
} yield mkJobLog(job, log)
resp <- jAndL.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found"))) resp <- jAndL.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found")))
} yield resp } yield resp

View File

@ -46,7 +46,7 @@ object ScanMailboxTask {
userId = ctx.args.account.user userId = ctx.args.account.user
imapConn = ctx.args.imapConnection imapConn = ctx.args.imapConnection
_ <- ctx.logger.info( _ <- ctx.logger.info(
s"Reading mails for user ${userId.id} from ${imapConn.id}/${folders}" s"Reading mails for user ${userId.id} from ${imapConn.id}/$folders"
) )
_ <- importMails(cfg, mailCfg, emil, upload, joex, ctx) _ <- importMails(cfg, mailCfg, emil, upload, joex, ctx)
} yield () } yield ()

View File

@ -10,11 +10,10 @@ import cats.implicits._
import docspell.common.Priority import docspell.common.Priority
/** A counting scheme to indicate a ratio between scheduling high and /** A counting scheme to indicate a ratio between scheduling high and low priority jobs.
* low priority jobs.
* *
* For example high=4, low=1 means: schedule 4 high priority jobs * For example high=4, low=1 means: schedule 4 high priority jobs and then 1 low
* and then 1 low priority job. * priority job.
*/ */
case class CountingScheme(high: Int, low: Int, counter: Int = 0) { case class CountingScheme(high: Int, low: Int, counter: Int = 0) {

View File

@ -14,14 +14,13 @@ import docspell.common.syntax.all._
import io.circe.Decoder import io.circe.Decoder
/** Binds a Task to a name. This is required to lookup the code based /** Binds a Task to a name. This is required to lookup the code based on the taskName in
* on the taskName in the RJob data and to execute it given the * the RJob data and to execute it given the arguments that have to be read from a
* arguments that have to be read from a string. * string.
* *
* Since the scheduler only has a string for the task argument, this * Since the scheduler only has a string for the task argument, this only works for Task
* only works for Task impls that accept a string. There is a * impls that accept a string. There is a convenience constructor that uses circe to
* convenience constructor that uses circe to decode json into some * decode json into some type A.
* type A.
*/ */
case class JobTask[F[_]]( case class JobTask[F[_]](
name: Ident, name: Ident,

View File

@ -8,9 +8,8 @@ package docspell.joex.scheduler
import docspell.common.Ident import docspell.common.Ident
/** This is a mapping from some identifier to a task. This is used by /** This is a mapping from some identifier to a task. This is used by the scheduler to
* the scheduler to lookup an implementation using the taskName field * lookup an implementation using the taskName field of the RJob database record.
* of the RJob database record.
*/ */
final class JobTaskRegistry[F[_]](tasks: Map[Ident, JobTask[F]]) { final class JobTaskRegistry[F[_]](tasks: Map[Ident, JobTask[F]]) {

View File

@ -13,14 +13,12 @@ import fs2.concurrent.SignallingRef
import docspell.joexapi.client.JoexClient import docspell.joexapi.client.JoexClient
import docspell.store.queue._ import docspell.store.queue._
/** A periodic scheduler takes care to submit periodic tasks to the /** A periodic scheduler takes care to submit periodic tasks to the job queue.
* job queue.
* *
* It is run in the background to regularily find a periodic task to * It is run in the background to regularily find a periodic task to execute. If the task
* execute. If the task is due, it will be submitted into the job * is due, it will be submitted into the job queue where it will be picked up by the
* queue where it will be picked up by the scheduler from some joex * scheduler from some joex instance. If it is due in the future, a notification is
* instance. If it is due in the future, a notification is scheduled * scheduled to be received at that time so the task can be looked up again.
* to be received at that time so the task can be looked up again.
*/ */
trait PeriodicScheduler[F[_]] { trait PeriodicScheduler[F[_]] {

View File

@ -53,8 +53,8 @@ final class PeriodicSchedulerImpl[F[_]: Async](
// internal // internal
/** On startup, get all periodic jobs from this scheduler and remove /** On startup, get all periodic jobs from this scheduler and remove the mark, so they
* the mark, so they get picked up again. * get picked up again.
*/ */
def init: F[Unit] = def init: F[Unit] =
logError("Error clearing marks")(store.clearMarks(config.name)) logError("Error clearing marks")(store.clearMarks(config.name))
@ -68,7 +68,7 @@ final class PeriodicSchedulerImpl[F[_]: Async](
go <- logThrow("Error getting next task")( go <- logThrow("Error getting next task")(
store store
.takeNext(config.name, None) .takeNext(config.name, None)
.use({ .use {
case Marked.Found(pj) => case Marked.Found(pj) =>
logger logger
.fdebug(s"Found periodic task '${pj.subject}/${pj.timer.asString}'") *> .fdebug(s"Found periodic task '${pj.subject}/${pj.timer.asString}'") *>
@ -79,7 +79,7 @@ final class PeriodicSchedulerImpl[F[_]: Async](
case Marked.NotMarkable => case Marked.NotMarkable =>
logger.fdebug("Periodic job cannot be marked. Trying again.") *> true logger.fdebug("Periodic job cannot be marked. Trying again.") *> true
.pure[F] .pure[F]
}) }
) )
} yield go } yield go
@ -90,7 +90,7 @@ final class PeriodicSchedulerImpl[F[_]: Async](
else ().pure[F] else ().pure[F]
) )
.flatMap(if (_) Stream.empty else Stream.eval(cancelNotify *> body)) .flatMap(if (_) Stream.empty else Stream.eval(cancelNotify *> body))
.flatMap({ .flatMap {
case true => case true =>
mainLoop mainLoop
case false => case false =>
@ -98,7 +98,7 @@ final class PeriodicSchedulerImpl[F[_]: Async](
waiter.discrete.take(2).drain ++ waiter.discrete.take(2).drain ++
logger.sdebug(s"Notify signal, going into main loop") ++ logger.sdebug(s"Notify signal, going into main loop") ++
mainLoop mainLoop
}) }
} }
def isTriggered(pj: RPeriodicTask, now: Timestamp): Boolean = def isTriggered(pj: RPeriodicTask, now: Timestamp): Boolean =
@ -107,7 +107,7 @@ final class PeriodicSchedulerImpl[F[_]: Async](
def submitJob(pj: RPeriodicTask): F[Boolean] = def submitJob(pj: RPeriodicTask): F[Boolean] =
store store
.findNonFinalJob(pj.id) .findNonFinalJob(pj.id)
.flatMap({ .flatMap {
case Some(job) => case Some(job) =>
logger.finfo[F]( logger.finfo[F](
s"There is already a job with non-final state '${job.state}' in the queue" s"There is already a job with non-final state '${job.state}' in the queue"
@ -116,7 +116,7 @@ final class PeriodicSchedulerImpl[F[_]: Async](
case None => case None =>
logger.finfo[F](s"Submitting job for periodic task '${pj.task.id}'") *> logger.finfo[F](s"Submitting job for periodic task '${pj.task.id}'") *>
pj.toJob.flatMap(queue.insert) *> notifyJoex *> true.pure[F] pj.toJob.flatMap(queue.insert) *> notifyJoex *> true.pure[F]
}) }
def notifyJoex: F[Unit] = def notifyJoex: F[Unit] =
sch.notifyChange *> store.findJoexNodes.flatMap( sch.notifyChange *> store.findJoexNodes.flatMap(
@ -145,12 +145,12 @@ final class PeriodicSchedulerImpl[F[_]: Async](
def cancelNotify: F[Unit] = def cancelNotify: F[Unit] =
state state
.modify(_.clearNotify) .modify(_.clearNotify)
.flatMap({ .flatMap {
case Some(fb) => case Some(fb) =>
fb.cancel fb.cancel
case None => case None =>
().pure[F] ().pure[F]
}) }
private def logError(msg: => String)(fa: F[Unit]): F[Unit] = private def logError(msg: => String)(fa: F[Unit]): F[Unit] =
fa.attempt.flatMap { fa.attempt.flatMap {
@ -159,12 +159,10 @@ final class PeriodicSchedulerImpl[F[_]: Async](
} }
private def logThrow[A](msg: => String)(fa: F[A]): F[A] = private def logThrow[A](msg: => String)(fa: F[A]): F[A] =
fa.attempt fa.attempt.flatMap {
.flatMap({
case r @ Right(_) => (r: Either[Throwable, A]).pure[F] case r @ Right(_) => (r: Either[Throwable, A]).pure[F]
case l @ Left(ex) => logger.ferror(ex)(msg).map(_ => (l: Either[Throwable, A])) case l @ Left(ex) => logger.ferror(ex)(msg).map(_ => (l: Either[Throwable, A]))
}) }.rethrow
.rethrow
} }
object PeriodicSchedulerImpl { object PeriodicSchedulerImpl {

View File

@ -26,13 +26,11 @@ trait Scheduler[F[_]] {
/** Requests to shutdown the scheduler. /** Requests to shutdown the scheduler.
* *
* The scheduler will not take any new jobs from the queue. If * The scheduler will not take any new jobs from the queue. If there are still running
* there are still running jobs, it waits for them to complete. * jobs, it waits for them to complete. when the cancelAll flag is set to true, it
* when the cancelAll flag is set to true, it cancels all running * cancels all running jobs.
* jobs.
* *
* The returned F[Unit] can be evaluated to wait for all that to * The returned F[Unit] can be evaluated to wait for all that to complete.
* complete.
*/ */
def shutdown(cancelAll: Boolean): F[Unit] def shutdown(cancelAll: Boolean): F[Unit]

View File

@ -36,8 +36,8 @@ final class SchedulerImpl[F[_]: Async](
private[this] val logger = getLogger private[this] val logger = getLogger
/** On startup, get all jobs in state running from this scheduler /** On startup, get all jobs in state running from this scheduler and put them into
* and put them into waiting state, so they get picked up again. * waiting state, so they get picked up again.
*/ */
def init: F[Unit] = def init: F[Unit] =
QJob.runningToWaiting(config.name, store) QJob.runningToWaiting(config.name, store)
@ -132,7 +132,7 @@ final class SchedulerImpl[F[_]: Async](
else ().pure[F] else ().pure[F]
) )
.flatMap(if (_) Stream.empty else Stream.eval(body)) .flatMap(if (_) Stream.empty else Stream.eval(body))
.flatMap({ .flatMap {
case true => case true =>
mainLoop mainLoop
case false => case false =>
@ -140,7 +140,7 @@ final class SchedulerImpl[F[_]: Async](
waiter.discrete.take(2).drain ++ waiter.discrete.take(2).drain ++
logger.sdebug(s"Notify signal, going into main loop") ++ logger.sdebug(s"Notify signal, going into main loop") ++
mainLoop mainLoop
}) }
} }
private def executeCancel(job: RJob): F[Unit] = { private def executeCancel(job: RJob): F[Unit] = {
@ -214,7 +214,7 @@ final class SchedulerImpl[F[_]: Async](
): Task[F, String, Unit] = ): Task[F, String, Unit] =
task task
.mapF(fa => onStart(job) *> logger.fdebug("Starting task now") *> fa) .mapF(fa => onStart(job) *> logger.fdebug("Starting task now") *> fa)
.mapF(_.attempt.flatMap({ .mapF(_.attempt.flatMap {
case Right(()) => case Right(()) =>
logger.info(s"Job execution successful: ${job.info}") logger.info(s"Job execution successful: ${job.info}")
ctx.logger.info("Job execution successful") *> ctx.logger.info("Job execution successful") *>
@ -239,7 +239,7 @@ final class SchedulerImpl[F[_]: Async](
.map(_ => JobState.Stuck: JobState) .map(_ => JobState.Stuck: JobState)
} }
} }
})) })
.mapF(_.attempt.flatMap { .mapF(_.attempt.flatMap {
case Right(jstate) => case Right(jstate) =>
onFinish(job, jstate) onFinish(job, jstate)
@ -262,12 +262,12 @@ final class SchedulerImpl[F[_]: Async](
.map(fiber => .map(fiber =>
logger.fdebug(s"Cancelling job ${job.info}") *> logger.fdebug(s"Cancelling job ${job.info}") *>
fiber.cancel *> fiber.cancel *>
onCancel.attempt.map({ onCancel.attempt.map {
case Right(_) => () case Right(_) => ()
case Left(ex) => case Left(ex) =>
logger.error(ex)(s"Task's cancelling code failed. Job ${job.info}.") logger.error(ex)(s"Task's cancelling code failed. Job ${job.info}.")
() ()
}) *> } *>
state.modify(_.markCancelled(job)) *> state.modify(_.markCancelled(job)) *>
onFinish(job, JobState.Cancelled) *> onFinish(job, JobState.Cancelled) *>
ctx.logger.warn("Job has been cancelled.") *> ctx.logger.warn("Job has been cancelled.") *>

View File

@ -51,7 +51,7 @@ object JoexClient {
if (succ) () if (succ) ()
else else
logger.warn( logger.warn(
s"Notifying Joex instance '${base.asString}' returned with failure: ${msg}" s"Notifying Joex instance '${base.asString}' returned with failure: $msg"
) )
case Left(ex) => case Left(ex) =>
logger.warn( logger.warn(

View File

@ -14,8 +14,7 @@ import docspell.query.ItemQuery.Expr.NotExpr
import docspell.query.ItemQuery.Expr.OrExpr import docspell.query.ItemQuery.Expr.OrExpr
import docspell.query.ItemQuery._ import docspell.query.ItemQuery._
/** Currently, fulltext in a query is only supported when in "root /** Currently, fulltext in a query is only supported when in "root AND" position
* AND" position
*/ */
object FulltextExtract { object FulltextExtract {
@ -45,15 +44,15 @@ object FulltextExtract {
def findFulltext(expr: Expr): Result = def findFulltext(expr: Expr): Result =
lookForFulltext(expr) lookForFulltext(expr)
/** Extracts the fulltext node from the given expr and returns it /** Extracts the fulltext node from the given expr and returns it together with the expr
* together with the expr without that node. * without that node.
*/ */
private def lookForFulltext(expr: Expr): Result = private def lookForFulltext(expr: Expr): Result =
expr match { expr match {
case Expr.Fulltext(ftq) => case Expr.Fulltext(ftq) =>
Result.SuccessNoExpr(ftq) Result.SuccessNoExpr(ftq)
case Expr.AndExpr(inner) => case Expr.AndExpr(inner) =>
inner.collect({ case Expr.Fulltext(fq) => fq }) match { inner.collect { case Expr.Fulltext(fq) => fq } match {
case Nil => case Nil =>
checkPosition(expr, 0) checkPosition(expr, 0)
case e :: Nil => case e :: Nil =>

View File

@ -10,12 +10,10 @@ import cats.data.{NonEmptyList => Nel}
import docspell.query.ItemQuery.Attr.{DateAttr, IntAttr, StringAttr} import docspell.query.ItemQuery.Attr.{DateAttr, IntAttr, StringAttr}
/** A query evaluates to `true` or `false` given enough details about /** A query evaluates to `true` or `false` given enough details about an item.
* an item.
* *
* It may consist of (field,op,value) tuples that specify some checks * It may consist of (field,op,value) tuples that specify some checks against a specific
* against a specific field of an item using some operator or a * field of an item using some operator or a combination thereof.
* combination thereof.
*/ */
final case class ItemQuery(expr: ItemQuery.Expr, raw: Option[String]) { final case class ItemQuery(expr: ItemQuery.Expr, raw: Option[String]) {
def findFulltext: FulltextExtract.Result = def findFulltext: FulltextExtract.Result =

View File

@ -45,7 +45,7 @@ object ParseFailure {
def render: String = { def render: String = {
val opts = expected.mkString(", ") val opts = expected.mkString(", ")
val dots = if (exhaustive) "" else "…" val dots = if (exhaustive) "" else "…"
s"Expected: ${opts}${dots}" s"Expected: $opts$dots"
} }
} }
@ -57,11 +57,11 @@ object ParseFailure {
) )
private[query] def packMsg(msg: Nel[Message]): Nel[Message] = { private[query] def packMsg(msg: Nel[Message]): Nel[Message] = {
val expectMsg = combineExpected(msg.collect({ case em: ExpectMessage => em })) val expectMsg = combineExpected(msg.collect { case em: ExpectMessage => em })
.sortBy(_.offset) .sortBy(_.offset)
.headOption .headOption
val simpleMsg = msg.collect({ case sm: SimpleMessage => sm }) val simpleMsg = msg.collect { case sm: SimpleMessage => sm }
Nel.fromListUnsafe((simpleMsg ++ expectMsg).sortBy(_.offset)) Nel.fromListUnsafe((simpleMsg ++ expectMsg).sortBy(_.offset))
} }
@ -69,13 +69,13 @@ object ParseFailure {
private[query] def combineExpected(msg: List[ExpectMessage]): List[ExpectMessage] = private[query] def combineExpected(msg: List[ExpectMessage]): List[ExpectMessage] =
msg msg
.groupBy(_.offset) .groupBy(_.offset)
.map({ case (offset, es) => .map { case (offset, es) =>
ExpectMessage( ExpectMessage(
offset, offset,
es.flatMap(_.expected).distinct.sorted, es.flatMap(_.expected).distinct.sorted,
es.forall(_.exhaustive) es.forall(_.exhaustive)
) )
}) }
.toList .toList
private[query] def expectationToMsg(e: Parser.Expectation): Message = private[query] def expectationToMsg(e: Parser.Expectation): Message =
@ -89,7 +89,7 @@ object ParseFailure {
case InRange(offset, lower, upper) => case InRange(offset, lower, upper) =>
if (lower == upper) ExpectMessage(offset, List(lower.toString), true) if (lower == upper) ExpectMessage(offset, List(lower.toString), true)
else { else {
val expect = s"${lower}-${upper}" val expect = s"$lower-$upper"
ExpectMessage(offset, List(expect), true) ExpectMessage(offset, List(expect), true)
} }

View File

@ -13,8 +13,8 @@ import docspell.query.ItemQuery._
object ExprUtil { object ExprUtil {
/** Does some basic transformation, like unfolding nested and trees /** Does some basic transformation, like unfolding nested and trees containing one value
* containing one value etc. * etc.
*/ */
def reduce(expr: Expr): Expr = def reduce(expr: Expr): Expr =
expr match { expr match {

View File

@ -54,12 +54,12 @@ object Config {
lazy val ipParts = ip.split('.') lazy val ipParts = ip.split('.')
def checkSingle(pattern: String): Boolean = def checkSingle(pattern: String): Boolean =
pattern == ip || (inet.isLoopbackAddress && pattern == "127.0.0.1") || (pattern pattern == ip || (inet.isLoopbackAddress && pattern == "127.0.0.1") || pattern
.split('.') .split('.')
.zip(ipParts) .zip(ipParts)
.foldLeft(true) { case (r, (a, b)) => .foldLeft(true) { case (r, (a, b)) =>
r && (a == "*" || a == b) r && (a == "*" || a == b)
}) }
ips.exists(checkSingle) ips.exists(checkSingle)
} }

View File

@ -579,7 +579,7 @@ trait Conversions {
) )
def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] = def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] =
timeId.map({ case (id, now) => timeId.map { case (id, now) =>
RSource( RSource(
id, id,
cid, cid,
@ -593,7 +593,7 @@ trait Conversions {
s.fileFilter, s.fileFilter,
s.language s.language
) )
}) }
def changeSource[F[_]](s: Source, coll: Ident): RSource = def changeSource[F[_]](s: Source, coll: Ident): RSource =
RSource( RSource(
@ -615,9 +615,9 @@ trait Conversions {
Equipment(re.eid, re.name, re.created, re.notes, re.use) Equipment(re.eid, re.name, re.created, re.notes, re.use)
def newEquipment[F[_]: Sync](e: Equipment, cid: Ident): F[REquipment] = def newEquipment[F[_]: Sync](e: Equipment, cid: Ident): F[REquipment] =
timeId.map({ case (id, now) => timeId.map { case (id, now) =>
REquipment(id, cid, e.name.trim, now, now, e.notes, e.use) REquipment(id, cid, e.name.trim, now, now, e.notes, e.use)
}) }
def changeEquipment[F[_]: Sync](e: Equipment, cid: Ident): F[REquipment] = def changeEquipment[F[_]: Sync](e: Equipment, cid: Ident): F[REquipment] =
Timestamp Timestamp

View File

@ -29,7 +29,7 @@ object ClientRequestInfo {
scheme <- NonEmptyList.fromList(getProtocol(req).toList) scheme <- NonEmptyList.fromList(getProtocol(req).toList)
host <- getHostname(req) host <- getHostname(req)
port = xForwardedPort(req).getOrElse(serverPort) port = xForwardedPort(req).getOrElse(serverPort)
hostPort = if (port == 80 || port == 443) host else s"${host}:${port}" hostPort = if (port == 80 || port == 443) host else s"$host:$port"
} yield LenientUri(scheme, Some(hostPort), LenientUri.EmptyPath, None, None) } yield LenientUri(scheme, Some(hostPort), LenientUri.EmptyPath, None, None)
def getHostname[F[_]](req: Request[F]): Option[String] = def getHostname[F[_]](req: Request[F]): Option[String] =

View File

@ -56,5 +56,5 @@ object AdminRoutes {
private def compareSecret(s1: String)(s2: String): Boolean = private def compareSecret(s1: String)(s2: String): Boolean =
s1.length > 0 && s1.length == s2.length && s1.length > 0 && s1.length == s2.length &&
s1.zip(s2).forall({ case (a, b) => a == b }) s1.zip(s2).forall { case (a, b) => a == b }
} }

View File

@ -22,12 +22,12 @@ import io.circe.{Decoder, Encoder}
trait DoobieMeta extends EmilDoobieMeta { trait DoobieMeta extends EmilDoobieMeta {
implicit val sqlLogging = LogHandler({ implicit val sqlLogging = LogHandler {
case e @ Success(_, _, _, _) => case e @ Success(_, _, _, _) =>
DoobieMeta.logger.trace("SQL " + e) DoobieMeta.logger.trace("SQL " + e)
case e => case e =>
DoobieMeta.logger.error(s"SQL Failure: $e") DoobieMeta.logger.error(s"SQL Failure: $e")
}) }
def jsonMeta[A](implicit d: Decoder[A], e: Encoder[A]): Meta[A] = def jsonMeta[A](implicit d: Decoder[A], e: Encoder[A]): Meta[A] =
Meta[String].imap(str => str.parseJsonAs[A].fold(ex => throw ex, identity))(a => Meta[String].imap(str => str.parseJsonAs[A].fold(ex => throw ex, identity))(a =>

View File

@ -22,7 +22,7 @@ object FlywayMigrate {
logger.info("Running db migrations...") logger.info("Running db migrations...")
val locations = jdbc.dbmsName match { val locations = jdbc.dbmsName match {
case Some(dbtype) => case Some(dbtype) =>
List(s"classpath:db/migration/${dbtype}") List(s"classpath:db/migration/$dbtype")
case None => case None =>
logger.warn( logger.warn(
s"Cannot read database name from jdbc url: ${jdbc.url}. Go with H2" s"Cannot read database name from jdbc url: ${jdbc.url}. Go with H2"

View File

@ -22,13 +22,12 @@ sealed trait FromExpr {
def leftJoin(sel: Select, alias: String, on: Condition): Joined = def leftJoin(sel: Select, alias: String, on: Condition): Joined =
leftJoin(Relation.SubSelect(sel, alias), on) leftJoin(Relation.SubSelect(sel, alias), on)
/** Prepends the given from expression to existing joins. It will /** Prepends the given from expression to existing joins. It will replace the current
* replace the current [[FromExpr.From]] value. * [[FromExpr.From]] value.
* *
* If this is a [[FromExpr.From]], it is replaced by the given * If this is a [[FromExpr.From]], it is replaced by the given expression. If this is a
* expression. If this is a [[FromExpr.Joined]] then the given * [[FromExpr.Joined]] then the given expression replaces the current `From` and the
* expression replaces the current `From` and the joins are * joins are prepended to the existing joins.
* prepended to the existing joins.
*/ */
def prepend(fe: FromExpr): FromExpr def prepend(fe: FromExpr): FromExpr
} }

View File

@ -46,9 +46,8 @@ object QAttachment {
.foldMonoid .foldMonoid
} }
/** Deletes an attachment, its related source and meta data records. /** Deletes an attachment, its related source and meta data records. It will only delete
* It will only delete an related archive file, if this is the last * an related archive file, if this is the last attachment in that archive.
* attachment in that archive.
*/ */
def deleteSingleAttachment[F[_]: Sync]( def deleteSingleAttachment[F[_]: Sync](
store: Store[F] store: Store[F]
@ -77,9 +76,9 @@ object QAttachment {
} yield n + k + f } yield n + k + f
} }
/** This deletes the attachment and *all* its related files. This used /** This deletes the attachment and *all* its related files. This used when deleting an
* when deleting an item and should not be used to delete a * item and should not be used to delete a *single* attachment where the item should
* *single* attachment where the item should stay. * stay.
*/ */
private def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] = private def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] =
for { for {

View File

@ -368,8 +368,8 @@ object QItem {
from.query[ListItem].stream from.query[ListItem].stream
} }
/** Same as `findItems` but resolves the tags for each item. Note that /** Same as `findItems` but resolves the tags for each item. Note that this is
* this is implemented by running an additional query per item. * implemented by running an additional query per item.
*/ */
def findItemsWithTags( def findItemsWithTags(
collective: Ident, collective: Ident,

View File

@ -43,14 +43,14 @@ object QJob {
else ().pure[F] else ().pure[F]
} }
.find(_.isRight) .find(_.isRight)
.flatMap({ .flatMap {
case Right(job) => case Right(job) =>
Stream.emit(job) Stream.emit(job)
case Left(_) => case Left(_) =>
Stream Stream
.eval(logger.fwarn[F]("Cannot mark job, even after retrying. Give up.")) .eval(logger.fwarn[F]("Cannot mark job, even after retrying. Give up."))
.map(_ => None) .map(_ => None)
}) }
.compile .compile
.last .last
.map(_.flatten) .map(_.flatten)

Some files were not shown because too many files have changed in this diff Show More