mirror of
https://github.com/TheAnachronism/docspell.git
synced 2025-06-06 15:15:58 +00:00
Reformat with scalafmt 3.0.0
This commit is contained in:
parent
5a2a0295ef
commit
e4fecefaea
20
build.sbt
20
build.sbt
@ -157,10 +157,10 @@ val buildInfoSettings = Seq(
|
|||||||
val openapiScalaSettings = Seq(
|
val openapiScalaSettings = Seq(
|
||||||
openapiScalaConfig := ScalaConfig()
|
openapiScalaConfig := ScalaConfig()
|
||||||
.withJson(ScalaJson.circeSemiauto)
|
.withJson(ScalaJson.circeSemiauto)
|
||||||
.addMapping(CustomMapping.forType({ case TypeDef("LocalDateTime", _) =>
|
.addMapping(CustomMapping.forType { case TypeDef("LocalDateTime", _) =>
|
||||||
TypeDef("Timestamp", Imports("docspell.common.Timestamp"))
|
TypeDef("Timestamp", Imports("docspell.common.Timestamp"))
|
||||||
}))
|
})
|
||||||
.addMapping(CustomMapping.forFormatType({
|
.addMapping(CustomMapping.forFormatType {
|
||||||
case "ident" =>
|
case "ident" =>
|
||||||
field => field.copy(typeDef = TypeDef("Ident", Imports("docspell.common.Ident")))
|
field => field.copy(typeDef = TypeDef("Ident", Imports("docspell.common.Ident")))
|
||||||
case "accountid" =>
|
case "accountid" =>
|
||||||
@ -246,7 +246,7 @@ val openapiScalaSettings = Seq(
|
|||||||
field =>
|
field =>
|
||||||
field
|
field
|
||||||
.copy(typeDef = TypeDef("Duration", Imports("docspell.common.Duration")))
|
.copy(typeDef = TypeDef("Duration", Imports("docspell.common.Duration")))
|
||||||
}))
|
})
|
||||||
)
|
)
|
||||||
|
|
||||||
// --- Modules
|
// --- Modules
|
||||||
@ -287,7 +287,7 @@ val files = project
|
|||||||
val files = (base ** (_.isFile)).pair(sbt.io.Path.relativeTo(base))
|
val files = (base ** (_.isFile)).pair(sbt.io.Path.relativeTo(base))
|
||||||
val lines = files.toList.map(_._2).map { s =>
|
val lines = files.toList.map(_._2).map { s =>
|
||||||
val ident = s.replaceAll("[^a-zA-Z0-9_]+", "_")
|
val ident = s.replaceAll("[^a-zA-Z0-9_]+", "_")
|
||||||
ident -> s"""val $ident = createUrl("${s}")"""
|
ident -> s"""val $ident = createUrl("$s")"""
|
||||||
}
|
}
|
||||||
val content = s"""package docspell.files
|
val content = s"""package docspell.files
|
||||||
|
|
||||||
@ -759,7 +759,7 @@ def packageTools(logger: Logger, dir: File, version: String): Seq[File] = {
|
|||||||
val target = dir / "target"
|
val target = dir / "target"
|
||||||
IO.delete(target)
|
IO.delete(target)
|
||||||
IO.createDirectory(target)
|
IO.createDirectory(target)
|
||||||
val archive = target / s"docspell-tools-${version}.zip"
|
val archive = target / s"docspell-tools-$version.zip"
|
||||||
logger.info(s"Packaging tools to $archive ...")
|
logger.info(s"Packaging tools to $archive ...")
|
||||||
val webext = target / "docspell-firefox-extension.xpi"
|
val webext = target / "docspell-firefox-extension.xpi"
|
||||||
val wx = dir / "webextension"
|
val wx = dir / "webextension"
|
||||||
@ -782,13 +782,13 @@ def packageTools(logger: Logger, dir: File, version: String): Seq[File] = {
|
|||||||
(dir ** "*")
|
(dir ** "*")
|
||||||
.filter(f => !excludes.exists(p => f.absolutePath.startsWith(p.absolutePath)))
|
.filter(f => !excludes.exists(p => f.absolutePath.startsWith(p.absolutePath)))
|
||||||
.pair(sbt.io.Path.relativeTo(dir))
|
.pair(sbt.io.Path.relativeTo(dir))
|
||||||
.map({ case (f, name) => (f, s"docspell-tools-${version}/$name") })
|
.map { case (f, name) => (f, s"docspell-tools-$version/$name") }
|
||||||
|
|
||||||
IO.zip(
|
IO.zip(
|
||||||
Seq(
|
Seq(
|
||||||
webext -> s"docspell-tools-${version}/firefox/docspell-extension.xpi",
|
webext -> s"docspell-tools-$version/firefox/docspell-extension.xpi",
|
||||||
wx / "native/app_manifest.json" -> s"docspell-tools-${version}/firefox/native/app_manifest.json",
|
wx / "native/app_manifest.json" -> s"docspell-tools-$version/firefox/native/app_manifest.json",
|
||||||
wx / "native/native.py" -> s"docspell-tools-${version}/firefox/native/native.py"
|
wx / "native/native.py" -> s"docspell-tools-$version/firefox/native/native.py"
|
||||||
) ++ files,
|
) ++ files,
|
||||||
archive,
|
archive,
|
||||||
None
|
None
|
||||||
|
@ -155,10 +155,8 @@ final class StanfordTextClassifier[F[_]: Async](cfg: TextClassifierConfig)
|
|||||||
case class TrainResult(score: Double, model: ClassifierModel)
|
case class TrainResult(score: Double, model: ClassifierModel)
|
||||||
|
|
||||||
def prepend(pre: String, data: Map[String, String]): Map[String, String] =
|
def prepend(pre: String, data: Map[String, String]): Map[String, String] =
|
||||||
data.toList
|
data.toList.map { case (k, v) =>
|
||||||
.map({ case (k, v) =>
|
|
||||||
if (k.startsWith(pre)) (k, v)
|
if (k.startsWith(pre)) (k, v)
|
||||||
else (pre + k, v)
|
else (pre + k, v)
|
||||||
})
|
}.toMap
|
||||||
.toMap
|
|
||||||
}
|
}
|
||||||
|
@ -32,7 +32,7 @@ object Domain {
|
|||||||
Tld
|
Tld
|
||||||
.findTld(str)
|
.findTld(str)
|
||||||
.map(tld => (str.dropRight(tld.length), tld))
|
.map(tld => (str.dropRight(tld.length), tld))
|
||||||
.map({ case (names, tld) =>
|
.map { case (names, tld) =>
|
||||||
names.split('.').toList match {
|
names.split('.').toList match {
|
||||||
case Nil => Left(s"Not a domain: $str")
|
case Nil => Left(s"Not a domain: $str")
|
||||||
case segs
|
case segs
|
||||||
@ -43,7 +43,7 @@ object Domain {
|
|||||||
Right(Domain(NonEmptyList.fromListUnsafe(segs), tld))
|
Right(Domain(NonEmptyList.fromListUnsafe(segs), tld))
|
||||||
case _ => Left(s"Not a domain: $str")
|
case _ => Left(s"Not a domain: $str")
|
||||||
}
|
}
|
||||||
})
|
}
|
||||||
.getOrElse(Left(s"Not a domain $str"))
|
.getOrElse(Left(s"Not a domain $str"))
|
||||||
|
|
||||||
def isDomain(str: String): Boolean =
|
def isDomain(str: String): Boolean =
|
||||||
|
@ -160,11 +160,11 @@ object DateFind {
|
|||||||
Reader(words => Nel.of(reader, more: _*).map(_.read(words)).reduce)
|
Reader(words => Nel.of(reader, more: _*).map(_.read(words)).reduce)
|
||||||
|
|
||||||
def readFirst[A](f: Word => Option[A]): Reader[A] =
|
def readFirst[A](f: Word => Option[A]): Reader[A] =
|
||||||
Reader({
|
Reader {
|
||||||
case Nil => Result.Failure
|
case Nil => Result.Failure
|
||||||
case a :: as =>
|
case a :: as =>
|
||||||
f(a).map(value => Result.Success(value, as)).getOrElse(Result.Failure)
|
f(a).map(value => Result.Success(value, as)).getOrElse(Result.Failure)
|
||||||
})
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
sealed trait Result[+A] {
|
sealed trait Result[+A] {
|
||||||
|
@ -15,7 +15,7 @@ object MonthName {
|
|||||||
|
|
||||||
private def merge(n0: List[List[String]], ns: List[List[String]]*): List[List[String]] =
|
private def merge(n0: List[List[String]], ns: List[List[String]]*): List[List[String]] =
|
||||||
ns.foldLeft(n0) { (res, el) =>
|
ns.foldLeft(n0) { (res, el) =>
|
||||||
res.zip(el).map({ case (a, b) => a ++ b })
|
res.zip(el).map { case (a, b) => a ++ b }
|
||||||
}
|
}
|
||||||
|
|
||||||
private def forLang(lang: Language): List[List[String]] =
|
private def forLang(lang: Language): List[List[String]] =
|
||||||
|
@ -39,8 +39,8 @@ object Annotator {
|
|||||||
* - full: the complete stanford pipeline is used
|
* - full: the complete stanford pipeline is used
|
||||||
* - basic: only the ner classifier is used
|
* - basic: only the ner classifier is used
|
||||||
*
|
*
|
||||||
* Additionally, if there is a regexNer-file specified, the regexner annotator is
|
* Additionally, if there is a regexNer-file specified, the regexner annotator is also
|
||||||
* also run. In case the full pipeline is used, this is already included.
|
* run. In case the full pipeline is used, this is already included.
|
||||||
*/
|
*/
|
||||||
def apply[F[_]: Sync](mode: NlpMode)(settings: NlpSettings): Annotator[F] =
|
def apply[F[_]: Sync](mode: NlpMode)(settings: NlpSettings): Annotator[F] =
|
||||||
mode match {
|
mode match {
|
||||||
|
@ -21,10 +21,9 @@ import edu.stanford.nlp.ie.crf.CRFClassifier
|
|||||||
import edu.stanford.nlp.ling.{CoreAnnotations, CoreLabel}
|
import edu.stanford.nlp.ling.{CoreAnnotations, CoreLabel}
|
||||||
import org.log4s.getLogger
|
import org.log4s.getLogger
|
||||||
|
|
||||||
/** This is only using the CRFClassifier without building an analysis
|
/** This is only using the CRFClassifier without building an analysis pipeline. The
|
||||||
* pipeline. The ner-classifier cannot use results from POS-tagging
|
* ner-classifier cannot use results from POS-tagging etc. and is therefore not as good
|
||||||
* etc. and is therefore not as good as the [[StanfordNerAnnotator]].
|
* as the [[StanfordNerAnnotator]]. But it uses less memory, while still being not bad.
|
||||||
* But it uses less memory, while still being not bad.
|
|
||||||
*/
|
*/
|
||||||
object BasicCRFAnnotator {
|
object BasicCRFAnnotator {
|
||||||
private[this] val logger = getLogger
|
private[this] val logger = getLogger
|
||||||
|
@ -17,8 +17,8 @@ import docspell.common._
|
|||||||
|
|
||||||
import org.log4s.getLogger
|
import org.log4s.getLogger
|
||||||
|
|
||||||
/** Creating the StanfordCoreNLP pipeline is quite expensive as it
|
/** Creating the StanfordCoreNLP pipeline is quite expensive as it involves IO and
|
||||||
* involves IO and initializing large objects.
|
* initializing large objects.
|
||||||
*
|
*
|
||||||
* Therefore, the instances are cached, because they are thread-safe.
|
* Therefore, the instances are cached, because they are thread-safe.
|
||||||
*
|
*
|
||||||
|
@ -22,13 +22,11 @@ object StanfordNerAnnotator {
|
|||||||
/** Runs named entity recognition on the given `text`.
|
/** Runs named entity recognition on the given `text`.
|
||||||
*
|
*
|
||||||
* This uses the classifier pipeline from stanford-nlp, see
|
* This uses the classifier pipeline from stanford-nlp, see
|
||||||
* https://nlp.stanford.edu/software/CRF-NER.html. Creating these
|
* https://nlp.stanford.edu/software/CRF-NER.html. Creating these classifiers is quite
|
||||||
* classifiers is quite expensive, it involves loading large model
|
* expensive, it involves loading large model files. The classifiers are thread-safe
|
||||||
* files. The classifiers are thread-safe and so they are cached.
|
* and so they are cached. The `cacheKey` defines the "slot" where classifiers are
|
||||||
* The `cacheKey` defines the "slot" where classifiers are stored
|
* stored and retrieved. If for a given `cacheKey` the `settings` change, a new
|
||||||
* and retrieved. If for a given `cacheKey` the `settings` change,
|
* classifier must be created. It will then replace the previous one.
|
||||||
* a new classifier must be created. It will then replace the
|
|
||||||
* previous one.
|
|
||||||
*/
|
*/
|
||||||
def nerAnnotate(nerClassifier: StanfordCoreNLP, text: String): Vector[NerLabel] = {
|
def nerAnnotate(nerClassifier: StanfordCoreNLP, text: String): Vector[NerLabel] = {
|
||||||
val doc = new CoreDocument(text)
|
val doc = new CoreDocument(text)
|
||||||
|
@ -17,18 +17,16 @@ object StanfordNerSettings {
|
|||||||
|
|
||||||
/** Settings for configuring the stanford NER pipeline.
|
/** Settings for configuring the stanford NER pipeline.
|
||||||
*
|
*
|
||||||
* The language is mandatory, only the provided ones are supported.
|
* The language is mandatory, only the provided ones are supported. The `highRecall`
|
||||||
* The `highRecall` only applies for non-English languages. For
|
* only applies for non-English languages. For non-English languages the english
|
||||||
* non-English languages the english classifier is run as second
|
* classifier is run as second classifier and if `highRecall` is true, then it will be
|
||||||
* classifier and if `highRecall` is true, then it will be used to
|
* used to tag untagged tokens. This may lead to a lot of false positives, but since
|
||||||
* tag untagged tokens. This may lead to a lot of false positives,
|
* English is omnipresent in other languages, too it depends on the use case for
|
||||||
* but since English is omnipresent in other languages, too it
|
* whether this is useful or not.
|
||||||
* depends on the use case for whether this is useful or not.
|
|
||||||
*
|
*
|
||||||
* The `regexNer` allows to specify a text file as described here:
|
* The `regexNer` allows to specify a text file as described here:
|
||||||
* https://nlp.stanford.edu/software/regexner.html. This will be used
|
* https://nlp.stanford.edu/software/regexner.html. This will be used as a last step to
|
||||||
* as a last step to tag untagged tokens using the provided list of
|
* tag untagged tokens using the provided list of regexps.
|
||||||
* regexps.
|
|
||||||
*/
|
*/
|
||||||
case class Full(
|
case class Full(
|
||||||
lang: NLPLanguage,
|
lang: NLPLanguage,
|
||||||
@ -36,7 +34,8 @@ object StanfordNerSettings {
|
|||||||
regexNer: Option[Path]
|
regexNer: Option[Path]
|
||||||
) extends StanfordNerSettings
|
) extends StanfordNerSettings
|
||||||
|
|
||||||
/** Not all languages are supported with predefined statistical models. This allows to provide regexps only.
|
/** Not all languages are supported with predefined statistical models. This allows to
|
||||||
|
* provide regexps only.
|
||||||
*/
|
*/
|
||||||
case class RegexOnly(regexNerFile: Path) extends StanfordNerSettings
|
case class RegexOnly(regexNerFile: Path) extends StanfordNerSettings
|
||||||
|
|
||||||
|
@ -37,9 +37,9 @@ class StanfordTextClassifierSuite extends FunSuite {
|
|||||||
.repeat
|
.repeat
|
||||||
.take(10)
|
.take(10)
|
||||||
)
|
)
|
||||||
.flatMap({ case (a, b) =>
|
.flatMap { case (a, b) =>
|
||||||
Stream.emits(Seq(a, b))
|
Stream.emits(Seq(a, b))
|
||||||
})
|
}
|
||||||
.covary[IO]
|
.covary[IO]
|
||||||
|
|
||||||
val modelExists = {
|
val modelExists = {
|
||||||
|
@ -41,6 +41,6 @@ private[auth] object TokenUtil {
|
|||||||
|
|
||||||
def constTimeEq(s1: String, s2: String): Boolean =
|
def constTimeEq(s1: String, s2: String): Boolean =
|
||||||
s1.zip(s2)
|
s1.zip(s2)
|
||||||
.foldLeft(true)({ case (r, (c1, c2)) => r & c1 == c2 }) & s1.length == s2.length
|
.foldLeft(true) { case (r, (c1, c2)) => r & c1 == c2 } & s1.length == s2.length
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -18,8 +18,8 @@ import docspell.store.queries.QItem
|
|||||||
|
|
||||||
trait CreateIndex[F[_]] {
|
trait CreateIndex[F[_]] {
|
||||||
|
|
||||||
/** Low-level function to re-index data. It is not submitted as a job,
|
/** Low-level function to re-index data. It is not submitted as a job, but invoked on
|
||||||
* but invoked on the current machine.
|
* the current machine.
|
||||||
*/
|
*/
|
||||||
def reIndexData(
|
def reIndexData(
|
||||||
logger: Logger[F],
|
logger: Logger[F],
|
||||||
|
@ -84,9 +84,9 @@ object Merge {
|
|||||||
nextPos <- store.transact(RAttachment.nextPosition(target))
|
nextPos <- store.transact(RAttachment.nextPosition(target))
|
||||||
attachs <- store.transact(items.tail.traverse(id => RAttachment.findByItem(id)))
|
attachs <- store.transact(items.tail.traverse(id => RAttachment.findByItem(id)))
|
||||||
attachFlat = attachs.flatMap(_.toList)
|
attachFlat = attachs.flatMap(_.toList)
|
||||||
n <- attachFlat.zipWithIndex.traverse({ case (a, idx) =>
|
n <- attachFlat.zipWithIndex.traverse { case (a, idx) =>
|
||||||
store.transact(RAttachment.updateItemId(a.id, target, nextPos + idx))
|
store.transact(RAttachment.updateItemId(a.id, target, nextPos + idx))
|
||||||
})
|
}
|
||||||
} yield n.sum
|
} yield n.sum
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -63,8 +63,8 @@ trait OCollective[F[_]] {
|
|||||||
|
|
||||||
def startEmptyTrash(args: EmptyTrashArgs): F[Unit]
|
def startEmptyTrash(args: EmptyTrashArgs): F[Unit]
|
||||||
|
|
||||||
/** Submits a task that (re)generates the preview images for all
|
/** Submits a task that (re)generates the preview images for all attachments of the
|
||||||
* attachments of the given collective.
|
* given collective.
|
||||||
*/
|
*/
|
||||||
def generatePreviews(
|
def generatePreviews(
|
||||||
storeMode: MakePreviewArgs.StoreMode,
|
storeMode: MakePreviewArgs.StoreMode,
|
||||||
|
@ -23,9 +23,8 @@ trait OFolder[F[_]] {
|
|||||||
|
|
||||||
def findById(id: Ident, account: AccountId): F[Option[OFolder.FolderDetail]]
|
def findById(id: Ident, account: AccountId): F[Option[OFolder.FolderDetail]]
|
||||||
|
|
||||||
/** Adds a new folder. If `login` is non-empty, the `folder.user`
|
/** Adds a new folder. If `login` is non-empty, the `folder.user` property is ignored
|
||||||
* property is ignored and the user-id is determined by the given
|
* and the user-id is determined by the given login name.
|
||||||
* login name.
|
|
||||||
*/
|
*/
|
||||||
def add(folder: RFolder, login: Option[Ident]): F[AddResult]
|
def add(folder: RFolder, login: Option[Ident]): F[AddResult]
|
||||||
|
|
||||||
|
@ -49,13 +49,12 @@ trait OFulltext[F[_]] {
|
|||||||
def findIndexOnlySummary(account: AccountId, fts: OFulltext.FtsInput): F[SearchSummary]
|
def findIndexOnlySummary(account: AccountId, fts: OFulltext.FtsInput): F[SearchSummary]
|
||||||
def findItemsSummary(q: Query, fts: OFulltext.FtsInput): F[SearchSummary]
|
def findItemsSummary(q: Query, fts: OFulltext.FtsInput): F[SearchSummary]
|
||||||
|
|
||||||
/** Clears the full-text index completely and launches a task that
|
/** Clears the full-text index completely and launches a task that indexes all data.
|
||||||
* indexes all data.
|
|
||||||
*/
|
*/
|
||||||
def reindexAll: F[Unit]
|
def reindexAll: F[Unit]
|
||||||
|
|
||||||
/** Clears the full-text index for the given collective and starts a
|
/** Clears the full-text index for the given collective and starts a task indexing all
|
||||||
* task indexing all their data.
|
* their data.
|
||||||
*/
|
*/
|
||||||
def reindexCollective(account: AccountId): F[Unit]
|
def reindexCollective(account: AccountId): F[Unit]
|
||||||
}
|
}
|
||||||
@ -125,7 +124,7 @@ object OFulltext {
|
|||||||
FtsQuery.HighlightSetting(ftsQ.highlightPre, ftsQ.highlightPost)
|
FtsQuery.HighlightSetting(ftsQ.highlightPre, ftsQ.highlightPost)
|
||||||
)
|
)
|
||||||
for {
|
for {
|
||||||
_ <- logger.ftrace(s"Find index only: ${ftsQ.query}/${batch}")
|
_ <- logger.ftrace(s"Find index only: ${ftsQ.query}/$batch")
|
||||||
folders <- store.transact(QFolder.getMemberFolders(account))
|
folders <- store.transact(QFolder.getMemberFolders(account))
|
||||||
ftsR <- fts.search(fq.withFolders(folders))
|
ftsR <- fts.search(fq.withFolders(folders))
|
||||||
ftsItems = ftsR.results.groupBy(_.itemId)
|
ftsItems = ftsR.results.groupBy(_.itemId)
|
||||||
@ -154,7 +153,7 @@ object OFulltext {
|
|||||||
res =
|
res =
|
||||||
itemsWithTags
|
itemsWithTags
|
||||||
.collect(convertFtsData(ftsR, ftsItems))
|
.collect(convertFtsData(ftsR, ftsItems))
|
||||||
.map({ case (li, fd) => FtsItemWithTags(li, fd) })
|
.map { case (li, fd) => FtsItemWithTags(li, fd) }
|
||||||
} yield res
|
} yield res
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -203,7 +202,7 @@ object OFulltext {
|
|||||||
)
|
)
|
||||||
.drop(batch.offset.toLong)
|
.drop(batch.offset.toLong)
|
||||||
.take(batch.limit.toLong)
|
.take(batch.limit.toLong)
|
||||||
.map({ case (li, fd) => FtsItem(li, fd) })
|
.map { case (li, fd) => FtsItem(li, fd) }
|
||||||
.compile
|
.compile
|
||||||
.toVector
|
.toVector
|
||||||
|
|
||||||
@ -221,7 +220,7 @@ object OFulltext {
|
|||||||
)
|
)
|
||||||
.drop(batch.offset.toLong)
|
.drop(batch.offset.toLong)
|
||||||
.take(batch.limit.toLong)
|
.take(batch.limit.toLong)
|
||||||
.map({ case (li, fd) => FtsItemWithTags(li, fd) })
|
.map { case (li, fd) => FtsItemWithTags(li, fd) }
|
||||||
.compile
|
.compile
|
||||||
.toVector
|
.toVector
|
||||||
|
|
||||||
|
@ -28,9 +28,8 @@ trait OItem[F[_]] {
|
|||||||
/** Sets the given tags (removing all existing ones). */
|
/** Sets the given tags (removing all existing ones). */
|
||||||
def setTags(item: Ident, tagIds: List[String], collective: Ident): F[UpdateResult]
|
def setTags(item: Ident, tagIds: List[String], collective: Ident): F[UpdateResult]
|
||||||
|
|
||||||
/** Sets tags for multiple items. The tags of the items will be
|
/** Sets tags for multiple items. The tags of the items will be replaced with the given
|
||||||
* replaced with the given ones. Same as `setTags` but for multiple
|
* ones. Same as `setTags` but for multiple items.
|
||||||
* items.
|
|
||||||
*/
|
*/
|
||||||
def setTagsMultipleItems(
|
def setTagsMultipleItems(
|
||||||
items: NonEmptyList[Ident],
|
items: NonEmptyList[Ident],
|
||||||
@ -41,8 +40,8 @@ trait OItem[F[_]] {
|
|||||||
/** Create a new tag and add it to the item. */
|
/** Create a new tag and add it to the item. */
|
||||||
def addNewTag(item: Ident, tag: RTag): F[AddResult]
|
def addNewTag(item: Ident, tag: RTag): F[AddResult]
|
||||||
|
|
||||||
/** Apply all tags to the given item. Tags must exist, but can be IDs
|
/** Apply all tags to the given item. Tags must exist, but can be IDs or names. Existing
|
||||||
* or names. Existing tags on the item are left unchanged.
|
* tags on the item are left unchanged.
|
||||||
*/
|
*/
|
||||||
def linkTags(item: Ident, tags: List[String], collective: Ident): F[UpdateResult]
|
def linkTags(item: Ident, tags: List[String], collective: Ident): F[UpdateResult]
|
||||||
|
|
||||||
@ -163,10 +162,9 @@ trait OItem[F[_]] {
|
|||||||
collective: Ident
|
collective: Ident
|
||||||
): F[UpdateResult]
|
): F[UpdateResult]
|
||||||
|
|
||||||
/** Submits the item for re-processing. The list of attachment ids can
|
/** Submits the item for re-processing. The list of attachment ids can be used to only
|
||||||
* be used to only re-process a subset of the item's attachments.
|
* re-process a subset of the item's attachments. If this list is empty, all
|
||||||
* If this list is empty, all attachments are reprocessed. This
|
* attachments are reprocessed. This call only submits the job into the queue.
|
||||||
* call only submits the job into the queue.
|
|
||||||
*/
|
*/
|
||||||
def reprocess(
|
def reprocess(
|
||||||
item: Ident,
|
item: Ident,
|
||||||
@ -181,9 +179,8 @@ trait OItem[F[_]] {
|
|||||||
notifyJoex: Boolean
|
notifyJoex: Boolean
|
||||||
): F[UpdateResult]
|
): F[UpdateResult]
|
||||||
|
|
||||||
/** Submits a task that finds all non-converted pdfs and triggers
|
/** Submits a task that finds all non-converted pdfs and triggers converting them using
|
||||||
* converting them using ocrmypdf. Each file is converted by a
|
* ocrmypdf. Each file is converted by a separate task.
|
||||||
* separate task.
|
|
||||||
*/
|
*/
|
||||||
def convertAllPdf(
|
def convertAllPdf(
|
||||||
collective: Option[Ident],
|
collective: Option[Ident],
|
||||||
@ -191,8 +188,7 @@ trait OItem[F[_]] {
|
|||||||
notifyJoex: Boolean
|
notifyJoex: Boolean
|
||||||
): F[UpdateResult]
|
): F[UpdateResult]
|
||||||
|
|
||||||
/** Submits a task that (re)generates the preview image for an
|
/** Submits a task that (re)generates the preview image for an attachment.
|
||||||
* attachment.
|
|
||||||
*/
|
*/
|
||||||
def generatePreview(
|
def generatePreview(
|
||||||
args: MakePreviewArgs,
|
args: MakePreviewArgs,
|
||||||
@ -200,8 +196,7 @@ trait OItem[F[_]] {
|
|||||||
notifyJoex: Boolean
|
notifyJoex: Boolean
|
||||||
): F[UpdateResult]
|
): F[UpdateResult]
|
||||||
|
|
||||||
/** Submits a task that (re)generates the preview images for all
|
/** Submits a task that (re)generates the preview images for all attachments.
|
||||||
* attachments.
|
|
||||||
*/
|
*/
|
||||||
def generateAllPreviews(
|
def generateAllPreviews(
|
||||||
storeMode: MakePreviewArgs.StoreMode,
|
storeMode: MakePreviewArgs.StoreMode,
|
||||||
|
@ -183,7 +183,7 @@ object OItemSearch {
|
|||||||
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] =
|
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] =
|
||||||
store
|
store
|
||||||
.transact(RAttachment.findByIdAndCollective(id, collective))
|
.transact(RAttachment.findByIdAndCollective(id, collective))
|
||||||
.flatMap({
|
.flatMap {
|
||||||
case Some(ra) =>
|
case Some(ra) =>
|
||||||
makeBinaryData(ra.fileId) { m =>
|
makeBinaryData(ra.fileId) { m =>
|
||||||
AttachmentData[F](
|
AttachmentData[F](
|
||||||
@ -195,7 +195,7 @@ object OItemSearch {
|
|||||||
|
|
||||||
case None =>
|
case None =>
|
||||||
(None: Option[AttachmentData[F]]).pure[F]
|
(None: Option[AttachmentData[F]]).pure[F]
|
||||||
})
|
}
|
||||||
|
|
||||||
def findAttachmentSource(
|
def findAttachmentSource(
|
||||||
id: Ident,
|
id: Ident,
|
||||||
@ -203,7 +203,7 @@ object OItemSearch {
|
|||||||
): F[Option[AttachmentSourceData[F]]] =
|
): F[Option[AttachmentSourceData[F]]] =
|
||||||
store
|
store
|
||||||
.transact(RAttachmentSource.findByIdAndCollective(id, collective))
|
.transact(RAttachmentSource.findByIdAndCollective(id, collective))
|
||||||
.flatMap({
|
.flatMap {
|
||||||
case Some(ra) =>
|
case Some(ra) =>
|
||||||
makeBinaryData(ra.fileId) { m =>
|
makeBinaryData(ra.fileId) { m =>
|
||||||
AttachmentSourceData[F](
|
AttachmentSourceData[F](
|
||||||
@ -215,7 +215,7 @@ object OItemSearch {
|
|||||||
|
|
||||||
case None =>
|
case None =>
|
||||||
(None: Option[AttachmentSourceData[F]]).pure[F]
|
(None: Option[AttachmentSourceData[F]]).pure[F]
|
||||||
})
|
}
|
||||||
|
|
||||||
def findAttachmentPreview(
|
def findAttachmentPreview(
|
||||||
id: Ident,
|
id: Ident,
|
||||||
@ -223,7 +223,7 @@ object OItemSearch {
|
|||||||
): F[Option[AttachmentPreviewData[F]]] =
|
): F[Option[AttachmentPreviewData[F]]] =
|
||||||
store
|
store
|
||||||
.transact(RAttachmentPreview.findByIdAndCollective(id, collective))
|
.transact(RAttachmentPreview.findByIdAndCollective(id, collective))
|
||||||
.flatMap({
|
.flatMap {
|
||||||
case Some(ra) =>
|
case Some(ra) =>
|
||||||
makeBinaryData(ra.fileId) { m =>
|
makeBinaryData(ra.fileId) { m =>
|
||||||
AttachmentPreviewData[F](
|
AttachmentPreviewData[F](
|
||||||
@ -235,7 +235,7 @@ object OItemSearch {
|
|||||||
|
|
||||||
case None =>
|
case None =>
|
||||||
(None: Option[AttachmentPreviewData[F]]).pure[F]
|
(None: Option[AttachmentPreviewData[F]]).pure[F]
|
||||||
})
|
}
|
||||||
|
|
||||||
def findItemPreview(
|
def findItemPreview(
|
||||||
item: Ident,
|
item: Ident,
|
||||||
@ -243,7 +243,7 @@ object OItemSearch {
|
|||||||
): F[Option[AttachmentPreviewData[F]]] =
|
): F[Option[AttachmentPreviewData[F]]] =
|
||||||
store
|
store
|
||||||
.transact(RAttachmentPreview.findByItemAndCollective(item, collective))
|
.transact(RAttachmentPreview.findByItemAndCollective(item, collective))
|
||||||
.flatMap({
|
.flatMap {
|
||||||
case Some(ra) =>
|
case Some(ra) =>
|
||||||
makeBinaryData(ra.fileId) { m =>
|
makeBinaryData(ra.fileId) { m =>
|
||||||
AttachmentPreviewData[F](
|
AttachmentPreviewData[F](
|
||||||
@ -255,7 +255,7 @@ object OItemSearch {
|
|||||||
|
|
||||||
case None =>
|
case None =>
|
||||||
(None: Option[AttachmentPreviewData[F]]).pure[F]
|
(None: Option[AttachmentPreviewData[F]]).pure[F]
|
||||||
})
|
}
|
||||||
|
|
||||||
def findAttachmentArchive(
|
def findAttachmentArchive(
|
||||||
id: Ident,
|
id: Ident,
|
||||||
@ -263,7 +263,7 @@ object OItemSearch {
|
|||||||
): F[Option[AttachmentArchiveData[F]]] =
|
): F[Option[AttachmentArchiveData[F]]] =
|
||||||
store
|
store
|
||||||
.transact(RAttachmentArchive.findByIdAndCollective(id, collective))
|
.transact(RAttachmentArchive.findByIdAndCollective(id, collective))
|
||||||
.flatMap({
|
.flatMap {
|
||||||
case Some(ra) =>
|
case Some(ra) =>
|
||||||
makeBinaryData(ra.fileId) { m =>
|
makeBinaryData(ra.fileId) { m =>
|
||||||
AttachmentArchiveData[F](
|
AttachmentArchiveData[F](
|
||||||
@ -275,7 +275,7 @@ object OItemSearch {
|
|||||||
|
|
||||||
case None =>
|
case None =>
|
||||||
(None: Option[AttachmentArchiveData[F]]).pure[F]
|
(None: Option[AttachmentArchiveData[F]]).pure[F]
|
||||||
})
|
}
|
||||||
|
|
||||||
private def makeBinaryData[A](fileId: Ident)(f: FileMeta => A): F[Option[A]] =
|
private def makeBinaryData[A](fileId: Ident)(f: FileMeta => A): F[Option[A]] =
|
||||||
store.bitpeace
|
store.bitpeace
|
||||||
|
@ -64,14 +64,14 @@ object OOrganization {
|
|||||||
): F[Vector[OrgAndContacts]] =
|
): F[Vector[OrgAndContacts]] =
|
||||||
store
|
store
|
||||||
.transact(QOrganization.findOrgAndContact(account.collective, query, _.name))
|
.transact(QOrganization.findOrgAndContact(account.collective, query, _.name))
|
||||||
.map({ case (org, cont) => OrgAndContacts(org, cont) })
|
.map { case (org, cont) => OrgAndContacts(org, cont) }
|
||||||
.compile
|
.compile
|
||||||
.toVector
|
.toVector
|
||||||
|
|
||||||
def findOrg(account: AccountId, orgId: Ident): F[Option[OrgAndContacts]] =
|
def findOrg(account: AccountId, orgId: Ident): F[Option[OrgAndContacts]] =
|
||||||
store
|
store
|
||||||
.transact(QOrganization.getOrgAndContact(account.collective, orgId))
|
.transact(QOrganization.getOrgAndContact(account.collective, orgId))
|
||||||
.map(_.map({ case (org, cont) => OrgAndContacts(org, cont) }))
|
.map(_.map { case (org, cont) => OrgAndContacts(org, cont) })
|
||||||
|
|
||||||
def findAllOrgRefs(
|
def findAllOrgRefs(
|
||||||
account: AccountId,
|
account: AccountId,
|
||||||
@ -91,14 +91,14 @@ object OOrganization {
|
|||||||
): F[Vector[PersonAndContacts]] =
|
): F[Vector[PersonAndContacts]] =
|
||||||
store
|
store
|
||||||
.transact(QOrganization.findPersonAndContact(account.collective, query, _.name))
|
.transact(QOrganization.findPersonAndContact(account.collective, query, _.name))
|
||||||
.map({ case (person, org, cont) => PersonAndContacts(person, org, cont) })
|
.map { case (person, org, cont) => PersonAndContacts(person, org, cont) }
|
||||||
.compile
|
.compile
|
||||||
.toVector
|
.toVector
|
||||||
|
|
||||||
def findPerson(account: AccountId, persId: Ident): F[Option[PersonAndContacts]] =
|
def findPerson(account: AccountId, persId: Ident): F[Option[PersonAndContacts]] =
|
||||||
store
|
store
|
||||||
.transact(QOrganization.getPersonAndContact(account.collective, persId))
|
.transact(QOrganization.getPersonAndContact(account.collective, persId))
|
||||||
.map(_.map({ case (pers, org, cont) => PersonAndContacts(pers, org, cont) }))
|
.map(_.map { case (pers, org, cont) => PersonAndContacts(pers, org, cont) })
|
||||||
|
|
||||||
def findAllPersonRefs(
|
def findAllPersonRefs(
|
||||||
account: AccountId,
|
account: AccountId,
|
||||||
|
@ -19,31 +19,27 @@ import docspell.store.queries.SearchSummary
|
|||||||
|
|
||||||
import org.log4s.getLogger
|
import org.log4s.getLogger
|
||||||
|
|
||||||
/** A "porcelain" api on top of OFulltext and OItemSearch. This takes
|
/** A "porcelain" api on top of OFulltext and OItemSearch. This takes care of restricting
|
||||||
* care of restricting the items to a subset, e.g. only items that
|
* the items to a subset, e.g. only items that have a "valid" state.
|
||||||
* have a "valid" state.
|
|
||||||
*/
|
*/
|
||||||
trait OSimpleSearch[F[_]] {
|
trait OSimpleSearch[F[_]] {
|
||||||
|
|
||||||
/** Search for items using the given query and optional fulltext
|
/** Search for items using the given query and optional fulltext search.
|
||||||
* search.
|
|
||||||
*
|
*
|
||||||
* When using fulltext search only (the query is empty), only the
|
* When using fulltext search only (the query is empty), only the index is searched. It
|
||||||
* index is searched. It is assumed that the index doesn't contain
|
* is assumed that the index doesn't contain "invalid" items. When using a query, then
|
||||||
* "invalid" items. When using a query, then a condition to select
|
* a condition to select only valid items is added to it.
|
||||||
* only valid items is added to it.
|
|
||||||
*/
|
*/
|
||||||
def search(settings: Settings)(q: Query, fulltextQuery: Option[String]): F[Items]
|
def search(settings: Settings)(q: Query, fulltextQuery: Option[String]): F[Items]
|
||||||
|
|
||||||
/** Using the same arguments as in `search`, this returns a summary
|
/** Using the same arguments as in `search`, this returns a summary and not the results.
|
||||||
* and not the results.
|
|
||||||
*/
|
*/
|
||||||
def searchSummary(
|
def searchSummary(
|
||||||
settings: StatsSettings
|
settings: StatsSettings
|
||||||
)(q: Query, fulltextQuery: Option[String]): F[SearchSummary]
|
)(q: Query, fulltextQuery: Option[String]): F[SearchSummary]
|
||||||
|
|
||||||
/** Calls `search` by parsing the given query string into a query that
|
/** Calls `search` by parsing the given query string into a query that is then amended
|
||||||
* is then amended wtih the given `fix` query.
|
* wtih the given `fix` query.
|
||||||
*/
|
*/
|
||||||
final def searchByString(
|
final def searchByString(
|
||||||
settings: Settings
|
settings: Settings
|
||||||
@ -52,8 +48,7 @@ trait OSimpleSearch[F[_]] {
|
|||||||
): F[StringSearchResult[Items]] =
|
): F[StringSearchResult[Items]] =
|
||||||
OSimpleSearch.applySearch[F, Items](fix, q)((iq, fts) => search(settings)(iq, fts))
|
OSimpleSearch.applySearch[F, Items](fix, q)((iq, fts) => search(settings)(iq, fts))
|
||||||
|
|
||||||
/** Same as `searchByString` but returning a summary instead of the
|
/** Same as `searchByString` but returning a summary instead of the results.
|
||||||
* results.
|
|
||||||
*/
|
*/
|
||||||
final def searchSummaryByString(
|
final def searchSummaryByString(
|
||||||
settings: StatsSettings
|
settings: StatsSettings
|
||||||
@ -190,8 +185,8 @@ object OSimpleSearch {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Calls `run` with one of the success results when extracting the
|
/** Calls `run` with one of the success results when extracting the fulltext search node
|
||||||
* fulltext search node from the query.
|
* from the query.
|
||||||
*/
|
*/
|
||||||
private def runQuery[F[_]: Applicative, A](
|
private def runQuery[F[_]: Applicative, A](
|
||||||
itemQuery: Option[ItemQuery]
|
itemQuery: Option[ItemQuery]
|
||||||
@ -211,10 +206,9 @@ object OSimpleSearch {
|
|||||||
final class Impl[F[_]: Sync](fts: OFulltext[F], is: OItemSearch[F])
|
final class Impl[F[_]: Sync](fts: OFulltext[F], is: OItemSearch[F])
|
||||||
extends OSimpleSearch[F] {
|
extends OSimpleSearch[F] {
|
||||||
|
|
||||||
/** Implements searching like this: it exploits the fact that teh
|
/** Implements searching like this: it exploits the fact that teh fulltext index only
|
||||||
* fulltext index only contains valid items. When searching via
|
* contains valid items. When searching via sql the query expression selecting only
|
||||||
* sql the query expression selecting only valid items is added
|
* valid items is added here.
|
||||||
* here.
|
|
||||||
*/
|
*/
|
||||||
def search(
|
def search(
|
||||||
settings: Settings
|
settings: Settings
|
||||||
|
@ -31,10 +31,9 @@ trait OUpload[F[_]] {
|
|||||||
itemId: Option[Ident]
|
itemId: Option[Ident]
|
||||||
): F[OUpload.UploadResult]
|
): F[OUpload.UploadResult]
|
||||||
|
|
||||||
/** Submit files via a given source identifier. The source is looked
|
/** Submit files via a given source identifier. The source is looked up to identify the
|
||||||
* up to identify the collective the files belong to. Metadata
|
* collective the files belong to. Metadata defined in the source is used as a fallback
|
||||||
* defined in the source is used as a fallback to those specified
|
* to those specified here (in UploadData).
|
||||||
* here (in UploadData).
|
|
||||||
*/
|
*/
|
||||||
def submit(
|
def submit(
|
||||||
data: OUpload.UploadData[F],
|
data: OUpload.UploadData[F],
|
||||||
@ -103,8 +102,7 @@ object OUpload {
|
|||||||
|
|
||||||
def noSource: UploadResult = NoSource
|
def noSource: UploadResult = NoSource
|
||||||
|
|
||||||
/** When adding files to an item, no item was found using the given
|
/** When adding files to an item, no item was found using the given item-id.
|
||||||
* item-id.
|
|
||||||
*/
|
*/
|
||||||
case object NoItem extends UploadResult
|
case object NoItem extends UploadResult
|
||||||
|
|
||||||
|
@ -37,8 +37,7 @@ trait OUserTask[F[_]] {
|
|||||||
task: UserTask[ScanMailboxArgs]
|
task: UserTask[ScanMailboxArgs]
|
||||||
): F[Unit]
|
): F[Unit]
|
||||||
|
|
||||||
/** Return the settings for all the notify-due-items task of the
|
/** Return the settings for all the notify-due-items task of the current user.
|
||||||
* current user.
|
|
||||||
*/
|
*/
|
||||||
def getNotifyDueItems(scope: UserTaskScope): Stream[F, UserTask[NotifyDueItemsArgs]]
|
def getNotifyDueItems(scope: UserTaskScope): Stream[F, UserTask[NotifyDueItemsArgs]]
|
||||||
|
|
||||||
@ -59,9 +58,8 @@ trait OUserTask[F[_]] {
|
|||||||
/** Removes a user task with the given id. */
|
/** Removes a user task with the given id. */
|
||||||
def deleteTask(scope: UserTaskScope, id: Ident): F[Unit]
|
def deleteTask(scope: UserTaskScope, id: Ident): F[Unit]
|
||||||
|
|
||||||
/** Discards the schedule and immediately submits the task to the job
|
/** Discards the schedule and immediately submits the task to the job executor's queue.
|
||||||
* executor's queue. It will not update the corresponding periodic
|
* It will not update the corresponding periodic task.
|
||||||
* task.
|
|
||||||
*/
|
*/
|
||||||
def executeNow[A](scope: UserTaskScope, subject: Option[String], task: UserTask[A])(
|
def executeNow[A](scope: UserTaskScope, subject: Option[String], task: UserTask[A])(
|
||||||
implicit E: Encoder[A]
|
implicit E: Encoder[A]
|
||||||
|
@ -16,8 +16,7 @@ object SendResult {
|
|||||||
*/
|
*/
|
||||||
case class Success(id: Ident) extends SendResult
|
case class Success(id: Ident) extends SendResult
|
||||||
|
|
||||||
/** There was a failure sending the mail. The mail is then not saved
|
/** There was a failure sending the mail. The mail is then not saved to db.
|
||||||
* to db.
|
|
||||||
*/
|
*/
|
||||||
case class SendFailure(ex: Throwable) extends SendResult
|
case class SendFailure(ex: Throwable) extends SendResult
|
||||||
|
|
||||||
@ -25,8 +24,7 @@ object SendResult {
|
|||||||
*/
|
*/
|
||||||
case class StoreFailure(ex: Throwable) extends SendResult
|
case class StoreFailure(ex: Throwable) extends SendResult
|
||||||
|
|
||||||
/** Something could not be found required for sending (mail configs,
|
/** Something could not be found required for sending (mail configs, items etc).
|
||||||
* items etc).
|
|
||||||
*/
|
*/
|
||||||
case object NotFound extends SendResult
|
case object NotFound extends SendResult
|
||||||
}
|
}
|
||||||
|
@ -9,12 +9,11 @@ package docspell.common
|
|||||||
import io.circe.generic.semiauto._
|
import io.circe.generic.semiauto._
|
||||||
import io.circe.{Decoder, Encoder}
|
import io.circe.{Decoder, Encoder}
|
||||||
|
|
||||||
/** Arguments for the `AllPreviewsTask` that submits tasks to
|
/** Arguments for the `AllPreviewsTask` that submits tasks to generates a preview image
|
||||||
* generates a preview image for attachments.
|
* for attachments.
|
||||||
*
|
*
|
||||||
* It can replace the current preview image or only generate one, if
|
* It can replace the current preview image or only generate one, if it is missing. If no
|
||||||
* it is missing. If no collective is specified, it considers all
|
* collective is specified, it considers all attachments.
|
||||||
* attachments.
|
|
||||||
*/
|
*/
|
||||||
case class AllPreviewsArgs(
|
case class AllPreviewsArgs(
|
||||||
collective: Option[Ident],
|
collective: Option[Ident],
|
||||||
|
@ -15,17 +15,15 @@ object CollectiveState {
|
|||||||
/** A normal active collective */
|
/** A normal active collective */
|
||||||
case object Active extends CollectiveState
|
case object Active extends CollectiveState
|
||||||
|
|
||||||
/** A collective may be readonly in cases it is implicitly closed
|
/** A collective may be readonly in cases it is implicitly closed (e.g. no payment).
|
||||||
* (e.g. no payment). Users can still see there data and
|
* Users can still see there data and download, but have no write access.
|
||||||
* download, but have no write access.
|
|
||||||
*/
|
*/
|
||||||
case object ReadOnly extends CollectiveState
|
case object ReadOnly extends CollectiveState
|
||||||
|
|
||||||
/** A collective that has been explicitely closed. */
|
/** A collective that has been explicitely closed. */
|
||||||
case object Closed extends CollectiveState
|
case object Closed extends CollectiveState
|
||||||
|
|
||||||
/** A collective blocked by a super user, usually some emergency
|
/** A collective blocked by a super user, usually some emergency action.
|
||||||
* action.
|
|
||||||
*/
|
*/
|
||||||
case object Blocked extends CollectiveState
|
case object Blocked extends CollectiveState
|
||||||
|
|
||||||
|
@ -9,14 +9,12 @@ package docspell.common
|
|||||||
import io.circe._
|
import io.circe._
|
||||||
import io.circe.generic.semiauto._
|
import io.circe.generic.semiauto._
|
||||||
|
|
||||||
/** Arguments for the task that finds all pdf files that have not been
|
/** Arguments for the task that finds all pdf files that have not been converted and
|
||||||
* converted and submits for each a job that will convert the file
|
* submits for each a job that will convert the file using ocrmypdf.
|
||||||
* using ocrmypdf.
|
|
||||||
*
|
*
|
||||||
* If the `collective` argument is present, then this task and the
|
* If the `collective` argument is present, then this task and the ones that are
|
||||||
* ones that are submitted by this task run in the realm of the
|
* submitted by this task run in the realm of the collective (and only their files are
|
||||||
* collective (and only their files are considered). If it is empty,
|
* considered). If it is empty, it is a system task and all files are considered.
|
||||||
* it is a system task and all files are considered.
|
|
||||||
*/
|
*/
|
||||||
case class ConvertAllPdfArgs(collective: Option[Ident])
|
case class ConvertAllPdfArgs(collective: Option[Ident])
|
||||||
|
|
||||||
|
@ -14,8 +14,8 @@ import io.circe.generic.semiauto._
|
|||||||
|
|
||||||
/** Arguments to the empty-trash task.
|
/** Arguments to the empty-trash task.
|
||||||
*
|
*
|
||||||
* This task is run periodically to really delete all soft-deleted
|
* This task is run periodically to really delete all soft-deleted items. These are items
|
||||||
* items. These are items with state `ItemState.Deleted`.
|
* with state `ItemState.Deleted`.
|
||||||
*/
|
*/
|
||||||
case class EmptyTrashArgs(
|
case class EmptyTrashArgs(
|
||||||
collective: Ident,
|
collective: Ident,
|
||||||
|
@ -14,8 +14,8 @@ case class FileName private (name: String) {
|
|||||||
case n => (name.take(n), Some(name.drop(n + 1)))
|
case n => (name.take(n), Some(name.drop(n + 1)))
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Returns the name part without the extension. If there is no
|
/** Returns the name part without the extension. If there is no extension, it is the
|
||||||
* extension, it is the same as fullname.
|
* same as fullname.
|
||||||
*/
|
*/
|
||||||
def baseName: String =
|
def baseName: String =
|
||||||
base
|
base
|
||||||
@ -27,20 +27,20 @@ case class FileName private (name: String) {
|
|||||||
def fullName: String =
|
def fullName: String =
|
||||||
name
|
name
|
||||||
|
|
||||||
/** Creates a new name where part is spliced into the name before the
|
/** Creates a new name where part is spliced into the name before the extension,
|
||||||
* extension, separated by separator.
|
* separated by separator.
|
||||||
*/
|
*/
|
||||||
def withPart(part: String, sep: Char): FileName =
|
def withPart(part: String, sep: Char): FileName =
|
||||||
if (part.isEmpty()) this
|
if (part.isEmpty()) this
|
||||||
else
|
else
|
||||||
ext
|
ext
|
||||||
.map(e => new FileName(s"${base}${sep}${part}.${e}"))
|
.map(e => new FileName(s"$base$sep$part.$e"))
|
||||||
.getOrElse(new FileName(s"${base}${sep}${part}"))
|
.getOrElse(new FileName(s"$base$sep$part"))
|
||||||
|
|
||||||
/** Create a new name using the given extension. */
|
/** Create a new name using the given extension. */
|
||||||
def withExtension(newExt: String): FileName =
|
def withExtension(newExt: String): FileName =
|
||||||
if (newExt.isEmpty()) new FileName(base)
|
if (newExt.isEmpty()) new FileName(base)
|
||||||
else new FileName(s"${base}.${newExt}")
|
else new FileName(s"$base.$newExt")
|
||||||
|
|
||||||
}
|
}
|
||||||
object FileName {
|
object FileName {
|
||||||
|
@ -16,14 +16,11 @@ trait Glob {
|
|||||||
/** Matches the input string against this glob. */
|
/** Matches the input string against this glob. */
|
||||||
def matches(caseSensitive: Boolean)(in: String): Boolean
|
def matches(caseSensitive: Boolean)(in: String): Boolean
|
||||||
|
|
||||||
/** If this glob consists of multiple segments, it is the same as
|
/** If this glob consists of multiple segments, it is the same as `matches`. If it is
|
||||||
* `matches`. If it is only a single segment, it is matched against
|
* only a single segment, it is matched against the last segment of the input string
|
||||||
* the last segment of the input string that is assumed to be a
|
* that is assumed to be a pathname separated by slash.
|
||||||
* pathname separated by slash.
|
|
||||||
*
|
*
|
||||||
* Example:
|
* Example: test.* <> "/a/b/test.txt" => true /test.* <> "/a/b/test.txt" => false
|
||||||
* test.* <> "/a/b/test.txt" => true
|
|
||||||
* /test.* <> "/a/b/test.txt" => false
|
|
||||||
*/
|
*/
|
||||||
def matchFilenameOrPath(in: String): Boolean
|
def matchFilenameOrPath(in: String): Boolean
|
||||||
|
|
||||||
|
@ -20,8 +20,7 @@ object JobState {
|
|||||||
/** Waiting for being executed. */
|
/** Waiting for being executed. */
|
||||||
case object Waiting extends JobState {}
|
case object Waiting extends JobState {}
|
||||||
|
|
||||||
/** A scheduler has picked up this job and will pass it to the next
|
/** A scheduler has picked up this job and will pass it to the next free slot.
|
||||||
* free slot.
|
|
||||||
*/
|
*/
|
||||||
case object Scheduled extends JobState {}
|
case object Scheduled extends JobState {}
|
||||||
|
|
||||||
|
@ -13,9 +13,9 @@ import io.circe.generic.semiauto._
|
|||||||
|
|
||||||
/** Arguments to the classify-item task.
|
/** Arguments to the classify-item task.
|
||||||
*
|
*
|
||||||
* This task is run periodically and learns from existing documents
|
* This task is run periodically and learns from existing documents to create a model for
|
||||||
* to create a model for predicting tags of new documents. The user
|
* predicting tags of new documents. The user must give a tag category as a subset of
|
||||||
* must give a tag category as a subset of possible tags..
|
* possible tags..
|
||||||
*/
|
*/
|
||||||
case class LearnClassifierArgs(
|
case class LearnClassifierArgs(
|
||||||
collective: Ident
|
collective: Ident
|
||||||
|
@ -9,9 +9,8 @@ package docspell.common
|
|||||||
import io.circe.generic.semiauto._
|
import io.circe.generic.semiauto._
|
||||||
import io.circe.{Decoder, Encoder}
|
import io.circe.{Decoder, Encoder}
|
||||||
|
|
||||||
/** Arguments for the `MakePageCountTask` that reads the number of
|
/** Arguments for the `MakePageCountTask` that reads the number of pages for an attachment
|
||||||
* pages for an attachment and stores it into the meta data of the
|
* and stores it into the meta data of the attachment.
|
||||||
* attachment.
|
|
||||||
*/
|
*/
|
||||||
case class MakePageCountArgs(
|
case class MakePageCountArgs(
|
||||||
attachment: Ident
|
attachment: Ident
|
||||||
|
@ -9,11 +9,9 @@ package docspell.common
|
|||||||
import io.circe.generic.semiauto._
|
import io.circe.generic.semiauto._
|
||||||
import io.circe.{Decoder, Encoder}
|
import io.circe.{Decoder, Encoder}
|
||||||
|
|
||||||
/** Arguments for the `MakePreviewTask` that generates a preview image
|
/** Arguments for the `MakePreviewTask` that generates a preview image for an attachment.
|
||||||
* for an attachment.
|
|
||||||
*
|
*
|
||||||
* It can replace the current preview image or only generate one, if
|
* It can replace the current preview image or only generate one, if it is missing.
|
||||||
* it is missing.
|
|
||||||
*/
|
*/
|
||||||
case class MakePreviewArgs(
|
case class MakePreviewArgs(
|
||||||
attachment: Ident,
|
attachment: Ident,
|
||||||
|
@ -20,14 +20,12 @@ import io.circe.generic.semiauto._
|
|||||||
|
|
||||||
/** A proposed meta data to an item.
|
/** A proposed meta data to an item.
|
||||||
*
|
*
|
||||||
* There is only one value for each proposal type. The list of
|
* There is only one value for each proposal type. The list of candidates is meant to be
|
||||||
* candidates is meant to be ordered from the best match to the
|
* ordered from the best match to the lowest match.
|
||||||
* lowest match.
|
|
||||||
*
|
*
|
||||||
* The candidate is already "resolved" against the database and
|
* The candidate is already "resolved" against the database and contains a valid record
|
||||||
* contains a valid record (with its ID and a human readable name).
|
* (with its ID and a human readable name). Additionally it carries a set of "labels"
|
||||||
* Additionally it carries a set of "labels" (which may be empty)
|
* (which may be empty) that are the source of this candidate.
|
||||||
* that are the source of this candidate.
|
|
||||||
*/
|
*/
|
||||||
case class MetaProposal(proposalType: MetaProposalType, values: NonEmptyList[Candidate]) {
|
case class MetaProposal(proposalType: MetaProposalType, values: NonEmptyList[Candidate]) {
|
||||||
|
|
||||||
@ -96,8 +94,8 @@ object MetaProposal {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Merges candidates with same `IdRef` values and concatenates their
|
/** Merges candidates with same `IdRef` values and concatenates their respective labels.
|
||||||
* respective labels. The candidate order is preserved.
|
* The candidate order is preserved.
|
||||||
*/
|
*/
|
||||||
def flatten(s: NonEmptyList[Candidate]): NonEmptyList[Candidate] = {
|
def flatten(s: NonEmptyList[Candidate]): NonEmptyList[Candidate] = {
|
||||||
def mergeInto(
|
def mergeInto(
|
||||||
|
@ -91,13 +91,12 @@ object MetaProposalList {
|
|||||||
.getOrElse(empty)
|
.getOrElse(empty)
|
||||||
|
|
||||||
def fromMap(m: Map[MetaProposalType, MetaProposal]): MetaProposalList =
|
def fromMap(m: Map[MetaProposalType, MetaProposal]): MetaProposalList =
|
||||||
new MetaProposalList(m.toList.map({ case (k, v) => v.copy(proposalType = k) }))
|
new MetaProposalList(m.toList.map { case (k, v) => v.copy(proposalType = k) })
|
||||||
|
|
||||||
/** Flattens the given list of meta-proposals into a single list,
|
/** Flattens the given list of meta-proposals into a single list, where each
|
||||||
* where each meta-proposal type exists at most once. Candidates to
|
* meta-proposal type exists at most once. Candidates to equal proposal-types are
|
||||||
* equal proposal-types are merged together. The candidate's order
|
* merged together. The candidate's order is preserved and candidates of proposals are
|
||||||
* is preserved and candidates of proposals are appended as given
|
* appended as given by the order of the given `seq'.
|
||||||
* by the order of the given `seq'.
|
|
||||||
*/
|
*/
|
||||||
def flatten(ml: Seq[MetaProposalList]): MetaProposalList =
|
def flatten(ml: Seq[MetaProposalList]): MetaProposalList =
|
||||||
flatten0(
|
flatten0(
|
||||||
|
@ -13,11 +13,10 @@ import io.circe.generic.semiauto._
|
|||||||
|
|
||||||
/** Arguments to the notification task.
|
/** Arguments to the notification task.
|
||||||
*
|
*
|
||||||
* This tasks queries items with a due date and informs the user via
|
* This tasks queries items with a due date and informs the user via mail.
|
||||||
* mail.
|
|
||||||
*
|
*
|
||||||
* If the structure changes, there must be some database migration to
|
* If the structure changes, there must be some database migration to update or remove
|
||||||
* update or remove the json data of the corresponding task.
|
* the json data of the corresponding task.
|
||||||
*/
|
*/
|
||||||
case class NotifyDueItemsArgs(
|
case class NotifyDueItemsArgs(
|
||||||
account: AccountId,
|
account: AccountId,
|
||||||
|
@ -14,11 +14,11 @@ import io.circe.generic.semiauto._
|
|||||||
|
|
||||||
/** Arguments to the process-item task.
|
/** Arguments to the process-item task.
|
||||||
*
|
*
|
||||||
* This task is run for each new file to create a new item from it or
|
* This task is run for each new file to create a new item from it or to add this file as
|
||||||
* to add this file as an attachment to an existing item.
|
* an attachment to an existing item.
|
||||||
*
|
*
|
||||||
* If the `itemId' is set to some value, the item is tried to load to
|
* If the `itemId' is set to some value, the item is tried to load to ammend with the
|
||||||
* ammend with the given files. Otherwise a new item is created.
|
* given files. Otherwise a new item is created.
|
||||||
*
|
*
|
||||||
* It is also re-used by the 'ReProcessItem' task.
|
* It is also re-used by the 'ReProcessItem' task.
|
||||||
*/
|
*/
|
||||||
|
@ -11,10 +11,9 @@ import io.circe.{Decoder, Encoder}
|
|||||||
|
|
||||||
/** Arguments when re-processing an item.
|
/** Arguments when re-processing an item.
|
||||||
*
|
*
|
||||||
* The `itemId` must exist and point to some item. If the attachment
|
* The `itemId` must exist and point to some item. If the attachment list is non-empty,
|
||||||
* list is non-empty, only those attachments are re-processed. They
|
* only those attachments are re-processed. They must belong to the given item. If the
|
||||||
* must belong to the given item. If the list is empty, then all
|
* list is empty, then all attachments are re-processed.
|
||||||
* attachments are re-processed.
|
|
||||||
*/
|
*/
|
||||||
case class ReProcessItemArgs(itemId: Ident, attachments: List[Ident])
|
case class ReProcessItemArgs(itemId: Ident, attachments: List[Ident])
|
||||||
|
|
||||||
|
@ -13,11 +13,10 @@ import io.circe.generic.semiauto._
|
|||||||
|
|
||||||
/** Arguments to the poll-mailbox task.
|
/** Arguments to the poll-mailbox task.
|
||||||
*
|
*
|
||||||
* This tasks queries user mailboxes and pushes found mails into
|
* This tasks queries user mailboxes and pushes found mails into docspell for processing.
|
||||||
* docspell for processing.
|
|
||||||
*
|
*
|
||||||
* If the structure changes, there must be some database migration to
|
* If the structure changes, there must be some database migration to update or remove
|
||||||
* update or remove the json data of the corresponding task.
|
* the json data of the corresponding task.
|
||||||
*/
|
*/
|
||||||
case class ScanMailboxArgs(
|
case class ScanMailboxArgs(
|
||||||
// the docspell user account
|
// the docspell user account
|
||||||
|
@ -19,10 +19,9 @@ sealed trait ConversionResult[F[_]] {
|
|||||||
|
|
||||||
object ConversionResult {
|
object ConversionResult {
|
||||||
|
|
||||||
/** The conversion is done by external tools that write files to the
|
/** The conversion is done by external tools that write files to the file system. These
|
||||||
* file system. These are temporary files and they will be deleted
|
* are temporary files and they will be deleted once the process finishes. This handler
|
||||||
* once the process finishes. This handler is used to do something
|
* is used to do something relevant with the resulting files.
|
||||||
* relevant with the resulting files.
|
|
||||||
*/
|
*/
|
||||||
type Handler[F[_], A] = Kleisli[F, ConversionResult[F], A]
|
type Handler[F[_], A] = Kleisli[F, ConversionResult[F], A]
|
||||||
|
|
||||||
|
@ -12,11 +12,10 @@ import scodec.bits.ByteVector
|
|||||||
@FunctionalInterface
|
@FunctionalInterface
|
||||||
trait SanitizeHtml {
|
trait SanitizeHtml {
|
||||||
|
|
||||||
/** The given `bytes' are html which can be modified to strip out
|
/** The given `bytes' are html which can be modified to strip out unwanted content.
|
||||||
* unwanted content.
|
|
||||||
*
|
*
|
||||||
* The result should use the same character encoding as the given
|
* The result should use the same character encoding as the given charset implies, or
|
||||||
* charset implies, or utf8 if not specified.
|
* utf8 if not specified.
|
||||||
*/
|
*/
|
||||||
def apply(bytes: ByteVector, charset: Option[Charset]): ByteVector
|
def apply(bytes: ByteVector, charset: Option[Charset]): ByteVector
|
||||||
|
|
||||||
|
@ -132,7 +132,7 @@ private[extern] object ExternConv {
|
|||||||
): Pipe[F, Byte, Unit] =
|
): Pipe[F, Byte, Unit] =
|
||||||
in =>
|
in =>
|
||||||
Stream
|
Stream
|
||||||
.eval(logger.debug(s"Storing input to file ${inFile} for running $name"))
|
.eval(logger.debug(s"Storing input to file $inFile for running $name"))
|
||||||
.drain ++
|
.drain ++
|
||||||
Stream.eval(storeFile(in, inFile))
|
Stream.eval(storeFile(in, inFile))
|
||||||
|
|
||||||
|
@ -150,12 +150,12 @@ class ConversionTest extends FunSuite with FileChecks {
|
|||||||
conversion
|
conversion
|
||||||
.use { conv =>
|
.use { conv =>
|
||||||
def check: Handler[IO, Unit] =
|
def check: Handler[IO, Unit] =
|
||||||
Kleisli({
|
Kleisli {
|
||||||
case ConversionResult.InputMalformed(_, _) =>
|
case ConversionResult.InputMalformed(_, _) =>
|
||||||
().pure[IO]
|
().pure[IO]
|
||||||
case cr =>
|
case cr =>
|
||||||
IO.raiseError(new Exception(s"Unexpected result: $cr"))
|
IO.raiseError(new Exception(s"Unexpected result: $cr"))
|
||||||
})
|
}
|
||||||
|
|
||||||
runConversion(bombs, _ => check, conv).compile.drain
|
runConversion(bombs, _ => check, conv).compile.drain
|
||||||
}
|
}
|
||||||
@ -171,12 +171,12 @@ class ConversionTest extends FunSuite with FileChecks {
|
|||||||
.emits(uris)
|
.emits(uris)
|
||||||
.covary[IO]
|
.covary[IO]
|
||||||
.zipWithIndex
|
.zipWithIndex
|
||||||
.evalMap({ case (uri, index) =>
|
.evalMap { case (uri, index) =>
|
||||||
val load = uri.readURL[IO](8192)
|
val load = uri.readURL[IO](8192)
|
||||||
val dataType = DataType.filename(uri.path.segments.last)
|
val dataType = DataType.filename(uri.path.segments.last)
|
||||||
logger.info(s"Processing file ${uri.path.asString}") *>
|
logger.info(s"Processing file ${uri.path.asString}") *>
|
||||||
conv.toPDF(dataType, Language.German, handler(index))(load)
|
conv.toPDF(dataType, Language.German, handler(index))(load)
|
||||||
})
|
}
|
||||||
|
|
||||||
def commandsExist: Boolean =
|
def commandsExist: Boolean =
|
||||||
commandExists(convertConfig.unoconv.command.program) &&
|
commandExists(convertConfig.unoconv.command.program) &&
|
||||||
|
@ -48,7 +48,7 @@ trait FileChecks {
|
|||||||
storePdfTxtHandler(file, file.resolveSibling("unexpected.txt")).map(_._1)
|
storePdfTxtHandler(file, file.resolveSibling("unexpected.txt")).map(_._1)
|
||||||
|
|
||||||
def storePdfTxtHandler(filePdf: Path, fileTxt: Path): Handler[IO, (Path, Path)] =
|
def storePdfTxtHandler(filePdf: Path, fileTxt: Path): Handler[IO, (Path, Path)] =
|
||||||
Kleisli({
|
Kleisli {
|
||||||
case ConversionResult.SuccessPdfTxt(pdf, txt) =>
|
case ConversionResult.SuccessPdfTxt(pdf, txt) =>
|
||||||
for {
|
for {
|
||||||
pout <- pdf.through(storeFile(filePdf)).compile.lastOrError
|
pout <- pdf.through(storeFile(filePdf)).compile.lastOrError
|
||||||
@ -64,7 +64,7 @@ trait FileChecks {
|
|||||||
|
|
||||||
case cr =>
|
case cr =>
|
||||||
throw new Exception(s"Unexpected result: $cr")
|
throw new Exception(s"Unexpected result: $cr")
|
||||||
})
|
}
|
||||||
|
|
||||||
def commandExists(cmd: String): Boolean =
|
def commandExists(cmd: String): Boolean =
|
||||||
Runtime.getRuntime.exec(Array("which", cmd)).waitFor() == 0
|
Runtime.getRuntime.exec(Array("which", cmd)).waitFor() == 0
|
||||||
|
@ -62,8 +62,8 @@ object Ocr {
|
|||||||
): Stream[F, String] =
|
): Stream[F, String] =
|
||||||
runTesseractFile(img, logger, lang, config)
|
runTesseractFile(img, logger, lang, config)
|
||||||
|
|
||||||
/** Run ghostscript to extract all pdf pages into tiff files. The
|
/** Run ghostscript to extract all pdf pages into tiff files. The files are stored to a
|
||||||
* files are stored to a temporary location on disk and returned.
|
* temporary location on disk and returned.
|
||||||
*/
|
*/
|
||||||
private[extract] def runGhostscript[F[_]: Async](
|
private[extract] def runGhostscript[F[_]: Async](
|
||||||
pdf: Stream[F, Byte],
|
pdf: Stream[F, Byte],
|
||||||
@ -88,8 +88,8 @@ object Ocr {
|
|||||||
.flatMap(_ => File.listFiles(pathEndsWith(".tif"), wd))
|
.flatMap(_ => File.listFiles(pathEndsWith(".tif"), wd))
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Run ghostscript to extract all pdf pages into tiff files. The
|
/** Run ghostscript to extract all pdf pages into tiff files. The files are stored to a
|
||||||
* files are stored to a temporary location on disk and returned.
|
* temporary location on disk and returned.
|
||||||
*/
|
*/
|
||||||
private[extract] def runGhostscriptFile[F[_]: Async](
|
private[extract] def runGhostscriptFile[F[_]: Async](
|
||||||
pdf: Path,
|
pdf: Path,
|
||||||
@ -111,8 +111,8 @@ object Ocr {
|
|||||||
private def pathEndsWith(ext: String): Path => Boolean =
|
private def pathEndsWith(ext: String): Path => Boolean =
|
||||||
p => p.fileName.toString.endsWith(ext)
|
p => p.fileName.toString.endsWith(ext)
|
||||||
|
|
||||||
/** Run unpaper to optimize the image for ocr. The
|
/** Run unpaper to optimize the image for ocr. The files are stored to a temporary
|
||||||
* files are stored to a temporary location on disk and returned.
|
* location on disk and returned.
|
||||||
*/
|
*/
|
||||||
private[extract] def runUnpaperFile[F[_]: Async](
|
private[extract] def runUnpaperFile[F[_]: Async](
|
||||||
img: Path,
|
img: Path,
|
||||||
@ -139,8 +139,7 @@ object Ocr {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Run tesseract on the given image file and return the extracted
|
/** Run tesseract on the given image file and return the extracted text.
|
||||||
* text.
|
|
||||||
*/
|
*/
|
||||||
private[extract] def runTesseractFile[F[_]: Async](
|
private[extract] def runTesseractFile[F[_]: Async](
|
||||||
img: Path,
|
img: Path,
|
||||||
@ -160,8 +159,7 @@ object Ocr {
|
|||||||
.map(_.stdout)
|
.map(_.stdout)
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Run tesseract on the given image file and return the extracted
|
/** Run tesseract on the given image file and return the extracted text.
|
||||||
* text.
|
|
||||||
*/
|
*/
|
||||||
private[extract] def runTesseractStdin[F[_]: Async](
|
private[extract] def runTesseractStdin[F[_]: Async](
|
||||||
img: Stream[F, Byte],
|
img: Stream[F, Byte],
|
||||||
|
@ -31,7 +31,7 @@ object TextExtract {
|
|||||||
): Stream[F, Text] =
|
): Stream[F, Text] =
|
||||||
Stream
|
Stream
|
||||||
.eval(TikaMimetype.detect(in, MimeTypeHint.none))
|
.eval(TikaMimetype.detect(in, MimeTypeHint.none))
|
||||||
.flatMap({
|
.flatMap {
|
||||||
case MimeType.pdf =>
|
case MimeType.pdf =>
|
||||||
Stream.eval(Ocr.extractPdf(in, logger, lang, config)).unNoneTerminate
|
Stream.eval(Ocr.extractPdf(in, logger, lang, config)).unNoneTerminate
|
||||||
|
|
||||||
@ -40,7 +40,7 @@ object TextExtract {
|
|||||||
|
|
||||||
case mt =>
|
case mt =>
|
||||||
raiseError(s"File `$mt` not supported")
|
raiseError(s"File `$mt` not supported")
|
||||||
})
|
}
|
||||||
.map(Text.apply)
|
.map(Text.apply)
|
||||||
|
|
||||||
private def raiseError[F[_]: Sync](msg: String): Stream[F, Nothing] =
|
private def raiseError[F[_]: Sync](msg: String): Stream[F, Nothing] =
|
||||||
|
@ -49,17 +49,13 @@ object PoiExtract {
|
|||||||
case PoiType.docx =>
|
case PoiType.docx =>
|
||||||
getDocx(data)
|
getDocx(data)
|
||||||
case PoiType.msoffice =>
|
case PoiType.msoffice =>
|
||||||
EitherT(getDoc[F](data))
|
EitherT(getDoc[F](data)).recoverWith { case _ =>
|
||||||
.recoverWith({ case _ =>
|
|
||||||
EitherT(getXls[F](data))
|
EitherT(getXls[F](data))
|
||||||
})
|
}.value
|
||||||
.value
|
|
||||||
case PoiType.ooxml =>
|
case PoiType.ooxml =>
|
||||||
EitherT(getDocx[F](data))
|
EitherT(getDocx[F](data)).recoverWith { case _ =>
|
||||||
.recoverWith({ case _ =>
|
|
||||||
EitherT(getXlsx[F](data))
|
EitherT(getXlsx[F](data))
|
||||||
})
|
}.value
|
||||||
.value
|
|
||||||
case mt =>
|
case mt =>
|
||||||
Sync[F].pure(Left(new Exception(s"Unsupported content: ${mt.asString}")))
|
Sync[F].pure(Left(new Exception(s"Unsupported content: ${mt.asString}")))
|
||||||
}
|
}
|
||||||
|
@ -20,20 +20,17 @@ import fs2.io.file.Path
|
|||||||
|
|
||||||
object ImageSize {
|
object ImageSize {
|
||||||
|
|
||||||
/** Return the image size from its header without reading
|
/** Return the image size from its header without reading the whole image into memory.
|
||||||
* the whole image into memory.
|
|
||||||
*/
|
*/
|
||||||
def get(file: Path): Option[Dimension] =
|
def get(file: Path): Option[Dimension] =
|
||||||
Using(new FileImageInputStream(file.toNioPath.toFile))(getDimension).toOption.flatten
|
Using(new FileImageInputStream(file.toNioPath.toFile))(getDimension).toOption.flatten
|
||||||
|
|
||||||
/** Return the image size from its header without reading
|
/** Return the image size from its header without reading the whole image into memory.
|
||||||
* the whole image into memory.
|
|
||||||
*/
|
*/
|
||||||
def get(in: InputStream): Option[Dimension] =
|
def get(in: InputStream): Option[Dimension] =
|
||||||
Option(ImageIO.createImageInputStream(in)).flatMap(getDimension)
|
Option(ImageIO.createImageInputStream(in)).flatMap(getDimension)
|
||||||
|
|
||||||
/** Return the image size from its header without reading
|
/** Return the image size from its header without reading the whole image into memory.
|
||||||
* the whole image into memory.
|
|
||||||
*/
|
*/
|
||||||
def get[F[_]: Sync](data: Stream[F, Byte]): F[Option[Dimension]] =
|
def get[F[_]: Sync](data: Stream[F, Byte]): F[Option[Dimension]] =
|
||||||
data.take(768).compile.to(Array).map { ar =>
|
data.take(768).compile.to(Array).map { ar =>
|
||||||
|
@ -14,32 +14,28 @@ import docspell.common._
|
|||||||
|
|
||||||
import org.log4s.getLogger
|
import org.log4s.getLogger
|
||||||
|
|
||||||
/** The fts client is the interface for docspell to a fulltext search
|
/** The fts client is the interface for docspell to a fulltext search engine.
|
||||||
* engine.
|
|
||||||
*
|
*
|
||||||
* It defines all operations required for integration into docspell.
|
* It defines all operations required for integration into docspell. It uses data
|
||||||
* It uses data structures from docspell. Implementation modules need
|
* structures from docspell. Implementation modules need to translate it to the engine
|
||||||
* to translate it to the engine that provides the features.
|
* that provides the features.
|
||||||
*/
|
*/
|
||||||
trait FtsClient[F[_]] {
|
trait FtsClient[F[_]] {
|
||||||
|
|
||||||
/** Initialization tasks. This can be used to setup the fulltext
|
/** Initialization tasks. This can be used to setup the fulltext search engine. The
|
||||||
* search engine. The implementation is expected to keep track of
|
* implementation is expected to keep track of run migrations, so that running these is
|
||||||
* run migrations, so that running these is idempotent. For
|
* idempotent. For example, it may be run on each application start.
|
||||||
* example, it may be run on each application start.
|
|
||||||
*
|
*
|
||||||
* Initialization may involve re-indexing all data, therefore it
|
* Initialization may involve re-indexing all data, therefore it must run outside the
|
||||||
* must run outside the scope of this client. The migration may
|
* scope of this client. The migration may include a task that applies any work and/or
|
||||||
* include a task that applies any work and/or it can return a
|
* it can return a result indicating that after this task a re-index is necessary.
|
||||||
* result indicating that after this task a re-index is necessary.
|
|
||||||
*/
|
*/
|
||||||
def initialize: F[List[FtsMigration[F]]]
|
def initialize: F[List[FtsMigration[F]]]
|
||||||
|
|
||||||
/** A list of initialization tasks that can be run when re-creating
|
/** A list of initialization tasks that can be run when re-creating the index.
|
||||||
* the index.
|
|
||||||
*
|
*
|
||||||
* This is not run on startup, but only when required, for example
|
* This is not run on startup, but only when required, for example when re-creating the
|
||||||
* when re-creating the entire index.
|
* entire index.
|
||||||
*/
|
*/
|
||||||
def initializeNew: List[FtsMigration[F]]
|
def initializeNew: List[FtsMigration[F]]
|
||||||
|
|
||||||
@ -53,18 +49,16 @@ trait FtsClient[F[_]] {
|
|||||||
else Stream.emit(result) ++ searchAll(q.nextPage)
|
else Stream.emit(result) ++ searchAll(q.nextPage)
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Push all data to the index. Data with same `id' is replaced.
|
/** Push all data to the index. Data with same `id' is replaced. Values that are `None'
|
||||||
* Values that are `None' are removed from the index (or set to an
|
* are removed from the index (or set to an empty string).
|
||||||
* empty string).
|
|
||||||
*/
|
*/
|
||||||
def indexData(logger: Logger[F], data: Stream[F, TextData]): F[Unit]
|
def indexData(logger: Logger[F], data: Stream[F, TextData]): F[Unit]
|
||||||
|
|
||||||
def indexData(logger: Logger[F], data: TextData*): F[Unit] =
|
def indexData(logger: Logger[F], data: TextData*): F[Unit] =
|
||||||
indexData(logger, Stream.emits(data))
|
indexData(logger, Stream.emits(data))
|
||||||
|
|
||||||
/** Push all data to the index, but only update existing entries. No
|
/** Push all data to the index, but only update existing entries. No new entries are
|
||||||
* new entries are created and values that are given as `None' are
|
* created and values that are given as `None' are skipped.
|
||||||
* skipped.
|
|
||||||
*/
|
*/
|
||||||
def updateIndex(logger: Logger[F], data: Stream[F, TextData]): F[Unit]
|
def updateIndex(logger: Logger[F], data: Stream[F, TextData]): F[Unit]
|
||||||
|
|
||||||
|
@ -10,16 +10,14 @@ import docspell.common._
|
|||||||
|
|
||||||
/** A fulltext query.
|
/** A fulltext query.
|
||||||
*
|
*
|
||||||
* The query itself is a raw string. Each implementation may
|
* The query itself is a raw string. Each implementation may interpret it according to
|
||||||
* interpret it according to the system in use.
|
* the system in use.
|
||||||
*
|
*
|
||||||
* Searches must only look for given collective and in the given list
|
* Searches must only look for given collective and in the given list of item ids, if it
|
||||||
* of item ids, if it is non-empty. If the item set is empty, then
|
* is non-empty. If the item set is empty, then don't restrict the result in this way.
|
||||||
* don't restrict the result in this way.
|
|
||||||
*
|
*
|
||||||
* The set of folders must be used to restrict the results only to
|
* The set of folders must be used to restrict the results only to items that have one of
|
||||||
* items that have one of the folders set or no folder set. If the
|
* the folders set or no folder set. If the set is empty, the restriction does not apply.
|
||||||
* set is empty, the restriction does not apply.
|
|
||||||
*/
|
*/
|
||||||
final case class FtsQuery(
|
final case class FtsQuery(
|
||||||
q: String,
|
q: String,
|
||||||
|
@ -22,10 +22,9 @@ trait JoexApp[F[_]] {
|
|||||||
|
|
||||||
/** Shuts down the job executor.
|
/** Shuts down the job executor.
|
||||||
*
|
*
|
||||||
* It will immediately stop taking new jobs, waiting for currently
|
* It will immediately stop taking new jobs, waiting for currently running jobs to
|
||||||
* running jobs to complete normally (i.e. running jobs are not
|
* complete normally (i.e. running jobs are not canceled). After this completed, the
|
||||||
* canceled). After this completed, the webserver stops and the
|
* webserver stops and the main loop will exit.
|
||||||
* main loop will exit.
|
|
||||||
*/
|
*/
|
||||||
def initShutdown: F[Unit]
|
def initShutdown: F[Unit]
|
||||||
}
|
}
|
||||||
|
@ -97,7 +97,7 @@ object NerFile {
|
|||||||
|
|
||||||
private def sanitizeRegex(str: String): String =
|
private def sanitizeRegex(str: String): String =
|
||||||
str.trim.toLowerCase.foldLeft("") { (res, ch) =>
|
str.trim.toLowerCase.foldLeft("") { (res, ch) =>
|
||||||
if (invalidChars.contains(ch)) s"${res}\\$ch"
|
if (invalidChars.contains(ch)) s"$res\\$ch"
|
||||||
else s"$res$ch"
|
else s"$res$ch"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -22,8 +22,7 @@ import docspell.store.records.RPerson
|
|||||||
import io.circe.syntax._
|
import io.circe.syntax._
|
||||||
import org.log4s.getLogger
|
import org.log4s.getLogger
|
||||||
|
|
||||||
/** Maintains a custom regex-ner file per collective for stanford's
|
/** Maintains a custom regex-ner file per collective for stanford's regexner annotator.
|
||||||
* regexner annotator.
|
|
||||||
*/
|
*/
|
||||||
trait RegexNerFile[F[_]] {
|
trait RegexNerFile[F[_]] {
|
||||||
|
|
||||||
@ -64,7 +63,7 @@ object RegexNerFile {
|
|||||||
val dur = Duration.between(nf.creation, now)
|
val dur = Duration.between(nf.creation, now)
|
||||||
if (dur > cfg.minTime)
|
if (dur > cfg.minTime)
|
||||||
logger.fdebug(
|
logger.fdebug(
|
||||||
s"Cache time elapsed (${dur} > ${cfg.minTime}). Check for new state."
|
s"Cache time elapsed ($dur > ${cfg.minTime}). Check for new state."
|
||||||
) *> updateFile(
|
) *> updateFile(
|
||||||
collective,
|
collective,
|
||||||
now,
|
now,
|
||||||
|
@ -52,7 +52,7 @@ object EmptyTrashTask {
|
|||||||
s"Starting removing all soft-deleted items older than ${maxDate.asString}"
|
s"Starting removing all soft-deleted items older than ${maxDate.asString}"
|
||||||
)
|
)
|
||||||
nDeleted <- deleteAll(ctx.args, maxDate, itemOps, itemSearchOps, ctx)
|
nDeleted <- deleteAll(ctx.args, maxDate, itemOps, itemSearchOps, ctx)
|
||||||
_ <- ctx.logger.info(s"Finished deleting ${nDeleted} items")
|
_ <- ctx.logger.info(s"Finished deleting $nDeleted items")
|
||||||
} yield ()
|
} yield ()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -101,7 +101,7 @@ object FtsWork {
|
|||||||
def recoverWith(
|
def recoverWith(
|
||||||
other: FtsWork[F]
|
other: FtsWork[F]
|
||||||
)(implicit ev: ApplicativeError[F, Throwable]): FtsWork[F] =
|
)(implicit ev: ApplicativeError[F, Throwable]): FtsWork[F] =
|
||||||
Kleisli(ctx => mt.run(ctx).onError({ case _ => other.run(ctx) }))
|
Kleisli(ctx => mt.run(ctx).onError { case _ => other.run(ctx) })
|
||||||
|
|
||||||
def forContext(
|
def forContext(
|
||||||
cfg: Config.FullTextSearch,
|
cfg: Config.FullTextSearch,
|
||||||
|
@ -19,9 +19,8 @@ import docspell.store.Store
|
|||||||
|
|
||||||
/** Migrating the index from the previous version to this version.
|
/** Migrating the index from the previous version to this version.
|
||||||
*
|
*
|
||||||
* The migration asks the fulltext search client for a list of
|
* The migration asks the fulltext search client for a list of migration tasks to run. It
|
||||||
* migration tasks to run. It may be empty when there is no migration
|
* may be empty when there is no migration required.
|
||||||
* required.
|
|
||||||
*/
|
*/
|
||||||
case class Migration[F[_]](
|
case class Migration[F[_]](
|
||||||
version: Int,
|
version: Int,
|
||||||
|
@ -10,8 +10,7 @@ import cats.data.Kleisli
|
|||||||
|
|
||||||
package object fts {
|
package object fts {
|
||||||
|
|
||||||
/** Some work that must be done to advance the schema of the fulltext
|
/** Some work that must be done to advance the schema of the fulltext index.
|
||||||
* index.
|
|
||||||
*/
|
*/
|
||||||
type FtsWork[F[_]] = Kleisli[F, FtsContext[F], Unit]
|
type FtsWork[F[_]] = Kleisli[F, FtsContext[F], Unit]
|
||||||
|
|
||||||
|
@ -23,7 +23,7 @@ object ClassifierName {
|
|||||||
private val categoryPrefix = "tagcategory-"
|
private val categoryPrefix = "tagcategory-"
|
||||||
|
|
||||||
def tagCategory(cat: String): ClassifierName =
|
def tagCategory(cat: String): ClassifierName =
|
||||||
apply(s"${categoryPrefix}${cat}")
|
apply(s"$categoryPrefix$cat")
|
||||||
|
|
||||||
val concernedPerson: ClassifierName =
|
val concernedPerson: ClassifierName =
|
||||||
apply("concernedperson")
|
apply("concernedperson")
|
||||||
@ -56,7 +56,7 @@ object ClassifierName {
|
|||||||
def findOrphanTagModels[F[_]](coll: Ident): ConnectionIO[List[RClassifierModel]] =
|
def findOrphanTagModels[F[_]](coll: Ident): ConnectionIO[List[RClassifierModel]] =
|
||||||
for {
|
for {
|
||||||
cats <- RClassifierSetting.getActiveCategories(coll)
|
cats <- RClassifierSetting.getActiveCategories(coll)
|
||||||
allModels = RClassifierModel.findAllByQuery(coll, s"${categoryPrefix}%")
|
allModels = RClassifierModel.findAllByQuery(coll, s"$categoryPrefix%")
|
||||||
result <- NonEmptyList.fromList(cats) match {
|
result <- NonEmptyList.fromList(cats) match {
|
||||||
case Some(nel) =>
|
case Some(nel) =>
|
||||||
allModels.flatMap(all =>
|
allModels.flatMap(all =>
|
||||||
|
@ -47,7 +47,7 @@ object Classify {
|
|||||||
.flatMap(_ => classifier.classify(logger, ClassifierModel(modelFile), text))
|
.flatMap(_ => classifier.classify(logger, ClassifierModel(modelFile), text))
|
||||||
}).filter(_ != LearnClassifierTask.noClass)
|
}).filter(_ != LearnClassifierTask.noClass)
|
||||||
.flatTapNone(logger.debug("Guessed: <none>"))
|
.flatTapNone(logger.debug("Guessed: <none>"))
|
||||||
_ <- OptionT.liftF(logger.debug(s"Guessed: ${cls}"))
|
_ <- OptionT.liftF(logger.debug(s"Guessed: $cls"))
|
||||||
} yield cls).value
|
} yield cls).value
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -26,9 +26,8 @@ import bitpeace.RangeDef
|
|||||||
import io.circe.generic.semiauto._
|
import io.circe.generic.semiauto._
|
||||||
import io.circe.{Decoder, Encoder}
|
import io.circe.{Decoder, Encoder}
|
||||||
|
|
||||||
/** Converts the given attachment file using ocrmypdf if it is a pdf
|
/** Converts the given attachment file using ocrmypdf if it is a pdf and has not already
|
||||||
* and has not already been converted (the source file is the same as
|
* been converted (the source file is the same as in the attachment).
|
||||||
* in the attachment).
|
|
||||||
*/
|
*/
|
||||||
object PdfConvTask {
|
object PdfConvTask {
|
||||||
case class Args(attachId: Ident)
|
case class Args(attachId: Ident)
|
||||||
@ -100,7 +99,7 @@ object PdfConvTask {
|
|||||||
.through(bp.fetchData2(RangeDef.all))
|
.through(bp.fetchData2(RangeDef.all))
|
||||||
|
|
||||||
val storeResult: ConversionResult.Handler[F, Unit] =
|
val storeResult: ConversionResult.Handler[F, Unit] =
|
||||||
Kleisli({
|
Kleisli {
|
||||||
case ConversionResult.SuccessPdf(file) =>
|
case ConversionResult.SuccessPdf(file) =>
|
||||||
storeToAttachment(ctx, in, file)
|
storeToAttachment(ctx, in, file)
|
||||||
|
|
||||||
@ -109,15 +108,15 @@ object PdfConvTask {
|
|||||||
|
|
||||||
case ConversionResult.UnsupportedFormat(mime) =>
|
case ConversionResult.UnsupportedFormat(mime) =>
|
||||||
ctx.logger.warn(
|
ctx.logger.warn(
|
||||||
s"Unable to convert '${mime}' file ${ctx.args}: unsupported format."
|
s"Unable to convert '$mime' file ${ctx.args}: unsupported format."
|
||||||
)
|
)
|
||||||
|
|
||||||
case ConversionResult.InputMalformed(mime, reason) =>
|
case ConversionResult.InputMalformed(mime, reason) =>
|
||||||
ctx.logger.warn(s"Unable to convert '${mime}' file ${ctx.args}: $reason")
|
ctx.logger.warn(s"Unable to convert '$mime' file ${ctx.args}: $reason")
|
||||||
|
|
||||||
case ConversionResult.Failure(ex) =>
|
case ConversionResult.Failure(ex) =>
|
||||||
Sync[F].raiseError(ex)
|
Sync[F].raiseError(ex)
|
||||||
})
|
}
|
||||||
|
|
||||||
def ocrMyPdf(lang: Language): F[Unit] =
|
def ocrMyPdf(lang: Language): F[Unit] =
|
||||||
OcrMyPdf.toPDF[F, Unit](
|
OcrMyPdf.toPDF[F, Unit](
|
||||||
|
@ -22,9 +22,8 @@ import docspell.store.syntax.MimeTypes._
|
|||||||
|
|
||||||
import bitpeace.{Mimetype, RangeDef}
|
import bitpeace.{Mimetype, RangeDef}
|
||||||
|
|
||||||
/** Goes through all attachments that must be already converted into a
|
/** Goes through all attachments that must be already converted into a pdf. If it is a
|
||||||
* pdf. If it is a pdf, the number of pages are retrieved and stored
|
* pdf, the number of pages are retrieved and stored in the attachment metadata.
|
||||||
* in the attachment metadata.
|
|
||||||
*/
|
*/
|
||||||
object AttachmentPageCount {
|
object AttachmentPageCount {
|
||||||
|
|
||||||
|
@ -24,9 +24,9 @@ import docspell.store.syntax.MimeTypes._
|
|||||||
|
|
||||||
import bitpeace.{Mimetype, MimetypeHint, RangeDef}
|
import bitpeace.{Mimetype, MimetypeHint, RangeDef}
|
||||||
|
|
||||||
/** Goes through all attachments that must be already converted into a
|
/** Goes through all attachments that must be already converted into a pdf. If it is a
|
||||||
* pdf. If it is a pdf, the first page is converted into a small
|
* pdf, the first page is converted into a small preview png image and linked to the
|
||||||
* preview png image and linked to the attachment.
|
* attachment.
|
||||||
*/
|
*/
|
||||||
object AttachmentPreview {
|
object AttachmentPreview {
|
||||||
|
|
||||||
|
@ -23,19 +23,16 @@ import docspell.store.syntax.MimeTypes._
|
|||||||
|
|
||||||
import bitpeace.{Mimetype, MimetypeHint, RangeDef}
|
import bitpeace.{Mimetype, MimetypeHint, RangeDef}
|
||||||
|
|
||||||
/** Goes through all attachments and creates a PDF version of it where
|
/** Goes through all attachments and creates a PDF version of it where supported.
|
||||||
* supported.
|
|
||||||
*
|
*
|
||||||
* The `attachment` record is updated with the PDF version while the
|
* The `attachment` record is updated with the PDF version while the original file has
|
||||||
* original file has been stored in the `attachment_source` record.
|
* been stored in the `attachment_source` record.
|
||||||
*
|
*
|
||||||
* If pdf conversion is not possible or if the input is already a
|
* If pdf conversion is not possible or if the input is already a pdf, both files are
|
||||||
* pdf, both files are identical. That is, the `file_id`s point to
|
* identical. That is, the `file_id`s point to the same file. Since the name of an
|
||||||
* the same file. Since the name of an attachment may be changed by
|
* attachment may be changed by the user, the `attachment_origin` record keeps that, too.
|
||||||
* the user, the `attachment_origin` record keeps that, too.
|
|
||||||
*
|
*
|
||||||
* This step assumes an existing premature item, it traverses its
|
* This step assumes an existing premature item, it traverses its attachments.
|
||||||
* attachments.
|
|
||||||
*/
|
*/
|
||||||
object ConvertPdf {
|
object ConvertPdf {
|
||||||
|
|
||||||
@ -104,7 +101,7 @@ object ConvertPdf {
|
|||||||
ra: RAttachment,
|
ra: RAttachment,
|
||||||
item: ItemData
|
item: ItemData
|
||||||
): Handler[F, (RAttachment, Option[RAttachmentMeta])] =
|
): Handler[F, (RAttachment, Option[RAttachmentMeta])] =
|
||||||
Kleisli({
|
Kleisli {
|
||||||
case ConversionResult.SuccessPdf(pdf) =>
|
case ConversionResult.SuccessPdf(pdf) =>
|
||||||
ctx.logger.info(s"Conversion to pdf successful. Saving file.") *>
|
ctx.logger.info(s"Conversion to pdf successful. Saving file.") *>
|
||||||
storePDF(ctx, cfg, ra, pdf)
|
storePDF(ctx, cfg, ra, pdf)
|
||||||
@ -142,7 +139,7 @@ object ConvertPdf {
|
|||||||
ctx.logger
|
ctx.logger
|
||||||
.error(s"PDF conversion failed: ${ex.getMessage}. Go without PDF file") *>
|
.error(s"PDF conversion failed: ${ex.getMessage}. Go without PDF file") *>
|
||||||
(ra, None: Option[RAttachmentMeta]).pure[F]
|
(ra, None: Option[RAttachmentMeta]).pure[F]
|
||||||
})
|
}
|
||||||
|
|
||||||
private def storePDF[F[_]: Sync](
|
private def storePDF[F[_]: Sync](
|
||||||
ctx: Context[F, ProcessItemArgs],
|
ctx: Context[F, ProcessItemArgs],
|
||||||
@ -196,7 +193,7 @@ object ConvertPdf {
|
|||||||
case Right(_) => ().pure[F]
|
case Right(_) => ().pure[F]
|
||||||
case Left(ex) =>
|
case Left(ex) =>
|
||||||
ctx.logger
|
ctx.logger
|
||||||
.error(ex)(s"Cannot delete previous attachment file: ${raPrev}")
|
.error(ex)(s"Cannot delete previous attachment file: $raPrev")
|
||||||
|
|
||||||
}
|
}
|
||||||
} yield ()
|
} yield ()
|
||||||
|
@ -45,9 +45,9 @@ object CreateItem {
|
|||||||
Stream
|
Stream
|
||||||
.emits(ctx.args.files)
|
.emits(ctx.args.files)
|
||||||
.flatMap(f => ctx.store.bitpeace.get(f.fileMetaId.id).map(fm => (f, fm)))
|
.flatMap(f => ctx.store.bitpeace.get(f.fileMetaId.id).map(fm => (f, fm)))
|
||||||
.collect({ case (f, Some(fm)) if isValidFile(fm) => f })
|
.collect { case (f, Some(fm)) if isValidFile(fm) => f }
|
||||||
.zipWithIndex
|
.zipWithIndex
|
||||||
.evalMap({ case (f, index) =>
|
.evalMap { case (f, index) =>
|
||||||
Ident
|
Ident
|
||||||
.randomId[F]
|
.randomId[F]
|
||||||
.map(id =>
|
.map(id =>
|
||||||
@ -60,7 +60,7 @@ object CreateItem {
|
|||||||
f.name
|
f.name
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
})
|
}
|
||||||
}
|
}
|
||||||
.compile
|
.compile
|
||||||
.toVector
|
.toVector
|
||||||
@ -152,7 +152,7 @@ object CreateItem {
|
|||||||
.transact(RAttachment.findByItemCollectiveSource(ri.id, ri.cid, fids))
|
.transact(RAttachment.findByItemCollectiveSource(ri.id, ri.cid, fids))
|
||||||
.flatTap(ats =>
|
.flatTap(ats =>
|
||||||
ctx.logger.debug(
|
ctx.logger.debug(
|
||||||
s"Found ${ats.size} attachments. Use only those from task args: ${fileMetaIds}"
|
s"Found ${ats.size} attachments. Use only those from task args: $fileMetaIds"
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
@ -14,12 +14,10 @@ import cats.implicits._
|
|||||||
import docspell.common._
|
import docspell.common._
|
||||||
import docspell.joex.scheduler.Task
|
import docspell.joex.scheduler.Task
|
||||||
|
|
||||||
/** After candidates have been determined, the set is reduced by doing
|
/** After candidates have been determined, the set is reduced by doing some cross checks.
|
||||||
* some cross checks. For example: if a organization is suggested as
|
* For example: if a organization is suggested as correspondent, the correspondent person
|
||||||
* correspondent, the correspondent person must be linked to that
|
* must be linked to that organization. So this *removes all* person candidates that are
|
||||||
* organization. So this *removes all* person candidates that are not
|
* not linked to the first organization candidate (which will be linked to the item).
|
||||||
* linked to the first organization candidate (which will be linked
|
|
||||||
* to the item).
|
|
||||||
*/
|
*/
|
||||||
object CrossCheckProposals {
|
object CrossCheckProposals {
|
||||||
|
|
||||||
|
@ -52,7 +52,7 @@ object DuplicateCheck {
|
|||||||
val fname = ctx.args.files.find(_.fileMetaId.id == fd.fm.id).flatMap(_.name)
|
val fname = ctx.args.files.find(_.fileMetaId.id == fd.fm.id).flatMap(_.name)
|
||||||
if (fd.exists)
|
if (fd.exists)
|
||||||
ctx.logger
|
ctx.logger
|
||||||
.info(s"Deleting duplicate file ${fname}!") *> ctx.store.bitpeace
|
.info(s"Deleting duplicate file $fname!") *> ctx.store.bitpeace
|
||||||
.delete(fd.fm.id)
|
.delete(fd.fm.id)
|
||||||
.compile
|
.compile
|
||||||
.drain
|
.drain
|
||||||
|
@ -15,8 +15,7 @@ import docspell.common._
|
|||||||
import docspell.joex.scheduler.{Context, Task}
|
import docspell.joex.scheduler.{Context, Task}
|
||||||
import docspell.store.records.{RAttachmentMeta, RPerson}
|
import docspell.store.records.{RAttachmentMeta, RPerson}
|
||||||
|
|
||||||
/** Calculate weights for candidates that adds the most likely
|
/** Calculate weights for candidates that adds the most likely candidate a lower number.
|
||||||
* candidate a lower number.
|
|
||||||
*/
|
*/
|
||||||
object EvalProposals {
|
object EvalProposals {
|
||||||
|
|
||||||
|
@ -25,16 +25,13 @@ import docspell.store.syntax.MimeTypes._
|
|||||||
import bitpeace.{Mimetype, MimetypeHint, RangeDef}
|
import bitpeace.{Mimetype, MimetypeHint, RangeDef}
|
||||||
import emil.Mail
|
import emil.Mail
|
||||||
|
|
||||||
/** Goes through all attachments and extracts archive files, like zip
|
/** Goes through all attachments and extracts archive files, like zip files. The process
|
||||||
* files. The process is recursive, until all archives have been
|
* is recursive, until all archives have been extracted.
|
||||||
* extracted.
|
|
||||||
*
|
*
|
||||||
* The archive file is stored as a `attachment_archive` record that
|
* The archive file is stored as a `attachment_archive` record that references all its
|
||||||
* references all its elements. If there are inner archive, only the
|
* elements. If there are inner archive, only the outer archive file is preserved.
|
||||||
* outer archive file is preserved.
|
|
||||||
*
|
*
|
||||||
* This step assumes an existing premature item, it traverses its
|
* This step assumes an existing premature item, it traverses its attachments.
|
||||||
* attachments.
|
|
||||||
*/
|
*/
|
||||||
object ExtractArchive {
|
object ExtractArchive {
|
||||||
|
|
||||||
@ -78,11 +75,10 @@ object ExtractArchive {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/** After all files have been extracted, the `extract' contains the
|
/** After all files have been extracted, the `extract' contains the whole (combined)
|
||||||
* whole (combined) result. This fixes positions of the attachments
|
* result. This fixes positions of the attachments such that the elements of an archive
|
||||||
* such that the elements of an archive are "spliced" into the
|
* are "spliced" into the attachment list at the position of the archive. If there is
|
||||||
* attachment list at the position of the archive. If there is no
|
* no archive, positions don't need to be fixed.
|
||||||
* archive, positions don't need to be fixed.
|
|
||||||
*/
|
*/
|
||||||
private def fixPositions(extract: Extracted): Extracted =
|
private def fixPositions(extract: Extracted): Extracted =
|
||||||
if (extract.archives.isEmpty) extract
|
if (extract.archives.isEmpty) extract
|
||||||
@ -267,7 +263,7 @@ object ExtractArchive {
|
|||||||
val sorted = nel.sorted
|
val sorted = nel.sorted
|
||||||
val offset = sorted.head.first
|
val offset = sorted.head.first
|
||||||
val pos =
|
val pos =
|
||||||
sorted.zipWithIndex.map({ case (p, i) => p.id -> (i + offset) }).toList.toMap
|
sorted.zipWithIndex.map { case (p, i) => p.id -> (i + offset) }.toList.toMap
|
||||||
val nf =
|
val nf =
|
||||||
files.map(f => pos.get(f.id).map(n => f.copy(position = n)).getOrElse(f))
|
files.map(f => pos.get(f.id).map(n => f.copy(position = n)).getOrElse(f))
|
||||||
copy(files = nf)
|
copy(files = nf)
|
||||||
|
@ -19,8 +19,8 @@ import docspell.joex.Config
|
|||||||
import docspell.joex.scheduler.{Context, Task}
|
import docspell.joex.scheduler.{Context, Task}
|
||||||
import docspell.store.records._
|
import docspell.store.records._
|
||||||
|
|
||||||
/** Super simple approach to find corresponding meta data to an item
|
/** Super simple approach to find corresponding meta data to an item by looking up values
|
||||||
* by looking up values from NER in the users address book.
|
* from NER in the users address book.
|
||||||
*/
|
*/
|
||||||
object FindProposal {
|
object FindProposal {
|
||||||
type Args = ProcessItemArgs
|
type Args = ProcessItemArgs
|
||||||
|
@ -12,18 +12,23 @@ import docspell.store.records.{RAttachment, RAttachmentMeta, RItem}
|
|||||||
|
|
||||||
/** Data that is carried across all processing tasks.
|
/** Data that is carried across all processing tasks.
|
||||||
*
|
*
|
||||||
* @param item the stored item record
|
* @param item
|
||||||
* @param attachments the attachments belonging to the item
|
* the stored item record
|
||||||
* @param metas the meta data to each attachment; depending on the
|
* @param attachments
|
||||||
* state of processing, this may be empty
|
* the attachments belonging to the item
|
||||||
* @param dateLabels a separate list of found dates
|
* @param metas
|
||||||
* @param originFile a mapping from an attachment id to a filemeta-id
|
* the meta data to each attachment; depending on the state of processing, this may be
|
||||||
* containng the source or origin file
|
* empty
|
||||||
* @param givenMeta meta data to this item that was not "guessed"
|
* @param dateLabels
|
||||||
* from an attachment but given and thus is always correct
|
* a separate list of found dates
|
||||||
* @param classifyProposals these are proposals that were obtained by
|
* @param originFile
|
||||||
* a trained classifier. There are no ner-tags, it will only provide a
|
* a mapping from an attachment id to a filemeta-id containng the source or origin file
|
||||||
* single label
|
* @param givenMeta
|
||||||
|
* meta data to this item that was not "guessed" from an attachment but given and thus
|
||||||
|
* is always correct
|
||||||
|
* @param classifyProposals
|
||||||
|
* these are proposals that were obtained by a trained classifier. There are no
|
||||||
|
* ner-tags, it will only provide a single label
|
||||||
*/
|
*/
|
||||||
case class ItemData(
|
case class ItemData(
|
||||||
item: RItem,
|
item: RItem,
|
||||||
@ -39,9 +44,8 @@ case class ItemData(
|
|||||||
classifyTags: List[String]
|
classifyTags: List[String]
|
||||||
) {
|
) {
|
||||||
|
|
||||||
/** sort by weight; order of equal weights is not important, just
|
/** sort by weight; order of equal weights is not important, just choose one others are
|
||||||
* choose one others are then suggestions
|
* then suggestions doc-date is only set when given explicitely, not from "guessing"
|
||||||
* doc-date is only set when given explicitely, not from "guessing"
|
|
||||||
*/
|
*/
|
||||||
def finalProposals: MetaProposalList =
|
def finalProposals: MetaProposalList =
|
||||||
MetaProposalList
|
MetaProposalList
|
||||||
|
@ -77,7 +77,7 @@ object ItemHandler {
|
|||||||
)(data: ItemData): Task[F, Args, ItemData] =
|
)(data: ItemData): Task[F, Args, ItemData] =
|
||||||
isLastRetry[F].flatMap {
|
isLastRetry[F].flatMap {
|
||||||
case true =>
|
case true =>
|
||||||
ProcessItem[F](cfg, itemOps, fts, analyser, regexNer)(data).attempt.flatMap({
|
ProcessItem[F](cfg, itemOps, fts, analyser, regexNer)(data).attempt.flatMap {
|
||||||
case Right(d) =>
|
case Right(d) =>
|
||||||
Task.pure(d)
|
Task.pure(d)
|
||||||
case Left(ex) =>
|
case Left(ex) =>
|
||||||
@ -85,7 +85,7 @@ object ItemHandler {
|
|||||||
"Processing failed on last retry. Creating item but without proposals."
|
"Processing failed on last retry. Creating item but without proposals."
|
||||||
).flatMap(_ => itemStateTask(ItemState.Created)(data))
|
).flatMap(_ => itemStateTask(ItemState.Created)(data))
|
||||||
.andThen(_ => Sync[F].raiseError(ex))
|
.andThen(_ => Sync[F].raiseError(ex))
|
||||||
})
|
}
|
||||||
case false =>
|
case false =>
|
||||||
ProcessItem[F](cfg, itemOps, fts, analyser, regexNer)(data)
|
ProcessItem[F](cfg, itemOps, fts, analyser, regexNer)(data)
|
||||||
.flatMap(itemStateTask(ItemState.Created))
|
.flatMap(itemStateTask(ItemState.Created))
|
||||||
|
@ -57,7 +57,7 @@ object LinkProposal {
|
|||||||
case Some(a) =>
|
case Some(a) =>
|
||||||
val ids = a.values.map(_.ref.id.id)
|
val ids = a.values.map(_.ref.id.id)
|
||||||
ctx.logger.info(
|
ctx.logger.info(
|
||||||
s"Found many (${a.size}, ${ids}) candidates for ${a.proposalType}. Setting first."
|
s"Found many (${a.size}, $ids) candidates for ${a.proposalType}. Setting first."
|
||||||
) *>
|
) *>
|
||||||
setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).map(_ =>
|
setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).map(_ =>
|
||||||
Result.multiple(mpt)
|
Result.multiple(mpt)
|
||||||
|
@ -149,14 +149,14 @@ object ReProcessItem {
|
|||||||
isLastRetry[F].flatMap {
|
isLastRetry[F].flatMap {
|
||||||
case true =>
|
case true =>
|
||||||
processFiles[F](cfg, fts, itemOps, analyser, regexNer, data).attempt
|
processFiles[F](cfg, fts, itemOps, analyser, regexNer, data).attempt
|
||||||
.flatMap({
|
.flatMap {
|
||||||
case Right(d) =>
|
case Right(d) =>
|
||||||
Task.pure(d)
|
Task.pure(d)
|
||||||
case Left(ex) =>
|
case Left(ex) =>
|
||||||
logWarn[F](
|
logWarn[F](
|
||||||
"Processing failed on last retry."
|
"Processing failed on last retry."
|
||||||
).andThen(_ => Sync[F].raiseError(ex))
|
).andThen(_ => Sync[F].raiseError(ex))
|
||||||
})
|
}
|
||||||
case false =>
|
case false =>
|
||||||
processFiles[F](cfg, fts, itemOps, analyser, regexNer, data)
|
processFiles[F](cfg, fts, itemOps, analyser, regexNer, data)
|
||||||
}
|
}
|
||||||
|
@ -67,7 +67,7 @@ object SetGivenData {
|
|||||||
val tags =
|
val tags =
|
||||||
(ctx.args.meta.tags.getOrElse(Nil) ++ data.tags ++ data.classifyTags).distinct
|
(ctx.args.meta.tags.getOrElse(Nil) ++ data.tags ++ data.classifyTags).distinct
|
||||||
for {
|
for {
|
||||||
_ <- ctx.logger.info(s"Set tags from given data: ${tags}")
|
_ <- ctx.logger.info(s"Set tags from given data: $tags")
|
||||||
e <- ops.linkTags(itemId, tags, collective).attempt
|
e <- ops.linkTags(itemId, tags, collective).attempt
|
||||||
_ <- e.fold(
|
_ <- e.fold(
|
||||||
ex => ctx.logger.warn(s"Error setting tags: ${ex.getMessage}"),
|
ex => ctx.logger.warn(s"Error setting tags: ${ex.getMessage}"),
|
||||||
|
@ -158,7 +158,7 @@ object TextExtraction {
|
|||||||
val extr = Extraction.create[F](ctx.logger, cfg)
|
val extr = Extraction.create[F](ctx.logger, cfg)
|
||||||
|
|
||||||
extractText[F](ctx, extr, lang)(id)
|
extractText[F](ctx, extr, lang)(id)
|
||||||
.flatMap({
|
.flatMap {
|
||||||
case res @ ExtractResult.Success(_, _) =>
|
case res @ ExtractResult.Success(_, _) =>
|
||||||
res.some.pure[F]
|
res.some.pure[F]
|
||||||
|
|
||||||
@ -173,15 +173,14 @@ object TextExtraction {
|
|||||||
ctx.logger
|
ctx.logger
|
||||||
.warn(s"Cannot extract text: ${ex.getMessage}. Try with converted file")
|
.warn(s"Cannot extract text: ${ex.getMessage}. Try with converted file")
|
||||||
.flatMap(_ => extractTextFallback[F](ctx, cfg, ra, lang)(rest))
|
.flatMap(_ => extractTextFallback[F](ctx, cfg, ra, lang)(rest))
|
||||||
})
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Returns the fileIds to extract text from. First, the source file
|
/** Returns the fileIds to extract text from. First, the source file is tried. If that
|
||||||
* is tried. If that fails, the converted file is tried.
|
* fails, the converted file is tried.
|
||||||
*
|
*
|
||||||
* If the source file is a PDF, then use the converted file. This
|
* If the source file is a PDF, then use the converted file. This may then already
|
||||||
* may then already contain the text if ocrmypdf is enabled. If it
|
* contain the text if ocrmypdf is enabled. If it is disabled, both files are the same.
|
||||||
* is disabled, both files are the same.
|
|
||||||
*/
|
*/
|
||||||
private def filesToExtract[F[_]: Sync](ctx: Context[F, _])(
|
private def filesToExtract[F[_]: Sync](ctx: Context[F, _])(
|
||||||
item: ItemData,
|
item: ItemData,
|
||||||
|
@ -50,7 +50,10 @@ object JoexRoutes {
|
|||||||
for {
|
for {
|
||||||
optJob <- app.scheduler.getRunning.map(_.find(_.id == id))
|
optJob <- app.scheduler.getRunning.map(_.find(_.id == id))
|
||||||
optLog <- optJob.traverse(j => app.findLogs(j.id))
|
optLog <- optJob.traverse(j => app.findLogs(j.id))
|
||||||
jAndL = for { job <- optJob; log <- optLog } yield mkJobLog(job, log)
|
jAndL = for {
|
||||||
|
job <- optJob
|
||||||
|
log <- optLog
|
||||||
|
} yield mkJobLog(job, log)
|
||||||
resp <- jAndL.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found")))
|
resp <- jAndL.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found")))
|
||||||
} yield resp
|
} yield resp
|
||||||
|
|
||||||
|
@ -46,7 +46,7 @@ object ScanMailboxTask {
|
|||||||
userId = ctx.args.account.user
|
userId = ctx.args.account.user
|
||||||
imapConn = ctx.args.imapConnection
|
imapConn = ctx.args.imapConnection
|
||||||
_ <- ctx.logger.info(
|
_ <- ctx.logger.info(
|
||||||
s"Reading mails for user ${userId.id} from ${imapConn.id}/${folders}"
|
s"Reading mails for user ${userId.id} from ${imapConn.id}/$folders"
|
||||||
)
|
)
|
||||||
_ <- importMails(cfg, mailCfg, emil, upload, joex, ctx)
|
_ <- importMails(cfg, mailCfg, emil, upload, joex, ctx)
|
||||||
} yield ()
|
} yield ()
|
||||||
|
@ -10,11 +10,10 @@ import cats.implicits._
|
|||||||
|
|
||||||
import docspell.common.Priority
|
import docspell.common.Priority
|
||||||
|
|
||||||
/** A counting scheme to indicate a ratio between scheduling high and
|
/** A counting scheme to indicate a ratio between scheduling high and low priority jobs.
|
||||||
* low priority jobs.
|
|
||||||
*
|
*
|
||||||
* For example high=4, low=1 means: ”schedule 4 high priority jobs
|
* For example high=4, low=1 means: ”schedule 4 high priority jobs and then 1 low
|
||||||
* and then 1 low priority job“.
|
* priority job“.
|
||||||
*/
|
*/
|
||||||
case class CountingScheme(high: Int, low: Int, counter: Int = 0) {
|
case class CountingScheme(high: Int, low: Int, counter: Int = 0) {
|
||||||
|
|
||||||
|
@ -14,14 +14,13 @@ import docspell.common.syntax.all._
|
|||||||
|
|
||||||
import io.circe.Decoder
|
import io.circe.Decoder
|
||||||
|
|
||||||
/** Binds a Task to a name. This is required to lookup the code based
|
/** Binds a Task to a name. This is required to lookup the code based on the taskName in
|
||||||
* on the taskName in the RJob data and to execute it given the
|
* the RJob data and to execute it given the arguments that have to be read from a
|
||||||
* arguments that have to be read from a string.
|
* string.
|
||||||
*
|
*
|
||||||
* Since the scheduler only has a string for the task argument, this
|
* Since the scheduler only has a string for the task argument, this only works for Task
|
||||||
* only works for Task impls that accept a string. There is a
|
* impls that accept a string. There is a convenience constructor that uses circe to
|
||||||
* convenience constructor that uses circe to decode json into some
|
* decode json into some type A.
|
||||||
* type A.
|
|
||||||
*/
|
*/
|
||||||
case class JobTask[F[_]](
|
case class JobTask[F[_]](
|
||||||
name: Ident,
|
name: Ident,
|
||||||
|
@ -8,9 +8,8 @@ package docspell.joex.scheduler
|
|||||||
|
|
||||||
import docspell.common.Ident
|
import docspell.common.Ident
|
||||||
|
|
||||||
/** This is a mapping from some identifier to a task. This is used by
|
/** This is a mapping from some identifier to a task. This is used by the scheduler to
|
||||||
* the scheduler to lookup an implementation using the taskName field
|
* lookup an implementation using the taskName field of the RJob database record.
|
||||||
* of the RJob database record.
|
|
||||||
*/
|
*/
|
||||||
final class JobTaskRegistry[F[_]](tasks: Map[Ident, JobTask[F]]) {
|
final class JobTaskRegistry[F[_]](tasks: Map[Ident, JobTask[F]]) {
|
||||||
|
|
||||||
|
@ -13,14 +13,12 @@ import fs2.concurrent.SignallingRef
|
|||||||
import docspell.joexapi.client.JoexClient
|
import docspell.joexapi.client.JoexClient
|
||||||
import docspell.store.queue._
|
import docspell.store.queue._
|
||||||
|
|
||||||
/** A periodic scheduler takes care to submit periodic tasks to the
|
/** A periodic scheduler takes care to submit periodic tasks to the job queue.
|
||||||
* job queue.
|
|
||||||
*
|
*
|
||||||
* It is run in the background to regularily find a periodic task to
|
* It is run in the background to regularily find a periodic task to execute. If the task
|
||||||
* execute. If the task is due, it will be submitted into the job
|
* is due, it will be submitted into the job queue where it will be picked up by the
|
||||||
* queue where it will be picked up by the scheduler from some joex
|
* scheduler from some joex instance. If it is due in the future, a notification is
|
||||||
* instance. If it is due in the future, a notification is scheduled
|
* scheduled to be received at that time so the task can be looked up again.
|
||||||
* to be received at that time so the task can be looked up again.
|
|
||||||
*/
|
*/
|
||||||
trait PeriodicScheduler[F[_]] {
|
trait PeriodicScheduler[F[_]] {
|
||||||
|
|
||||||
|
@ -53,8 +53,8 @@ final class PeriodicSchedulerImpl[F[_]: Async](
|
|||||||
|
|
||||||
// internal
|
// internal
|
||||||
|
|
||||||
/** On startup, get all periodic jobs from this scheduler and remove
|
/** On startup, get all periodic jobs from this scheduler and remove the mark, so they
|
||||||
* the mark, so they get picked up again.
|
* get picked up again.
|
||||||
*/
|
*/
|
||||||
def init: F[Unit] =
|
def init: F[Unit] =
|
||||||
logError("Error clearing marks")(store.clearMarks(config.name))
|
logError("Error clearing marks")(store.clearMarks(config.name))
|
||||||
@ -68,7 +68,7 @@ final class PeriodicSchedulerImpl[F[_]: Async](
|
|||||||
go <- logThrow("Error getting next task")(
|
go <- logThrow("Error getting next task")(
|
||||||
store
|
store
|
||||||
.takeNext(config.name, None)
|
.takeNext(config.name, None)
|
||||||
.use({
|
.use {
|
||||||
case Marked.Found(pj) =>
|
case Marked.Found(pj) =>
|
||||||
logger
|
logger
|
||||||
.fdebug(s"Found periodic task '${pj.subject}/${pj.timer.asString}'") *>
|
.fdebug(s"Found periodic task '${pj.subject}/${pj.timer.asString}'") *>
|
||||||
@ -79,7 +79,7 @@ final class PeriodicSchedulerImpl[F[_]: Async](
|
|||||||
case Marked.NotMarkable =>
|
case Marked.NotMarkable =>
|
||||||
logger.fdebug("Periodic job cannot be marked. Trying again.") *> true
|
logger.fdebug("Periodic job cannot be marked. Trying again.") *> true
|
||||||
.pure[F]
|
.pure[F]
|
||||||
})
|
}
|
||||||
)
|
)
|
||||||
} yield go
|
} yield go
|
||||||
|
|
||||||
@ -90,7 +90,7 @@ final class PeriodicSchedulerImpl[F[_]: Async](
|
|||||||
else ().pure[F]
|
else ().pure[F]
|
||||||
)
|
)
|
||||||
.flatMap(if (_) Stream.empty else Stream.eval(cancelNotify *> body))
|
.flatMap(if (_) Stream.empty else Stream.eval(cancelNotify *> body))
|
||||||
.flatMap({
|
.flatMap {
|
||||||
case true =>
|
case true =>
|
||||||
mainLoop
|
mainLoop
|
||||||
case false =>
|
case false =>
|
||||||
@ -98,7 +98,7 @@ final class PeriodicSchedulerImpl[F[_]: Async](
|
|||||||
waiter.discrete.take(2).drain ++
|
waiter.discrete.take(2).drain ++
|
||||||
logger.sdebug(s"Notify signal, going into main loop") ++
|
logger.sdebug(s"Notify signal, going into main loop") ++
|
||||||
mainLoop
|
mainLoop
|
||||||
})
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
def isTriggered(pj: RPeriodicTask, now: Timestamp): Boolean =
|
def isTriggered(pj: RPeriodicTask, now: Timestamp): Boolean =
|
||||||
@ -107,7 +107,7 @@ final class PeriodicSchedulerImpl[F[_]: Async](
|
|||||||
def submitJob(pj: RPeriodicTask): F[Boolean] =
|
def submitJob(pj: RPeriodicTask): F[Boolean] =
|
||||||
store
|
store
|
||||||
.findNonFinalJob(pj.id)
|
.findNonFinalJob(pj.id)
|
||||||
.flatMap({
|
.flatMap {
|
||||||
case Some(job) =>
|
case Some(job) =>
|
||||||
logger.finfo[F](
|
logger.finfo[F](
|
||||||
s"There is already a job with non-final state '${job.state}' in the queue"
|
s"There is already a job with non-final state '${job.state}' in the queue"
|
||||||
@ -116,7 +116,7 @@ final class PeriodicSchedulerImpl[F[_]: Async](
|
|||||||
case None =>
|
case None =>
|
||||||
logger.finfo[F](s"Submitting job for periodic task '${pj.task.id}'") *>
|
logger.finfo[F](s"Submitting job for periodic task '${pj.task.id}'") *>
|
||||||
pj.toJob.flatMap(queue.insert) *> notifyJoex *> true.pure[F]
|
pj.toJob.flatMap(queue.insert) *> notifyJoex *> true.pure[F]
|
||||||
})
|
}
|
||||||
|
|
||||||
def notifyJoex: F[Unit] =
|
def notifyJoex: F[Unit] =
|
||||||
sch.notifyChange *> store.findJoexNodes.flatMap(
|
sch.notifyChange *> store.findJoexNodes.flatMap(
|
||||||
@ -145,12 +145,12 @@ final class PeriodicSchedulerImpl[F[_]: Async](
|
|||||||
def cancelNotify: F[Unit] =
|
def cancelNotify: F[Unit] =
|
||||||
state
|
state
|
||||||
.modify(_.clearNotify)
|
.modify(_.clearNotify)
|
||||||
.flatMap({
|
.flatMap {
|
||||||
case Some(fb) =>
|
case Some(fb) =>
|
||||||
fb.cancel
|
fb.cancel
|
||||||
case None =>
|
case None =>
|
||||||
().pure[F]
|
().pure[F]
|
||||||
})
|
}
|
||||||
|
|
||||||
private def logError(msg: => String)(fa: F[Unit]): F[Unit] =
|
private def logError(msg: => String)(fa: F[Unit]): F[Unit] =
|
||||||
fa.attempt.flatMap {
|
fa.attempt.flatMap {
|
||||||
@ -159,12 +159,10 @@ final class PeriodicSchedulerImpl[F[_]: Async](
|
|||||||
}
|
}
|
||||||
|
|
||||||
private def logThrow[A](msg: => String)(fa: F[A]): F[A] =
|
private def logThrow[A](msg: => String)(fa: F[A]): F[A] =
|
||||||
fa.attempt
|
fa.attempt.flatMap {
|
||||||
.flatMap({
|
|
||||||
case r @ Right(_) => (r: Either[Throwable, A]).pure[F]
|
case r @ Right(_) => (r: Either[Throwable, A]).pure[F]
|
||||||
case l @ Left(ex) => logger.ferror(ex)(msg).map(_ => (l: Either[Throwable, A]))
|
case l @ Left(ex) => logger.ferror(ex)(msg).map(_ => (l: Either[Throwable, A]))
|
||||||
})
|
}.rethrow
|
||||||
.rethrow
|
|
||||||
}
|
}
|
||||||
|
|
||||||
object PeriodicSchedulerImpl {
|
object PeriodicSchedulerImpl {
|
||||||
|
@ -26,13 +26,11 @@ trait Scheduler[F[_]] {
|
|||||||
|
|
||||||
/** Requests to shutdown the scheduler.
|
/** Requests to shutdown the scheduler.
|
||||||
*
|
*
|
||||||
* The scheduler will not take any new jobs from the queue. If
|
* The scheduler will not take any new jobs from the queue. If there are still running
|
||||||
* there are still running jobs, it waits for them to complete.
|
* jobs, it waits for them to complete. when the cancelAll flag is set to true, it
|
||||||
* when the cancelAll flag is set to true, it cancels all running
|
* cancels all running jobs.
|
||||||
* jobs.
|
|
||||||
*
|
*
|
||||||
* The returned F[Unit] can be evaluated to wait for all that to
|
* The returned F[Unit] can be evaluated to wait for all that to complete.
|
||||||
* complete.
|
|
||||||
*/
|
*/
|
||||||
def shutdown(cancelAll: Boolean): F[Unit]
|
def shutdown(cancelAll: Boolean): F[Unit]
|
||||||
|
|
||||||
|
@ -36,8 +36,8 @@ final class SchedulerImpl[F[_]: Async](
|
|||||||
|
|
||||||
private[this] val logger = getLogger
|
private[this] val logger = getLogger
|
||||||
|
|
||||||
/** On startup, get all jobs in state running from this scheduler
|
/** On startup, get all jobs in state running from this scheduler and put them into
|
||||||
* and put them into waiting state, so they get picked up again.
|
* waiting state, so they get picked up again.
|
||||||
*/
|
*/
|
||||||
def init: F[Unit] =
|
def init: F[Unit] =
|
||||||
QJob.runningToWaiting(config.name, store)
|
QJob.runningToWaiting(config.name, store)
|
||||||
@ -132,7 +132,7 @@ final class SchedulerImpl[F[_]: Async](
|
|||||||
else ().pure[F]
|
else ().pure[F]
|
||||||
)
|
)
|
||||||
.flatMap(if (_) Stream.empty else Stream.eval(body))
|
.flatMap(if (_) Stream.empty else Stream.eval(body))
|
||||||
.flatMap({
|
.flatMap {
|
||||||
case true =>
|
case true =>
|
||||||
mainLoop
|
mainLoop
|
||||||
case false =>
|
case false =>
|
||||||
@ -140,7 +140,7 @@ final class SchedulerImpl[F[_]: Async](
|
|||||||
waiter.discrete.take(2).drain ++
|
waiter.discrete.take(2).drain ++
|
||||||
logger.sdebug(s"Notify signal, going into main loop") ++
|
logger.sdebug(s"Notify signal, going into main loop") ++
|
||||||
mainLoop
|
mainLoop
|
||||||
})
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private def executeCancel(job: RJob): F[Unit] = {
|
private def executeCancel(job: RJob): F[Unit] = {
|
||||||
@ -214,7 +214,7 @@ final class SchedulerImpl[F[_]: Async](
|
|||||||
): Task[F, String, Unit] =
|
): Task[F, String, Unit] =
|
||||||
task
|
task
|
||||||
.mapF(fa => onStart(job) *> logger.fdebug("Starting task now") *> fa)
|
.mapF(fa => onStart(job) *> logger.fdebug("Starting task now") *> fa)
|
||||||
.mapF(_.attempt.flatMap({
|
.mapF(_.attempt.flatMap {
|
||||||
case Right(()) =>
|
case Right(()) =>
|
||||||
logger.info(s"Job execution successful: ${job.info}")
|
logger.info(s"Job execution successful: ${job.info}")
|
||||||
ctx.logger.info("Job execution successful") *>
|
ctx.logger.info("Job execution successful") *>
|
||||||
@ -239,7 +239,7 @@ final class SchedulerImpl[F[_]: Async](
|
|||||||
.map(_ => JobState.Stuck: JobState)
|
.map(_ => JobState.Stuck: JobState)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}))
|
})
|
||||||
.mapF(_.attempt.flatMap {
|
.mapF(_.attempt.flatMap {
|
||||||
case Right(jstate) =>
|
case Right(jstate) =>
|
||||||
onFinish(job, jstate)
|
onFinish(job, jstate)
|
||||||
@ -262,12 +262,12 @@ final class SchedulerImpl[F[_]: Async](
|
|||||||
.map(fiber =>
|
.map(fiber =>
|
||||||
logger.fdebug(s"Cancelling job ${job.info}") *>
|
logger.fdebug(s"Cancelling job ${job.info}") *>
|
||||||
fiber.cancel *>
|
fiber.cancel *>
|
||||||
onCancel.attempt.map({
|
onCancel.attempt.map {
|
||||||
case Right(_) => ()
|
case Right(_) => ()
|
||||||
case Left(ex) =>
|
case Left(ex) =>
|
||||||
logger.error(ex)(s"Task's cancelling code failed. Job ${job.info}.")
|
logger.error(ex)(s"Task's cancelling code failed. Job ${job.info}.")
|
||||||
()
|
()
|
||||||
}) *>
|
} *>
|
||||||
state.modify(_.markCancelled(job)) *>
|
state.modify(_.markCancelled(job)) *>
|
||||||
onFinish(job, JobState.Cancelled) *>
|
onFinish(job, JobState.Cancelled) *>
|
||||||
ctx.logger.warn("Job has been cancelled.") *>
|
ctx.logger.warn("Job has been cancelled.") *>
|
||||||
|
@ -51,7 +51,7 @@ object JoexClient {
|
|||||||
if (succ) ()
|
if (succ) ()
|
||||||
else
|
else
|
||||||
logger.warn(
|
logger.warn(
|
||||||
s"Notifying Joex instance '${base.asString}' returned with failure: ${msg}"
|
s"Notifying Joex instance '${base.asString}' returned with failure: $msg"
|
||||||
)
|
)
|
||||||
case Left(ex) =>
|
case Left(ex) =>
|
||||||
logger.warn(
|
logger.warn(
|
||||||
|
@ -14,8 +14,7 @@ import docspell.query.ItemQuery.Expr.NotExpr
|
|||||||
import docspell.query.ItemQuery.Expr.OrExpr
|
import docspell.query.ItemQuery.Expr.OrExpr
|
||||||
import docspell.query.ItemQuery._
|
import docspell.query.ItemQuery._
|
||||||
|
|
||||||
/** Currently, fulltext in a query is only supported when in "root
|
/** Currently, fulltext in a query is only supported when in "root AND" position
|
||||||
* AND" position
|
|
||||||
*/
|
*/
|
||||||
object FulltextExtract {
|
object FulltextExtract {
|
||||||
|
|
||||||
@ -45,15 +44,15 @@ object FulltextExtract {
|
|||||||
def findFulltext(expr: Expr): Result =
|
def findFulltext(expr: Expr): Result =
|
||||||
lookForFulltext(expr)
|
lookForFulltext(expr)
|
||||||
|
|
||||||
/** Extracts the fulltext node from the given expr and returns it
|
/** Extracts the fulltext node from the given expr and returns it together with the expr
|
||||||
* together with the expr without that node.
|
* without that node.
|
||||||
*/
|
*/
|
||||||
private def lookForFulltext(expr: Expr): Result =
|
private def lookForFulltext(expr: Expr): Result =
|
||||||
expr match {
|
expr match {
|
||||||
case Expr.Fulltext(ftq) =>
|
case Expr.Fulltext(ftq) =>
|
||||||
Result.SuccessNoExpr(ftq)
|
Result.SuccessNoExpr(ftq)
|
||||||
case Expr.AndExpr(inner) =>
|
case Expr.AndExpr(inner) =>
|
||||||
inner.collect({ case Expr.Fulltext(fq) => fq }) match {
|
inner.collect { case Expr.Fulltext(fq) => fq } match {
|
||||||
case Nil =>
|
case Nil =>
|
||||||
checkPosition(expr, 0)
|
checkPosition(expr, 0)
|
||||||
case e :: Nil =>
|
case e :: Nil =>
|
||||||
|
@ -10,12 +10,10 @@ import cats.data.{NonEmptyList => Nel}
|
|||||||
|
|
||||||
import docspell.query.ItemQuery.Attr.{DateAttr, IntAttr, StringAttr}
|
import docspell.query.ItemQuery.Attr.{DateAttr, IntAttr, StringAttr}
|
||||||
|
|
||||||
/** A query evaluates to `true` or `false` given enough details about
|
/** A query evaluates to `true` or `false` given enough details about an item.
|
||||||
* an item.
|
|
||||||
*
|
*
|
||||||
* It may consist of (field,op,value) tuples that specify some checks
|
* It may consist of (field,op,value) tuples that specify some checks against a specific
|
||||||
* against a specific field of an item using some operator or a
|
* field of an item using some operator or a combination thereof.
|
||||||
* combination thereof.
|
|
||||||
*/
|
*/
|
||||||
final case class ItemQuery(expr: ItemQuery.Expr, raw: Option[String]) {
|
final case class ItemQuery(expr: ItemQuery.Expr, raw: Option[String]) {
|
||||||
def findFulltext: FulltextExtract.Result =
|
def findFulltext: FulltextExtract.Result =
|
||||||
|
@ -45,7 +45,7 @@ object ParseFailure {
|
|||||||
def render: String = {
|
def render: String = {
|
||||||
val opts = expected.mkString(", ")
|
val opts = expected.mkString(", ")
|
||||||
val dots = if (exhaustive) "" else "…"
|
val dots = if (exhaustive) "" else "…"
|
||||||
s"Expected: ${opts}${dots}"
|
s"Expected: $opts$dots"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -57,11 +57,11 @@ object ParseFailure {
|
|||||||
)
|
)
|
||||||
|
|
||||||
private[query] def packMsg(msg: Nel[Message]): Nel[Message] = {
|
private[query] def packMsg(msg: Nel[Message]): Nel[Message] = {
|
||||||
val expectMsg = combineExpected(msg.collect({ case em: ExpectMessage => em }))
|
val expectMsg = combineExpected(msg.collect { case em: ExpectMessage => em })
|
||||||
.sortBy(_.offset)
|
.sortBy(_.offset)
|
||||||
.headOption
|
.headOption
|
||||||
|
|
||||||
val simpleMsg = msg.collect({ case sm: SimpleMessage => sm })
|
val simpleMsg = msg.collect { case sm: SimpleMessage => sm }
|
||||||
|
|
||||||
Nel.fromListUnsafe((simpleMsg ++ expectMsg).sortBy(_.offset))
|
Nel.fromListUnsafe((simpleMsg ++ expectMsg).sortBy(_.offset))
|
||||||
}
|
}
|
||||||
@ -69,13 +69,13 @@ object ParseFailure {
|
|||||||
private[query] def combineExpected(msg: List[ExpectMessage]): List[ExpectMessage] =
|
private[query] def combineExpected(msg: List[ExpectMessage]): List[ExpectMessage] =
|
||||||
msg
|
msg
|
||||||
.groupBy(_.offset)
|
.groupBy(_.offset)
|
||||||
.map({ case (offset, es) =>
|
.map { case (offset, es) =>
|
||||||
ExpectMessage(
|
ExpectMessage(
|
||||||
offset,
|
offset,
|
||||||
es.flatMap(_.expected).distinct.sorted,
|
es.flatMap(_.expected).distinct.sorted,
|
||||||
es.forall(_.exhaustive)
|
es.forall(_.exhaustive)
|
||||||
)
|
)
|
||||||
})
|
}
|
||||||
.toList
|
.toList
|
||||||
|
|
||||||
private[query] def expectationToMsg(e: Parser.Expectation): Message =
|
private[query] def expectationToMsg(e: Parser.Expectation): Message =
|
||||||
@ -89,7 +89,7 @@ object ParseFailure {
|
|||||||
case InRange(offset, lower, upper) =>
|
case InRange(offset, lower, upper) =>
|
||||||
if (lower == upper) ExpectMessage(offset, List(lower.toString), true)
|
if (lower == upper) ExpectMessage(offset, List(lower.toString), true)
|
||||||
else {
|
else {
|
||||||
val expect = s"${lower}-${upper}"
|
val expect = s"$lower-$upper"
|
||||||
ExpectMessage(offset, List(expect), true)
|
ExpectMessage(offset, List(expect), true)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -13,8 +13,8 @@ import docspell.query.ItemQuery._
|
|||||||
|
|
||||||
object ExprUtil {
|
object ExprUtil {
|
||||||
|
|
||||||
/** Does some basic transformation, like unfolding nested and trees
|
/** Does some basic transformation, like unfolding nested and trees containing one value
|
||||||
* containing one value etc.
|
* etc.
|
||||||
*/
|
*/
|
||||||
def reduce(expr: Expr): Expr =
|
def reduce(expr: Expr): Expr =
|
||||||
expr match {
|
expr match {
|
||||||
|
@ -54,12 +54,12 @@ object Config {
|
|||||||
lazy val ipParts = ip.split('.')
|
lazy val ipParts = ip.split('.')
|
||||||
|
|
||||||
def checkSingle(pattern: String): Boolean =
|
def checkSingle(pattern: String): Boolean =
|
||||||
pattern == ip || (inet.isLoopbackAddress && pattern == "127.0.0.1") || (pattern
|
pattern == ip || (inet.isLoopbackAddress && pattern == "127.0.0.1") || pattern
|
||||||
.split('.')
|
.split('.')
|
||||||
.zip(ipParts)
|
.zip(ipParts)
|
||||||
.foldLeft(true) { case (r, (a, b)) =>
|
.foldLeft(true) { case (r, (a, b)) =>
|
||||||
r && (a == "*" || a == b)
|
r && (a == "*" || a == b)
|
||||||
})
|
}
|
||||||
|
|
||||||
ips.exists(checkSingle)
|
ips.exists(checkSingle)
|
||||||
}
|
}
|
||||||
|
@ -579,7 +579,7 @@ trait Conversions {
|
|||||||
)
|
)
|
||||||
|
|
||||||
def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] =
|
def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] =
|
||||||
timeId.map({ case (id, now) =>
|
timeId.map { case (id, now) =>
|
||||||
RSource(
|
RSource(
|
||||||
id,
|
id,
|
||||||
cid,
|
cid,
|
||||||
@ -593,7 +593,7 @@ trait Conversions {
|
|||||||
s.fileFilter,
|
s.fileFilter,
|
||||||
s.language
|
s.language
|
||||||
)
|
)
|
||||||
})
|
}
|
||||||
|
|
||||||
def changeSource[F[_]](s: Source, coll: Ident): RSource =
|
def changeSource[F[_]](s: Source, coll: Ident): RSource =
|
||||||
RSource(
|
RSource(
|
||||||
@ -615,9 +615,9 @@ trait Conversions {
|
|||||||
Equipment(re.eid, re.name, re.created, re.notes, re.use)
|
Equipment(re.eid, re.name, re.created, re.notes, re.use)
|
||||||
|
|
||||||
def newEquipment[F[_]: Sync](e: Equipment, cid: Ident): F[REquipment] =
|
def newEquipment[F[_]: Sync](e: Equipment, cid: Ident): F[REquipment] =
|
||||||
timeId.map({ case (id, now) =>
|
timeId.map { case (id, now) =>
|
||||||
REquipment(id, cid, e.name.trim, now, now, e.notes, e.use)
|
REquipment(id, cid, e.name.trim, now, now, e.notes, e.use)
|
||||||
})
|
}
|
||||||
|
|
||||||
def changeEquipment[F[_]: Sync](e: Equipment, cid: Ident): F[REquipment] =
|
def changeEquipment[F[_]: Sync](e: Equipment, cid: Ident): F[REquipment] =
|
||||||
Timestamp
|
Timestamp
|
||||||
|
@ -29,7 +29,7 @@ object ClientRequestInfo {
|
|||||||
scheme <- NonEmptyList.fromList(getProtocol(req).toList)
|
scheme <- NonEmptyList.fromList(getProtocol(req).toList)
|
||||||
host <- getHostname(req)
|
host <- getHostname(req)
|
||||||
port = xForwardedPort(req).getOrElse(serverPort)
|
port = xForwardedPort(req).getOrElse(serverPort)
|
||||||
hostPort = if (port == 80 || port == 443) host else s"${host}:${port}"
|
hostPort = if (port == 80 || port == 443) host else s"$host:$port"
|
||||||
} yield LenientUri(scheme, Some(hostPort), LenientUri.EmptyPath, None, None)
|
} yield LenientUri(scheme, Some(hostPort), LenientUri.EmptyPath, None, None)
|
||||||
|
|
||||||
def getHostname[F[_]](req: Request[F]): Option[String] =
|
def getHostname[F[_]](req: Request[F]): Option[String] =
|
||||||
|
@ -56,5 +56,5 @@ object AdminRoutes {
|
|||||||
|
|
||||||
private def compareSecret(s1: String)(s2: String): Boolean =
|
private def compareSecret(s1: String)(s2: String): Boolean =
|
||||||
s1.length > 0 && s1.length == s2.length &&
|
s1.length > 0 && s1.length == s2.length &&
|
||||||
s1.zip(s2).forall({ case (a, b) => a == b })
|
s1.zip(s2).forall { case (a, b) => a == b }
|
||||||
}
|
}
|
||||||
|
@ -22,12 +22,12 @@ import io.circe.{Decoder, Encoder}
|
|||||||
|
|
||||||
trait DoobieMeta extends EmilDoobieMeta {
|
trait DoobieMeta extends EmilDoobieMeta {
|
||||||
|
|
||||||
implicit val sqlLogging = LogHandler({
|
implicit val sqlLogging = LogHandler {
|
||||||
case e @ Success(_, _, _, _) =>
|
case e @ Success(_, _, _, _) =>
|
||||||
DoobieMeta.logger.trace("SQL " + e)
|
DoobieMeta.logger.trace("SQL " + e)
|
||||||
case e =>
|
case e =>
|
||||||
DoobieMeta.logger.error(s"SQL Failure: $e")
|
DoobieMeta.logger.error(s"SQL Failure: $e")
|
||||||
})
|
}
|
||||||
|
|
||||||
def jsonMeta[A](implicit d: Decoder[A], e: Encoder[A]): Meta[A] =
|
def jsonMeta[A](implicit d: Decoder[A], e: Encoder[A]): Meta[A] =
|
||||||
Meta[String].imap(str => str.parseJsonAs[A].fold(ex => throw ex, identity))(a =>
|
Meta[String].imap(str => str.parseJsonAs[A].fold(ex => throw ex, identity))(a =>
|
||||||
|
@ -22,7 +22,7 @@ object FlywayMigrate {
|
|||||||
logger.info("Running db migrations...")
|
logger.info("Running db migrations...")
|
||||||
val locations = jdbc.dbmsName match {
|
val locations = jdbc.dbmsName match {
|
||||||
case Some(dbtype) =>
|
case Some(dbtype) =>
|
||||||
List(s"classpath:db/migration/${dbtype}")
|
List(s"classpath:db/migration/$dbtype")
|
||||||
case None =>
|
case None =>
|
||||||
logger.warn(
|
logger.warn(
|
||||||
s"Cannot read database name from jdbc url: ${jdbc.url}. Go with H2"
|
s"Cannot read database name from jdbc url: ${jdbc.url}. Go with H2"
|
||||||
|
@ -22,13 +22,12 @@ sealed trait FromExpr {
|
|||||||
def leftJoin(sel: Select, alias: String, on: Condition): Joined =
|
def leftJoin(sel: Select, alias: String, on: Condition): Joined =
|
||||||
leftJoin(Relation.SubSelect(sel, alias), on)
|
leftJoin(Relation.SubSelect(sel, alias), on)
|
||||||
|
|
||||||
/** Prepends the given from expression to existing joins. It will
|
/** Prepends the given from expression to existing joins. It will replace the current
|
||||||
* replace the current [[FromExpr.From]] value.
|
* [[FromExpr.From]] value.
|
||||||
*
|
*
|
||||||
* If this is a [[FromExpr.From]], it is replaced by the given
|
* If this is a [[FromExpr.From]], it is replaced by the given expression. If this is a
|
||||||
* expression. If this is a [[FromExpr.Joined]] then the given
|
* [[FromExpr.Joined]] then the given expression replaces the current `From` and the
|
||||||
* expression replaces the current `From` and the joins are
|
* joins are prepended to the existing joins.
|
||||||
* prepended to the existing joins.
|
|
||||||
*/
|
*/
|
||||||
def prepend(fe: FromExpr): FromExpr
|
def prepend(fe: FromExpr): FromExpr
|
||||||
}
|
}
|
||||||
|
@ -46,9 +46,8 @@ object QAttachment {
|
|||||||
.foldMonoid
|
.foldMonoid
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Deletes an attachment, its related source and meta data records.
|
/** Deletes an attachment, its related source and meta data records. It will only delete
|
||||||
* It will only delete an related archive file, if this is the last
|
* an related archive file, if this is the last attachment in that archive.
|
||||||
* attachment in that archive.
|
|
||||||
*/
|
*/
|
||||||
def deleteSingleAttachment[F[_]: Sync](
|
def deleteSingleAttachment[F[_]: Sync](
|
||||||
store: Store[F]
|
store: Store[F]
|
||||||
@ -77,9 +76,9 @@ object QAttachment {
|
|||||||
} yield n + k + f
|
} yield n + k + f
|
||||||
}
|
}
|
||||||
|
|
||||||
/** This deletes the attachment and *all* its related files. This used
|
/** This deletes the attachment and *all* its related files. This used when deleting an
|
||||||
* when deleting an item and should not be used to delete a
|
* item and should not be used to delete a *single* attachment where the item should
|
||||||
* *single* attachment where the item should stay.
|
* stay.
|
||||||
*/
|
*/
|
||||||
private def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] =
|
private def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] =
|
||||||
for {
|
for {
|
||||||
|
@ -368,8 +368,8 @@ object QItem {
|
|||||||
from.query[ListItem].stream
|
from.query[ListItem].stream
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Same as `findItems` but resolves the tags for each item. Note that
|
/** Same as `findItems` but resolves the tags for each item. Note that this is
|
||||||
* this is implemented by running an additional query per item.
|
* implemented by running an additional query per item.
|
||||||
*/
|
*/
|
||||||
def findItemsWithTags(
|
def findItemsWithTags(
|
||||||
collective: Ident,
|
collective: Ident,
|
||||||
|
@ -43,14 +43,14 @@ object QJob {
|
|||||||
else ().pure[F]
|
else ().pure[F]
|
||||||
}
|
}
|
||||||
.find(_.isRight)
|
.find(_.isRight)
|
||||||
.flatMap({
|
.flatMap {
|
||||||
case Right(job) =>
|
case Right(job) =>
|
||||||
Stream.emit(job)
|
Stream.emit(job)
|
||||||
case Left(_) =>
|
case Left(_) =>
|
||||||
Stream
|
Stream
|
||||||
.eval(logger.fwarn[F]("Cannot mark job, even after retrying. Give up."))
|
.eval(logger.fwarn[F]("Cannot mark job, even after retrying. Give up."))
|
||||||
.map(_ => None)
|
.map(_ => None)
|
||||||
})
|
}
|
||||||
.compile
|
.compile
|
||||||
.last
|
.last
|
||||||
.map(_.flatten)
|
.map(_.flatten)
|
||||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user