TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-03-28 09:45:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

sbt scalafmtAll

Browse Source
This commit is contained in:
Eike Kettner 2020-02-25 20:55:00 +01:00
parent 4dbf75dd8f
commit 2f87065b2e
86 changed files with 582 additions and 525 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
modules/analysis/src/main/scala/docspell/analysis/contact/Contact.scala
View File

@ -12,13 +12,13 @@ object Contact {
def annotate(text: String): Vector[NerLabel] =
TextSplitter
.splitToken[Nothing](text, " \t\r\n".toSet)
.map({ token =>
.map { token =>
if (isEmailAddress(token.value))
NerLabel(token.value, NerTag.Email, token.begin, token.end).some
else if (isWebsite(token.value))
NerLabel(token.value, NerTag.Website, token.begin, token.end).some
else None
})
}
.flatMap(_.map(Stream.emit).getOrElse(Stream.empty))
.toVector

2
modules/backend/src/main/scala/docspell/backend/auth/AuthToken.scala
View File

@ -52,7 +52,7 @@ object AuthToken {
def user[F[_]: Sync](accountId: AccountId, key: ByteVector): F[AuthToken] =
for {
salt <- Common.genSaltString[F]
salt <- Common.genSaltString[F]
millis = Instant.now.toEpochMilli
cd = AuthToken(millis, accountId, salt, "")
sig = sign(cd, key)

2
modules/backend/src/main/scala/docspell/backend/auth/Login.scala
View File

@ -72,7 +72,7 @@ object Login {
data <- store.transact(QLogin.findUser(acc))
_ <- Sync[F].delay(logger.trace(s"Account lookup: $data"))
res <- if (data.exists(check(up.pass))) okResult
else Result.invalidAuth.pure[F]
else Result.invalidAuth.pure[F]
} yield res
case Left(_) =>
Result.invalidAuth.pure[F]

6
modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala
View File

@ -111,10 +111,10 @@ object OCollective {
): F[PassChangeResult] = {
val q = for {
optUser <- RUser.findByAccount(accountId)
check = optUser.map(_.password).map(p => PasswordCrypt.check(current, p))
check = optUser.map(_.password).map(p => PasswordCrypt.check(current, p))
n <- check
.filter(identity)
.traverse(_ => RUser.updatePassword(accountId, PasswordCrypt.crypt(newPass)))
.filter(identity)
.traverse(_ => RUser.updatePassword(accountId, PasswordCrypt.crypt(newPass)))
res = check match {
case Some(true) =>
if (n.getOrElse(0) > 0) PassChangeResult.success else PassChangeResult.updateFailed

38
modules/backend/src/main/scala/docspell/backend/ops/OItem.scala
View File

@ -11,7 +11,14 @@ import docspell.store.queries.{QAttachment, QItem}
import OItem.{AttachmentData, AttachmentSourceData, ItemData, ListItem, Query}
import bitpeace.{FileMeta, RangeDef}
import docspell.common.{Direction, Ident, ItemState, MetaProposalList, Timestamp}
import docspell.store.records.{RAttachment, RAttachmentMeta, RAttachmentSource, RItem, RSource, RTagItem}
import docspell.store.records.{
RAttachment,
RAttachmentMeta,
RAttachmentSource,
RItem,
RSource,
RTagItem
}
trait OItem[F[_]] {
@ -75,14 +82,17 @@ object OItem {
def fileId: Ident
}
case class AttachmentData[F[_]](ra: RAttachment, meta: FileMeta, data: Stream[F, Byte])
extends BinaryData[F] {
val name = ra.name
extends BinaryData[F] {
val name = ra.name
val fileId = ra.fileId
}
case class AttachmentSourceData[F[_]](rs: RAttachmentSource, meta: FileMeta, data: Stream[F, Byte])
extends BinaryData[F] {
val name = rs.name
case class AttachmentSourceData[F[_]](
rs: RAttachmentSource,
meta: FileMeta,
data: Stream[F, Byte]
) extends BinaryData[F] {
val name = rs.name
val fileId = rs.fileId
}
@ -131,18 +141,22 @@ object OItem {
private def makeBinaryData[A](fileId: Ident)(f: FileMeta => A): F[Option[A]] =
store.bitpeace
.get(fileId.id).unNoneTerminate.compile.last.map(
_.map(m => f(m))
)
.get(fileId.id)
.unNoneTerminate
.compile
.last
.map(
_.map(m => f(m))
)
def setTags(item: Ident, tagIds: List[Ident], collective: Ident): F[AddResult] = {
val db = for {
cid <- RItem.getCollective(item)
nd <- if (cid.contains(collective)) RTagItem.deleteItemTags(item)
else 0.pure[ConnectionIO]
else 0.pure[ConnectionIO]
ni <- if (tagIds.nonEmpty && cid.contains(collective))
RTagItem.insertItemTags(item, tagIds)
else 0.pure[ConnectionIO]
RTagItem.insertItemTags(item, tagIds)
else 0.pure[ConnectionIO]
} yield nd + ni
store.transact(db).attempt.map(AddResult.fromUpdate)

16
modules/backend/src/main/scala/docspell/backend/ops/OJob.scala
View File

@ -61,9 +61,9 @@ object OJob {
mustCancel(j.some).isEmpty
val tryDelete = for {
job <- RJob.findByIdAndGroup(id, collective)
job <- RJob.findByIdAndGroup(id, collective)
jobm = job.filter(canDelete)
del <- jobm.traverse(j => RJob.delete(j.id))
del <- jobm.traverse(j => RJob.delete(j.id))
} yield del match {
case Some(_) => Right(JobCancelResult.Removed: JobCancelResult)
case None => Left(mustCancel(job))
@ -77,12 +77,12 @@ object OJob {
for {
tryDel <- store.transact(tryDelete)
result <- tryDel match {
case Right(r) => r.pure[F]
case Left(Some((job, worker))) =>
tryCancel(job, worker)
case Left(None) =>
(JobCancelResult.JobNotFound: OJob.JobCancelResult).pure[F]
}
case Right(r) => r.pure[F]
case Left(Some((job, worker))) =>
tryCancel(job, worker)
case Left(None) =>
(JobCancelResult.JobNotFound: OJob.JobCancelResult).pure[F]
}
} yield result
}
})

32
modules/backend/src/main/scala/docspell/backend/ops/OMail.scala
View File

@ -113,10 +113,10 @@ object OMail {
def createSettings(accId: AccountId, s: SmtpSettings): F[AddResult] =
(for {
ru <- OptionT(store.transact(s.toRecord(accId).value))
ru <- OptionT(store.transact(s.toRecord(accId).value))
ins = RUserEmail.insert(ru)
exists = RUserEmail.exists(ru.uid, ru.name)
res <- OptionT.liftF(store.add(ins, exists))
res <- OptionT.liftF(store.add(ins, exists))
} yield res).getOrElse(AddResult.Failure(new Exception("User not found")))
def updateSettings(accId: AccountId, name: Ident, data: SmtpSettings): F[Int] = {
@ -143,10 +143,10 @@ object OMail {
for {
_ <- OptionT.liftF(store.transact(RItem.existsById(m.item))).filter(identity)
ras <- OptionT.liftF(
store.transact(
RAttachment.findByItemAndCollectiveWithMeta(m.item, accId.collective)
)
)
store.transact(
RAttachment.findByItemAndCollectiveWithMeta(m.item, accId.collective)
)
)
} yield {
val addAttach = m.attach.filter(ras).map { a =>
Attach[F](Stream.emit(a._2).through(store.bitpeace.fetchData2(RangeDef.all)))
@ -171,15 +171,15 @@ object OMail {
def storeMail(msgId: String, cfg: RUserEmail): F[Either[SendResult, Ident]] = {
val save = for {
data <- RSentMail.forItem(
m.item,
accId,
msgId,
cfg.mailFrom,
name,
m.subject,
m.recipients,
m.body
)
m.item,
accId,
msgId,
cfg.mailFrom,
name,
m.subject,
m.recipients,
m.body
)
_ <- OptionT.liftF(RSentMail.insert(data._1))
_ <- OptionT.liftF(RSentMailItem.insert(data._2))
} yield data._1.id
@ -197,7 +197,7 @@ object OMail {
mail <- createMail(mailCfg)
mid <- OptionT.liftF(sendMail(mailCfg.toMailConfig, mail))
res <- mid.traverse(id => OptionT.liftF(storeMail(id, mailCfg)))
conv = res.fold(identity, _.fold(identity, id => SendResult.Success(id)))
conv = res.fold(identity, _.fold(identity, id => SendResult.Success(id)))
} yield conv).getOrElse(SendResult.NotFound)
}

6
modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala
View File

@ -81,7 +81,7 @@ object OUpload {
def submit(data: OUpload.UploadData[F], sourceId: Ident): F[OUpload.UploadResult] =
for {
sOpt <- store.transact(RSource.find(sourceId)).map(_.toRight(UploadResult.NoSource))
sOpt <- store.transact(RSource.find(sourceId)).map(_.toRight(UploadResult.NoSource))
abbrev = sOpt.map(_.abbrev).toOption.getOrElse(data.meta.sourceAbbrev)
updata = data.copy(meta = data.meta.copy(sourceAbbrev = abbrev))
accId = sOpt.map(source => AccountId(source.cid, source.sid))
@ -131,8 +131,8 @@ object OUpload {
)
for {
id <- Ident.randomId[F]
now <- Timestamp.current[F]
id <- Ident.randomId[F]
now <- Timestamp.current[F]
jobs = args.map(a => create(id, now, a))
} yield jobs

17
modules/backend/src/main/scala/docspell/backend/signup/OSignup.scala
View File

@ -47,15 +47,16 @@ object OSignup {
for {
now <- Timestamp.current[F]
min = now.minus(cfg.inviteTime)
ok <- store.transact(RInvitation.useInvite(inv, min))
ok <- store.transact(RInvitation.useInvite(inv, min))
res <- if (ok) addUser(data).map(SignupResult.fromAddResult)
else SignupResult.invalidInvitationKey.pure[F]
else SignupResult.invalidInvitationKey.pure[F]
_ <- if (retryInvite(res))
logger.fdebug(s"Adding account failed ($res). Allow retry with invite.") *> store
.transact(
RInvitation.insert(RInvitation(inv, now))
)
else 0.pure[F]
logger
.fdebug(s"Adding account failed ($res). Allow retry with invite.") *> store
.transact(
RInvitation.insert(RInvitation(inv, now))
)
else 0.pure[F]
} yield res
case None =>
SignupResult.invalidInvitationKey.pure[F]
@ -81,7 +82,7 @@ object OSignup {
for {
id2 <- Ident.randomId[F]
now <- Timestamp.current[F]
c = RCollective(data.collName, CollectiveState.Active, Language.German, now)
c = RCollective(data.collName, CollectiveState.Active, Language.German, now)
u = RUser(
id2,
data.login,

4
modules/common/src/main/scala/docspell/common/AccountId.scala
View File

@ -26,9 +26,7 @@ object AccountId {
invalid
}
val separated = sepearatorChars.foldRight(invalid) { (c, v) =>
v.orElse(parse0(c))
}
val separated = sepearatorChars.foldRight(invalid)((c, v) => v.orElse(parse0(c)))
separated.orElse(Ident.fromString(str).map(id => AccountId(id, id)))
}

5
modules/common/src/main/scala/docspell/common/DataType.scala
View File

@ -1,8 +1,6 @@
package docspell.common
sealed trait DataType {
}
sealed trait DataType {}
object DataType {
@ -10,7 +8,6 @@ object DataType {
case class Hint(hint: MimeTypeHint) extends DataType
def apply(mt: MimeType): DataType =
Exact(mt)

10
modules/common/src/main/scala/docspell/common/File.scala
View File

@ -65,11 +65,13 @@ object File {
javaList.asScala.toList.sortBy(_.getFileName.toString)
}
def readAll[F[_]: Sync: ContextShift](file: Path, blocker: Blocker, chunkSize: Int): Stream[F, Byte] =
def readAll[F[_]: Sync: ContextShift](
file: Path,
blocker: Blocker,
chunkSize: Int
): Stream[F, Byte] =
fs2.io.file.readAll(file, blocker, chunkSize)
def readText[F[_]: Sync: ContextShift](file: Path, blocker: Blocker): F[String] =
readAll[F](file, blocker, 8192).
through(fs2.text.utf8Decode).
compile.foldMonoid
readAll[F](file, blocker, 8192).through(fs2.text.utf8Decode).compile.foldMonoid
}

4
modules/common/src/main/scala/docspell/common/LenientUri.scala
View File

@ -66,9 +66,7 @@ case class LenientUri(
)
def readText[F[_]: Sync: ContextShift](chunkSize: Int, blocker: Blocker): F[String] =
readURL[F](chunkSize, blocker).
through(fs2.text.utf8Decode).
compile.foldMonoid
readURL[F](chunkSize, blocker).through(fs2.text.utf8Decode).compile.foldMonoid
def host: Option[String] =
authority.map(a =>

3
modules/common/src/main/scala/docspell/common/Logger.scala
View File

@ -17,7 +17,6 @@ trait Logger[F[_]] {
object Logger {
def log4s[F[_]: Sync](log: Log4sLogger): Logger[F] = new Logger[F] {
def trace(msg: => String): F[Unit] =
log.ftrace(msg)
@ -38,4 +37,4 @@ object Logger {
log.ferror(msg)
}
}
}

4
modules/common/src/main/scala/docspell/common/MetaProposalList.scala
View File

@ -66,9 +66,7 @@ object MetaProposalList {
case None => map.updated(mp.proposalType, mp)
}
val merged = ml.foldLeft(init) { (map, el) =>
el.proposals.foldLeft(map)(updateMap)
}
val merged = ml.foldLeft(init)((map, el) => el.proposals.foldLeft(map)(updateMap))
fromMap(merged)
}

42
modules/common/src/main/scala/docspell/common/SystemCommand.scala
View File

@ -23,7 +23,8 @@ object SystemCommand {
repl.foldLeft(s) {
case (res, (k, v)) =>
res.replace(k, v)
})
}
)
def toCmd: List[String] =
program :: args.toList
@ -47,10 +48,10 @@ object SystemCommand {
_ <- writeToProcess(stdin, proc, blocker)
term <- Sync[F].delay(proc.waitFor(cmd.timeout.seconds, TimeUnit.SECONDS))
_ <- if (term) logger.debug(s"Command `${cmd.cmdString}` finished: ${proc.exitValue}")
else
logger.warn(
s"Command `${cmd.cmdString}` did not finish in ${cmd.timeout.formatExact}!"
)
else
logger.warn(
s"Command `${cmd.cmdString}` did not finish in ${cmd.timeout.formatExact}!"
)
_ <- if (!term) timeoutError(proc, cmd) else Sync[F].pure(())
out <- if (term) inputStreamToString(proc.getInputStream, blocker) else Sync[F].pure("")
err <- if (term) inputStreamToString(proc.getErrorStream, blocker) else Sync[F].pure("")
@ -75,25 +76,30 @@ object SystemCommand {
else Stream.emit(r)
}
private def startProcess[F[_]: Sync, A](cmd: Config, wd: Option[Path], logger: Logger[F], stdin: Stream[F, Byte])(
private def startProcess[F[_]: Sync, A](
cmd: Config,
wd: Option[Path],
logger: Logger[F],
stdin: Stream[F, Byte]
)(
f: Process => Stream[F, A]
): Stream[F, A] = {
val log = logger.debug(s"Running external command: ${cmd.cmdString}")
val log = logger.debug(s"Running external command: ${cmd.cmdString}")
val hasStdin = stdin.take(1).compile.last.map(_.isDefined)
val proc = log *> hasStdin.flatMap(flag => Sync[F].delay {
val pb = new ProcessBuilder(cmd.toCmd.asJava)
.redirectInput(if (flag) Redirect.PIPE else Redirect.INHERIT)
.redirectError(Redirect.PIPE)
.redirectOutput(Redirect.PIPE)
val proc = log *> hasStdin.flatMap(flag =>
Sync[F].delay {
val pb = new ProcessBuilder(cmd.toCmd.asJava)
.redirectInput(if (flag) Redirect.PIPE else Redirect.INHERIT)
.redirectError(Redirect.PIPE)
.redirectOutput(Redirect.PIPE)
wd.map(_.toFile).foreach(pb.directory)
pb.start()
})
wd.map(_.toFile).foreach(pb.directory)
pb.start()
}
)
Stream
.bracket(proc)(p =>
logger.debug(s"Closing process: `${cmd.cmdString}`").map { _ =>
p.destroy()
}
logger.debug(s"Closing process: `${cmd.cmdString}`").map(_ => p.destroy())
)
.flatMap(f)
}

4
modules/common/src/main/scala/docspell/common/syntax/StreamSyntax.scala
View File

@ -17,8 +17,8 @@ trait StreamSyntax {
.map(optStr =>
for {
str <- optStr
.map(_.trim)
.toRight(new Exception("Empty string cannot be parsed into a value"))
.map(_.trim)
.toRight(new Exception("Empty string cannot be parsed into a value"))
json <- parse(str).leftMap(_.underlying)
value <- json.as[A]
} yield value

16
modules/convert/src/main/scala/docspell/convert/Conversion.scala
View File

@ -13,7 +13,9 @@ import docspell.files.{ImageSize, TikaMimetype}
trait Conversion[F[_]] {
def toPDF[A](dataType: DataType, lang: Language, handler: Handler[F, A])(in: Stream[F, Byte]): F[A]
def toPDF[A](dataType: DataType, lang: Language, handler: Handler[F, A])(
in: Stream[F, Byte]
): F[A]
}
@ -26,7 +28,9 @@ object Conversion {
): Resource[F, Conversion[F]] =
Resource.pure(new Conversion[F] {
def toPDF[A](dataType: DataType, lang: Language, handler: Handler[F, A])(in: Stream[F, Byte]): F[A] =
def toPDF[A](dataType: DataType, lang: Language, handler: Handler[F, A])(
in: Stream[F, Byte]
): F[A] =
TikaMimetype.resolve(dataType, in).flatMap {
case MimeType.pdf =>
handler.run(ConversionResult.successPdf(in))
@ -112,10 +116,10 @@ object Conversion {
def unapply(mt: MimeType): Option[MimeType] =
mt match {
case Office(_) => Some(mt)
case Texts(_) => Some(mt)
case Images(_) => Some(mt)
case Office(_) => Some(mt)
case Texts(_) => Some(mt)
case Images(_) => Some(mt)
case MimeType.html => Some(mt)
case _ => None
case _ => None
}
}

14
modules/convert/src/main/scala/docspell/convert/ConvertConfig.scala
View File

@ -3,9 +3,11 @@ package docspell.convert
import docspell.convert.extern.{TesseractConfig, UnoconvConfig, WkHtmlPdfConfig}
import docspell.convert.flexmark.MarkdownConfig
case class ConvertConfig(chunkSize: Int,
maxImageSize: Int,
markdown: MarkdownConfig,
wkhtmlpdf: WkHtmlPdfConfig,
tesseract: TesseractConfig,
unoconv: UnoconvConfig)
case class ConvertConfig(
chunkSize: Int,
maxImageSize: Int,
markdown: MarkdownConfig,
wkhtmlpdf: WkHtmlPdfConfig,
tesseract: TesseractConfig,
unoconv: UnoconvConfig
)

32
modules/convert/src/main/scala/docspell/convert/extern/ExternConv.scala vendored
View File

@ -20,7 +20,9 @@ private[extern] object ExternConv {
logger: Logger[F],
reader: (Path, SystemCommand.Result) => F[ConversionResult[F]]
)(in: Stream[F, Byte], handler: Handler[F, A]): F[A] =
Stream.resource(File.withTempDir[F](wd, s"docspell-$name")).flatMap { dir =>
Stream
.resource(File.withTempDir[F](wd, s"docspell-$name"))
.flatMap { dir =>
val inFile = dir.resolve("infile").toAbsolutePath.normalize
val out = dir.resolve("out.pdf").toAbsolutePath.normalize
val sysCfg =
@ -40,12 +42,12 @@ private[extern] object ExternConv {
SystemCommand
.execSuccess[F](sysCfg, blocker, logger, Some(dir), if (useStdin) in else Stream.empty)
.evalMap(result =>
logResult(name, result, logger).
flatMap(_ => reader(out, result)).
flatMap(handler.run)
logResult(name, result, logger).flatMap(_ => reader(out, result)).flatMap(handler.run)
)
}
}.compile.lastOrError
}
.compile
.lastOrError
def readResult[F[_]: Sync: ContextShift](
blocker: Blocker,
@ -60,9 +62,11 @@ private[extern] object ExternConv {
successPdf(File.readAll(out, blocker, chunkSize)).pure[F]
case false =>
ConversionResult.failure[F](
new Exception(s"Command result=${result.rc}. No output file found.")
).pure[F]
ConversionResult
.failure[F](
new Exception(s"Command result=${result.rc}. No output file found.")
)
.pure[F]
}
def readResultTesseract[F[_]: Sync: ContextShift](
@ -75,7 +79,7 @@ private[extern] object ExternConv {
File.existsNonEmpty[F](outPdf).flatMap {
case true =>
val outTxt = out.resolveSibling(s"$outPrefix.txt")
File.exists(outTxt).flatMap(txtExists => {
File.exists(outTxt).flatMap { txtExists =>
val pdfData = File.readAll(out, blocker, chunkSize)
if (result.rc == 0) {
if (txtExists) successPdfTxt(pdfData, File.readText(outTxt, blocker)).pure[F]
@ -84,12 +88,14 @@ private[extern] object ExternConv {
logger.warn(s"Command not successful (rc=${result.rc}), but file exists.") *>
successPdf(pdfData).pure[F]
}
})
}
case false =>
ConversionResult.failure[F](
new Exception(s"Command result=${result.rc}. No output file found.")
).pure[F]
ConversionResult
.failure[F](
new Exception(s"Command result=${result.rc}. No output file found.")
)
.pure[F]
}
}

10
modules/convert/src/main/scala/docspell/convert/extern/Tesseract.scala vendored
View File

@ -21,7 +21,15 @@ object Tesseract {
val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
ExternConv.readResultTesseract[F](outBase, blocker, chunkSize, logger)
ExternConv.toPDF[F, A]("tesseract", cfg.command.replace(Map("{{lang}}" -> lang.iso3)), cfg.workingDir, false, blocker, logger, reader)(in, handler)
ExternConv.toPDF[F, A](
"tesseract",
cfg.command.replace(Map("{{lang}}" -> lang.iso3)),
cfg.workingDir,
false,
blocker,
logger,
reader
)(in, handler)
}
}

2
modules/convert/src/main/scala/docspell/convert/extern/TesseractConfig.scala vendored
View File

@ -4,4 +4,4 @@ import java.nio.file.Path
import docspell.common.SystemCommand
case class TesseractConfig (command: SystemCommand.Config, workingDir: Path)
case class TesseractConfig(command: SystemCommand.Config, workingDir: Path)

5
modules/convert/src/main/scala/docspell/convert/extern/Unoconv.scala vendored
View File

@ -19,7 +19,10 @@ object Unoconv {
val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
ExternConv.readResult[F](blocker, chunkSize, logger)
ExternConv.toPDF[F, A]("unoconv", cfg.command, cfg.workingDir, false, blocker, logger, reader)(in, handler)
ExternConv.toPDF[F, A]("unoconv", cfg.command, cfg.workingDir, false, blocker, logger, reader)(
in,
handler
)
}
}

2
modules/convert/src/main/scala/docspell/convert/extern/UnoconvConfig.scala vendored
View File

@ -4,4 +4,4 @@ import java.nio.file.Path
import docspell.common.SystemCommand
case class UnoconvConfig (command: SystemCommand.Config, workingDir: Path)
case class UnoconvConfig(command: SystemCommand.Config, workingDir: Path)

8
modules/convert/src/main/scala/docspell/convert/extern/WkHtmlPdf.scala vendored
View File

@ -14,12 +14,16 @@ object WkHtmlPdf {
cfg: WkHtmlPdfConfig,
chunkSize: Int,
blocker: Blocker,
logger: Logger[F],
logger: Logger[F]
)(in: Stream[F, Byte], handler: Handler[F, A]): F[A] = {
val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
ExternConv.readResult[F](blocker, chunkSize, logger)
ExternConv.toPDF[F, A]("wkhtmltopdf", cfg.command, cfg.workingDir, true, blocker, logger, reader)(in, handler)
ExternConv
.toPDF[F, A]("wkhtmltopdf", cfg.command, cfg.workingDir, true, blocker, logger, reader)(
in,
handler
)
}
}

2
modules/convert/src/main/scala/docspell/convert/extern/WkHtmlPdfConfig.scala vendored
View File

@ -4,4 +4,4 @@ import java.nio.file.Path
import docspell.common.SystemCommand
case class WkHtmlPdfConfig (command: SystemCommand.Config, workingDir: Path)
case class WkHtmlPdfConfig(command: SystemCommand.Config, workingDir: Path)

20
modules/convert/src/main/scala/docspell/convert/flexmark/Markdown.scala
View File

@ -22,24 +22,22 @@ object Markdown {
val r = createRenderer()
Try {
val reader = new InputStreamReader(is, StandardCharsets.UTF_8)
val doc = p.parseReader(reader)
val doc = p.parseReader(reader)
wrapHtml(r.render(doc), cfg)
}.toEither
}
def toHtml(md: String, cfg: MarkdownConfig): String = {
val p = createParser()
val r = createRenderer()
val p = createParser()
val r = createRenderer()
val doc = p.parse(md)
wrapHtml(r.render(doc), cfg)
}
def toHtml[F[_]: Sync](data: Stream[F, Byte], cfg: MarkdownConfig): F[String] =
data.through(fs2.text.utf8Decode).compile.foldMonoid.
map(str => toHtml(str, cfg))
data.through(fs2.text.utf8Decode).compile.foldMonoid.map(str => toHtml(str, cfg))
private def wrapHtml(body: String, cfg: MarkdownConfig): String = {
private def wrapHtml(body: String, cfg: MarkdownConfig): String =
s"""<!DOCTYPE html>
|<html>
|<head>
@ -53,13 +51,13 @@ object Markdown {
|</body>
|</html>
|""".stripMargin
}
private def createParser(): Parser = {
val opts = new MutableDataSet()
opts.set(Parser.EXTENSIONS.asInstanceOf[DataKey[util.Collection[_]]],
util.Arrays.asList(TablesExtension.create(),
StrikethroughExtension.create()));
opts.set(
Parser.EXTENSIONS.asInstanceOf[DataKey[util.Collection[_]]],
util.Arrays.asList(TablesExtension.create(), StrikethroughExtension.create())
);
Parser.builder(opts).build()
}

1
modules/convert/src/test/scala/docspell/convert/FileChecks.scala
View File

@ -55,5 +55,4 @@ trait FileChecks {
def commandExists(cmd: String): Boolean =
Runtime.getRuntime.exec(Array("which", cmd)).waitFor() == 0
}

1
modules/convert/src/test/scala/docspell/convert/extern/ExternConvTest.scala vendored
View File

@ -103,5 +103,4 @@ object ExternConvTest extends SimpleTestSuite with FileChecks {
}
}
}

31
modules/extract/src/main/scala/docspell/extract/Extraction.scala
View File

@ -29,7 +29,7 @@ object Extraction {
data: Stream[F, Byte],
dataType: DataType,
lang: Language
): F[ExtractResult] = {
): F[ExtractResult] =
TikaMimetype.resolve(dataType, data).flatMap {
case MimeType.pdf =>
PdfExtract
@ -50,39 +50,46 @@ object Extraction {
.extractOCR(data, blocker, logger, lang.iso3, cfg.ocr)
.compile
.lastOrError
.map(_.trim)
.attempt
.map(ExtractResult.fromEither)
ImageSize.get(data).flatMap {
case Some(dim) =>
if (dim.product > cfg.ocr.maxImageSize) {
logger.info(s"Image size (${dim.product}) is too large (max ${cfg.ocr.maxImageSize}).") *>
ExtractResult.failure(new Exception(
s"Image size (${dim.width}x${dim.height}) is too large (max ${cfg.ocr.maxImageSize}).")
).pure[F]
logger.info(
s"Image size (${dim.product}) is too large (max ${cfg.ocr.maxImageSize})."
) *>
ExtractResult
.failure(
new Exception(
s"Image size (${dim.width}x${dim.height}) is too large (max ${cfg.ocr.maxImageSize})."
)
)
.pure[F]
} else {
doExtract
}
case None =>
logger.info(s"Cannot read image data from ${mt.asString}. Extracting anyways.") *>
doExtract
doExtract
}
case OdfType.container =>
logger.info(s"File detected as ${OdfType.container}. Try to read as OpenDocument file.") *>
logger
.info(s"File detected as ${OdfType.container}. Try to read as OpenDocument file.") *>
OdfExtract.get(data).map(ExtractResult.fromEither)
case mt@MimeType("text", sub) if !sub.contains("html") =>
case mt @ MimeType("text", sub) if !sub.contains("html") =>
logger.info(s"File detected as ${mt.asString}. Returning itself as text.") *>
data.through(fs2.text.utf8Decode).compile.last.map { txt =>
ExtractResult.success(txt.getOrElse("").trim)
}
data.through(fs2.text.utf8Decode).compile.last.map { txt =>
ExtractResult.success(txt.getOrElse("").trim)
}
case mt =>
ExtractResult.unsupportedFormat(mt).pure[F]
}
}
}
}

2
modules/extract/src/main/scala/docspell/extract/PdfConfig.scala
View File

@ -1,3 +1,3 @@
package docspell.extract
case class PdfConfig (minTextLen: Int)
case class PdfConfig(minTextLen: Int)

3
modules/extract/src/main/scala/docspell/extract/PdfExtract.scala
View File

@ -33,7 +33,8 @@ object PdfExtract {
//maybe better: inspect the pdf and decide whether ocr or not
for {
pdfboxRes <- logger.debug("Trying to strip text from pdf using pdfbox.") *> PdfboxExtract.get[F](in)
pdfboxRes <- logger.debug("Trying to strip text from pdf using pdfbox.") *> PdfboxExtract
.get[F](in)
res <- pdfboxRes.fold(
ex =>
logger.info(

7
modules/extract/src/main/scala/docspell/extract/ocr/OcrConfig.scala
View File

@ -5,13 +5,12 @@ import java.nio.file.{Path, Paths}
import docspell.common._
case class OcrConfig(
maxImageSize: Int,
ghostscript: OcrConfig.Ghostscript,
maxImageSize: Int,
ghostscript: OcrConfig.Ghostscript,
pageRange: OcrConfig.PageRange,
unpaper: OcrConfig.Unpaper,
tesseract: OcrConfig.Tesseract
) {
}
) {}
object OcrConfig {

4
modules/extract/src/main/scala/docspell/extract/ocr/OcrType.scala
View File

@ -5,9 +5,9 @@ import docspell.common.MimeType
object OcrType {
val jpeg = MimeType.jpeg
val png = MimeType.png
val png = MimeType.png
val tiff = MimeType.tiff
val pdf = MimeType.pdf
val pdf = MimeType.pdf
val all = Set(jpeg, png, tiff, pdf)

18
modules/extract/src/main/scala/docspell/extract/odf/OdfExtract.scala
View File

@ -17,14 +17,14 @@ object OdfExtract {
def get[F[_]: Sync](data: Stream[F, Byte]): F[Either[Throwable, String]] =
data.compile.to(Array).map(new ByteArrayInputStream(_)).map(get)
def get(is: InputStream) = Try {
val handler = new BodyContentHandler()
val pctx = new ParseContext()
val meta = new Metadata()
val ooparser = new OpenDocumentParser()
ooparser.parse(is, handler, meta, pctx)
handler.toString.trim
}.toEither
def get(is: InputStream) =
Try {
val handler = new BodyContentHandler()
val pctx = new ParseContext()
val meta = new Metadata()
val ooparser = new OpenDocumentParser()
ooparser.parse(is, handler, meta, pctx)
handler.toString.trim
}.toEither
}

4
modules/extract/src/main/scala/docspell/extract/odf/OdfType.scala
View File

@ -4,8 +4,8 @@ import docspell.common.MimeType
object OdfType {
val odt = MimeType.application("vnd.oasis.opendocument.text")
val ods = MimeType.application("vnd.oasis.opendocument.spreadsheet")
val odt = MimeType.application("vnd.oasis.opendocument.text")
val ods = MimeType.application("vnd.oasis.opendocument.spreadsheet")
val odtAlias = MimeType.application("x-vnd.oasis.opendocument.text")
val odsAlias = MimeType.application("x-vnd.oasis.opendocument.spreadsheet")

4
modules/extract/src/main/scala/docspell/extract/pdfbox/PdfboxExtract.scala
View File

@ -14,9 +14,7 @@ import fs2.Stream
object PdfboxExtract {
def get[F[_]: Sync](data: Stream[F, Byte]): F[Either[Throwable, String]] =
data.compile.to(Array).map { bytes =>
Using(PDDocument.load(bytes))(readText).toEither.flatten
}
data.compile.to(Array).map(bytes => Using(PDDocument.load(bytes))(readText).toEither.flatten)
def get(is: InputStream): Either[Throwable, String] =
Using(PDDocument.load(is))(readText).toEither.flatten

8
modules/extract/src/main/scala/docspell/extract/poi/PoiExtract.scala
View File

@ -52,25 +52,25 @@ object PoiExtract {
def getDocx(is: InputStream): Either[Throwable, String] =
Try {
val xt = new XWPFWordExtractor(new XWPFDocument(is))
xt.getText.trim
Option(xt.getText).map(_.trim).getOrElse("")
}.toEither
def getDoc(is: InputStream): Either[Throwable, String] =
Try {
val xt = new WordExtractor(is)
xt.getText.trim
Option(xt.getText).map(_.trim).getOrElse("")
}.toEither
def getXlsx(is: InputStream): Either[Throwable, String] =
Try {
val xt = new XSSFExcelExtractor(new XSSFWorkbook(is))
xt.getText.trim
Option(xt.getText).map(_.trim).getOrElse("")
}.toEither
def getXls(is: InputStream): Either[Throwable, String] =
Try {
val xt = new ExcelExtractor(new HSSFWorkbook(is))
xt.getText.trim
Option(xt.getText).map(_.trim).getOrElse("")
}.toEither
def getDocx[F[_]: Sync](data: Stream[F, Byte]): F[Either[Throwable, String]] =

10
modules/extract/src/main/scala/docspell/extract/poi/PoiType.scala
View File

@ -5,11 +5,11 @@ import docspell.common.MimeType
object PoiType {
val msoffice = MimeType.application("x-tika-msoffice")
val ooxml = MimeType.application("x-tika-ooxml")
val docx = MimeType.application("vnd.openxmlformats-officedocument.wordprocessingml.document")
val xlsx = MimeType.application("vnd.openxmlformats-officedocument.spreadsheetml.sheet")
val xls = MimeType.application("vnd.ms-excel")
val doc = MimeType.application("msword")
val ooxml = MimeType.application("x-tika-ooxml")
val docx = MimeType.application("vnd.openxmlformats-officedocument.wordprocessingml.document")
val xlsx = MimeType.application("vnd.openxmlformats-officedocument.spreadsheetml.sheet")
val xls = MimeType.application("vnd.ms-excel")
val doc = MimeType.application("msword")
val all = Set(msoffice, ooxml, docx, xlsx, xls, doc)

17
modules/extract/src/test/scala/docspell/extract/odf/OdfExtractTest.scala
View File

@ -5,7 +5,7 @@ import docspell.files.{ExampleFiles, TestFiles}
import minitest.SimpleTestSuite
object OdfExtractTest extends SimpleTestSuite {
val blocker = TestFiles.blocker
val blocker = TestFiles.blocker
implicit val CS = TestFiles.CS
val files = List(
@ -14,14 +14,15 @@ object OdfExtractTest extends SimpleTestSuite {
)
test("test extract from odt") {
files.foreach { case (file, len) =>
val is = file.toJavaUrl.map(_.openStream()).fold(sys.error, identity)
val str1 = OdfExtract.get(is).fold(throw _, identity)
assertEquals(str1.length, len)
files.foreach {
case (file, len) =>
val is = file.toJavaUrl.map(_.openStream()).fold(sys.error, identity)
val str1 = OdfExtract.get(is).fold(throw _, identity)
assertEquals(str1.length, len)
val data = file.readURL[IO](8192, blocker)
val str2 = OdfExtract.get[IO](data).unsafeRunSync().fold(throw _, identity)
assertEquals(str2, str1)
val data = file.readURL[IO](8192, blocker)
val str2 = OdfExtract.get[IO](data).unsafeRunSync().fold(throw _, identity)
assertEquals(str2, str1)
}
}

4
modules/extract/src/test/scala/docspell/extract/rtf/RtfExtractTest.scala
View File

@ -7,8 +7,8 @@ object RtfExtractTest extends SimpleTestSuite {
test("extract text from rtf using java input-stream") {
val file = ExampleFiles.examples_sample_rtf
val is = file.toJavaUrl.map(_.openStream()).fold(sys.error, identity)
val str = RtfExtract.get(is).fold(throw _, identity)
val is = file.toJavaUrl.map(_.openStream()).fold(sys.error, identity)
val str = RtfExtract.get(is).fold(throw _, identity)
assertEquals(str.length, 7342)
}
}

7
modules/files/src/main/scala/docspell/files/ImageSize.scala
View File

@ -29,13 +29,12 @@ object ImageSize {
/** Return the image size from its header without reading
* the whole image into memory.
*/
def get[F[_]: Sync](data: Stream[F, Byte]): F[Option[Dimension]] = {
data.take(768).compile.to(Array).map(ar => {
def get[F[_]: Sync](data: Stream[F, Byte]): F[Option[Dimension]] =
data.take(768).compile.to(Array).map { ar =>
val iis = ImageIO.createImageInputStream(new ByteArrayInputStream(ar))
if (iis == null) sys.error("no reader given for the array")
else getDimension(iis)
})
}
}
private def getDimension(in: ImageInputStream): Option[Dimension] =
ImageIO

4
modules/files/src/main/scala/docspell/files/TikaMimetype.scala
View File

@ -52,8 +52,8 @@ object TikaMimetype {
def detect[F[_]: Sync](file: Path): F[MimeType] =
Sync[F].delay {
val hint = MimeTypeHint.filename(file.getFileName.toString)
Using(new BufferedInputStream(Files.newInputStream(file), 64))({ in =>
Using(new BufferedInputStream(Files.newInputStream(file), 64)) { in =>
convert(tika.detect(in, makeMetadata(hint)))
}).toEither
}.toEither
}.rethrow
}

3
modules/files/src/test/scala/docspell/files/ExampleFilesSupport.scala
View File

@ -7,8 +7,7 @@ trait ExampleFilesSupport {
def createUrl(resource: String): LenientUri =
Option(getClass.getResource("/" + resource)) match {
case Some(u) => LenientUri.fromJava(u)
case None => sys.error(s"Resource '$resource' not found")
case None => sys.error(s"Resource '$resource' not found")
}
}

5
modules/files/src/test/scala/docspell/files/Playing.scala
View File

@ -8,15 +8,14 @@ import scala.concurrent.ExecutionContext
object Playing extends IOApp {
val blocker = Blocker.liftExecutionContext(ExecutionContext.global)
def run(args: List[String]): IO[ExitCode] = IO {
//val ods = ExampleFiles.examples_sample_ods.readURL[IO](8192, blocker)
//val odt = ExampleFiles.examples_sample_odt.readURL[IO](8192, blocker)
val rtf = ExampleFiles.examples_sample_rtf.readURL[IO](8192, blocker)
val x = for {
odsm1 <- TikaMimetype.detect(rtf,
MimeTypeHint.filename(ExampleFiles.examples_sample_rtf.path.segments.last))
odsm1 <- TikaMimetype
.detect(rtf, MimeTypeHint.filename(ExampleFiles.examples_sample_rtf.path.segments.last))
odsm2 <- TikaMimetype.detect(rtf, MimeTypeHint.none)
} yield (odsm1, odsm2)
println(x.unsafeRunSync())

14
modules/joex/src/main/scala/docspell/joex/Config.scala
View File

@ -7,13 +7,13 @@ import docspell.convert.ConvertConfig
import docspell.extract.ExtractConfig
case class Config(
appId: Ident,
baseUrl: LenientUri,
bind: Config.Bind,
jdbc: JdbcConfig,
scheduler: SchedulerConfig,
extraction: ExtractConfig,
convert: ConvertConfig
appId: Ident,
baseUrl: LenientUri,
bind: Config.Bind,
jdbc: JdbcConfig,
scheduler: SchedulerConfig,
extraction: ExtractConfig,
convert: ConvertConfig
)
object Config {

18
modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala
View File

@ -52,15 +52,15 @@ object JoexAppImpl {
store <- Store.create(cfg.jdbc, connectEC, blocker)
nodeOps <- ONode(store)
sch <- SchedulerBuilder(cfg.scheduler, blocker, store)
.withTask(
JobTask.json(
ProcessItemArgs.taskName,
ItemHandler[F](cfg),
ItemHandler.onCancel[F]
)
)
.resource
app = new JoexAppImpl(cfg, nodeOps, store, termSignal, sch)
.withTask(
JobTask.json(
ProcessItemArgs.taskName,
ItemHandler[F](cfg),
ItemHandler.onCancel[F]
)
)
.resource
app = new JoexAppImpl(cfg, nodeOps, store, termSignal, sch)
appR <- Resource.make(app.init.map(_ => app))(_.shutdown)
} yield appR
}

20
modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala
View File

@ -68,7 +68,9 @@ object ConvertPdf {
.through(ctx.store.bitpeace.fetchData2(RangeDef.all))
val handler = conversionHandler[F](ctx, cfg, ra, item)
ctx.logger.info(s"Converting file ${ra.name} (${mime.asString}) into a PDF") *>
conv.toPDF(DataType(MimeType(mime.primary, mime.sub)), ctx.args.meta.language, handler)(data)
conv.toPDF(DataType(MimeType(mime.primary, mime.sub)), ctx.args.meta.language, handler)(
data
)
}
}
@ -107,19 +109,21 @@ object ConvertPdf {
})
private def storePDF[F[_]: Sync](
ctx: Context[F, ProcessItemArgs],
cfg: ConvertConfig,
ra: RAttachment,
pdf: Stream[F, Byte]
) = {
val hint = MimeTypeHint.advertised(MimeType.pdf).withName(ra.name.getOrElse("file.pdf"))
ctx: Context[F, ProcessItemArgs],
cfg: ConvertConfig,
ra: RAttachment,
pdf: Stream[F, Byte]
) = {
val hint = MimeTypeHint.advertised(MimeType.pdf).withName(ra.name.getOrElse("file.pdf"))
val newName = ra.name.map(n => s"$n.pdf")
ctx.store.bitpeace
.saveNew(pdf, cfg.chunkSize, MimetypeHint(hint.filename, hint.advertised))
.compile
.lastOrError
.map(fm => Ident.unsafe(fm.id))
.flatMap(fmId => ctx.store.transact(RAttachment.updateFileIdAndName(ra.id, fmId, newName)).map(_ => fmId))
.flatMap(fmId =>
ctx.store.transact(RAttachment.updateFileIdAndName(ra.id, fmId, newName)).map(_ => fmId)
)
.map(fmId => ra.copy(fileId = fmId, name = newName))
}
}

18
modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala
View File

@ -25,7 +25,7 @@ object CreateItem {
Task { ctx =>
def isValidFile(fm: FileMeta) =
ctx.args.meta.validFileTypes.isEmpty ||
ctx.args.meta.validFileTypes.map(_.asString).toSet.contains(fm.mimetype.baseType)
ctx.args.meta.validFileTypes.map(_.asString).toSet.contains(fm.mimetype.baseType)
def fileMetas(itemId: Ident, now: Timestamp) =
Stream
@ -77,18 +77,18 @@ object CreateItem {
for {
cand <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId)))
_ <- if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.")
else ().pure[F]
else ().pure[F]
ht <- cand.drop(1).traverse(ri => QItem.delete(ctx.store)(ri.id, ri.cid))
_ <- if (ht.sum > 0) ctx.logger.warn(s"Removed ${ht.sum} items with same attachments")
else ().pure[F]
else ().pure[F]
rms <- OptionT(
cand.headOption.traverse(ri =>
ctx.store.transact(RAttachment.findByItemAndCollective(ri.id, ri.cid))
)
).getOrElse(Vector.empty)
cand.headOption.traverse(ri =>
ctx.store.transact(RAttachment.findByItemAndCollective(ri.id, ri.cid))
)
).getOrElse(Vector.empty)
orig <- rms.traverse(a =>
ctx.store.transact(RAttachmentSource.findById(a.id)).map(s => (a, s))
)
ctx.store.transact(RAttachmentSource.findById(a.id)).map(s => (a, s))
)
origMap = orig
.map(originFileTuple)
.toMap

4
modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala
View File

@ -95,10 +95,10 @@ object FindProposal {
labels => self.find(labels).map(f)
def next(f: Finder[F])(implicit F: FlatMap[F], F3: Applicative[F]): Finder[F] =
flatMap({ ml0 =>
flatMap { ml0 =>
if (ml0.hasResultsAll) Finder.unit[F](ml0)
else f.map(ml1 => ml0.fillEmptyFrom(ml1))
})
}
def nextWhenEmpty(f: Finder[F], mt0: MetaProposalType, mts: MetaProposalType*)(
implicit F: FlatMap[F],

6
modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala
View File

@ -19,14 +19,12 @@ object ItemHandler {
.map(_ => ())
def itemStateTask[F[_]: Sync, A](state: ItemState)(data: ItemData): Task[F, A, ItemData] =
Task { ctx =>
ctx.store.transact(RItem.updateState(data.item.id, state)).map(_ => data)
}
Task(ctx => ctx.store.transact(RItem.updateState(data.item.id, state)).map(_ => data))
def isLastRetry[F[_]: Sync, A](ctx: Context[F, A]): F[Boolean] =
for {
current <- ctx.store.transact(RJob.getRetries(ctx.jobId))
last = ctx.config.retries == current.getOrElse(0)
last = ctx.config.retries == current.getOrElse(0)
} yield last
def safeProcess[F[_]: Sync: ContextShift](

4
modules/joex/src/main/scala/docspell/joex/process/TestTasks.scala
View File

@ -11,9 +11,7 @@ object TestTasks {
private[this] val logger = getLogger
def success[F[_]]: Task[F, ProcessItemArgs, Unit] =
Task { ctx =>
ctx.logger.info(s"Running task now: ${ctx.args}")
}
Task(ctx => ctx.logger.info(s"Running task now: ${ctx.args}"))
def failing[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] =
Task { ctx =>

4
modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala
View File

@ -20,8 +20,8 @@ object TextAnalysis {
t <- item.metas.toList.traverse(annotateAttachment[F](ctx.args.meta.language))
_ <- ctx.logger.debug(s"Storing tags: ${t.map(_._1.copy(content = None))}")
_ <- t.traverse(m =>
ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels))
)
ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels))
)
e <- s
_ <- ctx.logger.info(s"Text-Analysis finished in ${e.formatExact}")
v = t.toVector

54
modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala
View File

@ -12,8 +12,8 @@ import docspell.store.records.{RAttachment, RAttachmentMeta, RFileMeta}
object TextExtraction {
def apply[F[_]: Sync: ContextShift](
cfg: ExtractConfig,
item: ItemData
cfg: ExtractConfig,
item: ItemData
): Task[F, ProcessItemArgs, ItemData] =
Task { ctx =>
for {
@ -28,11 +28,11 @@ object TextExtraction {
}
def extractTextIfEmpty[F[_]: Sync: ContextShift](
ctx: Context[F, _],
cfg: ExtractConfig,
lang: Language,
item: ItemData
)(ra: RAttachment): F[RAttachmentMeta] = {
ctx: Context[F, _],
cfg: ExtractConfig,
lang: Language,
item: ItemData
)(ra: RAttachment): F[RAttachmentMeta] = {
val rm = item.findOrCreate(ra.id)
rm.content match {
case Some(_) =>
@ -50,14 +50,14 @@ object TextExtraction {
item: ItemData
)(ra: RAttachment): F[RAttachmentMeta] =
for {
_ <- ctx.logger.debug(s"Extracting text for attachment ${stripAttachmentName(ra)}")
dst <- Duration.stopTime[F]
txt <- extractTextFallback(ctx, cfg, ra, lang)(filesToExtract(item, ra))
_ <- ctx.logger.debug(s"Extracting text for attachment ${stripAttachmentName(ra)}")
dst <- Duration.stopTime[F]
txt <- extractTextFallback(ctx, cfg, ra, lang)(filesToExtract(item, ra))
meta = item.changeMeta(ra.id, rm => rm.setContentIfEmpty(txt.map(_.trim).filter(_.nonEmpty)))
est <- dst
est <- dst
_ <- ctx.logger.debug(
s"Extracting text for attachment ${stripAttachmentName(ra)} finished in ${est.formatExact}"
)
s"Extracting text for attachment ${stripAttachmentName(ra)} finished in ${est.formatExact}"
)
} yield meta
def extractText[F[_]: Sync: ContextShift](
@ -76,16 +76,15 @@ object TextExtraction {
.getOrElse(Mimetype.`application/octet-stream`)
findMime
.flatMap(mt =>
extr.extractText(data, DataType(MimeType(mt.primary, mt.sub)), lang))
.flatMap(mt => extr.extractText(data, DataType(MimeType(mt.primary, mt.sub)), lang))
}
private def extractTextFallback[F[_]: Sync: ContextShift](
ctx: Context[F, _],
cfg: ExtractConfig,
ra: RAttachment,
lang: Language,
)(fileIds: List[Ident]): F[Option[String]] = {
ctx: Context[F, _],
cfg: ExtractConfig,
ra: RAttachment,
lang: Language
)(fileIds: List[Ident]): F[Option[String]] =
fileIds match {
case Nil =>
ctx.logger.error(s"Cannot extract text").map(_ => None)
@ -99,15 +98,18 @@ object TextExtraction {
txt.some.pure[F]
case ExtractResult.UnsupportedFormat(mt) =>
ctx.logger.warn(s"Cannot extract text from file ${stripAttachmentName(ra)}: unsupported format ${mt.asString}. Try with converted file.").
flatMap(_ => extractTextFallback[F](ctx, cfg, ra, lang)(rest))
ctx.logger
.warn(
s"Cannot extract text from file ${stripAttachmentName(ra)}: unsupported format ${mt.asString}. Try with converted file."
)
.flatMap(_ => extractTextFallback[F](ctx, cfg, ra, lang)(rest))
case ExtractResult.Failure(ex) =>
ctx.logger.warn(s"Cannot extract text: ${ex.getMessage}. Try with converted file").
flatMap(_ => extractTextFallback[F](ctx, cfg, ra, lang)(rest))
ctx.logger
.warn(s"Cannot extract text: ${ex.getMessage}. Try with converted file")
.flatMap(_ => extractTextFallback[F](ctx, cfg, ra, lang)(rest))
})
}
}
/** Returns the fileIds to extract text from. First, the source file
* is tried. If that fails, the converted file is tried.
@ -115,7 +117,7 @@ object TextExtraction {
private def filesToExtract(item: ItemData, ra: RAttachment): List[Ident] =
item.originFile.get(ra.id) match {
case Some(sid) => List(sid, ra.fileId).distinct
case None => List(ra.fileId)
case None => List(ra.fileId)
}
private def stripAttachmentName(ra: RAttachment): String =

10
modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala
View File

@ -25,15 +25,15 @@ object JoexRoutes {
case GET -> Root / "running" =>
for {
jobs <- app.scheduler.getRunning
jj = jobs.map(mkJob)
jj = jobs.map(mkJob)
resp <- Ok(JobList(jj.toList))
} yield resp
case POST -> Root / "shutdownAndExit" =>
for {
_ <- ConcurrentEffect[F].start(
Timer[F].sleep(Duration.seconds(1).toScala) *> app.initShutdown
)
Timer[F].sleep(Duration.seconds(1).toScala) *> app.initShutdown
)
resp <- Ok(BasicResult(true, "Shutdown initiated."))
} yield resp
@ -41,8 +41,8 @@ object JoexRoutes {
for {
optJob <- app.scheduler.getRunning.map(_.find(_.id == id))
optLog <- optJob.traverse(j => app.findLogs(j.id))
jAndL = for { job <- optJob; log <- optLog } yield mkJobLog(job, log)
resp <- jAndL.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found")))
jAndL = for { job <- optJob; log <- optLog } yield mkJobLog(job, log)
resp <- jAndL.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found")))
} yield resp
case POST -> Root / "job" / Ident(id) / "cancel" =>

2
modules/joex/src/main/scala/docspell/joex/scheduler/Context.scala
View File

@ -54,7 +54,7 @@ object Context {
_ <- log.ftrace("Creating logger for task run")
logger <- QueueLogger(job.id, job.info, config.logBufferSize, logSink)
_ <- log.ftrace("Logger created, instantiating context")
ctx = create[F, A](job, arg, config, logger, store, blocker)
ctx = create[F, A](job, arg, config, logger, store, blocker)
} yield ctx
final private class ContextImpl[F[_]: Functor, A](

4
modules/joex/src/main/scala/docspell/joex/scheduler/QueueLogger.scala
View File

@ -38,9 +38,9 @@ object QueueLogger {
sink: LogSink[F]
): F[Logger[F]] =
for {
q <- Queue.circularBuffer[F, LogEvent](bufferSize)
q <- Queue.circularBuffer[F, LogEvent](bufferSize)
log = create(jobId, jobInfo, q)
_ <- Concurrent[F].start(q.dequeue.through(sink.receive).compile.drain)
_ <- Concurrent[F].start(q.dequeue.through(sink.receive).compile.drain)
} yield log
}

20
modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerImpl.scala
View File

@ -91,12 +91,12 @@ final class SchedulerImpl[F[_]: ConcurrentEffect: ContextShift](
_ <- logger.fdebug("New permit acquired")
down <- state.get.map(_.shutdownRequest)
rjob <- if (down) logger.finfo("") *> permits.release *> (None: Option[RJob]).pure[F]
else
queue.nextJob(
group => state.modify(_.nextPrio(group, config.countingScheme)),
config.name,
config.retryDelay
)
else
queue.nextJob(
group => state.modify(_.nextPrio(group, config.countingScheme)),
config.name,
config.retryDelay
)
_ <- logger.fdebug(s"Next job found: ${rjob.map(_.info)}")
_ <- rjob.map(execute).getOrElse(permits.release)
} yield rjob.isDefined
@ -122,8 +122,8 @@ final class SchedulerImpl[F[_]: ConcurrentEffect: ContextShift](
def execute(job: RJob): F[Unit] = {
val task = for {
jobtask <- tasks
.find(job.task)
.toRight(s"This executor cannot run tasks with name: ${job.task}")
.find(job.task)
.toRight(s"This executor cannot run tasks with name: ${job.task}")
} yield jobtask
task match {
@ -144,8 +144,8 @@ final class SchedulerImpl[F[_]: ConcurrentEffect: ContextShift](
for {
_ <- logger.fdebug(s"Job ${job.info} done $finalState. Releasing resources.")
_ <- permits.release *> permits.available.flatMap(a =>
logger.fdebug(s"Permit released ($a free)")
)
logger.fdebug(s"Permit released ($a free)")
)
_ <- state.modify(_.removeRunning(job))
_ <- QJob.setFinalState(job.id, finalState, store)
} yield ()

3
modules/microsite/docs/doc/nix.md
View File

@ -128,6 +128,9 @@ Please see the `nix/module-server.nix` and `nix/module-joex.nix` files
for the set of options. The nixos options are modelled after the
default configuration file.
The modules files are only applicable to the newest version of
Docspell. If you really need an older version, checkout the
appropriate commit.
## NixOs Example

4
modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala
View File

@ -27,8 +27,8 @@ object RestAppImpl {
): Resource[F, RestApp[F]] =
for {
backend <- BackendApp(cfg.backend, connectEC, httpClientEc, blocker)
app = new RestAppImpl[F](cfg, backend)
appR <- Resource.make(app.init.map(_ => app))(_.shutdown)
app = new RestAppImpl[F](cfg, backend)
appR <- Resource.make(app.init.map(_ => app))(_.shutdown)
} yield appR
}

4
modules/restserver/src/main/scala/docspell/restserver/auth/CookieData.scala
View File

@ -35,8 +35,8 @@ object CookieData {
for {
header <- headers.Cookie.from(req.headers).toRight("Cookie parsing error")
cookie <- header.values.toList
.find(_.name == cookieName)
.toRight("Couldn't find the authcookie")
.find(_.name == cookieName)
.toRight("Couldn't find the authcookie")
} yield cookie.content
def fromHeader[F[_]](req: Request[F]): Either[String, String] =

5
modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
View File

@ -84,7 +84,7 @@ trait Conversions {
data.inReplyTo.map(mkIdName),
data.item.dueDate,
data.item.notes,
data.attachments.map((mkAttachment(data)_).tupled).toList,
data.attachments.map((mkAttachment(data) _).tupled).toList,
data.sources.map((mkAttachmentSource _).tupled).toList,
data.tags.map(mkTag).toList
)
@ -204,7 +204,8 @@ trait Conversions {
val files = mp.parts
.filter(p => p.name.forall(s => !s.equalsIgnoreCase("meta")))
.map(p => OUpload.File(p.filename, p.headers.get(`Content-Type`).map(fromContentType), p.body)
.map(p =>
OUpload.File(p.filename, p.headers.get(`Content-Type`).map(fromContentType), p.body)
)
for {
metaData <- meta

34
modules/restserver/src/main/scala/docspell/restserver/routes/AttachmentRoutes.scala
View File

@ -45,21 +45,21 @@ object AttachmentRoutes {
for {
fileData <- backend.item.findAttachment(id, user.account.collective)
resp <- fileData
.map(data => withResponseHeaders(Ok())(data))
.getOrElse(NotFound(BasicResult(false, "Not found")))
.map(data => withResponseHeaders(Ok())(data))
.getOrElse(NotFound(BasicResult(false, "Not found")))
} yield resp
case req @ GET -> Root / Ident(id) =>
for {
fileData <- backend.item.findAttachment(id, user.account.collective)
inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
matches = matchETag(fileData.map(_.meta), inm)
inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
matches = matchETag(fileData.map(_.meta), inm)
resp <- fileData
.map({ data =>
if (matches) withResponseHeaders(NotModified())(data)
else makeByteResp(data)
})
.getOrElse(NotFound(BasicResult(false, "Not found")))
.map { data =>
if (matches) withResponseHeaders(NotModified())(data)
else makeByteResp(data)
}
.getOrElse(NotFound(BasicResult(false, "Not found")))
} yield resp
case HEAD -> Root / Ident(id) / "original" =>
@ -73,13 +73,13 @@ object AttachmentRoutes {
case req @ GET -> Root / Ident(id) / "original" =>
for {
fileData <- backend.item.findAttachmentSource(id, user.account.collective)
inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
matches = matchETag(fileData.map(_.meta), inm)
inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
matches = matchETag(fileData.map(_.meta), inm)
resp <- fileData
.map({ data =>
.map { data =>
if (matches) withResponseHeaders(NotModified())(data)
else makeByteResp(data)
})
}
.getOrElse(NotFound(BasicResult(false, "Not found")))
} yield resp
@ -92,16 +92,16 @@ object AttachmentRoutes {
case GET -> Root / Ident(id) / "meta" =>
for {
rm <- backend.item.findAttachmentMeta(id, user.account.collective)
md = rm.map(Conversions.mkAttachmentMeta)
rm <- backend.item.findAttachmentMeta(id, user.account.collective)
md = rm.map(Conversions.mkAttachmentMeta)
resp <- md.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found.")))
} yield resp
}
}
private def matchETag[F[_]](
fileData: Option[FileMeta],
noneMatch: Option[NonEmptyList[EntityTag]]
fileData: Option[FileMeta],
noneMatch: Option[NonEmptyList[EntityTag]]
): Boolean =
(fileData, noneMatch) match {
case (Some(meta), Some(nm)) =>

16
modules/restserver/src/main/scala/docspell/restserver/routes/CollectiveRoutes.scala
View File

@ -35,25 +35,25 @@ object CollectiveRoutes {
case GET -> Root / "settings" =>
for {
collDb <- backend.collective.find(user.account.collective)
sett = collDb.map(c => CollectiveSettings(c.language))
resp <- sett.toResponse()
sett = collDb.map(c => CollectiveSettings(c.language))
resp <- sett.toResponse()
} yield resp
case GET -> Root / "contacts" :? QueryParam.QueryOpt(q) +& QueryParam.ContactKindOpt(kind) =>
for {
res <- backend.collective
.getContacts(user.account.collective, q.map(_.q), kind)
.take(50)
.compile
.toList
.getContacts(user.account.collective, q.map(_.q), kind)
.take(50)
.compile
.toList
resp <- Ok(ContactList(res.map(Conversions.mkContact)))
} yield resp
case GET -> Root =>
for {
collDb <- backend.collective.find(user.account.collective)
coll = collDb.map(c => Collective(c.id, c.state, c.created))
resp <- coll.toResponse()
coll = collDb.map(c => Collective(c.id, c.state, c.created))
resp <- coll.toResponse()
} yield resp
}
}

6
modules/restserver/src/main/scala/docspell/restserver/routes/EquipmentRoutes.scala
View File

@ -37,10 +37,10 @@ object EquipmentRoutes {
case req @ PUT -> Root =>
for {
data <- req.as[Equipment]
data <- req.as[Equipment]
equip = changeEquipment(data, user.account.collective)
res <- backend.equipment.update(equip)
resp <- Ok(basicResult(res, "Equipment updated."))
res <- backend.equipment.update(equip)
resp <- Ok(basicResult(res, "Equipment updated."))
} yield resp
case DELETE -> Root / Ident(id) =>

16
modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala
View File

@ -24,8 +24,8 @@ object ItemRoutes {
HttpRoutes.of {
case req @ POST -> Root / "search" =>
for {
mask <- req.as[ItemSearch]
_ <- logger.ftrace(s"Got search mask: $mask")
mask <- req.as[ItemSearch]
_ <- logger.ftrace(s"Got search mask: $mask")
query = Conversions.mkQuery(mask, user.account.collective)
_ <- logger.ftrace(s"Running query: $query")
items <- backend.item.findItems(query, 100)
@ -34,9 +34,9 @@ object ItemRoutes {
case GET -> Root / Ident(id) =>
for {
item <- backend.item.findItem(id, user.account.collective)
item <- backend.item.findItem(id, user.account.collective)
result = item.map(Conversions.mkItemDetail)
resp <- result.map(r => Ok(r)).getOrElse(NotFound(BasicResult(false, "Not found.")))
resp <- result.map(r => Ok(r)).getOrElse(NotFound(BasicResult(false, "Not found.")))
} yield resp
case POST -> Root / Ident(id) / "confirm" =>
@ -125,15 +125,15 @@ object ItemRoutes {
case GET -> Root / Ident(id) / "proposals" =>
for {
ml <- backend.item.getProposals(id, user.account.collective)
ip = Conversions.mkItemProposals(ml)
ml <- backend.item.getProposals(id, user.account.collective)
ip = Conversions.mkItemProposals(ml)
resp <- Ok(ip)
} yield resp
case DELETE -> Root / Ident(id) =>
for {
n <- backend.item.delete(id, user.account.collective)
res = BasicResult(n > 0, if (n > 0) "Item deleted" else "Item deletion failed.")
n <- backend.item.delete(id, user.account.collective)
res = BasicResult(n > 0, if (n > 0) "Item deleted" else "Item deletion failed.")
resp <- Ok(res)
} yield resp
}

4
modules/restserver/src/main/scala/docspell/restserver/routes/JobQueueRoutes.scala
View File

@ -19,8 +19,8 @@ object JobQueueRoutes {
HttpRoutes.of {
case GET -> Root / "state" =>
for {
js <- backend.job.queueState(user.account.collective, 200)
res = Conversions.mkJobQueueState(js)
js <- backend.job.queueState(user.account.collective, 200)
res = Conversions.mkJobQueueState(js)
resp <- Ok(res)
} yield resp

18
modules/restserver/src/main/scala/docspell/restserver/routes/LoginRoutes.scala
View File

@ -54,15 +54,15 @@ object LoginRoutes {
for {
cd <- AuthToken.user(token.account, cfg.auth.serverSecret).map(CookieData.apply)
resp <- Ok(
AuthResult(
token.account.collective.id,
token.account.user.id,
true,
"Login successful",
Some(cd.asString),
cfg.auth.sessionValid.millis
)
).map(_.addCookie(cd.asCookie(cfg)))
AuthResult(
token.account.collective.id,
token.account.user.id,
true,
"Login successful",
Some(cd.asString),
cfg.auth.sessionValid.millis
)
).map(_.addCookie(cd.asCookie(cfg)))
} yield resp
case _ =>
Ok(AuthResult("", account, false, "Login failed.", None, 0L))

12
modules/restserver/src/main/scala/docspell/restserver/routes/MailSendRoutes.scala
View File

@ -24,13 +24,13 @@ object MailSendRoutes {
HttpRoutes.of {
case req @ POST -> Root / Ident(name) / Ident(id) =>
for {
in <- req.as[SimpleMail]
in <- req.as[SimpleMail]
mail = convertIn(id, in)
res <- mail.traverse(m => backend.mail.sendMail(user.account, name, m))
res <- mail.traverse(m => backend.mail.sendMail(user.account, name, m))
resp <- res.fold(
err => Ok(BasicResult(false, s"Invalid mail data: $err")),
res => Ok(convertOut(res))
)
err => Ok(BasicResult(false, s"Invalid mail data: $err")),
res => Ok(convertOut(res))
)
} yield resp
}
}
@ -39,7 +39,7 @@ object MailSendRoutes {
for {
rec <- s.recipients.traverse(EmilUtil.readMailAddress)
fileIds <- s.attachmentIds.traverse(Ident.fromString)
sel = if (s.addAllAttachments) AttachSelection.All else AttachSelection.Selected(fileIds)
sel = if (s.addAllAttachments) AttachSelection.All else AttachSelection.Selected(fileIds)
} yield ItemMail(item, s.subject, rec, s.body, sel)
def convertOut(res: SendResult): BasicResult =

40
modules/restserver/src/main/scala/docspell/restserver/routes/MailSettingsRoutes.scala
View File

@ -29,7 +29,7 @@ object MailSettingsRoutes {
case GET -> Root :? QueryParam.QueryOpt(q) =>
for {
list <- backend.mail.getSettings(user.account, q.map(_.q))
res = list.map(convert)
res = list.map(convert)
resp <- Ok(EmailSettingsList(res.toList))
} yield resp
@ -45,13 +45,13 @@ object MailSettingsRoutes {
ru = makeSettings(in)
up <- OptionT.liftF(ru.traverse(r => backend.mail.createSettings(user.account, r)))
resp <- OptionT.liftF(
Ok(
up.fold(
err => BasicResult(false, err),
ar => Conversions.basicResult(ar, "Mail settings stored.")
)
)
)
Ok(
up.fold(
err => BasicResult(false, err),
ar => Conversions.basicResult(ar, "Mail settings stored.")
)
)
)
} yield resp).getOrElseF(NotFound())
case req @ PUT -> Root / Ident(name) =>
@ -60,24 +60,24 @@ object MailSettingsRoutes {
ru = makeSettings(in)
up <- OptionT.liftF(ru.traverse(r => backend.mail.updateSettings(user.account, name, r)))
resp <- OptionT.liftF(
Ok(
up.fold(
err => BasicResult(false, err),
n =>
if (n > 0) BasicResult(true, "Mail settings stored.")
else BasicResult(false, "Mail settings could not be saved")
)
)
)
Ok(
up.fold(
err => BasicResult(false, err),
n =>
if (n > 0) BasicResult(true, "Mail settings stored.")
else BasicResult(false, "Mail settings could not be saved")
)
)
)
} yield resp).getOrElseF(NotFound())
case DELETE -> Root / Ident(name) =>
for {
n <- backend.mail.deleteSettings(user.account, name)
resp <- Ok(
if (n > 0) BasicResult(true, "Mail settings removed")
else BasicResult(false, "Mail settings could not be removed")
)
if (n > 0) BasicResult(true, "Mail settings removed")
else BasicResult(false, "Mail settings could not be removed")
)
} yield resp
}

4
modules/restserver/src/main/scala/docspell/restserver/routes/SourceRoutes.scala
View File

@ -36,8 +36,8 @@ object SourceRoutes {
case req @ PUT -> Root =>
for {
data <- req.as[Source]
src = changeSource(data, user.account.collective)
data <- req.as[Source]
src = changeSource(data, user.account.collective)
updated <- backend.source.update(src)
resp <- Ok(basicResult(updated, "Source updated."))
} yield resp

2
modules/restserver/src/main/scala/docspell/restserver/routes/TagRoutes.scala
View File

@ -37,7 +37,7 @@ object TagRoutes {
case req @ PUT -> Root =>
for {
data <- req.as[Tag]
tag = changeTag(data, user.account.collective)
tag = changeTag(data, user.account.collective)
res <- backend.tag.update(tag)
resp <- Ok(basicResult(res, "Tag successfully updated."))
} yield resp

10
modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala
View File

@ -28,11 +28,11 @@ object UploadRoutes {
for {
multipart <- req.as[Multipart[F]]
updata <- readMultipart(
multipart,
logger,
Priority.High,
cfg.backend.files.validMimeTypes
)
multipart,
logger,
Priority.High,
cfg.backend.files.validMimeTypes
)
result <- backend.upload.submit(updata, user.account)
res <- Ok(basicResult(result))
} yield res

12
modules/restserver/src/main/scala/docspell/restserver/routes/UserRoutes.scala
View File

@ -24,10 +24,10 @@ object UserRoutes {
for {
data <- req.as[PasswordChange]
res <- backend.collective.changePassword(
user.account,
data.currentPassword,
data.newPassword
)
user.account,
data.currentPassword,
data.newPassword
)
resp <- Ok(basicResult(res))
} yield resp
@ -47,8 +47,8 @@ object UserRoutes {
case req @ PUT -> Root =>
for {
data <- req.as[User]
nuser = changeUser(data, user.account.collective)
data <- req.as[User]
nuser = changeUser(data, user.account.collective)
update <- backend.collective.update(nuser)
resp <- Ok(basicResult(update, "User updated."))
} yield resp

2
modules/store/src/main/scala/docspell/store/Store.scala
View File

@ -40,7 +40,7 @@ object Store {
for {
xa <- hxa
st = new StoreImpl[F](jdbc, xa)
_ <- Resource.liftF(st.migrate)
_ <- Resource.liftF(st.migrate)
} yield st
}
}

33
modules/store/src/main/scala/docspell/store/queries/QAttachment.scala
View File

@ -14,25 +14,32 @@ object QAttachment {
def deleteById[F[_]: Sync](store: Store[F])(attachId: Ident, coll: Ident): F[Int] =
for {
raFile <- store.transact(RAttachment.findByIdAndCollective(attachId, coll)).map(_.map(_.fileId))
rsFile <- store.transact(RAttachmentSource.findByIdAndCollective(attachId, coll)).map(_.map(_.fileId))
n <- store.transact(RAttachment.delete(attachId))
f <- Stream.emits(raFile.toSeq ++ rsFile.toSeq)
.map(_.id)
.flatMap(store.bitpeace.delete)
.map(flag => if (flag) 1 else 0)
.compile
.foldMonoid
raFile <- store
.transact(RAttachment.findByIdAndCollective(attachId, coll))
.map(_.map(_.fileId))
rsFile <- store
.transact(RAttachmentSource.findByIdAndCollective(attachId, coll))
.map(_.map(_.fileId))
n <- store.transact(RAttachment.delete(attachId))
f <- Stream
.emits(raFile.toSeq ++ rsFile.toSeq)
.map(_.id)
.flatMap(store.bitpeace.delete)
.map(flag => if (flag) 1 else 0)
.compile
.foldMonoid
} yield n + f
def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] =
for {
s <- store.transact(RAttachmentSource.findById(ra.id))
n <- store.transact(RAttachment.delete(ra.id))
f <- Stream.emits(ra.fileId.id +: s.map(_.fileId.id).toSeq).
flatMap(store.bitpeace.delete).
map(flag => if (flag) 1 else 0).
compile.foldMonoid
f <- Stream
.emits(ra.fileId.id +: s.map(_.fileId.id).toSeq)
.flatMap(store.bitpeace.delete)
.map(flag => if (flag) 1 else 0)
.compile
.foldMonoid
} yield n + f
def deleteItemAttachments[F[_]: Sync](store: Store[F])(itemId: Ident, coll: Ident): F[Int] =

2
modules/store/src/main/scala/docspell/store/queries/QCollective.scala
View File

@ -27,7 +27,6 @@ object QCollective {
and(IC.cid.is(coll), IC.incoming.is(Direction.outgoing))
).query[Int].unique
val fileSize = sql"""
select sum(length) from (
with attachs as
@ -42,7 +41,6 @@ object QCollective {
inner join filemeta m on m.id = a.file_id where a.id in (select aid from attachs)
) as t""".query[Option[Long]].unique
val q3 = fr"SELECT" ++ commas(
TC.name.prefix("t").f,
fr"count(" ++ RC.itemId.prefix("r").f ++ fr")"

48
modules/store/src/main/scala/docspell/store/queries/QItem.scala
View File

@ -23,7 +23,7 @@ object QItem {
concEquip: Option[REquipment],
inReplyTo: Option[IdRef],
tags: Vector[RTag],
attachments: Vector[(RAttachment, FileMeta)],
attachments: Vector[(RAttachment, FileMeta)],
sources: Vector[(RAttachmentSource, FileMeta)]
) {
@ -39,23 +39,24 @@ object QItem {
val EC = REquipment.Columns.all.map(_.prefix("e"))
val ICC = List(RItem.Columns.id, RItem.Columns.name).map(_.prefix("ref"))
val cq = selectSimple(IC ++ OC ++ P0C ++ P1C ++ EC ++ ICC, RItem.table ++ fr"i", Fragment.empty) ++
fr"LEFT JOIN" ++ ROrganization.table ++ fr"o ON" ++ RItem.Columns.corrOrg
.prefix("i")
.is(ROrganization.Columns.oid.prefix("o")) ++
fr"LEFT JOIN" ++ RPerson.table ++ fr"p0 ON" ++ RItem.Columns.corrPerson
.prefix("i")
.is(RPerson.Columns.pid.prefix("p0")) ++
fr"LEFT JOIN" ++ RPerson.table ++ fr"p1 ON" ++ RItem.Columns.concPerson
.prefix("i")
.is(RPerson.Columns.pid.prefix("p1")) ++
fr"LEFT JOIN" ++ REquipment.table ++ fr"e ON" ++ RItem.Columns.concEquipment
.prefix("i")
.is(REquipment.Columns.eid.prefix("e")) ++
fr"LEFT JOIN" ++ RItem.table ++ fr"ref ON" ++ RItem.Columns.inReplyTo
.prefix("i")
.is(RItem.Columns.id.prefix("ref")) ++
fr"WHERE" ++ RItem.Columns.id.prefix("i").is(id)
val cq =
selectSimple(IC ++ OC ++ P0C ++ P1C ++ EC ++ ICC, RItem.table ++ fr"i", Fragment.empty) ++
fr"LEFT JOIN" ++ ROrganization.table ++ fr"o ON" ++ RItem.Columns.corrOrg
.prefix("i")
.is(ROrganization.Columns.oid.prefix("o")) ++
fr"LEFT JOIN" ++ RPerson.table ++ fr"p0 ON" ++ RItem.Columns.corrPerson
.prefix("i")
.is(RPerson.Columns.pid.prefix("p0")) ++
fr"LEFT JOIN" ++ RPerson.table ++ fr"p1 ON" ++ RItem.Columns.concPerson
.prefix("i")
.is(RPerson.Columns.pid.prefix("p1")) ++
fr"LEFT JOIN" ++ REquipment.table ++ fr"e ON" ++ RItem.Columns.concEquipment
.prefix("i")
.is(REquipment.Columns.eid.prefix("e")) ++
fr"LEFT JOIN" ++ RItem.table ++ fr"ref ON" ++ RItem.Columns.inReplyTo
.prefix("i")
.is(RItem.Columns.id.prefix("ref")) ++
fr"WHERE" ++ RItem.Columns.id.prefix("i").is(id)
val q = cq
.query[
@ -235,11 +236,12 @@ object QItem {
def findByFileIds(fileMetaIds: List[Ident]): ConnectionIO[Vector[RItem]] = {
val IC = RItem.Columns
val AC = RAttachment.Columns
val q = fr"SELECT DISTINCT" ++ commas(IC.all.map(_.prefix("i").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId
.prefix("a")
.is(IC.id.prefix("i")) ++
fr"WHERE" ++ AC.fileId.isOneOf(fileMetaIds) ++ orderBy(IC.created.prefix("i").asc)
val q =
fr"SELECT DISTINCT" ++ commas(IC.all.map(_.prefix("i").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId
.prefix("a")
.is(IC.id.prefix("i")) ++
fr"WHERE" ++ AC.fileId.isOneOf(fileMetaIds) ++ orderBy(IC.created.prefix("i").asc)
q.query[RItem].to[Vector]
}

13
modules/store/src/main/scala/docspell/store/queries/QJob.scala
View File

@ -21,11 +21,11 @@ object QJob {
Stream
.range(0, 10)
.evalMap(n => takeNextJob1(store)(priority, worker, retryPause, n))
.evalTap({ x =>
.evalTap { x =>
if (x.isLeft)
logger.fdebug[F]("Cannot mark job, probably due to concurrent updates. Will retry.")
else ().pure[F]
})
}
.find(_.isRight)
.flatMap({
case Right(job) =>
@ -50,7 +50,7 @@ object QJob {
store.transact(for {
n <- RJob.setScheduled(job.id, worker)
_ <- if (n == 1) RJobGroupUse.setGroup(RJobGroupUse(worker, job.group))
else 0.pure[ConnectionIO]
else 0.pure[ConnectionIO]
} yield if (n == 1) Right(job) else Left(()))
for {
@ -61,8 +61,8 @@ object QJob {
prio <- group.map(priority).getOrElse((Priority.Low: Priority).pure[F])
_ <- logger.ftrace[F](s"Looking for job of prio $prio")
job <- group
.map(g => store.transact(selectNextJob(g, prio, retryPause, now)))
.getOrElse((None: Option[RJob]).pure[F])
.map(g => store.transact(selectNextJob(g, prio, retryPause, now)))
.getOrElse((None: Option[RJob]).pure[F])
_ <- logger.ftrace[F](s"Found job: ${job.map(_.info)}")
res <- job.traverse(j => markJob(j))
} yield res.map(_.map(_.some)).getOrElse {
@ -97,7 +97,8 @@ object QJob {
val sql2 = fr"SELECT min(" ++ jgroup.f ++ fr") as g FROM" ++ RJob.table ++ fr"a" ++
fr"WHERE" ++ stateCond
val union = sql"SELECT g FROM ((" ++ sql1 ++ sql") UNION ALL (" ++ sql2 ++ sql")) as t0 WHERE g is not null"
val union =
sql"SELECT g FROM ((" ++ sql1 ++ sql") UNION ALL (" ++ sql2 ++ sql")) as t0 WHERE g is not null"
union
.query[Ident]

4
modules/store/src/main/scala/docspell/store/queue/JobQueue.scala
View File

@ -34,11 +34,11 @@ object JobQueue {
def insert(job: RJob): F[Unit] =
store
.transact(RJob.insert(job))
.flatMap({ n =>
.flatMap { n =>
if (n != 1)
Effect[F].raiseError(new Exception(s"Inserting job failed. Update count: $n"))
else ().pure[F]
})
}
def insertAll(jobs: Seq[RJob]): F[Unit] =
jobs.toList

9
modules/store/src/main/scala/docspell/store/records/RAttachment.scala
View File

@ -47,10 +47,10 @@ object RAttachment {
def findMeta(attachId: Ident): ConnectionIO[Option[FileMeta]] = {
import bitpeace.sql._
val cols = RFileMeta.Columns.all.map(_.prefix("m"))
val aId = id.prefix("a")
val cols = RFileMeta.Columns.all.map(_.prefix("m"))
val aId = id.prefix("a")
val aFileMeta = fileId.prefix("a")
val mId = RFileMeta.Columns.id.prefix("m")
val mId = RFileMeta.Columns.id.prefix("m")
val from = table ++ fr"a INNER JOIN" ++ RFileMeta.table ++ fr"m ON" ++ aFileMeta.is(mId)
val cond = aId.is(attachId)
@ -104,7 +104,8 @@ object RAttachment {
def findByItemWithMeta(id: Ident): ConnectionIO[Vector[(RAttachment, FileMeta)]] = {
import bitpeace.sql._
val q = fr"SELECT a.*,m.* FROM" ++ table ++ fr"a, filemeta m WHERE a.filemetaid = m.id AND a.itemid = $id ORDER BY a.position ASC"
val q =
fr"SELECT a.*,m.* FROM" ++ table ++ fr"a, filemeta m WHERE a.filemetaid = m.id AND a.itemid = $id ORDER BY a.position ASC"
q.query[(RAttachment, FileMeta)].to[Vector]
}

25
modules/store/src/main/scala/docspell/store/records/RAttachmentSource.scala
View File

@ -38,18 +38,20 @@ object RAttachmentSource {
def insert(v: RAttachmentSource): ConnectionIO[Int] =
insertRow(table, all, fr"${v.id},${v.fileId},${v.name},${v.created}").update.run
def findById(attachId: Ident): ConnectionIO[Option[RAttachmentSource]] =
selectSimple(all, table, id.is(attachId)).query[RAttachmentSource].option
def delete(attachId: Ident): ConnectionIO[Int] =
deleteFrom(table, id.is(attachId)).update.run
def findByIdAndCollective(attachId: Ident, collective: Ident): ConnectionIO[Option[RAttachmentSource]] = {
val bId = RAttachment.Columns.id.prefix("b")
val aId = Columns.id.prefix("a")
def findByIdAndCollective(
attachId: Ident,
collective: Ident
): ConnectionIO[Option[RAttachmentSource]] = {
val bId = RAttachment.Columns.id.prefix("b")
val aId = Columns.id.prefix("a")
val bItem = RAttachment.Columns.itemId.prefix("b")
val iId = RItem.Columns.id.prefix("i")
val iId = RItem.Columns.id.prefix("i")
val iColl = RItem.Columns.cid.prefix("i")
val from = table ++ fr"a INNER JOIN" ++
@ -64,11 +66,11 @@ object RAttachmentSource {
def findByItemWithMeta(id: Ident): ConnectionIO[Vector[(RAttachmentSource, FileMeta)]] = {
import bitpeace.sql._
val aId = Columns.id.prefix("a")
val aId = Columns.id.prefix("a")
val afileMeta = fileId.prefix("a")
val bPos = RAttachment.Columns.position.prefix("b")
val bId = RAttachment.Columns.id.prefix("b")
val bItem = RAttachment.Columns.itemId.prefix("b")
val bPos = RAttachment.Columns.position.prefix("b")
val bId = RAttachment.Columns.id.prefix("b")
val bItem = RAttachment.Columns.itemId.prefix("b")
val mId = RFileMeta.Columns.id.prefix("m")
val cols = all.map(_.prefix("a")) ++ RFileMeta.Columns.all.map(_.prefix("m"))
@ -77,8 +79,9 @@ object RAttachmentSource {
RAttachment.table ++ fr"b ON" ++ aId.is(bId)
val where = bItem.is(id)
(selectSimple(cols, from, where) ++ orderBy(bPos.asc)).
query[(RAttachmentSource, FileMeta)].to[Vector]
(selectSimple(cols, from, where) ++ orderBy(bPos.asc))
.query[(RAttachmentSource, FileMeta)]
.to[Vector]
}
}

112
modules/store/src/main/scala/docspell/store/records/RItem.scala
View File

@ -124,140 +124,140 @@ object RItem {
for {
t <- currentTime
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(state.setTo(itemState), updated.setTo(t))
).update.run
table,
and(id.is(itemId), cid.is(coll)),
commas(state.setTo(itemState), updated.setTo(t))
).update.run
} yield n
def updateDirection(itemId: Ident, coll: Ident, dir: Direction): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(incoming.setTo(dir), updated.setTo(t))
).update.run
table,
and(id.is(itemId), cid.is(coll)),
commas(incoming.setTo(dir), updated.setTo(t))
).update.run
} yield n
def updateCorrOrg(itemId: Ident, coll: Ident, org: Option[Ident]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(corrOrg.setTo(org), updated.setTo(t))
).update.run
table,
and(id.is(itemId), cid.is(coll)),
commas(corrOrg.setTo(org), updated.setTo(t))
).update.run
} yield n
def removeCorrOrg(coll: Ident, currentOrg: Ident): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(
table,
and(cid.is(coll), corrOrg.is(Some(currentOrg))),
commas(corrOrg.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
table,
and(cid.is(coll), corrOrg.is(Some(currentOrg))),
commas(corrOrg.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
} yield n
def updateCorrPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(corrPerson.setTo(person), updated.setTo(t))
).update.run
table,
and(id.is(itemId), cid.is(coll)),
commas(corrPerson.setTo(person), updated.setTo(t))
).update.run
} yield n
def removeCorrPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(
table,
and(cid.is(coll), corrPerson.is(Some(currentPerson))),
commas(corrPerson.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
table,
and(cid.is(coll), corrPerson.is(Some(currentPerson))),
commas(corrPerson.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
} yield n
def updateConcPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(concPerson.setTo(person), updated.setTo(t))
).update.run
table,
and(id.is(itemId), cid.is(coll)),
commas(concPerson.setTo(person), updated.setTo(t))
).update.run
} yield n
def removeConcPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(
table,
and(cid.is(coll), concPerson.is(Some(currentPerson))),
commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
table,
and(cid.is(coll), concPerson.is(Some(currentPerson))),
commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
} yield n
def updateConcEquip(itemId: Ident, coll: Ident, equip: Option[Ident]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(concEquipment.setTo(equip), updated.setTo(t))
).update.run
table,
and(id.is(itemId), cid.is(coll)),
commas(concEquipment.setTo(equip), updated.setTo(t))
).update.run
} yield n
def removeConcEquip(coll: Ident, currentEquip: Ident): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(
table,
and(cid.is(coll), concEquipment.is(Some(currentEquip))),
commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
table,
and(cid.is(coll), concEquipment.is(Some(currentEquip))),
commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t))
).update.run
} yield n
def updateNotes(itemId: Ident, coll: Ident, text: Option[String]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(notes.setTo(text), updated.setTo(t))
).update.run
table,
and(id.is(itemId), cid.is(coll)),
commas(notes.setTo(text), updated.setTo(t))
).update.run
} yield n
def updateName(itemId: Ident, coll: Ident, itemName: String): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(name.setTo(itemName), updated.setTo(t))
).update.run
table,
and(id.is(itemId), cid.is(coll)),
commas(name.setTo(itemName), updated.setTo(t))
).update.run
} yield n
def updateDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(itemDate.setTo(date), updated.setTo(t))
).update.run
table,
and(id.is(itemId), cid.is(coll)),
commas(itemDate.setTo(date), updated.setTo(t))
).update.run
} yield n
def updateDueDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] =
for {
t <- currentTime
n <- updateRow(
table,
and(id.is(itemId), cid.is(coll)),
commas(dueDate.setTo(date), updated.setTo(t))
).update.run
table,
and(id.is(itemId), cid.is(coll)),
commas(dueDate.setTo(date), updated.setTo(t))
).update.run
} yield n
def deleteByIdAndCollective(itemId: Ident, coll: Ident): ConnectionIO[Int] =

22
modules/store/src/main/scala/docspell/store/records/RJob.scala
View File

@ -162,17 +162,17 @@ object RJob {
for {
_ <- incrementRetries(jobId)
n <- updateRow(
table,
and(
id.is(jobId),
or(worker.isNull, worker.is(workerId)),
state.isOneOf(Seq[JobState](JobState.Waiting, JobState.Stuck))
),
commas(
state.setTo(JobState.Scheduled: JobState),
worker.setTo(workerId)
)
).update.run
table,
and(
id.is(jobId),
or(worker.isNull, worker.is(workerId)),
state.isOneOf(Seq[JobState](JobState.Waiting, JobState.Stuck))
),
commas(
state.setTo(JobState.Scheduled: JobState),
worker.setTo(workerId)
)
).update.run
} yield n
def setSuccess(jobId: Ident, now: Timestamp): ConnectionIO[Int] =

20
modules/store/src/main/scala/docspell/store/records/RSentMail.scala
View File

@ -52,16 +52,16 @@ object RSentMail {
for {
user <- OptionT(RUser.findByAccount(accId))
sm <- OptionT.liftF(
RSentMail[ConnectionIO](
user.uid,
messageId,
sender,
connName,
subject,
recipients,
body
)
)
RSentMail[ConnectionIO](
user.uid,
messageId,
sender,
connName,
subject,
recipients,
body
)
)
si <- OptionT.liftF(RSentMailItem[ConnectionIO](itemId, sm.id, Some(sm.created)))
} yield (sm, si)

6
modules/store/src/main/scala/docspell/store/records/RTagItem.scala
View File

@ -33,9 +33,9 @@ object RTagItem {
def insertItemTags(item: Ident, tags: Seq[Ident]): ConnectionIO[Int] =
for {
tagValues <- tags.toList.traverse(id =>
Ident.randomId[ConnectionIO].map(rid => RTagItem(rid, item, id))
)
Ident.randomId[ConnectionIO].map(rid => RTagItem(rid, item, id))
)
tagFrag = tagValues.map(v => fr"${v.tagItemId},${v.itemId},${v.tagId}")
ins <- insertRows(table, all, tagFrag).update.run
ins <- insertRows(table, all, tagFrag).update.run
} yield ins
}