sbt scalafmtAll

This commit is contained in:
Eike Kettner 2020-02-25 20:55:00 +01:00
parent 4dbf75dd8f
commit 2f87065b2e
86 changed files with 582 additions and 525 deletions

View File

@ -12,13 +12,13 @@ object Contact {
def annotate(text: String): Vector[NerLabel] = def annotate(text: String): Vector[NerLabel] =
TextSplitter TextSplitter
.splitToken[Nothing](text, " \t\r\n".toSet) .splitToken[Nothing](text, " \t\r\n".toSet)
.map({ token => .map { token =>
if (isEmailAddress(token.value)) if (isEmailAddress(token.value))
NerLabel(token.value, NerTag.Email, token.begin, token.end).some NerLabel(token.value, NerTag.Email, token.begin, token.end).some
else if (isWebsite(token.value)) else if (isWebsite(token.value))
NerLabel(token.value, NerTag.Website, token.begin, token.end).some NerLabel(token.value, NerTag.Website, token.begin, token.end).some
else None else None
}) }
.flatMap(_.map(Stream.emit).getOrElse(Stream.empty)) .flatMap(_.map(Stream.emit).getOrElse(Stream.empty))
.toVector .toVector

View File

@ -52,7 +52,7 @@ object AuthToken {
def user[F[_]: Sync](accountId: AccountId, key: ByteVector): F[AuthToken] = def user[F[_]: Sync](accountId: AccountId, key: ByteVector): F[AuthToken] =
for { for {
salt <- Common.genSaltString[F] salt <- Common.genSaltString[F]
millis = Instant.now.toEpochMilli millis = Instant.now.toEpochMilli
cd = AuthToken(millis, accountId, salt, "") cd = AuthToken(millis, accountId, salt, "")
sig = sign(cd, key) sig = sign(cd, key)

View File

@ -72,7 +72,7 @@ object Login {
data <- store.transact(QLogin.findUser(acc)) data <- store.transact(QLogin.findUser(acc))
_ <- Sync[F].delay(logger.trace(s"Account lookup: $data")) _ <- Sync[F].delay(logger.trace(s"Account lookup: $data"))
res <- if (data.exists(check(up.pass))) okResult res <- if (data.exists(check(up.pass))) okResult
else Result.invalidAuth.pure[F] else Result.invalidAuth.pure[F]
} yield res } yield res
case Left(_) => case Left(_) =>
Result.invalidAuth.pure[F] Result.invalidAuth.pure[F]

View File

@ -111,10 +111,10 @@ object OCollective {
): F[PassChangeResult] = { ): F[PassChangeResult] = {
val q = for { val q = for {
optUser <- RUser.findByAccount(accountId) optUser <- RUser.findByAccount(accountId)
check = optUser.map(_.password).map(p => PasswordCrypt.check(current, p)) check = optUser.map(_.password).map(p => PasswordCrypt.check(current, p))
n <- check n <- check
.filter(identity) .filter(identity)
.traverse(_ => RUser.updatePassword(accountId, PasswordCrypt.crypt(newPass))) .traverse(_ => RUser.updatePassword(accountId, PasswordCrypt.crypt(newPass)))
res = check match { res = check match {
case Some(true) => case Some(true) =>
if (n.getOrElse(0) > 0) PassChangeResult.success else PassChangeResult.updateFailed if (n.getOrElse(0) > 0) PassChangeResult.success else PassChangeResult.updateFailed

View File

@ -11,7 +11,14 @@ import docspell.store.queries.{QAttachment, QItem}
import OItem.{AttachmentData, AttachmentSourceData, ItemData, ListItem, Query} import OItem.{AttachmentData, AttachmentSourceData, ItemData, ListItem, Query}
import bitpeace.{FileMeta, RangeDef} import bitpeace.{FileMeta, RangeDef}
import docspell.common.{Direction, Ident, ItemState, MetaProposalList, Timestamp} import docspell.common.{Direction, Ident, ItemState, MetaProposalList, Timestamp}
import docspell.store.records.{RAttachment, RAttachmentMeta, RAttachmentSource, RItem, RSource, RTagItem} import docspell.store.records.{
RAttachment,
RAttachmentMeta,
RAttachmentSource,
RItem,
RSource,
RTagItem
}
trait OItem[F[_]] { trait OItem[F[_]] {
@ -75,14 +82,17 @@ object OItem {
def fileId: Ident def fileId: Ident
} }
case class AttachmentData[F[_]](ra: RAttachment, meta: FileMeta, data: Stream[F, Byte]) case class AttachmentData[F[_]](ra: RAttachment, meta: FileMeta, data: Stream[F, Byte])
extends BinaryData[F] { extends BinaryData[F] {
val name = ra.name val name = ra.name
val fileId = ra.fileId val fileId = ra.fileId
} }
case class AttachmentSourceData[F[_]](rs: RAttachmentSource, meta: FileMeta, data: Stream[F, Byte]) case class AttachmentSourceData[F[_]](
extends BinaryData[F] { rs: RAttachmentSource,
val name = rs.name meta: FileMeta,
data: Stream[F, Byte]
) extends BinaryData[F] {
val name = rs.name
val fileId = rs.fileId val fileId = rs.fileId
} }
@ -131,18 +141,22 @@ object OItem {
private def makeBinaryData[A](fileId: Ident)(f: FileMeta => A): F[Option[A]] = private def makeBinaryData[A](fileId: Ident)(f: FileMeta => A): F[Option[A]] =
store.bitpeace store.bitpeace
.get(fileId.id).unNoneTerminate.compile.last.map( .get(fileId.id)
_.map(m => f(m)) .unNoneTerminate
) .compile
.last
.map(
_.map(m => f(m))
)
def setTags(item: Ident, tagIds: List[Ident], collective: Ident): F[AddResult] = { def setTags(item: Ident, tagIds: List[Ident], collective: Ident): F[AddResult] = {
val db = for { val db = for {
cid <- RItem.getCollective(item) cid <- RItem.getCollective(item)
nd <- if (cid.contains(collective)) RTagItem.deleteItemTags(item) nd <- if (cid.contains(collective)) RTagItem.deleteItemTags(item)
else 0.pure[ConnectionIO] else 0.pure[ConnectionIO]
ni <- if (tagIds.nonEmpty && cid.contains(collective)) ni <- if (tagIds.nonEmpty && cid.contains(collective))
RTagItem.insertItemTags(item, tagIds) RTagItem.insertItemTags(item, tagIds)
else 0.pure[ConnectionIO] else 0.pure[ConnectionIO]
} yield nd + ni } yield nd + ni
store.transact(db).attempt.map(AddResult.fromUpdate) store.transact(db).attempt.map(AddResult.fromUpdate)

View File

@ -61,9 +61,9 @@ object OJob {
mustCancel(j.some).isEmpty mustCancel(j.some).isEmpty
val tryDelete = for { val tryDelete = for {
job <- RJob.findByIdAndGroup(id, collective) job <- RJob.findByIdAndGroup(id, collective)
jobm = job.filter(canDelete) jobm = job.filter(canDelete)
del <- jobm.traverse(j => RJob.delete(j.id)) del <- jobm.traverse(j => RJob.delete(j.id))
} yield del match { } yield del match {
case Some(_) => Right(JobCancelResult.Removed: JobCancelResult) case Some(_) => Right(JobCancelResult.Removed: JobCancelResult)
case None => Left(mustCancel(job)) case None => Left(mustCancel(job))
@ -77,12 +77,12 @@ object OJob {
for { for {
tryDel <- store.transact(tryDelete) tryDel <- store.transact(tryDelete)
result <- tryDel match { result <- tryDel match {
case Right(r) => r.pure[F] case Right(r) => r.pure[F]
case Left(Some((job, worker))) => case Left(Some((job, worker))) =>
tryCancel(job, worker) tryCancel(job, worker)
case Left(None) => case Left(None) =>
(JobCancelResult.JobNotFound: OJob.JobCancelResult).pure[F] (JobCancelResult.JobNotFound: OJob.JobCancelResult).pure[F]
} }
} yield result } yield result
} }
}) })

View File

@ -113,10 +113,10 @@ object OMail {
def createSettings(accId: AccountId, s: SmtpSettings): F[AddResult] = def createSettings(accId: AccountId, s: SmtpSettings): F[AddResult] =
(for { (for {
ru <- OptionT(store.transact(s.toRecord(accId).value)) ru <- OptionT(store.transact(s.toRecord(accId).value))
ins = RUserEmail.insert(ru) ins = RUserEmail.insert(ru)
exists = RUserEmail.exists(ru.uid, ru.name) exists = RUserEmail.exists(ru.uid, ru.name)
res <- OptionT.liftF(store.add(ins, exists)) res <- OptionT.liftF(store.add(ins, exists))
} yield res).getOrElse(AddResult.Failure(new Exception("User not found"))) } yield res).getOrElse(AddResult.Failure(new Exception("User not found")))
def updateSettings(accId: AccountId, name: Ident, data: SmtpSettings): F[Int] = { def updateSettings(accId: AccountId, name: Ident, data: SmtpSettings): F[Int] = {
@ -143,10 +143,10 @@ object OMail {
for { for {
_ <- OptionT.liftF(store.transact(RItem.existsById(m.item))).filter(identity) _ <- OptionT.liftF(store.transact(RItem.existsById(m.item))).filter(identity)
ras <- OptionT.liftF( ras <- OptionT.liftF(
store.transact( store.transact(
RAttachment.findByItemAndCollectiveWithMeta(m.item, accId.collective) RAttachment.findByItemAndCollectiveWithMeta(m.item, accId.collective)
) )
) )
} yield { } yield {
val addAttach = m.attach.filter(ras).map { a => val addAttach = m.attach.filter(ras).map { a =>
Attach[F](Stream.emit(a._2).through(store.bitpeace.fetchData2(RangeDef.all))) Attach[F](Stream.emit(a._2).through(store.bitpeace.fetchData2(RangeDef.all)))
@ -171,15 +171,15 @@ object OMail {
def storeMail(msgId: String, cfg: RUserEmail): F[Either[SendResult, Ident]] = { def storeMail(msgId: String, cfg: RUserEmail): F[Either[SendResult, Ident]] = {
val save = for { val save = for {
data <- RSentMail.forItem( data <- RSentMail.forItem(
m.item, m.item,
accId, accId,
msgId, msgId,
cfg.mailFrom, cfg.mailFrom,
name, name,
m.subject, m.subject,
m.recipients, m.recipients,
m.body m.body
) )
_ <- OptionT.liftF(RSentMail.insert(data._1)) _ <- OptionT.liftF(RSentMail.insert(data._1))
_ <- OptionT.liftF(RSentMailItem.insert(data._2)) _ <- OptionT.liftF(RSentMailItem.insert(data._2))
} yield data._1.id } yield data._1.id
@ -197,7 +197,7 @@ object OMail {
mail <- createMail(mailCfg) mail <- createMail(mailCfg)
mid <- OptionT.liftF(sendMail(mailCfg.toMailConfig, mail)) mid <- OptionT.liftF(sendMail(mailCfg.toMailConfig, mail))
res <- mid.traverse(id => OptionT.liftF(storeMail(id, mailCfg))) res <- mid.traverse(id => OptionT.liftF(storeMail(id, mailCfg)))
conv = res.fold(identity, _.fold(identity, id => SendResult.Success(id))) conv = res.fold(identity, _.fold(identity, id => SendResult.Success(id)))
} yield conv).getOrElse(SendResult.NotFound) } yield conv).getOrElse(SendResult.NotFound)
} }

View File

@ -81,7 +81,7 @@ object OUpload {
def submit(data: OUpload.UploadData[F], sourceId: Ident): F[OUpload.UploadResult] = def submit(data: OUpload.UploadData[F], sourceId: Ident): F[OUpload.UploadResult] =
for { for {
sOpt <- store.transact(RSource.find(sourceId)).map(_.toRight(UploadResult.NoSource)) sOpt <- store.transact(RSource.find(sourceId)).map(_.toRight(UploadResult.NoSource))
abbrev = sOpt.map(_.abbrev).toOption.getOrElse(data.meta.sourceAbbrev) abbrev = sOpt.map(_.abbrev).toOption.getOrElse(data.meta.sourceAbbrev)
updata = data.copy(meta = data.meta.copy(sourceAbbrev = abbrev)) updata = data.copy(meta = data.meta.copy(sourceAbbrev = abbrev))
accId = sOpt.map(source => AccountId(source.cid, source.sid)) accId = sOpt.map(source => AccountId(source.cid, source.sid))
@ -131,8 +131,8 @@ object OUpload {
) )
for { for {
id <- Ident.randomId[F] id <- Ident.randomId[F]
now <- Timestamp.current[F] now <- Timestamp.current[F]
jobs = args.map(a => create(id, now, a)) jobs = args.map(a => create(id, now, a))
} yield jobs } yield jobs

View File

@ -47,15 +47,16 @@ object OSignup {
for { for {
now <- Timestamp.current[F] now <- Timestamp.current[F]
min = now.minus(cfg.inviteTime) min = now.minus(cfg.inviteTime)
ok <- store.transact(RInvitation.useInvite(inv, min)) ok <- store.transact(RInvitation.useInvite(inv, min))
res <- if (ok) addUser(data).map(SignupResult.fromAddResult) res <- if (ok) addUser(data).map(SignupResult.fromAddResult)
else SignupResult.invalidInvitationKey.pure[F] else SignupResult.invalidInvitationKey.pure[F]
_ <- if (retryInvite(res)) _ <- if (retryInvite(res))
logger.fdebug(s"Adding account failed ($res). Allow retry with invite.") *> store logger
.transact( .fdebug(s"Adding account failed ($res). Allow retry with invite.") *> store
RInvitation.insert(RInvitation(inv, now)) .transact(
) RInvitation.insert(RInvitation(inv, now))
else 0.pure[F] )
else 0.pure[F]
} yield res } yield res
case None => case None =>
SignupResult.invalidInvitationKey.pure[F] SignupResult.invalidInvitationKey.pure[F]
@ -81,7 +82,7 @@ object OSignup {
for { for {
id2 <- Ident.randomId[F] id2 <- Ident.randomId[F]
now <- Timestamp.current[F] now <- Timestamp.current[F]
c = RCollective(data.collName, CollectiveState.Active, Language.German, now) c = RCollective(data.collName, CollectiveState.Active, Language.German, now)
u = RUser( u = RUser(
id2, id2,
data.login, data.login,

View File

@ -26,9 +26,7 @@ object AccountId {
invalid invalid
} }
val separated = sepearatorChars.foldRight(invalid) { (c, v) => val separated = sepearatorChars.foldRight(invalid)((c, v) => v.orElse(parse0(c)))
v.orElse(parse0(c))
}
separated.orElse(Ident.fromString(str).map(id => AccountId(id, id))) separated.orElse(Ident.fromString(str).map(id => AccountId(id, id)))
} }

View File

@ -1,8 +1,6 @@
package docspell.common package docspell.common
sealed trait DataType { sealed trait DataType {}
}
object DataType { object DataType {
@ -10,7 +8,6 @@ object DataType {
case class Hint(hint: MimeTypeHint) extends DataType case class Hint(hint: MimeTypeHint) extends DataType
def apply(mt: MimeType): DataType = def apply(mt: MimeType): DataType =
Exact(mt) Exact(mt)

View File

@ -65,11 +65,13 @@ object File {
javaList.asScala.toList.sortBy(_.getFileName.toString) javaList.asScala.toList.sortBy(_.getFileName.toString)
} }
def readAll[F[_]: Sync: ContextShift](file: Path, blocker: Blocker, chunkSize: Int): Stream[F, Byte] = def readAll[F[_]: Sync: ContextShift](
file: Path,
blocker: Blocker,
chunkSize: Int
): Stream[F, Byte] =
fs2.io.file.readAll(file, blocker, chunkSize) fs2.io.file.readAll(file, blocker, chunkSize)
def readText[F[_]: Sync: ContextShift](file: Path, blocker: Blocker): F[String] = def readText[F[_]: Sync: ContextShift](file: Path, blocker: Blocker): F[String] =
readAll[F](file, blocker, 8192). readAll[F](file, blocker, 8192).through(fs2.text.utf8Decode).compile.foldMonoid
through(fs2.text.utf8Decode).
compile.foldMonoid
} }

View File

@ -66,9 +66,7 @@ case class LenientUri(
) )
def readText[F[_]: Sync: ContextShift](chunkSize: Int, blocker: Blocker): F[String] = def readText[F[_]: Sync: ContextShift](chunkSize: Int, blocker: Blocker): F[String] =
readURL[F](chunkSize, blocker). readURL[F](chunkSize, blocker).through(fs2.text.utf8Decode).compile.foldMonoid
through(fs2.text.utf8Decode).
compile.foldMonoid
def host: Option[String] = def host: Option[String] =
authority.map(a => authority.map(a =>

View File

@ -17,7 +17,6 @@ trait Logger[F[_]] {
object Logger { object Logger {
def log4s[F[_]: Sync](log: Log4sLogger): Logger[F] = new Logger[F] { def log4s[F[_]: Sync](log: Log4sLogger): Logger[F] = new Logger[F] {
def trace(msg: => String): F[Unit] = def trace(msg: => String): F[Unit] =
log.ftrace(msg) log.ftrace(msg)
@ -38,4 +37,4 @@ object Logger {
log.ferror(msg) log.ferror(msg)
} }
} }

View File

@ -66,9 +66,7 @@ object MetaProposalList {
case None => map.updated(mp.proposalType, mp) case None => map.updated(mp.proposalType, mp)
} }
val merged = ml.foldLeft(init) { (map, el) => val merged = ml.foldLeft(init)((map, el) => el.proposals.foldLeft(map)(updateMap))
el.proposals.foldLeft(map)(updateMap)
}
fromMap(merged) fromMap(merged)
} }

View File

@ -23,7 +23,8 @@ object SystemCommand {
repl.foldLeft(s) { repl.foldLeft(s) {
case (res, (k, v)) => case (res, (k, v)) =>
res.replace(k, v) res.replace(k, v)
}) }
)
def toCmd: List[String] = def toCmd: List[String] =
program :: args.toList program :: args.toList
@ -47,10 +48,10 @@ object SystemCommand {
_ <- writeToProcess(stdin, proc, blocker) _ <- writeToProcess(stdin, proc, blocker)
term <- Sync[F].delay(proc.waitFor(cmd.timeout.seconds, TimeUnit.SECONDS)) term <- Sync[F].delay(proc.waitFor(cmd.timeout.seconds, TimeUnit.SECONDS))
_ <- if (term) logger.debug(s"Command `${cmd.cmdString}` finished: ${proc.exitValue}") _ <- if (term) logger.debug(s"Command `${cmd.cmdString}` finished: ${proc.exitValue}")
else else
logger.warn( logger.warn(
s"Command `${cmd.cmdString}` did not finish in ${cmd.timeout.formatExact}!" s"Command `${cmd.cmdString}` did not finish in ${cmd.timeout.formatExact}!"
) )
_ <- if (!term) timeoutError(proc, cmd) else Sync[F].pure(()) _ <- if (!term) timeoutError(proc, cmd) else Sync[F].pure(())
out <- if (term) inputStreamToString(proc.getInputStream, blocker) else Sync[F].pure("") out <- if (term) inputStreamToString(proc.getInputStream, blocker) else Sync[F].pure("")
err <- if (term) inputStreamToString(proc.getErrorStream, blocker) else Sync[F].pure("") err <- if (term) inputStreamToString(proc.getErrorStream, blocker) else Sync[F].pure("")
@ -75,25 +76,30 @@ object SystemCommand {
else Stream.emit(r) else Stream.emit(r)
} }
private def startProcess[F[_]: Sync, A](cmd: Config, wd: Option[Path], logger: Logger[F], stdin: Stream[F, Byte])( private def startProcess[F[_]: Sync, A](
cmd: Config,
wd: Option[Path],
logger: Logger[F],
stdin: Stream[F, Byte]
)(
f: Process => Stream[F, A] f: Process => Stream[F, A]
): Stream[F, A] = { ): Stream[F, A] = {
val log = logger.debug(s"Running external command: ${cmd.cmdString}") val log = logger.debug(s"Running external command: ${cmd.cmdString}")
val hasStdin = stdin.take(1).compile.last.map(_.isDefined) val hasStdin = stdin.take(1).compile.last.map(_.isDefined)
val proc = log *> hasStdin.flatMap(flag => Sync[F].delay { val proc = log *> hasStdin.flatMap(flag =>
val pb = new ProcessBuilder(cmd.toCmd.asJava) Sync[F].delay {
.redirectInput(if (flag) Redirect.PIPE else Redirect.INHERIT) val pb = new ProcessBuilder(cmd.toCmd.asJava)
.redirectError(Redirect.PIPE) .redirectInput(if (flag) Redirect.PIPE else Redirect.INHERIT)
.redirectOutput(Redirect.PIPE) .redirectError(Redirect.PIPE)
.redirectOutput(Redirect.PIPE)
wd.map(_.toFile).foreach(pb.directory) wd.map(_.toFile).foreach(pb.directory)
pb.start() pb.start()
}) }
)
Stream Stream
.bracket(proc)(p => .bracket(proc)(p =>
logger.debug(s"Closing process: `${cmd.cmdString}`").map { _ => logger.debug(s"Closing process: `${cmd.cmdString}`").map(_ => p.destroy())
p.destroy()
}
) )
.flatMap(f) .flatMap(f)
} }

View File

@ -17,8 +17,8 @@ trait StreamSyntax {
.map(optStr => .map(optStr =>
for { for {
str <- optStr str <- optStr
.map(_.trim) .map(_.trim)
.toRight(new Exception("Empty string cannot be parsed into a value")) .toRight(new Exception("Empty string cannot be parsed into a value"))
json <- parse(str).leftMap(_.underlying) json <- parse(str).leftMap(_.underlying)
value <- json.as[A] value <- json.as[A]
} yield value } yield value

View File

@ -13,7 +13,9 @@ import docspell.files.{ImageSize, TikaMimetype}
trait Conversion[F[_]] { trait Conversion[F[_]] {
def toPDF[A](dataType: DataType, lang: Language, handler: Handler[F, A])(in: Stream[F, Byte]): F[A] def toPDF[A](dataType: DataType, lang: Language, handler: Handler[F, A])(
in: Stream[F, Byte]
): F[A]
} }
@ -26,7 +28,9 @@ object Conversion {
): Resource[F, Conversion[F]] = ): Resource[F, Conversion[F]] =
Resource.pure(new Conversion[F] { Resource.pure(new Conversion[F] {
def toPDF[A](dataType: DataType, lang: Language, handler: Handler[F, A])(in: Stream[F, Byte]): F[A] = def toPDF[A](dataType: DataType, lang: Language, handler: Handler[F, A])(
in: Stream[F, Byte]
): F[A] =
TikaMimetype.resolve(dataType, in).flatMap { TikaMimetype.resolve(dataType, in).flatMap {
case MimeType.pdf => case MimeType.pdf =>
handler.run(ConversionResult.successPdf(in)) handler.run(ConversionResult.successPdf(in))
@ -112,10 +116,10 @@ object Conversion {
def unapply(mt: MimeType): Option[MimeType] = def unapply(mt: MimeType): Option[MimeType] =
mt match { mt match {
case Office(_) => Some(mt) case Office(_) => Some(mt)
case Texts(_) => Some(mt) case Texts(_) => Some(mt)
case Images(_) => Some(mt) case Images(_) => Some(mt)
case MimeType.html => Some(mt) case MimeType.html => Some(mt)
case _ => None case _ => None
} }
} }

View File

@ -3,9 +3,11 @@ package docspell.convert
import docspell.convert.extern.{TesseractConfig, UnoconvConfig, WkHtmlPdfConfig} import docspell.convert.extern.{TesseractConfig, UnoconvConfig, WkHtmlPdfConfig}
import docspell.convert.flexmark.MarkdownConfig import docspell.convert.flexmark.MarkdownConfig
case class ConvertConfig(chunkSize: Int, case class ConvertConfig(
maxImageSize: Int, chunkSize: Int,
markdown: MarkdownConfig, maxImageSize: Int,
wkhtmlpdf: WkHtmlPdfConfig, markdown: MarkdownConfig,
tesseract: TesseractConfig, wkhtmlpdf: WkHtmlPdfConfig,
unoconv: UnoconvConfig) tesseract: TesseractConfig,
unoconv: UnoconvConfig
)

View File

@ -20,7 +20,9 @@ private[extern] object ExternConv {
logger: Logger[F], logger: Logger[F],
reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] reader: (Path, SystemCommand.Result) => F[ConversionResult[F]]
)(in: Stream[F, Byte], handler: Handler[F, A]): F[A] = )(in: Stream[F, Byte], handler: Handler[F, A]): F[A] =
Stream.resource(File.withTempDir[F](wd, s"docspell-$name")).flatMap { dir => Stream
.resource(File.withTempDir[F](wd, s"docspell-$name"))
.flatMap { dir =>
val inFile = dir.resolve("infile").toAbsolutePath.normalize val inFile = dir.resolve("infile").toAbsolutePath.normalize
val out = dir.resolve("out.pdf").toAbsolutePath.normalize val out = dir.resolve("out.pdf").toAbsolutePath.normalize
val sysCfg = val sysCfg =
@ -40,12 +42,12 @@ private[extern] object ExternConv {
SystemCommand SystemCommand
.execSuccess[F](sysCfg, blocker, logger, Some(dir), if (useStdin) in else Stream.empty) .execSuccess[F](sysCfg, blocker, logger, Some(dir), if (useStdin) in else Stream.empty)
.evalMap(result => .evalMap(result =>
logResult(name, result, logger). logResult(name, result, logger).flatMap(_ => reader(out, result)).flatMap(handler.run)
flatMap(_ => reader(out, result)).
flatMap(handler.run)
) )
} }
}.compile.lastOrError }
.compile
.lastOrError
def readResult[F[_]: Sync: ContextShift]( def readResult[F[_]: Sync: ContextShift](
blocker: Blocker, blocker: Blocker,
@ -60,9 +62,11 @@ private[extern] object ExternConv {
successPdf(File.readAll(out, blocker, chunkSize)).pure[F] successPdf(File.readAll(out, blocker, chunkSize)).pure[F]
case false => case false =>
ConversionResult.failure[F]( ConversionResult
new Exception(s"Command result=${result.rc}. No output file found.") .failure[F](
).pure[F] new Exception(s"Command result=${result.rc}. No output file found.")
)
.pure[F]
} }
def readResultTesseract[F[_]: Sync: ContextShift]( def readResultTesseract[F[_]: Sync: ContextShift](
@ -75,7 +79,7 @@ private[extern] object ExternConv {
File.existsNonEmpty[F](outPdf).flatMap { File.existsNonEmpty[F](outPdf).flatMap {
case true => case true =>
val outTxt = out.resolveSibling(s"$outPrefix.txt") val outTxt = out.resolveSibling(s"$outPrefix.txt")
File.exists(outTxt).flatMap(txtExists => { File.exists(outTxt).flatMap { txtExists =>
val pdfData = File.readAll(out, blocker, chunkSize) val pdfData = File.readAll(out, blocker, chunkSize)
if (result.rc == 0) { if (result.rc == 0) {
if (txtExists) successPdfTxt(pdfData, File.readText(outTxt, blocker)).pure[F] if (txtExists) successPdfTxt(pdfData, File.readText(outTxt, blocker)).pure[F]
@ -84,12 +88,14 @@ private[extern] object ExternConv {
logger.warn(s"Command not successful (rc=${result.rc}), but file exists.") *> logger.warn(s"Command not successful (rc=${result.rc}), but file exists.") *>
successPdf(pdfData).pure[F] successPdf(pdfData).pure[F]
} }
}) }
case false => case false =>
ConversionResult.failure[F]( ConversionResult
new Exception(s"Command result=${result.rc}. No output file found.") .failure[F](
).pure[F] new Exception(s"Command result=${result.rc}. No output file found.")
)
.pure[F]
} }
} }

View File

@ -21,7 +21,15 @@ object Tesseract {
val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] = val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
ExternConv.readResultTesseract[F](outBase, blocker, chunkSize, logger) ExternConv.readResultTesseract[F](outBase, blocker, chunkSize, logger)
ExternConv.toPDF[F, A]("tesseract", cfg.command.replace(Map("{{lang}}" -> lang.iso3)), cfg.workingDir, false, blocker, logger, reader)(in, handler) ExternConv.toPDF[F, A](
"tesseract",
cfg.command.replace(Map("{{lang}}" -> lang.iso3)),
cfg.workingDir,
false,
blocker,
logger,
reader
)(in, handler)
} }
} }

View File

@ -4,4 +4,4 @@ import java.nio.file.Path
import docspell.common.SystemCommand import docspell.common.SystemCommand
case class TesseractConfig (command: SystemCommand.Config, workingDir: Path) case class TesseractConfig(command: SystemCommand.Config, workingDir: Path)

View File

@ -19,7 +19,10 @@ object Unoconv {
val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] = val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
ExternConv.readResult[F](blocker, chunkSize, logger) ExternConv.readResult[F](blocker, chunkSize, logger)
ExternConv.toPDF[F, A]("unoconv", cfg.command, cfg.workingDir, false, blocker, logger, reader)(in, handler) ExternConv.toPDF[F, A]("unoconv", cfg.command, cfg.workingDir, false, blocker, logger, reader)(
in,
handler
)
} }
} }

View File

@ -4,4 +4,4 @@ import java.nio.file.Path
import docspell.common.SystemCommand import docspell.common.SystemCommand
case class UnoconvConfig (command: SystemCommand.Config, workingDir: Path) case class UnoconvConfig(command: SystemCommand.Config, workingDir: Path)

View File

@ -14,12 +14,16 @@ object WkHtmlPdf {
cfg: WkHtmlPdfConfig, cfg: WkHtmlPdfConfig,
chunkSize: Int, chunkSize: Int,
blocker: Blocker, blocker: Blocker,
logger: Logger[F], logger: Logger[F]
)(in: Stream[F, Byte], handler: Handler[F, A]): F[A] = { )(in: Stream[F, Byte], handler: Handler[F, A]): F[A] = {
val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] = val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
ExternConv.readResult[F](blocker, chunkSize, logger) ExternConv.readResult[F](blocker, chunkSize, logger)
ExternConv.toPDF[F, A]("wkhtmltopdf", cfg.command, cfg.workingDir, true, blocker, logger, reader)(in, handler) ExternConv
.toPDF[F, A]("wkhtmltopdf", cfg.command, cfg.workingDir, true, blocker, logger, reader)(
in,
handler
)
} }
} }

View File

@ -4,4 +4,4 @@ import java.nio.file.Path
import docspell.common.SystemCommand import docspell.common.SystemCommand
case class WkHtmlPdfConfig (command: SystemCommand.Config, workingDir: Path) case class WkHtmlPdfConfig(command: SystemCommand.Config, workingDir: Path)

View File

@ -22,24 +22,22 @@ object Markdown {
val r = createRenderer() val r = createRenderer()
Try { Try {
val reader = new InputStreamReader(is, StandardCharsets.UTF_8) val reader = new InputStreamReader(is, StandardCharsets.UTF_8)
val doc = p.parseReader(reader) val doc = p.parseReader(reader)
wrapHtml(r.render(doc), cfg) wrapHtml(r.render(doc), cfg)
}.toEither }.toEither
} }
def toHtml(md: String, cfg: MarkdownConfig): String = { def toHtml(md: String, cfg: MarkdownConfig): String = {
val p = createParser() val p = createParser()
val r = createRenderer() val r = createRenderer()
val doc = p.parse(md) val doc = p.parse(md)
wrapHtml(r.render(doc), cfg) wrapHtml(r.render(doc), cfg)
} }
def toHtml[F[_]: Sync](data: Stream[F, Byte], cfg: MarkdownConfig): F[String] = def toHtml[F[_]: Sync](data: Stream[F, Byte], cfg: MarkdownConfig): F[String] =
data.through(fs2.text.utf8Decode).compile.foldMonoid. data.through(fs2.text.utf8Decode).compile.foldMonoid.map(str => toHtml(str, cfg))
map(str => toHtml(str, cfg))
private def wrapHtml(body: String, cfg: MarkdownConfig): String = { private def wrapHtml(body: String, cfg: MarkdownConfig): String =
s"""<!DOCTYPE html> s"""<!DOCTYPE html>
|<html> |<html>
|<head> |<head>
@ -53,13 +51,13 @@ object Markdown {
|</body> |</body>
|</html> |</html>
|""".stripMargin |""".stripMargin
}
private def createParser(): Parser = { private def createParser(): Parser = {
val opts = new MutableDataSet() val opts = new MutableDataSet()
opts.set(Parser.EXTENSIONS.asInstanceOf[DataKey[util.Collection[_]]], opts.set(
util.Arrays.asList(TablesExtension.create(), Parser.EXTENSIONS.asInstanceOf[DataKey[util.Collection[_]]],
StrikethroughExtension.create())); util.Arrays.asList(TablesExtension.create(), StrikethroughExtension.create())
);
Parser.builder(opts).build() Parser.builder(opts).build()
} }

View File

@ -55,5 +55,4 @@ trait FileChecks {
def commandExists(cmd: String): Boolean = def commandExists(cmd: String): Boolean =
Runtime.getRuntime.exec(Array("which", cmd)).waitFor() == 0 Runtime.getRuntime.exec(Array("which", cmd)).waitFor() == 0
} }

View File

@ -103,5 +103,4 @@ object ExternConvTest extends SimpleTestSuite with FileChecks {
} }
} }
} }

View File

@ -29,7 +29,7 @@ object Extraction {
data: Stream[F, Byte], data: Stream[F, Byte],
dataType: DataType, dataType: DataType,
lang: Language lang: Language
): F[ExtractResult] = { ): F[ExtractResult] =
TikaMimetype.resolve(dataType, data).flatMap { TikaMimetype.resolve(dataType, data).flatMap {
case MimeType.pdf => case MimeType.pdf =>
PdfExtract PdfExtract
@ -50,39 +50,46 @@ object Extraction {
.extractOCR(data, blocker, logger, lang.iso3, cfg.ocr) .extractOCR(data, blocker, logger, lang.iso3, cfg.ocr)
.compile .compile
.lastOrError .lastOrError
.map(_.trim)
.attempt .attempt
.map(ExtractResult.fromEither) .map(ExtractResult.fromEither)
ImageSize.get(data).flatMap { ImageSize.get(data).flatMap {
case Some(dim) => case Some(dim) =>
if (dim.product > cfg.ocr.maxImageSize) { if (dim.product > cfg.ocr.maxImageSize) {
logger.info(s"Image size (${dim.product}) is too large (max ${cfg.ocr.maxImageSize}).") *> logger.info(
ExtractResult.failure(new Exception( s"Image size (${dim.product}) is too large (max ${cfg.ocr.maxImageSize})."
s"Image size (${dim.width}x${dim.height}) is too large (max ${cfg.ocr.maxImageSize}).") ) *>
).pure[F] ExtractResult
.failure(
new Exception(
s"Image size (${dim.width}x${dim.height}) is too large (max ${cfg.ocr.maxImageSize})."
)
)
.pure[F]
} else { } else {
doExtract doExtract
} }
case None => case None =>
logger.info(s"Cannot read image data from ${mt.asString}. Extracting anyways.") *> logger.info(s"Cannot read image data from ${mt.asString}. Extracting anyways.") *>
doExtract doExtract
} }
case OdfType.container => case OdfType.container =>
logger.info(s"File detected as ${OdfType.container}. Try to read as OpenDocument file.") *> logger
.info(s"File detected as ${OdfType.container}. Try to read as OpenDocument file.") *>
OdfExtract.get(data).map(ExtractResult.fromEither) OdfExtract.get(data).map(ExtractResult.fromEither)
case mt@MimeType("text", sub) if !sub.contains("html") => case mt @ MimeType("text", sub) if !sub.contains("html") =>
logger.info(s"File detected as ${mt.asString}. Returning itself as text.") *> logger.info(s"File detected as ${mt.asString}. Returning itself as text.") *>
data.through(fs2.text.utf8Decode).compile.last.map { txt => data.through(fs2.text.utf8Decode).compile.last.map { txt =>
ExtractResult.success(txt.getOrElse("").trim) ExtractResult.success(txt.getOrElse("").trim)
} }
case mt => case mt =>
ExtractResult.unsupportedFormat(mt).pure[F] ExtractResult.unsupportedFormat(mt).pure[F]
} }
}
} }
} }

View File

@ -1,3 +1,3 @@
package docspell.extract package docspell.extract
case class PdfConfig (minTextLen: Int) case class PdfConfig(minTextLen: Int)

View File

@ -33,7 +33,8 @@ object PdfExtract {
//maybe better: inspect the pdf and decide whether ocr or not //maybe better: inspect the pdf and decide whether ocr or not
for { for {
pdfboxRes <- logger.debug("Trying to strip text from pdf using pdfbox.") *> PdfboxExtract.get[F](in) pdfboxRes <- logger.debug("Trying to strip text from pdf using pdfbox.") *> PdfboxExtract
.get[F](in)
res <- pdfboxRes.fold( res <- pdfboxRes.fold(
ex => ex =>
logger.info( logger.info(

View File

@ -5,13 +5,12 @@ import java.nio.file.{Path, Paths}
import docspell.common._ import docspell.common._
case class OcrConfig( case class OcrConfig(
maxImageSize: Int, maxImageSize: Int,
ghostscript: OcrConfig.Ghostscript, ghostscript: OcrConfig.Ghostscript,
pageRange: OcrConfig.PageRange, pageRange: OcrConfig.PageRange,
unpaper: OcrConfig.Unpaper, unpaper: OcrConfig.Unpaper,
tesseract: OcrConfig.Tesseract tesseract: OcrConfig.Tesseract
) { ) {}
}
object OcrConfig { object OcrConfig {

View File

@ -5,9 +5,9 @@ import docspell.common.MimeType
object OcrType { object OcrType {
val jpeg = MimeType.jpeg val jpeg = MimeType.jpeg
val png = MimeType.png val png = MimeType.png
val tiff = MimeType.tiff val tiff = MimeType.tiff
val pdf = MimeType.pdf val pdf = MimeType.pdf
val all = Set(jpeg, png, tiff, pdf) val all = Set(jpeg, png, tiff, pdf)

View File

@ -17,14 +17,14 @@ object OdfExtract {
def get[F[_]: Sync](data: Stream[F, Byte]): F[Either[Throwable, String]] = def get[F[_]: Sync](data: Stream[F, Byte]): F[Either[Throwable, String]] =
data.compile.to(Array).map(new ByteArrayInputStream(_)).map(get) data.compile.to(Array).map(new ByteArrayInputStream(_)).map(get)
def get(is: InputStream) =
def get(is: InputStream) = Try { Try {
val handler = new BodyContentHandler() val handler = new BodyContentHandler()
val pctx = new ParseContext() val pctx = new ParseContext()
val meta = new Metadata() val meta = new Metadata()
val ooparser = new OpenDocumentParser() val ooparser = new OpenDocumentParser()
ooparser.parse(is, handler, meta, pctx) ooparser.parse(is, handler, meta, pctx)
handler.toString.trim handler.toString.trim
}.toEither }.toEither
} }

View File

@ -4,8 +4,8 @@ import docspell.common.MimeType
object OdfType { object OdfType {
val odt = MimeType.application("vnd.oasis.opendocument.text") val odt = MimeType.application("vnd.oasis.opendocument.text")
val ods = MimeType.application("vnd.oasis.opendocument.spreadsheet") val ods = MimeType.application("vnd.oasis.opendocument.spreadsheet")
val odtAlias = MimeType.application("x-vnd.oasis.opendocument.text") val odtAlias = MimeType.application("x-vnd.oasis.opendocument.text")
val odsAlias = MimeType.application("x-vnd.oasis.opendocument.spreadsheet") val odsAlias = MimeType.application("x-vnd.oasis.opendocument.spreadsheet")

View File

@ -14,9 +14,7 @@ import fs2.Stream
object PdfboxExtract { object PdfboxExtract {
def get[F[_]: Sync](data: Stream[F, Byte]): F[Either[Throwable, String]] = def get[F[_]: Sync](data: Stream[F, Byte]): F[Either[Throwable, String]] =
data.compile.to(Array).map { bytes => data.compile.to(Array).map(bytes => Using(PDDocument.load(bytes))(readText).toEither.flatten)
Using(PDDocument.load(bytes))(readText).toEither.flatten
}
def get(is: InputStream): Either[Throwable, String] = def get(is: InputStream): Either[Throwable, String] =
Using(PDDocument.load(is))(readText).toEither.flatten Using(PDDocument.load(is))(readText).toEither.flatten

View File

@ -52,25 +52,25 @@ object PoiExtract {
def getDocx(is: InputStream): Either[Throwable, String] = def getDocx(is: InputStream): Either[Throwable, String] =
Try { Try {
val xt = new XWPFWordExtractor(new XWPFDocument(is)) val xt = new XWPFWordExtractor(new XWPFDocument(is))
xt.getText.trim Option(xt.getText).map(_.trim).getOrElse("")
}.toEither }.toEither
def getDoc(is: InputStream): Either[Throwable, String] = def getDoc(is: InputStream): Either[Throwable, String] =
Try { Try {
val xt = new WordExtractor(is) val xt = new WordExtractor(is)
xt.getText.trim Option(xt.getText).map(_.trim).getOrElse("")
}.toEither }.toEither
def getXlsx(is: InputStream): Either[Throwable, String] = def getXlsx(is: InputStream): Either[Throwable, String] =
Try { Try {
val xt = new XSSFExcelExtractor(new XSSFWorkbook(is)) val xt = new XSSFExcelExtractor(new XSSFWorkbook(is))
xt.getText.trim Option(xt.getText).map(_.trim).getOrElse("")
}.toEither }.toEither
def getXls(is: InputStream): Either[Throwable, String] = def getXls(is: InputStream): Either[Throwable, String] =
Try { Try {
val xt = new ExcelExtractor(new HSSFWorkbook(is)) val xt = new ExcelExtractor(new HSSFWorkbook(is))
xt.getText.trim Option(xt.getText).map(_.trim).getOrElse("")
}.toEither }.toEither
def getDocx[F[_]: Sync](data: Stream[F, Byte]): F[Either[Throwable, String]] = def getDocx[F[_]: Sync](data: Stream[F, Byte]): F[Either[Throwable, String]] =

View File

@ -5,11 +5,11 @@ import docspell.common.MimeType
object PoiType { object PoiType {
val msoffice = MimeType.application("x-tika-msoffice") val msoffice = MimeType.application("x-tika-msoffice")
val ooxml = MimeType.application("x-tika-ooxml") val ooxml = MimeType.application("x-tika-ooxml")
val docx = MimeType.application("vnd.openxmlformats-officedocument.wordprocessingml.document") val docx = MimeType.application("vnd.openxmlformats-officedocument.wordprocessingml.document")
val xlsx = MimeType.application("vnd.openxmlformats-officedocument.spreadsheetml.sheet") val xlsx = MimeType.application("vnd.openxmlformats-officedocument.spreadsheetml.sheet")
val xls = MimeType.application("vnd.ms-excel") val xls = MimeType.application("vnd.ms-excel")
val doc = MimeType.application("msword") val doc = MimeType.application("msword")
val all = Set(msoffice, ooxml, docx, xlsx, xls, doc) val all = Set(msoffice, ooxml, docx, xlsx, xls, doc)

View File

@ -5,7 +5,7 @@ import docspell.files.{ExampleFiles, TestFiles}
import minitest.SimpleTestSuite import minitest.SimpleTestSuite
object OdfExtractTest extends SimpleTestSuite { object OdfExtractTest extends SimpleTestSuite {
val blocker = TestFiles.blocker val blocker = TestFiles.blocker
implicit val CS = TestFiles.CS implicit val CS = TestFiles.CS
val files = List( val files = List(
@ -14,14 +14,15 @@ object OdfExtractTest extends SimpleTestSuite {
) )
test("test extract from odt") { test("test extract from odt") {
files.foreach { case (file, len) => files.foreach {
val is = file.toJavaUrl.map(_.openStream()).fold(sys.error, identity) case (file, len) =>
val str1 = OdfExtract.get(is).fold(throw _, identity) val is = file.toJavaUrl.map(_.openStream()).fold(sys.error, identity)
assertEquals(str1.length, len) val str1 = OdfExtract.get(is).fold(throw _, identity)
assertEquals(str1.length, len)
val data = file.readURL[IO](8192, blocker) val data = file.readURL[IO](8192, blocker)
val str2 = OdfExtract.get[IO](data).unsafeRunSync().fold(throw _, identity) val str2 = OdfExtract.get[IO](data).unsafeRunSync().fold(throw _, identity)
assertEquals(str2, str1) assertEquals(str2, str1)
} }
} }

View File

@ -7,8 +7,8 @@ object RtfExtractTest extends SimpleTestSuite {
test("extract text from rtf using java input-stream") { test("extract text from rtf using java input-stream") {
val file = ExampleFiles.examples_sample_rtf val file = ExampleFiles.examples_sample_rtf
val is = file.toJavaUrl.map(_.openStream()).fold(sys.error, identity) val is = file.toJavaUrl.map(_.openStream()).fold(sys.error, identity)
val str = RtfExtract.get(is).fold(throw _, identity) val str = RtfExtract.get(is).fold(throw _, identity)
assertEquals(str.length, 7342) assertEquals(str.length, 7342)
} }
} }

View File

@ -29,13 +29,12 @@ object ImageSize {
/** Return the image size from its header without reading /** Return the image size from its header without reading
* the whole image into memory. * the whole image into memory.
*/ */
def get[F[_]: Sync](data: Stream[F, Byte]): F[Option[Dimension]] = { def get[F[_]: Sync](data: Stream[F, Byte]): F[Option[Dimension]] =
data.take(768).compile.to(Array).map(ar => { data.take(768).compile.to(Array).map { ar =>
val iis = ImageIO.createImageInputStream(new ByteArrayInputStream(ar)) val iis = ImageIO.createImageInputStream(new ByteArrayInputStream(ar))
if (iis == null) sys.error("no reader given for the array") if (iis == null) sys.error("no reader given for the array")
else getDimension(iis) else getDimension(iis)
}) }
}
private def getDimension(in: ImageInputStream): Option[Dimension] = private def getDimension(in: ImageInputStream): Option[Dimension] =
ImageIO ImageIO

View File

@ -52,8 +52,8 @@ object TikaMimetype {
def detect[F[_]: Sync](file: Path): F[MimeType] = def detect[F[_]: Sync](file: Path): F[MimeType] =
Sync[F].delay { Sync[F].delay {
val hint = MimeTypeHint.filename(file.getFileName.toString) val hint = MimeTypeHint.filename(file.getFileName.toString)
Using(new BufferedInputStream(Files.newInputStream(file), 64))({ in => Using(new BufferedInputStream(Files.newInputStream(file), 64)) { in =>
convert(tika.detect(in, makeMetadata(hint))) convert(tika.detect(in, makeMetadata(hint)))
}).toEither }.toEither
}.rethrow }.rethrow
} }

View File

@ -7,8 +7,7 @@ trait ExampleFilesSupport {
def createUrl(resource: String): LenientUri = def createUrl(resource: String): LenientUri =
Option(getClass.getResource("/" + resource)) match { Option(getClass.getResource("/" + resource)) match {
case Some(u) => LenientUri.fromJava(u) case Some(u) => LenientUri.fromJava(u)
case None => sys.error(s"Resource '$resource' not found") case None => sys.error(s"Resource '$resource' not found")
} }
} }

View File

@ -8,15 +8,14 @@ import scala.concurrent.ExecutionContext
object Playing extends IOApp { object Playing extends IOApp {
val blocker = Blocker.liftExecutionContext(ExecutionContext.global) val blocker = Blocker.liftExecutionContext(ExecutionContext.global)
def run(args: List[String]): IO[ExitCode] = IO { def run(args: List[String]): IO[ExitCode] = IO {
//val ods = ExampleFiles.examples_sample_ods.readURL[IO](8192, blocker) //val ods = ExampleFiles.examples_sample_ods.readURL[IO](8192, blocker)
//val odt = ExampleFiles.examples_sample_odt.readURL[IO](8192, blocker) //val odt = ExampleFiles.examples_sample_odt.readURL[IO](8192, blocker)
val rtf = ExampleFiles.examples_sample_rtf.readURL[IO](8192, blocker) val rtf = ExampleFiles.examples_sample_rtf.readURL[IO](8192, blocker)
val x = for { val x = for {
odsm1 <- TikaMimetype.detect(rtf, odsm1 <- TikaMimetype
MimeTypeHint.filename(ExampleFiles.examples_sample_rtf.path.segments.last)) .detect(rtf, MimeTypeHint.filename(ExampleFiles.examples_sample_rtf.path.segments.last))
odsm2 <- TikaMimetype.detect(rtf, MimeTypeHint.none) odsm2 <- TikaMimetype.detect(rtf, MimeTypeHint.none)
} yield (odsm1, odsm2) } yield (odsm1, odsm2)
println(x.unsafeRunSync()) println(x.unsafeRunSync())

View File

@ -7,13 +7,13 @@ import docspell.convert.ConvertConfig
import docspell.extract.ExtractConfig import docspell.extract.ExtractConfig
case class Config( case class Config(
appId: Ident, appId: Ident,
baseUrl: LenientUri, baseUrl: LenientUri,
bind: Config.Bind, bind: Config.Bind,
jdbc: JdbcConfig, jdbc: JdbcConfig,
scheduler: SchedulerConfig, scheduler: SchedulerConfig,
extraction: ExtractConfig, extraction: ExtractConfig,
convert: ConvertConfig convert: ConvertConfig
) )
object Config { object Config {

View File

@ -52,15 +52,15 @@ object JoexAppImpl {
store <- Store.create(cfg.jdbc, connectEC, blocker) store <- Store.create(cfg.jdbc, connectEC, blocker)
nodeOps <- ONode(store) nodeOps <- ONode(store)
sch <- SchedulerBuilder(cfg.scheduler, blocker, store) sch <- SchedulerBuilder(cfg.scheduler, blocker, store)
.withTask( .withTask(
JobTask.json( JobTask.json(
ProcessItemArgs.taskName, ProcessItemArgs.taskName,
ItemHandler[F](cfg), ItemHandler[F](cfg),
ItemHandler.onCancel[F] ItemHandler.onCancel[F]
) )
) )
.resource .resource
app = new JoexAppImpl(cfg, nodeOps, store, termSignal, sch) app = new JoexAppImpl(cfg, nodeOps, store, termSignal, sch)
appR <- Resource.make(app.init.map(_ => app))(_.shutdown) appR <- Resource.make(app.init.map(_ => app))(_.shutdown)
} yield appR } yield appR
} }

View File

@ -68,7 +68,9 @@ object ConvertPdf {
.through(ctx.store.bitpeace.fetchData2(RangeDef.all)) .through(ctx.store.bitpeace.fetchData2(RangeDef.all))
val handler = conversionHandler[F](ctx, cfg, ra, item) val handler = conversionHandler[F](ctx, cfg, ra, item)
ctx.logger.info(s"Converting file ${ra.name} (${mime.asString}) into a PDF") *> ctx.logger.info(s"Converting file ${ra.name} (${mime.asString}) into a PDF") *>
conv.toPDF(DataType(MimeType(mime.primary, mime.sub)), ctx.args.meta.language, handler)(data) conv.toPDF(DataType(MimeType(mime.primary, mime.sub)), ctx.args.meta.language, handler)(
data
)
} }
} }
@ -107,19 +109,21 @@ object ConvertPdf {
}) })
private def storePDF[F[_]: Sync]( private def storePDF[F[_]: Sync](
ctx: Context[F, ProcessItemArgs], ctx: Context[F, ProcessItemArgs],
cfg: ConvertConfig, cfg: ConvertConfig,
ra: RAttachment, ra: RAttachment,
pdf: Stream[F, Byte] pdf: Stream[F, Byte]
) = { ) = {
val hint = MimeTypeHint.advertised(MimeType.pdf).withName(ra.name.getOrElse("file.pdf")) val hint = MimeTypeHint.advertised(MimeType.pdf).withName(ra.name.getOrElse("file.pdf"))
val newName = ra.name.map(n => s"$n.pdf") val newName = ra.name.map(n => s"$n.pdf")
ctx.store.bitpeace ctx.store.bitpeace
.saveNew(pdf, cfg.chunkSize, MimetypeHint(hint.filename, hint.advertised)) .saveNew(pdf, cfg.chunkSize, MimetypeHint(hint.filename, hint.advertised))
.compile .compile
.lastOrError .lastOrError
.map(fm => Ident.unsafe(fm.id)) .map(fm => Ident.unsafe(fm.id))
.flatMap(fmId => ctx.store.transact(RAttachment.updateFileIdAndName(ra.id, fmId, newName)).map(_ => fmId)) .flatMap(fmId =>
ctx.store.transact(RAttachment.updateFileIdAndName(ra.id, fmId, newName)).map(_ => fmId)
)
.map(fmId => ra.copy(fileId = fmId, name = newName)) .map(fmId => ra.copy(fileId = fmId, name = newName))
} }
} }

View File

@ -25,7 +25,7 @@ object CreateItem {
Task { ctx => Task { ctx =>
def isValidFile(fm: FileMeta) = def isValidFile(fm: FileMeta) =
ctx.args.meta.validFileTypes.isEmpty || ctx.args.meta.validFileTypes.isEmpty ||
ctx.args.meta.validFileTypes.map(_.asString).toSet.contains(fm.mimetype.baseType) ctx.args.meta.validFileTypes.map(_.asString).toSet.contains(fm.mimetype.baseType)
def fileMetas(itemId: Ident, now: Timestamp) = def fileMetas(itemId: Ident, now: Timestamp) =
Stream Stream
@ -77,18 +77,18 @@ object CreateItem {
for { for {
cand <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId))) cand <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId)))
_ <- if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.") _ <- if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.")
else ().pure[F] else ().pure[F]
ht <- cand.drop(1).traverse(ri => QItem.delete(ctx.store)(ri.id, ri.cid)) ht <- cand.drop(1).traverse(ri => QItem.delete(ctx.store)(ri.id, ri.cid))
_ <- if (ht.sum > 0) ctx.logger.warn(s"Removed ${ht.sum} items with same attachments") _ <- if (ht.sum > 0) ctx.logger.warn(s"Removed ${ht.sum} items with same attachments")
else ().pure[F] else ().pure[F]
rms <- OptionT( rms <- OptionT(
cand.headOption.traverse(ri => cand.headOption.traverse(ri =>
ctx.store.transact(RAttachment.findByItemAndCollective(ri.id, ri.cid)) ctx.store.transact(RAttachment.findByItemAndCollective(ri.id, ri.cid))
) )
).getOrElse(Vector.empty) ).getOrElse(Vector.empty)
orig <- rms.traverse(a => orig <- rms.traverse(a =>
ctx.store.transact(RAttachmentSource.findById(a.id)).map(s => (a, s)) ctx.store.transact(RAttachmentSource.findById(a.id)).map(s => (a, s))
) )
origMap = orig origMap = orig
.map(originFileTuple) .map(originFileTuple)
.toMap .toMap

View File

@ -95,10 +95,10 @@ object FindProposal {
labels => self.find(labels).map(f) labels => self.find(labels).map(f)
def next(f: Finder[F])(implicit F: FlatMap[F], F3: Applicative[F]): Finder[F] = def next(f: Finder[F])(implicit F: FlatMap[F], F3: Applicative[F]): Finder[F] =
flatMap({ ml0 => flatMap { ml0 =>
if (ml0.hasResultsAll) Finder.unit[F](ml0) if (ml0.hasResultsAll) Finder.unit[F](ml0)
else f.map(ml1 => ml0.fillEmptyFrom(ml1)) else f.map(ml1 => ml0.fillEmptyFrom(ml1))
}) }
def nextWhenEmpty(f: Finder[F], mt0: MetaProposalType, mts: MetaProposalType*)( def nextWhenEmpty(f: Finder[F], mt0: MetaProposalType, mts: MetaProposalType*)(
implicit F: FlatMap[F], implicit F: FlatMap[F],

View File

@ -19,14 +19,12 @@ object ItemHandler {
.map(_ => ()) .map(_ => ())
def itemStateTask[F[_]: Sync, A](state: ItemState)(data: ItemData): Task[F, A, ItemData] = def itemStateTask[F[_]: Sync, A](state: ItemState)(data: ItemData): Task[F, A, ItemData] =
Task { ctx => Task(ctx => ctx.store.transact(RItem.updateState(data.item.id, state)).map(_ => data))
ctx.store.transact(RItem.updateState(data.item.id, state)).map(_ => data)
}
def isLastRetry[F[_]: Sync, A](ctx: Context[F, A]): F[Boolean] = def isLastRetry[F[_]: Sync, A](ctx: Context[F, A]): F[Boolean] =
for { for {
current <- ctx.store.transact(RJob.getRetries(ctx.jobId)) current <- ctx.store.transact(RJob.getRetries(ctx.jobId))
last = ctx.config.retries == current.getOrElse(0) last = ctx.config.retries == current.getOrElse(0)
} yield last } yield last
def safeProcess[F[_]: Sync: ContextShift]( def safeProcess[F[_]: Sync: ContextShift](

View File

@ -11,9 +11,7 @@ object TestTasks {
private[this] val logger = getLogger private[this] val logger = getLogger
def success[F[_]]: Task[F, ProcessItemArgs, Unit] = def success[F[_]]: Task[F, ProcessItemArgs, Unit] =
Task { ctx => Task(ctx => ctx.logger.info(s"Running task now: ${ctx.args}"))
ctx.logger.info(s"Running task now: ${ctx.args}")
}
def failing[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] = def failing[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] =
Task { ctx => Task { ctx =>

View File

@ -20,8 +20,8 @@ object TextAnalysis {
t <- item.metas.toList.traverse(annotateAttachment[F](ctx.args.meta.language)) t <- item.metas.toList.traverse(annotateAttachment[F](ctx.args.meta.language))
_ <- ctx.logger.debug(s"Storing tags: ${t.map(_._1.copy(content = None))}") _ <- ctx.logger.debug(s"Storing tags: ${t.map(_._1.copy(content = None))}")
_ <- t.traverse(m => _ <- t.traverse(m =>
ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels)) ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels))
) )
e <- s e <- s
_ <- ctx.logger.info(s"Text-Analysis finished in ${e.formatExact}") _ <- ctx.logger.info(s"Text-Analysis finished in ${e.formatExact}")
v = t.toVector v = t.toVector

View File

@ -12,8 +12,8 @@ import docspell.store.records.{RAttachment, RAttachmentMeta, RFileMeta}
object TextExtraction { object TextExtraction {
def apply[F[_]: Sync: ContextShift]( def apply[F[_]: Sync: ContextShift](
cfg: ExtractConfig, cfg: ExtractConfig,
item: ItemData item: ItemData
): Task[F, ProcessItemArgs, ItemData] = ): Task[F, ProcessItemArgs, ItemData] =
Task { ctx => Task { ctx =>
for { for {
@ -28,11 +28,11 @@ object TextExtraction {
} }
def extractTextIfEmpty[F[_]: Sync: ContextShift]( def extractTextIfEmpty[F[_]: Sync: ContextShift](
ctx: Context[F, _], ctx: Context[F, _],
cfg: ExtractConfig, cfg: ExtractConfig,
lang: Language, lang: Language,
item: ItemData item: ItemData
)(ra: RAttachment): F[RAttachmentMeta] = { )(ra: RAttachment): F[RAttachmentMeta] = {
val rm = item.findOrCreate(ra.id) val rm = item.findOrCreate(ra.id)
rm.content match { rm.content match {
case Some(_) => case Some(_) =>
@ -50,14 +50,14 @@ object TextExtraction {
item: ItemData item: ItemData
)(ra: RAttachment): F[RAttachmentMeta] = )(ra: RAttachment): F[RAttachmentMeta] =
for { for {
_ <- ctx.logger.debug(s"Extracting text for attachment ${stripAttachmentName(ra)}") _ <- ctx.logger.debug(s"Extracting text for attachment ${stripAttachmentName(ra)}")
dst <- Duration.stopTime[F] dst <- Duration.stopTime[F]
txt <- extractTextFallback(ctx, cfg, ra, lang)(filesToExtract(item, ra)) txt <- extractTextFallback(ctx, cfg, ra, lang)(filesToExtract(item, ra))
meta = item.changeMeta(ra.id, rm => rm.setContentIfEmpty(txt.map(_.trim).filter(_.nonEmpty))) meta = item.changeMeta(ra.id, rm => rm.setContentIfEmpty(txt.map(_.trim).filter(_.nonEmpty)))
est <- dst est <- dst
_ <- ctx.logger.debug( _ <- ctx.logger.debug(
s"Extracting text for attachment ${stripAttachmentName(ra)} finished in ${est.formatExact}" s"Extracting text for attachment ${stripAttachmentName(ra)} finished in ${est.formatExact}"
) )
} yield meta } yield meta
def extractText[F[_]: Sync: ContextShift]( def extractText[F[_]: Sync: ContextShift](
@ -76,16 +76,15 @@ object TextExtraction {
.getOrElse(Mimetype.`application/octet-stream`) .getOrElse(Mimetype.`application/octet-stream`)
findMime findMime
.flatMap(mt => .flatMap(mt => extr.extractText(data, DataType(MimeType(mt.primary, mt.sub)), lang))
extr.extractText(data, DataType(MimeType(mt.primary, mt.sub)), lang))
} }
private def extractTextFallback[F[_]: Sync: ContextShift]( private def extractTextFallback[F[_]: Sync: ContextShift](
ctx: Context[F, _], ctx: Context[F, _],
cfg: ExtractConfig, cfg: ExtractConfig,
ra: RAttachment, ra: RAttachment,
lang: Language, lang: Language
)(fileIds: List[Ident]): F[Option[String]] = { )(fileIds: List[Ident]): F[Option[String]] =
fileIds match { fileIds match {
case Nil => case Nil =>
ctx.logger.error(s"Cannot extract text").map(_ => None) ctx.logger.error(s"Cannot extract text").map(_ => None)
@ -99,15 +98,18 @@ object TextExtraction {
txt.some.pure[F] txt.some.pure[F]
case ExtractResult.UnsupportedFormat(mt) => case ExtractResult.UnsupportedFormat(mt) =>
ctx.logger.warn(s"Cannot extract text from file ${stripAttachmentName(ra)}: unsupported format ${mt.asString}. Try with converted file."). ctx.logger
flatMap(_ => extractTextFallback[F](ctx, cfg, ra, lang)(rest)) .warn(
s"Cannot extract text from file ${stripAttachmentName(ra)}: unsupported format ${mt.asString}. Try with converted file."
)
.flatMap(_ => extractTextFallback[F](ctx, cfg, ra, lang)(rest))
case ExtractResult.Failure(ex) => case ExtractResult.Failure(ex) =>
ctx.logger.warn(s"Cannot extract text: ${ex.getMessage}. Try with converted file"). ctx.logger
flatMap(_ => extractTextFallback[F](ctx, cfg, ra, lang)(rest)) .warn(s"Cannot extract text: ${ex.getMessage}. Try with converted file")
.flatMap(_ => extractTextFallback[F](ctx, cfg, ra, lang)(rest))
}) })
} }
}
/** Returns the fileIds to extract text from. First, the source file /** Returns the fileIds to extract text from. First, the source file
* is tried. If that fails, the converted file is tried. * is tried. If that fails, the converted file is tried.
@ -115,7 +117,7 @@ object TextExtraction {
private def filesToExtract(item: ItemData, ra: RAttachment): List[Ident] = private def filesToExtract(item: ItemData, ra: RAttachment): List[Ident] =
item.originFile.get(ra.id) match { item.originFile.get(ra.id) match {
case Some(sid) => List(sid, ra.fileId).distinct case Some(sid) => List(sid, ra.fileId).distinct
case None => List(ra.fileId) case None => List(ra.fileId)
} }
private def stripAttachmentName(ra: RAttachment): String = private def stripAttachmentName(ra: RAttachment): String =

View File

@ -25,15 +25,15 @@ object JoexRoutes {
case GET -> Root / "running" => case GET -> Root / "running" =>
for { for {
jobs <- app.scheduler.getRunning jobs <- app.scheduler.getRunning
jj = jobs.map(mkJob) jj = jobs.map(mkJob)
resp <- Ok(JobList(jj.toList)) resp <- Ok(JobList(jj.toList))
} yield resp } yield resp
case POST -> Root / "shutdownAndExit" => case POST -> Root / "shutdownAndExit" =>
for { for {
_ <- ConcurrentEffect[F].start( _ <- ConcurrentEffect[F].start(
Timer[F].sleep(Duration.seconds(1).toScala) *> app.initShutdown Timer[F].sleep(Duration.seconds(1).toScala) *> app.initShutdown
) )
resp <- Ok(BasicResult(true, "Shutdown initiated.")) resp <- Ok(BasicResult(true, "Shutdown initiated."))
} yield resp } yield resp
@ -41,8 +41,8 @@ object JoexRoutes {
for { for {
optJob <- app.scheduler.getRunning.map(_.find(_.id == id)) optJob <- app.scheduler.getRunning.map(_.find(_.id == id))
optLog <- optJob.traverse(j => app.findLogs(j.id)) optLog <- optJob.traverse(j => app.findLogs(j.id))
jAndL = for { job <- optJob; log <- optLog } yield mkJobLog(job, log) jAndL = for { job <- optJob; log <- optLog } yield mkJobLog(job, log)
resp <- jAndL.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found"))) resp <- jAndL.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found")))
} yield resp } yield resp
case POST -> Root / "job" / Ident(id) / "cancel" => case POST -> Root / "job" / Ident(id) / "cancel" =>

View File

@ -54,7 +54,7 @@ object Context {
_ <- log.ftrace("Creating logger for task run") _ <- log.ftrace("Creating logger for task run")
logger <- QueueLogger(job.id, job.info, config.logBufferSize, logSink) logger <- QueueLogger(job.id, job.info, config.logBufferSize, logSink)
_ <- log.ftrace("Logger created, instantiating context") _ <- log.ftrace("Logger created, instantiating context")
ctx = create[F, A](job, arg, config, logger, store, blocker) ctx = create[F, A](job, arg, config, logger, store, blocker)
} yield ctx } yield ctx
final private class ContextImpl[F[_]: Functor, A]( final private class ContextImpl[F[_]: Functor, A](

View File

@ -38,9 +38,9 @@ object QueueLogger {
sink: LogSink[F] sink: LogSink[F]
): F[Logger[F]] = ): F[Logger[F]] =
for { for {
q <- Queue.circularBuffer[F, LogEvent](bufferSize) q <- Queue.circularBuffer[F, LogEvent](bufferSize)
log = create(jobId, jobInfo, q) log = create(jobId, jobInfo, q)
_ <- Concurrent[F].start(q.dequeue.through(sink.receive).compile.drain) _ <- Concurrent[F].start(q.dequeue.through(sink.receive).compile.drain)
} yield log } yield log
} }

View File

@ -91,12 +91,12 @@ final class SchedulerImpl[F[_]: ConcurrentEffect: ContextShift](
_ <- logger.fdebug("New permit acquired") _ <- logger.fdebug("New permit acquired")
down <- state.get.map(_.shutdownRequest) down <- state.get.map(_.shutdownRequest)
rjob <- if (down) logger.finfo("") *> permits.release *> (None: Option[RJob]).pure[F] rjob <- if (down) logger.finfo("") *> permits.release *> (None: Option[RJob]).pure[F]
else else
queue.nextJob( queue.nextJob(
group => state.modify(_.nextPrio(group, config.countingScheme)), group => state.modify(_.nextPrio(group, config.countingScheme)),
config.name, config.name,
config.retryDelay config.retryDelay
) )
_ <- logger.fdebug(s"Next job found: ${rjob.map(_.info)}") _ <- logger.fdebug(s"Next job found: ${rjob.map(_.info)}")
_ <- rjob.map(execute).getOrElse(permits.release) _ <- rjob.map(execute).getOrElse(permits.release)
} yield rjob.isDefined } yield rjob.isDefined
@ -122,8 +122,8 @@ final class SchedulerImpl[F[_]: ConcurrentEffect: ContextShift](
def execute(job: RJob): F[Unit] = { def execute(job: RJob): F[Unit] = {
val task = for { val task = for {
jobtask <- tasks jobtask <- tasks
.find(job.task) .find(job.task)
.toRight(s"This executor cannot run tasks with name: ${job.task}") .toRight(s"This executor cannot run tasks with name: ${job.task}")
} yield jobtask } yield jobtask
task match { task match {
@ -144,8 +144,8 @@ final class SchedulerImpl[F[_]: ConcurrentEffect: ContextShift](
for { for {
_ <- logger.fdebug(s"Job ${job.info} done $finalState. Releasing resources.") _ <- logger.fdebug(s"Job ${job.info} done $finalState. Releasing resources.")
_ <- permits.release *> permits.available.flatMap(a => _ <- permits.release *> permits.available.flatMap(a =>
logger.fdebug(s"Permit released ($a free)") logger.fdebug(s"Permit released ($a free)")
) )
_ <- state.modify(_.removeRunning(job)) _ <- state.modify(_.removeRunning(job))
_ <- QJob.setFinalState(job.id, finalState, store) _ <- QJob.setFinalState(job.id, finalState, store)
} yield () } yield ()

View File

@ -128,6 +128,9 @@ Please see the `nix/module-server.nix` and `nix/module-joex.nix` files
for the set of options. The nixos options are modelled after the for the set of options. The nixos options are modelled after the
default configuration file. default configuration file.
The modules files are only applicable to the newest version of
Docspell. If you really need an older version, checkout the
appropriate commit.
## NixOs Example ## NixOs Example

View File

@ -27,8 +27,8 @@ object RestAppImpl {
): Resource[F, RestApp[F]] = ): Resource[F, RestApp[F]] =
for { for {
backend <- BackendApp(cfg.backend, connectEC, httpClientEc, blocker) backend <- BackendApp(cfg.backend, connectEC, httpClientEc, blocker)
app = new RestAppImpl[F](cfg, backend) app = new RestAppImpl[F](cfg, backend)
appR <- Resource.make(app.init.map(_ => app))(_.shutdown) appR <- Resource.make(app.init.map(_ => app))(_.shutdown)
} yield appR } yield appR
} }

View File

@ -35,8 +35,8 @@ object CookieData {
for { for {
header <- headers.Cookie.from(req.headers).toRight("Cookie parsing error") header <- headers.Cookie.from(req.headers).toRight("Cookie parsing error")
cookie <- header.values.toList cookie <- header.values.toList
.find(_.name == cookieName) .find(_.name == cookieName)
.toRight("Couldn't find the authcookie") .toRight("Couldn't find the authcookie")
} yield cookie.content } yield cookie.content
def fromHeader[F[_]](req: Request[F]): Either[String, String] = def fromHeader[F[_]](req: Request[F]): Either[String, String] =

View File

@ -84,7 +84,7 @@ trait Conversions {
data.inReplyTo.map(mkIdName), data.inReplyTo.map(mkIdName),
data.item.dueDate, data.item.dueDate,
data.item.notes, data.item.notes,
data.attachments.map((mkAttachment(data)_).tupled).toList, data.attachments.map((mkAttachment(data) _).tupled).toList,
data.sources.map((mkAttachmentSource _).tupled).toList, data.sources.map((mkAttachmentSource _).tupled).toList,
data.tags.map(mkTag).toList data.tags.map(mkTag).toList
) )
@ -204,7 +204,8 @@ trait Conversions {
val files = mp.parts val files = mp.parts
.filter(p => p.name.forall(s => !s.equalsIgnoreCase("meta"))) .filter(p => p.name.forall(s => !s.equalsIgnoreCase("meta")))
.map(p => OUpload.File(p.filename, p.headers.get(`Content-Type`).map(fromContentType), p.body) .map(p =>
OUpload.File(p.filename, p.headers.get(`Content-Type`).map(fromContentType), p.body)
) )
for { for {
metaData <- meta metaData <- meta

View File

@ -45,21 +45,21 @@ object AttachmentRoutes {
for { for {
fileData <- backend.item.findAttachment(id, user.account.collective) fileData <- backend.item.findAttachment(id, user.account.collective)
resp <- fileData resp <- fileData
.map(data => withResponseHeaders(Ok())(data)) .map(data => withResponseHeaders(Ok())(data))
.getOrElse(NotFound(BasicResult(false, "Not found"))) .getOrElse(NotFound(BasicResult(false, "Not found")))
} yield resp } yield resp
case req @ GET -> Root / Ident(id) => case req @ GET -> Root / Ident(id) =>
for { for {
fileData <- backend.item.findAttachment(id, user.account.collective) fileData <- backend.item.findAttachment(id, user.account.collective)
inm = req.headers.get(`If-None-Match`).flatMap(_.tags) inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
matches = matchETag(fileData.map(_.meta), inm) matches = matchETag(fileData.map(_.meta), inm)
resp <- fileData resp <- fileData
.map({ data => .map { data =>
if (matches) withResponseHeaders(NotModified())(data) if (matches) withResponseHeaders(NotModified())(data)
else makeByteResp(data) else makeByteResp(data)
}) }
.getOrElse(NotFound(BasicResult(false, "Not found"))) .getOrElse(NotFound(BasicResult(false, "Not found")))
} yield resp } yield resp
case HEAD -> Root / Ident(id) / "original" => case HEAD -> Root / Ident(id) / "original" =>
@ -73,13 +73,13 @@ object AttachmentRoutes {
case req @ GET -> Root / Ident(id) / "original" => case req @ GET -> Root / Ident(id) / "original" =>
for { for {
fileData <- backend.item.findAttachmentSource(id, user.account.collective) fileData <- backend.item.findAttachmentSource(id, user.account.collective)
inm = req.headers.get(`If-None-Match`).flatMap(_.tags) inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
matches = matchETag(fileData.map(_.meta), inm) matches = matchETag(fileData.map(_.meta), inm)
resp <- fileData resp <- fileData
.map({ data => .map { data =>
if (matches) withResponseHeaders(NotModified())(data) if (matches) withResponseHeaders(NotModified())(data)
else makeByteResp(data) else makeByteResp(data)
}) }
.getOrElse(NotFound(BasicResult(false, "Not found"))) .getOrElse(NotFound(BasicResult(false, "Not found")))
} yield resp } yield resp
@ -92,16 +92,16 @@ object AttachmentRoutes {
case GET -> Root / Ident(id) / "meta" => case GET -> Root / Ident(id) / "meta" =>
for { for {
rm <- backend.item.findAttachmentMeta(id, user.account.collective) rm <- backend.item.findAttachmentMeta(id, user.account.collective)
md = rm.map(Conversions.mkAttachmentMeta) md = rm.map(Conversions.mkAttachmentMeta)
resp <- md.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found."))) resp <- md.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found.")))
} yield resp } yield resp
} }
} }
private def matchETag[F[_]]( private def matchETag[F[_]](
fileData: Option[FileMeta], fileData: Option[FileMeta],
noneMatch: Option[NonEmptyList[EntityTag]] noneMatch: Option[NonEmptyList[EntityTag]]
): Boolean = ): Boolean =
(fileData, noneMatch) match { (fileData, noneMatch) match {
case (Some(meta), Some(nm)) => case (Some(meta), Some(nm)) =>

View File

@ -35,25 +35,25 @@ object CollectiveRoutes {
case GET -> Root / "settings" => case GET -> Root / "settings" =>
for { for {
collDb <- backend.collective.find(user.account.collective) collDb <- backend.collective.find(user.account.collective)
sett = collDb.map(c => CollectiveSettings(c.language)) sett = collDb.map(c => CollectiveSettings(c.language))
resp <- sett.toResponse() resp <- sett.toResponse()
} yield resp } yield resp
case GET -> Root / "contacts" :? QueryParam.QueryOpt(q) +& QueryParam.ContactKindOpt(kind) => case GET -> Root / "contacts" :? QueryParam.QueryOpt(q) +& QueryParam.ContactKindOpt(kind) =>
for { for {
res <- backend.collective res <- backend.collective
.getContacts(user.account.collective, q.map(_.q), kind) .getContacts(user.account.collective, q.map(_.q), kind)
.take(50) .take(50)
.compile .compile
.toList .toList
resp <- Ok(ContactList(res.map(Conversions.mkContact))) resp <- Ok(ContactList(res.map(Conversions.mkContact)))
} yield resp } yield resp
case GET -> Root => case GET -> Root =>
for { for {
collDb <- backend.collective.find(user.account.collective) collDb <- backend.collective.find(user.account.collective)
coll = collDb.map(c => Collective(c.id, c.state, c.created)) coll = collDb.map(c => Collective(c.id, c.state, c.created))
resp <- coll.toResponse() resp <- coll.toResponse()
} yield resp } yield resp
} }
} }

View File

@ -37,10 +37,10 @@ object EquipmentRoutes {
case req @ PUT -> Root => case req @ PUT -> Root =>
for { for {
data <- req.as[Equipment] data <- req.as[Equipment]
equip = changeEquipment(data, user.account.collective) equip = changeEquipment(data, user.account.collective)
res <- backend.equipment.update(equip) res <- backend.equipment.update(equip)
resp <- Ok(basicResult(res, "Equipment updated.")) resp <- Ok(basicResult(res, "Equipment updated."))
} yield resp } yield resp
case DELETE -> Root / Ident(id) => case DELETE -> Root / Ident(id) =>

View File

@ -24,8 +24,8 @@ object ItemRoutes {
HttpRoutes.of { HttpRoutes.of {
case req @ POST -> Root / "search" => case req @ POST -> Root / "search" =>
for { for {
mask <- req.as[ItemSearch] mask <- req.as[ItemSearch]
_ <- logger.ftrace(s"Got search mask: $mask") _ <- logger.ftrace(s"Got search mask: $mask")
query = Conversions.mkQuery(mask, user.account.collective) query = Conversions.mkQuery(mask, user.account.collective)
_ <- logger.ftrace(s"Running query: $query") _ <- logger.ftrace(s"Running query: $query")
items <- backend.item.findItems(query, 100) items <- backend.item.findItems(query, 100)
@ -34,9 +34,9 @@ object ItemRoutes {
case GET -> Root / Ident(id) => case GET -> Root / Ident(id) =>
for { for {
item <- backend.item.findItem(id, user.account.collective) item <- backend.item.findItem(id, user.account.collective)
result = item.map(Conversions.mkItemDetail) result = item.map(Conversions.mkItemDetail)
resp <- result.map(r => Ok(r)).getOrElse(NotFound(BasicResult(false, "Not found."))) resp <- result.map(r => Ok(r)).getOrElse(NotFound(BasicResult(false, "Not found.")))
} yield resp } yield resp
case POST -> Root / Ident(id) / "confirm" => case POST -> Root / Ident(id) / "confirm" =>
@ -125,15 +125,15 @@ object ItemRoutes {
case GET -> Root / Ident(id) / "proposals" => case GET -> Root / Ident(id) / "proposals" =>
for { for {
ml <- backend.item.getProposals(id, user.account.collective) ml <- backend.item.getProposals(id, user.account.collective)
ip = Conversions.mkItemProposals(ml) ip = Conversions.mkItemProposals(ml)
resp <- Ok(ip) resp <- Ok(ip)
} yield resp } yield resp
case DELETE -> Root / Ident(id) => case DELETE -> Root / Ident(id) =>
for { for {
n <- backend.item.delete(id, user.account.collective) n <- backend.item.delete(id, user.account.collective)
res = BasicResult(n > 0, if (n > 0) "Item deleted" else "Item deletion failed.") res = BasicResult(n > 0, if (n > 0) "Item deleted" else "Item deletion failed.")
resp <- Ok(res) resp <- Ok(res)
} yield resp } yield resp
} }

View File

@ -19,8 +19,8 @@ object JobQueueRoutes {
HttpRoutes.of { HttpRoutes.of {
case GET -> Root / "state" => case GET -> Root / "state" =>
for { for {
js <- backend.job.queueState(user.account.collective, 200) js <- backend.job.queueState(user.account.collective, 200)
res = Conversions.mkJobQueueState(js) res = Conversions.mkJobQueueState(js)
resp <- Ok(res) resp <- Ok(res)
} yield resp } yield resp

View File

@ -54,15 +54,15 @@ object LoginRoutes {
for { for {
cd <- AuthToken.user(token.account, cfg.auth.serverSecret).map(CookieData.apply) cd <- AuthToken.user(token.account, cfg.auth.serverSecret).map(CookieData.apply)
resp <- Ok( resp <- Ok(
AuthResult( AuthResult(
token.account.collective.id, token.account.collective.id,
token.account.user.id, token.account.user.id,
true, true,
"Login successful", "Login successful",
Some(cd.asString), Some(cd.asString),
cfg.auth.sessionValid.millis cfg.auth.sessionValid.millis
) )
).map(_.addCookie(cd.asCookie(cfg))) ).map(_.addCookie(cd.asCookie(cfg)))
} yield resp } yield resp
case _ => case _ =>
Ok(AuthResult("", account, false, "Login failed.", None, 0L)) Ok(AuthResult("", account, false, "Login failed.", None, 0L))

View File

@ -24,13 +24,13 @@ object MailSendRoutes {
HttpRoutes.of { HttpRoutes.of {
case req @ POST -> Root / Ident(name) / Ident(id) => case req @ POST -> Root / Ident(name) / Ident(id) =>
for { for {
in <- req.as[SimpleMail] in <- req.as[SimpleMail]
mail = convertIn(id, in) mail = convertIn(id, in)
res <- mail.traverse(m => backend.mail.sendMail(user.account, name, m)) res <- mail.traverse(m => backend.mail.sendMail(user.account, name, m))
resp <- res.fold( resp <- res.fold(
err => Ok(BasicResult(false, s"Invalid mail data: $err")), err => Ok(BasicResult(false, s"Invalid mail data: $err")),
res => Ok(convertOut(res)) res => Ok(convertOut(res))
) )
} yield resp } yield resp
} }
} }
@ -39,7 +39,7 @@ object MailSendRoutes {
for { for {
rec <- s.recipients.traverse(EmilUtil.readMailAddress) rec <- s.recipients.traverse(EmilUtil.readMailAddress)
fileIds <- s.attachmentIds.traverse(Ident.fromString) fileIds <- s.attachmentIds.traverse(Ident.fromString)
sel = if (s.addAllAttachments) AttachSelection.All else AttachSelection.Selected(fileIds) sel = if (s.addAllAttachments) AttachSelection.All else AttachSelection.Selected(fileIds)
} yield ItemMail(item, s.subject, rec, s.body, sel) } yield ItemMail(item, s.subject, rec, s.body, sel)
def convertOut(res: SendResult): BasicResult = def convertOut(res: SendResult): BasicResult =

View File

@ -29,7 +29,7 @@ object MailSettingsRoutes {
case GET -> Root :? QueryParam.QueryOpt(q) => case GET -> Root :? QueryParam.QueryOpt(q) =>
for { for {
list <- backend.mail.getSettings(user.account, q.map(_.q)) list <- backend.mail.getSettings(user.account, q.map(_.q))
res = list.map(convert) res = list.map(convert)
resp <- Ok(EmailSettingsList(res.toList)) resp <- Ok(EmailSettingsList(res.toList))
} yield resp } yield resp
@ -45,13 +45,13 @@ object MailSettingsRoutes {
ru = makeSettings(in) ru = makeSettings(in)
up <- OptionT.liftF(ru.traverse(r => backend.mail.createSettings(user.account, r))) up <- OptionT.liftF(ru.traverse(r => backend.mail.createSettings(user.account, r)))
resp <- OptionT.liftF( resp <- OptionT.liftF(
Ok( Ok(
up.fold( up.fold(
err => BasicResult(false, err), err => BasicResult(false, err),
ar => Conversions.basicResult(ar, "Mail settings stored.") ar => Conversions.basicResult(ar, "Mail settings stored.")
) )
) )
) )
} yield resp).getOrElseF(NotFound()) } yield resp).getOrElseF(NotFound())
case req @ PUT -> Root / Ident(name) => case req @ PUT -> Root / Ident(name) =>
@ -60,24 +60,24 @@ object MailSettingsRoutes {
ru = makeSettings(in) ru = makeSettings(in)
up <- OptionT.liftF(ru.traverse(r => backend.mail.updateSettings(user.account, name, r))) up <- OptionT.liftF(ru.traverse(r => backend.mail.updateSettings(user.account, name, r)))
resp <- OptionT.liftF( resp <- OptionT.liftF(
Ok( Ok(
up.fold( up.fold(
err => BasicResult(false, err), err => BasicResult(false, err),
n => n =>
if (n > 0) BasicResult(true, "Mail settings stored.") if (n > 0) BasicResult(true, "Mail settings stored.")
else BasicResult(false, "Mail settings could not be saved") else BasicResult(false, "Mail settings could not be saved")
) )
) )
) )
} yield resp).getOrElseF(NotFound()) } yield resp).getOrElseF(NotFound())
case DELETE -> Root / Ident(name) => case DELETE -> Root / Ident(name) =>
for { for {
n <- backend.mail.deleteSettings(user.account, name) n <- backend.mail.deleteSettings(user.account, name)
resp <- Ok( resp <- Ok(
if (n > 0) BasicResult(true, "Mail settings removed") if (n > 0) BasicResult(true, "Mail settings removed")
else BasicResult(false, "Mail settings could not be removed") else BasicResult(false, "Mail settings could not be removed")
) )
} yield resp } yield resp
} }

View File

@ -36,8 +36,8 @@ object SourceRoutes {
case req @ PUT -> Root => case req @ PUT -> Root =>
for { for {
data <- req.as[Source] data <- req.as[Source]
src = changeSource(data, user.account.collective) src = changeSource(data, user.account.collective)
updated <- backend.source.update(src) updated <- backend.source.update(src)
resp <- Ok(basicResult(updated, "Source updated.")) resp <- Ok(basicResult(updated, "Source updated."))
} yield resp } yield resp

View File

@ -37,7 +37,7 @@ object TagRoutes {
case req @ PUT -> Root => case req @ PUT -> Root =>
for { for {
data <- req.as[Tag] data <- req.as[Tag]
tag = changeTag(data, user.account.collective) tag = changeTag(data, user.account.collective)
res <- backend.tag.update(tag) res <- backend.tag.update(tag)
resp <- Ok(basicResult(res, "Tag successfully updated.")) resp <- Ok(basicResult(res, "Tag successfully updated."))
} yield resp } yield resp

View File

@ -28,11 +28,11 @@ object UploadRoutes {
for { for {
multipart <- req.as[Multipart[F]] multipart <- req.as[Multipart[F]]
updata <- readMultipart( updata <- readMultipart(
multipart, multipart,
logger, logger,
Priority.High, Priority.High,
cfg.backend.files.validMimeTypes cfg.backend.files.validMimeTypes
) )
result <- backend.upload.submit(updata, user.account) result <- backend.upload.submit(updata, user.account)
res <- Ok(basicResult(result)) res <- Ok(basicResult(result))
} yield res } yield res

View File

@ -24,10 +24,10 @@ object UserRoutes {
for { for {
data <- req.as[PasswordChange] data <- req.as[PasswordChange]
res <- backend.collective.changePassword( res <- backend.collective.changePassword(
user.account, user.account,
data.currentPassword, data.currentPassword,
data.newPassword data.newPassword
) )
resp <- Ok(basicResult(res)) resp <- Ok(basicResult(res))
} yield resp } yield resp
@ -47,8 +47,8 @@ object UserRoutes {
case req @ PUT -> Root => case req @ PUT -> Root =>
for { for {
data <- req.as[User] data <- req.as[User]
nuser = changeUser(data, user.account.collective) nuser = changeUser(data, user.account.collective)
update <- backend.collective.update(nuser) update <- backend.collective.update(nuser)
resp <- Ok(basicResult(update, "User updated.")) resp <- Ok(basicResult(update, "User updated."))
} yield resp } yield resp

View File

@ -40,7 +40,7 @@ object Store {
for { for {
xa <- hxa xa <- hxa
st = new StoreImpl[F](jdbc, xa) st = new StoreImpl[F](jdbc, xa)
_ <- Resource.liftF(st.migrate) _ <- Resource.liftF(st.migrate)
} yield st } yield st
} }
} }

View File

@ -14,25 +14,32 @@ object QAttachment {
def deleteById[F[_]: Sync](store: Store[F])(attachId: Ident, coll: Ident): F[Int] = def deleteById[F[_]: Sync](store: Store[F])(attachId: Ident, coll: Ident): F[Int] =
for { for {
raFile <- store.transact(RAttachment.findByIdAndCollective(attachId, coll)).map(_.map(_.fileId)) raFile <- store
rsFile <- store.transact(RAttachmentSource.findByIdAndCollective(attachId, coll)).map(_.map(_.fileId)) .transact(RAttachment.findByIdAndCollective(attachId, coll))
n <- store.transact(RAttachment.delete(attachId)) .map(_.map(_.fileId))
f <- Stream.emits(raFile.toSeq ++ rsFile.toSeq) rsFile <- store
.map(_.id) .transact(RAttachmentSource.findByIdAndCollective(attachId, coll))
.flatMap(store.bitpeace.delete) .map(_.map(_.fileId))
.map(flag => if (flag) 1 else 0) n <- store.transact(RAttachment.delete(attachId))
.compile f <- Stream
.foldMonoid .emits(raFile.toSeq ++ rsFile.toSeq)
.map(_.id)
.flatMap(store.bitpeace.delete)
.map(flag => if (flag) 1 else 0)
.compile
.foldMonoid
} yield n + f } yield n + f
def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] = def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] =
for { for {
s <- store.transact(RAttachmentSource.findById(ra.id)) s <- store.transact(RAttachmentSource.findById(ra.id))
n <- store.transact(RAttachment.delete(ra.id)) n <- store.transact(RAttachment.delete(ra.id))
f <- Stream.emits(ra.fileId.id +: s.map(_.fileId.id).toSeq). f <- Stream
flatMap(store.bitpeace.delete). .emits(ra.fileId.id +: s.map(_.fileId.id).toSeq)
map(flag => if (flag) 1 else 0). .flatMap(store.bitpeace.delete)
compile.foldMonoid .map(flag => if (flag) 1 else 0)
.compile
.foldMonoid
} yield n + f } yield n + f
def deleteItemAttachments[F[_]: Sync](store: Store[F])(itemId: Ident, coll: Ident): F[Int] = def deleteItemAttachments[F[_]: Sync](store: Store[F])(itemId: Ident, coll: Ident): F[Int] =

View File

@ -27,7 +27,6 @@ object QCollective {
and(IC.cid.is(coll), IC.incoming.is(Direction.outgoing)) and(IC.cid.is(coll), IC.incoming.is(Direction.outgoing))
).query[Int].unique ).query[Int].unique
val fileSize = sql""" val fileSize = sql"""
select sum(length) from ( select sum(length) from (
with attachs as with attachs as
@ -42,7 +41,6 @@ object QCollective {
inner join filemeta m on m.id = a.file_id where a.id in (select aid from attachs) inner join filemeta m on m.id = a.file_id where a.id in (select aid from attachs)
) as t""".query[Option[Long]].unique ) as t""".query[Option[Long]].unique
val q3 = fr"SELECT" ++ commas( val q3 = fr"SELECT" ++ commas(
TC.name.prefix("t").f, TC.name.prefix("t").f,
fr"count(" ++ RC.itemId.prefix("r").f ++ fr")" fr"count(" ++ RC.itemId.prefix("r").f ++ fr")"

View File

@ -23,7 +23,7 @@ object QItem {
concEquip: Option[REquipment], concEquip: Option[REquipment],
inReplyTo: Option[IdRef], inReplyTo: Option[IdRef],
tags: Vector[RTag], tags: Vector[RTag],
attachments: Vector[(RAttachment, FileMeta)], attachments: Vector[(RAttachment, FileMeta)],
sources: Vector[(RAttachmentSource, FileMeta)] sources: Vector[(RAttachmentSource, FileMeta)]
) { ) {
@ -39,23 +39,24 @@ object QItem {
val EC = REquipment.Columns.all.map(_.prefix("e")) val EC = REquipment.Columns.all.map(_.prefix("e"))
val ICC = List(RItem.Columns.id, RItem.Columns.name).map(_.prefix("ref")) val ICC = List(RItem.Columns.id, RItem.Columns.name).map(_.prefix("ref"))
val cq = selectSimple(IC ++ OC ++ P0C ++ P1C ++ EC ++ ICC, RItem.table ++ fr"i", Fragment.empty) ++ val cq =
fr"LEFT JOIN" ++ ROrganization.table ++ fr"o ON" ++ RItem.Columns.corrOrg selectSimple(IC ++ OC ++ P0C ++ P1C ++ EC ++ ICC, RItem.table ++ fr"i", Fragment.empty) ++
.prefix("i") fr"LEFT JOIN" ++ ROrganization.table ++ fr"o ON" ++ RItem.Columns.corrOrg
.is(ROrganization.Columns.oid.prefix("o")) ++ .prefix("i")
fr"LEFT JOIN" ++ RPerson.table ++ fr"p0 ON" ++ RItem.Columns.corrPerson .is(ROrganization.Columns.oid.prefix("o")) ++
.prefix("i") fr"LEFT JOIN" ++ RPerson.table ++ fr"p0 ON" ++ RItem.Columns.corrPerson
.is(RPerson.Columns.pid.prefix("p0")) ++ .prefix("i")
fr"LEFT JOIN" ++ RPerson.table ++ fr"p1 ON" ++ RItem.Columns.concPerson .is(RPerson.Columns.pid.prefix("p0")) ++
.prefix("i") fr"LEFT JOIN" ++ RPerson.table ++ fr"p1 ON" ++ RItem.Columns.concPerson
.is(RPerson.Columns.pid.prefix("p1")) ++ .prefix("i")
fr"LEFT JOIN" ++ REquipment.table ++ fr"e ON" ++ RItem.Columns.concEquipment .is(RPerson.Columns.pid.prefix("p1")) ++
.prefix("i") fr"LEFT JOIN" ++ REquipment.table ++ fr"e ON" ++ RItem.Columns.concEquipment
.is(REquipment.Columns.eid.prefix("e")) ++ .prefix("i")
fr"LEFT JOIN" ++ RItem.table ++ fr"ref ON" ++ RItem.Columns.inReplyTo .is(REquipment.Columns.eid.prefix("e")) ++
.prefix("i") fr"LEFT JOIN" ++ RItem.table ++ fr"ref ON" ++ RItem.Columns.inReplyTo
.is(RItem.Columns.id.prefix("ref")) ++ .prefix("i")
fr"WHERE" ++ RItem.Columns.id.prefix("i").is(id) .is(RItem.Columns.id.prefix("ref")) ++
fr"WHERE" ++ RItem.Columns.id.prefix("i").is(id)
val q = cq val q = cq
.query[ .query[
@ -235,11 +236,12 @@ object QItem {
def findByFileIds(fileMetaIds: List[Ident]): ConnectionIO[Vector[RItem]] = { def findByFileIds(fileMetaIds: List[Ident]): ConnectionIO[Vector[RItem]] = {
val IC = RItem.Columns val IC = RItem.Columns
val AC = RAttachment.Columns val AC = RAttachment.Columns
val q = fr"SELECT DISTINCT" ++ commas(IC.all.map(_.prefix("i").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++ val q =
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId fr"SELECT DISTINCT" ++ commas(IC.all.map(_.prefix("i").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++
.prefix("a") fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId
.is(IC.id.prefix("i")) ++ .prefix("a")
fr"WHERE" ++ AC.fileId.isOneOf(fileMetaIds) ++ orderBy(IC.created.prefix("i").asc) .is(IC.id.prefix("i")) ++
fr"WHERE" ++ AC.fileId.isOneOf(fileMetaIds) ++ orderBy(IC.created.prefix("i").asc)
q.query[RItem].to[Vector] q.query[RItem].to[Vector]
} }

View File

@ -21,11 +21,11 @@ object QJob {
Stream Stream
.range(0, 10) .range(0, 10)
.evalMap(n => takeNextJob1(store)(priority, worker, retryPause, n)) .evalMap(n => takeNextJob1(store)(priority, worker, retryPause, n))
.evalTap({ x => .evalTap { x =>
if (x.isLeft) if (x.isLeft)
logger.fdebug[F]("Cannot mark job, probably due to concurrent updates. Will retry.") logger.fdebug[F]("Cannot mark job, probably due to concurrent updates. Will retry.")
else ().pure[F] else ().pure[F]
}) }
.find(_.isRight) .find(_.isRight)
.flatMap({ .flatMap({
case Right(job) => case Right(job) =>
@ -50,7 +50,7 @@ object QJob {
store.transact(for { store.transact(for {
n <- RJob.setScheduled(job.id, worker) n <- RJob.setScheduled(job.id, worker)
_ <- if (n == 1) RJobGroupUse.setGroup(RJobGroupUse(worker, job.group)) _ <- if (n == 1) RJobGroupUse.setGroup(RJobGroupUse(worker, job.group))
else 0.pure[ConnectionIO] else 0.pure[ConnectionIO]
} yield if (n == 1) Right(job) else Left(())) } yield if (n == 1) Right(job) else Left(()))
for { for {
@ -61,8 +61,8 @@ object QJob {
prio <- group.map(priority).getOrElse((Priority.Low: Priority).pure[F]) prio <- group.map(priority).getOrElse((Priority.Low: Priority).pure[F])
_ <- logger.ftrace[F](s"Looking for job of prio $prio") _ <- logger.ftrace[F](s"Looking for job of prio $prio")
job <- group job <- group
.map(g => store.transact(selectNextJob(g, prio, retryPause, now))) .map(g => store.transact(selectNextJob(g, prio, retryPause, now)))
.getOrElse((None: Option[RJob]).pure[F]) .getOrElse((None: Option[RJob]).pure[F])
_ <- logger.ftrace[F](s"Found job: ${job.map(_.info)}") _ <- logger.ftrace[F](s"Found job: ${job.map(_.info)}")
res <- job.traverse(j => markJob(j)) res <- job.traverse(j => markJob(j))
} yield res.map(_.map(_.some)).getOrElse { } yield res.map(_.map(_.some)).getOrElse {
@ -97,7 +97,8 @@ object QJob {
val sql2 = fr"SELECT min(" ++ jgroup.f ++ fr") as g FROM" ++ RJob.table ++ fr"a" ++ val sql2 = fr"SELECT min(" ++ jgroup.f ++ fr") as g FROM" ++ RJob.table ++ fr"a" ++
fr"WHERE" ++ stateCond fr"WHERE" ++ stateCond
val union = sql"SELECT g FROM ((" ++ sql1 ++ sql") UNION ALL (" ++ sql2 ++ sql")) as t0 WHERE g is not null" val union =
sql"SELECT g FROM ((" ++ sql1 ++ sql") UNION ALL (" ++ sql2 ++ sql")) as t0 WHERE g is not null"
union union
.query[Ident] .query[Ident]

View File

@ -34,11 +34,11 @@ object JobQueue {
def insert(job: RJob): F[Unit] = def insert(job: RJob): F[Unit] =
store store
.transact(RJob.insert(job)) .transact(RJob.insert(job))
.flatMap({ n => .flatMap { n =>
if (n != 1) if (n != 1)
Effect[F].raiseError(new Exception(s"Inserting job failed. Update count: $n")) Effect[F].raiseError(new Exception(s"Inserting job failed. Update count: $n"))
else ().pure[F] else ().pure[F]
}) }
def insertAll(jobs: Seq[RJob]): F[Unit] = def insertAll(jobs: Seq[RJob]): F[Unit] =
jobs.toList jobs.toList

View File

@ -47,10 +47,10 @@ object RAttachment {
def findMeta(attachId: Ident): ConnectionIO[Option[FileMeta]] = { def findMeta(attachId: Ident): ConnectionIO[Option[FileMeta]] = {
import bitpeace.sql._ import bitpeace.sql._
val cols = RFileMeta.Columns.all.map(_.prefix("m")) val cols = RFileMeta.Columns.all.map(_.prefix("m"))
val aId = id.prefix("a") val aId = id.prefix("a")
val aFileMeta = fileId.prefix("a") val aFileMeta = fileId.prefix("a")
val mId = RFileMeta.Columns.id.prefix("m") val mId = RFileMeta.Columns.id.prefix("m")
val from = table ++ fr"a INNER JOIN" ++ RFileMeta.table ++ fr"m ON" ++ aFileMeta.is(mId) val from = table ++ fr"a INNER JOIN" ++ RFileMeta.table ++ fr"m ON" ++ aFileMeta.is(mId)
val cond = aId.is(attachId) val cond = aId.is(attachId)
@ -104,7 +104,8 @@ object RAttachment {
def findByItemWithMeta(id: Ident): ConnectionIO[Vector[(RAttachment, FileMeta)]] = { def findByItemWithMeta(id: Ident): ConnectionIO[Vector[(RAttachment, FileMeta)]] = {
import bitpeace.sql._ import bitpeace.sql._
val q = fr"SELECT a.*,m.* FROM" ++ table ++ fr"a, filemeta m WHERE a.filemetaid = m.id AND a.itemid = $id ORDER BY a.position ASC" val q =
fr"SELECT a.*,m.* FROM" ++ table ++ fr"a, filemeta m WHERE a.filemetaid = m.id AND a.itemid = $id ORDER BY a.position ASC"
q.query[(RAttachment, FileMeta)].to[Vector] q.query[(RAttachment, FileMeta)].to[Vector]
} }

View File

@ -38,18 +38,20 @@ object RAttachmentSource {
def insert(v: RAttachmentSource): ConnectionIO[Int] = def insert(v: RAttachmentSource): ConnectionIO[Int] =
insertRow(table, all, fr"${v.id},${v.fileId},${v.name},${v.created}").update.run insertRow(table, all, fr"${v.id},${v.fileId},${v.name},${v.created}").update.run
def findById(attachId: Ident): ConnectionIO[Option[RAttachmentSource]] = def findById(attachId: Ident): ConnectionIO[Option[RAttachmentSource]] =
selectSimple(all, table, id.is(attachId)).query[RAttachmentSource].option selectSimple(all, table, id.is(attachId)).query[RAttachmentSource].option
def delete(attachId: Ident): ConnectionIO[Int] = def delete(attachId: Ident): ConnectionIO[Int] =
deleteFrom(table, id.is(attachId)).update.run deleteFrom(table, id.is(attachId)).update.run
def findByIdAndCollective(attachId: Ident, collective: Ident): ConnectionIO[Option[RAttachmentSource]] = { def findByIdAndCollective(
val bId = RAttachment.Columns.id.prefix("b") attachId: Ident,
val aId = Columns.id.prefix("a") collective: Ident
): ConnectionIO[Option[RAttachmentSource]] = {
val bId = RAttachment.Columns.id.prefix("b")
val aId = Columns.id.prefix("a")
val bItem = RAttachment.Columns.itemId.prefix("b") val bItem = RAttachment.Columns.itemId.prefix("b")
val iId = RItem.Columns.id.prefix("i") val iId = RItem.Columns.id.prefix("i")
val iColl = RItem.Columns.cid.prefix("i") val iColl = RItem.Columns.cid.prefix("i")
val from = table ++ fr"a INNER JOIN" ++ val from = table ++ fr"a INNER JOIN" ++
@ -64,11 +66,11 @@ object RAttachmentSource {
def findByItemWithMeta(id: Ident): ConnectionIO[Vector[(RAttachmentSource, FileMeta)]] = { def findByItemWithMeta(id: Ident): ConnectionIO[Vector[(RAttachmentSource, FileMeta)]] = {
import bitpeace.sql._ import bitpeace.sql._
val aId = Columns.id.prefix("a") val aId = Columns.id.prefix("a")
val afileMeta = fileId.prefix("a") val afileMeta = fileId.prefix("a")
val bPos = RAttachment.Columns.position.prefix("b") val bPos = RAttachment.Columns.position.prefix("b")
val bId = RAttachment.Columns.id.prefix("b") val bId = RAttachment.Columns.id.prefix("b")
val bItem = RAttachment.Columns.itemId.prefix("b") val bItem = RAttachment.Columns.itemId.prefix("b")
val mId = RFileMeta.Columns.id.prefix("m") val mId = RFileMeta.Columns.id.prefix("m")
val cols = all.map(_.prefix("a")) ++ RFileMeta.Columns.all.map(_.prefix("m")) val cols = all.map(_.prefix("a")) ++ RFileMeta.Columns.all.map(_.prefix("m"))
@ -77,8 +79,9 @@ object RAttachmentSource {
RAttachment.table ++ fr"b ON" ++ aId.is(bId) RAttachment.table ++ fr"b ON" ++ aId.is(bId)
val where = bItem.is(id) val where = bItem.is(id)
(selectSimple(cols, from, where) ++ orderBy(bPos.asc)). (selectSimple(cols, from, where) ++ orderBy(bPos.asc))
query[(RAttachmentSource, FileMeta)].to[Vector] .query[(RAttachmentSource, FileMeta)]
.to[Vector]
} }
} }

View File

@ -124,140 +124,140 @@ object RItem {
for { for {
t <- currentTime t <- currentTime
n <- updateRow( n <- updateRow(
table, table,
and(id.is(itemId), cid.is(coll)), and(id.is(itemId), cid.is(coll)),
commas(state.setTo(itemState), updated.setTo(t)) commas(state.setTo(itemState), updated.setTo(t))
).update.run ).update.run
} yield n } yield n
def updateDirection(itemId: Ident, coll: Ident, dir: Direction): ConnectionIO[Int] = def updateDirection(itemId: Ident, coll: Ident, dir: Direction): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow( n <- updateRow(
table, table,
and(id.is(itemId), cid.is(coll)), and(id.is(itemId), cid.is(coll)),
commas(incoming.setTo(dir), updated.setTo(t)) commas(incoming.setTo(dir), updated.setTo(t))
).update.run ).update.run
} yield n } yield n
def updateCorrOrg(itemId: Ident, coll: Ident, org: Option[Ident]): ConnectionIO[Int] = def updateCorrOrg(itemId: Ident, coll: Ident, org: Option[Ident]): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow( n <- updateRow(
table, table,
and(id.is(itemId), cid.is(coll)), and(id.is(itemId), cid.is(coll)),
commas(corrOrg.setTo(org), updated.setTo(t)) commas(corrOrg.setTo(org), updated.setTo(t))
).update.run ).update.run
} yield n } yield n
def removeCorrOrg(coll: Ident, currentOrg: Ident): ConnectionIO[Int] = def removeCorrOrg(coll: Ident, currentOrg: Ident): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow( n <- updateRow(
table, table,
and(cid.is(coll), corrOrg.is(Some(currentOrg))), and(cid.is(coll), corrOrg.is(Some(currentOrg))),
commas(corrOrg.setTo(None: Option[Ident]), updated.setTo(t)) commas(corrOrg.setTo(None: Option[Ident]), updated.setTo(t))
).update.run ).update.run
} yield n } yield n
def updateCorrPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] = def updateCorrPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow( n <- updateRow(
table, table,
and(id.is(itemId), cid.is(coll)), and(id.is(itemId), cid.is(coll)),
commas(corrPerson.setTo(person), updated.setTo(t)) commas(corrPerson.setTo(person), updated.setTo(t))
).update.run ).update.run
} yield n } yield n
def removeCorrPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] = def removeCorrPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow( n <- updateRow(
table, table,
and(cid.is(coll), corrPerson.is(Some(currentPerson))), and(cid.is(coll), corrPerson.is(Some(currentPerson))),
commas(corrPerson.setTo(None: Option[Ident]), updated.setTo(t)) commas(corrPerson.setTo(None: Option[Ident]), updated.setTo(t))
).update.run ).update.run
} yield n } yield n
def updateConcPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] = def updateConcPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow( n <- updateRow(
table, table,
and(id.is(itemId), cid.is(coll)), and(id.is(itemId), cid.is(coll)),
commas(concPerson.setTo(person), updated.setTo(t)) commas(concPerson.setTo(person), updated.setTo(t))
).update.run ).update.run
} yield n } yield n
def removeConcPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] = def removeConcPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow( n <- updateRow(
table, table,
and(cid.is(coll), concPerson.is(Some(currentPerson))), and(cid.is(coll), concPerson.is(Some(currentPerson))),
commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t)) commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t))
).update.run ).update.run
} yield n } yield n
def updateConcEquip(itemId: Ident, coll: Ident, equip: Option[Ident]): ConnectionIO[Int] = def updateConcEquip(itemId: Ident, coll: Ident, equip: Option[Ident]): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow( n <- updateRow(
table, table,
and(id.is(itemId), cid.is(coll)), and(id.is(itemId), cid.is(coll)),
commas(concEquipment.setTo(equip), updated.setTo(t)) commas(concEquipment.setTo(equip), updated.setTo(t))
).update.run ).update.run
} yield n } yield n
def removeConcEquip(coll: Ident, currentEquip: Ident): ConnectionIO[Int] = def removeConcEquip(coll: Ident, currentEquip: Ident): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow( n <- updateRow(
table, table,
and(cid.is(coll), concEquipment.is(Some(currentEquip))), and(cid.is(coll), concEquipment.is(Some(currentEquip))),
commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t)) commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t))
).update.run ).update.run
} yield n } yield n
def updateNotes(itemId: Ident, coll: Ident, text: Option[String]): ConnectionIO[Int] = def updateNotes(itemId: Ident, coll: Ident, text: Option[String]): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow( n <- updateRow(
table, table,
and(id.is(itemId), cid.is(coll)), and(id.is(itemId), cid.is(coll)),
commas(notes.setTo(text), updated.setTo(t)) commas(notes.setTo(text), updated.setTo(t))
).update.run ).update.run
} yield n } yield n
def updateName(itemId: Ident, coll: Ident, itemName: String): ConnectionIO[Int] = def updateName(itemId: Ident, coll: Ident, itemName: String): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow( n <- updateRow(
table, table,
and(id.is(itemId), cid.is(coll)), and(id.is(itemId), cid.is(coll)),
commas(name.setTo(itemName), updated.setTo(t)) commas(name.setTo(itemName), updated.setTo(t))
).update.run ).update.run
} yield n } yield n
def updateDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] = def updateDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow( n <- updateRow(
table, table,
and(id.is(itemId), cid.is(coll)), and(id.is(itemId), cid.is(coll)),
commas(itemDate.setTo(date), updated.setTo(t)) commas(itemDate.setTo(date), updated.setTo(t))
).update.run ).update.run
} yield n } yield n
def updateDueDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] = def updateDueDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] =
for { for {
t <- currentTime t <- currentTime
n <- updateRow( n <- updateRow(
table, table,
and(id.is(itemId), cid.is(coll)), and(id.is(itemId), cid.is(coll)),
commas(dueDate.setTo(date), updated.setTo(t)) commas(dueDate.setTo(date), updated.setTo(t))
).update.run ).update.run
} yield n } yield n
def deleteByIdAndCollective(itemId: Ident, coll: Ident): ConnectionIO[Int] = def deleteByIdAndCollective(itemId: Ident, coll: Ident): ConnectionIO[Int] =

View File

@ -162,17 +162,17 @@ object RJob {
for { for {
_ <- incrementRetries(jobId) _ <- incrementRetries(jobId)
n <- updateRow( n <- updateRow(
table, table,
and( and(
id.is(jobId), id.is(jobId),
or(worker.isNull, worker.is(workerId)), or(worker.isNull, worker.is(workerId)),
state.isOneOf(Seq[JobState](JobState.Waiting, JobState.Stuck)) state.isOneOf(Seq[JobState](JobState.Waiting, JobState.Stuck))
), ),
commas( commas(
state.setTo(JobState.Scheduled: JobState), state.setTo(JobState.Scheduled: JobState),
worker.setTo(workerId) worker.setTo(workerId)
) )
).update.run ).update.run
} yield n } yield n
def setSuccess(jobId: Ident, now: Timestamp): ConnectionIO[Int] = def setSuccess(jobId: Ident, now: Timestamp): ConnectionIO[Int] =

View File

@ -52,16 +52,16 @@ object RSentMail {
for { for {
user <- OptionT(RUser.findByAccount(accId)) user <- OptionT(RUser.findByAccount(accId))
sm <- OptionT.liftF( sm <- OptionT.liftF(
RSentMail[ConnectionIO]( RSentMail[ConnectionIO](
user.uid, user.uid,
messageId, messageId,
sender, sender,
connName, connName,
subject, subject,
recipients, recipients,
body body
) )
) )
si <- OptionT.liftF(RSentMailItem[ConnectionIO](itemId, sm.id, Some(sm.created))) si <- OptionT.liftF(RSentMailItem[ConnectionIO](itemId, sm.id, Some(sm.created)))
} yield (sm, si) } yield (sm, si)

View File

@ -33,9 +33,9 @@ object RTagItem {
def insertItemTags(item: Ident, tags: Seq[Ident]): ConnectionIO[Int] = def insertItemTags(item: Ident, tags: Seq[Ident]): ConnectionIO[Int] =
for { for {
tagValues <- tags.toList.traverse(id => tagValues <- tags.toList.traverse(id =>
Ident.randomId[ConnectionIO].map(rid => RTagItem(rid, item, id)) Ident.randomId[ConnectionIO].map(rid => RTagItem(rid, item, id))
) )
tagFrag = tagValues.map(v => fr"${v.tagItemId},${v.itemId},${v.tagId}") tagFrag = tagValues.map(v => fr"${v.tagItemId},${v.itemId},${v.tagId}")
ins <- insertRows(table, all, tagFrag).update.run ins <- insertRows(table, all, tagFrag).update.run
} yield ins } yield ins
} }