mirror of
https://github.com/TheAnachronism/docspell.git
synced 2025-06-04 06:05:59 +00:00
sbt scalafmtAll
This commit is contained in:
parent
4dbf75dd8f
commit
2f87065b2e
@ -12,13 +12,13 @@ object Contact {
|
|||||||
def annotate(text: String): Vector[NerLabel] =
|
def annotate(text: String): Vector[NerLabel] =
|
||||||
TextSplitter
|
TextSplitter
|
||||||
.splitToken[Nothing](text, " \t\r\n".toSet)
|
.splitToken[Nothing](text, " \t\r\n".toSet)
|
||||||
.map({ token =>
|
.map { token =>
|
||||||
if (isEmailAddress(token.value))
|
if (isEmailAddress(token.value))
|
||||||
NerLabel(token.value, NerTag.Email, token.begin, token.end).some
|
NerLabel(token.value, NerTag.Email, token.begin, token.end).some
|
||||||
else if (isWebsite(token.value))
|
else if (isWebsite(token.value))
|
||||||
NerLabel(token.value, NerTag.Website, token.begin, token.end).some
|
NerLabel(token.value, NerTag.Website, token.begin, token.end).some
|
||||||
else None
|
else None
|
||||||
})
|
}
|
||||||
.flatMap(_.map(Stream.emit).getOrElse(Stream.empty))
|
.flatMap(_.map(Stream.emit).getOrElse(Stream.empty))
|
||||||
.toVector
|
.toVector
|
||||||
|
|
||||||
|
@ -52,7 +52,7 @@ object AuthToken {
|
|||||||
|
|
||||||
def user[F[_]: Sync](accountId: AccountId, key: ByteVector): F[AuthToken] =
|
def user[F[_]: Sync](accountId: AccountId, key: ByteVector): F[AuthToken] =
|
||||||
for {
|
for {
|
||||||
salt <- Common.genSaltString[F]
|
salt <- Common.genSaltString[F]
|
||||||
millis = Instant.now.toEpochMilli
|
millis = Instant.now.toEpochMilli
|
||||||
cd = AuthToken(millis, accountId, salt, "")
|
cd = AuthToken(millis, accountId, salt, "")
|
||||||
sig = sign(cd, key)
|
sig = sign(cd, key)
|
||||||
|
@ -72,7 +72,7 @@ object Login {
|
|||||||
data <- store.transact(QLogin.findUser(acc))
|
data <- store.transact(QLogin.findUser(acc))
|
||||||
_ <- Sync[F].delay(logger.trace(s"Account lookup: $data"))
|
_ <- Sync[F].delay(logger.trace(s"Account lookup: $data"))
|
||||||
res <- if (data.exists(check(up.pass))) okResult
|
res <- if (data.exists(check(up.pass))) okResult
|
||||||
else Result.invalidAuth.pure[F]
|
else Result.invalidAuth.pure[F]
|
||||||
} yield res
|
} yield res
|
||||||
case Left(_) =>
|
case Left(_) =>
|
||||||
Result.invalidAuth.pure[F]
|
Result.invalidAuth.pure[F]
|
||||||
|
@ -111,10 +111,10 @@ object OCollective {
|
|||||||
): F[PassChangeResult] = {
|
): F[PassChangeResult] = {
|
||||||
val q = for {
|
val q = for {
|
||||||
optUser <- RUser.findByAccount(accountId)
|
optUser <- RUser.findByAccount(accountId)
|
||||||
check = optUser.map(_.password).map(p => PasswordCrypt.check(current, p))
|
check = optUser.map(_.password).map(p => PasswordCrypt.check(current, p))
|
||||||
n <- check
|
n <- check
|
||||||
.filter(identity)
|
.filter(identity)
|
||||||
.traverse(_ => RUser.updatePassword(accountId, PasswordCrypt.crypt(newPass)))
|
.traverse(_ => RUser.updatePassword(accountId, PasswordCrypt.crypt(newPass)))
|
||||||
res = check match {
|
res = check match {
|
||||||
case Some(true) =>
|
case Some(true) =>
|
||||||
if (n.getOrElse(0) > 0) PassChangeResult.success else PassChangeResult.updateFailed
|
if (n.getOrElse(0) > 0) PassChangeResult.success else PassChangeResult.updateFailed
|
||||||
|
@ -11,7 +11,14 @@ import docspell.store.queries.{QAttachment, QItem}
|
|||||||
import OItem.{AttachmentData, AttachmentSourceData, ItemData, ListItem, Query}
|
import OItem.{AttachmentData, AttachmentSourceData, ItemData, ListItem, Query}
|
||||||
import bitpeace.{FileMeta, RangeDef}
|
import bitpeace.{FileMeta, RangeDef}
|
||||||
import docspell.common.{Direction, Ident, ItemState, MetaProposalList, Timestamp}
|
import docspell.common.{Direction, Ident, ItemState, MetaProposalList, Timestamp}
|
||||||
import docspell.store.records.{RAttachment, RAttachmentMeta, RAttachmentSource, RItem, RSource, RTagItem}
|
import docspell.store.records.{
|
||||||
|
RAttachment,
|
||||||
|
RAttachmentMeta,
|
||||||
|
RAttachmentSource,
|
||||||
|
RItem,
|
||||||
|
RSource,
|
||||||
|
RTagItem
|
||||||
|
}
|
||||||
|
|
||||||
trait OItem[F[_]] {
|
trait OItem[F[_]] {
|
||||||
|
|
||||||
@ -75,14 +82,17 @@ object OItem {
|
|||||||
def fileId: Ident
|
def fileId: Ident
|
||||||
}
|
}
|
||||||
case class AttachmentData[F[_]](ra: RAttachment, meta: FileMeta, data: Stream[F, Byte])
|
case class AttachmentData[F[_]](ra: RAttachment, meta: FileMeta, data: Stream[F, Byte])
|
||||||
extends BinaryData[F] {
|
extends BinaryData[F] {
|
||||||
val name = ra.name
|
val name = ra.name
|
||||||
val fileId = ra.fileId
|
val fileId = ra.fileId
|
||||||
}
|
}
|
||||||
|
|
||||||
case class AttachmentSourceData[F[_]](rs: RAttachmentSource, meta: FileMeta, data: Stream[F, Byte])
|
case class AttachmentSourceData[F[_]](
|
||||||
extends BinaryData[F] {
|
rs: RAttachmentSource,
|
||||||
val name = rs.name
|
meta: FileMeta,
|
||||||
|
data: Stream[F, Byte]
|
||||||
|
) extends BinaryData[F] {
|
||||||
|
val name = rs.name
|
||||||
val fileId = rs.fileId
|
val fileId = rs.fileId
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -131,18 +141,22 @@ object OItem {
|
|||||||
|
|
||||||
private def makeBinaryData[A](fileId: Ident)(f: FileMeta => A): F[Option[A]] =
|
private def makeBinaryData[A](fileId: Ident)(f: FileMeta => A): F[Option[A]] =
|
||||||
store.bitpeace
|
store.bitpeace
|
||||||
.get(fileId.id).unNoneTerminate.compile.last.map(
|
.get(fileId.id)
|
||||||
_.map(m => f(m))
|
.unNoneTerminate
|
||||||
)
|
.compile
|
||||||
|
.last
|
||||||
|
.map(
|
||||||
|
_.map(m => f(m))
|
||||||
|
)
|
||||||
|
|
||||||
def setTags(item: Ident, tagIds: List[Ident], collective: Ident): F[AddResult] = {
|
def setTags(item: Ident, tagIds: List[Ident], collective: Ident): F[AddResult] = {
|
||||||
val db = for {
|
val db = for {
|
||||||
cid <- RItem.getCollective(item)
|
cid <- RItem.getCollective(item)
|
||||||
nd <- if (cid.contains(collective)) RTagItem.deleteItemTags(item)
|
nd <- if (cid.contains(collective)) RTagItem.deleteItemTags(item)
|
||||||
else 0.pure[ConnectionIO]
|
else 0.pure[ConnectionIO]
|
||||||
ni <- if (tagIds.nonEmpty && cid.contains(collective))
|
ni <- if (tagIds.nonEmpty && cid.contains(collective))
|
||||||
RTagItem.insertItemTags(item, tagIds)
|
RTagItem.insertItemTags(item, tagIds)
|
||||||
else 0.pure[ConnectionIO]
|
else 0.pure[ConnectionIO]
|
||||||
} yield nd + ni
|
} yield nd + ni
|
||||||
|
|
||||||
store.transact(db).attempt.map(AddResult.fromUpdate)
|
store.transact(db).attempt.map(AddResult.fromUpdate)
|
||||||
|
@ -61,9 +61,9 @@ object OJob {
|
|||||||
mustCancel(j.some).isEmpty
|
mustCancel(j.some).isEmpty
|
||||||
|
|
||||||
val tryDelete = for {
|
val tryDelete = for {
|
||||||
job <- RJob.findByIdAndGroup(id, collective)
|
job <- RJob.findByIdAndGroup(id, collective)
|
||||||
jobm = job.filter(canDelete)
|
jobm = job.filter(canDelete)
|
||||||
del <- jobm.traverse(j => RJob.delete(j.id))
|
del <- jobm.traverse(j => RJob.delete(j.id))
|
||||||
} yield del match {
|
} yield del match {
|
||||||
case Some(_) => Right(JobCancelResult.Removed: JobCancelResult)
|
case Some(_) => Right(JobCancelResult.Removed: JobCancelResult)
|
||||||
case None => Left(mustCancel(job))
|
case None => Left(mustCancel(job))
|
||||||
@ -77,12 +77,12 @@ object OJob {
|
|||||||
for {
|
for {
|
||||||
tryDel <- store.transact(tryDelete)
|
tryDel <- store.transact(tryDelete)
|
||||||
result <- tryDel match {
|
result <- tryDel match {
|
||||||
case Right(r) => r.pure[F]
|
case Right(r) => r.pure[F]
|
||||||
case Left(Some((job, worker))) =>
|
case Left(Some((job, worker))) =>
|
||||||
tryCancel(job, worker)
|
tryCancel(job, worker)
|
||||||
case Left(None) =>
|
case Left(None) =>
|
||||||
(JobCancelResult.JobNotFound: OJob.JobCancelResult).pure[F]
|
(JobCancelResult.JobNotFound: OJob.JobCancelResult).pure[F]
|
||||||
}
|
}
|
||||||
} yield result
|
} yield result
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
@ -113,10 +113,10 @@ object OMail {
|
|||||||
|
|
||||||
def createSettings(accId: AccountId, s: SmtpSettings): F[AddResult] =
|
def createSettings(accId: AccountId, s: SmtpSettings): F[AddResult] =
|
||||||
(for {
|
(for {
|
||||||
ru <- OptionT(store.transact(s.toRecord(accId).value))
|
ru <- OptionT(store.transact(s.toRecord(accId).value))
|
||||||
ins = RUserEmail.insert(ru)
|
ins = RUserEmail.insert(ru)
|
||||||
exists = RUserEmail.exists(ru.uid, ru.name)
|
exists = RUserEmail.exists(ru.uid, ru.name)
|
||||||
res <- OptionT.liftF(store.add(ins, exists))
|
res <- OptionT.liftF(store.add(ins, exists))
|
||||||
} yield res).getOrElse(AddResult.Failure(new Exception("User not found")))
|
} yield res).getOrElse(AddResult.Failure(new Exception("User not found")))
|
||||||
|
|
||||||
def updateSettings(accId: AccountId, name: Ident, data: SmtpSettings): F[Int] = {
|
def updateSettings(accId: AccountId, name: Ident, data: SmtpSettings): F[Int] = {
|
||||||
@ -143,10 +143,10 @@ object OMail {
|
|||||||
for {
|
for {
|
||||||
_ <- OptionT.liftF(store.transact(RItem.existsById(m.item))).filter(identity)
|
_ <- OptionT.liftF(store.transact(RItem.existsById(m.item))).filter(identity)
|
||||||
ras <- OptionT.liftF(
|
ras <- OptionT.liftF(
|
||||||
store.transact(
|
store.transact(
|
||||||
RAttachment.findByItemAndCollectiveWithMeta(m.item, accId.collective)
|
RAttachment.findByItemAndCollectiveWithMeta(m.item, accId.collective)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
} yield {
|
} yield {
|
||||||
val addAttach = m.attach.filter(ras).map { a =>
|
val addAttach = m.attach.filter(ras).map { a =>
|
||||||
Attach[F](Stream.emit(a._2).through(store.bitpeace.fetchData2(RangeDef.all)))
|
Attach[F](Stream.emit(a._2).through(store.bitpeace.fetchData2(RangeDef.all)))
|
||||||
@ -171,15 +171,15 @@ object OMail {
|
|||||||
def storeMail(msgId: String, cfg: RUserEmail): F[Either[SendResult, Ident]] = {
|
def storeMail(msgId: String, cfg: RUserEmail): F[Either[SendResult, Ident]] = {
|
||||||
val save = for {
|
val save = for {
|
||||||
data <- RSentMail.forItem(
|
data <- RSentMail.forItem(
|
||||||
m.item,
|
m.item,
|
||||||
accId,
|
accId,
|
||||||
msgId,
|
msgId,
|
||||||
cfg.mailFrom,
|
cfg.mailFrom,
|
||||||
name,
|
name,
|
||||||
m.subject,
|
m.subject,
|
||||||
m.recipients,
|
m.recipients,
|
||||||
m.body
|
m.body
|
||||||
)
|
)
|
||||||
_ <- OptionT.liftF(RSentMail.insert(data._1))
|
_ <- OptionT.liftF(RSentMail.insert(data._1))
|
||||||
_ <- OptionT.liftF(RSentMailItem.insert(data._2))
|
_ <- OptionT.liftF(RSentMailItem.insert(data._2))
|
||||||
} yield data._1.id
|
} yield data._1.id
|
||||||
@ -197,7 +197,7 @@ object OMail {
|
|||||||
mail <- createMail(mailCfg)
|
mail <- createMail(mailCfg)
|
||||||
mid <- OptionT.liftF(sendMail(mailCfg.toMailConfig, mail))
|
mid <- OptionT.liftF(sendMail(mailCfg.toMailConfig, mail))
|
||||||
res <- mid.traverse(id => OptionT.liftF(storeMail(id, mailCfg)))
|
res <- mid.traverse(id => OptionT.liftF(storeMail(id, mailCfg)))
|
||||||
conv = res.fold(identity, _.fold(identity, id => SendResult.Success(id)))
|
conv = res.fold(identity, _.fold(identity, id => SendResult.Success(id)))
|
||||||
} yield conv).getOrElse(SendResult.NotFound)
|
} yield conv).getOrElse(SendResult.NotFound)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -81,7 +81,7 @@ object OUpload {
|
|||||||
|
|
||||||
def submit(data: OUpload.UploadData[F], sourceId: Ident): F[OUpload.UploadResult] =
|
def submit(data: OUpload.UploadData[F], sourceId: Ident): F[OUpload.UploadResult] =
|
||||||
for {
|
for {
|
||||||
sOpt <- store.transact(RSource.find(sourceId)).map(_.toRight(UploadResult.NoSource))
|
sOpt <- store.transact(RSource.find(sourceId)).map(_.toRight(UploadResult.NoSource))
|
||||||
abbrev = sOpt.map(_.abbrev).toOption.getOrElse(data.meta.sourceAbbrev)
|
abbrev = sOpt.map(_.abbrev).toOption.getOrElse(data.meta.sourceAbbrev)
|
||||||
updata = data.copy(meta = data.meta.copy(sourceAbbrev = abbrev))
|
updata = data.copy(meta = data.meta.copy(sourceAbbrev = abbrev))
|
||||||
accId = sOpt.map(source => AccountId(source.cid, source.sid))
|
accId = sOpt.map(source => AccountId(source.cid, source.sid))
|
||||||
@ -131,8 +131,8 @@ object OUpload {
|
|||||||
)
|
)
|
||||||
|
|
||||||
for {
|
for {
|
||||||
id <- Ident.randomId[F]
|
id <- Ident.randomId[F]
|
||||||
now <- Timestamp.current[F]
|
now <- Timestamp.current[F]
|
||||||
jobs = args.map(a => create(id, now, a))
|
jobs = args.map(a => create(id, now, a))
|
||||||
} yield jobs
|
} yield jobs
|
||||||
|
|
||||||
|
@ -47,15 +47,16 @@ object OSignup {
|
|||||||
for {
|
for {
|
||||||
now <- Timestamp.current[F]
|
now <- Timestamp.current[F]
|
||||||
min = now.minus(cfg.inviteTime)
|
min = now.minus(cfg.inviteTime)
|
||||||
ok <- store.transact(RInvitation.useInvite(inv, min))
|
ok <- store.transact(RInvitation.useInvite(inv, min))
|
||||||
res <- if (ok) addUser(data).map(SignupResult.fromAddResult)
|
res <- if (ok) addUser(data).map(SignupResult.fromAddResult)
|
||||||
else SignupResult.invalidInvitationKey.pure[F]
|
else SignupResult.invalidInvitationKey.pure[F]
|
||||||
_ <- if (retryInvite(res))
|
_ <- if (retryInvite(res))
|
||||||
logger.fdebug(s"Adding account failed ($res). Allow retry with invite.") *> store
|
logger
|
||||||
.transact(
|
.fdebug(s"Adding account failed ($res). Allow retry with invite.") *> store
|
||||||
RInvitation.insert(RInvitation(inv, now))
|
.transact(
|
||||||
)
|
RInvitation.insert(RInvitation(inv, now))
|
||||||
else 0.pure[F]
|
)
|
||||||
|
else 0.pure[F]
|
||||||
} yield res
|
} yield res
|
||||||
case None =>
|
case None =>
|
||||||
SignupResult.invalidInvitationKey.pure[F]
|
SignupResult.invalidInvitationKey.pure[F]
|
||||||
@ -81,7 +82,7 @@ object OSignup {
|
|||||||
for {
|
for {
|
||||||
id2 <- Ident.randomId[F]
|
id2 <- Ident.randomId[F]
|
||||||
now <- Timestamp.current[F]
|
now <- Timestamp.current[F]
|
||||||
c = RCollective(data.collName, CollectiveState.Active, Language.German, now)
|
c = RCollective(data.collName, CollectiveState.Active, Language.German, now)
|
||||||
u = RUser(
|
u = RUser(
|
||||||
id2,
|
id2,
|
||||||
data.login,
|
data.login,
|
||||||
|
@ -26,9 +26,7 @@ object AccountId {
|
|||||||
invalid
|
invalid
|
||||||
}
|
}
|
||||||
|
|
||||||
val separated = sepearatorChars.foldRight(invalid) { (c, v) =>
|
val separated = sepearatorChars.foldRight(invalid)((c, v) => v.orElse(parse0(c)))
|
||||||
v.orElse(parse0(c))
|
|
||||||
}
|
|
||||||
|
|
||||||
separated.orElse(Ident.fromString(str).map(id => AccountId(id, id)))
|
separated.orElse(Ident.fromString(str).map(id => AccountId(id, id)))
|
||||||
}
|
}
|
||||||
|
@ -1,8 +1,6 @@
|
|||||||
package docspell.common
|
package docspell.common
|
||||||
|
|
||||||
sealed trait DataType {
|
sealed trait DataType {}
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
object DataType {
|
object DataType {
|
||||||
|
|
||||||
@ -10,7 +8,6 @@ object DataType {
|
|||||||
|
|
||||||
case class Hint(hint: MimeTypeHint) extends DataType
|
case class Hint(hint: MimeTypeHint) extends DataType
|
||||||
|
|
||||||
|
|
||||||
def apply(mt: MimeType): DataType =
|
def apply(mt: MimeType): DataType =
|
||||||
Exact(mt)
|
Exact(mt)
|
||||||
|
|
||||||
|
@ -65,11 +65,13 @@ object File {
|
|||||||
javaList.asScala.toList.sortBy(_.getFileName.toString)
|
javaList.asScala.toList.sortBy(_.getFileName.toString)
|
||||||
}
|
}
|
||||||
|
|
||||||
def readAll[F[_]: Sync: ContextShift](file: Path, blocker: Blocker, chunkSize: Int): Stream[F, Byte] =
|
def readAll[F[_]: Sync: ContextShift](
|
||||||
|
file: Path,
|
||||||
|
blocker: Blocker,
|
||||||
|
chunkSize: Int
|
||||||
|
): Stream[F, Byte] =
|
||||||
fs2.io.file.readAll(file, blocker, chunkSize)
|
fs2.io.file.readAll(file, blocker, chunkSize)
|
||||||
|
|
||||||
def readText[F[_]: Sync: ContextShift](file: Path, blocker: Blocker): F[String] =
|
def readText[F[_]: Sync: ContextShift](file: Path, blocker: Blocker): F[String] =
|
||||||
readAll[F](file, blocker, 8192).
|
readAll[F](file, blocker, 8192).through(fs2.text.utf8Decode).compile.foldMonoid
|
||||||
through(fs2.text.utf8Decode).
|
|
||||||
compile.foldMonoid
|
|
||||||
}
|
}
|
||||||
|
@ -66,9 +66,7 @@ case class LenientUri(
|
|||||||
)
|
)
|
||||||
|
|
||||||
def readText[F[_]: Sync: ContextShift](chunkSize: Int, blocker: Blocker): F[String] =
|
def readText[F[_]: Sync: ContextShift](chunkSize: Int, blocker: Blocker): F[String] =
|
||||||
readURL[F](chunkSize, blocker).
|
readURL[F](chunkSize, blocker).through(fs2.text.utf8Decode).compile.foldMonoid
|
||||||
through(fs2.text.utf8Decode).
|
|
||||||
compile.foldMonoid
|
|
||||||
|
|
||||||
def host: Option[String] =
|
def host: Option[String] =
|
||||||
authority.map(a =>
|
authority.map(a =>
|
||||||
|
@ -17,7 +17,6 @@ trait Logger[F[_]] {
|
|||||||
|
|
||||||
object Logger {
|
object Logger {
|
||||||
|
|
||||||
|
|
||||||
def log4s[F[_]: Sync](log: Log4sLogger): Logger[F] = new Logger[F] {
|
def log4s[F[_]: Sync](log: Log4sLogger): Logger[F] = new Logger[F] {
|
||||||
def trace(msg: => String): F[Unit] =
|
def trace(msg: => String): F[Unit] =
|
||||||
log.ftrace(msg)
|
log.ftrace(msg)
|
||||||
@ -38,4 +37,4 @@ object Logger {
|
|||||||
log.ferror(msg)
|
log.ferror(msg)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -66,9 +66,7 @@ object MetaProposalList {
|
|||||||
case None => map.updated(mp.proposalType, mp)
|
case None => map.updated(mp.proposalType, mp)
|
||||||
}
|
}
|
||||||
|
|
||||||
val merged = ml.foldLeft(init) { (map, el) =>
|
val merged = ml.foldLeft(init)((map, el) => el.proposals.foldLeft(map)(updateMap))
|
||||||
el.proposals.foldLeft(map)(updateMap)
|
|
||||||
}
|
|
||||||
|
|
||||||
fromMap(merged)
|
fromMap(merged)
|
||||||
}
|
}
|
||||||
|
@ -23,7 +23,8 @@ object SystemCommand {
|
|||||||
repl.foldLeft(s) {
|
repl.foldLeft(s) {
|
||||||
case (res, (k, v)) =>
|
case (res, (k, v)) =>
|
||||||
res.replace(k, v)
|
res.replace(k, v)
|
||||||
})
|
}
|
||||||
|
)
|
||||||
|
|
||||||
def toCmd: List[String] =
|
def toCmd: List[String] =
|
||||||
program :: args.toList
|
program :: args.toList
|
||||||
@ -47,10 +48,10 @@ object SystemCommand {
|
|||||||
_ <- writeToProcess(stdin, proc, blocker)
|
_ <- writeToProcess(stdin, proc, blocker)
|
||||||
term <- Sync[F].delay(proc.waitFor(cmd.timeout.seconds, TimeUnit.SECONDS))
|
term <- Sync[F].delay(proc.waitFor(cmd.timeout.seconds, TimeUnit.SECONDS))
|
||||||
_ <- if (term) logger.debug(s"Command `${cmd.cmdString}` finished: ${proc.exitValue}")
|
_ <- if (term) logger.debug(s"Command `${cmd.cmdString}` finished: ${proc.exitValue}")
|
||||||
else
|
else
|
||||||
logger.warn(
|
logger.warn(
|
||||||
s"Command `${cmd.cmdString}` did not finish in ${cmd.timeout.formatExact}!"
|
s"Command `${cmd.cmdString}` did not finish in ${cmd.timeout.formatExact}!"
|
||||||
)
|
)
|
||||||
_ <- if (!term) timeoutError(proc, cmd) else Sync[F].pure(())
|
_ <- if (!term) timeoutError(proc, cmd) else Sync[F].pure(())
|
||||||
out <- if (term) inputStreamToString(proc.getInputStream, blocker) else Sync[F].pure("")
|
out <- if (term) inputStreamToString(proc.getInputStream, blocker) else Sync[F].pure("")
|
||||||
err <- if (term) inputStreamToString(proc.getErrorStream, blocker) else Sync[F].pure("")
|
err <- if (term) inputStreamToString(proc.getErrorStream, blocker) else Sync[F].pure("")
|
||||||
@ -75,25 +76,30 @@ object SystemCommand {
|
|||||||
else Stream.emit(r)
|
else Stream.emit(r)
|
||||||
}
|
}
|
||||||
|
|
||||||
private def startProcess[F[_]: Sync, A](cmd: Config, wd: Option[Path], logger: Logger[F], stdin: Stream[F, Byte])(
|
private def startProcess[F[_]: Sync, A](
|
||||||
|
cmd: Config,
|
||||||
|
wd: Option[Path],
|
||||||
|
logger: Logger[F],
|
||||||
|
stdin: Stream[F, Byte]
|
||||||
|
)(
|
||||||
f: Process => Stream[F, A]
|
f: Process => Stream[F, A]
|
||||||
): Stream[F, A] = {
|
): Stream[F, A] = {
|
||||||
val log = logger.debug(s"Running external command: ${cmd.cmdString}")
|
val log = logger.debug(s"Running external command: ${cmd.cmdString}")
|
||||||
val hasStdin = stdin.take(1).compile.last.map(_.isDefined)
|
val hasStdin = stdin.take(1).compile.last.map(_.isDefined)
|
||||||
val proc = log *> hasStdin.flatMap(flag => Sync[F].delay {
|
val proc = log *> hasStdin.flatMap(flag =>
|
||||||
val pb = new ProcessBuilder(cmd.toCmd.asJava)
|
Sync[F].delay {
|
||||||
.redirectInput(if (flag) Redirect.PIPE else Redirect.INHERIT)
|
val pb = new ProcessBuilder(cmd.toCmd.asJava)
|
||||||
.redirectError(Redirect.PIPE)
|
.redirectInput(if (flag) Redirect.PIPE else Redirect.INHERIT)
|
||||||
.redirectOutput(Redirect.PIPE)
|
.redirectError(Redirect.PIPE)
|
||||||
|
.redirectOutput(Redirect.PIPE)
|
||||||
|
|
||||||
wd.map(_.toFile).foreach(pb.directory)
|
wd.map(_.toFile).foreach(pb.directory)
|
||||||
pb.start()
|
pb.start()
|
||||||
})
|
}
|
||||||
|
)
|
||||||
Stream
|
Stream
|
||||||
.bracket(proc)(p =>
|
.bracket(proc)(p =>
|
||||||
logger.debug(s"Closing process: `${cmd.cmdString}`").map { _ =>
|
logger.debug(s"Closing process: `${cmd.cmdString}`").map(_ => p.destroy())
|
||||||
p.destroy()
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
.flatMap(f)
|
.flatMap(f)
|
||||||
}
|
}
|
||||||
|
@ -17,8 +17,8 @@ trait StreamSyntax {
|
|||||||
.map(optStr =>
|
.map(optStr =>
|
||||||
for {
|
for {
|
||||||
str <- optStr
|
str <- optStr
|
||||||
.map(_.trim)
|
.map(_.trim)
|
||||||
.toRight(new Exception("Empty string cannot be parsed into a value"))
|
.toRight(new Exception("Empty string cannot be parsed into a value"))
|
||||||
json <- parse(str).leftMap(_.underlying)
|
json <- parse(str).leftMap(_.underlying)
|
||||||
value <- json.as[A]
|
value <- json.as[A]
|
||||||
} yield value
|
} yield value
|
||||||
|
@ -13,7 +13,9 @@ import docspell.files.{ImageSize, TikaMimetype}
|
|||||||
|
|
||||||
trait Conversion[F[_]] {
|
trait Conversion[F[_]] {
|
||||||
|
|
||||||
def toPDF[A](dataType: DataType, lang: Language, handler: Handler[F, A])(in: Stream[F, Byte]): F[A]
|
def toPDF[A](dataType: DataType, lang: Language, handler: Handler[F, A])(
|
||||||
|
in: Stream[F, Byte]
|
||||||
|
): F[A]
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -26,7 +28,9 @@ object Conversion {
|
|||||||
): Resource[F, Conversion[F]] =
|
): Resource[F, Conversion[F]] =
|
||||||
Resource.pure(new Conversion[F] {
|
Resource.pure(new Conversion[F] {
|
||||||
|
|
||||||
def toPDF[A](dataType: DataType, lang: Language, handler: Handler[F, A])(in: Stream[F, Byte]): F[A] =
|
def toPDF[A](dataType: DataType, lang: Language, handler: Handler[F, A])(
|
||||||
|
in: Stream[F, Byte]
|
||||||
|
): F[A] =
|
||||||
TikaMimetype.resolve(dataType, in).flatMap {
|
TikaMimetype.resolve(dataType, in).flatMap {
|
||||||
case MimeType.pdf =>
|
case MimeType.pdf =>
|
||||||
handler.run(ConversionResult.successPdf(in))
|
handler.run(ConversionResult.successPdf(in))
|
||||||
@ -112,10 +116,10 @@ object Conversion {
|
|||||||
|
|
||||||
def unapply(mt: MimeType): Option[MimeType] =
|
def unapply(mt: MimeType): Option[MimeType] =
|
||||||
mt match {
|
mt match {
|
||||||
case Office(_) => Some(mt)
|
case Office(_) => Some(mt)
|
||||||
case Texts(_) => Some(mt)
|
case Texts(_) => Some(mt)
|
||||||
case Images(_) => Some(mt)
|
case Images(_) => Some(mt)
|
||||||
case MimeType.html => Some(mt)
|
case MimeType.html => Some(mt)
|
||||||
case _ => None
|
case _ => None
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -3,9 +3,11 @@ package docspell.convert
|
|||||||
import docspell.convert.extern.{TesseractConfig, UnoconvConfig, WkHtmlPdfConfig}
|
import docspell.convert.extern.{TesseractConfig, UnoconvConfig, WkHtmlPdfConfig}
|
||||||
import docspell.convert.flexmark.MarkdownConfig
|
import docspell.convert.flexmark.MarkdownConfig
|
||||||
|
|
||||||
case class ConvertConfig(chunkSize: Int,
|
case class ConvertConfig(
|
||||||
maxImageSize: Int,
|
chunkSize: Int,
|
||||||
markdown: MarkdownConfig,
|
maxImageSize: Int,
|
||||||
wkhtmlpdf: WkHtmlPdfConfig,
|
markdown: MarkdownConfig,
|
||||||
tesseract: TesseractConfig,
|
wkhtmlpdf: WkHtmlPdfConfig,
|
||||||
unoconv: UnoconvConfig)
|
tesseract: TesseractConfig,
|
||||||
|
unoconv: UnoconvConfig
|
||||||
|
)
|
||||||
|
@ -20,7 +20,9 @@ private[extern] object ExternConv {
|
|||||||
logger: Logger[F],
|
logger: Logger[F],
|
||||||
reader: (Path, SystemCommand.Result) => F[ConversionResult[F]]
|
reader: (Path, SystemCommand.Result) => F[ConversionResult[F]]
|
||||||
)(in: Stream[F, Byte], handler: Handler[F, A]): F[A] =
|
)(in: Stream[F, Byte], handler: Handler[F, A]): F[A] =
|
||||||
Stream.resource(File.withTempDir[F](wd, s"docspell-$name")).flatMap { dir =>
|
Stream
|
||||||
|
.resource(File.withTempDir[F](wd, s"docspell-$name"))
|
||||||
|
.flatMap { dir =>
|
||||||
val inFile = dir.resolve("infile").toAbsolutePath.normalize
|
val inFile = dir.resolve("infile").toAbsolutePath.normalize
|
||||||
val out = dir.resolve("out.pdf").toAbsolutePath.normalize
|
val out = dir.resolve("out.pdf").toAbsolutePath.normalize
|
||||||
val sysCfg =
|
val sysCfg =
|
||||||
@ -40,12 +42,12 @@ private[extern] object ExternConv {
|
|||||||
SystemCommand
|
SystemCommand
|
||||||
.execSuccess[F](sysCfg, blocker, logger, Some(dir), if (useStdin) in else Stream.empty)
|
.execSuccess[F](sysCfg, blocker, logger, Some(dir), if (useStdin) in else Stream.empty)
|
||||||
.evalMap(result =>
|
.evalMap(result =>
|
||||||
logResult(name, result, logger).
|
logResult(name, result, logger).flatMap(_ => reader(out, result)).flatMap(handler.run)
|
||||||
flatMap(_ => reader(out, result)).
|
|
||||||
flatMap(handler.run)
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}.compile.lastOrError
|
}
|
||||||
|
.compile
|
||||||
|
.lastOrError
|
||||||
|
|
||||||
def readResult[F[_]: Sync: ContextShift](
|
def readResult[F[_]: Sync: ContextShift](
|
||||||
blocker: Blocker,
|
blocker: Blocker,
|
||||||
@ -60,9 +62,11 @@ private[extern] object ExternConv {
|
|||||||
successPdf(File.readAll(out, blocker, chunkSize)).pure[F]
|
successPdf(File.readAll(out, blocker, chunkSize)).pure[F]
|
||||||
|
|
||||||
case false =>
|
case false =>
|
||||||
ConversionResult.failure[F](
|
ConversionResult
|
||||||
new Exception(s"Command result=${result.rc}. No output file found.")
|
.failure[F](
|
||||||
).pure[F]
|
new Exception(s"Command result=${result.rc}. No output file found.")
|
||||||
|
)
|
||||||
|
.pure[F]
|
||||||
}
|
}
|
||||||
|
|
||||||
def readResultTesseract[F[_]: Sync: ContextShift](
|
def readResultTesseract[F[_]: Sync: ContextShift](
|
||||||
@ -75,7 +79,7 @@ private[extern] object ExternConv {
|
|||||||
File.existsNonEmpty[F](outPdf).flatMap {
|
File.existsNonEmpty[F](outPdf).flatMap {
|
||||||
case true =>
|
case true =>
|
||||||
val outTxt = out.resolveSibling(s"$outPrefix.txt")
|
val outTxt = out.resolveSibling(s"$outPrefix.txt")
|
||||||
File.exists(outTxt).flatMap(txtExists => {
|
File.exists(outTxt).flatMap { txtExists =>
|
||||||
val pdfData = File.readAll(out, blocker, chunkSize)
|
val pdfData = File.readAll(out, blocker, chunkSize)
|
||||||
if (result.rc == 0) {
|
if (result.rc == 0) {
|
||||||
if (txtExists) successPdfTxt(pdfData, File.readText(outTxt, blocker)).pure[F]
|
if (txtExists) successPdfTxt(pdfData, File.readText(outTxt, blocker)).pure[F]
|
||||||
@ -84,12 +88,14 @@ private[extern] object ExternConv {
|
|||||||
logger.warn(s"Command not successful (rc=${result.rc}), but file exists.") *>
|
logger.warn(s"Command not successful (rc=${result.rc}), but file exists.") *>
|
||||||
successPdf(pdfData).pure[F]
|
successPdf(pdfData).pure[F]
|
||||||
}
|
}
|
||||||
})
|
}
|
||||||
|
|
||||||
case false =>
|
case false =>
|
||||||
ConversionResult.failure[F](
|
ConversionResult
|
||||||
new Exception(s"Command result=${result.rc}. No output file found.")
|
.failure[F](
|
||||||
).pure[F]
|
new Exception(s"Command result=${result.rc}. No output file found.")
|
||||||
|
)
|
||||||
|
.pure[F]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -21,7 +21,15 @@ object Tesseract {
|
|||||||
val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
|
val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
|
||||||
ExternConv.readResultTesseract[F](outBase, blocker, chunkSize, logger)
|
ExternConv.readResultTesseract[F](outBase, blocker, chunkSize, logger)
|
||||||
|
|
||||||
ExternConv.toPDF[F, A]("tesseract", cfg.command.replace(Map("{{lang}}" -> lang.iso3)), cfg.workingDir, false, blocker, logger, reader)(in, handler)
|
ExternConv.toPDF[F, A](
|
||||||
|
"tesseract",
|
||||||
|
cfg.command.replace(Map("{{lang}}" -> lang.iso3)),
|
||||||
|
cfg.workingDir,
|
||||||
|
false,
|
||||||
|
blocker,
|
||||||
|
logger,
|
||||||
|
reader
|
||||||
|
)(in, handler)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -4,4 +4,4 @@ import java.nio.file.Path
|
|||||||
|
|
||||||
import docspell.common.SystemCommand
|
import docspell.common.SystemCommand
|
||||||
|
|
||||||
case class TesseractConfig (command: SystemCommand.Config, workingDir: Path)
|
case class TesseractConfig(command: SystemCommand.Config, workingDir: Path)
|
||||||
|
@ -19,7 +19,10 @@ object Unoconv {
|
|||||||
val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
|
val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
|
||||||
ExternConv.readResult[F](blocker, chunkSize, logger)
|
ExternConv.readResult[F](blocker, chunkSize, logger)
|
||||||
|
|
||||||
ExternConv.toPDF[F, A]("unoconv", cfg.command, cfg.workingDir, false, blocker, logger, reader)(in, handler)
|
ExternConv.toPDF[F, A]("unoconv", cfg.command, cfg.workingDir, false, blocker, logger, reader)(
|
||||||
|
in,
|
||||||
|
handler
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -4,4 +4,4 @@ import java.nio.file.Path
|
|||||||
|
|
||||||
import docspell.common.SystemCommand
|
import docspell.common.SystemCommand
|
||||||
|
|
||||||
case class UnoconvConfig (command: SystemCommand.Config, workingDir: Path)
|
case class UnoconvConfig(command: SystemCommand.Config, workingDir: Path)
|
||||||
|
@ -14,12 +14,16 @@ object WkHtmlPdf {
|
|||||||
cfg: WkHtmlPdfConfig,
|
cfg: WkHtmlPdfConfig,
|
||||||
chunkSize: Int,
|
chunkSize: Int,
|
||||||
blocker: Blocker,
|
blocker: Blocker,
|
||||||
logger: Logger[F],
|
logger: Logger[F]
|
||||||
)(in: Stream[F, Byte], handler: Handler[F, A]): F[A] = {
|
)(in: Stream[F, Byte], handler: Handler[F, A]): F[A] = {
|
||||||
val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
|
val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
|
||||||
ExternConv.readResult[F](blocker, chunkSize, logger)
|
ExternConv.readResult[F](blocker, chunkSize, logger)
|
||||||
|
|
||||||
ExternConv.toPDF[F, A]("wkhtmltopdf", cfg.command, cfg.workingDir, true, blocker, logger, reader)(in, handler)
|
ExternConv
|
||||||
|
.toPDF[F, A]("wkhtmltopdf", cfg.command, cfg.workingDir, true, blocker, logger, reader)(
|
||||||
|
in,
|
||||||
|
handler
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -4,4 +4,4 @@ import java.nio.file.Path
|
|||||||
|
|
||||||
import docspell.common.SystemCommand
|
import docspell.common.SystemCommand
|
||||||
|
|
||||||
case class WkHtmlPdfConfig (command: SystemCommand.Config, workingDir: Path)
|
case class WkHtmlPdfConfig(command: SystemCommand.Config, workingDir: Path)
|
||||||
|
@ -22,24 +22,22 @@ object Markdown {
|
|||||||
val r = createRenderer()
|
val r = createRenderer()
|
||||||
Try {
|
Try {
|
||||||
val reader = new InputStreamReader(is, StandardCharsets.UTF_8)
|
val reader = new InputStreamReader(is, StandardCharsets.UTF_8)
|
||||||
val doc = p.parseReader(reader)
|
val doc = p.parseReader(reader)
|
||||||
wrapHtml(r.render(doc), cfg)
|
wrapHtml(r.render(doc), cfg)
|
||||||
}.toEither
|
}.toEither
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def toHtml(md: String, cfg: MarkdownConfig): String = {
|
def toHtml(md: String, cfg: MarkdownConfig): String = {
|
||||||
val p = createParser()
|
val p = createParser()
|
||||||
val r = createRenderer()
|
val r = createRenderer()
|
||||||
val doc = p.parse(md)
|
val doc = p.parse(md)
|
||||||
wrapHtml(r.render(doc), cfg)
|
wrapHtml(r.render(doc), cfg)
|
||||||
}
|
}
|
||||||
|
|
||||||
def toHtml[F[_]: Sync](data: Stream[F, Byte], cfg: MarkdownConfig): F[String] =
|
def toHtml[F[_]: Sync](data: Stream[F, Byte], cfg: MarkdownConfig): F[String] =
|
||||||
data.through(fs2.text.utf8Decode).compile.foldMonoid.
|
data.through(fs2.text.utf8Decode).compile.foldMonoid.map(str => toHtml(str, cfg))
|
||||||
map(str => toHtml(str, cfg))
|
|
||||||
|
|
||||||
private def wrapHtml(body: String, cfg: MarkdownConfig): String = {
|
private def wrapHtml(body: String, cfg: MarkdownConfig): String =
|
||||||
s"""<!DOCTYPE html>
|
s"""<!DOCTYPE html>
|
||||||
|<html>
|
|<html>
|
||||||
|<head>
|
|<head>
|
||||||
@ -53,13 +51,13 @@ object Markdown {
|
|||||||
|</body>
|
|</body>
|
||||||
|</html>
|
|</html>
|
||||||
|""".stripMargin
|
|""".stripMargin
|
||||||
}
|
|
||||||
|
|
||||||
private def createParser(): Parser = {
|
private def createParser(): Parser = {
|
||||||
val opts = new MutableDataSet()
|
val opts = new MutableDataSet()
|
||||||
opts.set(Parser.EXTENSIONS.asInstanceOf[DataKey[util.Collection[_]]],
|
opts.set(
|
||||||
util.Arrays.asList(TablesExtension.create(),
|
Parser.EXTENSIONS.asInstanceOf[DataKey[util.Collection[_]]],
|
||||||
StrikethroughExtension.create()));
|
util.Arrays.asList(TablesExtension.create(), StrikethroughExtension.create())
|
||||||
|
);
|
||||||
|
|
||||||
Parser.builder(opts).build()
|
Parser.builder(opts).build()
|
||||||
}
|
}
|
||||||
|
@ -55,5 +55,4 @@ trait FileChecks {
|
|||||||
def commandExists(cmd: String): Boolean =
|
def commandExists(cmd: String): Boolean =
|
||||||
Runtime.getRuntime.exec(Array("which", cmd)).waitFor() == 0
|
Runtime.getRuntime.exec(Array("which", cmd)).waitFor() == 0
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -103,5 +103,4 @@ object ExternConvTest extends SimpleTestSuite with FileChecks {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -29,7 +29,7 @@ object Extraction {
|
|||||||
data: Stream[F, Byte],
|
data: Stream[F, Byte],
|
||||||
dataType: DataType,
|
dataType: DataType,
|
||||||
lang: Language
|
lang: Language
|
||||||
): F[ExtractResult] = {
|
): F[ExtractResult] =
|
||||||
TikaMimetype.resolve(dataType, data).flatMap {
|
TikaMimetype.resolve(dataType, data).flatMap {
|
||||||
case MimeType.pdf =>
|
case MimeType.pdf =>
|
||||||
PdfExtract
|
PdfExtract
|
||||||
@ -50,39 +50,46 @@ object Extraction {
|
|||||||
.extractOCR(data, blocker, logger, lang.iso3, cfg.ocr)
|
.extractOCR(data, blocker, logger, lang.iso3, cfg.ocr)
|
||||||
.compile
|
.compile
|
||||||
.lastOrError
|
.lastOrError
|
||||||
|
.map(_.trim)
|
||||||
.attempt
|
.attempt
|
||||||
.map(ExtractResult.fromEither)
|
.map(ExtractResult.fromEither)
|
||||||
|
|
||||||
ImageSize.get(data).flatMap {
|
ImageSize.get(data).flatMap {
|
||||||
case Some(dim) =>
|
case Some(dim) =>
|
||||||
if (dim.product > cfg.ocr.maxImageSize) {
|
if (dim.product > cfg.ocr.maxImageSize) {
|
||||||
logger.info(s"Image size (${dim.product}) is too large (max ${cfg.ocr.maxImageSize}).") *>
|
logger.info(
|
||||||
ExtractResult.failure(new Exception(
|
s"Image size (${dim.product}) is too large (max ${cfg.ocr.maxImageSize})."
|
||||||
s"Image size (${dim.width}x${dim.height}) is too large (max ${cfg.ocr.maxImageSize}).")
|
) *>
|
||||||
).pure[F]
|
ExtractResult
|
||||||
|
.failure(
|
||||||
|
new Exception(
|
||||||
|
s"Image size (${dim.width}x${dim.height}) is too large (max ${cfg.ocr.maxImageSize})."
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.pure[F]
|
||||||
} else {
|
} else {
|
||||||
doExtract
|
doExtract
|
||||||
}
|
}
|
||||||
case None =>
|
case None =>
|
||||||
logger.info(s"Cannot read image data from ${mt.asString}. Extracting anyways.") *>
|
logger.info(s"Cannot read image data from ${mt.asString}. Extracting anyways.") *>
|
||||||
doExtract
|
doExtract
|
||||||
}
|
}
|
||||||
|
|
||||||
case OdfType.container =>
|
case OdfType.container =>
|
||||||
logger.info(s"File detected as ${OdfType.container}. Try to read as OpenDocument file.") *>
|
logger
|
||||||
|
.info(s"File detected as ${OdfType.container}. Try to read as OpenDocument file.") *>
|
||||||
OdfExtract.get(data).map(ExtractResult.fromEither)
|
OdfExtract.get(data).map(ExtractResult.fromEither)
|
||||||
|
|
||||||
case mt@MimeType("text", sub) if !sub.contains("html") =>
|
case mt @ MimeType("text", sub) if !sub.contains("html") =>
|
||||||
logger.info(s"File detected as ${mt.asString}. Returning itself as text.") *>
|
logger.info(s"File detected as ${mt.asString}. Returning itself as text.") *>
|
||||||
data.through(fs2.text.utf8Decode).compile.last.map { txt =>
|
data.through(fs2.text.utf8Decode).compile.last.map { txt =>
|
||||||
ExtractResult.success(txt.getOrElse("").trim)
|
ExtractResult.success(txt.getOrElse("").trim)
|
||||||
}
|
}
|
||||||
|
|
||||||
case mt =>
|
case mt =>
|
||||||
ExtractResult.unsupportedFormat(mt).pure[F]
|
ExtractResult.unsupportedFormat(mt).pure[F]
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -1,3 +1,3 @@
|
|||||||
package docspell.extract
|
package docspell.extract
|
||||||
|
|
||||||
case class PdfConfig (minTextLen: Int)
|
case class PdfConfig(minTextLen: Int)
|
||||||
|
@ -33,7 +33,8 @@ object PdfExtract {
|
|||||||
|
|
||||||
//maybe better: inspect the pdf and decide whether ocr or not
|
//maybe better: inspect the pdf and decide whether ocr or not
|
||||||
for {
|
for {
|
||||||
pdfboxRes <- logger.debug("Trying to strip text from pdf using pdfbox.") *> PdfboxExtract.get[F](in)
|
pdfboxRes <- logger.debug("Trying to strip text from pdf using pdfbox.") *> PdfboxExtract
|
||||||
|
.get[F](in)
|
||||||
res <- pdfboxRes.fold(
|
res <- pdfboxRes.fold(
|
||||||
ex =>
|
ex =>
|
||||||
logger.info(
|
logger.info(
|
||||||
|
@ -5,13 +5,12 @@ import java.nio.file.{Path, Paths}
|
|||||||
import docspell.common._
|
import docspell.common._
|
||||||
|
|
||||||
case class OcrConfig(
|
case class OcrConfig(
|
||||||
maxImageSize: Int,
|
maxImageSize: Int,
|
||||||
ghostscript: OcrConfig.Ghostscript,
|
ghostscript: OcrConfig.Ghostscript,
|
||||||
pageRange: OcrConfig.PageRange,
|
pageRange: OcrConfig.PageRange,
|
||||||
unpaper: OcrConfig.Unpaper,
|
unpaper: OcrConfig.Unpaper,
|
||||||
tesseract: OcrConfig.Tesseract
|
tesseract: OcrConfig.Tesseract
|
||||||
) {
|
) {}
|
||||||
}
|
|
||||||
|
|
||||||
object OcrConfig {
|
object OcrConfig {
|
||||||
|
|
||||||
|
@ -5,9 +5,9 @@ import docspell.common.MimeType
|
|||||||
object OcrType {
|
object OcrType {
|
||||||
|
|
||||||
val jpeg = MimeType.jpeg
|
val jpeg = MimeType.jpeg
|
||||||
val png = MimeType.png
|
val png = MimeType.png
|
||||||
val tiff = MimeType.tiff
|
val tiff = MimeType.tiff
|
||||||
val pdf = MimeType.pdf
|
val pdf = MimeType.pdf
|
||||||
|
|
||||||
val all = Set(jpeg, png, tiff, pdf)
|
val all = Set(jpeg, png, tiff, pdf)
|
||||||
|
|
||||||
|
@ -17,14 +17,14 @@ object OdfExtract {
|
|||||||
def get[F[_]: Sync](data: Stream[F, Byte]): F[Either[Throwable, String]] =
|
def get[F[_]: Sync](data: Stream[F, Byte]): F[Either[Throwable, String]] =
|
||||||
data.compile.to(Array).map(new ByteArrayInputStream(_)).map(get)
|
data.compile.to(Array).map(new ByteArrayInputStream(_)).map(get)
|
||||||
|
|
||||||
|
def get(is: InputStream) =
|
||||||
def get(is: InputStream) = Try {
|
Try {
|
||||||
val handler = new BodyContentHandler()
|
val handler = new BodyContentHandler()
|
||||||
val pctx = new ParseContext()
|
val pctx = new ParseContext()
|
||||||
val meta = new Metadata()
|
val meta = new Metadata()
|
||||||
val ooparser = new OpenDocumentParser()
|
val ooparser = new OpenDocumentParser()
|
||||||
ooparser.parse(is, handler, meta, pctx)
|
ooparser.parse(is, handler, meta, pctx)
|
||||||
handler.toString.trim
|
handler.toString.trim
|
||||||
}.toEither
|
}.toEither
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -4,8 +4,8 @@ import docspell.common.MimeType
|
|||||||
|
|
||||||
object OdfType {
|
object OdfType {
|
||||||
|
|
||||||
val odt = MimeType.application("vnd.oasis.opendocument.text")
|
val odt = MimeType.application("vnd.oasis.opendocument.text")
|
||||||
val ods = MimeType.application("vnd.oasis.opendocument.spreadsheet")
|
val ods = MimeType.application("vnd.oasis.opendocument.spreadsheet")
|
||||||
val odtAlias = MimeType.application("x-vnd.oasis.opendocument.text")
|
val odtAlias = MimeType.application("x-vnd.oasis.opendocument.text")
|
||||||
val odsAlias = MimeType.application("x-vnd.oasis.opendocument.spreadsheet")
|
val odsAlias = MimeType.application("x-vnd.oasis.opendocument.spreadsheet")
|
||||||
|
|
||||||
|
@ -14,9 +14,7 @@ import fs2.Stream
|
|||||||
object PdfboxExtract {
|
object PdfboxExtract {
|
||||||
|
|
||||||
def get[F[_]: Sync](data: Stream[F, Byte]): F[Either[Throwable, String]] =
|
def get[F[_]: Sync](data: Stream[F, Byte]): F[Either[Throwable, String]] =
|
||||||
data.compile.to(Array).map { bytes =>
|
data.compile.to(Array).map(bytes => Using(PDDocument.load(bytes))(readText).toEither.flatten)
|
||||||
Using(PDDocument.load(bytes))(readText).toEither.flatten
|
|
||||||
}
|
|
||||||
|
|
||||||
def get(is: InputStream): Either[Throwable, String] =
|
def get(is: InputStream): Either[Throwable, String] =
|
||||||
Using(PDDocument.load(is))(readText).toEither.flatten
|
Using(PDDocument.load(is))(readText).toEither.flatten
|
||||||
|
@ -52,25 +52,25 @@ object PoiExtract {
|
|||||||
def getDocx(is: InputStream): Either[Throwable, String] =
|
def getDocx(is: InputStream): Either[Throwable, String] =
|
||||||
Try {
|
Try {
|
||||||
val xt = new XWPFWordExtractor(new XWPFDocument(is))
|
val xt = new XWPFWordExtractor(new XWPFDocument(is))
|
||||||
xt.getText.trim
|
Option(xt.getText).map(_.trim).getOrElse("")
|
||||||
}.toEither
|
}.toEither
|
||||||
|
|
||||||
def getDoc(is: InputStream): Either[Throwable, String] =
|
def getDoc(is: InputStream): Either[Throwable, String] =
|
||||||
Try {
|
Try {
|
||||||
val xt = new WordExtractor(is)
|
val xt = new WordExtractor(is)
|
||||||
xt.getText.trim
|
Option(xt.getText).map(_.trim).getOrElse("")
|
||||||
}.toEither
|
}.toEither
|
||||||
|
|
||||||
def getXlsx(is: InputStream): Either[Throwable, String] =
|
def getXlsx(is: InputStream): Either[Throwable, String] =
|
||||||
Try {
|
Try {
|
||||||
val xt = new XSSFExcelExtractor(new XSSFWorkbook(is))
|
val xt = new XSSFExcelExtractor(new XSSFWorkbook(is))
|
||||||
xt.getText.trim
|
Option(xt.getText).map(_.trim).getOrElse("")
|
||||||
}.toEither
|
}.toEither
|
||||||
|
|
||||||
def getXls(is: InputStream): Either[Throwable, String] =
|
def getXls(is: InputStream): Either[Throwable, String] =
|
||||||
Try {
|
Try {
|
||||||
val xt = new ExcelExtractor(new HSSFWorkbook(is))
|
val xt = new ExcelExtractor(new HSSFWorkbook(is))
|
||||||
xt.getText.trim
|
Option(xt.getText).map(_.trim).getOrElse("")
|
||||||
}.toEither
|
}.toEither
|
||||||
|
|
||||||
def getDocx[F[_]: Sync](data: Stream[F, Byte]): F[Either[Throwable, String]] =
|
def getDocx[F[_]: Sync](data: Stream[F, Byte]): F[Either[Throwable, String]] =
|
||||||
|
@ -5,11 +5,11 @@ import docspell.common.MimeType
|
|||||||
object PoiType {
|
object PoiType {
|
||||||
|
|
||||||
val msoffice = MimeType.application("x-tika-msoffice")
|
val msoffice = MimeType.application("x-tika-msoffice")
|
||||||
val ooxml = MimeType.application("x-tika-ooxml")
|
val ooxml = MimeType.application("x-tika-ooxml")
|
||||||
val docx = MimeType.application("vnd.openxmlformats-officedocument.wordprocessingml.document")
|
val docx = MimeType.application("vnd.openxmlformats-officedocument.wordprocessingml.document")
|
||||||
val xlsx = MimeType.application("vnd.openxmlformats-officedocument.spreadsheetml.sheet")
|
val xlsx = MimeType.application("vnd.openxmlformats-officedocument.spreadsheetml.sheet")
|
||||||
val xls = MimeType.application("vnd.ms-excel")
|
val xls = MimeType.application("vnd.ms-excel")
|
||||||
val doc = MimeType.application("msword")
|
val doc = MimeType.application("msword")
|
||||||
|
|
||||||
val all = Set(msoffice, ooxml, docx, xlsx, xls, doc)
|
val all = Set(msoffice, ooxml, docx, xlsx, xls, doc)
|
||||||
|
|
||||||
|
@ -5,7 +5,7 @@ import docspell.files.{ExampleFiles, TestFiles}
|
|||||||
import minitest.SimpleTestSuite
|
import minitest.SimpleTestSuite
|
||||||
|
|
||||||
object OdfExtractTest extends SimpleTestSuite {
|
object OdfExtractTest extends SimpleTestSuite {
|
||||||
val blocker = TestFiles.blocker
|
val blocker = TestFiles.blocker
|
||||||
implicit val CS = TestFiles.CS
|
implicit val CS = TestFiles.CS
|
||||||
|
|
||||||
val files = List(
|
val files = List(
|
||||||
@ -14,14 +14,15 @@ object OdfExtractTest extends SimpleTestSuite {
|
|||||||
)
|
)
|
||||||
|
|
||||||
test("test extract from odt") {
|
test("test extract from odt") {
|
||||||
files.foreach { case (file, len) =>
|
files.foreach {
|
||||||
val is = file.toJavaUrl.map(_.openStream()).fold(sys.error, identity)
|
case (file, len) =>
|
||||||
val str1 = OdfExtract.get(is).fold(throw _, identity)
|
val is = file.toJavaUrl.map(_.openStream()).fold(sys.error, identity)
|
||||||
assertEquals(str1.length, len)
|
val str1 = OdfExtract.get(is).fold(throw _, identity)
|
||||||
|
assertEquals(str1.length, len)
|
||||||
|
|
||||||
val data = file.readURL[IO](8192, blocker)
|
val data = file.readURL[IO](8192, blocker)
|
||||||
val str2 = OdfExtract.get[IO](data).unsafeRunSync().fold(throw _, identity)
|
val str2 = OdfExtract.get[IO](data).unsafeRunSync().fold(throw _, identity)
|
||||||
assertEquals(str2, str1)
|
assertEquals(str2, str1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -7,8 +7,8 @@ object RtfExtractTest extends SimpleTestSuite {
|
|||||||
|
|
||||||
test("extract text from rtf using java input-stream") {
|
test("extract text from rtf using java input-stream") {
|
||||||
val file = ExampleFiles.examples_sample_rtf
|
val file = ExampleFiles.examples_sample_rtf
|
||||||
val is = file.toJavaUrl.map(_.openStream()).fold(sys.error, identity)
|
val is = file.toJavaUrl.map(_.openStream()).fold(sys.error, identity)
|
||||||
val str = RtfExtract.get(is).fold(throw _, identity)
|
val str = RtfExtract.get(is).fold(throw _, identity)
|
||||||
assertEquals(str.length, 7342)
|
assertEquals(str.length, 7342)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -29,13 +29,12 @@ object ImageSize {
|
|||||||
/** Return the image size from its header without reading
|
/** Return the image size from its header without reading
|
||||||
* the whole image into memory.
|
* the whole image into memory.
|
||||||
*/
|
*/
|
||||||
def get[F[_]: Sync](data: Stream[F, Byte]): F[Option[Dimension]] = {
|
def get[F[_]: Sync](data: Stream[F, Byte]): F[Option[Dimension]] =
|
||||||
data.take(768).compile.to(Array).map(ar => {
|
data.take(768).compile.to(Array).map { ar =>
|
||||||
val iis = ImageIO.createImageInputStream(new ByteArrayInputStream(ar))
|
val iis = ImageIO.createImageInputStream(new ByteArrayInputStream(ar))
|
||||||
if (iis == null) sys.error("no reader given for the array")
|
if (iis == null) sys.error("no reader given for the array")
|
||||||
else getDimension(iis)
|
else getDimension(iis)
|
||||||
})
|
}
|
||||||
}
|
|
||||||
|
|
||||||
private def getDimension(in: ImageInputStream): Option[Dimension] =
|
private def getDimension(in: ImageInputStream): Option[Dimension] =
|
||||||
ImageIO
|
ImageIO
|
||||||
|
@ -52,8 +52,8 @@ object TikaMimetype {
|
|||||||
def detect[F[_]: Sync](file: Path): F[MimeType] =
|
def detect[F[_]: Sync](file: Path): F[MimeType] =
|
||||||
Sync[F].delay {
|
Sync[F].delay {
|
||||||
val hint = MimeTypeHint.filename(file.getFileName.toString)
|
val hint = MimeTypeHint.filename(file.getFileName.toString)
|
||||||
Using(new BufferedInputStream(Files.newInputStream(file), 64))({ in =>
|
Using(new BufferedInputStream(Files.newInputStream(file), 64)) { in =>
|
||||||
convert(tika.detect(in, makeMetadata(hint)))
|
convert(tika.detect(in, makeMetadata(hint)))
|
||||||
}).toEither
|
}.toEither
|
||||||
}.rethrow
|
}.rethrow
|
||||||
}
|
}
|
||||||
|
@ -7,8 +7,7 @@ trait ExampleFilesSupport {
|
|||||||
def createUrl(resource: String): LenientUri =
|
def createUrl(resource: String): LenientUri =
|
||||||
Option(getClass.getResource("/" + resource)) match {
|
Option(getClass.getResource("/" + resource)) match {
|
||||||
case Some(u) => LenientUri.fromJava(u)
|
case Some(u) => LenientUri.fromJava(u)
|
||||||
case None => sys.error(s"Resource '$resource' not found")
|
case None => sys.error(s"Resource '$resource' not found")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -8,15 +8,14 @@ import scala.concurrent.ExecutionContext
|
|||||||
object Playing extends IOApp {
|
object Playing extends IOApp {
|
||||||
val blocker = Blocker.liftExecutionContext(ExecutionContext.global)
|
val blocker = Blocker.liftExecutionContext(ExecutionContext.global)
|
||||||
|
|
||||||
|
|
||||||
def run(args: List[String]): IO[ExitCode] = IO {
|
def run(args: List[String]): IO[ExitCode] = IO {
|
||||||
//val ods = ExampleFiles.examples_sample_ods.readURL[IO](8192, blocker)
|
//val ods = ExampleFiles.examples_sample_ods.readURL[IO](8192, blocker)
|
||||||
//val odt = ExampleFiles.examples_sample_odt.readURL[IO](8192, blocker)
|
//val odt = ExampleFiles.examples_sample_odt.readURL[IO](8192, blocker)
|
||||||
val rtf = ExampleFiles.examples_sample_rtf.readURL[IO](8192, blocker)
|
val rtf = ExampleFiles.examples_sample_rtf.readURL[IO](8192, blocker)
|
||||||
|
|
||||||
val x = for {
|
val x = for {
|
||||||
odsm1 <- TikaMimetype.detect(rtf,
|
odsm1 <- TikaMimetype
|
||||||
MimeTypeHint.filename(ExampleFiles.examples_sample_rtf.path.segments.last))
|
.detect(rtf, MimeTypeHint.filename(ExampleFiles.examples_sample_rtf.path.segments.last))
|
||||||
odsm2 <- TikaMimetype.detect(rtf, MimeTypeHint.none)
|
odsm2 <- TikaMimetype.detect(rtf, MimeTypeHint.none)
|
||||||
} yield (odsm1, odsm2)
|
} yield (odsm1, odsm2)
|
||||||
println(x.unsafeRunSync())
|
println(x.unsafeRunSync())
|
||||||
|
@ -7,13 +7,13 @@ import docspell.convert.ConvertConfig
|
|||||||
import docspell.extract.ExtractConfig
|
import docspell.extract.ExtractConfig
|
||||||
|
|
||||||
case class Config(
|
case class Config(
|
||||||
appId: Ident,
|
appId: Ident,
|
||||||
baseUrl: LenientUri,
|
baseUrl: LenientUri,
|
||||||
bind: Config.Bind,
|
bind: Config.Bind,
|
||||||
jdbc: JdbcConfig,
|
jdbc: JdbcConfig,
|
||||||
scheduler: SchedulerConfig,
|
scheduler: SchedulerConfig,
|
||||||
extraction: ExtractConfig,
|
extraction: ExtractConfig,
|
||||||
convert: ConvertConfig
|
convert: ConvertConfig
|
||||||
)
|
)
|
||||||
|
|
||||||
object Config {
|
object Config {
|
||||||
|
@ -52,15 +52,15 @@ object JoexAppImpl {
|
|||||||
store <- Store.create(cfg.jdbc, connectEC, blocker)
|
store <- Store.create(cfg.jdbc, connectEC, blocker)
|
||||||
nodeOps <- ONode(store)
|
nodeOps <- ONode(store)
|
||||||
sch <- SchedulerBuilder(cfg.scheduler, blocker, store)
|
sch <- SchedulerBuilder(cfg.scheduler, blocker, store)
|
||||||
.withTask(
|
.withTask(
|
||||||
JobTask.json(
|
JobTask.json(
|
||||||
ProcessItemArgs.taskName,
|
ProcessItemArgs.taskName,
|
||||||
ItemHandler[F](cfg),
|
ItemHandler[F](cfg),
|
||||||
ItemHandler.onCancel[F]
|
ItemHandler.onCancel[F]
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
.resource
|
.resource
|
||||||
app = new JoexAppImpl(cfg, nodeOps, store, termSignal, sch)
|
app = new JoexAppImpl(cfg, nodeOps, store, termSignal, sch)
|
||||||
appR <- Resource.make(app.init.map(_ => app))(_.shutdown)
|
appR <- Resource.make(app.init.map(_ => app))(_.shutdown)
|
||||||
} yield appR
|
} yield appR
|
||||||
}
|
}
|
||||||
|
@ -68,7 +68,9 @@ object ConvertPdf {
|
|||||||
.through(ctx.store.bitpeace.fetchData2(RangeDef.all))
|
.through(ctx.store.bitpeace.fetchData2(RangeDef.all))
|
||||||
val handler = conversionHandler[F](ctx, cfg, ra, item)
|
val handler = conversionHandler[F](ctx, cfg, ra, item)
|
||||||
ctx.logger.info(s"Converting file ${ra.name} (${mime.asString}) into a PDF") *>
|
ctx.logger.info(s"Converting file ${ra.name} (${mime.asString}) into a PDF") *>
|
||||||
conv.toPDF(DataType(MimeType(mime.primary, mime.sub)), ctx.args.meta.language, handler)(data)
|
conv.toPDF(DataType(MimeType(mime.primary, mime.sub)), ctx.args.meta.language, handler)(
|
||||||
|
data
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -107,19 +109,21 @@ object ConvertPdf {
|
|||||||
})
|
})
|
||||||
|
|
||||||
private def storePDF[F[_]: Sync](
|
private def storePDF[F[_]: Sync](
|
||||||
ctx: Context[F, ProcessItemArgs],
|
ctx: Context[F, ProcessItemArgs],
|
||||||
cfg: ConvertConfig,
|
cfg: ConvertConfig,
|
||||||
ra: RAttachment,
|
ra: RAttachment,
|
||||||
pdf: Stream[F, Byte]
|
pdf: Stream[F, Byte]
|
||||||
) = {
|
) = {
|
||||||
val hint = MimeTypeHint.advertised(MimeType.pdf).withName(ra.name.getOrElse("file.pdf"))
|
val hint = MimeTypeHint.advertised(MimeType.pdf).withName(ra.name.getOrElse("file.pdf"))
|
||||||
val newName = ra.name.map(n => s"$n.pdf")
|
val newName = ra.name.map(n => s"$n.pdf")
|
||||||
ctx.store.bitpeace
|
ctx.store.bitpeace
|
||||||
.saveNew(pdf, cfg.chunkSize, MimetypeHint(hint.filename, hint.advertised))
|
.saveNew(pdf, cfg.chunkSize, MimetypeHint(hint.filename, hint.advertised))
|
||||||
.compile
|
.compile
|
||||||
.lastOrError
|
.lastOrError
|
||||||
.map(fm => Ident.unsafe(fm.id))
|
.map(fm => Ident.unsafe(fm.id))
|
||||||
.flatMap(fmId => ctx.store.transact(RAttachment.updateFileIdAndName(ra.id, fmId, newName)).map(_ => fmId))
|
.flatMap(fmId =>
|
||||||
|
ctx.store.transact(RAttachment.updateFileIdAndName(ra.id, fmId, newName)).map(_ => fmId)
|
||||||
|
)
|
||||||
.map(fmId => ra.copy(fileId = fmId, name = newName))
|
.map(fmId => ra.copy(fileId = fmId, name = newName))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -25,7 +25,7 @@ object CreateItem {
|
|||||||
Task { ctx =>
|
Task { ctx =>
|
||||||
def isValidFile(fm: FileMeta) =
|
def isValidFile(fm: FileMeta) =
|
||||||
ctx.args.meta.validFileTypes.isEmpty ||
|
ctx.args.meta.validFileTypes.isEmpty ||
|
||||||
ctx.args.meta.validFileTypes.map(_.asString).toSet.contains(fm.mimetype.baseType)
|
ctx.args.meta.validFileTypes.map(_.asString).toSet.contains(fm.mimetype.baseType)
|
||||||
|
|
||||||
def fileMetas(itemId: Ident, now: Timestamp) =
|
def fileMetas(itemId: Ident, now: Timestamp) =
|
||||||
Stream
|
Stream
|
||||||
@ -77,18 +77,18 @@ object CreateItem {
|
|||||||
for {
|
for {
|
||||||
cand <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId)))
|
cand <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId)))
|
||||||
_ <- if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.")
|
_ <- if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.")
|
||||||
else ().pure[F]
|
else ().pure[F]
|
||||||
ht <- cand.drop(1).traverse(ri => QItem.delete(ctx.store)(ri.id, ri.cid))
|
ht <- cand.drop(1).traverse(ri => QItem.delete(ctx.store)(ri.id, ri.cid))
|
||||||
_ <- if (ht.sum > 0) ctx.logger.warn(s"Removed ${ht.sum} items with same attachments")
|
_ <- if (ht.sum > 0) ctx.logger.warn(s"Removed ${ht.sum} items with same attachments")
|
||||||
else ().pure[F]
|
else ().pure[F]
|
||||||
rms <- OptionT(
|
rms <- OptionT(
|
||||||
cand.headOption.traverse(ri =>
|
cand.headOption.traverse(ri =>
|
||||||
ctx.store.transact(RAttachment.findByItemAndCollective(ri.id, ri.cid))
|
ctx.store.transact(RAttachment.findByItemAndCollective(ri.id, ri.cid))
|
||||||
)
|
)
|
||||||
).getOrElse(Vector.empty)
|
).getOrElse(Vector.empty)
|
||||||
orig <- rms.traverse(a =>
|
orig <- rms.traverse(a =>
|
||||||
ctx.store.transact(RAttachmentSource.findById(a.id)).map(s => (a, s))
|
ctx.store.transact(RAttachmentSource.findById(a.id)).map(s => (a, s))
|
||||||
)
|
)
|
||||||
origMap = orig
|
origMap = orig
|
||||||
.map(originFileTuple)
|
.map(originFileTuple)
|
||||||
.toMap
|
.toMap
|
||||||
|
@ -95,10 +95,10 @@ object FindProposal {
|
|||||||
labels => self.find(labels).map(f)
|
labels => self.find(labels).map(f)
|
||||||
|
|
||||||
def next(f: Finder[F])(implicit F: FlatMap[F], F3: Applicative[F]): Finder[F] =
|
def next(f: Finder[F])(implicit F: FlatMap[F], F3: Applicative[F]): Finder[F] =
|
||||||
flatMap({ ml0 =>
|
flatMap { ml0 =>
|
||||||
if (ml0.hasResultsAll) Finder.unit[F](ml0)
|
if (ml0.hasResultsAll) Finder.unit[F](ml0)
|
||||||
else f.map(ml1 => ml0.fillEmptyFrom(ml1))
|
else f.map(ml1 => ml0.fillEmptyFrom(ml1))
|
||||||
})
|
}
|
||||||
|
|
||||||
def nextWhenEmpty(f: Finder[F], mt0: MetaProposalType, mts: MetaProposalType*)(
|
def nextWhenEmpty(f: Finder[F], mt0: MetaProposalType, mts: MetaProposalType*)(
|
||||||
implicit F: FlatMap[F],
|
implicit F: FlatMap[F],
|
||||||
|
@ -19,14 +19,12 @@ object ItemHandler {
|
|||||||
.map(_ => ())
|
.map(_ => ())
|
||||||
|
|
||||||
def itemStateTask[F[_]: Sync, A](state: ItemState)(data: ItemData): Task[F, A, ItemData] =
|
def itemStateTask[F[_]: Sync, A](state: ItemState)(data: ItemData): Task[F, A, ItemData] =
|
||||||
Task { ctx =>
|
Task(ctx => ctx.store.transact(RItem.updateState(data.item.id, state)).map(_ => data))
|
||||||
ctx.store.transact(RItem.updateState(data.item.id, state)).map(_ => data)
|
|
||||||
}
|
|
||||||
|
|
||||||
def isLastRetry[F[_]: Sync, A](ctx: Context[F, A]): F[Boolean] =
|
def isLastRetry[F[_]: Sync, A](ctx: Context[F, A]): F[Boolean] =
|
||||||
for {
|
for {
|
||||||
current <- ctx.store.transact(RJob.getRetries(ctx.jobId))
|
current <- ctx.store.transact(RJob.getRetries(ctx.jobId))
|
||||||
last = ctx.config.retries == current.getOrElse(0)
|
last = ctx.config.retries == current.getOrElse(0)
|
||||||
} yield last
|
} yield last
|
||||||
|
|
||||||
def safeProcess[F[_]: Sync: ContextShift](
|
def safeProcess[F[_]: Sync: ContextShift](
|
||||||
|
@ -11,9 +11,7 @@ object TestTasks {
|
|||||||
private[this] val logger = getLogger
|
private[this] val logger = getLogger
|
||||||
|
|
||||||
def success[F[_]]: Task[F, ProcessItemArgs, Unit] =
|
def success[F[_]]: Task[F, ProcessItemArgs, Unit] =
|
||||||
Task { ctx =>
|
Task(ctx => ctx.logger.info(s"Running task now: ${ctx.args}"))
|
||||||
ctx.logger.info(s"Running task now: ${ctx.args}")
|
|
||||||
}
|
|
||||||
|
|
||||||
def failing[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] =
|
def failing[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] =
|
||||||
Task { ctx =>
|
Task { ctx =>
|
||||||
|
@ -20,8 +20,8 @@ object TextAnalysis {
|
|||||||
t <- item.metas.toList.traverse(annotateAttachment[F](ctx.args.meta.language))
|
t <- item.metas.toList.traverse(annotateAttachment[F](ctx.args.meta.language))
|
||||||
_ <- ctx.logger.debug(s"Storing tags: ${t.map(_._1.copy(content = None))}")
|
_ <- ctx.logger.debug(s"Storing tags: ${t.map(_._1.copy(content = None))}")
|
||||||
_ <- t.traverse(m =>
|
_ <- t.traverse(m =>
|
||||||
ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels))
|
ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels))
|
||||||
)
|
)
|
||||||
e <- s
|
e <- s
|
||||||
_ <- ctx.logger.info(s"Text-Analysis finished in ${e.formatExact}")
|
_ <- ctx.logger.info(s"Text-Analysis finished in ${e.formatExact}")
|
||||||
v = t.toVector
|
v = t.toVector
|
||||||
|
@ -12,8 +12,8 @@ import docspell.store.records.{RAttachment, RAttachmentMeta, RFileMeta}
|
|||||||
object TextExtraction {
|
object TextExtraction {
|
||||||
|
|
||||||
def apply[F[_]: Sync: ContextShift](
|
def apply[F[_]: Sync: ContextShift](
|
||||||
cfg: ExtractConfig,
|
cfg: ExtractConfig,
|
||||||
item: ItemData
|
item: ItemData
|
||||||
): Task[F, ProcessItemArgs, ItemData] =
|
): Task[F, ProcessItemArgs, ItemData] =
|
||||||
Task { ctx =>
|
Task { ctx =>
|
||||||
for {
|
for {
|
||||||
@ -28,11 +28,11 @@ object TextExtraction {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def extractTextIfEmpty[F[_]: Sync: ContextShift](
|
def extractTextIfEmpty[F[_]: Sync: ContextShift](
|
||||||
ctx: Context[F, _],
|
ctx: Context[F, _],
|
||||||
cfg: ExtractConfig,
|
cfg: ExtractConfig,
|
||||||
lang: Language,
|
lang: Language,
|
||||||
item: ItemData
|
item: ItemData
|
||||||
)(ra: RAttachment): F[RAttachmentMeta] = {
|
)(ra: RAttachment): F[RAttachmentMeta] = {
|
||||||
val rm = item.findOrCreate(ra.id)
|
val rm = item.findOrCreate(ra.id)
|
||||||
rm.content match {
|
rm.content match {
|
||||||
case Some(_) =>
|
case Some(_) =>
|
||||||
@ -50,14 +50,14 @@ object TextExtraction {
|
|||||||
item: ItemData
|
item: ItemData
|
||||||
)(ra: RAttachment): F[RAttachmentMeta] =
|
)(ra: RAttachment): F[RAttachmentMeta] =
|
||||||
for {
|
for {
|
||||||
_ <- ctx.logger.debug(s"Extracting text for attachment ${stripAttachmentName(ra)}")
|
_ <- ctx.logger.debug(s"Extracting text for attachment ${stripAttachmentName(ra)}")
|
||||||
dst <- Duration.stopTime[F]
|
dst <- Duration.stopTime[F]
|
||||||
txt <- extractTextFallback(ctx, cfg, ra, lang)(filesToExtract(item, ra))
|
txt <- extractTextFallback(ctx, cfg, ra, lang)(filesToExtract(item, ra))
|
||||||
meta = item.changeMeta(ra.id, rm => rm.setContentIfEmpty(txt.map(_.trim).filter(_.nonEmpty)))
|
meta = item.changeMeta(ra.id, rm => rm.setContentIfEmpty(txt.map(_.trim).filter(_.nonEmpty)))
|
||||||
est <- dst
|
est <- dst
|
||||||
_ <- ctx.logger.debug(
|
_ <- ctx.logger.debug(
|
||||||
s"Extracting text for attachment ${stripAttachmentName(ra)} finished in ${est.formatExact}"
|
s"Extracting text for attachment ${stripAttachmentName(ra)} finished in ${est.formatExact}"
|
||||||
)
|
)
|
||||||
} yield meta
|
} yield meta
|
||||||
|
|
||||||
def extractText[F[_]: Sync: ContextShift](
|
def extractText[F[_]: Sync: ContextShift](
|
||||||
@ -76,16 +76,15 @@ object TextExtraction {
|
|||||||
.getOrElse(Mimetype.`application/octet-stream`)
|
.getOrElse(Mimetype.`application/octet-stream`)
|
||||||
|
|
||||||
findMime
|
findMime
|
||||||
.flatMap(mt =>
|
.flatMap(mt => extr.extractText(data, DataType(MimeType(mt.primary, mt.sub)), lang))
|
||||||
extr.extractText(data, DataType(MimeType(mt.primary, mt.sub)), lang))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private def extractTextFallback[F[_]: Sync: ContextShift](
|
private def extractTextFallback[F[_]: Sync: ContextShift](
|
||||||
ctx: Context[F, _],
|
ctx: Context[F, _],
|
||||||
cfg: ExtractConfig,
|
cfg: ExtractConfig,
|
||||||
ra: RAttachment,
|
ra: RAttachment,
|
||||||
lang: Language,
|
lang: Language
|
||||||
)(fileIds: List[Ident]): F[Option[String]] = {
|
)(fileIds: List[Ident]): F[Option[String]] =
|
||||||
fileIds match {
|
fileIds match {
|
||||||
case Nil =>
|
case Nil =>
|
||||||
ctx.logger.error(s"Cannot extract text").map(_ => None)
|
ctx.logger.error(s"Cannot extract text").map(_ => None)
|
||||||
@ -99,15 +98,18 @@ object TextExtraction {
|
|||||||
txt.some.pure[F]
|
txt.some.pure[F]
|
||||||
|
|
||||||
case ExtractResult.UnsupportedFormat(mt) =>
|
case ExtractResult.UnsupportedFormat(mt) =>
|
||||||
ctx.logger.warn(s"Cannot extract text from file ${stripAttachmentName(ra)}: unsupported format ${mt.asString}. Try with converted file.").
|
ctx.logger
|
||||||
flatMap(_ => extractTextFallback[F](ctx, cfg, ra, lang)(rest))
|
.warn(
|
||||||
|
s"Cannot extract text from file ${stripAttachmentName(ra)}: unsupported format ${mt.asString}. Try with converted file."
|
||||||
|
)
|
||||||
|
.flatMap(_ => extractTextFallback[F](ctx, cfg, ra, lang)(rest))
|
||||||
|
|
||||||
case ExtractResult.Failure(ex) =>
|
case ExtractResult.Failure(ex) =>
|
||||||
ctx.logger.warn(s"Cannot extract text: ${ex.getMessage}. Try with converted file").
|
ctx.logger
|
||||||
flatMap(_ => extractTextFallback[F](ctx, cfg, ra, lang)(rest))
|
.warn(s"Cannot extract text: ${ex.getMessage}. Try with converted file")
|
||||||
|
.flatMap(_ => extractTextFallback[F](ctx, cfg, ra, lang)(rest))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
/** Returns the fileIds to extract text from. First, the source file
|
/** Returns the fileIds to extract text from. First, the source file
|
||||||
* is tried. If that fails, the converted file is tried.
|
* is tried. If that fails, the converted file is tried.
|
||||||
@ -115,7 +117,7 @@ object TextExtraction {
|
|||||||
private def filesToExtract(item: ItemData, ra: RAttachment): List[Ident] =
|
private def filesToExtract(item: ItemData, ra: RAttachment): List[Ident] =
|
||||||
item.originFile.get(ra.id) match {
|
item.originFile.get(ra.id) match {
|
||||||
case Some(sid) => List(sid, ra.fileId).distinct
|
case Some(sid) => List(sid, ra.fileId).distinct
|
||||||
case None => List(ra.fileId)
|
case None => List(ra.fileId)
|
||||||
}
|
}
|
||||||
|
|
||||||
private def stripAttachmentName(ra: RAttachment): String =
|
private def stripAttachmentName(ra: RAttachment): String =
|
||||||
|
@ -25,15 +25,15 @@ object JoexRoutes {
|
|||||||
case GET -> Root / "running" =>
|
case GET -> Root / "running" =>
|
||||||
for {
|
for {
|
||||||
jobs <- app.scheduler.getRunning
|
jobs <- app.scheduler.getRunning
|
||||||
jj = jobs.map(mkJob)
|
jj = jobs.map(mkJob)
|
||||||
resp <- Ok(JobList(jj.toList))
|
resp <- Ok(JobList(jj.toList))
|
||||||
} yield resp
|
} yield resp
|
||||||
|
|
||||||
case POST -> Root / "shutdownAndExit" =>
|
case POST -> Root / "shutdownAndExit" =>
|
||||||
for {
|
for {
|
||||||
_ <- ConcurrentEffect[F].start(
|
_ <- ConcurrentEffect[F].start(
|
||||||
Timer[F].sleep(Duration.seconds(1).toScala) *> app.initShutdown
|
Timer[F].sleep(Duration.seconds(1).toScala) *> app.initShutdown
|
||||||
)
|
)
|
||||||
resp <- Ok(BasicResult(true, "Shutdown initiated."))
|
resp <- Ok(BasicResult(true, "Shutdown initiated."))
|
||||||
} yield resp
|
} yield resp
|
||||||
|
|
||||||
@ -41,8 +41,8 @@ object JoexRoutes {
|
|||||||
for {
|
for {
|
||||||
optJob <- app.scheduler.getRunning.map(_.find(_.id == id))
|
optJob <- app.scheduler.getRunning.map(_.find(_.id == id))
|
||||||
optLog <- optJob.traverse(j => app.findLogs(j.id))
|
optLog <- optJob.traverse(j => app.findLogs(j.id))
|
||||||
jAndL = for { job <- optJob; log <- optLog } yield mkJobLog(job, log)
|
jAndL = for { job <- optJob; log <- optLog } yield mkJobLog(job, log)
|
||||||
resp <- jAndL.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found")))
|
resp <- jAndL.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found")))
|
||||||
} yield resp
|
} yield resp
|
||||||
|
|
||||||
case POST -> Root / "job" / Ident(id) / "cancel" =>
|
case POST -> Root / "job" / Ident(id) / "cancel" =>
|
||||||
|
@ -54,7 +54,7 @@ object Context {
|
|||||||
_ <- log.ftrace("Creating logger for task run")
|
_ <- log.ftrace("Creating logger for task run")
|
||||||
logger <- QueueLogger(job.id, job.info, config.logBufferSize, logSink)
|
logger <- QueueLogger(job.id, job.info, config.logBufferSize, logSink)
|
||||||
_ <- log.ftrace("Logger created, instantiating context")
|
_ <- log.ftrace("Logger created, instantiating context")
|
||||||
ctx = create[F, A](job, arg, config, logger, store, blocker)
|
ctx = create[F, A](job, arg, config, logger, store, blocker)
|
||||||
} yield ctx
|
} yield ctx
|
||||||
|
|
||||||
final private class ContextImpl[F[_]: Functor, A](
|
final private class ContextImpl[F[_]: Functor, A](
|
||||||
|
@ -38,9 +38,9 @@ object QueueLogger {
|
|||||||
sink: LogSink[F]
|
sink: LogSink[F]
|
||||||
): F[Logger[F]] =
|
): F[Logger[F]] =
|
||||||
for {
|
for {
|
||||||
q <- Queue.circularBuffer[F, LogEvent](bufferSize)
|
q <- Queue.circularBuffer[F, LogEvent](bufferSize)
|
||||||
log = create(jobId, jobInfo, q)
|
log = create(jobId, jobInfo, q)
|
||||||
_ <- Concurrent[F].start(q.dequeue.through(sink.receive).compile.drain)
|
_ <- Concurrent[F].start(q.dequeue.through(sink.receive).compile.drain)
|
||||||
} yield log
|
} yield log
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -91,12 +91,12 @@ final class SchedulerImpl[F[_]: ConcurrentEffect: ContextShift](
|
|||||||
_ <- logger.fdebug("New permit acquired")
|
_ <- logger.fdebug("New permit acquired")
|
||||||
down <- state.get.map(_.shutdownRequest)
|
down <- state.get.map(_.shutdownRequest)
|
||||||
rjob <- if (down) logger.finfo("") *> permits.release *> (None: Option[RJob]).pure[F]
|
rjob <- if (down) logger.finfo("") *> permits.release *> (None: Option[RJob]).pure[F]
|
||||||
else
|
else
|
||||||
queue.nextJob(
|
queue.nextJob(
|
||||||
group => state.modify(_.nextPrio(group, config.countingScheme)),
|
group => state.modify(_.nextPrio(group, config.countingScheme)),
|
||||||
config.name,
|
config.name,
|
||||||
config.retryDelay
|
config.retryDelay
|
||||||
)
|
)
|
||||||
_ <- logger.fdebug(s"Next job found: ${rjob.map(_.info)}")
|
_ <- logger.fdebug(s"Next job found: ${rjob.map(_.info)}")
|
||||||
_ <- rjob.map(execute).getOrElse(permits.release)
|
_ <- rjob.map(execute).getOrElse(permits.release)
|
||||||
} yield rjob.isDefined
|
} yield rjob.isDefined
|
||||||
@ -122,8 +122,8 @@ final class SchedulerImpl[F[_]: ConcurrentEffect: ContextShift](
|
|||||||
def execute(job: RJob): F[Unit] = {
|
def execute(job: RJob): F[Unit] = {
|
||||||
val task = for {
|
val task = for {
|
||||||
jobtask <- tasks
|
jobtask <- tasks
|
||||||
.find(job.task)
|
.find(job.task)
|
||||||
.toRight(s"This executor cannot run tasks with name: ${job.task}")
|
.toRight(s"This executor cannot run tasks with name: ${job.task}")
|
||||||
} yield jobtask
|
} yield jobtask
|
||||||
|
|
||||||
task match {
|
task match {
|
||||||
@ -144,8 +144,8 @@ final class SchedulerImpl[F[_]: ConcurrentEffect: ContextShift](
|
|||||||
for {
|
for {
|
||||||
_ <- logger.fdebug(s"Job ${job.info} done $finalState. Releasing resources.")
|
_ <- logger.fdebug(s"Job ${job.info} done $finalState. Releasing resources.")
|
||||||
_ <- permits.release *> permits.available.flatMap(a =>
|
_ <- permits.release *> permits.available.flatMap(a =>
|
||||||
logger.fdebug(s"Permit released ($a free)")
|
logger.fdebug(s"Permit released ($a free)")
|
||||||
)
|
)
|
||||||
_ <- state.modify(_.removeRunning(job))
|
_ <- state.modify(_.removeRunning(job))
|
||||||
_ <- QJob.setFinalState(job.id, finalState, store)
|
_ <- QJob.setFinalState(job.id, finalState, store)
|
||||||
} yield ()
|
} yield ()
|
||||||
|
@ -128,6 +128,9 @@ Please see the `nix/module-server.nix` and `nix/module-joex.nix` files
|
|||||||
for the set of options. The nixos options are modelled after the
|
for the set of options. The nixos options are modelled after the
|
||||||
default configuration file.
|
default configuration file.
|
||||||
|
|
||||||
|
The modules files are only applicable to the newest version of
|
||||||
|
Docspell. If you really need an older version, checkout the
|
||||||
|
appropriate commit.
|
||||||
|
|
||||||
## NixOs Example
|
## NixOs Example
|
||||||
|
|
||||||
|
@ -27,8 +27,8 @@ object RestAppImpl {
|
|||||||
): Resource[F, RestApp[F]] =
|
): Resource[F, RestApp[F]] =
|
||||||
for {
|
for {
|
||||||
backend <- BackendApp(cfg.backend, connectEC, httpClientEc, blocker)
|
backend <- BackendApp(cfg.backend, connectEC, httpClientEc, blocker)
|
||||||
app = new RestAppImpl[F](cfg, backend)
|
app = new RestAppImpl[F](cfg, backend)
|
||||||
appR <- Resource.make(app.init.map(_ => app))(_.shutdown)
|
appR <- Resource.make(app.init.map(_ => app))(_.shutdown)
|
||||||
} yield appR
|
} yield appR
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -35,8 +35,8 @@ object CookieData {
|
|||||||
for {
|
for {
|
||||||
header <- headers.Cookie.from(req.headers).toRight("Cookie parsing error")
|
header <- headers.Cookie.from(req.headers).toRight("Cookie parsing error")
|
||||||
cookie <- header.values.toList
|
cookie <- header.values.toList
|
||||||
.find(_.name == cookieName)
|
.find(_.name == cookieName)
|
||||||
.toRight("Couldn't find the authcookie")
|
.toRight("Couldn't find the authcookie")
|
||||||
} yield cookie.content
|
} yield cookie.content
|
||||||
|
|
||||||
def fromHeader[F[_]](req: Request[F]): Either[String, String] =
|
def fromHeader[F[_]](req: Request[F]): Either[String, String] =
|
||||||
|
@ -84,7 +84,7 @@ trait Conversions {
|
|||||||
data.inReplyTo.map(mkIdName),
|
data.inReplyTo.map(mkIdName),
|
||||||
data.item.dueDate,
|
data.item.dueDate,
|
||||||
data.item.notes,
|
data.item.notes,
|
||||||
data.attachments.map((mkAttachment(data)_).tupled).toList,
|
data.attachments.map((mkAttachment(data) _).tupled).toList,
|
||||||
data.sources.map((mkAttachmentSource _).tupled).toList,
|
data.sources.map((mkAttachmentSource _).tupled).toList,
|
||||||
data.tags.map(mkTag).toList
|
data.tags.map(mkTag).toList
|
||||||
)
|
)
|
||||||
@ -204,7 +204,8 @@ trait Conversions {
|
|||||||
|
|
||||||
val files = mp.parts
|
val files = mp.parts
|
||||||
.filter(p => p.name.forall(s => !s.equalsIgnoreCase("meta")))
|
.filter(p => p.name.forall(s => !s.equalsIgnoreCase("meta")))
|
||||||
.map(p => OUpload.File(p.filename, p.headers.get(`Content-Type`).map(fromContentType), p.body)
|
.map(p =>
|
||||||
|
OUpload.File(p.filename, p.headers.get(`Content-Type`).map(fromContentType), p.body)
|
||||||
)
|
)
|
||||||
for {
|
for {
|
||||||
metaData <- meta
|
metaData <- meta
|
||||||
|
@ -45,21 +45,21 @@ object AttachmentRoutes {
|
|||||||
for {
|
for {
|
||||||
fileData <- backend.item.findAttachment(id, user.account.collective)
|
fileData <- backend.item.findAttachment(id, user.account.collective)
|
||||||
resp <- fileData
|
resp <- fileData
|
||||||
.map(data => withResponseHeaders(Ok())(data))
|
.map(data => withResponseHeaders(Ok())(data))
|
||||||
.getOrElse(NotFound(BasicResult(false, "Not found")))
|
.getOrElse(NotFound(BasicResult(false, "Not found")))
|
||||||
} yield resp
|
} yield resp
|
||||||
|
|
||||||
case req @ GET -> Root / Ident(id) =>
|
case req @ GET -> Root / Ident(id) =>
|
||||||
for {
|
for {
|
||||||
fileData <- backend.item.findAttachment(id, user.account.collective)
|
fileData <- backend.item.findAttachment(id, user.account.collective)
|
||||||
inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
|
inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
|
||||||
matches = matchETag(fileData.map(_.meta), inm)
|
matches = matchETag(fileData.map(_.meta), inm)
|
||||||
resp <- fileData
|
resp <- fileData
|
||||||
.map({ data =>
|
.map { data =>
|
||||||
if (matches) withResponseHeaders(NotModified())(data)
|
if (matches) withResponseHeaders(NotModified())(data)
|
||||||
else makeByteResp(data)
|
else makeByteResp(data)
|
||||||
})
|
}
|
||||||
.getOrElse(NotFound(BasicResult(false, "Not found")))
|
.getOrElse(NotFound(BasicResult(false, "Not found")))
|
||||||
} yield resp
|
} yield resp
|
||||||
|
|
||||||
case HEAD -> Root / Ident(id) / "original" =>
|
case HEAD -> Root / Ident(id) / "original" =>
|
||||||
@ -73,13 +73,13 @@ object AttachmentRoutes {
|
|||||||
case req @ GET -> Root / Ident(id) / "original" =>
|
case req @ GET -> Root / Ident(id) / "original" =>
|
||||||
for {
|
for {
|
||||||
fileData <- backend.item.findAttachmentSource(id, user.account.collective)
|
fileData <- backend.item.findAttachmentSource(id, user.account.collective)
|
||||||
inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
|
inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
|
||||||
matches = matchETag(fileData.map(_.meta), inm)
|
matches = matchETag(fileData.map(_.meta), inm)
|
||||||
resp <- fileData
|
resp <- fileData
|
||||||
.map({ data =>
|
.map { data =>
|
||||||
if (matches) withResponseHeaders(NotModified())(data)
|
if (matches) withResponseHeaders(NotModified())(data)
|
||||||
else makeByteResp(data)
|
else makeByteResp(data)
|
||||||
})
|
}
|
||||||
.getOrElse(NotFound(BasicResult(false, "Not found")))
|
.getOrElse(NotFound(BasicResult(false, "Not found")))
|
||||||
} yield resp
|
} yield resp
|
||||||
|
|
||||||
@ -92,16 +92,16 @@ object AttachmentRoutes {
|
|||||||
|
|
||||||
case GET -> Root / Ident(id) / "meta" =>
|
case GET -> Root / Ident(id) / "meta" =>
|
||||||
for {
|
for {
|
||||||
rm <- backend.item.findAttachmentMeta(id, user.account.collective)
|
rm <- backend.item.findAttachmentMeta(id, user.account.collective)
|
||||||
md = rm.map(Conversions.mkAttachmentMeta)
|
md = rm.map(Conversions.mkAttachmentMeta)
|
||||||
resp <- md.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found.")))
|
resp <- md.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found.")))
|
||||||
} yield resp
|
} yield resp
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private def matchETag[F[_]](
|
private def matchETag[F[_]](
|
||||||
fileData: Option[FileMeta],
|
fileData: Option[FileMeta],
|
||||||
noneMatch: Option[NonEmptyList[EntityTag]]
|
noneMatch: Option[NonEmptyList[EntityTag]]
|
||||||
): Boolean =
|
): Boolean =
|
||||||
(fileData, noneMatch) match {
|
(fileData, noneMatch) match {
|
||||||
case (Some(meta), Some(nm)) =>
|
case (Some(meta), Some(nm)) =>
|
||||||
|
@ -35,25 +35,25 @@ object CollectiveRoutes {
|
|||||||
case GET -> Root / "settings" =>
|
case GET -> Root / "settings" =>
|
||||||
for {
|
for {
|
||||||
collDb <- backend.collective.find(user.account.collective)
|
collDb <- backend.collective.find(user.account.collective)
|
||||||
sett = collDb.map(c => CollectiveSettings(c.language))
|
sett = collDb.map(c => CollectiveSettings(c.language))
|
||||||
resp <- sett.toResponse()
|
resp <- sett.toResponse()
|
||||||
} yield resp
|
} yield resp
|
||||||
|
|
||||||
case GET -> Root / "contacts" :? QueryParam.QueryOpt(q) +& QueryParam.ContactKindOpt(kind) =>
|
case GET -> Root / "contacts" :? QueryParam.QueryOpt(q) +& QueryParam.ContactKindOpt(kind) =>
|
||||||
for {
|
for {
|
||||||
res <- backend.collective
|
res <- backend.collective
|
||||||
.getContacts(user.account.collective, q.map(_.q), kind)
|
.getContacts(user.account.collective, q.map(_.q), kind)
|
||||||
.take(50)
|
.take(50)
|
||||||
.compile
|
.compile
|
||||||
.toList
|
.toList
|
||||||
resp <- Ok(ContactList(res.map(Conversions.mkContact)))
|
resp <- Ok(ContactList(res.map(Conversions.mkContact)))
|
||||||
} yield resp
|
} yield resp
|
||||||
|
|
||||||
case GET -> Root =>
|
case GET -> Root =>
|
||||||
for {
|
for {
|
||||||
collDb <- backend.collective.find(user.account.collective)
|
collDb <- backend.collective.find(user.account.collective)
|
||||||
coll = collDb.map(c => Collective(c.id, c.state, c.created))
|
coll = collDb.map(c => Collective(c.id, c.state, c.created))
|
||||||
resp <- coll.toResponse()
|
resp <- coll.toResponse()
|
||||||
} yield resp
|
} yield resp
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -37,10 +37,10 @@ object EquipmentRoutes {
|
|||||||
|
|
||||||
case req @ PUT -> Root =>
|
case req @ PUT -> Root =>
|
||||||
for {
|
for {
|
||||||
data <- req.as[Equipment]
|
data <- req.as[Equipment]
|
||||||
equip = changeEquipment(data, user.account.collective)
|
equip = changeEquipment(data, user.account.collective)
|
||||||
res <- backend.equipment.update(equip)
|
res <- backend.equipment.update(equip)
|
||||||
resp <- Ok(basicResult(res, "Equipment updated."))
|
resp <- Ok(basicResult(res, "Equipment updated."))
|
||||||
} yield resp
|
} yield resp
|
||||||
|
|
||||||
case DELETE -> Root / Ident(id) =>
|
case DELETE -> Root / Ident(id) =>
|
||||||
|
@ -24,8 +24,8 @@ object ItemRoutes {
|
|||||||
HttpRoutes.of {
|
HttpRoutes.of {
|
||||||
case req @ POST -> Root / "search" =>
|
case req @ POST -> Root / "search" =>
|
||||||
for {
|
for {
|
||||||
mask <- req.as[ItemSearch]
|
mask <- req.as[ItemSearch]
|
||||||
_ <- logger.ftrace(s"Got search mask: $mask")
|
_ <- logger.ftrace(s"Got search mask: $mask")
|
||||||
query = Conversions.mkQuery(mask, user.account.collective)
|
query = Conversions.mkQuery(mask, user.account.collective)
|
||||||
_ <- logger.ftrace(s"Running query: $query")
|
_ <- logger.ftrace(s"Running query: $query")
|
||||||
items <- backend.item.findItems(query, 100)
|
items <- backend.item.findItems(query, 100)
|
||||||
@ -34,9 +34,9 @@ object ItemRoutes {
|
|||||||
|
|
||||||
case GET -> Root / Ident(id) =>
|
case GET -> Root / Ident(id) =>
|
||||||
for {
|
for {
|
||||||
item <- backend.item.findItem(id, user.account.collective)
|
item <- backend.item.findItem(id, user.account.collective)
|
||||||
result = item.map(Conversions.mkItemDetail)
|
result = item.map(Conversions.mkItemDetail)
|
||||||
resp <- result.map(r => Ok(r)).getOrElse(NotFound(BasicResult(false, "Not found.")))
|
resp <- result.map(r => Ok(r)).getOrElse(NotFound(BasicResult(false, "Not found.")))
|
||||||
} yield resp
|
} yield resp
|
||||||
|
|
||||||
case POST -> Root / Ident(id) / "confirm" =>
|
case POST -> Root / Ident(id) / "confirm" =>
|
||||||
@ -125,15 +125,15 @@ object ItemRoutes {
|
|||||||
|
|
||||||
case GET -> Root / Ident(id) / "proposals" =>
|
case GET -> Root / Ident(id) / "proposals" =>
|
||||||
for {
|
for {
|
||||||
ml <- backend.item.getProposals(id, user.account.collective)
|
ml <- backend.item.getProposals(id, user.account.collective)
|
||||||
ip = Conversions.mkItemProposals(ml)
|
ip = Conversions.mkItemProposals(ml)
|
||||||
resp <- Ok(ip)
|
resp <- Ok(ip)
|
||||||
} yield resp
|
} yield resp
|
||||||
|
|
||||||
case DELETE -> Root / Ident(id) =>
|
case DELETE -> Root / Ident(id) =>
|
||||||
for {
|
for {
|
||||||
n <- backend.item.delete(id, user.account.collective)
|
n <- backend.item.delete(id, user.account.collective)
|
||||||
res = BasicResult(n > 0, if (n > 0) "Item deleted" else "Item deletion failed.")
|
res = BasicResult(n > 0, if (n > 0) "Item deleted" else "Item deletion failed.")
|
||||||
resp <- Ok(res)
|
resp <- Ok(res)
|
||||||
} yield resp
|
} yield resp
|
||||||
}
|
}
|
||||||
|
@ -19,8 +19,8 @@ object JobQueueRoutes {
|
|||||||
HttpRoutes.of {
|
HttpRoutes.of {
|
||||||
case GET -> Root / "state" =>
|
case GET -> Root / "state" =>
|
||||||
for {
|
for {
|
||||||
js <- backend.job.queueState(user.account.collective, 200)
|
js <- backend.job.queueState(user.account.collective, 200)
|
||||||
res = Conversions.mkJobQueueState(js)
|
res = Conversions.mkJobQueueState(js)
|
||||||
resp <- Ok(res)
|
resp <- Ok(res)
|
||||||
} yield resp
|
} yield resp
|
||||||
|
|
||||||
|
@ -54,15 +54,15 @@ object LoginRoutes {
|
|||||||
for {
|
for {
|
||||||
cd <- AuthToken.user(token.account, cfg.auth.serverSecret).map(CookieData.apply)
|
cd <- AuthToken.user(token.account, cfg.auth.serverSecret).map(CookieData.apply)
|
||||||
resp <- Ok(
|
resp <- Ok(
|
||||||
AuthResult(
|
AuthResult(
|
||||||
token.account.collective.id,
|
token.account.collective.id,
|
||||||
token.account.user.id,
|
token.account.user.id,
|
||||||
true,
|
true,
|
||||||
"Login successful",
|
"Login successful",
|
||||||
Some(cd.asString),
|
Some(cd.asString),
|
||||||
cfg.auth.sessionValid.millis
|
cfg.auth.sessionValid.millis
|
||||||
)
|
)
|
||||||
).map(_.addCookie(cd.asCookie(cfg)))
|
).map(_.addCookie(cd.asCookie(cfg)))
|
||||||
} yield resp
|
} yield resp
|
||||||
case _ =>
|
case _ =>
|
||||||
Ok(AuthResult("", account, false, "Login failed.", None, 0L))
|
Ok(AuthResult("", account, false, "Login failed.", None, 0L))
|
||||||
|
@ -24,13 +24,13 @@ object MailSendRoutes {
|
|||||||
HttpRoutes.of {
|
HttpRoutes.of {
|
||||||
case req @ POST -> Root / Ident(name) / Ident(id) =>
|
case req @ POST -> Root / Ident(name) / Ident(id) =>
|
||||||
for {
|
for {
|
||||||
in <- req.as[SimpleMail]
|
in <- req.as[SimpleMail]
|
||||||
mail = convertIn(id, in)
|
mail = convertIn(id, in)
|
||||||
res <- mail.traverse(m => backend.mail.sendMail(user.account, name, m))
|
res <- mail.traverse(m => backend.mail.sendMail(user.account, name, m))
|
||||||
resp <- res.fold(
|
resp <- res.fold(
|
||||||
err => Ok(BasicResult(false, s"Invalid mail data: $err")),
|
err => Ok(BasicResult(false, s"Invalid mail data: $err")),
|
||||||
res => Ok(convertOut(res))
|
res => Ok(convertOut(res))
|
||||||
)
|
)
|
||||||
} yield resp
|
} yield resp
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -39,7 +39,7 @@ object MailSendRoutes {
|
|||||||
for {
|
for {
|
||||||
rec <- s.recipients.traverse(EmilUtil.readMailAddress)
|
rec <- s.recipients.traverse(EmilUtil.readMailAddress)
|
||||||
fileIds <- s.attachmentIds.traverse(Ident.fromString)
|
fileIds <- s.attachmentIds.traverse(Ident.fromString)
|
||||||
sel = if (s.addAllAttachments) AttachSelection.All else AttachSelection.Selected(fileIds)
|
sel = if (s.addAllAttachments) AttachSelection.All else AttachSelection.Selected(fileIds)
|
||||||
} yield ItemMail(item, s.subject, rec, s.body, sel)
|
} yield ItemMail(item, s.subject, rec, s.body, sel)
|
||||||
|
|
||||||
def convertOut(res: SendResult): BasicResult =
|
def convertOut(res: SendResult): BasicResult =
|
||||||
|
@ -29,7 +29,7 @@ object MailSettingsRoutes {
|
|||||||
case GET -> Root :? QueryParam.QueryOpt(q) =>
|
case GET -> Root :? QueryParam.QueryOpt(q) =>
|
||||||
for {
|
for {
|
||||||
list <- backend.mail.getSettings(user.account, q.map(_.q))
|
list <- backend.mail.getSettings(user.account, q.map(_.q))
|
||||||
res = list.map(convert)
|
res = list.map(convert)
|
||||||
resp <- Ok(EmailSettingsList(res.toList))
|
resp <- Ok(EmailSettingsList(res.toList))
|
||||||
} yield resp
|
} yield resp
|
||||||
|
|
||||||
@ -45,13 +45,13 @@ object MailSettingsRoutes {
|
|||||||
ru = makeSettings(in)
|
ru = makeSettings(in)
|
||||||
up <- OptionT.liftF(ru.traverse(r => backend.mail.createSettings(user.account, r)))
|
up <- OptionT.liftF(ru.traverse(r => backend.mail.createSettings(user.account, r)))
|
||||||
resp <- OptionT.liftF(
|
resp <- OptionT.liftF(
|
||||||
Ok(
|
Ok(
|
||||||
up.fold(
|
up.fold(
|
||||||
err => BasicResult(false, err),
|
err => BasicResult(false, err),
|
||||||
ar => Conversions.basicResult(ar, "Mail settings stored.")
|
ar => Conversions.basicResult(ar, "Mail settings stored.")
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
} yield resp).getOrElseF(NotFound())
|
} yield resp).getOrElseF(NotFound())
|
||||||
|
|
||||||
case req @ PUT -> Root / Ident(name) =>
|
case req @ PUT -> Root / Ident(name) =>
|
||||||
@ -60,24 +60,24 @@ object MailSettingsRoutes {
|
|||||||
ru = makeSettings(in)
|
ru = makeSettings(in)
|
||||||
up <- OptionT.liftF(ru.traverse(r => backend.mail.updateSettings(user.account, name, r)))
|
up <- OptionT.liftF(ru.traverse(r => backend.mail.updateSettings(user.account, name, r)))
|
||||||
resp <- OptionT.liftF(
|
resp <- OptionT.liftF(
|
||||||
Ok(
|
Ok(
|
||||||
up.fold(
|
up.fold(
|
||||||
err => BasicResult(false, err),
|
err => BasicResult(false, err),
|
||||||
n =>
|
n =>
|
||||||
if (n > 0) BasicResult(true, "Mail settings stored.")
|
if (n > 0) BasicResult(true, "Mail settings stored.")
|
||||||
else BasicResult(false, "Mail settings could not be saved")
|
else BasicResult(false, "Mail settings could not be saved")
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
} yield resp).getOrElseF(NotFound())
|
} yield resp).getOrElseF(NotFound())
|
||||||
|
|
||||||
case DELETE -> Root / Ident(name) =>
|
case DELETE -> Root / Ident(name) =>
|
||||||
for {
|
for {
|
||||||
n <- backend.mail.deleteSettings(user.account, name)
|
n <- backend.mail.deleteSettings(user.account, name)
|
||||||
resp <- Ok(
|
resp <- Ok(
|
||||||
if (n > 0) BasicResult(true, "Mail settings removed")
|
if (n > 0) BasicResult(true, "Mail settings removed")
|
||||||
else BasicResult(false, "Mail settings could not be removed")
|
else BasicResult(false, "Mail settings could not be removed")
|
||||||
)
|
)
|
||||||
} yield resp
|
} yield resp
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -36,8 +36,8 @@ object SourceRoutes {
|
|||||||
|
|
||||||
case req @ PUT -> Root =>
|
case req @ PUT -> Root =>
|
||||||
for {
|
for {
|
||||||
data <- req.as[Source]
|
data <- req.as[Source]
|
||||||
src = changeSource(data, user.account.collective)
|
src = changeSource(data, user.account.collective)
|
||||||
updated <- backend.source.update(src)
|
updated <- backend.source.update(src)
|
||||||
resp <- Ok(basicResult(updated, "Source updated."))
|
resp <- Ok(basicResult(updated, "Source updated."))
|
||||||
} yield resp
|
} yield resp
|
||||||
|
@ -37,7 +37,7 @@ object TagRoutes {
|
|||||||
case req @ PUT -> Root =>
|
case req @ PUT -> Root =>
|
||||||
for {
|
for {
|
||||||
data <- req.as[Tag]
|
data <- req.as[Tag]
|
||||||
tag = changeTag(data, user.account.collective)
|
tag = changeTag(data, user.account.collective)
|
||||||
res <- backend.tag.update(tag)
|
res <- backend.tag.update(tag)
|
||||||
resp <- Ok(basicResult(res, "Tag successfully updated."))
|
resp <- Ok(basicResult(res, "Tag successfully updated."))
|
||||||
} yield resp
|
} yield resp
|
||||||
|
@ -28,11 +28,11 @@ object UploadRoutes {
|
|||||||
for {
|
for {
|
||||||
multipart <- req.as[Multipart[F]]
|
multipart <- req.as[Multipart[F]]
|
||||||
updata <- readMultipart(
|
updata <- readMultipart(
|
||||||
multipart,
|
multipart,
|
||||||
logger,
|
logger,
|
||||||
Priority.High,
|
Priority.High,
|
||||||
cfg.backend.files.validMimeTypes
|
cfg.backend.files.validMimeTypes
|
||||||
)
|
)
|
||||||
result <- backend.upload.submit(updata, user.account)
|
result <- backend.upload.submit(updata, user.account)
|
||||||
res <- Ok(basicResult(result))
|
res <- Ok(basicResult(result))
|
||||||
} yield res
|
} yield res
|
||||||
|
@ -24,10 +24,10 @@ object UserRoutes {
|
|||||||
for {
|
for {
|
||||||
data <- req.as[PasswordChange]
|
data <- req.as[PasswordChange]
|
||||||
res <- backend.collective.changePassword(
|
res <- backend.collective.changePassword(
|
||||||
user.account,
|
user.account,
|
||||||
data.currentPassword,
|
data.currentPassword,
|
||||||
data.newPassword
|
data.newPassword
|
||||||
)
|
)
|
||||||
resp <- Ok(basicResult(res))
|
resp <- Ok(basicResult(res))
|
||||||
} yield resp
|
} yield resp
|
||||||
|
|
||||||
@ -47,8 +47,8 @@ object UserRoutes {
|
|||||||
|
|
||||||
case req @ PUT -> Root =>
|
case req @ PUT -> Root =>
|
||||||
for {
|
for {
|
||||||
data <- req.as[User]
|
data <- req.as[User]
|
||||||
nuser = changeUser(data, user.account.collective)
|
nuser = changeUser(data, user.account.collective)
|
||||||
update <- backend.collective.update(nuser)
|
update <- backend.collective.update(nuser)
|
||||||
resp <- Ok(basicResult(update, "User updated."))
|
resp <- Ok(basicResult(update, "User updated."))
|
||||||
} yield resp
|
} yield resp
|
||||||
|
@ -40,7 +40,7 @@ object Store {
|
|||||||
for {
|
for {
|
||||||
xa <- hxa
|
xa <- hxa
|
||||||
st = new StoreImpl[F](jdbc, xa)
|
st = new StoreImpl[F](jdbc, xa)
|
||||||
_ <- Resource.liftF(st.migrate)
|
_ <- Resource.liftF(st.migrate)
|
||||||
} yield st
|
} yield st
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -14,25 +14,32 @@ object QAttachment {
|
|||||||
|
|
||||||
def deleteById[F[_]: Sync](store: Store[F])(attachId: Ident, coll: Ident): F[Int] =
|
def deleteById[F[_]: Sync](store: Store[F])(attachId: Ident, coll: Ident): F[Int] =
|
||||||
for {
|
for {
|
||||||
raFile <- store.transact(RAttachment.findByIdAndCollective(attachId, coll)).map(_.map(_.fileId))
|
raFile <- store
|
||||||
rsFile <- store.transact(RAttachmentSource.findByIdAndCollective(attachId, coll)).map(_.map(_.fileId))
|
.transact(RAttachment.findByIdAndCollective(attachId, coll))
|
||||||
n <- store.transact(RAttachment.delete(attachId))
|
.map(_.map(_.fileId))
|
||||||
f <- Stream.emits(raFile.toSeq ++ rsFile.toSeq)
|
rsFile <- store
|
||||||
.map(_.id)
|
.transact(RAttachmentSource.findByIdAndCollective(attachId, coll))
|
||||||
.flatMap(store.bitpeace.delete)
|
.map(_.map(_.fileId))
|
||||||
.map(flag => if (flag) 1 else 0)
|
n <- store.transact(RAttachment.delete(attachId))
|
||||||
.compile
|
f <- Stream
|
||||||
.foldMonoid
|
.emits(raFile.toSeq ++ rsFile.toSeq)
|
||||||
|
.map(_.id)
|
||||||
|
.flatMap(store.bitpeace.delete)
|
||||||
|
.map(flag => if (flag) 1 else 0)
|
||||||
|
.compile
|
||||||
|
.foldMonoid
|
||||||
} yield n + f
|
} yield n + f
|
||||||
|
|
||||||
def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] =
|
def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] =
|
||||||
for {
|
for {
|
||||||
s <- store.transact(RAttachmentSource.findById(ra.id))
|
s <- store.transact(RAttachmentSource.findById(ra.id))
|
||||||
n <- store.transact(RAttachment.delete(ra.id))
|
n <- store.transact(RAttachment.delete(ra.id))
|
||||||
f <- Stream.emits(ra.fileId.id +: s.map(_.fileId.id).toSeq).
|
f <- Stream
|
||||||
flatMap(store.bitpeace.delete).
|
.emits(ra.fileId.id +: s.map(_.fileId.id).toSeq)
|
||||||
map(flag => if (flag) 1 else 0).
|
.flatMap(store.bitpeace.delete)
|
||||||
compile.foldMonoid
|
.map(flag => if (flag) 1 else 0)
|
||||||
|
.compile
|
||||||
|
.foldMonoid
|
||||||
} yield n + f
|
} yield n + f
|
||||||
|
|
||||||
def deleteItemAttachments[F[_]: Sync](store: Store[F])(itemId: Ident, coll: Ident): F[Int] =
|
def deleteItemAttachments[F[_]: Sync](store: Store[F])(itemId: Ident, coll: Ident): F[Int] =
|
||||||
|
@ -27,7 +27,6 @@ object QCollective {
|
|||||||
and(IC.cid.is(coll), IC.incoming.is(Direction.outgoing))
|
and(IC.cid.is(coll), IC.incoming.is(Direction.outgoing))
|
||||||
).query[Int].unique
|
).query[Int].unique
|
||||||
|
|
||||||
|
|
||||||
val fileSize = sql"""
|
val fileSize = sql"""
|
||||||
select sum(length) from (
|
select sum(length) from (
|
||||||
with attachs as
|
with attachs as
|
||||||
@ -42,7 +41,6 @@ object QCollective {
|
|||||||
inner join filemeta m on m.id = a.file_id where a.id in (select aid from attachs)
|
inner join filemeta m on m.id = a.file_id where a.id in (select aid from attachs)
|
||||||
) as t""".query[Option[Long]].unique
|
) as t""".query[Option[Long]].unique
|
||||||
|
|
||||||
|
|
||||||
val q3 = fr"SELECT" ++ commas(
|
val q3 = fr"SELECT" ++ commas(
|
||||||
TC.name.prefix("t").f,
|
TC.name.prefix("t").f,
|
||||||
fr"count(" ++ RC.itemId.prefix("r").f ++ fr")"
|
fr"count(" ++ RC.itemId.prefix("r").f ++ fr")"
|
||||||
|
@ -23,7 +23,7 @@ object QItem {
|
|||||||
concEquip: Option[REquipment],
|
concEquip: Option[REquipment],
|
||||||
inReplyTo: Option[IdRef],
|
inReplyTo: Option[IdRef],
|
||||||
tags: Vector[RTag],
|
tags: Vector[RTag],
|
||||||
attachments: Vector[(RAttachment, FileMeta)],
|
attachments: Vector[(RAttachment, FileMeta)],
|
||||||
sources: Vector[(RAttachmentSource, FileMeta)]
|
sources: Vector[(RAttachmentSource, FileMeta)]
|
||||||
) {
|
) {
|
||||||
|
|
||||||
@ -39,23 +39,24 @@ object QItem {
|
|||||||
val EC = REquipment.Columns.all.map(_.prefix("e"))
|
val EC = REquipment.Columns.all.map(_.prefix("e"))
|
||||||
val ICC = List(RItem.Columns.id, RItem.Columns.name).map(_.prefix("ref"))
|
val ICC = List(RItem.Columns.id, RItem.Columns.name).map(_.prefix("ref"))
|
||||||
|
|
||||||
val cq = selectSimple(IC ++ OC ++ P0C ++ P1C ++ EC ++ ICC, RItem.table ++ fr"i", Fragment.empty) ++
|
val cq =
|
||||||
fr"LEFT JOIN" ++ ROrganization.table ++ fr"o ON" ++ RItem.Columns.corrOrg
|
selectSimple(IC ++ OC ++ P0C ++ P1C ++ EC ++ ICC, RItem.table ++ fr"i", Fragment.empty) ++
|
||||||
.prefix("i")
|
fr"LEFT JOIN" ++ ROrganization.table ++ fr"o ON" ++ RItem.Columns.corrOrg
|
||||||
.is(ROrganization.Columns.oid.prefix("o")) ++
|
.prefix("i")
|
||||||
fr"LEFT JOIN" ++ RPerson.table ++ fr"p0 ON" ++ RItem.Columns.corrPerson
|
.is(ROrganization.Columns.oid.prefix("o")) ++
|
||||||
.prefix("i")
|
fr"LEFT JOIN" ++ RPerson.table ++ fr"p0 ON" ++ RItem.Columns.corrPerson
|
||||||
.is(RPerson.Columns.pid.prefix("p0")) ++
|
.prefix("i")
|
||||||
fr"LEFT JOIN" ++ RPerson.table ++ fr"p1 ON" ++ RItem.Columns.concPerson
|
.is(RPerson.Columns.pid.prefix("p0")) ++
|
||||||
.prefix("i")
|
fr"LEFT JOIN" ++ RPerson.table ++ fr"p1 ON" ++ RItem.Columns.concPerson
|
||||||
.is(RPerson.Columns.pid.prefix("p1")) ++
|
.prefix("i")
|
||||||
fr"LEFT JOIN" ++ REquipment.table ++ fr"e ON" ++ RItem.Columns.concEquipment
|
.is(RPerson.Columns.pid.prefix("p1")) ++
|
||||||
.prefix("i")
|
fr"LEFT JOIN" ++ REquipment.table ++ fr"e ON" ++ RItem.Columns.concEquipment
|
||||||
.is(REquipment.Columns.eid.prefix("e")) ++
|
.prefix("i")
|
||||||
fr"LEFT JOIN" ++ RItem.table ++ fr"ref ON" ++ RItem.Columns.inReplyTo
|
.is(REquipment.Columns.eid.prefix("e")) ++
|
||||||
.prefix("i")
|
fr"LEFT JOIN" ++ RItem.table ++ fr"ref ON" ++ RItem.Columns.inReplyTo
|
||||||
.is(RItem.Columns.id.prefix("ref")) ++
|
.prefix("i")
|
||||||
fr"WHERE" ++ RItem.Columns.id.prefix("i").is(id)
|
.is(RItem.Columns.id.prefix("ref")) ++
|
||||||
|
fr"WHERE" ++ RItem.Columns.id.prefix("i").is(id)
|
||||||
|
|
||||||
val q = cq
|
val q = cq
|
||||||
.query[
|
.query[
|
||||||
@ -235,11 +236,12 @@ object QItem {
|
|||||||
def findByFileIds(fileMetaIds: List[Ident]): ConnectionIO[Vector[RItem]] = {
|
def findByFileIds(fileMetaIds: List[Ident]): ConnectionIO[Vector[RItem]] = {
|
||||||
val IC = RItem.Columns
|
val IC = RItem.Columns
|
||||||
val AC = RAttachment.Columns
|
val AC = RAttachment.Columns
|
||||||
val q = fr"SELECT DISTINCT" ++ commas(IC.all.map(_.prefix("i").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++
|
val q =
|
||||||
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId
|
fr"SELECT DISTINCT" ++ commas(IC.all.map(_.prefix("i").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++
|
||||||
.prefix("a")
|
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId
|
||||||
.is(IC.id.prefix("i")) ++
|
.prefix("a")
|
||||||
fr"WHERE" ++ AC.fileId.isOneOf(fileMetaIds) ++ orderBy(IC.created.prefix("i").asc)
|
.is(IC.id.prefix("i")) ++
|
||||||
|
fr"WHERE" ++ AC.fileId.isOneOf(fileMetaIds) ++ orderBy(IC.created.prefix("i").asc)
|
||||||
|
|
||||||
q.query[RItem].to[Vector]
|
q.query[RItem].to[Vector]
|
||||||
}
|
}
|
||||||
|
@ -21,11 +21,11 @@ object QJob {
|
|||||||
Stream
|
Stream
|
||||||
.range(0, 10)
|
.range(0, 10)
|
||||||
.evalMap(n => takeNextJob1(store)(priority, worker, retryPause, n))
|
.evalMap(n => takeNextJob1(store)(priority, worker, retryPause, n))
|
||||||
.evalTap({ x =>
|
.evalTap { x =>
|
||||||
if (x.isLeft)
|
if (x.isLeft)
|
||||||
logger.fdebug[F]("Cannot mark job, probably due to concurrent updates. Will retry.")
|
logger.fdebug[F]("Cannot mark job, probably due to concurrent updates. Will retry.")
|
||||||
else ().pure[F]
|
else ().pure[F]
|
||||||
})
|
}
|
||||||
.find(_.isRight)
|
.find(_.isRight)
|
||||||
.flatMap({
|
.flatMap({
|
||||||
case Right(job) =>
|
case Right(job) =>
|
||||||
@ -50,7 +50,7 @@ object QJob {
|
|||||||
store.transact(for {
|
store.transact(for {
|
||||||
n <- RJob.setScheduled(job.id, worker)
|
n <- RJob.setScheduled(job.id, worker)
|
||||||
_ <- if (n == 1) RJobGroupUse.setGroup(RJobGroupUse(worker, job.group))
|
_ <- if (n == 1) RJobGroupUse.setGroup(RJobGroupUse(worker, job.group))
|
||||||
else 0.pure[ConnectionIO]
|
else 0.pure[ConnectionIO]
|
||||||
} yield if (n == 1) Right(job) else Left(()))
|
} yield if (n == 1) Right(job) else Left(()))
|
||||||
|
|
||||||
for {
|
for {
|
||||||
@ -61,8 +61,8 @@ object QJob {
|
|||||||
prio <- group.map(priority).getOrElse((Priority.Low: Priority).pure[F])
|
prio <- group.map(priority).getOrElse((Priority.Low: Priority).pure[F])
|
||||||
_ <- logger.ftrace[F](s"Looking for job of prio $prio")
|
_ <- logger.ftrace[F](s"Looking for job of prio $prio")
|
||||||
job <- group
|
job <- group
|
||||||
.map(g => store.transact(selectNextJob(g, prio, retryPause, now)))
|
.map(g => store.transact(selectNextJob(g, prio, retryPause, now)))
|
||||||
.getOrElse((None: Option[RJob]).pure[F])
|
.getOrElse((None: Option[RJob]).pure[F])
|
||||||
_ <- logger.ftrace[F](s"Found job: ${job.map(_.info)}")
|
_ <- logger.ftrace[F](s"Found job: ${job.map(_.info)}")
|
||||||
res <- job.traverse(j => markJob(j))
|
res <- job.traverse(j => markJob(j))
|
||||||
} yield res.map(_.map(_.some)).getOrElse {
|
} yield res.map(_.map(_.some)).getOrElse {
|
||||||
@ -97,7 +97,8 @@ object QJob {
|
|||||||
val sql2 = fr"SELECT min(" ++ jgroup.f ++ fr") as g FROM" ++ RJob.table ++ fr"a" ++
|
val sql2 = fr"SELECT min(" ++ jgroup.f ++ fr") as g FROM" ++ RJob.table ++ fr"a" ++
|
||||||
fr"WHERE" ++ stateCond
|
fr"WHERE" ++ stateCond
|
||||||
|
|
||||||
val union = sql"SELECT g FROM ((" ++ sql1 ++ sql") UNION ALL (" ++ sql2 ++ sql")) as t0 WHERE g is not null"
|
val union =
|
||||||
|
sql"SELECT g FROM ((" ++ sql1 ++ sql") UNION ALL (" ++ sql2 ++ sql")) as t0 WHERE g is not null"
|
||||||
|
|
||||||
union
|
union
|
||||||
.query[Ident]
|
.query[Ident]
|
||||||
|
@ -34,11 +34,11 @@ object JobQueue {
|
|||||||
def insert(job: RJob): F[Unit] =
|
def insert(job: RJob): F[Unit] =
|
||||||
store
|
store
|
||||||
.transact(RJob.insert(job))
|
.transact(RJob.insert(job))
|
||||||
.flatMap({ n =>
|
.flatMap { n =>
|
||||||
if (n != 1)
|
if (n != 1)
|
||||||
Effect[F].raiseError(new Exception(s"Inserting job failed. Update count: $n"))
|
Effect[F].raiseError(new Exception(s"Inserting job failed. Update count: $n"))
|
||||||
else ().pure[F]
|
else ().pure[F]
|
||||||
})
|
}
|
||||||
|
|
||||||
def insertAll(jobs: Seq[RJob]): F[Unit] =
|
def insertAll(jobs: Seq[RJob]): F[Unit] =
|
||||||
jobs.toList
|
jobs.toList
|
||||||
|
@ -47,10 +47,10 @@ object RAttachment {
|
|||||||
def findMeta(attachId: Ident): ConnectionIO[Option[FileMeta]] = {
|
def findMeta(attachId: Ident): ConnectionIO[Option[FileMeta]] = {
|
||||||
import bitpeace.sql._
|
import bitpeace.sql._
|
||||||
|
|
||||||
val cols = RFileMeta.Columns.all.map(_.prefix("m"))
|
val cols = RFileMeta.Columns.all.map(_.prefix("m"))
|
||||||
val aId = id.prefix("a")
|
val aId = id.prefix("a")
|
||||||
val aFileMeta = fileId.prefix("a")
|
val aFileMeta = fileId.prefix("a")
|
||||||
val mId = RFileMeta.Columns.id.prefix("m")
|
val mId = RFileMeta.Columns.id.prefix("m")
|
||||||
|
|
||||||
val from = table ++ fr"a INNER JOIN" ++ RFileMeta.table ++ fr"m ON" ++ aFileMeta.is(mId)
|
val from = table ++ fr"a INNER JOIN" ++ RFileMeta.table ++ fr"m ON" ++ aFileMeta.is(mId)
|
||||||
val cond = aId.is(attachId)
|
val cond = aId.is(attachId)
|
||||||
@ -104,7 +104,8 @@ object RAttachment {
|
|||||||
def findByItemWithMeta(id: Ident): ConnectionIO[Vector[(RAttachment, FileMeta)]] = {
|
def findByItemWithMeta(id: Ident): ConnectionIO[Vector[(RAttachment, FileMeta)]] = {
|
||||||
import bitpeace.sql._
|
import bitpeace.sql._
|
||||||
|
|
||||||
val q = fr"SELECT a.*,m.* FROM" ++ table ++ fr"a, filemeta m WHERE a.filemetaid = m.id AND a.itemid = $id ORDER BY a.position ASC"
|
val q =
|
||||||
|
fr"SELECT a.*,m.* FROM" ++ table ++ fr"a, filemeta m WHERE a.filemetaid = m.id AND a.itemid = $id ORDER BY a.position ASC"
|
||||||
q.query[(RAttachment, FileMeta)].to[Vector]
|
q.query[(RAttachment, FileMeta)].to[Vector]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -38,18 +38,20 @@ object RAttachmentSource {
|
|||||||
def insert(v: RAttachmentSource): ConnectionIO[Int] =
|
def insert(v: RAttachmentSource): ConnectionIO[Int] =
|
||||||
insertRow(table, all, fr"${v.id},${v.fileId},${v.name},${v.created}").update.run
|
insertRow(table, all, fr"${v.id},${v.fileId},${v.name},${v.created}").update.run
|
||||||
|
|
||||||
|
|
||||||
def findById(attachId: Ident): ConnectionIO[Option[RAttachmentSource]] =
|
def findById(attachId: Ident): ConnectionIO[Option[RAttachmentSource]] =
|
||||||
selectSimple(all, table, id.is(attachId)).query[RAttachmentSource].option
|
selectSimple(all, table, id.is(attachId)).query[RAttachmentSource].option
|
||||||
|
|
||||||
def delete(attachId: Ident): ConnectionIO[Int] =
|
def delete(attachId: Ident): ConnectionIO[Int] =
|
||||||
deleteFrom(table, id.is(attachId)).update.run
|
deleteFrom(table, id.is(attachId)).update.run
|
||||||
|
|
||||||
def findByIdAndCollective(attachId: Ident, collective: Ident): ConnectionIO[Option[RAttachmentSource]] = {
|
def findByIdAndCollective(
|
||||||
val bId = RAttachment.Columns.id.prefix("b")
|
attachId: Ident,
|
||||||
val aId = Columns.id.prefix("a")
|
collective: Ident
|
||||||
|
): ConnectionIO[Option[RAttachmentSource]] = {
|
||||||
|
val bId = RAttachment.Columns.id.prefix("b")
|
||||||
|
val aId = Columns.id.prefix("a")
|
||||||
val bItem = RAttachment.Columns.itemId.prefix("b")
|
val bItem = RAttachment.Columns.itemId.prefix("b")
|
||||||
val iId = RItem.Columns.id.prefix("i")
|
val iId = RItem.Columns.id.prefix("i")
|
||||||
val iColl = RItem.Columns.cid.prefix("i")
|
val iColl = RItem.Columns.cid.prefix("i")
|
||||||
|
|
||||||
val from = table ++ fr"a INNER JOIN" ++
|
val from = table ++ fr"a INNER JOIN" ++
|
||||||
@ -64,11 +66,11 @@ object RAttachmentSource {
|
|||||||
def findByItemWithMeta(id: Ident): ConnectionIO[Vector[(RAttachmentSource, FileMeta)]] = {
|
def findByItemWithMeta(id: Ident): ConnectionIO[Vector[(RAttachmentSource, FileMeta)]] = {
|
||||||
import bitpeace.sql._
|
import bitpeace.sql._
|
||||||
|
|
||||||
val aId = Columns.id.prefix("a")
|
val aId = Columns.id.prefix("a")
|
||||||
val afileMeta = fileId.prefix("a")
|
val afileMeta = fileId.prefix("a")
|
||||||
val bPos = RAttachment.Columns.position.prefix("b")
|
val bPos = RAttachment.Columns.position.prefix("b")
|
||||||
val bId = RAttachment.Columns.id.prefix("b")
|
val bId = RAttachment.Columns.id.prefix("b")
|
||||||
val bItem = RAttachment.Columns.itemId.prefix("b")
|
val bItem = RAttachment.Columns.itemId.prefix("b")
|
||||||
val mId = RFileMeta.Columns.id.prefix("m")
|
val mId = RFileMeta.Columns.id.prefix("m")
|
||||||
|
|
||||||
val cols = all.map(_.prefix("a")) ++ RFileMeta.Columns.all.map(_.prefix("m"))
|
val cols = all.map(_.prefix("a")) ++ RFileMeta.Columns.all.map(_.prefix("m"))
|
||||||
@ -77,8 +79,9 @@ object RAttachmentSource {
|
|||||||
RAttachment.table ++ fr"b ON" ++ aId.is(bId)
|
RAttachment.table ++ fr"b ON" ++ aId.is(bId)
|
||||||
val where = bItem.is(id)
|
val where = bItem.is(id)
|
||||||
|
|
||||||
(selectSimple(cols, from, where) ++ orderBy(bPos.asc)).
|
(selectSimple(cols, from, where) ++ orderBy(bPos.asc))
|
||||||
query[(RAttachmentSource, FileMeta)].to[Vector]
|
.query[(RAttachmentSource, FileMeta)]
|
||||||
|
.to[Vector]
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -124,140 +124,140 @@ object RItem {
|
|||||||
for {
|
for {
|
||||||
t <- currentTime
|
t <- currentTime
|
||||||
n <- updateRow(
|
n <- updateRow(
|
||||||
table,
|
table,
|
||||||
and(id.is(itemId), cid.is(coll)),
|
and(id.is(itemId), cid.is(coll)),
|
||||||
commas(state.setTo(itemState), updated.setTo(t))
|
commas(state.setTo(itemState), updated.setTo(t))
|
||||||
).update.run
|
).update.run
|
||||||
} yield n
|
} yield n
|
||||||
|
|
||||||
def updateDirection(itemId: Ident, coll: Ident, dir: Direction): ConnectionIO[Int] =
|
def updateDirection(itemId: Ident, coll: Ident, dir: Direction): ConnectionIO[Int] =
|
||||||
for {
|
for {
|
||||||
t <- currentTime
|
t <- currentTime
|
||||||
n <- updateRow(
|
n <- updateRow(
|
||||||
table,
|
table,
|
||||||
and(id.is(itemId), cid.is(coll)),
|
and(id.is(itemId), cid.is(coll)),
|
||||||
commas(incoming.setTo(dir), updated.setTo(t))
|
commas(incoming.setTo(dir), updated.setTo(t))
|
||||||
).update.run
|
).update.run
|
||||||
} yield n
|
} yield n
|
||||||
|
|
||||||
def updateCorrOrg(itemId: Ident, coll: Ident, org: Option[Ident]): ConnectionIO[Int] =
|
def updateCorrOrg(itemId: Ident, coll: Ident, org: Option[Ident]): ConnectionIO[Int] =
|
||||||
for {
|
for {
|
||||||
t <- currentTime
|
t <- currentTime
|
||||||
n <- updateRow(
|
n <- updateRow(
|
||||||
table,
|
table,
|
||||||
and(id.is(itemId), cid.is(coll)),
|
and(id.is(itemId), cid.is(coll)),
|
||||||
commas(corrOrg.setTo(org), updated.setTo(t))
|
commas(corrOrg.setTo(org), updated.setTo(t))
|
||||||
).update.run
|
).update.run
|
||||||
} yield n
|
} yield n
|
||||||
|
|
||||||
def removeCorrOrg(coll: Ident, currentOrg: Ident): ConnectionIO[Int] =
|
def removeCorrOrg(coll: Ident, currentOrg: Ident): ConnectionIO[Int] =
|
||||||
for {
|
for {
|
||||||
t <- currentTime
|
t <- currentTime
|
||||||
n <- updateRow(
|
n <- updateRow(
|
||||||
table,
|
table,
|
||||||
and(cid.is(coll), corrOrg.is(Some(currentOrg))),
|
and(cid.is(coll), corrOrg.is(Some(currentOrg))),
|
||||||
commas(corrOrg.setTo(None: Option[Ident]), updated.setTo(t))
|
commas(corrOrg.setTo(None: Option[Ident]), updated.setTo(t))
|
||||||
).update.run
|
).update.run
|
||||||
} yield n
|
} yield n
|
||||||
|
|
||||||
def updateCorrPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] =
|
def updateCorrPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] =
|
||||||
for {
|
for {
|
||||||
t <- currentTime
|
t <- currentTime
|
||||||
n <- updateRow(
|
n <- updateRow(
|
||||||
table,
|
table,
|
||||||
and(id.is(itemId), cid.is(coll)),
|
and(id.is(itemId), cid.is(coll)),
|
||||||
commas(corrPerson.setTo(person), updated.setTo(t))
|
commas(corrPerson.setTo(person), updated.setTo(t))
|
||||||
).update.run
|
).update.run
|
||||||
} yield n
|
} yield n
|
||||||
|
|
||||||
def removeCorrPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] =
|
def removeCorrPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] =
|
||||||
for {
|
for {
|
||||||
t <- currentTime
|
t <- currentTime
|
||||||
n <- updateRow(
|
n <- updateRow(
|
||||||
table,
|
table,
|
||||||
and(cid.is(coll), corrPerson.is(Some(currentPerson))),
|
and(cid.is(coll), corrPerson.is(Some(currentPerson))),
|
||||||
commas(corrPerson.setTo(None: Option[Ident]), updated.setTo(t))
|
commas(corrPerson.setTo(None: Option[Ident]), updated.setTo(t))
|
||||||
).update.run
|
).update.run
|
||||||
} yield n
|
} yield n
|
||||||
|
|
||||||
def updateConcPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] =
|
def updateConcPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] =
|
||||||
for {
|
for {
|
||||||
t <- currentTime
|
t <- currentTime
|
||||||
n <- updateRow(
|
n <- updateRow(
|
||||||
table,
|
table,
|
||||||
and(id.is(itemId), cid.is(coll)),
|
and(id.is(itemId), cid.is(coll)),
|
||||||
commas(concPerson.setTo(person), updated.setTo(t))
|
commas(concPerson.setTo(person), updated.setTo(t))
|
||||||
).update.run
|
).update.run
|
||||||
} yield n
|
} yield n
|
||||||
|
|
||||||
def removeConcPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] =
|
def removeConcPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] =
|
||||||
for {
|
for {
|
||||||
t <- currentTime
|
t <- currentTime
|
||||||
n <- updateRow(
|
n <- updateRow(
|
||||||
table,
|
table,
|
||||||
and(cid.is(coll), concPerson.is(Some(currentPerson))),
|
and(cid.is(coll), concPerson.is(Some(currentPerson))),
|
||||||
commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t))
|
commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t))
|
||||||
).update.run
|
).update.run
|
||||||
} yield n
|
} yield n
|
||||||
|
|
||||||
def updateConcEquip(itemId: Ident, coll: Ident, equip: Option[Ident]): ConnectionIO[Int] =
|
def updateConcEquip(itemId: Ident, coll: Ident, equip: Option[Ident]): ConnectionIO[Int] =
|
||||||
for {
|
for {
|
||||||
t <- currentTime
|
t <- currentTime
|
||||||
n <- updateRow(
|
n <- updateRow(
|
||||||
table,
|
table,
|
||||||
and(id.is(itemId), cid.is(coll)),
|
and(id.is(itemId), cid.is(coll)),
|
||||||
commas(concEquipment.setTo(equip), updated.setTo(t))
|
commas(concEquipment.setTo(equip), updated.setTo(t))
|
||||||
).update.run
|
).update.run
|
||||||
} yield n
|
} yield n
|
||||||
|
|
||||||
def removeConcEquip(coll: Ident, currentEquip: Ident): ConnectionIO[Int] =
|
def removeConcEquip(coll: Ident, currentEquip: Ident): ConnectionIO[Int] =
|
||||||
for {
|
for {
|
||||||
t <- currentTime
|
t <- currentTime
|
||||||
n <- updateRow(
|
n <- updateRow(
|
||||||
table,
|
table,
|
||||||
and(cid.is(coll), concEquipment.is(Some(currentEquip))),
|
and(cid.is(coll), concEquipment.is(Some(currentEquip))),
|
||||||
commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t))
|
commas(concPerson.setTo(None: Option[Ident]), updated.setTo(t))
|
||||||
).update.run
|
).update.run
|
||||||
} yield n
|
} yield n
|
||||||
|
|
||||||
def updateNotes(itemId: Ident, coll: Ident, text: Option[String]): ConnectionIO[Int] =
|
def updateNotes(itemId: Ident, coll: Ident, text: Option[String]): ConnectionIO[Int] =
|
||||||
for {
|
for {
|
||||||
t <- currentTime
|
t <- currentTime
|
||||||
n <- updateRow(
|
n <- updateRow(
|
||||||
table,
|
table,
|
||||||
and(id.is(itemId), cid.is(coll)),
|
and(id.is(itemId), cid.is(coll)),
|
||||||
commas(notes.setTo(text), updated.setTo(t))
|
commas(notes.setTo(text), updated.setTo(t))
|
||||||
).update.run
|
).update.run
|
||||||
} yield n
|
} yield n
|
||||||
|
|
||||||
def updateName(itemId: Ident, coll: Ident, itemName: String): ConnectionIO[Int] =
|
def updateName(itemId: Ident, coll: Ident, itemName: String): ConnectionIO[Int] =
|
||||||
for {
|
for {
|
||||||
t <- currentTime
|
t <- currentTime
|
||||||
n <- updateRow(
|
n <- updateRow(
|
||||||
table,
|
table,
|
||||||
and(id.is(itemId), cid.is(coll)),
|
and(id.is(itemId), cid.is(coll)),
|
||||||
commas(name.setTo(itemName), updated.setTo(t))
|
commas(name.setTo(itemName), updated.setTo(t))
|
||||||
).update.run
|
).update.run
|
||||||
} yield n
|
} yield n
|
||||||
|
|
||||||
def updateDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] =
|
def updateDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] =
|
||||||
for {
|
for {
|
||||||
t <- currentTime
|
t <- currentTime
|
||||||
n <- updateRow(
|
n <- updateRow(
|
||||||
table,
|
table,
|
||||||
and(id.is(itemId), cid.is(coll)),
|
and(id.is(itemId), cid.is(coll)),
|
||||||
commas(itemDate.setTo(date), updated.setTo(t))
|
commas(itemDate.setTo(date), updated.setTo(t))
|
||||||
).update.run
|
).update.run
|
||||||
} yield n
|
} yield n
|
||||||
|
|
||||||
def updateDueDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] =
|
def updateDueDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] =
|
||||||
for {
|
for {
|
||||||
t <- currentTime
|
t <- currentTime
|
||||||
n <- updateRow(
|
n <- updateRow(
|
||||||
table,
|
table,
|
||||||
and(id.is(itemId), cid.is(coll)),
|
and(id.is(itemId), cid.is(coll)),
|
||||||
commas(dueDate.setTo(date), updated.setTo(t))
|
commas(dueDate.setTo(date), updated.setTo(t))
|
||||||
).update.run
|
).update.run
|
||||||
} yield n
|
} yield n
|
||||||
|
|
||||||
def deleteByIdAndCollective(itemId: Ident, coll: Ident): ConnectionIO[Int] =
|
def deleteByIdAndCollective(itemId: Ident, coll: Ident): ConnectionIO[Int] =
|
||||||
|
@ -162,17 +162,17 @@ object RJob {
|
|||||||
for {
|
for {
|
||||||
_ <- incrementRetries(jobId)
|
_ <- incrementRetries(jobId)
|
||||||
n <- updateRow(
|
n <- updateRow(
|
||||||
table,
|
table,
|
||||||
and(
|
and(
|
||||||
id.is(jobId),
|
id.is(jobId),
|
||||||
or(worker.isNull, worker.is(workerId)),
|
or(worker.isNull, worker.is(workerId)),
|
||||||
state.isOneOf(Seq[JobState](JobState.Waiting, JobState.Stuck))
|
state.isOneOf(Seq[JobState](JobState.Waiting, JobState.Stuck))
|
||||||
),
|
),
|
||||||
commas(
|
commas(
|
||||||
state.setTo(JobState.Scheduled: JobState),
|
state.setTo(JobState.Scheduled: JobState),
|
||||||
worker.setTo(workerId)
|
worker.setTo(workerId)
|
||||||
)
|
)
|
||||||
).update.run
|
).update.run
|
||||||
} yield n
|
} yield n
|
||||||
|
|
||||||
def setSuccess(jobId: Ident, now: Timestamp): ConnectionIO[Int] =
|
def setSuccess(jobId: Ident, now: Timestamp): ConnectionIO[Int] =
|
||||||
|
@ -52,16 +52,16 @@ object RSentMail {
|
|||||||
for {
|
for {
|
||||||
user <- OptionT(RUser.findByAccount(accId))
|
user <- OptionT(RUser.findByAccount(accId))
|
||||||
sm <- OptionT.liftF(
|
sm <- OptionT.liftF(
|
||||||
RSentMail[ConnectionIO](
|
RSentMail[ConnectionIO](
|
||||||
user.uid,
|
user.uid,
|
||||||
messageId,
|
messageId,
|
||||||
sender,
|
sender,
|
||||||
connName,
|
connName,
|
||||||
subject,
|
subject,
|
||||||
recipients,
|
recipients,
|
||||||
body
|
body
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
si <- OptionT.liftF(RSentMailItem[ConnectionIO](itemId, sm.id, Some(sm.created)))
|
si <- OptionT.liftF(RSentMailItem[ConnectionIO](itemId, sm.id, Some(sm.created)))
|
||||||
} yield (sm, si)
|
} yield (sm, si)
|
||||||
|
|
||||||
|
@ -33,9 +33,9 @@ object RTagItem {
|
|||||||
def insertItemTags(item: Ident, tags: Seq[Ident]): ConnectionIO[Int] =
|
def insertItemTags(item: Ident, tags: Seq[Ident]): ConnectionIO[Int] =
|
||||||
for {
|
for {
|
||||||
tagValues <- tags.toList.traverse(id =>
|
tagValues <- tags.toList.traverse(id =>
|
||||||
Ident.randomId[ConnectionIO].map(rid => RTagItem(rid, item, id))
|
Ident.randomId[ConnectionIO].map(rid => RTagItem(rid, item, id))
|
||||||
)
|
)
|
||||||
tagFrag = tagValues.map(v => fr"${v.tagItemId},${v.itemId},${v.tagId}")
|
tagFrag = tagValues.map(v => fr"${v.tagItemId},${v.itemId},${v.tagId}")
|
||||||
ins <- insertRows(table, all, tagFrag).update.run
|
ins <- insertRows(table, all, tagFrag).update.run
|
||||||
} yield ins
|
} yield ins
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user