mirror of
https://github.com/TheAnachronism/docspell.git
synced 2025-06-21 09:58:26 +00:00
Update scalafmt to 2.5.1 + scalafmtAll
This commit is contained in:
@ -1,7 +1,6 @@
|
|||||||
version = "2.4.2"
|
version = "2.5.1"
|
||||||
|
|
||||||
align = more
|
preset = defaultWithAlign
|
||||||
#align.arrowEnumeratorGenerator = true
|
|
||||||
|
|
||||||
maxColumn = 90
|
maxColumn = 90
|
||||||
|
|
||||||
|
@ -49,7 +49,8 @@ object TextAnalyser {
|
|||||||
StanfordNerClassifier.nerAnnotate(lang)(text)
|
StanfordNerClassifier.nerAnnotate(lang)(text)
|
||||||
}
|
}
|
||||||
|
|
||||||
private def contactNer(text: String): F[Vector[NerLabel]] = Sync[F].delay {
|
private def contactNer(text: String): F[Vector[NerLabel]] =
|
||||||
|
Sync[F].delay {
|
||||||
Contact.annotate(text)
|
Contact.annotate(text)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -71,7 +71,8 @@ object Login {
|
|||||||
for {
|
for {
|
||||||
data <- store.transact(QLogin.findUser(acc))
|
data <- store.transact(QLogin.findUser(acc))
|
||||||
_ <- Sync[F].delay(logger.trace(s"Account lookup: $data"))
|
_ <- Sync[F].delay(logger.trace(s"Account lookup: $data"))
|
||||||
res <- if (data.exists(check(up.pass))) okResult
|
res <-
|
||||||
|
if (data.exists(check(up.pass))) okResult
|
||||||
else Result.invalidAuth.pure[F]
|
else Result.invalidAuth.pure[F]
|
||||||
} yield res
|
} yield res
|
||||||
case Left(_) =>
|
case Left(_) =>
|
||||||
|
@ -117,9 +117,12 @@ object OCollective {
|
|||||||
val q = for {
|
val q = for {
|
||||||
optUser <- RUser.findByAccount(accountId)
|
optUser <- RUser.findByAccount(accountId)
|
||||||
check = optUser.map(_.password).map(p => PasswordCrypt.check(current, p))
|
check = optUser.map(_.password).map(p => PasswordCrypt.check(current, p))
|
||||||
n <- check
|
n <-
|
||||||
|
check
|
||||||
.filter(identity)
|
.filter(identity)
|
||||||
.traverse(_ => RUser.updatePassword(accountId, PasswordCrypt.crypt(newPass)))
|
.traverse(_ =>
|
||||||
|
RUser.updatePassword(accountId, PasswordCrypt.crypt(newPass))
|
||||||
|
)
|
||||||
res = check match {
|
res = check match {
|
||||||
case Some(true) =>
|
case Some(true) =>
|
||||||
if (n.getOrElse(0) > 0) PassChangeResult.success
|
if (n.getOrElse(0) > 0) PassChangeResult.success
|
||||||
|
@ -199,9 +199,11 @@ object OItem {
|
|||||||
def setTags(item: Ident, tagIds: List[Ident], collective: Ident): F[AddResult] = {
|
def setTags(item: Ident, tagIds: List[Ident], collective: Ident): F[AddResult] = {
|
||||||
val db = for {
|
val db = for {
|
||||||
cid <- RItem.getCollective(item)
|
cid <- RItem.getCollective(item)
|
||||||
nd <- if (cid.contains(collective)) RTagItem.deleteItemTags(item)
|
nd <-
|
||||||
|
if (cid.contains(collective)) RTagItem.deleteItemTags(item)
|
||||||
else 0.pure[ConnectionIO]
|
else 0.pure[ConnectionIO]
|
||||||
ni <- if (tagIds.nonEmpty && cid.contains(collective))
|
ni <-
|
||||||
|
if (tagIds.nonEmpty && cid.contains(collective))
|
||||||
RTagItem.insertItemTags(item, tagIds)
|
RTagItem.insertItemTags(item, tagIds)
|
||||||
else 0.pure[ConnectionIO]
|
else 0.pure[ConnectionIO]
|
||||||
} yield nd + ni
|
} yield nd + ni
|
||||||
|
@ -72,7 +72,8 @@ object OUpload {
|
|||||||
data.meta.sourceAbbrev,
|
data.meta.sourceAbbrev,
|
||||||
data.meta.validFileTypes
|
data.meta.validFileTypes
|
||||||
)
|
)
|
||||||
args = if (data.multiple) files.map(f => ProcessItemArgs(meta, List(f)))
|
args =
|
||||||
|
if (data.multiple) files.map(f => ProcessItemArgs(meta, List(f)))
|
||||||
else Vector(ProcessItemArgs(meta, files.toList))
|
else Vector(ProcessItemArgs(meta, files.toList))
|
||||||
job <- pred.traverse(_ => makeJobs(args, account, data.priority, data.tracker))
|
job <- pred.traverse(_ => makeJobs(args, account, data.priority, data.tracker))
|
||||||
_ <- logger.fdebug(s"Storing jobs: $job")
|
_ <- logger.fdebug(s"Storing jobs: $job")
|
||||||
@ -84,7 +85,8 @@ object OUpload {
|
|||||||
|
|
||||||
def submit(data: OUpload.UploadData[F], sourceId: Ident): F[OUpload.UploadResult] =
|
def submit(data: OUpload.UploadData[F], sourceId: Ident): F[OUpload.UploadResult] =
|
||||||
for {
|
for {
|
||||||
sOpt <- store
|
sOpt <-
|
||||||
|
store
|
||||||
.transact(RSource.find(sourceId))
|
.transact(RSource.find(sourceId))
|
||||||
.map(_.toRight(UploadResult.NoSource))
|
.map(_.toRight(UploadResult.NoSource))
|
||||||
abbrev = sOpt.map(_.abbrev).toOption.getOrElse(data.meta.sourceAbbrev)
|
abbrev = sOpt.map(_.abbrev).toOption.getOrElse(data.meta.sourceAbbrev)
|
||||||
@ -108,10 +110,15 @@ object OUpload {
|
|||||||
.lastOrError
|
.lastOrError
|
||||||
.map(fm => Ident.unsafe(fm.id))
|
.map(fm => Ident.unsafe(fm.id))
|
||||||
.attempt
|
.attempt
|
||||||
.map(_.fold(ex => {
|
.map(
|
||||||
|
_.fold(
|
||||||
|
ex => {
|
||||||
logger.warn(ex)(s"Could not store file for processing!")
|
logger.warn(ex)(s"Could not store file for processing!")
|
||||||
None
|
None
|
||||||
}, id => Some(ProcessItemArgs.File(file.name, id))))
|
},
|
||||||
|
id => Some(ProcessItemArgs.File(file.name, id))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
private def checkFileList(
|
private def checkFileList(
|
||||||
files: Seq[ProcessItemArgs.File]
|
files: Seq[ProcessItemArgs.File]
|
||||||
|
@ -27,8 +27,8 @@ trait OUserTask[F[_]] {
|
|||||||
* executor's queue. It will not update the corresponding periodic
|
* executor's queue. It will not update the corresponding periodic
|
||||||
* task.
|
* task.
|
||||||
*/
|
*/
|
||||||
def executeNow[A](account: AccountId, task: UserTask[A])(
|
def executeNow[A](account: AccountId, task: UserTask[A])(implicit
|
||||||
implicit E: Encoder[A]
|
E: Encoder[A]
|
||||||
): F[Unit]
|
): F[Unit]
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -41,8 +41,8 @@ object OUserTask {
|
|||||||
): Resource[F, OUserTask[F]] =
|
): Resource[F, OUserTask[F]] =
|
||||||
Resource.pure[F, OUserTask[F]](new OUserTask[F] {
|
Resource.pure[F, OUserTask[F]](new OUserTask[F] {
|
||||||
|
|
||||||
def executeNow[A](account: AccountId, task: UserTask[A])(
|
def executeNow[A](account: AccountId, task: UserTask[A])(implicit
|
||||||
implicit E: Encoder[A]
|
E: Encoder[A]
|
||||||
): F[Unit] =
|
): F[Unit] =
|
||||||
for {
|
for {
|
||||||
ptask <- task.encode.toPeriodicTask(account)
|
ptask <- task.encode.toPeriodicTask(account)
|
||||||
|
@ -25,16 +25,15 @@ object OSignup {
|
|||||||
Resource.pure[F, OSignup[F]](new OSignup[F] {
|
Resource.pure[F, OSignup[F]](new OSignup[F] {
|
||||||
|
|
||||||
def newInvite(cfg: Config)(password: Password): F[NewInviteResult] =
|
def newInvite(cfg: Config)(password: Password): F[NewInviteResult] =
|
||||||
if (cfg.mode == Config.Mode.Invite) {
|
if (cfg.mode == Config.Mode.Invite)
|
||||||
if (cfg.newInvitePassword.isEmpty || cfg.newInvitePassword != password)
|
if (cfg.newInvitePassword.isEmpty || cfg.newInvitePassword != password)
|
||||||
NewInviteResult.passwordMismatch.pure[F]
|
NewInviteResult.passwordMismatch.pure[F]
|
||||||
else
|
else
|
||||||
store
|
store
|
||||||
.transact(RInvitation.insertNew)
|
.transact(RInvitation.insertNew)
|
||||||
.map(ri => NewInviteResult.success(ri.id))
|
.map(ri => NewInviteResult.success(ri.id))
|
||||||
} else {
|
else
|
||||||
Effect[F].pure(NewInviteResult.invitationClosed)
|
Effect[F].pure(NewInviteResult.invitationClosed)
|
||||||
}
|
|
||||||
|
|
||||||
def register(cfg: Config)(data: RegisterData): F[SignupResult] =
|
def register(cfg: Config)(data: RegisterData): F[SignupResult] =
|
||||||
cfg.mode match {
|
cfg.mode match {
|
||||||
@ -51,11 +50,15 @@ object OSignup {
|
|||||||
now <- Timestamp.current[F]
|
now <- Timestamp.current[F]
|
||||||
min = now.minus(cfg.inviteTime)
|
min = now.minus(cfg.inviteTime)
|
||||||
ok <- store.transact(RInvitation.useInvite(inv, min))
|
ok <- store.transact(RInvitation.useInvite(inv, min))
|
||||||
res <- if (ok) addUser(data).map(SignupResult.fromAddResult)
|
res <-
|
||||||
|
if (ok) addUser(data).map(SignupResult.fromAddResult)
|
||||||
else SignupResult.invalidInvitationKey.pure[F]
|
else SignupResult.invalidInvitationKey.pure[F]
|
||||||
_ <- if (retryInvite(res))
|
_ <-
|
||||||
|
if (retryInvite(res))
|
||||||
logger
|
logger
|
||||||
.fdebug(s"Adding account failed ($res). Allow retry with invite.") *> store
|
.fdebug(
|
||||||
|
s"Adding account failed ($res). Allow retry with invite."
|
||||||
|
) *> store
|
||||||
.transact(
|
.transact(
|
||||||
RInvitation.insert(RInvitation(inv, now))
|
RInvitation.insert(RInvitation(inv, now))
|
||||||
)
|
)
|
||||||
|
@ -18,7 +18,8 @@ object SignupResult {
|
|||||||
def failure(ex: Throwable): SignupResult = Failure(ex)
|
def failure(ex: Throwable): SignupResult = Failure(ex)
|
||||||
def success: SignupResult = Success
|
def success: SignupResult = Success
|
||||||
|
|
||||||
def fromAddResult(ar: AddResult): SignupResult = ar match {
|
def fromAddResult(ar: AddResult): SignupResult =
|
||||||
|
ar match {
|
||||||
case AddResult.Success => Success
|
case AddResult.Success => Success
|
||||||
case AddResult.Failure(ex) => Failure(ex)
|
case AddResult.Failure(ex) => Failure(ex)
|
||||||
case AddResult.EntityExists(_) => CollectiveExists
|
case AddResult.EntityExists(_) => CollectiveExists
|
||||||
|
@ -37,11 +37,10 @@ object Binary {
|
|||||||
Binary(name, MimeType.html.withCharset(cs), Stream.chunk(Chunk.byteVector(content)))
|
Binary(name, MimeType.html.withCharset(cs), Stream.chunk(Chunk.byteVector(content)))
|
||||||
|
|
||||||
def decode[F[_]](cs: Charset): Pipe[F, Byte, String] =
|
def decode[F[_]](cs: Charset): Pipe[F, Byte, String] =
|
||||||
if (cs == StandardCharsets.UTF_8) {
|
if (cs == StandardCharsets.UTF_8)
|
||||||
fs2.text.utf8Decode
|
fs2.text.utf8Decode
|
||||||
} else {
|
else
|
||||||
util.decode[F](cs)
|
util.decode[F](cs)
|
||||||
}
|
|
||||||
|
|
||||||
def loadAllBytes[F[_]: Sync](data: Stream[F, Byte]): F[ByteVector] =
|
def loadAllBytes[F[_]: Sync](data: Stream[F, Byte]): F[ByteVector] =
|
||||||
data.chunks.map(_.toByteVector).compile.fold(ByteVector.empty)((r, e) => r ++ e)
|
data.chunks.map(_.toByteVector).compile.fold(ByteVector.empty)((r, e) => r ++ e)
|
||||||
@ -78,17 +77,16 @@ object Binary {
|
|||||||
decoder.decode(byteBuffer, charBuffer, false)
|
decoder.decode(byteBuffer, charBuffer, false)
|
||||||
val nextStream = stream.consChunk(Chunk.byteBuffer(byteBuffer.slice()))
|
val nextStream = stream.consChunk(Chunk.byteBuffer(byteBuffer.slice()))
|
||||||
Pull.output1(charBuffer.flip().toString).as(Some(nextStream))
|
Pull.output1(charBuffer.flip().toString).as(Some(nextStream))
|
||||||
} else {
|
} else
|
||||||
Pull.output(Chunk.empty[String]).as(Some(stream))
|
Pull.output(Chunk.empty[String]).as(Some(stream))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
private def skipByteOrderMark[F[_]](chunk: Chunk[Byte]): Chunk[Byte] =
|
private def skipByteOrderMark[F[_]](chunk: Chunk[Byte]): Chunk[Byte] =
|
||||||
if (chunk.size >= 3 && chunk.take(3) == utf8Bom) {
|
if (chunk.size >= 3 && chunk.take(3) == utf8Bom)
|
||||||
chunk.drop(3)
|
chunk.drop(3)
|
||||||
} else chunk
|
else chunk
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -33,7 +33,8 @@ object CollectiveState {
|
|||||||
def unsafe(str: String): CollectiveState =
|
def unsafe(str: String): CollectiveState =
|
||||||
fromString(str).fold(sys.error, identity)
|
fromString(str).fold(sys.error, identity)
|
||||||
|
|
||||||
def asString(state: CollectiveState): String = state match {
|
def asString(state: CollectiveState): String =
|
||||||
|
state match {
|
||||||
case Active => "active"
|
case Active => "active"
|
||||||
case Blocked => "blocked"
|
case Blocked => "blocked"
|
||||||
case Closed => "closed"
|
case Closed => "closed"
|
||||||
|
@ -25,7 +25,8 @@ object File {
|
|||||||
): F[Path] =
|
): F[Path] =
|
||||||
mkDir(parent).map(p => Files.createTempFile(p, prefix, suffix.orNull))
|
mkDir(parent).map(p => Files.createTempFile(p, prefix, suffix.orNull))
|
||||||
|
|
||||||
def deleteDirectory[F[_]: Sync](dir: Path): F[Int] = Sync[F].delay {
|
def deleteDirectory[F[_]: Sync](dir: Path): F[Int] =
|
||||||
|
Sync[F].delay {
|
||||||
val count = new AtomicInteger(0)
|
val count = new AtomicInteger(0)
|
||||||
Files.walkFileTree(
|
Files.walkFileTree(
|
||||||
dir,
|
dir,
|
||||||
|
@ -20,7 +20,8 @@ object Ident {
|
|||||||
def randomUUID[F[_]: Sync]: F[Ident] =
|
def randomUUID[F[_]: Sync]: F[Ident] =
|
||||||
Sync[F].delay(unsafe(UUID.randomUUID.toString))
|
Sync[F].delay(unsafe(UUID.randomUUID.toString))
|
||||||
|
|
||||||
def randomId[F[_]: Sync]: F[Ident] = Sync[F].delay {
|
def randomId[F[_]: Sync]: F[Ident] =
|
||||||
|
Sync[F].delay {
|
||||||
val random = new SecureRandom()
|
val random = new SecureRandom()
|
||||||
val buffer = new Array[Byte](32)
|
val buffer = new Array[Byte](32)
|
||||||
random.nextBytes(buffer)
|
random.nextBytes(buffer)
|
||||||
|
@ -122,7 +122,8 @@ object LenientUri {
|
|||||||
val isRoot = false
|
val isRoot = false
|
||||||
def /(seg: String): Path =
|
def /(seg: String): Path =
|
||||||
copy(segs = segs.append(seg))
|
copy(segs = segs.append(seg))
|
||||||
def asString = segs.head match {
|
def asString =
|
||||||
|
segs.head match {
|
||||||
case "." => segments.map(percentEncode).mkString("/")
|
case "." => segments.map(percentEncode).mkString("/")
|
||||||
case ".." => segments.map(percentEncode).mkString("/")
|
case ".." => segments.map(percentEncode).mkString("/")
|
||||||
case _ => "/" + segments.map(percentEncode).mkString("/")
|
case _ => "/" + segments.map(percentEncode).mkString("/")
|
||||||
@ -136,7 +137,8 @@ object LenientUri {
|
|||||||
unsafe(u.toExternalForm)
|
unsafe(u.toExternalForm)
|
||||||
|
|
||||||
def parse(str: String): Either[String, LenientUri] = {
|
def parse(str: String): Either[String, LenientUri] = {
|
||||||
def makePath(str: String): Path = str.trim match {
|
def makePath(str: String): Path =
|
||||||
|
str.trim match {
|
||||||
case "/" => RootPath
|
case "/" => RootPath
|
||||||
case "" => EmptyPath
|
case "" => EmptyPath
|
||||||
case _ =>
|
case _ =>
|
||||||
|
@ -17,7 +17,8 @@ trait Logger[F[_]] {
|
|||||||
|
|
||||||
object Logger {
|
object Logger {
|
||||||
|
|
||||||
def log4s[F[_]: Sync](log: Log4sLogger): Logger[F] = new Logger[F] {
|
def log4s[F[_]: Sync](log: Log4sLogger): Logger[F] =
|
||||||
|
new Logger[F] {
|
||||||
def trace(msg: => String): F[Unit] =
|
def trace(msg: => String): F[Unit] =
|
||||||
log.ftrace(msg)
|
log.ftrace(msg)
|
||||||
|
|
||||||
|
@ -47,16 +47,19 @@ object SystemCommand {
|
|||||||
for {
|
for {
|
||||||
_ <- writeToProcess(stdin, proc, blocker)
|
_ <- writeToProcess(stdin, proc, blocker)
|
||||||
term <- Sync[F].delay(proc.waitFor(cmd.timeout.seconds, TimeUnit.SECONDS))
|
term <- Sync[F].delay(proc.waitFor(cmd.timeout.seconds, TimeUnit.SECONDS))
|
||||||
_ <- if (term)
|
_ <-
|
||||||
|
if (term)
|
||||||
logger.debug(s"Command `${cmd.cmdString}` finished: ${proc.exitValue}")
|
logger.debug(s"Command `${cmd.cmdString}` finished: ${proc.exitValue}")
|
||||||
else
|
else
|
||||||
logger.warn(
|
logger.warn(
|
||||||
s"Command `${cmd.cmdString}` did not finish in ${cmd.timeout.formatExact}!"
|
s"Command `${cmd.cmdString}` did not finish in ${cmd.timeout.formatExact}!"
|
||||||
)
|
)
|
||||||
_ <- if (!term) timeoutError(proc, cmd) else Sync[F].pure(())
|
_ <- if (!term) timeoutError(proc, cmd) else Sync[F].pure(())
|
||||||
out <- if (term) inputStreamToString(proc.getInputStream, blocker)
|
out <-
|
||||||
|
if (term) inputStreamToString(proc.getInputStream, blocker)
|
||||||
else Sync[F].pure("")
|
else Sync[F].pure("")
|
||||||
err <- if (term) inputStreamToString(proc.getErrorStream, blocker)
|
err <-
|
||||||
|
if (term) inputStreamToString(proc.getErrorStream, blocker)
|
||||||
else Sync[F].pure("")
|
else Sync[F].pure("")
|
||||||
} yield Result(proc.exitValue, out, err)
|
} yield Result(proc.exitValue, out, err)
|
||||||
}
|
}
|
||||||
|
@ -22,7 +22,8 @@ object UserState {
|
|||||||
def unsafe(str: String): UserState =
|
def unsafe(str: String): UserState =
|
||||||
fromString(str).fold(sys.error, identity)
|
fromString(str).fold(sys.error, identity)
|
||||||
|
|
||||||
def asString(s: UserState): String = s match {
|
def asString(s: UserState): String =
|
||||||
|
s match {
|
||||||
case Active => "active"
|
case Active => "active"
|
||||||
case Disabled => "disabled"
|
case Disabled => "disabled"
|
||||||
}
|
}
|
||||||
|
@ -3,14 +3,16 @@ package docspell.common.syntax
|
|||||||
trait EitherSyntax {
|
trait EitherSyntax {
|
||||||
|
|
||||||
implicit final class LeftStringEitherOps[A](e: Either[String, A]) {
|
implicit final class LeftStringEitherOps[A](e: Either[String, A]) {
|
||||||
def throwLeft: A = e match {
|
def throwLeft: A =
|
||||||
|
e match {
|
||||||
case Right(a) => a
|
case Right(a) => a
|
||||||
case Left(err) => sys.error(err)
|
case Left(err) => sys.error(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
implicit final class ThrowableLeftEitherOps[A](e: Either[Throwable, A]) {
|
implicit final class ThrowableLeftEitherOps[A](e: Either[Throwable, A]) {
|
||||||
def throwLeft: A = e match {
|
def throwLeft: A =
|
||||||
|
e match {
|
||||||
case Right(a) => a
|
case Right(a) => a
|
||||||
case Left(err) => throw err
|
case Left(err) => throw err
|
||||||
}
|
}
|
||||||
|
@ -16,7 +16,8 @@ trait StreamSyntax {
|
|||||||
.last
|
.last
|
||||||
.map(optStr =>
|
.map(optStr =>
|
||||||
for {
|
for {
|
||||||
str <- optStr
|
str <-
|
||||||
|
optStr
|
||||||
.map(_.trim)
|
.map(_.trim)
|
||||||
.toRight(new Exception("Empty string cannot be parsed into a value"))
|
.toRight(new Exception("Empty string cannot be parsed into a value"))
|
||||||
json <- parse(str).leftMap(_.underlying)
|
json <- parse(str).leftMap(_.underlying)
|
||||||
|
@ -63,7 +63,7 @@ object Conversion {
|
|||||||
case Images(mt) =>
|
case Images(mt) =>
|
||||||
ImageSize.get(in).flatMap {
|
ImageSize.get(in).flatMap {
|
||||||
case Some(dim) =>
|
case Some(dim) =>
|
||||||
if (dim.product > cfg.maxImageSize) {
|
if (dim.product > cfg.maxImageSize)
|
||||||
logger
|
logger
|
||||||
.info(
|
.info(
|
||||||
s"Image size (${dim.product}) is too large (max ${cfg.maxImageSize})."
|
s"Image size (${dim.product}) is too large (max ${cfg.maxImageSize})."
|
||||||
@ -74,12 +74,11 @@ object Conversion {
|
|||||||
s"Image size (${dim.width}x${dim.height}) is too large (max ${cfg.maxImageSize})."
|
s"Image size (${dim.width}x${dim.height}) is too large (max ${cfg.maxImageSize})."
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
} else {
|
else
|
||||||
Tesseract.toPDF(cfg.tesseract, lang, cfg.chunkSize, blocker, logger)(
|
Tesseract.toPDF(cfg.tesseract, lang, cfg.chunkSize, blocker, logger)(
|
||||||
in,
|
in,
|
||||||
handler
|
handler
|
||||||
)
|
)
|
||||||
}
|
|
||||||
|
|
||||||
case None =>
|
case None =>
|
||||||
logger.info(
|
logger.info(
|
||||||
|
@ -90,14 +90,13 @@ private[extern] object ExternConv {
|
|||||||
val outTxt = out.resolveSibling(s"$outPrefix.txt")
|
val outTxt = out.resolveSibling(s"$outPrefix.txt")
|
||||||
File.exists(outTxt).flatMap { txtExists =>
|
File.exists(outTxt).flatMap { txtExists =>
|
||||||
val pdfData = File.readAll(out, blocker, chunkSize)
|
val pdfData = File.readAll(out, blocker, chunkSize)
|
||||||
if (result.rc == 0) {
|
if (result.rc == 0)
|
||||||
if (txtExists) successPdfTxt(pdfData, File.readText(outTxt, blocker)).pure[F]
|
if (txtExists) successPdfTxt(pdfData, File.readText(outTxt, blocker)).pure[F]
|
||||||
else successPdf(pdfData).pure[F]
|
else successPdf(pdfData).pure[F]
|
||||||
} else {
|
else
|
||||||
logger.warn(s"Command not successful (rc=${result.rc}), but file exists.") *>
|
logger.warn(s"Command not successful (rc=${result.rc}), but file exists.") *>
|
||||||
successPdf(pdfData).pure[F]
|
successPdf(pdfData).pure[F]
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
case false =>
|
case false =>
|
||||||
ConversionResult
|
ConversionResult
|
||||||
|
@ -24,7 +24,7 @@ object ExternConvTest extends SimpleTestSuite with FileChecks {
|
|||||||
)
|
)
|
||||||
|
|
||||||
if (!commandExists(cfg.program)) ignore(s"Command ${cfg.program} not found")
|
if (!commandExists(cfg.program)) ignore(s"Command ${cfg.program} not found")
|
||||||
else {
|
else
|
||||||
File
|
File
|
||||||
.withTempDir[IO](target, "wkhtmltopdf")
|
.withTempDir[IO](target, "wkhtmltopdf")
|
||||||
.use(dir =>
|
.use(dir =>
|
||||||
@ -43,7 +43,6 @@ object ExternConvTest extends SimpleTestSuite with FileChecks {
|
|||||||
)
|
)
|
||||||
.unsafeRunSync
|
.unsafeRunSync
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
test("convert office to pdf") {
|
test("convert office to pdf") {
|
||||||
val cfg = SystemCommand.Config(
|
val cfg = SystemCommand.Config(
|
||||||
@ -53,7 +52,7 @@ object ExternConvTest extends SimpleTestSuite with FileChecks {
|
|||||||
)
|
)
|
||||||
|
|
||||||
if (!commandExists(cfg.program)) ignore(s"Command ${cfg.program} not found")
|
if (!commandExists(cfg.program)) ignore(s"Command ${cfg.program} not found")
|
||||||
else {
|
else
|
||||||
File
|
File
|
||||||
.withTempDir[IO](target, "unoconv")
|
.withTempDir[IO](target, "unoconv")
|
||||||
.use(dir =>
|
.use(dir =>
|
||||||
@ -72,7 +71,6 @@ object ExternConvTest extends SimpleTestSuite with FileChecks {
|
|||||||
)
|
)
|
||||||
.unsafeRunSync
|
.unsafeRunSync
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
test("convert image to pdf") {
|
test("convert image to pdf") {
|
||||||
val cfg = SystemCommand.Config(
|
val cfg = SystemCommand.Config(
|
||||||
@ -82,7 +80,7 @@ object ExternConvTest extends SimpleTestSuite with FileChecks {
|
|||||||
)
|
)
|
||||||
|
|
||||||
if (!commandExists(cfg.program)) ignore(s"Command ${cfg.program} not found")
|
if (!commandExists(cfg.program)) ignore(s"Command ${cfg.program} not found")
|
||||||
else {
|
else
|
||||||
File
|
File
|
||||||
.withTempDir[IO](target, "tesseract")
|
.withTempDir[IO](target, "tesseract")
|
||||||
.use(dir =>
|
.use(dir =>
|
||||||
@ -102,6 +100,5 @@ object ExternConvTest extends SimpleTestSuite with FileChecks {
|
|||||||
)
|
)
|
||||||
.unsafeRunSync
|
.unsafeRunSync
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -60,7 +60,7 @@ object Extraction {
|
|||||||
|
|
||||||
ImageSize.get(data).flatMap {
|
ImageSize.get(data).flatMap {
|
||||||
case Some(dim) =>
|
case Some(dim) =>
|
||||||
if (dim.product > cfg.ocr.maxImageSize) {
|
if (dim.product > cfg.ocr.maxImageSize)
|
||||||
logger.info(
|
logger.info(
|
||||||
s"Image size (${dim.product}) is too large (max ${cfg.ocr.maxImageSize})."
|
s"Image size (${dim.product}) is too large (max ${cfg.ocr.maxImageSize})."
|
||||||
) *>
|
) *>
|
||||||
@ -71,9 +71,8 @@ object Extraction {
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
.pure[F]
|
.pure[F]
|
||||||
} else {
|
else
|
||||||
doExtract
|
doExtract
|
||||||
}
|
|
||||||
case None =>
|
case None =>
|
||||||
logger.info(
|
logger.info(
|
||||||
s"Cannot read image data from ${mt.asString}. Extracting anyways."
|
s"Cannot read image data from ${mt.asString}. Extracting anyways."
|
||||||
|
@ -33,7 +33,8 @@ object PdfExtract {
|
|||||||
|
|
||||||
//maybe better: inspect the pdf and decide whether ocr or not
|
//maybe better: inspect the pdf and decide whether ocr or not
|
||||||
for {
|
for {
|
||||||
pdfboxRes <- logger.debug("Trying to strip text from pdf using pdfbox.") *> PdfboxExtract
|
pdfboxRes <-
|
||||||
|
logger.debug("Trying to strip text from pdf using pdfbox.") *> PdfboxExtract
|
||||||
.get[F](in)
|
.get[F](in)
|
||||||
res <- pdfboxRes.fold(
|
res <- pdfboxRes.fold(
|
||||||
ex =>
|
ex =>
|
||||||
@ -44,7 +45,9 @@ object PdfExtract {
|
|||||||
if (str.length >= stripMinLen) str.pure[F].attempt
|
if (str.length >= stripMinLen) str.pure[F].attempt
|
||||||
else
|
else
|
||||||
logger
|
logger
|
||||||
.info(s"Stripped text from PDF is small (${str.length}). Trying with OCR.") *>
|
.info(
|
||||||
|
s"Stripped text from PDF is small (${str.length}). Trying with OCR."
|
||||||
|
) *>
|
||||||
runOcr.flatMap(ocrStr => chooseResult(ocrStr, str)).attempt
|
runOcr.flatMap(ocrStr => chooseResult(ocrStr, str)).attempt
|
||||||
)
|
)
|
||||||
} yield res
|
} yield res
|
||||||
|
@ -36,7 +36,8 @@ object TikaMimetype {
|
|||||||
md
|
md
|
||||||
}
|
}
|
||||||
|
|
||||||
private def normalize(in: MimeType): MimeType = in match {
|
private def normalize(in: MimeType): MimeType =
|
||||||
|
in match {
|
||||||
case MimeType(_, sub, p) if sub contains "xhtml" =>
|
case MimeType(_, sub, p) if sub contains "xhtml" =>
|
||||||
MimeType.html.copy(params = p)
|
MimeType.html.copy(params = p)
|
||||||
case _ => in
|
case _ => in
|
||||||
@ -46,14 +47,14 @@ object TikaMimetype {
|
|||||||
val mt = convert(
|
val mt = convert(
|
||||||
tika.detect(new java.io.ByteArrayInputStream(bv), makeMetadata(hint))
|
tika.detect(new java.io.ByteArrayInputStream(bv), makeMetadata(hint))
|
||||||
)
|
)
|
||||||
if (mt.primary == "text") {
|
if (mt.primary == "text")
|
||||||
charsetFromBytes(bv, hint) match {
|
charsetFromBytes(bv, hint) match {
|
||||||
case Some(cs) =>
|
case Some(cs) =>
|
||||||
mt.withCharset(cs)
|
mt.withCharset(cs)
|
||||||
case None =>
|
case None =>
|
||||||
mt
|
mt
|
||||||
}
|
}
|
||||||
} else mt
|
else mt
|
||||||
}
|
}
|
||||||
|
|
||||||
private def charsetFromBytes(bv: Array[Byte], hint: MimeTypeHint): Option[Charset] =
|
private def charsetFromBytes(bv: Array[Byte], hint: MimeTypeHint): Option[Charset] =
|
||||||
|
@ -8,13 +8,15 @@ import scala.concurrent.ExecutionContext
|
|||||||
object Playing extends IOApp {
|
object Playing extends IOApp {
|
||||||
val blocker = Blocker.liftExecutionContext(ExecutionContext.global)
|
val blocker = Blocker.liftExecutionContext(ExecutionContext.global)
|
||||||
|
|
||||||
def run(args: List[String]): IO[ExitCode] = IO {
|
def run(args: List[String]): IO[ExitCode] =
|
||||||
|
IO {
|
||||||
//val ods = ExampleFiles.examples_sample_ods.readURL[IO](8192, blocker)
|
//val ods = ExampleFiles.examples_sample_ods.readURL[IO](8192, blocker)
|
||||||
//val odt = ExampleFiles.examples_sample_odt.readURL[IO](8192, blocker)
|
//val odt = ExampleFiles.examples_sample_odt.readURL[IO](8192, blocker)
|
||||||
val rtf = ExampleFiles.examples_sample_rtf.readURL[IO](8192, blocker)
|
val rtf = ExampleFiles.examples_sample_rtf.readURL[IO](8192, blocker)
|
||||||
|
|
||||||
val x = for {
|
val x = for {
|
||||||
odsm1 <- TikaMimetype
|
odsm1 <-
|
||||||
|
TikaMimetype
|
||||||
.detect(
|
.detect(
|
||||||
rtf,
|
rtf,
|
||||||
MimeTypeHint.filename(ExampleFiles.examples_sample_rtf.path.segments.last)
|
MimeTypeHint.filename(ExampleFiles.examples_sample_rtf.path.segments.last)
|
||||||
|
@ -28,7 +28,8 @@ object JoexServer {
|
|||||||
val app = for {
|
val app = for {
|
||||||
signal <- Resource.liftF(SignallingRef[F, Boolean](false))
|
signal <- Resource.liftF(SignallingRef[F, Boolean](false))
|
||||||
exitCode <- Resource.liftF(Ref[F].of(ExitCode.Success))
|
exitCode <- Resource.liftF(Ref[F].of(ExitCode.Success))
|
||||||
joexApp <- JoexAppImpl
|
joexApp <-
|
||||||
|
JoexAppImpl
|
||||||
.create[F](cfg, signal, pools.connectEC, pools.httpClientEC, pools.blocker)
|
.create[F](cfg, signal, pools.connectEC, pools.httpClientEC, pools.blocker)
|
||||||
|
|
||||||
httpApp = Router(
|
httpApp = Router(
|
||||||
|
@ -31,9 +31,8 @@ object Main extends IOApp {
|
|||||||
if (!Files.exists(path)) {
|
if (!Files.exists(path)) {
|
||||||
logger.info(s"Not using config file '$f' because it doesn't exist")
|
logger.info(s"Not using config file '$f' because it doesn't exist")
|
||||||
System.clearProperty("config.file")
|
System.clearProperty("config.file")
|
||||||
} else {
|
} else
|
||||||
logger.info(s"Using config file from system properties: $f")
|
logger.info(s"Using config file from system properties: $f")
|
||||||
}
|
|
||||||
case _ =>
|
case _ =>
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -11,7 +11,7 @@ object CleanupInvitesTask {
|
|||||||
|
|
||||||
def apply[F[_]: Sync](cfg: HouseKeepingConfig.CleanupInvites): Task[F, Unit, Unit] =
|
def apply[F[_]: Sync](cfg: HouseKeepingConfig.CleanupInvites): Task[F, Unit, Unit] =
|
||||||
Task { ctx =>
|
Task { ctx =>
|
||||||
if (cfg.enabled) {
|
if (cfg.enabled)
|
||||||
for {
|
for {
|
||||||
now <- Timestamp.current[F]
|
now <- Timestamp.current[F]
|
||||||
ts = now - cfg.olderThan
|
ts = now - cfg.olderThan
|
||||||
@ -19,8 +19,7 @@ object CleanupInvitesTask {
|
|||||||
n <- ctx.store.transact(RInvitation.deleteOlderThan(ts))
|
n <- ctx.store.transact(RInvitation.deleteOlderThan(ts))
|
||||||
_ <- ctx.logger.info(s"Removed $n invitations")
|
_ <- ctx.logger.info(s"Removed $n invitations")
|
||||||
} yield ()
|
} yield ()
|
||||||
} else {
|
else
|
||||||
ctx.logger.info("CleanupInvites task is disabled in the configuration")
|
ctx.logger.info("CleanupInvites task is disabled in the configuration")
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
@ -13,7 +13,7 @@ object CleanupJobsTask {
|
|||||||
|
|
||||||
def apply[F[_]: Sync](cfg: HouseKeepingConfig.CleanupJobs): Task[F, Unit, Unit] =
|
def apply[F[_]: Sync](cfg: HouseKeepingConfig.CleanupJobs): Task[F, Unit, Unit] =
|
||||||
Task { ctx =>
|
Task { ctx =>
|
||||||
if (cfg.enabled) {
|
if (cfg.enabled)
|
||||||
for {
|
for {
|
||||||
now <- Timestamp.current[F]
|
now <- Timestamp.current[F]
|
||||||
ts = now - cfg.olderThan
|
ts = now - cfg.olderThan
|
||||||
@ -21,10 +21,9 @@ object CleanupJobsTask {
|
|||||||
n <- deleteDoneJobs(ctx.store, ts, cfg.deleteBatch)
|
n <- deleteDoneJobs(ctx.store, ts, cfg.deleteBatch)
|
||||||
_ <- ctx.logger.info(s"Removed $n jobs")
|
_ <- ctx.logger.info(s"Removed $n jobs")
|
||||||
} yield ()
|
} yield ()
|
||||||
} else {
|
else
|
||||||
ctx.logger.info("CleanupJobs task is disabled in the configuration")
|
ctx.logger.info("CleanupJobs task is disabled in the configuration")
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
def deleteDoneJobs[F[_]: Sync](store: Store[F], ts: Timestamp, batch: Int): F[Int] =
|
def deleteDoneJobs[F[_]: Sync](store: Store[F], ts: Timestamp, batch: Int): F[Int] =
|
||||||
Stream
|
Stream
|
||||||
|
@ -69,7 +69,8 @@ object NotifyDueItemsTask {
|
|||||||
def findItems[F[_]: Sync](ctx: Context[F, Args]): F[Vector[QItem.ListItem]] =
|
def findItems[F[_]: Sync](ctx: Context[F, Args]): F[Vector[QItem.ListItem]] =
|
||||||
for {
|
for {
|
||||||
now <- Timestamp.current[F]
|
now <- Timestamp.current[F]
|
||||||
q = QItem.Query
|
q =
|
||||||
|
QItem.Query
|
||||||
.empty(ctx.args.account.collective)
|
.empty(ctx.args.account.collective)
|
||||||
.copy(
|
.copy(
|
||||||
states = ItemState.validStates,
|
states = ItemState.validStates,
|
||||||
|
@ -89,10 +89,12 @@ object CreateItem {
|
|||||||
Task { ctx =>
|
Task { ctx =>
|
||||||
for {
|
for {
|
||||||
cand <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId)))
|
cand <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId)))
|
||||||
_ <- if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.")
|
_ <-
|
||||||
|
if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.")
|
||||||
else ().pure[F]
|
else ().pure[F]
|
||||||
ht <- cand.drop(1).traverse(ri => QItem.delete(ctx.store)(ri.id, ri.cid))
|
ht <- cand.drop(1).traverse(ri => QItem.delete(ctx.store)(ri.id, ri.cid))
|
||||||
_ <- if (ht.sum > 0)
|
_ <-
|
||||||
|
if (ht.sum > 0)
|
||||||
ctx.logger.warn(s"Removed ${ht.sum} items with same attachments")
|
ctx.logger.warn(s"Removed ${ht.sum} items with same attachments")
|
||||||
else ().pure[F]
|
else ().pure[F]
|
||||||
rms <- OptionT(
|
rms <- OptionT(
|
||||||
@ -103,7 +105,8 @@ object CreateItem {
|
|||||||
orig <- rms.traverse(a =>
|
orig <- rms.traverse(a =>
|
||||||
ctx.store.transact(RAttachmentSource.findById(a.id)).map(s => (a, s))
|
ctx.store.transact(RAttachmentSource.findById(a.id)).map(s => (a, s))
|
||||||
)
|
)
|
||||||
origMap = orig
|
origMap =
|
||||||
|
orig
|
||||||
.map(originFileTuple)
|
.map(originFileTuple)
|
||||||
.toMap
|
.toMap
|
||||||
} yield cand.headOption.map(ri =>
|
} yield cand.headOption.map(ri =>
|
||||||
@ -116,13 +119,12 @@ object CreateItem {
|
|||||||
saved: Vector[RAttachment],
|
saved: Vector[RAttachment],
|
||||||
saveCount: Int
|
saveCount: Int
|
||||||
): F[Unit] =
|
): F[Unit] =
|
||||||
if (ctx.args.files.size != saved.size) {
|
if (ctx.args.files.size != saved.size)
|
||||||
ctx.logger.warn(
|
ctx.logger.warn(
|
||||||
s"Not all given files (${ctx.args.files.size}) have been stored. Files retained: ${saved.size}; saveCount=$saveCount"
|
s"Not all given files (${ctx.args.files.size}) have been stored. Files retained: ${saved.size}; saveCount=$saveCount"
|
||||||
)
|
)
|
||||||
} else {
|
else
|
||||||
().pure[F]
|
().pure[F]
|
||||||
}
|
|
||||||
|
|
||||||
private def storeItemError[F[_]: Sync](ctx: Context[F, ProcessItemArgs]): F[Unit] = {
|
private def storeItemError[F[_]: Sync](ctx: Context[F, ProcessItemArgs]): F[Unit] = {
|
||||||
val msg = "Inserting item failed. DB returned 0 update count!"
|
val msg = "Inserting item failed. DB returned 0 update count!"
|
||||||
|
@ -79,8 +79,10 @@ object EvalProposals {
|
|||||||
if (mt == MetaProposalType.CorrOrg) 0.8
|
if (mt == MetaProposalType.CorrOrg) 0.8
|
||||||
else 1.0
|
else 1.0
|
||||||
case NerTag.Person =>
|
case NerTag.Person =>
|
||||||
if (mt == MetaProposalType.CorrPerson ||
|
if (
|
||||||
mt == MetaProposalType.ConcPerson) 0.8
|
mt == MetaProposalType.CorrPerson ||
|
||||||
|
mt == MetaProposalType.ConcPerson
|
||||||
|
) 0.8
|
||||||
else 1.0
|
else 1.0
|
||||||
case NerTag.Website => 0.5
|
case NerTag.Website => 0.5
|
||||||
}
|
}
|
||||||
|
@ -103,7 +103,8 @@ object FindProposal {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def nextWhenEmpty(f: Finder[F], mt0: MetaProposalType, mts: MetaProposalType*)(
|
def nextWhenEmpty(f: Finder[F], mt0: MetaProposalType, mts: MetaProposalType*)(
|
||||||
implicit F: FlatMap[F],
|
implicit
|
||||||
|
F: FlatMap[F],
|
||||||
F2: Applicative[F]
|
F2: Applicative[F]
|
||||||
): Finder[F] =
|
): Finder[F] =
|
||||||
flatMap { res0 =>
|
flatMap { res0 =>
|
||||||
@ -139,11 +140,11 @@ object FindProposal {
|
|||||||
val minLength =
|
val minLength =
|
||||||
if (exact) 2 else 5
|
if (exact) 2 else 5
|
||||||
|
|
||||||
if (value.length < minLength) {
|
if (value.length < minLength)
|
||||||
ctx.logger
|
ctx.logger
|
||||||
.debug(s"Skipping too small value '$value' (original '${nt.label}').")
|
.debug(s"Skipping too small value '$value' (original '${nt.label}').")
|
||||||
.map(_ => MetaProposalList.empty)
|
.map(_ => MetaProposalList.empty)
|
||||||
} else
|
else
|
||||||
nt.tag match {
|
nt.tag match {
|
||||||
case NerTag.Organization =>
|
case NerTag.Organization =>
|
||||||
ctx.logger.debug(s"Looking for organizations: $value") *>
|
ctx.logger.debug(s"Looking for organizations: $value") *>
|
||||||
@ -191,9 +192,8 @@ object FindProposal {
|
|||||||
.map(s => s"%$s%")
|
.map(s => s"%$s%")
|
||||||
.getOrElse(value)
|
.getOrElse(value)
|
||||||
searchContact(nt, ContactKind.Website, searchString, ctx)
|
searchContact(nt, ContactKind.Website, searchString, ctx)
|
||||||
} else {
|
} else
|
||||||
searchContact(nt, ContactKind.Website, value, ctx)
|
searchContact(nt, ContactKind.Website, value, ctx)
|
||||||
}
|
|
||||||
|
|
||||||
case NerTag.Date =>
|
case NerTag.Date =>
|
||||||
// There is no database search required for this tag
|
// There is no database search required for this tag
|
||||||
|
@ -18,8 +18,11 @@ object TextAnalysis {
|
|||||||
for {
|
for {
|
||||||
_ <- ctx.logger.info("Starting text analysis")
|
_ <- ctx.logger.info("Starting text analysis")
|
||||||
s <- Duration.stopTime[F]
|
s <- Duration.stopTime[F]
|
||||||
t <- item.metas.toList
|
t <-
|
||||||
.traverse(annotateAttachment[F](ctx.args.meta.language, ctx.logger, analyser))
|
item.metas.toList
|
||||||
|
.traverse(
|
||||||
|
annotateAttachment[F](ctx.args.meta.language, ctx.logger, analyser)
|
||||||
|
)
|
||||||
_ <- ctx.logger.debug(s"Storing tags: ${t.map(_._1.copy(content = None))}")
|
_ <- ctx.logger.debug(s"Storing tags: ${t.map(_._1.copy(content = None))}")
|
||||||
_ <- t.traverse(m =>
|
_ <- t.traverse(m =>
|
||||||
ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels))
|
ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels))
|
||||||
|
@ -28,8 +28,8 @@ object JobTask {
|
|||||||
name: Ident,
|
name: Ident,
|
||||||
task: Task[F, A, Unit],
|
task: Task[F, A, Unit],
|
||||||
onCancel: Task[F, A, Unit]
|
onCancel: Task[F, A, Unit]
|
||||||
)(
|
)(implicit
|
||||||
implicit D: Decoder[A]
|
D: Decoder[A]
|
||||||
): JobTask[F] = {
|
): JobTask[F] = {
|
||||||
val convert: String => F[A] =
|
val convert: String => F[A] =
|
||||||
str =>
|
str =>
|
||||||
|
@ -96,7 +96,8 @@ final class SchedulerImpl[F[_]: ConcurrentEffect: ContextShift](
|
|||||||
_ <- permits.acquire
|
_ <- permits.acquire
|
||||||
_ <- logger.fdebug("New permit acquired")
|
_ <- logger.fdebug("New permit acquired")
|
||||||
down <- state.get.map(_.shutdownRequest)
|
down <- state.get.map(_.shutdownRequest)
|
||||||
rjob <- if (down)
|
rjob <-
|
||||||
|
if (down)
|
||||||
logger.finfo("") *> permits.release *> (None: Option[RJob]).pure[F]
|
logger.finfo("") *> permits.release *> (None: Option[RJob]).pure[F]
|
||||||
else
|
else
|
||||||
queue.nextJob(
|
queue.nextJob(
|
||||||
@ -128,7 +129,8 @@ final class SchedulerImpl[F[_]: ConcurrentEffect: ContextShift](
|
|||||||
|
|
||||||
def execute(job: RJob): F[Unit] = {
|
def execute(job: RJob): F[Unit] = {
|
||||||
val task = for {
|
val task = for {
|
||||||
jobtask <- tasks
|
jobtask <-
|
||||||
|
tasks
|
||||||
.find(job.task)
|
.find(job.task)
|
||||||
.toRight(s"This executor cannot run tasks with name: ${job.task}")
|
.toRight(s"This executor cannot run tasks with name: ${job.task}")
|
||||||
} yield jobtask
|
} yield jobtask
|
||||||
|
@ -25,8 +25,8 @@ trait Task[F[_], A, B] {
|
|||||||
def mapF[C](f: F[B] => F[C]): Task[F, A, C] =
|
def mapF[C](f: F[B] => F[C]): Task[F, A, C] =
|
||||||
Task(Task.toKleisli(this).mapF(f))
|
Task(Task.toKleisli(this).mapF(f))
|
||||||
|
|
||||||
def attempt(
|
def attempt(implicit
|
||||||
implicit F: ApplicativeError[F, Throwable]
|
F: ApplicativeError[F, Throwable]
|
||||||
): Task[F, A, Either[Throwable, B]] =
|
): Task[F, A, Either[Throwable, B]] =
|
||||||
mapF(_.attempt)
|
mapF(_.attempt)
|
||||||
|
|
||||||
|
@ -31,9 +31,8 @@ object Main extends IOApp {
|
|||||||
if (!Files.exists(path)) {
|
if (!Files.exists(path)) {
|
||||||
logger.info(s"Not using config file '$f' because it doesn't exist")
|
logger.info(s"Not using config file '$f' because it doesn't exist")
|
||||||
System.clearProperty("config.file")
|
System.clearProperty("config.file")
|
||||||
} else {
|
} else
|
||||||
logger.info(s"Using config file from system properties: $f")
|
logger.info(s"Using config file from system properties: $f")
|
||||||
}
|
|
||||||
case _ =>
|
case _ =>
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -23,7 +23,8 @@ object RestServer {
|
|||||||
|
|
||||||
val templates = TemplateRoutes[F](pools.blocker, cfg)
|
val templates = TemplateRoutes[F](pools.blocker, cfg)
|
||||||
val app = for {
|
val app = for {
|
||||||
restApp <- RestAppImpl
|
restApp <-
|
||||||
|
RestAppImpl
|
||||||
.create[F](cfg, pools.connectEC, pools.httpClientEC, pools.blocker)
|
.create[F](cfg, pools.connectEC, pools.httpClientEC, pools.blocker)
|
||||||
httpApp = Router(
|
httpApp = Router(
|
||||||
"/api/info" -> routes.InfoRoutes(),
|
"/api/info" -> routes.InfoRoutes(),
|
||||||
|
@ -34,7 +34,8 @@ object CookieData {
|
|||||||
def fromCookie[F[_]](req: Request[F]): Either[String, String] =
|
def fromCookie[F[_]](req: Request[F]): Either[String, String] =
|
||||||
for {
|
for {
|
||||||
header <- headers.Cookie.from(req.headers).toRight("Cookie parsing error")
|
header <- headers.Cookie.from(req.headers).toRight("Cookie parsing error")
|
||||||
cookie <- header.values.toList
|
cookie <-
|
||||||
|
header.values.toList
|
||||||
.find(_.name == cookieName)
|
.find(_.name == cookieName)
|
||||||
.toRight("Couldn't find the authcookie")
|
.toRight("Couldn't find the authcookie")
|
||||||
} yield cookie.content
|
} yield cookie.content
|
||||||
|
@ -199,10 +199,15 @@ trait Conversions {
|
|||||||
body
|
body
|
||||||
.through(fs2.text.utf8Decode)
|
.through(fs2.text.utf8Decode)
|
||||||
.parseJsonAs[ItemUploadMeta]
|
.parseJsonAs[ItemUploadMeta]
|
||||||
.map(_.fold(ex => {
|
.map(
|
||||||
|
_.fold(
|
||||||
|
ex => {
|
||||||
logger.error(ex)("Reading upload metadata failed.")
|
logger.error(ex)("Reading upload metadata failed.")
|
||||||
throw ex
|
throw ex
|
||||||
}, identity))
|
},
|
||||||
|
identity
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
val meta: F[(Boolean, UploadMeta)] = mp.parts
|
val meta: F[(Boolean, UploadMeta)] = mp.parts
|
||||||
.find(_.name.exists(_.equalsIgnoreCase("meta")))
|
.find(_.name.exists(_.equalsIgnoreCase("meta")))
|
||||||
@ -452,19 +457,23 @@ trait Conversions {
|
|||||||
BasicResult(true, "The job has been removed from the queue.")
|
BasicResult(true, "The job has been removed from the queue.")
|
||||||
}
|
}
|
||||||
|
|
||||||
def basicResult(ar: AddResult, successMsg: String): BasicResult = ar match {
|
def basicResult(ar: AddResult, successMsg: String): BasicResult =
|
||||||
|
ar match {
|
||||||
case AddResult.Success => BasicResult(true, successMsg)
|
case AddResult.Success => BasicResult(true, successMsg)
|
||||||
case AddResult.EntityExists(msg) => BasicResult(false, msg)
|
case AddResult.EntityExists(msg) => BasicResult(false, msg)
|
||||||
case AddResult.Failure(ex) => BasicResult(false, s"Internal error: ${ex.getMessage}")
|
case AddResult.Failure(ex) =>
|
||||||
|
BasicResult(false, s"Internal error: ${ex.getMessage}")
|
||||||
}
|
}
|
||||||
|
|
||||||
def basicResult(ur: OUpload.UploadResult): BasicResult = ur match {
|
def basicResult(ur: OUpload.UploadResult): BasicResult =
|
||||||
|
ur match {
|
||||||
case UploadResult.Success => BasicResult(true, "Files submitted.")
|
case UploadResult.Success => BasicResult(true, "Files submitted.")
|
||||||
case UploadResult.NoFiles => BasicResult(false, "There were no files to submit.")
|
case UploadResult.NoFiles => BasicResult(false, "There were no files to submit.")
|
||||||
case UploadResult.NoSource => BasicResult(false, "The source id is not valid.")
|
case UploadResult.NoSource => BasicResult(false, "The source id is not valid.")
|
||||||
}
|
}
|
||||||
|
|
||||||
def basicResult(cr: PassChangeResult): BasicResult = cr match {
|
def basicResult(cr: PassChangeResult): BasicResult =
|
||||||
|
cr match {
|
||||||
case PassChangeResult.Success => BasicResult(true, "Password changed.")
|
case PassChangeResult.Success => BasicResult(true, "Password changed.")
|
||||||
case PassChangeResult.UpdateFailed =>
|
case PassChangeResult.UpdateFailed =>
|
||||||
BasicResult(false, "The database update failed.")
|
BasicResult(false, "The database update failed.")
|
||||||
|
@ -9,8 +9,8 @@ trait ResponseGenerator[F[_]] {
|
|||||||
self: Http4sDsl[F] =>
|
self: Http4sDsl[F] =>
|
||||||
|
|
||||||
implicit final class EitherResponses[A, B](e: Either[A, B]) {
|
implicit final class EitherResponses[A, B](e: Either[A, B]) {
|
||||||
def toResponse(headers: Header*)(
|
def toResponse(headers: Header*)(implicit
|
||||||
implicit F: Applicative[F],
|
F: Applicative[F],
|
||||||
w0: EntityEncoder[F, A],
|
w0: EntityEncoder[F, A],
|
||||||
w1: EntityEncoder[F, B]
|
w1: EntityEncoder[F, B]
|
||||||
): F[Response[F]] =
|
): F[Response[F]] =
|
||||||
|
@ -46,7 +46,8 @@ object AttachmentRoutes {
|
|||||||
case HEAD -> Root / Ident(id) =>
|
case HEAD -> Root / Ident(id) =>
|
||||||
for {
|
for {
|
||||||
fileData <- backend.item.findAttachment(id, user.account.collective)
|
fileData <- backend.item.findAttachment(id, user.account.collective)
|
||||||
resp <- fileData
|
resp <-
|
||||||
|
fileData
|
||||||
.map(data => withResponseHeaders(Ok())(data))
|
.map(data => withResponseHeaders(Ok())(data))
|
||||||
.getOrElse(NotFound(BasicResult(false, "Not found")))
|
.getOrElse(NotFound(BasicResult(false, "Not found")))
|
||||||
} yield resp
|
} yield resp
|
||||||
@ -56,7 +57,8 @@ object AttachmentRoutes {
|
|||||||
fileData <- backend.item.findAttachment(id, user.account.collective)
|
fileData <- backend.item.findAttachment(id, user.account.collective)
|
||||||
inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
|
inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
|
||||||
matches = matchETag(fileData.map(_.meta), inm)
|
matches = matchETag(fileData.map(_.meta), inm)
|
||||||
resp <- fileData
|
resp <-
|
||||||
|
fileData
|
||||||
.map { data =>
|
.map { data =>
|
||||||
if (matches) withResponseHeaders(NotModified())(data)
|
if (matches) withResponseHeaders(NotModified())(data)
|
||||||
else makeByteResp(data)
|
else makeByteResp(data)
|
||||||
@ -67,7 +69,8 @@ object AttachmentRoutes {
|
|||||||
case HEAD -> Root / Ident(id) / "original" =>
|
case HEAD -> Root / Ident(id) / "original" =>
|
||||||
for {
|
for {
|
||||||
fileData <- backend.item.findAttachmentSource(id, user.account.collective)
|
fileData <- backend.item.findAttachmentSource(id, user.account.collective)
|
||||||
resp <- fileData
|
resp <-
|
||||||
|
fileData
|
||||||
.map(data => withResponseHeaders(Ok())(data))
|
.map(data => withResponseHeaders(Ok())(data))
|
||||||
.getOrElse(NotFound(BasicResult(false, "Not found")))
|
.getOrElse(NotFound(BasicResult(false, "Not found")))
|
||||||
} yield resp
|
} yield resp
|
||||||
@ -77,7 +80,8 @@ object AttachmentRoutes {
|
|||||||
fileData <- backend.item.findAttachmentSource(id, user.account.collective)
|
fileData <- backend.item.findAttachmentSource(id, user.account.collective)
|
||||||
inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
|
inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
|
||||||
matches = matchETag(fileData.map(_.meta), inm)
|
matches = matchETag(fileData.map(_.meta), inm)
|
||||||
resp <- fileData
|
resp <-
|
||||||
|
fileData
|
||||||
.map { data =>
|
.map { data =>
|
||||||
if (matches) withResponseHeaders(NotModified())(data)
|
if (matches) withResponseHeaders(NotModified())(data)
|
||||||
else makeByteResp(data)
|
else makeByteResp(data)
|
||||||
@ -88,7 +92,8 @@ object AttachmentRoutes {
|
|||||||
case HEAD -> Root / Ident(id) / "archive" =>
|
case HEAD -> Root / Ident(id) / "archive" =>
|
||||||
for {
|
for {
|
||||||
fileData <- backend.item.findAttachmentArchive(id, user.account.collective)
|
fileData <- backend.item.findAttachmentArchive(id, user.account.collective)
|
||||||
resp <- fileData
|
resp <-
|
||||||
|
fileData
|
||||||
.map(data => withResponseHeaders(Ok())(data))
|
.map(data => withResponseHeaders(Ok())(data))
|
||||||
.getOrElse(NotFound(BasicResult(false, "Not found")))
|
.getOrElse(NotFound(BasicResult(false, "Not found")))
|
||||||
} yield resp
|
} yield resp
|
||||||
@ -98,7 +103,8 @@ object AttachmentRoutes {
|
|||||||
fileData <- backend.item.findAttachmentArchive(id, user.account.collective)
|
fileData <- backend.item.findAttachmentArchive(id, user.account.collective)
|
||||||
inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
|
inm = req.headers.get(`If-None-Match`).flatMap(_.tags)
|
||||||
matches = matchETag(fileData.map(_.meta), inm)
|
matches = matchETag(fileData.map(_.meta), inm)
|
||||||
resp <- fileData
|
resp <-
|
||||||
|
fileData
|
||||||
.map { data =>
|
.map { data =>
|
||||||
if (matches) withResponseHeaders(NotModified())(data)
|
if (matches) withResponseHeaders(NotModified())(data)
|
||||||
else makeByteResp(data)
|
else makeByteResp(data)
|
||||||
@ -123,7 +129,8 @@ object AttachmentRoutes {
|
|||||||
case DELETE -> Root / Ident(id) =>
|
case DELETE -> Root / Ident(id) =>
|
||||||
for {
|
for {
|
||||||
n <- backend.item.deleteAttachment(id, user.account.collective)
|
n <- backend.item.deleteAttachment(id, user.account.collective)
|
||||||
res = if (n == 0) BasicResult(false, "Attachment not found")
|
res =
|
||||||
|
if (n == 0) BasicResult(false, "Attachment not found")
|
||||||
else BasicResult(true, "Attachment deleted.")
|
else BasicResult(true, "Attachment deleted.")
|
||||||
resp <- Ok(res)
|
resp <- Ok(res)
|
||||||
} yield resp
|
} yield resp
|
||||||
|
@ -28,7 +28,8 @@ object CollectiveRoutes {
|
|||||||
case req @ POST -> Root / "settings" =>
|
case req @ POST -> Root / "settings" =>
|
||||||
for {
|
for {
|
||||||
settings <- req.as[CollectiveSettings]
|
settings <- req.as[CollectiveSettings]
|
||||||
res <- backend.collective
|
res <-
|
||||||
|
backend.collective
|
||||||
.updateLanguage(user.account.collective, settings.language)
|
.updateLanguage(user.account.collective, settings.language)
|
||||||
resp <- Ok(Conversions.basicResult(res, "Language updated."))
|
resp <- Ok(Conversions.basicResult(res, "Language updated."))
|
||||||
} yield resp
|
} yield resp
|
||||||
@ -43,7 +44,8 @@ object CollectiveRoutes {
|
|||||||
case GET -> Root / "contacts" :? QueryParam.QueryOpt(q) +& QueryParam
|
case GET -> Root / "contacts" :? QueryParam.QueryOpt(q) +& QueryParam
|
||||||
.ContactKindOpt(kind) =>
|
.ContactKindOpt(kind) =>
|
||||||
for {
|
for {
|
||||||
res <- backend.collective
|
res <-
|
||||||
|
backend.collective
|
||||||
.getContacts(user.account.collective, q.map(_.q), kind)
|
.getContacts(user.account.collective, q.map(_.q), kind)
|
||||||
.take(50)
|
.take(50)
|
||||||
.compile
|
.compile
|
||||||
|
@ -36,7 +36,8 @@ object ItemRoutes {
|
|||||||
for {
|
for {
|
||||||
item <- backend.item.findItem(id, user.account.collective)
|
item <- backend.item.findItem(id, user.account.collective)
|
||||||
result = item.map(Conversions.mkItemDetail)
|
result = item.map(Conversions.mkItemDetail)
|
||||||
resp <- result
|
resp <-
|
||||||
|
result
|
||||||
.map(r => Ok(r))
|
.map(r => Ok(r))
|
||||||
.getOrElse(NotFound(BasicResult(false, "Not found.")))
|
.getOrElse(NotFound(BasicResult(false, "Not found.")))
|
||||||
} yield resp
|
} yield resp
|
||||||
|
@ -40,7 +40,8 @@ object MailSendRoutes {
|
|||||||
for {
|
for {
|
||||||
rec <- s.recipients.traverse(MailAddress.parse)
|
rec <- s.recipients.traverse(MailAddress.parse)
|
||||||
fileIds <- s.attachmentIds.traverse(Ident.fromString)
|
fileIds <- s.attachmentIds.traverse(Ident.fromString)
|
||||||
sel = if (s.addAllAttachments) AttachSelection.All
|
sel =
|
||||||
|
if (s.addAllAttachments) AttachSelection.All
|
||||||
else AttachSelection.Selected(fileIds)
|
else AttachSelection.Selected(fileIds)
|
||||||
} yield ItemMail(item, s.subject, rec, s.body, sel)
|
} yield ItemMail(item, s.subject, rec, s.body, sel)
|
||||||
|
|
||||||
|
@ -31,8 +31,8 @@ object NotifyDueItemsRoutes {
|
|||||||
for {
|
for {
|
||||||
data <- req.as[NotificationSettings]
|
data <- req.as[NotificationSettings]
|
||||||
task = makeTask(cfg, user.account, data)
|
task = makeTask(cfg, user.account, data)
|
||||||
res <- ut
|
res <-
|
||||||
.executeNow(user.account, task)
|
ut.executeNow(user.account, task)
|
||||||
.attempt
|
.attempt
|
||||||
.map(Conversions.basicResult(_, "Submitted successfully."))
|
.map(Conversions.basicResult(_, "Submitted successfully."))
|
||||||
resp <- Ok(res)
|
resp <- Ok(res)
|
||||||
@ -49,8 +49,8 @@ object NotifyDueItemsRoutes {
|
|||||||
for {
|
for {
|
||||||
data <- req.as[NotificationSettings]
|
data <- req.as[NotificationSettings]
|
||||||
task = makeTask(cfg, user.account, data)
|
task = makeTask(cfg, user.account, data)
|
||||||
res <- ut
|
res <-
|
||||||
.submitNotifyDueItems(user.account, task)
|
ut.submitNotifyDueItems(user.account, task)
|
||||||
.attempt
|
.attempt
|
||||||
.map(Conversions.basicResult(_, "Saved successfully."))
|
.map(Conversions.basicResult(_, "Saved successfully."))
|
||||||
resp <- Ok(res)
|
resp <- Ok(res)
|
||||||
@ -89,7 +89,8 @@ object NotifyDueItemsRoutes {
|
|||||||
for {
|
for {
|
||||||
tinc <- backend.tag.loadAll(task.args.tagsInclude)
|
tinc <- backend.tag.loadAll(task.args.tagsInclude)
|
||||||
texc <- backend.tag.loadAll(task.args.tagsExclude)
|
texc <- backend.tag.loadAll(task.args.tagsExclude)
|
||||||
conn <- backend.mail
|
conn <-
|
||||||
|
backend.mail
|
||||||
.getSettings(account, None)
|
.getSettings(account, None)
|
||||||
.map(
|
.map(
|
||||||
_.find(_.name == task.args.smtpConnection)
|
_.find(_.name == task.args.smtpConnection)
|
||||||
|
@ -21,17 +21,16 @@ object OrganizationRoutes {
|
|||||||
|
|
||||||
HttpRoutes.of {
|
HttpRoutes.of {
|
||||||
case GET -> Root :? QueryParam.FullOpt(full) +& QueryParam.QueryOpt(q) =>
|
case GET -> Root :? QueryParam.FullOpt(full) +& QueryParam.QueryOpt(q) =>
|
||||||
if (full.getOrElse(false)) {
|
if (full.getOrElse(false))
|
||||||
for {
|
for {
|
||||||
data <- backend.organization.findAllOrg(user.account, q.map(_.q))
|
data <- backend.organization.findAllOrg(user.account, q.map(_.q))
|
||||||
resp <- Ok(OrganizationList(data.map(mkOrg).toList))
|
resp <- Ok(OrganizationList(data.map(mkOrg).toList))
|
||||||
} yield resp
|
} yield resp
|
||||||
} else {
|
else
|
||||||
for {
|
for {
|
||||||
data <- backend.organization.findAllOrgRefs(user.account, q.map(_.q))
|
data <- backend.organization.findAllOrgRefs(user.account, q.map(_.q))
|
||||||
resp <- Ok(ReferenceList(data.map(mkIdName).toList))
|
resp <- Ok(ReferenceList(data.map(mkIdName).toList))
|
||||||
} yield resp
|
} yield resp
|
||||||
}
|
|
||||||
|
|
||||||
case req @ POST -> Root =>
|
case req @ POST -> Root =>
|
||||||
for {
|
for {
|
||||||
|
@ -24,17 +24,16 @@ object PersonRoutes {
|
|||||||
|
|
||||||
HttpRoutes.of {
|
HttpRoutes.of {
|
||||||
case GET -> Root :? QueryParam.FullOpt(full) +& QueryParam.QueryOpt(q) =>
|
case GET -> Root :? QueryParam.FullOpt(full) +& QueryParam.QueryOpt(q) =>
|
||||||
if (full.getOrElse(false)) {
|
if (full.getOrElse(false))
|
||||||
for {
|
for {
|
||||||
data <- backend.organization.findAllPerson(user.account, q.map(_.q))
|
data <- backend.organization.findAllPerson(user.account, q.map(_.q))
|
||||||
resp <- Ok(PersonList(data.map(mkPerson).toList))
|
resp <- Ok(PersonList(data.map(mkPerson).toList))
|
||||||
} yield resp
|
} yield resp
|
||||||
} else {
|
else
|
||||||
for {
|
for {
|
||||||
data <- backend.organization.findAllPersonRefs(user.account, q.map(_.q))
|
data <- backend.organization.findAllPersonRefs(user.account, q.map(_.q))
|
||||||
resp <- Ok(ReferenceList(data.map(mkIdName).toList))
|
resp <- Ok(ReferenceList(data.map(mkIdName).toList))
|
||||||
} yield resp
|
} yield resp
|
||||||
}
|
|
||||||
|
|
||||||
case req @ POST -> Root =>
|
case req @ POST -> Root =>
|
||||||
for {
|
for {
|
||||||
|
@ -38,7 +38,8 @@ object RegisterRoutes {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
def convert(r: NewInviteResult): InviteResult = r match {
|
def convert(r: NewInviteResult): InviteResult =
|
||||||
|
r match {
|
||||||
case NewInviteResult.Success(id) =>
|
case NewInviteResult.Success(id) =>
|
||||||
InviteResult(true, "New invitation created.", Some(id))
|
InviteResult(true, "New invitation created.", Some(id))
|
||||||
case NewInviteResult.InvitationDisabled =>
|
case NewInviteResult.InvitationDisabled =>
|
||||||
@ -47,7 +48,8 @@ object RegisterRoutes {
|
|||||||
InviteResult(false, "Password is invalid.", None)
|
InviteResult(false, "Password is invalid.", None)
|
||||||
}
|
}
|
||||||
|
|
||||||
def convert(r: SignupResult): BasicResult = r match {
|
def convert(r: SignupResult): BasicResult =
|
||||||
|
r match {
|
||||||
case SignupResult.CollectiveExists =>
|
case SignupResult.CollectiveExists =>
|
||||||
BasicResult(false, "A collective with this name already exists.")
|
BasicResult(false, "A collective with this name already exists.")
|
||||||
case SignupResult.InvalidInvitationKey =>
|
case SignupResult.InvalidInvitationKey =>
|
||||||
|
@ -26,8 +26,8 @@ object TemplateRoutes {
|
|||||||
def app: HttpRoutes[F]
|
def app: HttpRoutes[F]
|
||||||
}
|
}
|
||||||
|
|
||||||
def apply[F[_]: Effect](blocker: Blocker, cfg: Config)(
|
def apply[F[_]: Effect](blocker: Blocker, cfg: Config)(implicit
|
||||||
implicit C: ContextShift[F]
|
C: ContextShift[F]
|
||||||
): InnerRoutes[F] = {
|
): InnerRoutes[F] = {
|
||||||
val indexTemplate = memo(
|
val indexTemplate = memo(
|
||||||
loadResource("/index.html").flatMap(loadTemplate(_, blocker))
|
loadResource("/index.html").flatMap(loadTemplate(_, blocker))
|
||||||
@ -64,8 +64,8 @@ object TemplateRoutes {
|
|||||||
r.pure[F]
|
r.pure[F]
|
||||||
}
|
}
|
||||||
|
|
||||||
def loadUrl[F[_]: Sync](url: URL, blocker: Blocker)(
|
def loadUrl[F[_]: Sync](url: URL, blocker: Blocker)(implicit
|
||||||
implicit C: ContextShift[F]
|
C: ContextShift[F]
|
||||||
): F[String] =
|
): F[String] =
|
||||||
Stream
|
Stream
|
||||||
.bracket(Sync[F].delay(url.openStream))(in => Sync[F].delay(in.close()))
|
.bracket(Sync[F].delay(url.openStream))(in => Sync[F].delay(in.close()))
|
||||||
@ -82,8 +82,8 @@ object TemplateRoutes {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
def loadTemplate[F[_]: Sync](url: URL, blocker: Blocker)(
|
def loadTemplate[F[_]: Sync](url: URL, blocker: Blocker)(implicit
|
||||||
implicit C: ContextShift[F]
|
C: ContextShift[F]
|
||||||
): F[Template] =
|
): F[Template] =
|
||||||
loadUrl[F](url, blocker).flatMap(s => parseTemplate(s)).map { t =>
|
loadUrl[F](url, blocker).flatMap(s => parseTemplate(s)).map { t =>
|
||||||
logger.info(s"Compiled template $url")
|
logger.info(s"Compiled template $url")
|
||||||
|
@ -22,7 +22,8 @@ case class Column(name: String, ns: String = "", alias: String = "") {
|
|||||||
def is[A: Put](value: A): Fragment =
|
def is[A: Put](value: A): Fragment =
|
||||||
f ++ fr" = $value"
|
f ++ fr" = $value"
|
||||||
|
|
||||||
def is[A: Put](ov: Option[A]): Fragment = ov match {
|
def is[A: Put](ov: Option[A]): Fragment =
|
||||||
|
ov match {
|
||||||
case Some(v) => f ++ fr" = $v"
|
case Some(v) => f ++ fr" = $v"
|
||||||
case None => fr"is null"
|
case None => fr"is null"
|
||||||
}
|
}
|
||||||
|
@ -8,7 +8,8 @@ import org.log4s._
|
|||||||
object FlywayMigrate {
|
object FlywayMigrate {
|
||||||
private[this] val logger = getLogger
|
private[this] val logger = getLogger
|
||||||
|
|
||||||
def run[F[_]: Sync](jdbc: JdbcConfig): F[Int] = Sync[F].delay {
|
def run[F[_]: Sync](jdbc: JdbcConfig): F[Int] =
|
||||||
|
Sync[F].delay {
|
||||||
logger.info("Running db migrations...")
|
logger.info("Running db migrations...")
|
||||||
val locations = jdbc.dbmsName match {
|
val locations = jdbc.dbmsName match {
|
||||||
case Some(dbtype) =>
|
case Some(dbtype) =>
|
||||||
|
@ -30,10 +30,12 @@ object QAttachment {
|
|||||||
|
|
||||||
for {
|
for {
|
||||||
files <- store.transact(loadFiles)
|
files <- store.transact(loadFiles)
|
||||||
k <- if (files._3 == 1) deleteArchive(store)(attachId)
|
k <-
|
||||||
|
if (files._3 == 1) deleteArchive(store)(attachId)
|
||||||
else store.transact(RAttachmentArchive.delete(attachId))
|
else store.transact(RAttachmentArchive.delete(attachId))
|
||||||
n <- store.transact(RAttachment.delete(attachId))
|
n <- store.transact(RAttachment.delete(attachId))
|
||||||
f <- Stream
|
f <-
|
||||||
|
Stream
|
||||||
.emits(files._1.toSeq ++ files._2.toSeq)
|
.emits(files._1.toSeq ++ files._2.toSeq)
|
||||||
.map(_.id)
|
.map(_.id)
|
||||||
.flatMap(store.bitpeace.delete)
|
.flatMap(store.bitpeace.delete)
|
||||||
@ -55,7 +57,8 @@ object QAttachment {
|
|||||||
_ <- logger.fdebug[F](
|
_ <- logger.fdebug[F](
|
||||||
s"Deleted $n meta records (source, meta, archive). Deleting binaries now."
|
s"Deleted $n meta records (source, meta, archive). Deleting binaries now."
|
||||||
)
|
)
|
||||||
f <- Stream
|
f <-
|
||||||
|
Stream
|
||||||
.emits(ra.fileId.id +: (s.map(_.fileId.id).toSeq))
|
.emits(ra.fileId.id +: (s.map(_.fileId.id).toSeq))
|
||||||
.flatMap(store.bitpeace.delete)
|
.flatMap(store.bitpeace.delete)
|
||||||
.map(flag => if (flag) 1 else 0)
|
.map(flag => if (flag) 1 else 0)
|
||||||
@ -119,7 +122,9 @@ object QAttachment {
|
|||||||
val IC = RItem.Columns
|
val IC = RItem.Columns
|
||||||
|
|
||||||
val q =
|
val q =
|
||||||
fr"SELECT" ++ commas(MC.all.map(_.prefix("m").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++
|
fr"SELECT" ++ commas(
|
||||||
|
MC.all.map(_.prefix("m").f)
|
||||||
|
) ++ fr"FROM" ++ RItem.table ++ fr"i" ++
|
||||||
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ IC.id
|
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ IC.id
|
||||||
.prefix("i")
|
.prefix("i")
|
||||||
.is(AC.itemId.prefix("a")) ++
|
.is(AC.itemId.prefix("a")) ++
|
||||||
|
@ -96,7 +96,9 @@ object QCollective {
|
|||||||
RC.Columns.all,
|
RC.Columns.all,
|
||||||
RC.table,
|
RC.table,
|
||||||
and(
|
and(
|
||||||
Seq(or(RC.Columns.orgId.isIn(orgCond), RC.Columns.personId.isIn(persCond))) ++ queryCond ++ kindCond
|
Seq(
|
||||||
|
or(RC.Columns.orgId.isIn(orgCond), RC.Columns.personId.isIn(persCond))
|
||||||
|
) ++ queryCond ++ kindCond
|
||||||
)
|
)
|
||||||
) ++ orderBy(RC.Columns.value.f)
|
) ++ orderBy(RC.Columns.value.f)
|
||||||
|
|
||||||
|
@ -291,7 +291,9 @@ object QItem {
|
|||||||
val IC = RItem.Columns
|
val IC = RItem.Columns
|
||||||
val AC = RAttachment.Columns
|
val AC = RAttachment.Columns
|
||||||
val q =
|
val q =
|
||||||
fr"SELECT DISTINCT" ++ commas(IC.all.map(_.prefix("i").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++
|
fr"SELECT DISTINCT" ++ commas(
|
||||||
|
IC.all.map(_.prefix("i").f)
|
||||||
|
) ++ fr"FROM" ++ RItem.table ++ fr"i" ++
|
||||||
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId
|
fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId
|
||||||
.prefix("a")
|
.prefix("a")
|
||||||
.is(IC.id.prefix("i")) ++
|
.is(IC.id.prefix("i")) ++
|
||||||
|
@ -55,7 +55,8 @@ object QJob {
|
|||||||
def markJob(job: RJob): F[Either[Unit, RJob]] =
|
def markJob(job: RJob): F[Either[Unit, RJob]] =
|
||||||
store.transact(for {
|
store.transact(for {
|
||||||
n <- RJob.setScheduled(job.id, worker)
|
n <- RJob.setScheduled(job.id, worker)
|
||||||
_ <- if (n == 1) RJobGroupUse.setGroup(RJobGroupUse(worker, job.group))
|
_ <-
|
||||||
|
if (n == 1) RJobGroupUse.setGroup(RJobGroupUse(worker, job.group))
|
||||||
else 0.pure[ConnectionIO]
|
else 0.pure[ConnectionIO]
|
||||||
} yield if (n == 1) Right(job) else Left(()))
|
} yield if (n == 1) Right(job) else Left(()))
|
||||||
|
|
||||||
@ -68,7 +69,8 @@ object QJob {
|
|||||||
_ <- logger.ftrace[F](s"Choose group ${group.map(_.id)}")
|
_ <- logger.ftrace[F](s"Choose group ${group.map(_.id)}")
|
||||||
prio <- group.map(priority).getOrElse((Priority.Low: Priority).pure[F])
|
prio <- group.map(priority).getOrElse((Priority.Low: Priority).pure[F])
|
||||||
_ <- logger.ftrace[F](s"Looking for job of prio $prio")
|
_ <- logger.ftrace[F](s"Looking for job of prio $prio")
|
||||||
job <- group
|
job <-
|
||||||
|
group
|
||||||
.map(g => store.transact(selectNextJob(g, prio, retryPause, now)))
|
.map(g => store.transact(selectNextJob(g, prio, retryPause, now)))
|
||||||
.getOrElse((None: Option[RJob]).pure[F])
|
.getOrElse((None: Option[RJob]).pure[F])
|
||||||
_ <- logger.ftrace[F](s"Found job: ${job.map(_.info)}")
|
_ <- logger.ftrace[F](s"Found job: ${job.map(_.info)}")
|
||||||
|
@ -81,7 +81,8 @@ object PeriodicTaskStore {
|
|||||||
def unmark(job: RPeriodicTask): F[Unit] =
|
def unmark(job: RPeriodicTask): F[Unit] =
|
||||||
for {
|
for {
|
||||||
now <- Timestamp.current[F]
|
now <- Timestamp.current[F]
|
||||||
nextRun <- CalevFs2
|
nextRun <-
|
||||||
|
CalevFs2
|
||||||
.nextElapses[F](now.atUTC)(job.timer)
|
.nextElapses[F](now.atUTC)(job.timer)
|
||||||
.take(1)
|
.take(1)
|
||||||
.compile
|
.compile
|
||||||
|
@ -43,7 +43,11 @@ object RAttachment {
|
|||||||
fId: Ident,
|
fId: Ident,
|
||||||
fname: Option[String]
|
fname: Option[String]
|
||||||
): ConnectionIO[Int] =
|
): ConnectionIO[Int] =
|
||||||
updateRow(table, id.is(attachId), commas(fileId.setTo(fId), name.setTo(fname))).update.run
|
updateRow(
|
||||||
|
table,
|
||||||
|
id.is(attachId),
|
||||||
|
commas(fileId.setTo(fId), name.setTo(fname))
|
||||||
|
).update.run
|
||||||
|
|
||||||
def updatePosition(attachId: Ident, pos: Int): ConnectionIO[Int] =
|
def updatePosition(attachId: Ident, pos: Int): ConnectionIO[Int] =
|
||||||
updateRow(table, id.is(attachId), position.setTo(pos)).update.run
|
updateRow(table, id.is(attachId), position.setTo(pos)).update.run
|
||||||
|
@ -38,7 +38,11 @@ object RAttachmentArchive {
|
|||||||
RAttachmentArchive(ra.id, ra.fileId, ra.name, mId, ra.created)
|
RAttachmentArchive(ra.id, ra.fileId, ra.name, mId, ra.created)
|
||||||
|
|
||||||
def insert(v: RAttachmentArchive): ConnectionIO[Int] =
|
def insert(v: RAttachmentArchive): ConnectionIO[Int] =
|
||||||
insertRow(table, all, fr"${v.id},${v.fileId},${v.name},${v.messageId},${v.created}").update.run
|
insertRow(
|
||||||
|
table,
|
||||||
|
all,
|
||||||
|
fr"${v.id},${v.fileId},${v.name},${v.messageId},${v.created}"
|
||||||
|
).update.run
|
||||||
|
|
||||||
def findById(attachId: Ident): ConnectionIO[Option[RAttachmentArchive]] =
|
def findById(attachId: Ident): ConnectionIO[Option[RAttachmentArchive]] =
|
||||||
selectSimple(all, table, id.is(attachId)).query[RAttachmentArchive].option
|
selectSimple(all, table, id.is(attachId)).query[RAttachmentArchive].option
|
||||||
|
@ -35,7 +35,11 @@ object RAttachmentMeta {
|
|||||||
import Columns._
|
import Columns._
|
||||||
|
|
||||||
def insert(v: RAttachmentMeta): ConnectionIO[Int] =
|
def insert(v: RAttachmentMeta): ConnectionIO[Int] =
|
||||||
insertRow(table, all, fr"${v.id},${v.content},${v.nerlabels},${v.proposals}").update.run
|
insertRow(
|
||||||
|
table,
|
||||||
|
all,
|
||||||
|
fr"${v.id},${v.content},${v.nerlabels},${v.proposals}"
|
||||||
|
).update.run
|
||||||
|
|
||||||
def exists(attachId: Ident): ConnectionIO[Boolean] =
|
def exists(attachId: Ident): ConnectionIO[Boolean] =
|
||||||
selectCount(id, table, id.is(attachId)).query[Int].unique.map(_ > 0)
|
selectCount(id, table, id.is(attachId)).query[Int].unique.map(_ > 0)
|
||||||
|
@ -29,7 +29,11 @@ object RJobLog {
|
|||||||
import Columns._
|
import Columns._
|
||||||
|
|
||||||
def insert(v: RJobLog): ConnectionIO[Int] =
|
def insert(v: RJobLog): ConnectionIO[Int] =
|
||||||
insertRow(table, all, fr"${v.id},${v.jobId},${v.level},${v.created},${v.message}").update.run
|
insertRow(
|
||||||
|
table,
|
||||||
|
all,
|
||||||
|
fr"${v.id},${v.jobId},${v.level},${v.created},${v.message}"
|
||||||
|
).update.run
|
||||||
|
|
||||||
def findLogs(id: Ident): ConnectionIO[Vector[RJobLog]] =
|
def findLogs(id: Ident): ConnectionIO[Vector[RJobLog]] =
|
||||||
(selectSimple(all, table, jobId.is(id)) ++ orderBy(created.asc))
|
(selectSimple(all, table, jobId.is(id)) ++ orderBy(created.asc))
|
||||||
|
@ -34,7 +34,11 @@ object RNode {
|
|||||||
import Columns._
|
import Columns._
|
||||||
|
|
||||||
def insert(v: RNode): ConnectionIO[Int] =
|
def insert(v: RNode): ConnectionIO[Int] =
|
||||||
insertRow(table, all, fr"${v.id},${v.nodeType},${v.url},${v.updated},${v.created}").update.run
|
insertRow(
|
||||||
|
table,
|
||||||
|
all,
|
||||||
|
fr"${v.id},${v.nodeType},${v.url},${v.updated},${v.created}"
|
||||||
|
).update.run
|
||||||
|
|
||||||
def update(v: RNode): ConnectionIO[Int] =
|
def update(v: RNode): ConnectionIO[Int] =
|
||||||
updateRow(
|
updateRow(
|
||||||
|
@ -38,8 +38,8 @@ trait UserTaskStore[F[_]] {
|
|||||||
/** Return all tasks of the given name and user. The task's arguments
|
/** Return all tasks of the given name and user. The task's arguments
|
||||||
* are decoded using the given json decoder.
|
* are decoded using the given json decoder.
|
||||||
*/
|
*/
|
||||||
def getByName[A](account: AccountId, name: Ident)(
|
def getByName[A](account: AccountId, name: Ident)(implicit
|
||||||
implicit D: Decoder[A]
|
D: Decoder[A]
|
||||||
): Stream[F, UserTask[A]]
|
): Stream[F, UserTask[A]]
|
||||||
|
|
||||||
/** Updates or inserts the given task.
|
/** Updates or inserts the given task.
|
||||||
@ -65,8 +65,8 @@ trait UserTaskStore[F[_]] {
|
|||||||
* error is returned. The task's arguments are decoded using the
|
* error is returned. The task's arguments are decoded using the
|
||||||
* given json decoder.
|
* given json decoder.
|
||||||
*/
|
*/
|
||||||
def getOneByName[A](account: AccountId, name: Ident)(
|
def getOneByName[A](account: AccountId, name: Ident)(implicit
|
||||||
implicit D: Decoder[A]
|
D: Decoder[A]
|
||||||
): OptionT[F, UserTask[A]]
|
): OptionT[F, UserTask[A]]
|
||||||
|
|
||||||
/** Updates or inserts the given task.
|
/** Updates or inserts the given task.
|
||||||
@ -80,8 +80,8 @@ trait UserTaskStore[F[_]] {
|
|||||||
* the user `account`, they will all be removed and the given task
|
* the user `account`, they will all be removed and the given task
|
||||||
* inserted!
|
* inserted!
|
||||||
*/
|
*/
|
||||||
def updateOneTask[A](account: AccountId, ut: UserTask[A])(
|
def updateOneTask[A](account: AccountId, ut: UserTask[A])(implicit
|
||||||
implicit E: Encoder[A]
|
E: Encoder[A]
|
||||||
): F[UserTask[String]]
|
): F[UserTask[String]]
|
||||||
|
|
||||||
/** Delete all tasks of the given user that have name `name'.
|
/** Delete all tasks of the given user that have name `name'.
|
||||||
@ -100,16 +100,16 @@ object UserTaskStore {
|
|||||||
def getByNameRaw(account: AccountId, name: Ident): Stream[F, UserTask[String]] =
|
def getByNameRaw(account: AccountId, name: Ident): Stream[F, UserTask[String]] =
|
||||||
store.transact(QUserTask.findByName(account, name))
|
store.transact(QUserTask.findByName(account, name))
|
||||||
|
|
||||||
def getByName[A](account: AccountId, name: Ident)(
|
def getByName[A](account: AccountId, name: Ident)(implicit
|
||||||
implicit D: Decoder[A]
|
D: Decoder[A]
|
||||||
): Stream[F, UserTask[A]] =
|
): Stream[F, UserTask[A]] =
|
||||||
getByNameRaw(account, name).flatMap(_.decode match {
|
getByNameRaw(account, name).flatMap(_.decode match {
|
||||||
case Right(ua) => Stream.emit(ua)
|
case Right(ua) => Stream.emit(ua)
|
||||||
case Left(err) => Stream.raiseError[F](new Exception(err))
|
case Left(err) => Stream.raiseError[F](new Exception(err))
|
||||||
})
|
})
|
||||||
|
|
||||||
def updateTask[A](account: AccountId, ut: UserTask[A])(
|
def updateTask[A](account: AccountId, ut: UserTask[A])(implicit
|
||||||
implicit E: Encoder[A]
|
E: Encoder[A]
|
||||||
): F[Int] = {
|
): F[Int] = {
|
||||||
val exists = QUserTask.exists(ut.id)
|
val exists = QUserTask.exists(ut.id)
|
||||||
val insert = QUserTask.insert(account, ut.encode)
|
val insert = QUserTask.insert(account, ut.encode)
|
||||||
@ -142,8 +142,8 @@ object UserTaskStore {
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
def getOneByName[A](account: AccountId, name: Ident)(
|
def getOneByName[A](account: AccountId, name: Ident)(implicit
|
||||||
implicit D: Decoder[A]
|
D: Decoder[A]
|
||||||
): OptionT[F, UserTask[A]] =
|
): OptionT[F, UserTask[A]] =
|
||||||
getOneByNameRaw(account, name)
|
getOneByNameRaw(account, name)
|
||||||
.semiflatMap(_.decode match {
|
.semiflatMap(_.decode match {
|
||||||
@ -151,8 +151,8 @@ object UserTaskStore {
|
|||||||
case Left(err) => Effect[F].raiseError(new Exception(err))
|
case Left(err) => Effect[F].raiseError(new Exception(err))
|
||||||
})
|
})
|
||||||
|
|
||||||
def updateOneTask[A](account: AccountId, ut: UserTask[A])(
|
def updateOneTask[A](account: AccountId, ut: UserTask[A])(implicit
|
||||||
implicit E: Encoder[A]
|
E: Encoder[A]
|
||||||
): F[UserTask[String]] =
|
): F[UserTask[String]] =
|
||||||
getByNameRaw(account, ut.name).compile.toList.flatMap {
|
getByNameRaw(account, ut.name).compile.toList.flatMap {
|
||||||
case a :: rest =>
|
case a :: rest =>
|
||||||
|
Reference in New Issue
Block a user