From d3ef4b6166696fc0468a793507a5a4d52dabcabb Mon Sep 17 00:00:00 2001 From: eikek Date: Sat, 5 Mar 2022 14:46:27 +0100 Subject: [PATCH 01/16] Moved route definitions in RestApp --- .../scala/docspell/restserver/RestApp.scala | 6 +- .../docspell/restserver/RestAppImpl.scala | 124 +++++++++++++++- .../docspell/restserver/RestServer.scala | 134 ++---------------- 3 files changed, 137 insertions(+), 127 deletions(-) diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestApp.scala b/modules/restserver/src/main/scala/docspell/restserver/RestApp.scala index 44c80eaa..9ad2f34e 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestApp.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestApp.scala @@ -7,8 +7,9 @@ package docspell.restserver import fs2.Stream - import docspell.backend.BackendApp +import org.http4s.HttpRoutes +import org.http4s.server.websocket.WebSocketBuilder2 trait RestApp[F[_]] { @@ -25,4 +26,7 @@ trait RestApp[F[_]] { * via websocket. */ def subscriptions: Stream[F, Nothing] + + /** Http4s endpoint definitions. */ + def routes(wsb: WebSocketBuilder2[F]): HttpRoutes[F] } diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala index ce14ecd3..ea44065d 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala @@ -9,22 +9,30 @@ package docspell.restserver import cats.effect._ import fs2.Stream import fs2.concurrent.Topic - import docspell.backend.BackendApp +import docspell.backend.auth.{AuthToken, ShareToken} import docspell.ftsclient.FtsClient import docspell.ftssolr.SolrFtsClient import docspell.notification.api.NotificationModule import docspell.notification.impl.NotificationModuleImpl +import docspell.oidc.CodeFlowRoutes import docspell.pubsub.api.{PubSub, PubSubT} -import docspell.restserver.ws.OutputEvent +import docspell.restserver.auth.OpenId +import docspell.restserver.http4s.EnvMiddleware +import docspell.restserver.routes._ +import docspell.restserver.webapp.{TemplateRoutes, Templates, WebjarRoutes} +import docspell.restserver.ws.{OutputEvent, WebSocketRoutes} import docspell.store.Store - import emil.javamail.JavaMailEmil +import org.http4s.HttpRoutes import org.http4s.client.Client +import org.http4s.server.Router +import org.http4s.server.websocket.WebSocketBuilder2 final class RestAppImpl[F[_]: Async]( val config: Config, val backend: BackendApp[F], + httpClient: Client[F], notificationMod: NotificationModule[F], wsTopic: Topic[F, OutputEvent], pubSub: PubSubT[F] @@ -35,6 +43,107 @@ final class RestAppImpl[F[_]: Async]( def subscriptions: Stream[F, Nothing] = Subscriptions[F](wsTopic, pubSub) + + def routes(wsb: WebSocketBuilder2[F]): HttpRoutes[F] = + createHttpApp(wsb) + + val templates = TemplateRoutes[F](config, Templates[F]) + + def createHttpApp( + wsB: WebSocketBuilder2[F] + ) = + Router( + "/api/info" -> InfoRoutes(), + "/api/v1/open/" -> openRoutes(httpClient), + "/api/v1/sec/" -> Authenticate(backend.login, config.auth) { token => + securedRoutes(wsB, token) + }, + "/api/v1/admin" -> AdminAuth(config.adminEndpoint) { + adminRoutes + }, + "/api/v1/share" -> ShareAuth(backend.share, config.auth) { token => + shareRoutes(token) + }, + "/api/doc" -> templates.doc, + "/app/assets" -> EnvMiddleware(WebjarRoutes.appRoutes[F]), + "/app" -> EnvMiddleware(templates.app), + "/sw.js" -> EnvMiddleware(templates.serviceWorker) + ) + + def adminRoutes: HttpRoutes[F] = + Router( + "fts" -> FullTextIndexRoutes.admin(config, backend), + "user/otp" -> TotpRoutes.admin(backend), + "user" -> UserRoutes.admin(backend), + "info" -> InfoRoutes.admin(config), + "attachments" -> AttachmentRoutes.admin(backend) + ) + + def shareRoutes( + token: ShareToken + ): HttpRoutes[F] = + Router( + "search" -> ShareSearchRoutes(backend, config, token), + "attachment" -> ShareAttachmentRoutes(backend, token), + "item" -> ShareItemRoutes(backend, token), + "clientSettings" -> ClientSettingsRoutes.share(backend, token) + ) + def openRoutes( + client: Client[F] + ): HttpRoutes[F] = + Router( + "auth/openid" -> CodeFlowRoutes( + config.openIdEnabled, + OpenId.handle[F](backend, config), + OpenId.codeFlowConfig(config), + client + ), + "auth" -> LoginRoutes.login(backend.login, config), + "signup" -> RegisterRoutes(backend, config), + "upload" -> UploadRoutes.open(backend, config), + "checkfile" -> CheckFileRoutes.open(backend), + "integration" -> IntegrationEndpointRoutes.open(backend, config), + "share" -> ShareRoutes.verify(backend, config) + ) + + def securedRoutes( + wsB: WebSocketBuilder2[F], + token: AuthToken + ): HttpRoutes[F] = + Router( + "ws" -> WebSocketRoutes(token, backend, wsTopic, wsB), + "auth" -> LoginRoutes.session(backend.login, config, token), + "tag" -> TagRoutes(backend, token), + "equipment" -> EquipmentRoutes(backend, token), + "organization" -> OrganizationRoutes(backend, token), + "person" -> PersonRoutes(backend, token), + "source" -> SourceRoutes(backend, token), + "user/otp" -> TotpRoutes(backend, config, token), + "user" -> UserRoutes(backend, token), + "collective" -> CollectiveRoutes(backend, token), + "queue" -> JobQueueRoutes(backend, token), + "item" -> ItemRoutes(config, backend, token), + "items" -> ItemMultiRoutes(config, backend, token), + "attachment" -> AttachmentRoutes(backend, token), + "attachments" -> AttachmentMultiRoutes(backend, token), + "upload" -> UploadRoutes.secured(backend, config, token), + "checkfile" -> CheckFileRoutes.secured(backend, token), + "email/send" -> MailSendRoutes(backend, token), + "email/settings" -> MailSettingsRoutes(backend, token), + "email/sent" -> SentMailRoutes(backend, token), + "share" -> ShareRoutes.manage(backend, token), + "usertask/notifydueitems" -> NotifyDueItemsRoutes(config, backend, token), + "usertask/scanmailbox" -> ScanMailboxRoutes(backend, token), + "usertask/periodicquery" -> PeriodicQueryRoutes(config, backend, token), + "calevent/check" -> CalEventCheckRoutes(), + "fts" -> FullTextIndexRoutes.secured(config, backend, token), + "folder" -> FolderRoutes(backend, token), + "customfield" -> CustomFieldRoutes(backend, token), + "clientSettings" -> ClientSettingsRoutes(backend, token), + "notification" -> NotificationRoutes(config, backend, token), + "querybookmark" -> BookmarkRoutes(backend, token) + ) + } object RestAppImpl { @@ -58,7 +167,14 @@ object RestAppImpl { backend <- BackendApp .create[F](store, javaEmil, ftsClient, pubSubT, notificationMod) - app = new RestAppImpl[F](cfg, backend, notificationMod, wsTopic, pubSubT) + app = new RestAppImpl[F]( + cfg, + backend, + httpClient, + notificationMod, + wsTopic, + pubSubT + ) } yield app } diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala index 891d177e..0b58a584 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala @@ -7,30 +7,21 @@ package docspell.restserver import scala.concurrent.duration._ - import cats.effect._ import cats.implicits._ import fs2.Stream import fs2.concurrent.Topic - -import docspell.backend.auth.{AuthToken, ShareToken} import docspell.backend.msg.Topics import docspell.common._ -import docspell.oidc.CodeFlowRoutes import docspell.pubsub.naive.NaivePubSub -import docspell.restserver.auth.OpenId -import docspell.restserver.http4s.{EnvMiddleware, InternalHeader} -import docspell.restserver.routes._ -import docspell.restserver.webapp._ +import docspell.restserver.http4s.InternalHeader import docspell.restserver.ws.OutputEvent.KeepAlive -import docspell.restserver.ws.{OutputEvent, WebSocketRoutes} +import docspell.restserver.ws.OutputEvent import docspell.store.Store import docspell.store.records.RInternalSetting - import org.http4s._ import org.http4s.blaze.client.BlazeClientBuilder import org.http4s.blaze.server.BlazeServerBuilder -import org.http4s.client.Client import org.http4s.dsl.Http4sDsl import org.http4s.headers.Location import org.http4s.implicits._ @@ -51,7 +42,7 @@ object RestServer { server = Stream .resource(createApp(cfg, pools, wsTopic)) - .flatMap { case (restApp, pubSub, httpClient, setting) => + .flatMap { case (restApp, pubSub, setting) => Stream( restApp.subscriptions, restApp.eventConsume(2), @@ -59,7 +50,7 @@ object RestServer { .bindHttp(cfg.bind.port, cfg.bind.address) .withoutBanner .withHttpWebSocketApp( - createHttpApp(cfg, setting, httpClient, pubSub, restApp, wsTopic) + createHttpApp(setting, pubSub, restApp) ) .serve .drain @@ -76,7 +67,7 @@ object RestServer { wsTopic: Topic[F, OutputEvent] ): Resource[ F, - (RestApp[F], NaivePubSub[F], Client[F], RInternalSetting) + (RestApp[F], NaivePubSub[F], RInternalSetting) ] = for { httpClient <- BlazeClientBuilder[F].resource @@ -92,41 +83,22 @@ object RestServer { httpClient )(Topics.all.map(_.topic)) restApp <- RestAppImpl.create[F](cfg, store, httpClient, pubSub, wsTopic) - } yield (restApp, pubSub, httpClient, setting) + } yield (restApp, pubSub, setting) def createHttpApp[F[_]: Async]( - cfg: Config, internSettings: RInternalSetting, - httpClient: Client[F], pubSub: NaivePubSub[F], - restApp: RestApp[F], - topic: Topic[F, OutputEvent] + restApp: RestApp[F] )( wsB: WebSocketBuilder2[F] ) = { - val templates = TemplateRoutes[F](cfg, Templates[F]) - val httpApp = Router( + val internal = Router( + "/" -> redirectTo("/app"), "/internal" -> InternalHeader(internSettings.internalRouteKey) { internalRoutes(pubSub) - }, - "/api/info" -> routes.InfoRoutes(), - "/api/v1/open/" -> openRoutes(cfg, httpClient, restApp), - "/api/v1/sec/" -> Authenticate(restApp.backend.login, cfg.auth) { token => - securedRoutes(cfg, restApp, wsB, topic, token) - }, - "/api/v1/admin" -> AdminAuth(cfg.adminEndpoint) { - adminRoutes(cfg, restApp) - }, - "/api/v1/share" -> ShareAuth(restApp.backend.share, cfg.auth) { token => - shareRoutes(cfg, restApp, token) - }, - "/api/doc" -> templates.doc, - "/app/assets" -> EnvMiddleware(WebjarRoutes.appRoutes[F]), - "/app" -> EnvMiddleware(templates.app), - "/sw.js" -> EnvMiddleware(templates.serviceWorker), - "/" -> redirectTo("/app") - ).orNotFound - + } + ) + val httpApp = (internal <+> restApp.routes(wsB)).orNotFound Logger.httpApp(logHeaders = false, logBody = false)(httpApp) } @@ -135,88 +107,6 @@ object RestServer { "pubsub" -> pubSub.receiveRoute ) - def securedRoutes[F[_]: Async]( - cfg: Config, - restApp: RestApp[F], - wsB: WebSocketBuilder2[F], - topic: Topic[F, OutputEvent], - token: AuthToken - ): HttpRoutes[F] = - Router( - "ws" -> WebSocketRoutes(token, restApp.backend, topic, wsB), - "auth" -> LoginRoutes.session(restApp.backend.login, cfg, token), - "tag" -> TagRoutes(restApp.backend, token), - "equipment" -> EquipmentRoutes(restApp.backend, token), - "organization" -> OrganizationRoutes(restApp.backend, token), - "person" -> PersonRoutes(restApp.backend, token), - "source" -> SourceRoutes(restApp.backend, token), - "user/otp" -> TotpRoutes(restApp.backend, cfg, token), - "user" -> UserRoutes(restApp.backend, token), - "collective" -> CollectiveRoutes(restApp.backend, token), - "queue" -> JobQueueRoutes(restApp.backend, token), - "item" -> ItemRoutes(cfg, restApp.backend, token), - "items" -> ItemMultiRoutes(cfg, restApp.backend, token), - "attachment" -> AttachmentRoutes(restApp.backend, token), - "attachments" -> AttachmentMultiRoutes(restApp.backend, token), - "upload" -> UploadRoutes.secured(restApp.backend, cfg, token), - "checkfile" -> CheckFileRoutes.secured(restApp.backend, token), - "email/send" -> MailSendRoutes(restApp.backend, token), - "email/settings" -> MailSettingsRoutes(restApp.backend, token), - "email/sent" -> SentMailRoutes(restApp.backend, token), - "share" -> ShareRoutes.manage(restApp.backend, token), - "usertask/notifydueitems" -> NotifyDueItemsRoutes(cfg, restApp.backend, token), - "usertask/scanmailbox" -> ScanMailboxRoutes(restApp.backend, token), - "usertask/periodicquery" -> PeriodicQueryRoutes(cfg, restApp.backend, token), - "calevent/check" -> CalEventCheckRoutes(), - "fts" -> FullTextIndexRoutes.secured(cfg, restApp.backend, token), - "folder" -> FolderRoutes(restApp.backend, token), - "customfield" -> CustomFieldRoutes(restApp.backend, token), - "clientSettings" -> ClientSettingsRoutes(restApp.backend, token), - "notification" -> NotificationRoutes(cfg, restApp.backend, token), - "querybookmark" -> BookmarkRoutes(restApp.backend, token) - ) - - def openRoutes[F[_]: Async]( - cfg: Config, - client: Client[F], - restApp: RestApp[F] - ): HttpRoutes[F] = - Router( - "auth/openid" -> CodeFlowRoutes( - cfg.openIdEnabled, - OpenId.handle[F](restApp.backend, cfg), - OpenId.codeFlowConfig(cfg), - client - ), - "auth" -> LoginRoutes.login(restApp.backend.login, cfg), - "signup" -> RegisterRoutes(restApp.backend, cfg), - "upload" -> UploadRoutes.open(restApp.backend, cfg), - "checkfile" -> CheckFileRoutes.open(restApp.backend), - "integration" -> IntegrationEndpointRoutes.open(restApp.backend, cfg), - "share" -> ShareRoutes.verify(restApp.backend, cfg) - ) - - def adminRoutes[F[_]: Async](cfg: Config, restApp: RestApp[F]): HttpRoutes[F] = - Router( - "fts" -> FullTextIndexRoutes.admin(cfg, restApp.backend), - "user/otp" -> TotpRoutes.admin(restApp.backend), - "user" -> UserRoutes.admin(restApp.backend), - "info" -> InfoRoutes.admin(cfg), - "attachments" -> AttachmentRoutes.admin(restApp.backend) - ) - - def shareRoutes[F[_]: Async]( - cfg: Config, - restApp: RestApp[F], - token: ShareToken - ): HttpRoutes[F] = - Router( - "search" -> ShareSearchRoutes(restApp.backend, cfg, token), - "attachment" -> ShareAttachmentRoutes(restApp.backend, token), - "item" -> ShareItemRoutes(restApp.backend, token), - "clientSettings" -> ClientSettingsRoutes.share(restApp.backend, token) - ) - def redirectTo[F[_]: Async](path: String): HttpRoutes[F] = { val dsl = new Http4sDsl[F] {} import dsl._ From c812ea1009961a57834f3df76d89a5559b817bb6 Mon Sep 17 00:00:00 2001 From: eikek Date: Mon, 7 Mar 2022 12:06:29 +0100 Subject: [PATCH 02/16] Store files in different binary stores --- .../main/scala/docspell/joex/JoexServer.scala | 3 +- .../docspell/restserver/RestServer.scala | 3 +- .../src/main/scala/docspell/store/Store.scala | 6 +- .../docspell/store/file/BinnyUtils.scala | 62 ++++++++++++++++++- .../docspell/store/file/FileRepository.scala | 10 ++- .../store/file/FileRepositoryConfig.scala | 27 ++++++++ .../scala/docspell/store/StoreFixture.scala | 5 +- project/Dependencies.scala | 3 +- 8 files changed, 103 insertions(+), 16 deletions(-) create mode 100644 modules/store/src/main/scala/docspell/store/file/FileRepositoryConfig.scala diff --git a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala index ef06f730..1d0458d4 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala @@ -16,6 +16,7 @@ import docspell.common.Pools import docspell.joex.routes._ import docspell.pubsub.naive.NaivePubSub import docspell.store.Store +import docspell.store.file.FileRepositoryConfig import docspell.store.records.RInternalSetting import org.http4s.HttpApp @@ -41,7 +42,7 @@ object JoexServer { store <- Store.create[F]( cfg.jdbc, - cfg.files.chunkSize, + FileRepositoryConfig.Database(cfg.files.chunkSize), pools.connectEC ) settings <- Resource.eval(store.transact(RInternalSetting.create)) diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala index 0b58a584..57c77707 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala @@ -18,6 +18,7 @@ import docspell.restserver.http4s.InternalHeader import docspell.restserver.ws.OutputEvent.KeepAlive import docspell.restserver.ws.OutputEvent import docspell.store.Store +import docspell.store.file.FileRepositoryConfig import docspell.store.records.RInternalSetting import org.http4s._ import org.http4s.blaze.client.BlazeClientBuilder @@ -73,7 +74,7 @@ object RestServer { httpClient <- BlazeClientBuilder[F].resource store <- Store.create[F]( cfg.backend.jdbc, - cfg.backend.files.chunkSize, + FileRepositoryConfig.Database(cfg.backend.files.chunkSize), pools.connectEC ) setting <- Resource.eval(store.transact(RInternalSetting.create)) diff --git a/modules/store/src/main/scala/docspell/store/Store.scala b/modules/store/src/main/scala/docspell/store/Store.scala index 3e54a7a2..24c80b98 100644 --- a/modules/store/src/main/scala/docspell/store/Store.scala +++ b/modules/store/src/main/scala/docspell/store/Store.scala @@ -12,7 +12,7 @@ import cats.effect._ import cats.~> import fs2._ -import docspell.store.file.FileRepository +import docspell.store.file.{FileRepository, FileRepositoryConfig} import docspell.store.impl.StoreImpl import com.zaxxer.hikari.HikariDataSource @@ -35,7 +35,7 @@ object Store { def create[F[_]: Async]( jdbc: JdbcConfig, - chunkSize: Int, + fileRepoConfig: FileRepositoryConfig, connectEC: ExecutionContext ): Resource[F, Store[F]] = { val acquire = Sync[F].delay(new HikariDataSource()) @@ -50,7 +50,7 @@ object Store { ds.setDriverClassName(jdbc.driverClass) } xa = HikariTransactor(ds, connectEC) - fr = FileRepository.genericJDBC(xa, ds, chunkSize) + fr = FileRepository.apply(xa, ds, fileRepoConfig) st = new StoreImpl[F](fr, jdbc, xa) _ <- Resource.eval(st.migrate) } yield st diff --git a/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala b/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala index eef07da3..bdd09f48 100644 --- a/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala +++ b/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala @@ -6,12 +6,19 @@ package docspell.store.file -import docspell.common +import javax.sql.DataSource + +import cats.effect._ +import fs2.io.file.Path + import docspell.common._ import docspell.files.TikaMimetype import docspell.logging.Logger import binny._ +import binny.fs.{FsBinaryStore, FsStoreConfig, PathMapping} +import binny.jdbc.{GenericJdbcStore, JdbcStoreConfig} +import binny.minio.{MinioBinaryStore, MinioConfig, S3KeyMapping} import scodec.bits.ByteVector private[store] object BinnyUtils { @@ -26,7 +33,7 @@ private[store] object BinnyUtils { coll <- Ident.fromString(cId) cat <- FileCategory.fromString(catId) file <- Ident.fromString(fId) - } yield common.FileKey(coll, cat, file) + } yield FileKey(coll, cat, file) case _ => Left(s"Invalid format for file-key: $bid") } @@ -57,4 +64,55 @@ private[store] object BinnyUtils { .asString ) } + + val pathMapping: PathMapping = { + import binny.fs.PathMapping.syntax._ + + def toPath(base: Path, binaryId: BinaryId): Path = { + val fkey = unsafeBinaryIdToFileKey(binaryId) + base / fkey.collective.id / fkey.category.id.id / fkey.id.id / "file" + } + + def toId(file: Path): Option[BinaryId] = + for { + id <- file.parent + cat <- id.parent + fcat <- FileCategory.fromString(cat.asId.id).toOption + coll <- cat.parent + fkey = FileKey(Ident.unsafe(coll.asId.id), fcat, Ident.unsafe(id.asId.id)) + } yield fileKeyToBinaryId(fkey) + + PathMapping(toPath)(toId) + } + + def binaryStore[F[_]: Async]( + cfg: FileRepositoryConfig, + attrStore: AttributeStore[F], + ds: DataSource, + logger: Logger[F] + ): BinaryStore[F] = + cfg match { + case FileRepositoryConfig.Database(chunkSize) => + val jdbcConfig = + JdbcStoreConfig("filechunk", chunkSize, BinnyUtils.TikaContentTypeDetect) + GenericJdbcStore[F](ds, LoggerAdapter(logger), jdbcConfig, attrStore) + + case FileRepositoryConfig.S3(endpoint, accessKey, secretKey, bucket, chunkSize) => + val keyMapping = S3KeyMapping.constant(bucket) + val minioCfg = MinioConfig + .default(endpoint, accessKey, secretKey, keyMapping) + .copy(chunkSize = chunkSize, detect = BinnyUtils.TikaContentTypeDetect) + + MinioBinaryStore[F](minioCfg, attrStore, LoggerAdapter(logger)) + + case FileRepositoryConfig.Directory(path, chunkSize) => + val fsConfig = FsStoreConfig( + path, + BinnyUtils.TikaContentTypeDetect, + FsStoreConfig.OverwriteMode.Fail, + BinnyUtils.pathMapping, + chunkSize + ) + FsBinaryStore[F](fsConfig, LoggerAdapter(logger), attrStore) + } } diff --git a/modules/store/src/main/scala/docspell/store/file/FileRepository.scala b/modules/store/src/main/scala/docspell/store/file/FileRepository.scala index b3da6da3..b8bef362 100644 --- a/modules/store/src/main/scala/docspell/store/file/FileRepository.scala +++ b/modules/store/src/main/scala/docspell/store/file/FileRepository.scala @@ -13,8 +13,7 @@ import fs2._ import docspell.common._ -import binny.BinaryId -import binny.jdbc.{GenericJdbcStore, JdbcStoreConfig} +import binny.{BinaryId, BinaryStore} import doobie.Transactor trait FileRepository[F[_]] { @@ -33,16 +32,15 @@ trait FileRepository[F[_]] { object FileRepository { - def genericJDBC[F[_]: Sync]( + def apply[F[_]: Async]( xa: Transactor[F], ds: DataSource, - chunkSize: Int + cfg: FileRepositoryConfig ): FileRepository[F] = { val attrStore = new AttributeStore[F](xa) - val cfg = JdbcStoreConfig("filechunk", chunkSize, BinnyUtils.TikaContentTypeDetect) val log = docspell.logging.getLogger[F] - val binStore = GenericJdbcStore[F](ds, BinnyUtils.LoggerAdapter(log), cfg, attrStore) val keyFun: FileKey => BinaryId = BinnyUtils.fileKeyToBinaryId + val binStore: BinaryStore[F] = BinnyUtils.binaryStore(cfg, attrStore, ds, log) new FileRepositoryImpl[F](binStore, attrStore, keyFun) } diff --git a/modules/store/src/main/scala/docspell/store/file/FileRepositoryConfig.scala b/modules/store/src/main/scala/docspell/store/file/FileRepositoryConfig.scala new file mode 100644 index 00000000..5c575e3d --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/file/FileRepositoryConfig.scala @@ -0,0 +1,27 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.store.file + +import fs2.io.file.Path + +sealed trait FileRepositoryConfig {} + +object FileRepositoryConfig { + + final case class Database(chunkSize: Int) extends FileRepositoryConfig + + final case class S3( + endpoint: String, + accessKey: String, + secretKey: String, + bucketName: String, + chunkSize: Int + ) extends FileRepositoryConfig + + final case class Directory(path: Path, chunkSize: Int) extends FileRepositoryConfig + +} diff --git a/modules/store/src/test/scala/docspell/store/StoreFixture.scala b/modules/store/src/test/scala/docspell/store/StoreFixture.scala index 91441701..dc11ff41 100644 --- a/modules/store/src/test/scala/docspell/store/StoreFixture.scala +++ b/modules/store/src/test/scala/docspell/store/StoreFixture.scala @@ -11,7 +11,7 @@ import javax.sql.DataSource import cats.effect._ import docspell.common.LenientUri -import docspell.store.file.FileRepository +import docspell.store.file.{FileRepository, FileRepositoryConfig} import docspell.store.impl.StoreImpl import docspell.store.migrate.FlywayMigrate @@ -67,7 +67,8 @@ object StoreFixture { for { ds <- dataSource(jdbc) xa <- makeXA(ds) - fr = FileRepository.genericJDBC[IO](xa, ds, 64 * 1024) + cfg = FileRepositoryConfig.Database(64 * 1024) + fr = FileRepository[IO](xa, ds, cfg) store = new StoreImpl[IO](fr, jdbc, xa) _ <- Resource.eval(store.migrate) } yield store diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 34d7f735..a9093d36 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -307,7 +307,8 @@ object Dependencies { val binny = Seq( "com.github.eikek" %% "binny-core" % BinnyVersion, "com.github.eikek" %% "binny-jdbc" % BinnyVersion, - "com.github.eikek" %% "binny-minio" % BinnyVersion + "com.github.eikek" %% "binny-minio" % BinnyVersion, + "com.github.eikek" %% "binny-fs" % BinnyVersion ) // https://github.com/flyway/flyway From e82b00c582a98d6509fbd454c013e6253fa0adff Mon Sep 17 00:00:00 2001 From: eikek Date: Mon, 7 Mar 2022 17:21:38 +0100 Subject: [PATCH 03/16] Use different file stores based on config --- .../main/scala/docspell/backend/Config.scala | 45 ++++++++++++++++++- .../docspell/common/FileStoreConfig.scala | 36 +++++++++++++++ .../scala/docspell/common/FileStoreType.scala | 32 +++++++++++++ .../scala/docspell/config/Implicits.scala | 15 +++++++ .../joex/src/main/resources/reference.conf | 35 +++++++++++++++ .../main/scala/docspell/joex/ConfigFile.scala | 5 +-- .../main/scala/docspell/joex/JoexServer.scala | 3 +- .../src/main/resources/reference.conf | 35 +++++++++++++++ .../docspell/restserver/ConfigFile.scala | 11 ++++- .../docspell/restserver/RestServer.scala | 3 +- 10 files changed, 211 insertions(+), 9 deletions(-) create mode 100644 modules/common/src/main/scala/docspell/common/FileStoreConfig.scala create mode 100644 modules/common/src/main/scala/docspell/common/FileStoreType.scala diff --git a/modules/backend/src/main/scala/docspell/backend/Config.scala b/modules/backend/src/main/scala/docspell/backend/Config.scala index 8ddce838..e9cd3356 100644 --- a/modules/backend/src/main/scala/docspell/backend/Config.scala +++ b/modules/backend/src/main/scala/docspell/backend/Config.scala @@ -6,9 +6,13 @@ package docspell.backend +import cats.data.{Validated, ValidatedNec} +import cats.implicits._ + import docspell.backend.signup.{Config => SignupConfig} import docspell.common._ import docspell.store.JdbcConfig +import docspell.store.file.FileRepositoryConfig import emil.javamail.Settings @@ -21,10 +25,49 @@ case class Config( def mailSettings: Settings = Settings.defaultSettings.copy(debug = mailDebug) + } object Config { - case class Files(chunkSize: Int, validMimeTypes: Seq[MimeType]) + case class Files( + chunkSize: Int, + validMimeTypes: Seq[MimeType], + defaultStore: Ident, + stores: Map[Ident, FileStoreConfig] + ) { + val enabledStores: Map[Ident, FileStoreConfig] = + stores.view.filter(_._2.enabled).toMap + def defaultStoreConfig: FileStoreConfig = + enabledStores(defaultStore) + + def toFileRepositoryConfig: FileRepositoryConfig = + defaultStoreConfig match { + case FileStoreConfig.DefaultDatabase(_) => + FileRepositoryConfig.Database(chunkSize) + case FileStoreConfig.S3(_, endpoint, accessKey, secretKey, bucket) => + FileRepositoryConfig.S3(endpoint, accessKey, secretKey, bucket, chunkSize) + case FileStoreConfig.FileSystem(_, directory) => + FileRepositoryConfig.Directory(directory, chunkSize) + } + + def validate: ValidatedNec[String, Files] = { + val storesEmpty = + if (enabledStores.isEmpty) + Validated.invalidNec( + "No file stores defined! Make sure at least one enabled store is present." + ) + else Validated.validNec(()) + + val defaultStorePresent = + enabledStores.get(defaultStore) match { + case Some(_) => Validated.validNec(()) + case None => + Validated.invalidNec(s"Default file store not present: ${defaultStore.id}") + } + + (storesEmpty |+| defaultStorePresent).map(_ => this) + } + } } diff --git a/modules/common/src/main/scala/docspell/common/FileStoreConfig.scala b/modules/common/src/main/scala/docspell/common/FileStoreConfig.scala new file mode 100644 index 00000000..80652217 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/FileStoreConfig.scala @@ -0,0 +1,36 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import fs2.io.file.Path + +sealed trait FileStoreConfig { + def enabled: Boolean + def storeType: FileStoreType +} +object FileStoreConfig { + case class DefaultDatabase(enabled: Boolean) extends FileStoreConfig { + val storeType = FileStoreType.DefaultDatabase + } + + case class FileSystem( + enabled: Boolean, + directory: Path + ) extends FileStoreConfig { + val storeType = FileStoreType.FileSystem + } + + case class S3( + enabled: Boolean, + endpoint: String, + accessKey: String, + secretKey: String, + bucket: String + ) extends FileStoreConfig { + val storeType = FileStoreType.S3 + } +} diff --git a/modules/common/src/main/scala/docspell/common/FileStoreType.scala b/modules/common/src/main/scala/docspell/common/FileStoreType.scala new file mode 100644 index 00000000..f67b218d --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/FileStoreType.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import cats.data.NonEmptyList + +sealed trait FileStoreType { self: Product => + def name: String = + productPrefix.toLowerCase +} +object FileStoreType { + case object DefaultDatabase extends FileStoreType + + case object S3 extends FileStoreType + + case object FileSystem extends FileStoreType + + val all: NonEmptyList[FileStoreType] = + NonEmptyList.of(DefaultDatabase, S3, FileSystem) + + def fromString(str: String): Either[String, FileStoreType] = + all + .find(_.name.equalsIgnoreCase(str)) + .toRight(s"Invalid file store type: $str") + + def unsafeFromString(str: String): FileStoreType = + fromString(str).fold(sys.error, identity) +} diff --git a/modules/config/src/main/scala/docspell/config/Implicits.scala b/modules/config/src/main/scala/docspell/config/Implicits.scala index 0bc7dda4..e9b23348 100644 --- a/modules/config/src/main/scala/docspell/config/Implicits.scala +++ b/modules/config/src/main/scala/docspell/config/Implicits.scala @@ -18,9 +18,18 @@ import docspell.logging.{Level, LogConfig} import com.github.eikek.calev.CalEvent import pureconfig.ConfigReader import pureconfig.error.{CannotConvert, FailureReason} +import pureconfig.generic.{CoproductHint, FieldCoproductHint} import scodec.bits.ByteVector object Implicits { + // the value "s-3" looks strange, this is to allow to write "s3" in the config + implicit val fileStoreCoproductHint: CoproductHint[FileStoreConfig] = + new FieldCoproductHint[FileStoreConfig]("type") { + override def fieldValue(name: String) = + if (name.equalsIgnoreCase("S3")) "s3" + else super.fieldValue(name) + } + implicit val accountIdReader: ConfigReader[AccountId] = ConfigReader[String].emap(reason(AccountId.parse)) @@ -42,6 +51,9 @@ object Implicits { implicit val identReader: ConfigReader[Ident] = ConfigReader[String].emap(reason(Ident.fromString)) + implicit def identMapReader[B: ConfigReader]: ConfigReader[Map[Ident, B]] = + pureconfig.configurable.genericMapReader[Ident, B](reason(Ident.fromString)) + implicit val byteVectorReader: ConfigReader[ByteVector] = ConfigReader[String].emap(reason { str => if (str.startsWith("hex:")) @@ -70,6 +82,9 @@ object Implicits { implicit val logLevelReader: ConfigReader[Level] = ConfigReader[String].emap(reason(Level.fromString)) + implicit val fileStoreTypeReader: ConfigReader[FileStoreType] = + ConfigReader[String].emap(reason(FileStoreType.fromString)) + def reason[A: ClassTag]( f: String => Either[String, A] ): String => Either[FailureReason, A] = diff --git a/modules/joex/src/main/resources/reference.conf b/modules/joex/src/main/resources/reference.conf index 8516c916..444ef5ef 100644 --- a/modules/joex/src/main/resources/reference.conf +++ b/modules/joex/src/main/resources/reference.conf @@ -646,6 +646,41 @@ Docpell Update Check # restrict file types that should be handed over to processing. # By default all files are allowed. valid-mime-types = [ ] + + # The id of an enabled store from the `stores` array that should + # be used. + # + # IMPORTANT NOTE: All nodes must have the exact same file store + # configuration! + default-store = "database" + + # A list of possible file stores. Each entry must have a unique + # id. The `type` is one of: default-database, filesystem, s3. + # + # The enabled property serves currently to define target stores + # for te "copy files" task. All stores with enabled=false are + # removed from the list. The `default-store` must be enabled. + stores = { + database = + { enabled = true + type = "default-database" + } + + filesystem = + { enabled = false + type = "file-system" + directory = "/some/directory" + } + + minio = + { enabled = false + type = "s3" + endpoint = "http://localhost:9000" + access-key = "username" + secret-key = "password" + bucket = "docspell" + } + } } # Configuration of the full-text search engine. diff --git a/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala b/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala index a3663030..32049b16 100644 --- a/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala +++ b/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala @@ -19,8 +19,6 @@ import pureconfig.generic.auto._ import yamusca.imports._ object ConfigFile { - import Implicits._ - def loadConfig[F[_]: Async](args: List[String]): F[Config] = { val logger = docspell.logging.getLogger[F] ConfigFactory @@ -51,6 +49,7 @@ object ConfigFile { Validation.failWhen( cfg => cfg.updateCheck.enabled && cfg.updateCheck.subject.els.isEmpty, "No subject given for enabled update check!" - ) + ), + Validation(cfg => cfg.files.validate.map(_ => cfg)) ) } diff --git a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala index 1d0458d4..29b7b9e8 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala @@ -16,7 +16,6 @@ import docspell.common.Pools import docspell.joex.routes._ import docspell.pubsub.naive.NaivePubSub import docspell.store.Store -import docspell.store.file.FileRepositoryConfig import docspell.store.records.RInternalSetting import org.http4s.HttpApp @@ -42,7 +41,7 @@ object JoexServer { store <- Store.create[F]( cfg.jdbc, - FileRepositoryConfig.Database(cfg.files.chunkSize), + cfg.files.toFileRepositoryConfig, pools.connectEC ) settings <- Resource.eval(store.transact(RInternalSetting.create)) diff --git a/modules/restserver/src/main/resources/reference.conf b/modules/restserver/src/main/resources/reference.conf index 14b21372..d3306679 100644 --- a/modules/restserver/src/main/resources/reference.conf +++ b/modules/restserver/src/main/resources/reference.conf @@ -358,6 +358,41 @@ docspell.server { # restrict file types that should be handed over to processing. # By default all files are allowed. valid-mime-types = [ ] + + # The id of an enabled store from the `stores` array that should + # be used. + # + # IMPORTANT NOTE: All nodes must have the exact same file store + # configuration! + default-store = "database" + + # A list of possible file stores. Each entry must have a unique + # id. The `type` is one of: default-database, filesystem, s3. + # + # The enabled property serves currently to define target stores + # for te "copy files" task. All stores with enabled=false are + # removed from the list. The `default-store` must be enabled. + stores = { + database = + { enabled = true + type = "default-database" + } + + filesystem = + { enabled = false + type = "file-system" + directory = "/some/directory" + } + + minio = + { enabled = false + type = "s3" + endpoint = "http://localhost:9000" + access-key = "username" + secret-key = "password" + bucket = "docspell" + } + } } } } \ No newline at end of file diff --git a/modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala b/modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala index 5e72a1d4..2e225cae 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala @@ -24,12 +24,18 @@ import scodec.bits.ByteVector object ConfigFile { private[this] val unsafeLogger = docspell.logging.unsafeLogger + // IntelliJ is wrong, this is required import Implicits._ def loadConfig[F[_]: Async](args: List[String]): F[Config] = { val logger = docspell.logging.getLogger[F] val validate = - Validation.of(generateSecretIfEmpty, duplicateOpenIdProvider, signKeyVsUserUrl) + Validation.of( + generateSecretIfEmpty, + duplicateOpenIdProvider, + signKeyVsUserUrl, + filesValidate + ) ConfigFactory .default[F, Config](logger, "docspell.server")(args, validate) } @@ -97,4 +103,7 @@ object ConfigFile { .map(checkProvider) ) } + + def filesValidate: Validation[Config] = + Validation(cfg => cfg.backend.files.validate.map(_ => cfg)) } diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala index 57c77707..4546fc34 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala @@ -18,7 +18,6 @@ import docspell.restserver.http4s.InternalHeader import docspell.restserver.ws.OutputEvent.KeepAlive import docspell.restserver.ws.OutputEvent import docspell.store.Store -import docspell.store.file.FileRepositoryConfig import docspell.store.records.RInternalSetting import org.http4s._ import org.http4s.blaze.client.BlazeClientBuilder @@ -74,7 +73,7 @@ object RestServer { httpClient <- BlazeClientBuilder[F].resource store <- Store.create[F]( cfg.backend.jdbc, - FileRepositoryConfig.Database(cfg.backend.files.chunkSize), + cfg.backend.files.toFileRepositoryConfig, pools.connectEC ) setting <- Resource.eval(store.transact(RInternalSetting.create)) From 422c0905dc20e8c0e389c2df255a8927598607ed Mon Sep 17 00:00:00 2001 From: eikek Date: Tue, 8 Mar 2022 00:11:40 +0100 Subject: [PATCH 04/16] Add task to copy files --- .../scala/docspell/backend/BackendApp.scala | 3 + .../main/scala/docspell/backend/Config.scala | 14 +- .../scala/docspell/backend/JobFactory.scala | 20 +++ .../backend/ops/OFileRepository.scala | 40 ++++++ .../main/scala/docspell/common/Banner.scala | 4 +- .../docspell/common/DocspellSystem.scala | 2 + .../docspell/common/FileCopyTaskArgs.scala | 56 ++++++++ .../docspell/common/FileStoreConfig.scala | 3 + .../main/scala/docspell/joex/ConfigFile.scala | 3 + .../scala/docspell/joex/JoexAppImpl.scala | 8 ++ .../main/scala/docspell/joex/JoexServer.scala | 2 +- .../src/main/scala/docspell/joex/Main.scala | 3 +- .../docspell/joex/filecopy/FileCopyTask.scala | 133 ++++++++++++++++++ .../logging/impl/ScribeConfigure.scala | 6 +- .../src/main/resources/docspell-openapi.yml | 41 ++++++ .../main/scala/docspell/restserver/Main.scala | 3 +- .../scala/docspell/restserver/RestApp.scala | 2 + .../docspell/restserver/RestAppImpl.scala | 5 +- .../docspell/restserver/RestServer.scala | 7 +- .../routes/FileRepositoryRoutes.scala | 54 +++++++ .../src/main/scala/docspell/store/Store.scala | 9 +- .../docspell/store/file/AttributeStore.scala | 104 +++++++++----- .../docspell/store/file/BinnyUtils.scala | 2 +- .../docspell/store/file/FileRepository.scala | 24 +++- .../store/file/FileRepositoryConfig.scala | 11 ++ .../store/file/FileRepositoryImpl.scala | 5 +- .../scala/docspell/store/impl/StoreImpl.scala | 11 +- .../scala/docspell/store/StoreFixture.scala | 2 +- 28 files changed, 512 insertions(+), 65 deletions(-) create mode 100644 modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala create mode 100644 modules/common/src/main/scala/docspell/common/FileCopyTaskArgs.scala create mode 100644 modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala create mode 100644 modules/restserver/src/main/scala/docspell/restserver/routes/FileRepositoryRoutes.scala diff --git a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala index fd217fb5..9c24f29c 100644 --- a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala +++ b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala @@ -50,6 +50,7 @@ trait BackendApp[F[_]] { def events: EventExchange[F] def notification: ONotification[F] def bookmarks: OQueryBookmarks[F] + def fileRepository: OFileRepository[F] } object BackendApp { @@ -91,6 +92,7 @@ object BackendApp { ) notifyImpl <- ONotification(store, notificationMod) bookmarksImpl <- OQueryBookmarks(store) + fileRepoImpl <- OFileRepository(queue, joexImpl) } yield new BackendApp[F] { val pubSub = pubSubT val login = loginImpl @@ -118,5 +120,6 @@ object BackendApp { val events = notificationMod val notification = notifyImpl val bookmarks = bookmarksImpl + val fileRepository = fileRepoImpl } } diff --git a/modules/backend/src/main/scala/docspell/backend/Config.scala b/modules/backend/src/main/scala/docspell/backend/Config.scala index e9cd3356..efccb8dc 100644 --- a/modules/backend/src/main/scala/docspell/backend/Config.scala +++ b/modules/backend/src/main/scala/docspell/backend/Config.scala @@ -42,15 +42,11 @@ object Config { def defaultStoreConfig: FileStoreConfig = enabledStores(defaultStore) - def toFileRepositoryConfig: FileRepositoryConfig = - defaultStoreConfig match { - case FileStoreConfig.DefaultDatabase(_) => - FileRepositoryConfig.Database(chunkSize) - case FileStoreConfig.S3(_, endpoint, accessKey, secretKey, bucket) => - FileRepositoryConfig.S3(endpoint, accessKey, secretKey, bucket, chunkSize) - case FileStoreConfig.FileSystem(_, directory) => - FileRepositoryConfig.Directory(directory, chunkSize) - } + def defaultFileRepositoryConfig: FileRepositoryConfig = + FileRepositoryConfig.fromFileStoreConfig(chunkSize, defaultStoreConfig) + + def getFileRepositoryConfig(id: Ident): Option[FileRepositoryConfig] = + stores.get(id).map(FileRepositoryConfig.fromFileStoreConfig(chunkSize, _)) def validate: ValidatedNec[String, Files] = { val storesEmpty = diff --git a/modules/backend/src/main/scala/docspell/backend/JobFactory.scala b/modules/backend/src/main/scala/docspell/backend/JobFactory.scala index 05320b31..3f43b280 100644 --- a/modules/backend/src/main/scala/docspell/backend/JobFactory.scala +++ b/modules/backend/src/main/scala/docspell/backend/JobFactory.scala @@ -15,6 +15,26 @@ import docspell.notification.api.PeriodicQueryArgs import docspell.store.records.RJob object JobFactory extends MailAddressCodec { + def fileCopy[F[_]: Sync]( + args: FileCopyTaskArgs, + submitter: AccountId = DocspellSystem.account + ): F[RJob] = + for { + id <- Ident.randomId[F] + now <- Timestamp.current[F] + job = RJob.newJob( + id, + FileCopyTaskArgs.taskName, + submitter.collective, + args, + s"Copying all files", + now, + submitter.user, + Priority.High, + Some(FileCopyTaskArgs.taskName) + ) + } yield job + def periodicQuery[F[_]: Sync](args: PeriodicQueryArgs, submitter: AccountId): F[RJob] = for { id <- Ident.randomId[F] diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala new file mode 100644 index 00000000..270ab66e --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.backend.ops + +import cats.effect._ +import cats.implicits._ + +import docspell.backend.JobFactory +import docspell.common.FileCopyTaskArgs +import docspell.store.queue.JobQueue +import docspell.store.records.RJob + +trait OFileRepository[F[_]] { + + /** Inserts the job or return None if such a job already is running. */ + def cloneFileRepository(args: FileCopyTaskArgs, notifyJoex: Boolean): F[Option[RJob]] +} + +object OFileRepository { + + def apply[F[_]: Async]( + queue: JobQueue[F], + joex: OJoex[F] + ): Resource[F, OFileRepository[F]] = + Resource.pure(new OFileRepository[F] { + def cloneFileRepository( + args: FileCopyTaskArgs, + notifyJoex: Boolean + ): F[Option[RJob]] = + for { + job <- JobFactory.fileCopy(args) + flag <- queue.insertIfNew(job) + _ <- if (notifyJoex) joex.notifyAllNodes else ().pure[F] + } yield Option.when(flag)(job) + }) +} diff --git a/modules/common/src/main/scala/docspell/common/Banner.scala b/modules/common/src/main/scala/docspell/common/Banner.scala index 169c67cc..21a7f299 100644 --- a/modules/common/src/main/scala/docspell/common/Banner.scala +++ b/modules/common/src/main/scala/docspell/common/Banner.scala @@ -14,7 +14,8 @@ case class Banner( configFile: Option[String], appId: Ident, baseUrl: LenientUri, - ftsUrl: Option[LenientUri] + ftsUrl: Option[LenientUri], + fileStoreConfig: FileStoreConfig ) { private val banner = @@ -36,6 +37,7 @@ case class Banner( s"Database: ${jdbcUrl.asString}", s"Fts: ${ftsUrl.map(_.asString).getOrElse("-")}", s"Config: ${configFile.getOrElse("")}", + s"FileRepo: ${fileStoreConfig}", "" ) diff --git a/modules/common/src/main/scala/docspell/common/DocspellSystem.scala b/modules/common/src/main/scala/docspell/common/DocspellSystem.scala index 4ecfff2a..37319864 100644 --- a/modules/common/src/main/scala/docspell/common/DocspellSystem.scala +++ b/modules/common/src/main/scala/docspell/common/DocspellSystem.scala @@ -10,6 +10,8 @@ object DocspellSystem { val user = Ident.unsafe("docspell-system") val taskGroup = user + val account: AccountId = AccountId(taskGroup, user) + val migrationTaskTracker = Ident.unsafe("full-text-index-tracker") val allPreviewTaskTracker = Ident.unsafe("generate-all-previews") val allPageCountTaskTracker = Ident.unsafe("all-page-count-tracker") diff --git a/modules/common/src/main/scala/docspell/common/FileCopyTaskArgs.scala b/modules/common/src/main/scala/docspell/common/FileCopyTaskArgs.scala new file mode 100644 index 00000000..5026a775 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/FileCopyTaskArgs.scala @@ -0,0 +1,56 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import cats.data.NonEmptyList + +import docspell.common.FileCopyTaskArgs.Selection + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.syntax._ +import io.circe.{Decoder, Encoder} + +/** This is the input to the `FileCopyTask`. The task copies all files from on + * FileRepository to one ore more target repositories. + * + * If no `from` is given, the default file repository is used. For targets, a list of ids + * can be specified that must match a configured file store in the config file. When + * selecting "all", it means all enabled stores. + */ +final case class FileCopyTaskArgs(from: Option[Ident], to: Selection) + +object FileCopyTaskArgs { + val taskName = Ident.unsafe("copy-file-repositories") + + sealed trait Selection + + object Selection { + + case object All extends Selection + case class Stores(ids: NonEmptyList[Ident]) extends Selection + + implicit val jsonEncoder: Encoder[Selection] = + Encoder.instance { + case All => "!all".asJson + case Stores(ids) => ids.toList.asJson + } + + implicit val jsonDecoder: Decoder[Selection] = + Decoder.instance { cursor => + cursor.value.asString match { + case Some(s) if s.equalsIgnoreCase("!all") => Right(All) + case _ => cursor.value.as[NonEmptyList[Ident]].map(Stores.apply) + } + } + } + + implicit val jsonDecoder: Decoder[FileCopyTaskArgs] = + deriveDecoder + + implicit val jsonEncoder: Encoder[FileCopyTaskArgs] = + deriveEncoder +} diff --git a/modules/common/src/main/scala/docspell/common/FileStoreConfig.scala b/modules/common/src/main/scala/docspell/common/FileStoreConfig.scala index 80652217..cb9afa9d 100644 --- a/modules/common/src/main/scala/docspell/common/FileStoreConfig.scala +++ b/modules/common/src/main/scala/docspell/common/FileStoreConfig.scala @@ -32,5 +32,8 @@ object FileStoreConfig { bucket: String ) extends FileStoreConfig { val storeType = FileStoreType.S3 + + override def toString = + s"S3(enabled=$enabled, endpoint=$endpoint, bucket=$bucket, accessKey=$accessKey, secretKey=***)" } } diff --git a/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala b/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala index 32049b16..a17d94a3 100644 --- a/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala +++ b/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala @@ -19,6 +19,9 @@ import pureconfig.generic.auto._ import yamusca.imports._ object ConfigFile { + // IntelliJ is wrong, this is required + import Implicits._ + def loadConfig[F[_]: Async](args: List[String]): F[Config] = { val logger = docspell.logging.getLogger[F] ConfigFactory diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala index ce28e8d3..7881a8f4 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala @@ -20,6 +20,7 @@ import docspell.ftsclient.FtsClient import docspell.ftssolr.SolrFtsClient import docspell.joex.analysis.RegexNerFile import docspell.joex.emptytrash._ +import docspell.joex.filecopy.FileCopyTask import docspell.joex.fts.{MigrationTask, ReIndexTask} import docspell.joex.hk._ import docspell.joex.learn.LearnClassifierTask @@ -279,6 +280,13 @@ object JoexAppImpl extends MailAddressCodec { PeriodicDueItemsTask.onCancel[F] ) ) + .withTask( + JobTask.json( + FileCopyTaskArgs.taskName, + FileCopyTask[F](cfg), + FileCopyTask.onCancel[F] + ) + ) .resource psch <- PeriodicScheduler.create( cfg.periodicScheduler, diff --git a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala index 29b7b9e8..a13d4b1f 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala @@ -41,7 +41,7 @@ object JoexServer { store <- Store.create[F]( cfg.jdbc, - cfg.files.toFileRepositoryConfig, + cfg.files.defaultFileRepositoryConfig, pools.connectEC ) settings <- Resource.eval(store.transact(RInternalSetting.create)) diff --git a/modules/joex/src/main/scala/docspell/joex/Main.scala b/modules/joex/src/main/scala/docspell/joex/Main.scala index 7866d6d1..a7607a5f 100644 --- a/modules/joex/src/main/scala/docspell/joex/Main.scala +++ b/modules/joex/src/main/scala/docspell/joex/Main.scala @@ -31,7 +31,8 @@ object Main extends IOApp { Option(System.getProperty("config.file")), cfg.appId, cfg.baseUrl, - Some(cfg.fullTextSearch.solr.url).filter(_ => cfg.fullTextSearch.enabled) + Some(cfg.fullTextSearch.solr.url).filter(_ => cfg.fullTextSearch.enabled), + cfg.files.defaultStoreConfig ) _ <- logger.info(s"\n${banner.render("***>")}") _ <- diff --git a/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala b/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala new file mode 100644 index 00000000..986c094b --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala @@ -0,0 +1,133 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.filecopy + +import cats.data.NonEmptyList +import cats.effect._ +import cats.implicits._ + +import docspell.common.FileCopyTaskArgs.Selection +import docspell.common.{FileCopyTaskArgs, Ident} +import docspell.joex.Config +import docspell.joex.scheduler.Task +import docspell.logging.Logger +import docspell.store.file.{BinnyUtils, FileRepository, FileRepositoryConfig} + +import binny.CopyTool.Counter +import binny.{BinaryId, BinaryStore, CopyTool} +import io.circe.generic.semiauto.deriveCodec +import io.circe.{Codec, Decoder, Encoder} + +object FileCopyTask { + type Args = FileCopyTaskArgs + + case class CopyResult(success: Boolean, message: String, counter: List[Counter]) + object CopyResult { + def noSourceImpl: CopyResult = + CopyResult(false, "No source BinaryStore implementation found!", Nil) + + def noTargetImpl: CopyResult = + CopyResult(false, "No target BinaryStore implementation found!", Nil) + + def noSourceStore(id: Ident): CopyResult = + CopyResult( + false, + s"No source file repo found with id: ${id.id}. Make sure it is present in the config.", + Nil + ) + + def noTargetStore: CopyResult = + CopyResult(false, "No target file repositories defined", Nil) + + def success(counter: NonEmptyList[Counter]): CopyResult = + CopyResult(true, "Done", counter.toList) + + implicit val binaryIdCodec: Codec[BinaryId] = + Codec.from( + Decoder.decodeString.map(BinaryId.apply), + Encoder.encodeString.contramap(_.id) + ) + + implicit val counterEncoder: Codec[Counter] = + deriveCodec + implicit val jsonCodec: Codec[CopyResult] = + deriveCodec + } + + def onCancel[F[_]]: Task[F, Args, Unit] = + Task.log(_.warn(s"Cancelling ${FileCopyTaskArgs.taskName.id} task")) + + def apply[F[_]: Async](cfg: Config): Task[F, Args, CopyResult] = + Task { ctx => + val src = ctx.args.from + .map(id => + cfg.files.getFileRepositoryConfig(id).toRight(CopyResult.noSourceStore(id)) + ) + .getOrElse(Right(cfg.files.defaultFileRepositoryConfig)) + + val targets = ctx.args.to match { + case Selection.All => + cfg.files.enabledStores.values.toList + .map(FileRepositoryConfig.fromFileStoreConfig(cfg.files.chunkSize, _)) + case Selection.Stores(ids) => + ids.traverse(cfg.files.getFileRepositoryConfig).map(_.toList).getOrElse(Nil) + } + + // remove source from targets if present there + val data = + for { + srcConfig <- src + trgConfig <- NonEmptyList + .fromList(targets.filter(_ != srcConfig)) + .toRight(CopyResult.noTargetStore) + + srcRepo = ctx.store.createFileRepository(srcConfig, true) + targetRepos = trgConfig.map(ctx.store.createFileRepository(_, false)) + } yield (srcRepo, targetRepos) + + data match { + case Right((from, tos)) => + ctx.logger.info(s"Start copying all files from ") *> + copy(ctx.logger, from, tos).flatTap(r => + if (r.success) ctx.logger.info(s"Copying finished: ${r.counter}") + else ctx.logger.error(s"Copying failed: $r") + ) + + case Left(res) => + ctx.logger.error(s"Copying failed: $res") *> res.pure[F] + } + } + + def copy[F[_]: Async]( + logger: Logger[F], + from: FileRepository[F], + to: NonEmptyList[FileRepository[F]] + ): F[CopyResult] = + FileRepository.getDelegate(from) match { + case None => + CopyResult.noSourceImpl.pure[F] + + case Some((src, srcMeta)) => + to.traverse(FileRepository.getDelegate).map(_.map(_._1)) match { + case None => + CopyResult.noTargetImpl.pure[F] + + case Some(targets) => + val log = BinnyUtils.LoggerAdapter(logger) + val maxConcurrent = { + val nCores = Runtime.getRuntime.availableProcessors() + if (nCores > 2) nCores / 2 else 1 + } + + def copyTo(to: BinaryStore[F]) = + CopyTool.copyAll[F](log, src, srcMeta, to, 50, maxConcurrent) + + logger.info(s"Start copying ${from.config} -> ${to.map(_.config)}") *> + targets.traverse(copyTo).map(CopyResult.success) + } + } +} diff --git a/modules/logging/scribe/src/main/scala/docspell/logging/impl/ScribeConfigure.scala b/modules/logging/scribe/src/main/scala/docspell/logging/impl/ScribeConfigure.scala index 6bf3ad8d..81e6e662 100644 --- a/modules/logging/scribe/src/main/scala/docspell/logging/impl/ScribeConfigure.scala +++ b/modules/logging/scribe/src/main/scala/docspell/logging/impl/ScribeConfigure.scala @@ -22,10 +22,12 @@ object ScribeConfigure { Sync[F].delay { replaceJUL() val docspellLogger = scribe.Logger("docspell") - val flywayLogger = scribe.Logger("org.flywaydb") + unsafeConfigure(scribe.Logger.root, cfg.copy(minimumLevel = getRootMinimumLevel)) unsafeConfigure(docspellLogger, cfg) - unsafeConfigure(flywayLogger, cfg) + unsafeConfigure(scribe.Logger("org.flywaydb"), cfg) + unsafeConfigure(scribe.Logger("binny"), cfg) + unsafeConfigure(scribe.Logger("org.http4s"), cfg) } private[this] def getRootMinimumLevel: Level = diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 50dc715d..14cc07e2 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -2487,6 +2487,35 @@ paths: schema: $ref: "#/components/schemas/BasicResult" + /admin/files/cloneFileRepository: + post: + operationId: "admin-files-cloneFileRepository" + tags: [Admin] + summary: Copy all files into a new repository + description: | + Submits a task that will copy all files of the application + (from the default file repository) into another file + repository as specified in the request. The request may define + ids of file repository configurations that must be present in + the config file. An empty list means to copy to all enabled + file repositories from te default file repository. + security: + - adminHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/FileRepositoryCloneRequest" + responses: + 422: + description: BadRequest + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/source: get: operationId: "sec-source-get-all" @@ -5433,6 +5462,18 @@ paths: components: schemas: + FileRepositoryCloneRequest: + description: | + Clone the file repository to a new location. + required: + - targetRepositories + properties: + targetRepositories: + type: array + items: + type: string + format: ident + BookmarkedQuery: description: | A query bookmark. diff --git a/modules/restserver/src/main/scala/docspell/restserver/Main.scala b/modules/restserver/src/main/scala/docspell/restserver/Main.scala index c8a6a003..106052a1 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/Main.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/Main.scala @@ -28,7 +28,8 @@ object Main extends IOApp { Option(System.getProperty("config.file")), cfg.appId, cfg.baseUrl, - Some(cfg.fullTextSearch.solr.url).filter(_ => cfg.fullTextSearch.enabled) + Some(cfg.fullTextSearch.solr.url).filter(_ => cfg.fullTextSearch.enabled), + cfg.backend.files.defaultStoreConfig ) _ <- logger.info(s"\n${banner.render("***>")}") _ <- diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestApp.scala b/modules/restserver/src/main/scala/docspell/restserver/RestApp.scala index 9ad2f34e..b7288f60 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestApp.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestApp.scala @@ -7,7 +7,9 @@ package docspell.restserver import fs2.Stream + import docspell.backend.BackendApp + import org.http4s.HttpRoutes import org.http4s.server.websocket.WebSocketBuilder2 diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala index ea44065d..7ed9d2dc 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala @@ -9,6 +9,7 @@ package docspell.restserver import cats.effect._ import fs2.Stream import fs2.concurrent.Topic + import docspell.backend.BackendApp import docspell.backend.auth.{AuthToken, ShareToken} import docspell.ftsclient.FtsClient @@ -23,6 +24,7 @@ import docspell.restserver.routes._ import docspell.restserver.webapp.{TemplateRoutes, Templates, WebjarRoutes} import docspell.restserver.ws.{OutputEvent, WebSocketRoutes} import docspell.store.Store + import emil.javamail.JavaMailEmil import org.http4s.HttpRoutes import org.http4s.client.Client @@ -76,7 +78,8 @@ final class RestAppImpl[F[_]: Async]( "user/otp" -> TotpRoutes.admin(backend), "user" -> UserRoutes.admin(backend), "info" -> InfoRoutes.admin(config), - "attachments" -> AttachmentRoutes.admin(backend) + "attachments" -> AttachmentRoutes.admin(backend), + "files" -> FileRepositoryRoutes.admin(backend) ) def shareRoutes( diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala index 4546fc34..a4d4fb6d 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala @@ -7,18 +7,21 @@ package docspell.restserver import scala.concurrent.duration._ + import cats.effect._ import cats.implicits._ import fs2.Stream import fs2.concurrent.Topic + import docspell.backend.msg.Topics import docspell.common._ import docspell.pubsub.naive.NaivePubSub import docspell.restserver.http4s.InternalHeader -import docspell.restserver.ws.OutputEvent.KeepAlive import docspell.restserver.ws.OutputEvent +import docspell.restserver.ws.OutputEvent.KeepAlive import docspell.store.Store import docspell.store.records.RInternalSetting + import org.http4s._ import org.http4s.blaze.client.BlazeClientBuilder import org.http4s.blaze.server.BlazeServerBuilder @@ -73,7 +76,7 @@ object RestServer { httpClient <- BlazeClientBuilder[F].resource store <- Store.create[F]( cfg.backend.jdbc, - cfg.backend.files.toFileRepositoryConfig, + cfg.backend.files.defaultFileRepositoryConfig, pools.connectEC ) setting <- Resource.eval(store.transact(RInternalSetting.create)) diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/FileRepositoryRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/FileRepositoryRoutes.scala new file mode 100644 index 00000000..7a608283 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/FileRepositoryRoutes.scala @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.restserver.routes + +import cats.data.NonEmptyList +import cats.effect._ +import cats.implicits._ + +import docspell.backend.BackendApp +import docspell.common.FileCopyTaskArgs +import docspell.common.FileCopyTaskArgs.Selection +import docspell.restapi.model._ + +import org.http4s._ +import org.http4s.circe.CirceEntityDecoder._ +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.dsl.Http4sDsl + +object FileRepositoryRoutes { + + def admin[F[_]: Async](backend: BackendApp[F]): HttpRoutes[F] = { + val dsl = Http4sDsl[F] + import dsl._ + val logger = docspell.logging.getLogger[F] + + HttpRoutes.of { case req @ POST -> Root / "cloneFileRepository" => + for { + input <- req.as[FileRepositoryCloneRequest] + args = makeTaskArgs(input) + job <- backend.fileRepository.cloneFileRepository(args, true) + result = BasicResult( + job.isDefined, + job.fold(s"Job for '${FileCopyTaskArgs.taskName.id}' already running")(j => + s"Job for '${FileCopyTaskArgs.taskName.id}' submitted: ${j.id.id}" + ) + ) + _ <- logger.info(result.message) + resp <- Ok(result) + } yield resp + } + } + + def makeTaskArgs(input: FileRepositoryCloneRequest): FileCopyTaskArgs = + NonEmptyList.fromList(input.targetRepositories) match { + case Some(nel) => + FileCopyTaskArgs(None, Selection.Stores(nel)) + case None => + FileCopyTaskArgs(None, Selection.All) + } +} diff --git a/modules/store/src/main/scala/docspell/store/Store.scala b/modules/store/src/main/scala/docspell/store/Store.scala index 24c80b98..b7f611e3 100644 --- a/modules/store/src/main/scala/docspell/store/Store.scala +++ b/modules/store/src/main/scala/docspell/store/Store.scala @@ -28,6 +28,11 @@ trait Store[F[_]] { def fileRepo: FileRepository[F] + def createFileRepository( + cfg: FileRepositoryConfig, + withAttributeStore: Boolean + ): FileRepository[F] + def add(insert: ConnectionIO[Int], exists: ConnectionIO[Boolean]): F[AddResult] } @@ -50,8 +55,8 @@ object Store { ds.setDriverClassName(jdbc.driverClass) } xa = HikariTransactor(ds, connectEC) - fr = FileRepository.apply(xa, ds, fileRepoConfig) - st = new StoreImpl[F](fr, jdbc, xa) + fr = FileRepository.apply(xa, ds, fileRepoConfig, true) + st = new StoreImpl[F](fr, jdbc, ds, xa) _ <- Resource.eval(st.migrate) } yield st } diff --git a/modules/store/src/main/scala/docspell/store/file/AttributeStore.scala b/modules/store/src/main/scala/docspell/store/file/AttributeStore.scala index 4814a9dc..127788b2 100644 --- a/modules/store/src/main/scala/docspell/store/file/AttributeStore.scala +++ b/modules/store/src/main/scala/docspell/store/file/AttributeStore.scala @@ -6,6 +6,7 @@ package docspell.store.file +import cats.Applicative import cats.data.OptionT import cats.effect._ import cats.implicits._ @@ -17,40 +18,71 @@ import binny._ import doobie._ import doobie.implicits._ -final private[file] class AttributeStore[F[_]: Sync](xa: Transactor[F]) - extends BinaryAttributeStore[F] { - def saveAttr(id: BinaryId, attrs: F[BinaryAttributes]): F[Unit] = - for { - now <- Timestamp.current[F] - a <- attrs - fileKey <- makeFileKey(id) - fm = RFileMeta( - fileKey, - now, - MimeType.parse(a.contentType.contentType).getOrElse(MimeType.octetStream), - ByteSize(a.length), - a.sha256 - ) - _ <- RFileMeta.insert(fm).transact(xa) - } yield () - - def deleteAttr(id: BinaryId): F[Boolean] = - makeFileKey(id).flatMap(fileKey => RFileMeta.delete(fileKey).transact(xa).map(_ > 0)) - - def findAttr(id: BinaryId): OptionT[F, BinaryAttributes] = - findMeta(id).map(fm => - BinaryAttributes( - fm.checksum, - SimpleContentType(fm.mimetype.asString), - fm.length.bytes - ) - ) - - def findMeta(id: BinaryId): OptionT[F, RFileMeta] = - OptionT(makeFileKey(id).flatMap(fileKey => RFileMeta.findById(fileKey).transact(xa))) - - private def makeFileKey(binaryId: BinaryId): F[FileKey] = - Sync[F] - .pure(BinnyUtils.binaryIdToFileKey(binaryId).left.map(new IllegalStateException(_))) - .rethrow +private[file] trait AttributeStore[F[_]] extends BinaryAttributeStore[F] { + def findMeta(id: BinaryId): OptionT[F, RFileMeta] +} + +private[file] object AttributeStore { + def empty[F[_]: Applicative]: AttributeStore[F] = + new AttributeStore[F] { + val delegate = BinaryAttributeStore.empty[F] + + def findMeta(id: BinaryId) = + OptionT.none + + def saveAttr(id: BinaryId, attrs: F[BinaryAttributes]) = + delegate.saveAttr(id, attrs) + + def deleteAttr(id: BinaryId) = + delegate.deleteAttr(id) + + def findAttr(id: BinaryId) = + delegate.findAttr(id) + } + + def apply[F[_]: Sync](xa: Transactor[F]): AttributeStore[F] = + new Impl[F](xa) + + final private class Impl[F[_]: Sync](xa: Transactor[F]) extends AttributeStore[F] { + def saveAttr(id: BinaryId, attrs: F[BinaryAttributes]): F[Unit] = + for { + now <- Timestamp.current[F] + a <- attrs + fileKey <- makeFileKey(id) + fm = RFileMeta( + fileKey, + now, + MimeType.parse(a.contentType.contentType).getOrElse(MimeType.octetStream), + ByteSize(a.length), + a.sha256 + ) + _ <- RFileMeta.insert(fm).transact(xa) + } yield () + + def deleteAttr(id: BinaryId): F[Boolean] = + makeFileKey(id).flatMap(fileKey => + RFileMeta.delete(fileKey).transact(xa).map(_ > 0) + ) + + def findAttr(id: BinaryId): OptionT[F, BinaryAttributes] = + findMeta(id).map(fm => + BinaryAttributes( + fm.checksum, + SimpleContentType(fm.mimetype.asString), + fm.length.bytes + ) + ) + + def findMeta(id: BinaryId): OptionT[F, RFileMeta] = + OptionT( + makeFileKey(id).flatMap(fileKey => RFileMeta.findById(fileKey).transact(xa)) + ) + + private def makeFileKey(binaryId: BinaryId): F[FileKey] = + Sync[F] + .pure( + BinnyUtils.binaryIdToFileKey(binaryId).left.map(new IllegalStateException(_)) + ) + .rethrow + } } diff --git a/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala b/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala index bdd09f48..a9f309e7 100644 --- a/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala +++ b/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala @@ -21,7 +21,7 @@ import binny.jdbc.{GenericJdbcStore, JdbcStoreConfig} import binny.minio.{MinioBinaryStore, MinioConfig, S3KeyMapping} import scodec.bits.ByteVector -private[store] object BinnyUtils { +object BinnyUtils { def fileKeyToBinaryId(fk: FileKey): BinaryId = BinaryId(s"${fk.collective.id}/${fk.category.id.id}/${fk.id.id}") diff --git a/modules/store/src/main/scala/docspell/store/file/FileRepository.scala b/modules/store/src/main/scala/docspell/store/file/FileRepository.scala index b8bef362..48b30b71 100644 --- a/modules/store/src/main/scala/docspell/store/file/FileRepository.scala +++ b/modules/store/src/main/scala/docspell/store/file/FileRepository.scala @@ -13,10 +13,12 @@ import fs2._ import docspell.common._ -import binny.{BinaryId, BinaryStore} +import binny.{BinaryAttributeStore, BinaryId, BinaryStore} import doobie.Transactor trait FileRepository[F[_]] { + def config: FileRepositoryConfig + def getBytes(key: FileKey): Stream[F, Byte] def findMeta(key: FileKey): F[Option[FileMetadata]] @@ -35,13 +37,27 @@ object FileRepository { def apply[F[_]: Async]( xa: Transactor[F], ds: DataSource, - cfg: FileRepositoryConfig + cfg: FileRepositoryConfig, + withAttributeStore: Boolean ): FileRepository[F] = { - val attrStore = new AttributeStore[F](xa) + val attrStore = + if (withAttributeStore) AttributeStore[F](xa) + else AttributeStore.empty[F] val log = docspell.logging.getLogger[F] val keyFun: FileKey => BinaryId = BinnyUtils.fileKeyToBinaryId val binStore: BinaryStore[F] = BinnyUtils.binaryStore(cfg, attrStore, ds, log) - new FileRepositoryImpl[F](binStore, attrStore, keyFun) + new FileRepositoryImpl[F](cfg, binStore, attrStore, keyFun) } + + def getDelegate[F[_]]( + repo: FileRepository[F] + ): Option[(BinaryStore[F], BinaryAttributeStore[F])] = + repo match { + case n: FileRepositoryImpl[F] => + Some((n.bs, n.attrStore)) + + case _ => + None + } } diff --git a/modules/store/src/main/scala/docspell/store/file/FileRepositoryConfig.scala b/modules/store/src/main/scala/docspell/store/file/FileRepositoryConfig.scala index 5c575e3d..41aec6da 100644 --- a/modules/store/src/main/scala/docspell/store/file/FileRepositoryConfig.scala +++ b/modules/store/src/main/scala/docspell/store/file/FileRepositoryConfig.scala @@ -8,6 +8,8 @@ package docspell.store.file import fs2.io.file.Path +import docspell.common.FileStoreConfig + sealed trait FileRepositoryConfig {} object FileRepositoryConfig { @@ -24,4 +26,13 @@ object FileRepositoryConfig { final case class Directory(path: Path, chunkSize: Int) extends FileRepositoryConfig + def fromFileStoreConfig(chunkSize: Int, cfg: FileStoreConfig): FileRepositoryConfig = + cfg match { + case FileStoreConfig.DefaultDatabase(_) => + FileRepositoryConfig.Database(chunkSize) + case FileStoreConfig.S3(_, endpoint, accessKey, secretKey, bucket) => + FileRepositoryConfig.S3(endpoint, accessKey, secretKey, bucket, chunkSize) + case FileStoreConfig.FileSystem(_, directory) => + FileRepositoryConfig.Directory(directory, chunkSize) + } } diff --git a/modules/store/src/main/scala/docspell/store/file/FileRepositoryImpl.scala b/modules/store/src/main/scala/docspell/store/file/FileRepositoryImpl.scala index 7f820244..e605d79c 100644 --- a/modules/store/src/main/scala/docspell/store/file/FileRepositoryImpl.scala +++ b/modules/store/src/main/scala/docspell/store/file/FileRepositoryImpl.scala @@ -16,8 +16,9 @@ import docspell.common._ import binny._ final class FileRepositoryImpl[F[_]: Sync]( - bs: BinaryStore[F], - attrStore: AttributeStore[F], + val config: FileRepositoryConfig, + val bs: BinaryStore[F], + val attrStore: AttributeStore[F], keyFun: FileKey => BinaryId ) extends FileRepository[F] { diff --git a/modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala b/modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala index 54505649..1c774956 100644 --- a/modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala +++ b/modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala @@ -6,12 +6,14 @@ package docspell.store.impl +import javax.sql.DataSource + import cats.arrow.FunctionK import cats.effect.Async import cats.implicits._ import cats.~> -import docspell.store.file.FileRepository +import docspell.store.file.{FileRepository, FileRepositoryConfig} import docspell.store.migrate.FlywayMigrate import docspell.store.{AddResult, JdbcConfig, Store} @@ -21,9 +23,16 @@ import doobie.implicits._ final class StoreImpl[F[_]: Async]( val fileRepo: FileRepository[F], jdbc: JdbcConfig, + ds: DataSource, xa: Transactor[F] ) extends Store[F] { + def createFileRepository( + cfg: FileRepositoryConfig, + withAttributeStore: Boolean + ): FileRepository[F] = + FileRepository(xa, ds, cfg, withAttributeStore) + def transform: ConnectionIO ~> F = FunctionK.lift(transact) diff --git a/modules/store/src/test/scala/docspell/store/StoreFixture.scala b/modules/store/src/test/scala/docspell/store/StoreFixture.scala index dc11ff41..a59871e3 100644 --- a/modules/store/src/test/scala/docspell/store/StoreFixture.scala +++ b/modules/store/src/test/scala/docspell/store/StoreFixture.scala @@ -69,7 +69,7 @@ object StoreFixture { xa <- makeXA(ds) cfg = FileRepositoryConfig.Database(64 * 1024) fr = FileRepository[IO](xa, ds, cfg) - store = new StoreImpl[IO](fr, jdbc, xa) + store = new StoreImpl[IO](fr, jdbc, ds, xa) _ <- Resource.eval(store.migrate) } yield store } From c1ce0769eb241556bf775c50c521c6306a05d75b Mon Sep 17 00:00:00 2001 From: eikek Date: Thu, 10 Mar 2022 22:18:50 +0100 Subject: [PATCH 05/16] Check file integrity --- .../scala/docspell/backend/BackendApp.scala | 2 +- .../scala/docspell/backend/JobFactory.scala | 20 ++++++ .../backend/ops/OFileRepository.scala | 54 +++++++++++++- .../common/FileIntegrityCheckArgs.scala | 22 ++++++ .../main/scala/docspell/common/FileKey.scala | 5 +- .../scala/docspell/common/FileKeyPart.scala | 53 ++++++++++++++ .../scala/docspell/joex/JoexAppImpl.scala | 10 ++- .../filecopy/FileIntegrityCheckTask.scala | 71 +++++++++++++++++++ .../routes/FileRepositoryRoutes.scala | 43 +++++++---- .../docspell/store/file/BinnyUtils.scala | 8 +++ .../docspell/store/records/RFileMeta.scala | 14 ++++ .../scala/docspell/store/StoreFixture.scala | 2 +- 12 files changed, 285 insertions(+), 19 deletions(-) create mode 100644 modules/common/src/main/scala/docspell/common/FileIntegrityCheckArgs.scala create mode 100644 modules/common/src/main/scala/docspell/common/FileKeyPart.scala create mode 100644 modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala diff --git a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala index 9c24f29c..f3142016 100644 --- a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala +++ b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala @@ -92,7 +92,7 @@ object BackendApp { ) notifyImpl <- ONotification(store, notificationMod) bookmarksImpl <- OQueryBookmarks(store) - fileRepoImpl <- OFileRepository(queue, joexImpl) + fileRepoImpl <- OFileRepository(store, queue, joexImpl) } yield new BackendApp[F] { val pubSub = pubSubT val login = loginImpl diff --git a/modules/backend/src/main/scala/docspell/backend/JobFactory.scala b/modules/backend/src/main/scala/docspell/backend/JobFactory.scala index 3f43b280..f41911f4 100644 --- a/modules/backend/src/main/scala/docspell/backend/JobFactory.scala +++ b/modules/backend/src/main/scala/docspell/backend/JobFactory.scala @@ -15,6 +15,26 @@ import docspell.notification.api.PeriodicQueryArgs import docspell.store.records.RJob object JobFactory extends MailAddressCodec { + def integrityCheck[F[_]: Sync]( + args: FileIntegrityCheckArgs, + submitter: AccountId = DocspellSystem.account + ): F[RJob] = + for { + id <- Ident.randomId[F] + now <- Timestamp.current[F] + job = RJob.newJob( + id, + FileIntegrityCheckArgs.taskName, + submitter.collective, + args, + s"Check integrity of files", + now, + submitter.user, + Priority.High, + Some(FileIntegrityCheckArgs.taskName) + ) + } yield job + def fileCopy[F[_]: Sync]( args: FileCopyTaskArgs, submitter: AccountId = DocspellSystem.account diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala index 270ab66e..aec12823 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala @@ -6,27 +6,41 @@ package docspell.backend.ops +import cats.data.OptionT import cats.effect._ import cats.implicits._ import docspell.backend.JobFactory -import docspell.common.FileCopyTaskArgs +import docspell.backend.ops.OFileRepository.IntegrityResult +import docspell.common._ +import docspell.store.Store import docspell.store.queue.JobQueue import docspell.store.records.RJob +import scodec.bits.ByteVector + trait OFileRepository[F[_]] { /** Inserts the job or return None if such a job already is running. */ def cloneFileRepository(args: FileCopyTaskArgs, notifyJoex: Boolean): F[Option[RJob]] + + def checkIntegrityAll(part: FileKeyPart, notifyJoex: Boolean): F[Option[RJob]] + + def checkIntegrity(key: FileKey, hash: Option[ByteVector]): F[Option[IntegrityResult]] } object OFileRepository { + case class IntegrityResult(ok: Boolean, key: FileKey) + def apply[F[_]: Async]( + store: Store[F], queue: JobQueue[F], joex: OJoex[F] ): Resource[F, OFileRepository[F]] = Resource.pure(new OFileRepository[F] { + private[this] val logger = docspell.logging.getLogger[F] + def cloneFileRepository( args: FileCopyTaskArgs, notifyJoex: Boolean @@ -36,5 +50,43 @@ object OFileRepository { flag <- queue.insertIfNew(job) _ <- if (notifyJoex) joex.notifyAllNodes else ().pure[F] } yield Option.when(flag)(job) + + def checkIntegrityAll(part: FileKeyPart, notifyJoex: Boolean): F[Option[RJob]] = + for { + job <- JobFactory.integrityCheck(FileIntegrityCheckArgs(part)) + flag <- queue.insertIfNew(job) + _ <- if (notifyJoex) joex.notifyAllNodes else ().pure[F] + } yield Option.when(flag)(job) + + def checkIntegrity( + key: FileKey, + hash: Option[ByteVector] + ): F[Option[IntegrityResult]] = + (for { + _ <- OptionT.liftF( + logger.debugWith(s"Checking file $key")(_.data("fileKey", key)) + ) + expectedHash <- + hash.fold(OptionT(store.fileRepo.findMeta(key)).map(_.checksum))(h => + OptionT.pure[F](h) + ) + + actualHash <- + OptionT.liftF( + logger.debugWith(s"Calculating new hash for $key")( + _.data("fileKey", key) + ) *> + store.fileRepo + .getBytes(key) + .through(fs2.hash.sha256) + .compile + .foldChunks(ByteVector.empty)(_ ++ _.toByteVector) + ) + res = IntegrityResult(expectedHash == actualHash, key) + _ <- OptionT.liftF { + if (res.ok) logger.debug(s"File hashes match for $key") + else logger.warnWith(s"File hashes differ for: $key")(_.data("fileKey", key)) + } + } yield res).value }) } diff --git a/modules/common/src/main/scala/docspell/common/FileIntegrityCheckArgs.scala b/modules/common/src/main/scala/docspell/common/FileIntegrityCheckArgs.scala new file mode 100644 index 00000000..671596f3 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/FileIntegrityCheckArgs.scala @@ -0,0 +1,22 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +final case class FileIntegrityCheckArgs(pattern: FileKeyPart) {} + +object FileIntegrityCheckArgs { + val taskName: Ident = Ident.unsafe("all-file-integrity-check") + + implicit val jsonDecoder: Decoder[FileIntegrityCheckArgs] = + deriveDecoder + + implicit val jsonEncoder: Encoder[FileIntegrityCheckArgs] = + deriveEncoder +} diff --git a/modules/common/src/main/scala/docspell/common/FileKey.scala b/modules/common/src/main/scala/docspell/common/FileKey.scala index 17dc009e..46eb8409 100644 --- a/modules/common/src/main/scala/docspell/common/FileKey.scala +++ b/modules/common/src/main/scala/docspell/common/FileKey.scala @@ -9,7 +9,10 @@ package docspell.common import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} import io.circe.{Decoder, Encoder} -case class FileKey(collective: Ident, category: FileCategory, id: Ident) +final case class FileKey(collective: Ident, category: FileCategory, id: Ident) { + override def toString = + s"${collective.id}/${category.id.id}/${id.id}" +} object FileKey { diff --git a/modules/common/src/main/scala/docspell/common/FileKeyPart.scala b/modules/common/src/main/scala/docspell/common/FileKeyPart.scala new file mode 100644 index 00000000..fab01ab0 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/FileKeyPart.scala @@ -0,0 +1,53 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.common + +import cats.implicits._ + +import io.circe.syntax._ +import io.circe.{Decoder, DecodingFailure, Encoder} + +sealed trait FileKeyPart {} + +object FileKeyPart { + + case object Empty extends FileKeyPart + + final case class Collective(collective: Ident) extends FileKeyPart + + final case class Category(collective: Ident, category: FileCategory) extends FileKeyPart + + final case class Key(key: FileKey) extends FileKeyPart + + implicit val jsonEncoder: Encoder[FileKeyPart] = + Encoder.instance { + case Empty => ().asJson + case Collective(cid) => + Map("collective" -> cid.asJson).asJson + case Category(cid, cat) => + Map("collective" -> cid.asJson, "category" -> cat.asJson).asJson + case Key(key) => + key.asJson + } + + implicit val jsonDecoder: Decoder[FileKeyPart] = + Decoder.instance { cursor => + for { + cid <- cursor.getOrElse[Option[Ident]]("collective")(None) + cat <- cursor.getOrElse[Option[FileCategory]]("category")(None) + emptyObj = cursor.keys.exists(_.isEmpty) + + c3 = cursor.as[FileKey].map(Key).toOption + c2 = (cid, cat).mapN(Category) + c1 = cid.map(Collective) + c0 = Option.when(emptyObj)(Empty) + + c = c3.orElse(c2).orElse(c1).orElse(c0) + res <- c.toRight(DecodingFailure("", cursor.history)) + } yield res + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala index 7881a8f4..fc668f0f 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala @@ -20,7 +20,7 @@ import docspell.ftsclient.FtsClient import docspell.ftssolr.SolrFtsClient import docspell.joex.analysis.RegexNerFile import docspell.joex.emptytrash._ -import docspell.joex.filecopy.FileCopyTask +import docspell.joex.filecopy.{FileCopyTask, FileIntegrityCheckTask} import docspell.joex.fts.{MigrationTask, ReIndexTask} import docspell.joex.hk._ import docspell.joex.learn.LearnClassifierTask @@ -151,6 +151,7 @@ object JoexAppImpl extends MailAddressCodec { regexNer <- RegexNerFile(cfg.textAnalysis.regexNerFileConfig, store) updateCheck <- UpdateCheck.resource(httpClient) notification <- ONotification(store, notificationMod) + fileRepo <- OFileRepository(store, queue, joex) sch <- SchedulerBuilder(cfg.scheduler, store) .withQueue(queue) .withPubSub(pubSubT) @@ -287,6 +288,13 @@ object JoexAppImpl extends MailAddressCodec { FileCopyTask.onCancel[F] ) ) + .withTask( + JobTask.json( + FileIntegrityCheckArgs.taskName, + FileIntegrityCheckTask[F](fileRepo), + FileIntegrityCheckTask.onCancel[F] + ) + ) .resource psch <- PeriodicScheduler.create( cfg.periodicScheduler, diff --git a/modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala b/modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala new file mode 100644 index 00000000..17ee1386 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala @@ -0,0 +1,71 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.filecopy + +import cats.Monoid +import cats.effect._ +import cats.implicits._ + +import docspell.backend.ops.OFileRepository +import docspell.backend.ops.OFileRepository.IntegrityResult +import docspell.common.{FileIntegrityCheckArgs, FileKey} +import docspell.joex.scheduler.Task +import docspell.store.records.RFileMeta + +import io.circe.Encoder +import io.circe.generic.semiauto.deriveEncoder + +object FileIntegrityCheckTask { + type Args = FileIntegrityCheckArgs + + case class Result(ok: Int, failedKeys: Set[FileKey]) { + override def toString: String = + s"Result(ok=$ok, failed=${failedKeys.size}, keysFailed=$failedKeys)" + } + object Result { + val empty = Result(0, Set.empty) + + def from(r: IntegrityResult): Result = + if (r.ok) Result(1, Set.empty) else Result(0, Set(r.key)) + + implicit val monoid: Monoid[Result] = + Monoid.instance(empty, (a, b) => Result(a.ok + b.ok, a.failedKeys ++ b.failedKeys)) + + implicit val jsonEncoder: Encoder[Result] = + deriveEncoder + } + + def apply[F[_]: Sync](ops: OFileRepository[F]): Task[F, Args, Result] = + Task { ctx => + ctx.store + .transact( + RFileMeta + .findAll(ctx.args.pattern, 50) + ) + .chunks + .evalTap(c => ctx.logger.info(s"Checking next ${c.size} files…")) + .unchunks + .evalMap(meta => ops.checkIntegrity(meta.id, meta.checksum.some)) + .evalMap { + case Some(r) => + Result.from(r).pure[F] + case None => + ctx.logger.error(s"File not found").as(Result.empty) + } + .foldMonoid + .compile + .lastOrError + .flatTap(result => + ctx.logger + .infoWith(s"File check result: $result")(_.data("integrityCheck", result)) + ) + } + + def onCancel[F[_]]: Task[F, Args, Unit] = + Task.log(_.warn(s"Cancelling ${FileIntegrityCheckArgs.taskName.id} task")) + +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/FileRepositoryRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/FileRepositoryRoutes.scala index 7a608283..e35c020c 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/FileRepositoryRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/FileRepositoryRoutes.scala @@ -11,8 +11,8 @@ import cats.effect._ import cats.implicits._ import docspell.backend.BackendApp -import docspell.common.FileCopyTaskArgs import docspell.common.FileCopyTaskArgs.Selection +import docspell.common.{FileCopyTaskArgs, FileIntegrityCheckArgs, FileKeyPart} import docspell.restapi.model._ import org.http4s._ @@ -27,20 +27,35 @@ object FileRepositoryRoutes { import dsl._ val logger = docspell.logging.getLogger[F] - HttpRoutes.of { case req @ POST -> Root / "cloneFileRepository" => - for { - input <- req.as[FileRepositoryCloneRequest] - args = makeTaskArgs(input) - job <- backend.fileRepository.cloneFileRepository(args, true) - result = BasicResult( - job.isDefined, - job.fold(s"Job for '${FileCopyTaskArgs.taskName.id}' already running")(j => - s"Job for '${FileCopyTaskArgs.taskName.id}' submitted: ${j.id.id}" + HttpRoutes.of { + case req @ POST -> Root / "cloneFileRepository" => + for { + input <- req.as[FileRepositoryCloneRequest] + args = makeTaskArgs(input) + job <- backend.fileRepository.cloneFileRepository(args, true) + result = BasicResult( + job.isDefined, + job.fold(s"Job for '${FileCopyTaskArgs.taskName.id}' already running")(j => + s"Job for '${FileCopyTaskArgs.taskName.id}' submitted: ${j.id.id}" + ) ) - ) - _ <- logger.info(result.message) - resp <- Ok(result) - } yield resp + _ <- logger.info(result.message) + resp <- Ok(result) + } yield resp + + case req @ POST -> Root / "integrityCheckAll" => + for { + input <- req.as[FileKeyPart] + job <- backend.fileRepository.checkIntegrityAll(input, true) + result = BasicResult( + job.isDefined, + job.fold(s"Job for '${FileCopyTaskArgs.taskName.id}' already running")(j => + s"Job for '${FileIntegrityCheckArgs.taskName.id}' submitted: ${j.id.id}" + ) + ) + _ <- logger.info(result.message) + resp <- Ok(result) + } yield resp } } diff --git a/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala b/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala index a9f309e7..e87d9d3d 100644 --- a/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala +++ b/modules/store/src/main/scala/docspell/store/file/BinnyUtils.scala @@ -26,6 +26,14 @@ object BinnyUtils { def fileKeyToBinaryId(fk: FileKey): BinaryId = BinaryId(s"${fk.collective.id}/${fk.category.id.id}/${fk.id.id}") + def fileKeyPartToPrefix(fkp: FileKeyPart): Option[String] = + fkp match { + case FileKeyPart.Empty => None + case FileKeyPart.Collective(cid) => Some(s"${cid.id}/%") + case FileKeyPart.Category(cid, cat) => Some(s"${cid.id}/${cat.id.id}/%") + case FileKeyPart.Key(key) => Some(fileKeyToBinaryId(key).id) + } + def binaryIdToFileKey(bid: BinaryId): Either[String, FileKey] = bid.id.split('/').toList match { case cId :: catId :: fId :: Nil => diff --git a/modules/store/src/main/scala/docspell/store/records/RFileMeta.scala b/modules/store/src/main/scala/docspell/store/records/RFileMeta.scala index 8263e875..fb26272d 100644 --- a/modules/store/src/main/scala/docspell/store/records/RFileMeta.scala +++ b/modules/store/src/main/scala/docspell/store/records/RFileMeta.scala @@ -8,8 +8,10 @@ package docspell.store.records import cats.data.NonEmptyList import cats.implicits._ +import fs2.Stream import docspell.common.{FileKey, _} +import docspell.store.file.BinnyUtils import docspell.store.qb.DSL._ import docspell.store.qb._ @@ -44,6 +46,18 @@ object RFileMeta { def as(alias: String): Table = Table(Some(alias)) + def findAll(part: FileKeyPart, chunkSize: Int): Stream[ConnectionIO, RFileMeta] = { + val cond = BinnyUtils + .fileKeyPartToPrefix(part) + .map(prefix => T.id.cast[String].like(prefix)) + + Select( + select(T.all), + from(T), + cond.getOrElse(Condition.unit) + ).build.query[RFileMeta].streamWithChunkSize(chunkSize) + } + def insert(r: RFileMeta): ConnectionIO[Int] = DML.insert(T, T.all, fr"${r.id},${r.created},${r.mimetype},${r.length},${r.checksum}") diff --git a/modules/store/src/test/scala/docspell/store/StoreFixture.scala b/modules/store/src/test/scala/docspell/store/StoreFixture.scala index a59871e3..2933b51f 100644 --- a/modules/store/src/test/scala/docspell/store/StoreFixture.scala +++ b/modules/store/src/test/scala/docspell/store/StoreFixture.scala @@ -68,7 +68,7 @@ object StoreFixture { ds <- dataSource(jdbc) xa <- makeXA(ds) cfg = FileRepositoryConfig.Database(64 * 1024) - fr = FileRepository[IO](xa, ds, cfg) + fr = FileRepository[IO](xa, ds, cfg, true) store = new StoreImpl[IO](fr, jdbc, ds, xa) _ <- Resource.eval(store.migrate) } yield store From 290b4ca58b0c658a0a166d0cadb0cab4949b496d Mon Sep 17 00:00:00 2001 From: eikek Date: Fri, 11 Mar 2022 22:56:14 +0100 Subject: [PATCH 06/16] Provide tasks with ability to return data and human message To allow better communication from background tasks, tasks can return not only data (json), but also a human readable message which is send via notification channels --- .../docspell/joex/filecopy/FileCopyTask.scala | 16 ++++-- .../filecopy/FileIntegrityCheckTask.scala | 49 +++++++++++++------ .../docspell/joex/process/ItemData.scala | 24 ++++++++- .../docspell/joex/scheduler/JobTask.scala | 8 +-- .../joex/scheduler/JobTaskResult.scala | 27 ++++++++++ .../joex/scheduler/JobTaskResultEncoder.scala | 49 +++++++++++++++++++ .../joex/scheduler/SchedulerImpl.scala | 19 +++---- .../docspell/notification/api/Event.scala | 6 ++- .../notification/api/EventContext.scala | 27 ++++------ .../notification/api/EventMessage.scala | 13 +++++ .../impl/AbstractEventContext.scala | 22 ++++----- .../impl/EventContextSyntax.scala | 5 +- .../impl/context/JobDoneCtx.scala | 17 +++++-- .../impl/context/TagsChangedCtxTest.scala | 10 ++-- .../src/main/resources/docspell-openapi.yml | 32 ++++++++++++ .../routes/FileRepositoryRoutes.scala | 2 +- 16 files changed, 250 insertions(+), 76 deletions(-) create mode 100644 modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskResult.scala create mode 100644 modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskResultEncoder.scala create mode 100644 modules/notification/api/src/main/scala/docspell/notification/api/EventMessage.scala diff --git a/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala b/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala index 986c094b..67be2980 100644 --- a/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala @@ -9,14 +9,12 @@ package docspell.joex.filecopy import cats.data.NonEmptyList import cats.effect._ import cats.implicits._ - import docspell.common.FileCopyTaskArgs.Selection import docspell.common.{FileCopyTaskArgs, Ident} import docspell.joex.Config -import docspell.joex.scheduler.Task +import docspell.joex.scheduler.{JobTaskResultEncoder, Task} import docspell.logging.Logger import docspell.store.file.{BinnyUtils, FileRepository, FileRepositoryConfig} - import binny.CopyTool.Counter import binny.{BinaryId, BinaryStore, CopyTool} import io.circe.generic.semiauto.deriveCodec @@ -56,6 +54,16 @@ object FileCopyTask { deriveCodec implicit val jsonCodec: Codec[CopyResult] = deriveCodec + + implicit val jobTaskResultEncoder: JobTaskResultEncoder[CopyResult] = + JobTaskResultEncoder.fromJson[CopyResult].withMessage { result => + val allGood = result.counter.map(_.success).sum + val failed = result.counter.map(_.failed.size).sum + if (result.success) + s"Successfully copied $allGood files to ${result.counter.size} stores." + else + s"Copying files failed for ${failed} files! ${allGood} were copied successfully." + } } def onCancel[F[_]]: Task[F, Args, Unit] = @@ -91,7 +99,7 @@ object FileCopyTask { data match { case Right((from, tos)) => - ctx.logger.info(s"Start copying all files from ") *> + ctx.logger.info(s"Start copying all files from $from") *> copy(ctx.logger, from, tos).flatTap(r => if (r.success) ctx.logger.info(s"Copying finished: ${r.counter}") else ctx.logger.error(s"Copying failed: $r") diff --git a/modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala b/modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala index 17ee1386..0b64f131 100644 --- a/modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala @@ -9,34 +9,49 @@ package docspell.joex.filecopy import cats.Monoid import cats.effect._ import cats.implicits._ - import docspell.backend.ops.OFileRepository import docspell.backend.ops.OFileRepository.IntegrityResult import docspell.common.{FileIntegrityCheckArgs, FileKey} -import docspell.joex.scheduler.Task +import docspell.joex.scheduler.{JobTaskResultEncoder, Task} import docspell.store.records.RFileMeta - import io.circe.Encoder import io.circe.generic.semiauto.deriveEncoder object FileIntegrityCheckTask { type Args = FileIntegrityCheckArgs - case class Result(ok: Int, failedKeys: Set[FileKey]) { + case class Result(ok: Int, failedKeys: Set[FileKey], notFoundKeys: Set[FileKey]) { override def toString: String = - s"Result(ok=$ok, failed=${failedKeys.size}, keysFailed=$failedKeys)" + s"Result(ok=$ok, failed=${failedKeys.size}, notFound=${notFoundKeys.size}, " + + s"keysFailed=$failedKeys, notFoundKeys=$notFoundKeys)" } object Result { - val empty = Result(0, Set.empty) + val empty = Result(0, Set.empty, Set.empty) + + def notFound(key: FileKey) = Result(0, Set.empty, Set(key)) def from(r: IntegrityResult): Result = - if (r.ok) Result(1, Set.empty) else Result(0, Set(r.key)) + if (r.ok) Result(1, Set.empty, Set.empty) else Result(0, Set(r.key), Set.empty) implicit val monoid: Monoid[Result] = - Monoid.instance(empty, (a, b) => Result(a.ok + b.ok, a.failedKeys ++ b.failedKeys)) + Monoid.instance( + empty, + (a, b) => + Result( + a.ok + b.ok, + a.failedKeys ++ b.failedKeys, + a.notFoundKeys ++ b.notFoundKeys + ) + ) implicit val jsonEncoder: Encoder[Result] = deriveEncoder + + implicit val jobTaskResultEncoder: JobTaskResultEncoder[Result] = + JobTaskResultEncoder.fromJson[Result].withMessage { result => + s"Integrity check finished. Ok: ${result.ok}, " + + s"Failed: ${result.failedKeys.size}, Not found: ${result.notFoundKeys.size}" + } } def apply[F[_]: Sync](ops: OFileRepository[F]): Task[F, Args, Result] = @@ -49,13 +64,16 @@ object FileIntegrityCheckTask { .chunks .evalTap(c => ctx.logger.info(s"Checking next ${c.size} files…")) .unchunks - .evalMap(meta => ops.checkIntegrity(meta.id, meta.checksum.some)) - .evalMap { - case Some(r) => - Result.from(r).pure[F] - case None => - ctx.logger.error(s"File not found").as(Result.empty) - } + .evalMap(meta => + ops.checkIntegrity(meta.id, meta.checksum.some).flatMap { + case Some(r) => + Result.from(r).pure[F] + case None => + ctx.logger + .error(s"File '${meta.id.toString}' not found in file repository") + .as(Result.notFound(meta.id)) + } + ) .foldMonoid .compile .lastOrError @@ -67,5 +85,4 @@ object FileIntegrityCheckTask { def onCancel[F[_]]: Task[F, Args, Unit] = Task.log(_.warn(s"Cancelling ${FileIntegrityCheckArgs.taskName.id} task")) - } diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala index 4d1c03b5..3b5ac5ab 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala @@ -8,6 +8,7 @@ package docspell.joex.process import docspell.common._ import docspell.joex.process.ItemData.AttachmentDates +import docspell.joex.scheduler.JobTaskResultEncoder import docspell.store.records.{RAttachment, RAttachmentMeta, RItem} import io.circe.syntax.EncoderOps @@ -118,7 +119,28 @@ object ItemData { ) .asJson, "tags" -> data.tags.asJson, - "assumedTags" -> data.classifyTags.asJson + "assumedTags" -> data.classifyTags.asJson, + "assumedCorrOrg" -> data.finalProposals + .find(MetaProposalType.CorrOrg) + .map(_.values.head.ref) + .asJson ) } + + implicit val jobTaskResultEncoder: JobTaskResultEncoder[ItemData] = + JobTaskResultEncoder.fromJson[ItemData].withMessage { data => + val tags = + if (data.tags.isEmpty && data.classifyTags.isEmpty) "" + else (data.tags ++ data.classifyTags).mkString("[", ", ", "]") + + val corg = + data.finalProposals.find(MetaProposalType.CorrOrg).map(_.values.head.ref.name) + val cpers = + data.finalProposals.find(MetaProposalType.CorrPerson).map(_.values.head.ref.name) + val org = corg match { + case Some(o) => s" by $o" + cpers.map(p => s"/$p").getOrElse("") + case None => cpers.map(p => s" by $p").getOrElse("") + } + s"Processed '${data.item.name}' $tags$org" + } } diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/JobTask.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/JobTask.scala index fd291d0b..aec269b2 100644 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/JobTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/JobTask.scala @@ -12,7 +12,7 @@ import cats.implicits._ import docspell.common.Ident import docspell.common.syntax.all._ -import io.circe.{Decoder, Encoder, Json} +import io.circe.Decoder /** Binds a Task to a name. This is required to lookup the code based on the taskName in * the RJob data and to execute it given the arguments that have to be read from a @@ -24,7 +24,7 @@ import io.circe.{Decoder, Encoder, Json} */ case class JobTask[F[_]]( name: Ident, - task: Task[F, String, Json], + task: Task[F, String, JobTaskResult], onCancel: Task[F, String, Unit] ) @@ -36,7 +36,7 @@ object JobTask { onCancel: Task[F, A, Unit] )(implicit D: Decoder[A], - E: Encoder[B] + E: JobTaskResultEncoder[B] ): JobTask[F] = { val convert: String => F[A] = str => @@ -46,6 +46,6 @@ object JobTask { Sync[F].raiseError(new Exception(s"Cannot parse task arguments: $str", ex)) } - JobTask(name, task.contramap(convert).map(E.apply), onCancel.contramap(convert)) + JobTask(name, task.contramap(convert).map(E.encode), onCancel.contramap(convert)) } } diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskResult.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskResult.scala new file mode 100644 index 00000000..d735565e --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskResult.scala @@ -0,0 +1,27 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.scheduler + +import io.circe.Json + +final case class JobTaskResult(message: Option[String], json: Option[Json]) { + + def withMessage(m: String): JobTaskResult = + copy(message = Some(m)) + + def withJson(json: Json): JobTaskResult = + copy(json = Some(json)) +} + +object JobTaskResult { + + val empty: JobTaskResult = JobTaskResult(None, None) + + def message(msg: String): JobTaskResult = JobTaskResult(Some(msg), None) + + def json(json: Json): JobTaskResult = JobTaskResult(None, Some(json)) +} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskResultEncoder.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskResultEncoder.scala new file mode 100644 index 00000000..205d01ce --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskResultEncoder.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.scheduler + +import docspell.joex.scheduler.JobTaskResultEncoder.instance + +import io.circe.Encoder + +trait JobTaskResultEncoder[A] { self => + def encode(a: A): JobTaskResult + + final def contramap[B](f: B => A): JobTaskResultEncoder[B] = + JobTaskResultEncoder.instance(b => self.encode(f(b))) + + final def map(f: JobTaskResult => JobTaskResult): JobTaskResultEncoder[A] = + instance(a => f(self.encode(a))) + + final def modify(f: (A, JobTaskResult) => JobTaskResult): JobTaskResultEncoder[A] = + instance(a => f(a, self.encode(a))) + + final def withMessage(f: A => String): JobTaskResultEncoder[A] = + modify((a, r) => r.withMessage(f(a))) +} + +object JobTaskResultEncoder { + + def apply[A](implicit v: JobTaskResultEncoder[A]): JobTaskResultEncoder[A] = v + + def instance[A](f: A => JobTaskResult): JobTaskResultEncoder[A] = + (a: A) => f(a) + + def fromJson[A: Encoder]: JobTaskResultEncoder[A] = + instance(a => JobTaskResult.json(Encoder[A].apply(a))) + + implicit val unitJobTaskResultEncoder: JobTaskResultEncoder[Unit] = + instance(_ => JobTaskResult.empty) + + implicit def optionJobTaskResultEncoder[A](implicit + ea: JobTaskResultEncoder[A] + ): JobTaskResultEncoder[Option[A]] = + instance { + case Some(a) => ea.encode(a) + case None => JobTaskResult.empty + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerImpl.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerImpl.scala index be83f9d6..a4cc030a 100644 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerImpl.scala +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerImpl.scala @@ -167,7 +167,7 @@ final class SchedulerImpl[F[_]: Async]( ctx <- Context[F, String](job, job.args, config, logSink, store) _ <- t.onCancel.run(ctx) _ <- state.modify(_.markCancelled(job)) - _ <- onFinish(job, Json.Null, JobState.Cancelled) + _ <- onFinish(job, JobTaskResult.empty, JobState.Cancelled) _ <- ctx.logger.warn("Job has been cancelled.") _ <- logger.debug(s"Job ${job.info} has been cancelled.") } yield () @@ -196,7 +196,7 @@ final class SchedulerImpl[F[_]: Async]( } } - def onFinish(job: RJob, result: Json, finishState: JobState): F[Unit] = + def onFinish(job: RJob, result: JobTaskResult, finishState: JobState): F[Unit] = for { _ <- logger.debug(s"Job ${job.info} done $finishState. Releasing resources.") _ <- permits.release *> permits.available.flatMap(a => @@ -220,7 +220,8 @@ final class SchedulerImpl[F[_]: Async]( job.state, job.subject, job.submitter, - result + result.json.getOrElse(Json.Null), + result.message ) ) ) @@ -235,7 +236,7 @@ final class SchedulerImpl[F[_]: Async]( def wrapTask( job: RJob, - task: Task[F, String, Json], + task: Task[F, String, JobTaskResult], ctx: Context[F, String] ): Task[F, String, Unit] = task @@ -250,19 +251,19 @@ final class SchedulerImpl[F[_]: Async]( case true => logger.error(ex)(s"Job ${job.info} execution failed (cancel = true)") ctx.logger.error(ex)("Job execution failed (cancel = true)") *> - (JobState.Cancelled: JobState, Json.Null).pure[F] + (JobState.Cancelled: JobState, JobTaskResult.empty).pure[F] case false => QJob.exceedsRetries(job.id, config.retries, store).flatMap { case true => logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") ctx.logger .error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") - .map(_ => (JobState.Failed: JobState, Json.Null)) + .map(_ => (JobState.Failed: JobState, JobTaskResult.empty)) case false => logger.error(ex)(s"Job ${job.info} execution failed. Retrying later.") ctx.logger .error(ex)(s"Job ${job.info} execution failed. Retrying later.") - .map(_ => (JobState.Stuck: JobState, Json.Null)) + .map(_ => (JobState.Stuck: JobState, JobTaskResult.empty)) } } }) @@ -273,7 +274,7 @@ final class SchedulerImpl[F[_]: Async]( logger.error(ex)(s"Error happened during post-processing of ${job.info}!") // we don't know the real outcome here… // since tasks should be idempotent, set it to stuck. if above has failed, this might fail anyways - onFinish(job, Json.Null, JobState.Stuck) + onFinish(job, JobTaskResult.empty, JobState.Stuck) }) def forkRun( @@ -295,7 +296,7 @@ final class SchedulerImpl[F[_]: Async]( () } *> state.modify(_.markCancelled(job)) *> - onFinish(job, Json.Null, JobState.Cancelled) *> + onFinish(job, JobTaskResult.empty, JobState.Cancelled) *> ctx.logger.warn("Job has been cancelled.") *> logger.debug(s"Job ${job.info} has been cancelled.") ) diff --git a/modules/notification/api/src/main/scala/docspell/notification/api/Event.scala b/modules/notification/api/src/main/scala/docspell/notification/api/Event.scala index 3da27455..83f320bb 100644 --- a/modules/notification/api/src/main/scala/docspell/notification/api/Event.scala +++ b/modules/notification/api/src/main/scala/docspell/notification/api/Event.scala @@ -204,7 +204,8 @@ object Event { state: JobState, subject: String, submitter: Ident, - result: Json + resultData: Json, + resultMsg: Option[String] ) extends Event { val eventType = JobDone val baseUrl = None @@ -222,7 +223,8 @@ object Event { JobState.running, "Process 3 files", account.user, - Json.Null + Json.Null, + None ) } yield ev } diff --git a/modules/notification/api/src/main/scala/docspell/notification/api/EventContext.scala b/modules/notification/api/src/main/scala/docspell/notification/api/EventContext.scala index fc3f2984..e1bd8d27 100644 --- a/modules/notification/api/src/main/scala/docspell/notification/api/EventContext.scala +++ b/modules/notification/api/src/main/scala/docspell/notification/api/EventContext.scala @@ -31,30 +31,25 @@ trait EventContext { "content" -> content ) - def defaultTitle: Either[String, String] - def defaultTitleHtml: Either[String, String] - - def defaultBody: Either[String, String] - def defaultBodyHtml: Either[String, String] + def defaultMessage: Either[String, EventMessage] + def defaultMessageHtml: Either[String, EventMessage] def defaultBoth: Either[String, String] def defaultBothHtml: Either[String, String] lazy val asJsonWithMessage: Either[String, Json] = for { - tt1 <- defaultTitle - tb1 <- defaultBody - tt2 <- defaultTitleHtml - tb2 <- defaultBodyHtml + dm1 <- defaultMessage + dm2 <- defaultMessageHtml data = asJson msg = Json.obj( "message" -> Json.obj( - "title" -> tt1.asJson, - "body" -> tb1.asJson + "title" -> dm1.title.asJson, + "body" -> dm1.body.asJson ), "messageHtml" -> Json.obj( - "title" -> tt2.asJson, - "body" -> tb2.asJson + "title" -> dm2.title.asJson, + "body" -> dm2.body.asJson ) ) } yield data.withObject(o1 => msg.withObject(o2 => o1.deepMerge(o2).asJson)) @@ -65,10 +60,8 @@ object EventContext { new EventContext { val event = ev def content = Json.obj() - def defaultTitle = Right("") - def defaultTitleHtml = Right("") - def defaultBody = Right("") - def defaultBodyHtml = Right("") + def defaultMessage = Right(EventMessage.empty) + def defaultMessageHtml = Right(EventMessage.empty) def defaultBoth = Right("") def defaultBothHtml = Right("") } diff --git a/modules/notification/api/src/main/scala/docspell/notification/api/EventMessage.scala b/modules/notification/api/src/main/scala/docspell/notification/api/EventMessage.scala new file mode 100644 index 00000000..eb341cf7 --- /dev/null +++ b/modules/notification/api/src/main/scala/docspell/notification/api/EventMessage.scala @@ -0,0 +1,13 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.notification.api + +final case class EventMessage(title: String, body: String) + +object EventMessage { + val empty: EventMessage = EventMessage("", "") +} diff --git a/modules/notification/impl/src/main/scala/docspell/notification/impl/AbstractEventContext.scala b/modules/notification/impl/src/main/scala/docspell/notification/impl/AbstractEventContext.scala index 04dc3990..416c0760 100644 --- a/modules/notification/impl/src/main/scala/docspell/notification/impl/AbstractEventContext.scala +++ b/modules/notification/impl/src/main/scala/docspell/notification/impl/AbstractEventContext.scala @@ -6,7 +6,7 @@ package docspell.notification.impl -import docspell.notification.api.EventContext +import docspell.notification.api.{EventContext, EventMessage} import yamusca.circe._ import yamusca.implicits._ @@ -24,17 +24,17 @@ abstract class AbstractEventContext extends EventContext { def renderHtml(template: Template): String = Markdown.toHtml(render(template)) - lazy val defaultTitle: Either[String, String] = - titleTemplate.map(render) + lazy val defaultMessage: Either[String, EventMessage] = + for { + title <- titleTemplate.map(render) + body <- bodyTemplate.map(render) + } yield EventMessage(title, body) - lazy val defaultTitleHtml: Either[String, String] = - titleTemplate.map(renderHtml) - - lazy val defaultBody: Either[String, String] = - bodyTemplate.map(render) - - lazy val defaultBodyHtml: Either[String, String] = - bodyTemplate.map(renderHtml) + lazy val defaultMessageHtml: Either[String, EventMessage] = + for { + title <- titleTemplate.map(renderHtml) + body <- bodyTemplate.map(renderHtml) + } yield EventMessage(title, body) lazy val defaultBoth: Either[String, String] = for { diff --git a/modules/notification/impl/src/main/scala/docspell/notification/impl/EventContextSyntax.scala b/modules/notification/impl/src/main/scala/docspell/notification/impl/EventContextSyntax.scala index 1609105c..adf5b486 100644 --- a/modules/notification/impl/src/main/scala/docspell/notification/impl/EventContextSyntax.scala +++ b/modules/notification/impl/src/main/scala/docspell/notification/impl/EventContextSyntax.scala @@ -18,8 +18,9 @@ trait EventContextSyntax { implicit final class EventContextOps(self: EventContext) { def withDefault[F[_]](logger: Logger[F])(f: (String, String) => F[Unit]): F[Unit] = (for { - tt <- self.defaultTitle - tb <- self.defaultBody + dm <- self.defaultMessage + tt = dm.title + tb = dm.body } yield f(tt, tb)).fold(logError(logger), identity) def withJsonMessage[F[_]](logger: Logger[F])(f: Json => F[Unit]): F[Unit] = diff --git a/modules/notification/impl/src/main/scala/docspell/notification/impl/context/JobDoneCtx.scala b/modules/notification/impl/src/main/scala/docspell/notification/impl/context/JobDoneCtx.scala index 8ef19754..b23161fc 100644 --- a/modules/notification/impl/src/main/scala/docspell/notification/impl/context/JobDoneCtx.scala +++ b/modules/notification/impl/src/main/scala/docspell/notification/impl/context/JobDoneCtx.scala @@ -23,9 +23,14 @@ final case class JobDoneCtx(event: Event.JobDone, data: JobDoneCtx.Data) val content = data.asJson val titleTemplate = Right(mustache"{{eventType}} (by *{{account.user}}*)") - val bodyTemplate = Right( - mustache"""{{#content}}_'{{subject}}'_ finished {{/content}}""" - ) + val bodyTemplate = + data.resultMsg match { + case None => + Right(mustache"""{{#content}}_'{{subject}}'_ finished {{/content}}""") + case Some(msg) => + val tpl = s"""{{#content}}$msg{{/content}}""" + yamusca.imports.mustache.parse(tpl).left.map(_._2) + } } object JobDoneCtx { @@ -46,7 +51,8 @@ object JobDoneCtx { state: JobState, subject: String, submitter: Ident, - result: Json + resultData: Json, + resultMsg: Option[String] ) object Data { implicit val jsonEncoder: Encoder[Data] = @@ -61,7 +67,8 @@ object JobDoneCtx { ev.state, ev.subject, ev.submitter, - ev.result + ev.resultData, + ev.resultMsg ) } } diff --git a/modules/notification/impl/src/test/scala/docspell/notification/impl/context/TagsChangedCtxTest.scala b/modules/notification/impl/src/test/scala/docspell/notification/impl/context/TagsChangedCtxTest.scala index 094ae368..fe72545b 100644 --- a/modules/notification/impl/src/test/scala/docspell/notification/impl/context/TagsChangedCtxTest.scala +++ b/modules/notification/impl/src/test/scala/docspell/notification/impl/context/TagsChangedCtxTest.scala @@ -46,9 +46,10 @@ class TagsChangedCtxTest extends FunSuite { TagsChangedCtx.Data(account, List(item), List(tag), Nil, url.some.map(_.asString)) ) - assertEquals(ctx.defaultTitle.toOption.get, "TagsChanged (by *user2*)") + val dm = ctx.defaultMessage.toOption.get + assertEquals(dm.title, "TagsChanged (by *user2*)") assertEquals( - ctx.defaultBody.toOption.get, + dm.body, "Adding *tag-red* on [`Report 2`](http://test/item-1)." ) } @@ -65,9 +66,10 @@ class TagsChangedCtxTest extends FunSuite { ) ) - assertEquals(ctx.defaultTitle.toOption.get, "TagsChanged (by *user2*)") + val dm = ctx.defaultMessage.toOption.get + assertEquals(dm.title, "TagsChanged (by *user2*)") assertEquals( - ctx.defaultBody.toOption.get, + dm.body, "Adding *tag-red*; Removing *tag-blue* on [`Report 2`](http://test/item-1)." ) } diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 14cc07e2..669825e4 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -2516,6 +2516,30 @@ paths: schema: $ref: "#/components/schemas/BasicResult" + /admin/files/integrityCheck: + post: + operationId: "admin-files-integrityCheck" + tags: [ Admin ] + summary: Verifies the stored checksum + description: | + Submits a task that goes through the files and compares the + stored checksum (at the time of inserting) against a newly + calculated one. + security: + - adminHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/FileIntegrityCheckRequest" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/source: get: operationId: "sec-source-get-all" @@ -5462,6 +5486,14 @@ paths: components: schemas: + FileIntegrityCheckRequest: + description: | + Data for running a file integrity check + properties: + collective: + type: string + format: ident + FileRepositoryCloneRequest: description: | Clone the file repository to a new location. diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/FileRepositoryRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/FileRepositoryRoutes.scala index e35c020c..3d4592c2 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/FileRepositoryRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/FileRepositoryRoutes.scala @@ -43,7 +43,7 @@ object FileRepositoryRoutes { resp <- Ok(result) } yield resp - case req @ POST -> Root / "integrityCheckAll" => + case req @ POST -> Root / "integrityCheck" => for { input <- req.as[FileKeyPart] job <- backend.fileRepository.checkIntegrityAll(input, true) From b71085761bf9f496b7b93f3f81839c69b9240296 Mon Sep 17 00:00:00 2001 From: eikek Date: Sat, 12 Mar 2022 01:20:06 +0100 Subject: [PATCH 07/16] Add some docs to file backends --- .../docspell/joex/hk/HouseKeepingTask.scala | 5 +- website/site/content/docs/configure/_index.md | 172 +++++++++++++++++- website/styles/content.css | 7 + 3 files changed, 179 insertions(+), 5 deletions(-) diff --git a/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala index e6d0a86b..20b1b2e6 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala @@ -8,21 +8,20 @@ package docspell.joex.hk import cats.effect._ import cats.implicits._ - import docspell.common._ import docspell.joex.Config import docspell.joex.scheduler.Task import docspell.store.records._ import docspell.store.usertask.UserTaskScope - import com.github.eikek.calev._ +import docspell.backend.ops.OFileRepository object HouseKeepingTask { private val periodicId = Ident.unsafe("docspell-houskeeping") val taskName: Ident = Ident.unsafe("housekeeping") - def apply[F[_]: Async](cfg: Config): Task[F, Unit, Unit] = + def apply[F[_]: Async](cfg: Config, fileRepo: OFileRepository[F]): Task[F, Unit, Unit] = Task .log[F, Unit](_.info(s"Running house-keeping task now")) .flatMap(_ => CleanupInvitesTask(cfg.houseKeeping.cleanupInvites)) diff --git a/website/site/content/docs/configure/_index.md b/website/site/content/docs/configure/_index.md index b57cf1ea..4c09fce3 100644 --- a/website/site/content/docs/configure/_index.md +++ b/website/site/content/docs/configure/_index.md @@ -160,6 +160,22 @@ enabled by providing a secret: This secret must be provided to all requests to a `/api/v1/admin/` endpoint. +The most convenient way to execute admin tasks is to use the +[cli](@/docs/tools/cli.md). You get a list of possible admin commands +via `dsc admin help`. + +To see the output of the commands, there are these ways: + +1. looking at the joex logs, which gives most details. +2. Use the job-queue page when logged in as `docspell-system` +3. setup a [webhook](@/docs/webapp/notification.md) to be notified + when a job finishes. This way you get a small message. + +All admin tasks (and also some other system tasks) are run under the +account `docspell-system` (collective and user). You need to create +this account and setup the notification hooks in there - not in your +normal account. + ## Full-Text Search: SOLR @@ -204,6 +220,12 @@ a call: $ curl -XPOST -H "Docspell-Admin-Secret: test123" http://localhost:7880/api/v1/admin/fts/reIndexAll ``` +or use the [cli](@/docs/tools/cli.md): + +```bash +dsc admin -a test123 recreate-index +``` + Here the `test123` is the key defined with `admin-endpoint.secret`. If it is empty (the default), this call is disabled (all admin routes). Otherwise, the POST request will submit a system task that is executed @@ -445,6 +467,147 @@ If you find that these methods do not suffice for your case, please open an issue. +## File Backends + +Docspell allows to choose from different storage backends for binary +files. You can choose between: + +1. *Database (the recommended default)* + + The database can be used to store the files as well. It is the + default. It doesn't require any other configuration and works well + with multiple instances of restservers and joex nodes. +2. *S3* + + The S3 backend allows to store files in an S3 compatible storage. + It was tested with MinIO, which is possible to self host. + +3. *Filesystem* + + The filesystem can also be used directly, by specifying a + directory. Be aware that _all_ nodes must have read and write + access into this directory! When running multiple nodes over a + network, consider using one of the above instead. Docspell uses a + fixed structure for storing the files below the given directory, it + cannot be configured. + +When using S3 or filesystem, remember to backup the database *and* the +files! + +Note that Docspell not only stores the file that are uploaded, but +also some other files for internal use. + +### Configuring + +{% warningbubble(title="Note") %} + +Each node must have the same config for its file backend! When using +the filesystem, make sure all processes can access the directory with +read and write permissions. + +{% end %} + +The file storage backend can be configured inside the `files` section +(see the default configs below): + +```conf +files { + … + default-store = "database" + + stores = { + database = + { enabled = true + type = "default-database" + } + + filesystem = + { enabled = false + type = "file-system" + directory = "/some/directory" + } + + minio = + { enabled = false + type = "s3" + endpoint = "http://localhost:9000" + access-key = "username" + secret-key = "password" + bucket = "docspell" + } + } +} +``` + +The `stores` object defines a set of stores and the `default-store` +selects the one that should be used. All disabled store configurations +are removed from the list. Thus the `default-store` must be enabled. +Other enabled stores can be used as the target when copying files (see +below). + +A store configuration requires a `enabled` and `type` property. +Depending on the `type` property, other properties are required, they +are presented above. The available storage types are +`default-database`, `file-system` and `s3`. + +If you use the docker setup, you can find the corresponding +environment variables to the above config snippet +[below](#environment-variables). + +### Change Backends + +It is possible to change backends with a bit of manual effort. When +doing this, please make sure that the application is not used. It is +important that no file is uploaded during the following steps. + +The [cli](@/docs/tools/cli.md) will be used, please set it up first +and you need to enable the [admin endpoint](#admin-endpoint). Config +changes mentioned here must be applied to all nodes - joex and +restserver! + +1. In the config, enable a second file backend (besides the default) + you want to change to and start docspell as normal. Don't change + `default-store` yet. +2. Run the file integrity check in order to see whether all files are + ok as they are in the current store. This can be done using the + [cli](@/docs/tools/cli.md) by running: + + ```bash + dsc admin file-integrity-check + ``` +3. Run the copy files admin command which will copy all files from the + current `default-store` to all other enabled stores. + + ```bash + dsc admin clone-file-repository + ``` + + And wait until it's done :-). You can see the progress in the jobs + page when logged in as `docspell-system` or just look at the logs. +4. In the config, change the `default-store` to the one you just + copied all the files to and restart docspell. +5. Login and do some smoke tests. Then run the file integrity check + again: + + ```bash + dsc admin file-integrity-check + ``` + +If all is fine, then you are done and are now using the new file +backend. If the second integrity check fails, please open an issue. +You need then to revert the config change of step 4 to use the +previous `default-store` again. + +If you want to delete the files from the database, you can do so by +running the following SQL against the database: + +```sql +DELETE FROM filechunk +``` + +You can copy them back into the database using the steps above. + + ## File Processing Files are being processed by the joex component. So all the respective @@ -517,9 +680,14 @@ setting has significant impact, especially when your documents are in German. Here are some rough numbers on jvm heap usage (the same file was used for all tries): - +
- + + + + + + diff --git a/website/styles/content.css b/website/styles/content.css index e936f10e..0e2ffac6 100644 --- a/website/styles/content.css +++ b/website/styles/content.css @@ -3,6 +3,13 @@ @apply leading-relaxed text-left; } + .content table.striped-basic tbody tr { + @apply border-t dark:border-stone-600; + } + .content table { + @apply w-full my-2 px-4; + } + .content h1:not(.no-default) { @apply text-4xl font-serif font-bold mt-6 mb-3 py-1 border-b dark:border-stone-800 text-stone-700 dark:text-stone-200; } From cd3db6ea08bea5c9562cf99c3cff03570b6716e1 Mon Sep 17 00:00:00 2001 From: eikek Date: Sat, 12 Mar 2022 12:06:36 +0100 Subject: [PATCH 08/16] Run file integrity check in house keeping tasks --- .../joex/src/main/resources/reference.conf | 5 ++ .../scala/docspell/joex/JoexAppImpl.scala | 2 +- .../scala/docspell/joex/fts/ReIndexTask.scala | 3 +- .../docspell/joex/hk/CheckNodesTask.scala | 9 ++-- .../docspell/joex/hk/CleanupInvitesTask.scala | 9 ++-- .../docspell/joex/hk/CleanupJobsTask.scala | 9 ++-- .../joex/hk/CleanupRememberMeTask.scala | 10 ++-- .../docspell/joex/hk/CleanupResult.scala | 21 ++++++++ .../docspell/joex/hk/HouseKeepingConfig.scala | 4 +- .../docspell/joex/hk/HouseKeepingTask.scala | 52 ++++++++++++++++--- .../docspell/joex/hk/IntegrityCheckTask.scala | 32 ++++++++++++ .../joex/learn/LearnItemEntities.scala | 2 +- .../docspell/joex/process/ItemHandler.scala | 6 +-- .../docspell/joex/process/ProcessItem.scala | 1 + .../scala/docspell/joex/scheduler/Task.scala | 19 ++++--- 15 files changed, 150 insertions(+), 34 deletions(-) create mode 100644 modules/joex/src/main/scala/docspell/joex/hk/CleanupResult.scala create mode 100644 modules/joex/src/main/scala/docspell/joex/hk/IntegrityCheckTask.scala diff --git a/modules/joex/src/main/resources/reference.conf b/modules/joex/src/main/resources/reference.conf index 444ef5ef..318bdeff 100644 --- a/modules/joex/src/main/resources/reference.conf +++ b/modules/joex/src/main/resources/reference.conf @@ -194,6 +194,11 @@ docspell.joex { # How often the node must be unreachable, before it is removed. min-not-found = 2 } + + # Checks all files against their checksum + integrity-check { + enabled = true + } } # A periodic task to check for new releases of docspell. It can diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala index fc668f0f..ce34e43c 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala @@ -194,7 +194,7 @@ object JoexAppImpl extends MailAddressCodec { .withTask( JobTask.json( HouseKeepingTask.taskName, - HouseKeepingTask[F](cfg), + HouseKeepingTask[F](cfg, fileRepo), HouseKeepingTask.onCancel[F] ) ) diff --git a/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala b/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala index 37c1326b..7c7f81e0 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala @@ -7,6 +7,7 @@ package docspell.joex.fts import cats.effect._ +import cats.implicits._ import docspell.backend.fulltext.CreateIndex import docspell.common._ @@ -42,7 +43,7 @@ object ReIndexTask { (collective match { case Some(_) => FtsWork - .clearIndex(collective) + .clearIndex[F](collective) .recoverWith( FtsWork.log[F](_.info("Clearing data failed. Continue re-indexing.")) ) ++ diff --git a/modules/joex/src/main/scala/docspell/joex/hk/CheckNodesTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/CheckNodesTask.scala index 8e0bdb1c..09651893 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/CheckNodesTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/CheckNodesTask.scala @@ -18,10 +18,9 @@ import org.http4s.blaze.client.BlazeClientBuilder import org.http4s.client.Client object CheckNodesTask { - def apply[F[_]: Async]( cfg: HouseKeepingConfig.CheckNodes - ): Task[F, Unit, Unit] = + ): Task[F, Unit, CleanupResult] = Task { ctx => if (cfg.enabled) for { @@ -35,9 +34,11 @@ object CheckNodesTask { ) n <- removeNodes(ctx, cfg) _ <- ctx.logger.info(s"Removed $n nodes") - } yield () + } yield CleanupResult.of(n) else - ctx.logger.info("CheckNodes task is disabled in the configuration") + ctx.logger.info("CheckNodes task is disabled in the configuration") *> + CleanupResult.disabled.pure[F] + } def checkNodes[F[_]: Async](ctx: Context[F, _], client: Client[F]): F[Unit] = diff --git a/modules/joex/src/main/scala/docspell/joex/hk/CleanupInvitesTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/CleanupInvitesTask.scala index ae7b59e8..de23eb71 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/CleanupInvitesTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/CleanupInvitesTask.scala @@ -15,7 +15,9 @@ import docspell.store.records._ object CleanupInvitesTask { - def apply[F[_]: Sync](cfg: HouseKeepingConfig.CleanupInvites): Task[F, Unit, Unit] = + def apply[F[_]: Sync]( + cfg: HouseKeepingConfig.CleanupInvites + ): Task[F, Unit, CleanupResult] = Task { ctx => if (cfg.enabled) for { @@ -24,8 +26,9 @@ object CleanupInvitesTask { _ <- ctx.logger.info(s"Cleanup invitations older than $ts") n <- ctx.store.transact(RInvitation.deleteOlderThan(ts)) _ <- ctx.logger.info(s"Removed $n invitations") - } yield () + } yield CleanupResult.of(n) else - ctx.logger.info("CleanupInvites task is disabled in the configuration") + ctx.logger.info("CleanupInvites task is disabled in the configuration") *> + CleanupResult.disabled.pure[F] } } diff --git a/modules/joex/src/main/scala/docspell/joex/hk/CleanupJobsTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/CleanupJobsTask.scala index 4ae1c9e1..67a1e1a2 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/CleanupJobsTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/CleanupJobsTask.scala @@ -17,7 +17,9 @@ import docspell.store.records._ object CleanupJobsTask { - def apply[F[_]: Sync](cfg: HouseKeepingConfig.CleanupJobs): Task[F, Unit, Unit] = + def apply[F[_]: Sync]( + cfg: HouseKeepingConfig.CleanupJobs + ): Task[F, Unit, CleanupResult] = Task { ctx => if (cfg.enabled) for { @@ -26,9 +28,10 @@ object CleanupJobsTask { _ <- ctx.logger.info(s"Cleanup jobs older than $ts") n <- deleteDoneJobs(ctx.store, ts, cfg.deleteBatch) _ <- ctx.logger.info(s"Removed $n jobs") - } yield () + } yield CleanupResult.of(n) else - ctx.logger.info("CleanupJobs task is disabled in the configuration") + ctx.logger.info("CleanupJobs task is disabled in the configuration") *> + CleanupResult.disabled.pure[F] } def deleteDoneJobs[F[_]: Sync](store: Store[F], ts: Timestamp, batch: Int): F[Int] = diff --git a/modules/joex/src/main/scala/docspell/joex/hk/CleanupRememberMeTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/CleanupRememberMeTask.scala index b9cfdc98..120b8cc8 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/CleanupRememberMeTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/CleanupRememberMeTask.scala @@ -14,8 +14,9 @@ import docspell.joex.scheduler.Task import docspell.store.records._ object CleanupRememberMeTask { - - def apply[F[_]: Sync](cfg: HouseKeepingConfig.CleanupRememberMe): Task[F, Unit, Unit] = + def apply[F[_]: Sync]( + cfg: HouseKeepingConfig.CleanupRememberMe + ): Task[F, Unit, CleanupResult] = Task { ctx => if (cfg.enabled) for { @@ -24,8 +25,9 @@ object CleanupRememberMeTask { _ <- ctx.logger.info(s"Cleanup remember-me tokens older than $ts") n <- ctx.store.transact(RRememberMe.deleteOlderThan(ts)) _ <- ctx.logger.info(s"Removed $n tokens") - } yield () + } yield CleanupResult.of(n) else - ctx.logger.info("CleanupRememberMe task is disabled in the configuration") + ctx.logger.info("CleanupRememberMe task is disabled in the configuration") *> + CleanupResult.disabled.pure[F] } } diff --git a/modules/joex/src/main/scala/docspell/joex/hk/CleanupResult.scala b/modules/joex/src/main/scala/docspell/joex/hk/CleanupResult.scala new file mode 100644 index 00000000..931f6714 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/hk/CleanupResult.scala @@ -0,0 +1,21 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.hk + +import io.circe.Encoder +import io.circe.generic.semiauto.deriveEncoder + +case class CleanupResult(removed: Int, disabled: Boolean) { + def asString = if (disabled) "disabled" else s"$removed" +} +object CleanupResult { + def of(n: Int): CleanupResult = CleanupResult(n, false) + def disabled: CleanupResult = CleanupResult(0, true) + + implicit val jsonEncoder: Encoder[CleanupResult] = + deriveEncoder +} diff --git a/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingConfig.scala b/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingConfig.scala index 0d6b4778..2b5bc355 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingConfig.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingConfig.scala @@ -16,7 +16,8 @@ case class HouseKeepingConfig( cleanupInvites: CleanupInvites, cleanupJobs: CleanupJobs, cleanupRememberMe: CleanupRememberMe, - checkNodes: CheckNodes + checkNodes: CheckNodes, + integrityCheck: IntegrityCheck ) object HouseKeepingConfig { @@ -29,4 +30,5 @@ object HouseKeepingConfig { case class CheckNodes(enabled: Boolean, minNotFound: Int) + case class IntegrityCheck(enabled: Boolean) } diff --git a/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala index 20b1b2e6..c0c3d084 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala @@ -8,26 +8,41 @@ package docspell.joex.hk import cats.effect._ import cats.implicits._ + +import docspell.backend.ops.OFileRepository import docspell.common._ import docspell.joex.Config -import docspell.joex.scheduler.Task +import docspell.joex.filecopy.FileIntegrityCheckTask +import docspell.joex.scheduler.{JobTaskResultEncoder, Task} import docspell.store.records._ import docspell.store.usertask.UserTaskScope + import com.github.eikek.calev._ -import docspell.backend.ops.OFileRepository +import io.circe.Encoder +import io.circe.generic.semiauto.deriveEncoder object HouseKeepingTask { private val periodicId = Ident.unsafe("docspell-houskeeping") val taskName: Ident = Ident.unsafe("housekeeping") - def apply[F[_]: Async](cfg: Config, fileRepo: OFileRepository[F]): Task[F, Unit, Unit] = + def apply[F[_]: Async]( + cfg: Config, + fileRepo: OFileRepository[F] + ): Task[F, Unit, Result] = { + val combined = + ( + CheckNodesTask(cfg.houseKeeping.checkNodes), + CleanupInvitesTask(cfg.houseKeeping.cleanupInvites), + CleanupJobsTask(cfg.houseKeeping.cleanupJobs), + CleanupRememberMeTask(cfg.houseKeeping.cleanupRememberMe), + IntegrityCheckTask(cfg.houseKeeping.integrityCheck, fileRepo) + ).mapN(Result.apply) + Task .log[F, Unit](_.info(s"Running house-keeping task now")) - .flatMap(_ => CleanupInvitesTask(cfg.houseKeeping.cleanupInvites)) - .flatMap(_ => CleanupRememberMeTask(cfg.houseKeeping.cleanupRememberMe)) - .flatMap(_ => CleanupJobsTask(cfg.houseKeeping.cleanupJobs)) - .flatMap(_ => CheckNodesTask(cfg.houseKeeping.checkNodes)) + .flatMap(_ => combined) + } def onCancel[F[_]]: Task[F, Unit, Unit] = Task.log[F, Unit](_.warn("Cancelling house-keeping task")) @@ -45,4 +60,27 @@ object HouseKeepingTask { None ) .map(_.copy(id = periodicId)) + + case class Result( + checkNodes: CleanupResult, + cleanupInvites: CleanupResult, + cleanupJobs: CleanupResult, + cleanupRememberMe: CleanupResult, + integrityCheck: FileIntegrityCheckTask.Result + ) + + object Result { + implicit val jsonEncoder: Encoder[Result] = + deriveEncoder + + implicit val jobTaskResultEncoder: JobTaskResultEncoder[Result] = + JobTaskResultEncoder.fromJson[Result].withMessage { r => + s"- Nodes removed: ${r.checkNodes.asString}\n" + + s"- Invites removed: ${r.cleanupInvites.asString}\n" + + s"- Jobs removed: ${r.cleanupJobs.asString}\n" + + s"- RememberMe removed: ${r.cleanupRememberMe.asString}\n" + + s"- Integrity check: ok=${r.integrityCheck.ok}, failed=${r.integrityCheck.failedKeys.size}, notFound=${r.integrityCheck.notFoundKeys.size}" + } + + } } diff --git a/modules/joex/src/main/scala/docspell/joex/hk/IntegrityCheckTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/IntegrityCheckTask.scala new file mode 100644 index 00000000..42bb3333 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/hk/IntegrityCheckTask.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex.hk + +import cats.effect._ +import cats.implicits._ + +import docspell.backend.ops.OFileRepository +import docspell.common._ +import docspell.joex.filecopy.FileIntegrityCheckTask +import docspell.joex.scheduler.Task + +object IntegrityCheckTask { + + def apply[F[_]: Sync]( + cfg: HouseKeepingConfig.IntegrityCheck, + fileRepo: OFileRepository[F] + ): Task[F, Unit, FileIntegrityCheckTask.Result] = + Task { ctx => + if (cfg.enabled) + FileIntegrityCheckTask(fileRepo).run( + ctx.map(_ => FileIntegrityCheckArgs(FileKeyPart.Empty)) + ) + else + ctx.logger.info("Integrity check task is disabled in the configuration") *> + FileIntegrityCheckTask.Result.empty.pure[F] + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/learn/LearnItemEntities.scala b/modules/joex/src/main/scala/docspell/joex/learn/LearnItemEntities.scala index b0d4f181..9fc1b502 100644 --- a/modules/joex/src/main/scala/docspell/joex/learn/LearnItemEntities.scala +++ b/modules/joex/src/main/scala/docspell/joex/learn/LearnItemEntities.scala @@ -23,7 +23,7 @@ object LearnItemEntities { maxItems: Int, maxTextLen: Int ): Task[F, A, Unit] = - learnCorrOrg(analyser, collective, maxItems, maxTextLen) + learnCorrOrg[F, A](analyser, collective, maxItems, maxTextLen) .flatMap(_ => learnCorrPerson[F, A](analyser, collective, maxItems, maxTextLen)) .flatMap(_ => learnConcPerson(analyser, collective, maxItems, maxTextLen)) .flatMap(_ => learnConcEquip(analyser, collective, maxItems, maxTextLen)) diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala index fa1774e0..5da1e6d6 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala @@ -25,7 +25,7 @@ object ItemHandler { type Args = ProcessItemArgs def onCancel[F[_]: Sync]: Task[F, Args, Unit] = - logWarn("Now cancelling.").flatMap(_ => + logWarn[F]("Now cancelling.").flatMap(_ => markItemCreated.flatMap { case true => Task.pure(()) @@ -41,10 +41,10 @@ object ItemHandler { analyser: TextAnalyser[F], regexNer: RegexNerFile[F] ): Task[F, Args, Option[ItemData]] = - logBeginning.flatMap(_ => + logBeginning[F].flatMap(_ => DuplicateCheck[F] .flatMap(args => - if (args.files.isEmpty) logNoFiles.map(_ => None) + if (args.files.isEmpty) logNoFiles[F].map(_ => None) else { val create: Task[F, Args, ItemData] = CreateItem[F].contramap(_ => args.pure[F]) diff --git a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala index 18cb7a31..735d13be 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala @@ -7,6 +7,7 @@ package docspell.joex.process import cats.effect._ +import cats.implicits._ import docspell.analysis.TextAnalyser import docspell.backend.ops.OItem diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/Task.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/Task.scala index d211d5a0..4ba4d84d 100644 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/Task.scala +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/Task.scala @@ -18,12 +18,6 @@ trait Task[F[_], A, B] { def run(ctx: Context[F, A]): F[B] - def map[C](f: B => C)(implicit F: Functor[F]): Task[F, A, C] = - Task(Task.toKleisli(this).map(f)) - - def flatMap[C](f: B => Task[F, A, C])(implicit F: FlatMap[F]): Task[F, A, C] = - Task(Task.toKleisli(this).flatMap(a => Task.toKleisli(f(a)))) - def andThen[C](f: B => F[C])(implicit F: FlatMap[F]): Task[F, A, C] = Task(Task.toKleisli(this).andThen(f)) @@ -62,4 +56,17 @@ object Task { def log[F[_], A](f: Logger[F] => F[Unit]): Task[F, A, Unit] = Task(ctx => f(ctx.logger)) + + implicit def taskMonad[F[_]: Monad, T]: Monad[Task[F, T, *]] = + new Monad[Task[F, T, *]] { + def pure[A](x: A) = Task(_ => Monad[F].pure(x)) + def flatMap[A, B](fa: Task[F, T, A])(f: A => Task[F, T, B]) = + Task(Task.toKleisli(fa).flatMap(a => Task.toKleisli(f(a)))) + + def tailRecM[A, B](a: A)(f: A => Task[F, T, Either[A, B]]) = { + val monadK = Monad[Kleisli[F, Context[F, T], *]] + val r = monadK.tailRecM(a)(x => Task.toKleisli(f(x))) + Task(r) + } + } } From 0739957fd7a1f4603ac138905bffb708e095e897 Mon Sep 17 00:00:00 2001 From: eikek Date: Sat, 12 Mar 2022 12:58:08 +0100 Subject: [PATCH 09/16] Fix package name --- modules/store/src/main/scala/db/migration/MigrationTasks.scala | 2 +- .../src/main/scala/db/migration/data/NotifyDueItemsArgs.scala | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/store/src/main/scala/db/migration/MigrationTasks.scala b/modules/store/src/main/scala/db/migration/MigrationTasks.scala index f8e2bf55..39f53f58 100644 --- a/modules/store/src/main/scala/db/migration/MigrationTasks.scala +++ b/modules/store/src/main/scala/db/migration/MigrationTasks.scala @@ -15,7 +15,7 @@ import docspell.common.syntax.StringSyntax._ import docspell.notification.api._ import docspell.store.records._ -import db.migration.data.{PeriodicDueItemsArgsOld, PeriodicQueryArgsOld} +import db.migration.data._ import doobie._ import doobie.implicits._ import doobie.util.transactor.Strategy diff --git a/modules/store/src/main/scala/db/migration/data/NotifyDueItemsArgs.scala b/modules/store/src/main/scala/db/migration/data/NotifyDueItemsArgs.scala index e664b58f..0e88df9f 100644 --- a/modules/store/src/main/scala/db/migration/data/NotifyDueItemsArgs.scala +++ b/modules/store/src/main/scala/db/migration/data/NotifyDueItemsArgs.scala @@ -4,8 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -package docspell.common +package db.migration.data +import docspell.common._ import docspell.common.syntax.all._ import io.circe._ From 0ce3abb3ff35a45c7b03afae582aded1d317bf2c Mon Sep 17 00:00:00 2001 From: eikek Date: Sat, 12 Mar 2022 14:10:49 +0100 Subject: [PATCH 10/16] Move scheduler code into separate module --- build.sbt | 41 +- .../scala/docspell/scheduler/Context.scala | 82 ++++ .../docspell/scheduler/CountingScheme.scala | 44 +++ .../scala/docspell/scheduler/JobTask.scala | 51 +++ .../docspell/scheduler/JobTaskRegistry.scala | 30 ++ .../docspell/scheduler/JobTaskResult.scala | 27 ++ .../scheduler/JobTaskResultEncoder.scala | 49 +++ .../scala/docspell/scheduler/LogEvent.scala | 38 ++ .../scala/docspell/scheduler/LogSink.scala | 76 ++++ .../scheduler/PeriodicScheduler.scala | 32 ++ .../scheduler/PeriodicSchedulerConfig.scala | 14 + .../docspell/scheduler/QueueLogger.scala | 58 +++ .../scala/docspell/scheduler/Scheduler.scala | 38 ++ .../docspell/scheduler/SchedulerConfig.scala | 33 ++ .../main/scala/docspell/scheduler/Task.scala | 72 ++++ .../docspell/scheduler/msg/JobDone.scala | 32 ++ .../impl/PeriodicSchedulerBuilder.scala | 31 ++ .../impl/PeriodicSchedulerImpl.scala | 182 +++++++++ .../scheduler/impl/SchedulerBuilder.scala | 97 +++++ .../scheduler/impl/SchedulerImpl.scala | 352 ++++++++++++++++++ 20 files changed, 1376 insertions(+), 3 deletions(-) create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/CountingScheme.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/JobTask.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/JobTaskRegistry.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/JobTaskResult.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/JobTaskResultEncoder.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/LogEvent.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/LogSink.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/PeriodicScheduler.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/PeriodicSchedulerConfig.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/QueueLogger.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/Scheduler.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/SchedulerConfig.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/Task.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobDone.scala create mode 100644 modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerBuilder.scala create mode 100644 modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerImpl.scala create mode 100644 modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerBuilder.scala create mode 100644 modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala diff --git a/build.sbt b/build.sbt index aa74d056..9eb54d39 100644 --- a/build.sbt +++ b/build.sbt @@ -519,6 +519,29 @@ val pubsubNaive = project ) .dependsOn(common, pubsubApi, store % "compile->compile;test->test") +val schedulerApi = project + .in(file("modules/scheduler/api")) + .disablePlugins(RevolverPlugin) + .settings(sharedSettings) + .withTestSettingsDependsOn(loggingScribe) + .settings( + name := "docspell-scheduler-api", + libraryDependencies ++= + Dependencies.fs2Core ++ + Dependencies.circeCore + ) + .dependsOn(loggingApi, common, store, pubsubApi) + +val schedulerImpl = project + .in(file("modules/scheduler/impl")) + .disablePlugins(RevolverPlugin) + .settings(sharedSettings) + .withTestSettingsDependsOn(loggingScribe) + .settings( + name := "docspell-scheduler-impl" + ) + .dependsOn(schedulerApi, notificationApi, pubsubApi) + val extract = project .in(file("modules/extract")) .disablePlugins(RevolverPlugin) @@ -641,7 +664,16 @@ val backend = project Dependencies.http4sClient ++ Dependencies.emil ) - .dependsOn(store, notificationApi, joexapi, ftsclient, totp, pubsubApi, loggingApi) + .dependsOn( + store, + notificationApi, + joexapi, + ftsclient, + totp, + pubsubApi, + loggingApi, + schedulerApi + ) val oidc = project .in(file("modules/oidc")) @@ -732,7 +764,8 @@ val joex = project restapi, ftssolr, pubsubNaive, - notificationImpl + notificationImpl, + schedulerImpl ) val restserver = project @@ -902,7 +935,9 @@ val root = project pubsubApi, pubsubNaive, notificationApi, - notificationImpl + notificationImpl, + schedulerApi, + schedulerImpl ) // --- Helpers diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala new file mode 100644 index 00000000..d5380f94 --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala @@ -0,0 +1,82 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler + +import cats.effect._ +import cats.implicits._ +import cats.{Applicative, Functor} + +import docspell.common._ +import docspell.logging.Logger +import docspell.store.Store +import docspell.store.records.RJob + +trait Context[F[_], A] { self => + + def jobId: Ident + + def args: A + + def config: SchedulerConfig + + def logger: Logger[F] + + def setProgress(percent: Int): F[Unit] + + def store: Store[F] + + final def isLastRetry(implicit ev: Applicative[F]): F[Boolean] = + for { + current <- store.transact(RJob.getRetries(jobId)) + last = config.retries == current.getOrElse(0) + } yield last + + def map[C](f: A => C)(implicit F: Functor[F]): Context[F, C] = + new Context.ContextImpl[F, C](f(args), logger, store, config, jobId) +} + +object Context { + + def create[F[_]: Async, A]( + jobId: Ident, + arg: A, + config: SchedulerConfig, + log: Logger[F], + store: Store[F] + ): Context[F, A] = + new ContextImpl(arg, log, store, config, jobId) + + def apply[F[_]: Async, A]( + job: RJob, + arg: A, + config: SchedulerConfig, + logSink: LogSink[F], + store: Store[F] + ): F[Context[F, A]] = { + val log = docspell.logging.getLogger[F] + for { + _ <- log.trace("Creating logger for task run") + logger <- QueueLogger(job.id, job.info, config.logBufferSize, logSink) + _ <- log.trace("Logger created, instantiating context") + ctx = create[F, A](job.id, arg, config, logger, store) + } yield ctx + } + + final private class ContextImpl[F[_]: Functor, A]( + val args: A, + val logger: Logger[F], + val store: Store[F], + val config: SchedulerConfig, + val jobId: Ident + ) extends Context[F, A] { + + def setProgress(percent: Int): F[Unit] = { + val pval = math.min(100, math.max(0, percent)) + store.transact(RJob.setProgress(jobId, pval)).map(_ => ()) + } + } +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/CountingScheme.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/CountingScheme.scala new file mode 100644 index 00000000..86aecd1a --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/CountingScheme.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler + +import cats.implicits._ + +import docspell.common.Priority + +/** A counting scheme to indicate a ratio between scheduling high and low priority jobs. + * + * For example high=4, low=1 means: ”schedule 4 high priority jobs and then 1 low + * priority job“. + */ +case class CountingScheme(high: Int, low: Int, counter: Int = 0) { + + def nextPriority: (CountingScheme, Priority) = + if (counter <= 0) (increment, Priority.High) + else { + val rest = counter % (high + low) + if (rest < high) (increment, Priority.High) + else (increment, Priority.Low) + } + + def increment: CountingScheme = + copy(counter = counter + 1) +} + +object CountingScheme { + + def writeString(cs: CountingScheme): String = + s"${cs.high},${cs.low}" + + def readString(str: String): Either[String, CountingScheme] = + str.split(',') match { + case Array(h, l) => + Either.catchNonFatal(CountingScheme(h.toInt, l.toInt)).left.map(_.getMessage) + case _ => + Left(s"Invalid counting scheme: $str") + } +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/JobTask.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobTask.scala new file mode 100644 index 00000000..e4c5074f --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobTask.scala @@ -0,0 +1,51 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler + +import cats.effect.Sync +import cats.implicits._ + +import docspell.common.Ident +import docspell.common.syntax.all._ + +import io.circe.Decoder + +/** Binds a Task to a name. This is required to lookup the code based on the taskName in + * the RJob data and to execute it given the arguments that have to be read from a + * string. + * + * Since the scheduler only has a string for the task argument, this only works for Task + * impls that accept a string. There is a convenience constructor that uses circe to + * decode json into some type A. + */ +case class JobTask[F[_]]( + name: Ident, + task: Task[F, String, JobTaskResult], + onCancel: Task[F, String, Unit] +) + +object JobTask { + + def json[F[_]: Sync, A, B]( + name: Ident, + task: Task[F, A, B], + onCancel: Task[F, A, Unit] + )(implicit + D: Decoder[A], + E: JobTaskResultEncoder[B] + ): JobTask[F] = { + val convert: String => F[A] = + str => + str.parseJsonAs[A] match { + case Right(a) => a.pure[F] + case Left(ex) => + Sync[F].raiseError(new Exception(s"Cannot parse task arguments: $str", ex)) + } + + JobTask(name, task.contramap(convert).map(E.encode), onCancel.contramap(convert)) + } +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/JobTaskRegistry.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobTaskRegistry.scala new file mode 100644 index 00000000..baa12150 --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobTaskRegistry.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler + +import docspell.common.Ident + +/** This is a mapping from some identifier to a task. This is used by the scheduler to + * lookup an implementation using the taskName field of the RJob database record. + */ +final class JobTaskRegistry[F[_]](tasks: Map[Ident, JobTask[F]]) { + + def withTask(task: JobTask[F]): JobTaskRegistry[F] = + JobTaskRegistry(tasks.updated(task.name, task)) + + def find(taskName: Ident): Option[JobTask[F]] = + tasks.get(taskName) +} + +object JobTaskRegistry { + + def apply[F[_]](map: Map[Ident, JobTask[F]]): JobTaskRegistry[F] = + new JobTaskRegistry[F](map) + + def empty[F[_]]: JobTaskRegistry[F] = apply(Map.empty) + +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/JobTaskResult.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobTaskResult.scala new file mode 100644 index 00000000..c121fc47 --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobTaskResult.scala @@ -0,0 +1,27 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler + +import io.circe.Json + +final case class JobTaskResult(message: Option[String], json: Option[Json]) { + + def withMessage(m: String): JobTaskResult = + copy(message = Some(m)) + + def withJson(json: Json): JobTaskResult = + copy(json = Some(json)) +} + +object JobTaskResult { + + val empty: JobTaskResult = JobTaskResult(None, None) + + def message(msg: String): JobTaskResult = JobTaskResult(Some(msg), None) + + def json(json: Json): JobTaskResult = JobTaskResult(None, Some(json)) +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/JobTaskResultEncoder.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobTaskResultEncoder.scala new file mode 100644 index 00000000..a0ad96a5 --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobTaskResultEncoder.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler + +import docspell.scheduler.JobTaskResultEncoder.instance + +import io.circe.Encoder + +trait JobTaskResultEncoder[A] { self => + def encode(a: A): JobTaskResult + + final def contramap[B](f: B => A): JobTaskResultEncoder[B] = + JobTaskResultEncoder.instance(b => self.encode(f(b))) + + final def map(f: JobTaskResult => JobTaskResult): JobTaskResultEncoder[A] = + instance(a => f(self.encode(a))) + + final def modify(f: (A, JobTaskResult) => JobTaskResult): JobTaskResultEncoder[A] = + instance(a => f(a, self.encode(a))) + + final def withMessage(f: A => String): JobTaskResultEncoder[A] = + modify((a, r) => r.withMessage(f(a))) +} + +object JobTaskResultEncoder { + + def apply[A](implicit v: JobTaskResultEncoder[A]): JobTaskResultEncoder[A] = v + + def instance[A](f: A => JobTaskResult): JobTaskResultEncoder[A] = + (a: A) => f(a) + + def fromJson[A: Encoder]: JobTaskResultEncoder[A] = + instance(a => JobTaskResult.json(Encoder[A].apply(a))) + + implicit val unitJobTaskResultEncoder: JobTaskResultEncoder[Unit] = + instance(_ => JobTaskResult.empty) + + implicit def optionJobTaskResultEncoder[A](implicit + ea: JobTaskResultEncoder[A] + ): JobTaskResultEncoder[Option[A]] = + instance { + case Some(a) => ea.encode(a) + case None => JobTaskResult.empty + } +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/LogEvent.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/LogEvent.scala new file mode 100644 index 00000000..29a91631 --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/LogEvent.scala @@ -0,0 +1,38 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler + +import cats.effect.Sync +import cats.implicits._ + +import docspell.common._ + +case class LogEvent( + jobId: Ident, + jobInfo: String, + time: Timestamp, + level: LogLevel, + msg: String, + ex: Option[Throwable] = None +) { + + def logLine: String = + s">>> ${time.asString} $level $jobInfo: $msg" + +} + +object LogEvent { + + def create[F[_]: Sync]( + jobId: Ident, + jobInfo: String, + level: LogLevel, + msg: String + ): F[LogEvent] = + Timestamp.current[F].map(now => LogEvent(jobId, jobInfo, now, level, msg)) + +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/LogSink.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/LogSink.scala new file mode 100644 index 00000000..6bf34f05 --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/LogSink.scala @@ -0,0 +1,76 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler + +import cats.effect._ +import cats.implicits._ +import fs2.Pipe + +import docspell.common._ +import docspell.logging +import docspell.store.Store +import docspell.store.records.RJobLog + +trait LogSink[F[_]] { + + def receive: Pipe[F, LogEvent, Unit] + +} + +object LogSink { + + def apply[F[_]](sink: Pipe[F, LogEvent, Unit]): LogSink[F] = + new LogSink[F] { + val receive = sink + } + + def logInternal[F[_]: Sync](e: LogEvent): F[Unit] = { + val logger = docspell.logging.getLogger[F] + val addData: logging.LogEvent => logging.LogEvent = + _.data("jobId", e.jobId).data("jobInfo", e.jobInfo) + + e.level match { + case LogLevel.Info => + logger.infoWith(e.logLine)(addData) + case LogLevel.Debug => + logger.debugWith(e.logLine)(addData) + case LogLevel.Warn => + logger.warnWith(e.logLine)(addData) + case LogLevel.Error => + e.ex match { + case Some(exc) => + logger.errorWith(e.logLine)(addData.andThen(_.addError(exc))) + case None => + logger.errorWith(e.logLine)(addData) + } + } + } + + def printer[F[_]: Sync]: LogSink[F] = + LogSink(_.evalMap(e => logInternal(e))) + + def db[F[_]: Async](store: Store[F]): LogSink[F] = + LogSink( + _.evalMap(ev => + for { + id <- Ident.randomId[F] + joblog = RJobLog( + id, + ev.jobId, + ev.level, + ev.time, + ev.msg + ev.ex.map(th => ": " + th.getMessage).getOrElse("") + ) + _ <- logInternal(ev) + _ <- store.transact(RJobLog.insert(joblog)) + } yield () + ) + ) + + def dbAndLog[F[_]: Async](store: Store[F]): LogSink[F] = + LogSink(_.broadcastThrough(printer[F].receive, db[F](store).receive)) +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/PeriodicScheduler.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/PeriodicScheduler.scala new file mode 100644 index 00000000..284a9465 --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/PeriodicScheduler.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler + +import cats.effect._ +import fs2._ + +/** A periodic scheduler takes care to submit periodic tasks to the job queue. + * + * It is run in the background to regularly find a periodic task to execute. If the task + * is due, it will be submitted into the job queue where it will be picked up by the + * scheduler from some joex instance. If it is due in the future, a notification is + * scheduled to be received at that time so the task can be looked up again. + */ +trait PeriodicScheduler[F[_]] { + + def config: PeriodicSchedulerConfig + + def start: Stream[F, Nothing] + + def shutdown: F[Unit] + + def periodicAwake: F[Fiber[F, Throwable, Unit]] + + def notifyChange: F[Unit] +} + +object PeriodicScheduler {} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/PeriodicSchedulerConfig.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/PeriodicSchedulerConfig.scala new file mode 100644 index 00000000..7445e9e3 --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/PeriodicSchedulerConfig.scala @@ -0,0 +1,14 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler + +import docspell.common._ + +case class PeriodicSchedulerConfig( + name: Ident, + wakeupPeriod: Duration +) diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/QueueLogger.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/QueueLogger.scala new file mode 100644 index 00000000..b0f17d23 --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/QueueLogger.scala @@ -0,0 +1,58 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler + +import cats.effect._ +import cats.effect.std.Queue +import cats.implicits._ +import fs2.Stream + +import docspell.common._ +import docspell.logging +import docspell.logging.{Level, Logger} + +object QueueLogger { + + def create[F[_]: Sync]( + jobId: Ident, + jobInfo: String, + q: Queue[F, LogEvent] + ): Logger[F] = + new Logger[F] { + + def log(logEvent: logging.LogEvent) = + LogEvent + .create[F](jobId, jobInfo, level2Level(logEvent.level), logEvent.msg()) + .flatMap { ev => + val event = + logEvent.findErrors.headOption + .map(ex => ev.copy(ex = Some(ex))) + .getOrElse(ev) + + q.offer(event) + } + + def asUnsafe = Logger.off + } + + def apply[F[_]: Async]( + jobId: Ident, + jobInfo: String, + bufferSize: Int, + sink: LogSink[F] + ): F[Logger[F]] = + for { + q <- Queue.circularBuffer[F, LogEvent](bufferSize) + log = create(jobId, jobInfo, q) + _ <- Async[F].start( + Stream.fromQueueUnterminated(q).through(sink.receive).compile.drain + ) + } yield log + + private def level2Level(level: Level): LogLevel = + LogLevel.fromLevel(level) +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/Scheduler.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/Scheduler.scala new file mode 100644 index 00000000..720214c7 --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/Scheduler.scala @@ -0,0 +1,38 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler + +import cats.effect._ +import fs2.Stream + +import docspell.common.Ident +import docspell.store.records.RJob + +trait Scheduler[F[_]] { + + def config: SchedulerConfig + + def getRunning: F[Vector[RJob]] + + def requestCancel(jobId: Ident): F[Boolean] + + def notifyChange: F[Unit] + + def start: Stream[F, Nothing] + + /** Requests to shutdown the scheduler. + * + * The scheduler will not take any new jobs from the queue. If there are still running + * jobs, it waits for them to complete. when the cancelAll flag is set to true, it + * cancels all running jobs. + * + * The returned F[Unit] can be evaluated to wait for all that to complete. + */ + def shutdown(cancelAll: Boolean): F[Unit] + + def periodicAwake: F[Fiber[F, Throwable, Unit]] +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/SchedulerConfig.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/SchedulerConfig.scala new file mode 100644 index 00000000..fbbc4f3c --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/SchedulerConfig.scala @@ -0,0 +1,33 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler + +import docspell.common._ + +case class SchedulerConfig( + name: Ident, + poolSize: Int, + countingScheme: CountingScheme, + retries: Int, + retryDelay: Duration, + logBufferSize: Int, + wakeupPeriod: Duration +) + +object SchedulerConfig { + + val default = SchedulerConfig( + name = Ident.unsafe("default-scheduler"), + poolSize = 2 // math.max(2, Runtime.getRuntime.availableProcessors / 2) + , + countingScheme = CountingScheme(2, 1), + retries = 5, + retryDelay = Duration.seconds(30), + logBufferSize = 500, + wakeupPeriod = Duration.minutes(10) + ) +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/Task.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/Task.scala new file mode 100644 index 00000000..d6868a08 --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/Task.scala @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler + +import cats._ +import cats.data.Kleisli +import cats.effect.Sync +import cats.implicits._ + +import docspell.logging.Logger + +/** The code that is executed by the scheduler */ +trait Task[F[_], A, B] { + + def run(ctx: Context[F, A]): F[B] + + def andThen[C](f: B => F[C])(implicit F: FlatMap[F]): Task[F, A, C] = + Task(Task.toKleisli(this).andThen(f)) + + def mapF[C](f: F[B] => F[C]): Task[F, A, C] = + Task(Task.toKleisli(this).mapF(f)) + + def attempt(implicit + F: ApplicativeError[F, Throwable] + ): Task[F, A, Either[Throwable, B]] = + mapF(_.attempt) + + def contramap[C](f: C => F[A])(implicit F: FlatMap[F]): Task[F, C, B] = { + ctxc: Context[F, C] => f(ctxc.args).flatMap(a => run(ctxc.map(_ => a))) + } +} + +object Task { + + def pure[F[_]: Applicative, A, B](b: B): Task[F, A, B] = + Task(_ => b.pure[F]) + + def of[F[_], A, B](b: F[B]): Task[F, A, B] = + Task(_ => b) + + def apply[F[_], A, B](f: Context[F, A] => F[B]): Task[F, A, B] = + (ctx: Context[F, A]) => f(ctx) + + def apply[F[_], A, B](k: Kleisli[F, Context[F, A], B]): Task[F, A, B] = + c => k.run(c) + + def toKleisli[F[_], A, B](t: Task[F, A, B]): Kleisli[F, Context[F, A], B] = + Kleisli(t.run) + + def setProgress[F[_]: Sync, A, B](n: Int)(data: B): Task[F, A, B] = + Task(_.setProgress(n).map(_ => data)) + + def log[F[_], A](f: Logger[F] => F[Unit]): Task[F, A, Unit] = + Task(ctx => f(ctx.logger)) + + implicit def taskMonad[F[_]: Monad, T]: Monad[Task[F, T, *]] = + new Monad[Task[F, T, *]] { + def pure[A](x: A) = Task(_ => Monad[F].pure(x)) + def flatMap[A, B](fa: Task[F, T, A])(f: A => Task[F, T, B]) = + Task(Task.toKleisli(fa).flatMap(a => Task.toKleisli(f(a)))) + + def tailRecM[A, B](a: A)(f: A => Task[F, T, Either[A, B]]) = { + val monadK = Monad[Kleisli[F, Context[F, T], *]] + val r = monadK.tailRecM(a)(x => Task.toKleisli(f(x))) + Task(r) + } + } +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobDone.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobDone.scala new file mode 100644 index 00000000..61eff3f9 --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobDone.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler.msg + +import docspell.common._ +import docspell.pubsub.api.{Topic, TypedTopic} + +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import io.circe.{Decoder, Encoder} + +/** Message to notify about finished jobs. They have a final state. */ +final case class JobDone( + jobId: Ident, + group: Ident, + task: Ident, + args: String, + state: JobState +) +object JobDone { + implicit val jsonDecoder: Decoder[JobDone] = + deriveDecoder[JobDone] + + implicit val jsonEncoder: Encoder[JobDone] = + deriveEncoder[JobDone] + + val topic: TypedTopic[JobDone] = + TypedTopic(Topic("job-finished")) +} diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerBuilder.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerBuilder.scala new file mode 100644 index 00000000..5fbaada2 --- /dev/null +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerBuilder.scala @@ -0,0 +1,31 @@ +package docspell.scheduler.impl + +import cats.effect._ +import docspell.scheduler._ +import docspell.store.queue.{JobQueue, PeriodicTaskStore} +import fs2.concurrent.SignallingRef + +object PeriodicSchedulerBuilder { + + def build[F[_]: Async]( + cfg: PeriodicSchedulerConfig, + sch: Scheduler[F], + queue: JobQueue[F], + store: PeriodicTaskStore[F], + notifyJoex: F[Unit] + ): Resource[F, PeriodicScheduler[F]] = + for { + waiter <- Resource.eval(SignallingRef(true)) + state <- Resource.eval(SignallingRef(PeriodicSchedulerImpl.emptyState[F])) + psch = new PeriodicSchedulerImpl[F]( + cfg, + sch, + queue, + store, + notifyJoex, + waiter, + state + ) + _ <- Resource.eval(psch.init) + } yield psch +} diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerImpl.scala new file mode 100644 index 00000000..efdefeaf --- /dev/null +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerImpl.scala @@ -0,0 +1,182 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler.impl + +import cats.effect._ +import cats.implicits._ +import fs2._ +import fs2.concurrent.SignallingRef + +import docspell.common._ +import docspell.scheduler._ +import docspell.scheduler.impl.PeriodicSchedulerImpl.State +import docspell.store.queue._ +import docspell.store.records.RPeriodicTask + +import eu.timepit.fs2cron.calev.CalevScheduler + +final class PeriodicSchedulerImpl[F[_]: Async]( + val config: PeriodicSchedulerConfig, + sch: Scheduler[F], + queue: JobQueue[F], + store: PeriodicTaskStore[F], + joexNotifyAll: F[Unit], + waiter: SignallingRef[F, Boolean], + state: SignallingRef[F, State[F]] +) extends PeriodicScheduler[F] { + private[this] val logger = docspell.logging.getLogger[F] + + def start: Stream[F, Nothing] = + logger.stream.info("Starting periodic scheduler").drain ++ + mainLoop + + def shutdown: F[Unit] = + state.modify(_.requestShutdown) + + def periodicAwake: F[Fiber[F, Throwable, Unit]] = + Async[F].start( + Stream + .awakeEvery[F](config.wakeupPeriod.toScala) + .evalMap(_ => logger.debug("Periodic awake reached") *> notifyChange) + .compile + .drain + ) + + def notifyChange: F[Unit] = + waiter.update(b => !b) + + // internal + + /** On startup, get all periodic jobs from this scheduler and remove the mark, so they + * get picked up again. + */ + def init: F[Unit] = + logError("Error clearing marks")(store.clearMarks(config.name)) + + def mainLoop: Stream[F, Nothing] = { + val body: F[Boolean] = + for { + _ <- logger.debug(s"Going into main loop") + now <- Timestamp.current[F] + _ <- logger.debug(s"Looking for next periodic task") + go <- logThrow("Error getting next task")( + store + .takeNext(config.name, None) + .use { + case Marked.Found(pj) => + logger + .debug(s"Found periodic task '${pj.subject}/${pj.timer.asString}'") *> + (if (isTriggered(pj, now)) submitJob(pj) + else scheduleNotify(pj).map(_ => false)) + case Marked.NotFound => + logger.debug("No periodic task found") *> false.pure[F] + case Marked.NotMarkable => + logger.debug("Periodic job cannot be marked. Trying again.") *> true + .pure[F] + } + ) + } yield go + + Stream + .eval(state.get.map(_.shutdownRequest)) + .evalTap( + if (_) logger.info("Stopping main loop due to shutdown request.") + else ().pure[F] + ) + .flatMap(if (_) Stream.empty else Stream.eval(cancelNotify *> body)) + .flatMap { + case true => + mainLoop + case false => + logger.stream.debug(s"Waiting for notify").drain ++ + waiter.discrete.take(2).drain ++ + logger.stream.debug(s"Notify signal, going into main loop").drain ++ + mainLoop + } + } + + def isTriggered(pj: RPeriodicTask, now: Timestamp): Boolean = + pj.nextrun < now + + def submitJob(pj: RPeriodicTask): F[Boolean] = + store + .findNonFinalJob(pj.id) + .flatMap { + case Some(job) => + logger.info( + s"There is already a job with non-final state '${job.state}' in the queue" + ) *> scheduleNotify(pj) *> false.pure[F] + + case None => + logger.info(s"Submitting job for periodic task '${pj.task.id}'") *> + pj.toJob.flatMap(queue.insert) *> notifyJoex *> true.pure[F] + } + + def notifyJoex: F[Unit] = + sch.notifyChange *> joexNotifyAll + + def scheduleNotify(pj: RPeriodicTask): F[Unit] = + Timestamp + .current[F] + .flatMap(now => + logger.debug( + s"Scheduling next notify for timer ${pj.timer.asString} -> ${pj.timer.nextElapse(now.toUtcDateTime)}" + ) + ) *> + Async[F] + .start( + CalevScheduler + .utc[F] + .sleep(pj.timer) + .evalMap(_ => notifyChange) + .compile + .drain + ) + .flatMap(fb => state.modify(_.setNotify(fb))) + + def cancelNotify: F[Unit] = + state + .modify(_.clearNotify) + .flatMap { + case Some(fb) => + fb.cancel + case None => + ().pure[F] + } + + private def logError(msg: => String)(fa: F[Unit]): F[Unit] = + fa.attempt.flatMap { + case Right(_) => ().pure[F] + case Left(ex) => logger.error(ex)(msg).map(_ => ()) + } + + private def logThrow[A](msg: => String)(fa: F[A]): F[A] = + fa.attempt.flatMap { + case r @ Right(_) => (r: Either[Throwable, A]).pure[F] + case l @ Left(ex) => logger.error(ex)(msg).map(_ => (l: Either[Throwable, A])) + }.rethrow +} + +object PeriodicSchedulerImpl { + def emptyState[F[_]]: State[F] = + State(false, None) + + case class State[F[_]]( + shutdownRequest: Boolean, + scheduledNotify: Option[Fiber[F, Throwable, Unit]] + ) { + def requestShutdown: (State[F], Unit) = + (copy(shutdownRequest = true), ()) + + def setNotify(fb: Fiber[F, Throwable, Unit]): (State[F], Unit) = + (copy(scheduledNotify = Some(fb)), ()) + + def clearNotify: (State[F], Option[Fiber[F, Throwable, Unit]]) = + (copy(scheduledNotify = None), scheduledNotify) + + } +} diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerBuilder.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerBuilder.scala new file mode 100644 index 00000000..4e51231b --- /dev/null +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerBuilder.scala @@ -0,0 +1,97 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler.impl + +import cats.effect._ +import cats.effect.std.Semaphore +import cats.implicits._ +import fs2.concurrent.SignallingRef + +import docspell.scheduler._ +import docspell.notification.api.EventSink +import docspell.pubsub.api.PubSubT +import docspell.store.Store +import docspell.store.queue.JobQueue + +case class SchedulerBuilder[F[_]: Async]( + config: SchedulerConfig, + tasks: JobTaskRegistry[F], + store: Store[F], + queue: Resource[F, JobQueue[F]], + logSink: LogSink[F], + pubSub: PubSubT[F], + eventSink: EventSink[F] +) { + + def withConfig(cfg: SchedulerConfig): SchedulerBuilder[F] = + copy(config = cfg) + + def withTaskRegistry(reg: JobTaskRegistry[F]): SchedulerBuilder[F] = + copy(tasks = reg) + + def withTask[A](task: JobTask[F]): SchedulerBuilder[F] = + withTaskRegistry(tasks.withTask(task)) + + def withQueue(queue: Resource[F, JobQueue[F]]): SchedulerBuilder[F] = + copy(queue = queue) + + def withLogSink(sink: LogSink[F]): SchedulerBuilder[F] = + copy(logSink = sink) + + def withQueue(queue: JobQueue[F]): SchedulerBuilder[F] = + copy(queue = Resource.pure[F, JobQueue[F]](queue)) + + def withPubSub(pubSubT: PubSubT[F]): SchedulerBuilder[F] = + copy(pubSub = pubSubT) + + def withEventSink(sink: EventSink[F]): SchedulerBuilder[F] = + copy(eventSink = sink) + + def serve: Resource[F, Scheduler[F]] = + resource.evalMap(sch => Async[F].start(sch.start.compile.drain).map(_ => sch)) + + def resource: Resource[F, Scheduler[F]] = { + val scheduler: Resource[F, SchedulerImpl[F]] = for { + jq <- queue + waiter <- Resource.eval(SignallingRef(true)) + state <- Resource.eval(SignallingRef(SchedulerImpl.emptyState[F])) + perms <- Resource.eval(Semaphore(config.poolSize.toLong)) + } yield new SchedulerImpl[F]( + config, + jq, + pubSub, + eventSink, + tasks, + store, + logSink, + state, + waiter, + perms + ) + + scheduler.evalTap(_.init).map(s => s: Scheduler[F]) + } + +} + +object SchedulerBuilder { + + def apply[F[_]: Async]( + config: SchedulerConfig, + store: Store[F] + ): SchedulerBuilder[F] = + new SchedulerBuilder[F]( + config, + JobTaskRegistry.empty[F], + store, + JobQueue(store), + LogSink.db[F](store), + PubSubT.noop[F], + EventSink.silent[F] + ) + +} diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala new file mode 100644 index 00000000..d0de5991 --- /dev/null +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala @@ -0,0 +1,352 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler.impl + +import cats.data.OptionT +import cats.effect._ +import cats.effect.std.Semaphore +import cats.implicits._ +import fs2.Stream +import fs2.concurrent.SignallingRef + +import docspell.scheduler.msg.JobDone +import docspell.common._ +import docspell.scheduler._ +import docspell.scheduler.impl.SchedulerImpl._ +import docspell.notification.api.Event +import docspell.notification.api.EventSink +import docspell.pubsub.api.PubSubT +import docspell.store.Store +import docspell.store.queries.QJob +import docspell.store.queue.JobQueue +import docspell.store.records.RJob + +import io.circe.Json + +final class SchedulerImpl[F[_]: Async]( + val config: SchedulerConfig, + queue: JobQueue[F], + pubSub: PubSubT[F], + eventSink: EventSink[F], + tasks: JobTaskRegistry[F], + store: Store[F], + logSink: LogSink[F], + state: SignallingRef[F, State[F]], + waiter: SignallingRef[F, Boolean], + permits: Semaphore[F] +) extends Scheduler[F] { + + private[this] val logger = docspell.logging.getLogger[F] + + /** On startup, get all jobs in state running from this scheduler and put them into + * waiting state, so they get picked up again. + */ + def init: F[Unit] = + QJob.runningToWaiting(config.name, store) + + def periodicAwake: F[Fiber[F, Throwable, Unit]] = + Async[F].start( + Stream + .awakeEvery[F](config.wakeupPeriod.toScala) + .evalMap(_ => logger.debug("Periodic awake reached") *> notifyChange) + .compile + .drain + ) + + def getRunning: F[Vector[RJob]] = + state.get.flatMap(s => QJob.findAll(s.getRunning, store)) + + def requestCancel(jobId: Ident): F[Boolean] = + logger.info(s"Scheduler requested to cancel job: ${jobId.id}") *> + state.get.flatMap(_.cancelRequest(jobId) match { + case Some(ct) => ct.map(_ => true) + case None => + (for { + job <- OptionT(store.transact(RJob.findByIdAndWorker(jobId, config.name))) + _ <- OptionT.liftF( + if (job.isInProgress) executeCancel(job) + else ().pure[F] + ) + } yield true) + .getOrElseF( + logger.warn(s"Job ${jobId.id} not found, cannot cancel.").map(_ => false) + ) + }) + + def notifyChange: F[Unit] = + waiter.update(b => !b) + + def shutdown(cancelAll: Boolean): F[Unit] = { + val doCancel = + state.get.flatMap(_.cancelTokens.values.toList.traverse(identity)).map(_ => ()) + + val runShutdown = + state.modify(_.requestShutdown) *> (if (cancelAll) doCancel else ().pure[F]) + + val wait = Stream + .eval(runShutdown) + .evalMap(_ => logger.info("Scheduler is shutting down now.")) + .flatMap(_ => + Stream.eval(state.get) ++ Stream + .suspend(state.discrete.takeWhile(_.getRunning.nonEmpty)) + ) + .flatMap { state => + if (state.getRunning.isEmpty) Stream.eval(logger.info("No jobs running.")) + else + Stream.eval( + logger.info(s"Waiting for ${state.getRunning.size} jobs to finish.") + ) ++ + Stream.emit(state) + } + + (wait.drain ++ Stream.emit(())).compile.lastOrError + } + + def start: Stream[F, Nothing] = + logger.stream.info("Starting scheduler").drain ++ + mainLoop + + def mainLoop: Stream[F, Nothing] = { + val body: F[Boolean] = + for { + _ <- permits.available.flatMap(a => + logger.debug(s"Try to acquire permit ($a free)") + ) + _ <- permits.acquire + _ <- logger.debug("New permit acquired") + down <- state.get.map(_.shutdownRequest) + rjob <- + if (down) + logger.info("") *> permits.release *> (None: Option[RJob]).pure[F] + else + queue.nextJob( + group => state.modify(_.nextPrio(group, config.countingScheme)), + config.name, + config.retryDelay + ) + _ <- logger.debug(s"Next job found: ${rjob.map(_.info)}") + _ <- rjob.map(execute).getOrElse(permits.release) + } yield rjob.isDefined + + Stream + .eval(state.get.map(_.shutdownRequest)) + .evalTap( + if (_) logger.info("Stopping main loop due to shutdown request.") + else ().pure[F] + ) + .flatMap(if (_) Stream.empty else Stream.eval(body)) + .flatMap { + case true => + mainLoop + case false => + logger.stream.debug(s"Waiting for notify").drain ++ + waiter.discrete.take(2).drain ++ + logger.stream.debug(s"Notify signal, going into main loop").drain ++ + mainLoop + } + } + + private def executeCancel(job: RJob): F[Unit] = { + val task = for { + jobtask <- + tasks + .find(job.task) + .toRight(s"This executor cannot run tasks with name: ${job.task}") + } yield jobtask + + task match { + case Left(err) => + logger.error(s"Unable to run cancellation task for job ${job.info}: $err") + case Right(t) => + for { + _ <- + logger.debug(s"Creating context for job ${job.info} to run cancellation $t") + ctx <- Context[F, String](job, job.args, config, logSink, store) + _ <- t.onCancel.run(ctx) + _ <- state.modify(_.markCancelled(job)) + _ <- onFinish(job, JobTaskResult.empty, JobState.Cancelled) + _ <- ctx.logger.warn("Job has been cancelled.") + _ <- logger.debug(s"Job ${job.info} has been cancelled.") + } yield () + } + } + + def execute(job: RJob): F[Unit] = { + val task = for { + jobtask <- + tasks + .find(job.task) + .toRight(s"This executor cannot run tasks with name: ${job.task}") + } yield jobtask + + task match { + case Left(err) => + logger.error(s"Unable to start a task for job ${job.info}: $err") + case Right(t) => + for { + _ <- logger.debug(s"Creating context for job ${job.info} to run $t") + ctx <- Context[F, String](job, job.args, config, logSink, store) + jot = wrapTask(job, t.task, ctx) + tok <- forkRun(job, jot.run(ctx), t.onCancel.run(ctx), ctx) + _ <- state.modify(_.addRunning(job, tok)) + } yield () + } + } + + def onFinish(job: RJob, result: JobTaskResult, finishState: JobState): F[Unit] = + for { + _ <- logger.debug(s"Job ${job.info} done $finishState. Releasing resources.") + _ <- permits.release *> permits.available.flatMap(a => + logger.debug(s"Permit released ($a free)") + ) + _ <- state.modify(_.removeRunning(job)) + _ <- QJob.setFinalState(job.id, finishState, store) + _ <- Sync[F].whenA(JobState.isDone(finishState))( + pubSub.publish1IgnoreErrors( + JobDone.topic, + JobDone(job.id, job.group, job.task, job.args, finishState) + ) + ) + _ <- Sync[F].whenA(JobState.isDone(finishState))( + eventSink.offer( + Event.JobDone( + job.id, + job.group, + job.task, + job.args, + job.state, + job.subject, + job.submitter, + result.json.getOrElse(Json.Null), + result.message + ) + ) + ) + } yield () + + def onStart(job: RJob): F[Unit] = + QJob.setRunning( + job.id, + config.name, + store + ) // also increments retries if current state=stuck + + def wrapTask( + job: RJob, + task: Task[F, String, JobTaskResult], + ctx: Context[F, String] + ): Task[F, String, Unit] = + task + .mapF(fa => onStart(job) *> logger.debug("Starting task now") *> fa) + .mapF(_.attempt.flatMap { + case Right(result) => + logger.info(s"Job execution successful: ${job.info}") + ctx.logger.info("Job execution successful") *> + (JobState.Success: JobState, result).pure[F] + case Left(ex) => + state.get.map(_.wasCancelled(job)).flatMap { + case true => + logger.error(ex)(s"Job ${job.info} execution failed (cancel = true)") + ctx.logger.error(ex)("Job execution failed (cancel = true)") *> + (JobState.Cancelled: JobState, JobTaskResult.empty).pure[F] + case false => + QJob.exceedsRetries(job.id, config.retries, store).flatMap { + case true => + logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") + ctx.logger + .error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") + .map(_ => (JobState.Failed: JobState, JobTaskResult.empty)) + case false => + logger.error(ex)(s"Job ${job.info} execution failed. Retrying later.") + ctx.logger + .error(ex)(s"Job ${job.info} execution failed. Retrying later.") + .map(_ => (JobState.Stuck: JobState, JobTaskResult.empty)) + } + } + }) + .mapF(_.attempt.flatMap { + case Right((jstate, result)) => + onFinish(job, result, jstate) + case Left(ex) => + logger.error(ex)(s"Error happened during post-processing of ${job.info}!") + // we don't know the real outcome here… + // since tasks should be idempotent, set it to stuck. if above has failed, this might fail anyways + onFinish(job, JobTaskResult.empty, JobState.Stuck) + }) + + def forkRun( + job: RJob, + code: F[Unit], + onCancel: F[Unit], + ctx: Context[F, String] + ): F[F[Unit]] = + logger.debug(s"Forking job ${job.info}") *> + Async[F] + .start(code) + .map(fiber => + logger.debug(s"Cancelling job ${job.info}") *> + fiber.cancel *> + onCancel.attempt.map { + case Right(_) => () + case Left(ex) => + logger.error(ex)(s"Task's cancelling code failed. Job ${job.info}.") + () + } *> + state.modify(_.markCancelled(job)) *> + onFinish(job, JobTaskResult.empty, JobState.Cancelled) *> + ctx.logger.warn("Job has been cancelled.") *> + logger.debug(s"Job ${job.info} has been cancelled.") + ) +} + +object SchedulerImpl { + + type CancelToken[F[_]] = F[Unit] + + def emptyState[F[_]]: State[F] = + State(Map.empty, Set.empty, Map.empty, false) + + case class State[F[_]]( + counters: Map[Ident, CountingScheme], + cancelled: Set[Ident], + cancelTokens: Map[Ident, CancelToken[F]], + shutdownRequest: Boolean + ) { + + def nextPrio(group: Ident, initial: CountingScheme): (State[F], Priority) = { + val (cs, prio) = counters.getOrElse(group, initial).nextPriority + (copy(counters = counters.updated(group, cs)), prio) + } + + def addRunning(job: RJob, token: CancelToken[F]): (State[F], Unit) = + ( + State(counters, cancelled, cancelTokens.updated(job.id, token), shutdownRequest), + () + ) + + def removeRunning(job: RJob): (State[F], Unit) = + ( + copy(cancelled = cancelled - job.id, cancelTokens = cancelTokens.removed(job.id)), + () + ) + + def markCancelled(job: RJob): (State[F], Unit) = + (copy(cancelled = cancelled + job.id), ()) + + def wasCancelled(job: RJob): Boolean = + cancelled.contains(job.id) + + def cancelRequest(id: Ident): Option[F[Unit]] = + cancelTokens.get(id) + + def getRunning: Seq[Ident] = + cancelTokens.keys.toSeq + + def requestShutdown: (State[F], Unit) = + (copy(shutdownRequest = true), ()) + } +} From aafd9089068d0f7666ffdf38de9844d0171e83d1 Mon Sep 17 00:00:00 2001 From: eikek Date: Sat, 12 Mar 2022 14:15:36 +0100 Subject: [PATCH 11/16] Adopt code to use moved schedule api --- .../scala/docspell/backend/msg/JobDone.scala | 32 -- .../scala/docspell/backend/msg/Topics.scala | 2 +- .../src/main/scala/docspell/joex/Config.scala | 2 +- .../main/scala/docspell/joex/ConfigFile.scala | 2 +- .../main/scala/docspell/joex/JoexApp.scala | 2 +- .../scala/docspell/joex/JoexAppImpl.scala | 9 +- .../joex/emptytrash/EmptyTrashTask.scala | 2 +- .../docspell/joex/filecopy/FileCopyTask.scala | 2 +- .../filecopy/FileIntegrityCheckTask.scala | 2 +- .../scala/docspell/joex/fts/FtsContext.scala | 2 +- .../scala/docspell/joex/fts/FtsWork.scala | 2 +- .../docspell/joex/fts/MigrationTask.scala | 2 +- .../scala/docspell/joex/fts/ReIndexTask.scala | 2 +- .../docspell/joex/hk/CheckNodesTask.scala | 2 +- .../docspell/joex/hk/CleanupInvitesTask.scala | 2 +- .../docspell/joex/hk/CleanupJobsTask.scala | 2 +- .../joex/hk/CleanupRememberMeTask.scala | 2 +- .../docspell/joex/hk/HouseKeepingTask.scala | 2 +- .../docspell/joex/hk/IntegrityCheckTask.scala | 2 +- .../joex/learn/LearnClassifierTask.scala | 2 +- .../joex/learn/LearnItemEntities.scala | 2 +- .../scala/docspell/joex/learn/LearnTags.scala | 2 +- .../docspell/joex/learn/SelectItems.scala | 2 +- .../joex/learn/StoreClassifierModel.scala | 2 +- .../joex/notify/PeriodicDueItemsTask.scala | 4 +- .../joex/notify/PeriodicQueryTask.scala | 4 +- .../joex/pagecount/AllPageCountTask.scala | 4 +- .../joex/pagecount/PageCountTask.scala | 4 +- .../joex/pdfconv/ConvertAllPdfTask.scala | 2 +- .../docspell/joex/pdfconv/PdfConvTask.scala | 2 +- .../joex/preview/AllPreviewsTask.scala | 4 +- .../joex/preview/MakePreviewTask.scala | 4 +- .../joex/process/AttachmentPageCount.scala | 2 +- .../joex/process/AttachmentPreview.scala | 2 +- .../docspell/joex/process/ConvertPdf.scala | 2 +- .../docspell/joex/process/CreateItem.scala | 2 +- .../joex/process/CrossCheckProposals.scala | 2 +- .../joex/process/DuplicateCheck.scala | 2 +- .../docspell/joex/process/EvalProposals.scala | 2 +- .../joex/process/ExtractArchive.scala | 2 +- .../docspell/joex/process/FindProposal.scala | 2 +- .../docspell/joex/process/ItemData.scala | 2 +- .../docspell/joex/process/ItemHandler.scala | 2 +- .../docspell/joex/process/LinkProposal.scala | 2 +- .../docspell/joex/process/ProcessItem.scala | 2 +- .../docspell/joex/process/ReProcessItem.scala | 4 +- .../joex/process/RemoveEmptyItem.scala | 2 +- .../docspell/joex/process/SaveProposals.scala | 2 +- .../docspell/joex/process/SetGivenData.scala | 2 +- .../docspell/joex/process/TextAnalysis.scala | 4 +- .../joex/process/TextExtraction.scala | 2 +- .../joex/scanmailbox/ScanMailboxTask.scala | 2 +- .../docspell/joex/scheduler/Context.scala | 82 ---- .../joex/scheduler/CountingScheme.scala | 44 --- .../docspell/joex/scheduler/JobTask.scala | 51 --- .../joex/scheduler/JobTaskRegistry.scala | 30 -- .../joex/scheduler/JobTaskResult.scala | 27 -- .../joex/scheduler/JobTaskResultEncoder.scala | 49 --- .../docspell/joex/scheduler/LogEvent.scala | 38 -- .../docspell/joex/scheduler/LogSink.scala | 76 ---- .../joex/scheduler/PeriodicScheduler.scala | 60 --- .../scheduler/PeriodicSchedulerConfig.scala | 14 - .../scheduler/PeriodicSchedulerImpl.scala | 182 --------- .../docspell/joex/scheduler/QueueLogger.scala | 58 --- .../docspell/joex/scheduler/Scheduler.scala | 38 -- .../joex/scheduler/SchedulerBuilder.scala | 96 ----- .../joex/scheduler/SchedulerConfig.scala | 33 -- .../joex/scheduler/SchedulerImpl.scala | 351 ------------------ .../scala/docspell/joex/scheduler/Task.scala | 72 ---- .../joex/updatecheck/UpdateCheckTask.scala | 4 +- 70 files changed, 64 insertions(+), 1398 deletions(-) delete mode 100644 modules/backend/src/main/scala/docspell/backend/msg/JobDone.scala delete mode 100644 modules/joex/src/main/scala/docspell/joex/scheduler/Context.scala delete mode 100644 modules/joex/src/main/scala/docspell/joex/scheduler/CountingScheme.scala delete mode 100644 modules/joex/src/main/scala/docspell/joex/scheduler/JobTask.scala delete mode 100644 modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskRegistry.scala delete mode 100644 modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskResult.scala delete mode 100644 modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskResultEncoder.scala delete mode 100644 modules/joex/src/main/scala/docspell/joex/scheduler/LogEvent.scala delete mode 100644 modules/joex/src/main/scala/docspell/joex/scheduler/LogSink.scala delete mode 100644 modules/joex/src/main/scala/docspell/joex/scheduler/PeriodicScheduler.scala delete mode 100644 modules/joex/src/main/scala/docspell/joex/scheduler/PeriodicSchedulerConfig.scala delete mode 100644 modules/joex/src/main/scala/docspell/joex/scheduler/PeriodicSchedulerImpl.scala delete mode 100644 modules/joex/src/main/scala/docspell/joex/scheduler/QueueLogger.scala delete mode 100644 modules/joex/src/main/scala/docspell/joex/scheduler/Scheduler.scala delete mode 100644 modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerBuilder.scala delete mode 100644 modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerConfig.scala delete mode 100644 modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerImpl.scala delete mode 100644 modules/joex/src/main/scala/docspell/joex/scheduler/Task.scala diff --git a/modules/backend/src/main/scala/docspell/backend/msg/JobDone.scala b/modules/backend/src/main/scala/docspell/backend/msg/JobDone.scala deleted file mode 100644 index 0ac701c3..00000000 --- a/modules/backend/src/main/scala/docspell/backend/msg/JobDone.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.backend.msg - -import docspell.common._ -import docspell.pubsub.api.{Topic, TypedTopic} - -import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} -import io.circe.{Decoder, Encoder} - -/** Message to notify about finished jobs. They have a final state. */ -final case class JobDone( - jobId: Ident, - group: Ident, - task: Ident, - args: String, - state: JobState -) -object JobDone { - implicit val jsonDecoder: Decoder[JobDone] = - deriveDecoder[JobDone] - - implicit val jsonEncoder: Encoder[JobDone] = - deriveEncoder[JobDone] - - val topic: TypedTopic[JobDone] = - TypedTopic(Topic("job-finished")) -} diff --git a/modules/backend/src/main/scala/docspell/backend/msg/Topics.scala b/modules/backend/src/main/scala/docspell/backend/msg/Topics.scala index ae53d9f6..bda187d1 100644 --- a/modules/backend/src/main/scala/docspell/backend/msg/Topics.scala +++ b/modules/backend/src/main/scala/docspell/backend/msg/Topics.scala @@ -7,8 +7,8 @@ package docspell.backend.msg import cats.data.NonEmptyList - import docspell.pubsub.api.{Topic, TypedTopic} +import docspell.scheduler.msg.JobDone /** All topics used in Docspell. */ object Topics { diff --git a/modules/joex/src/main/scala/docspell/joex/Config.scala b/modules/joex/src/main/scala/docspell/joex/Config.scala index 549b24ca..0d0945d6 100644 --- a/modules/joex/src/main/scala/docspell/joex/Config.scala +++ b/modules/joex/src/main/scala/docspell/joex/Config.scala @@ -19,7 +19,7 @@ import docspell.ftssolr.SolrConfig import docspell.joex.analysis.RegexNerFile import docspell.joex.hk.HouseKeepingConfig import docspell.joex.routes.InternalHeader -import docspell.joex.scheduler.{PeriodicSchedulerConfig, SchedulerConfig} +import docspell.scheduler.{PeriodicSchedulerConfig, SchedulerConfig} import docspell.joex.updatecheck.UpdateCheckConfig import docspell.logging.LogConfig import docspell.pubsub.naive.PubSubConfig diff --git a/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala b/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala index a17d94a3..ec5e70da 100644 --- a/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala +++ b/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala @@ -10,7 +10,7 @@ import cats.effect.Async import docspell.config.Implicits._ import docspell.config.{ConfigFactory, Validation} -import docspell.joex.scheduler.CountingScheme +import docspell.scheduler.CountingScheme import emil.MailAddress import emil.javamail.syntax._ diff --git a/modules/joex/src/main/scala/docspell/joex/JoexApp.scala b/modules/joex/src/main/scala/docspell/joex/JoexApp.scala index 4f096f02..7fe53412 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexApp.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexApp.scala @@ -7,7 +7,7 @@ package docspell.joex import docspell.common.Ident -import docspell.joex.scheduler.{PeriodicScheduler, Scheduler} +import docspell.scheduler.{PeriodicScheduler, Scheduler} import docspell.store.records.RJobLog trait JoexApp[F[_]] { diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala index ce34e43c..7ba81536 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala @@ -9,7 +9,6 @@ package docspell.joex import cats.effect._ import cats.implicits._ import fs2.concurrent.SignallingRef - import docspell.analysis.TextAnalyser import docspell.backend.MailAddressCodec import docspell.backend.fulltext.CreateIndex @@ -32,7 +31,8 @@ import docspell.joex.preview._ import docspell.joex.process.ItemHandler import docspell.joex.process.ReProcessItem import docspell.joex.scanmailbox._ -import docspell.joex.scheduler._ +import docspell.scheduler._ +import docspell.scheduler.impl.{PeriodicSchedulerBuilder, SchedulerBuilder} import docspell.joex.updatecheck._ import docspell.notification.api.NotificationModule import docspell.notification.impl.NotificationModuleImpl @@ -42,7 +42,6 @@ import docspell.store.queue._ import docspell.store.records.{REmptyTrashSetting, RJobLog} import docspell.store.usertask.UserTaskScope import docspell.store.usertask.UserTaskStore - import emil.javamail._ import org.http4s.client.Client @@ -296,12 +295,12 @@ object JoexAppImpl extends MailAddressCodec { ) ) .resource - psch <- PeriodicScheduler.create( + psch <- PeriodicSchedulerBuilder.build( cfg.periodicScheduler, sch, queue, pstore, - joex + joex.notifyAllNodes ) app = new JoexAppImpl( cfg, diff --git a/modules/joex/src/main/scala/docspell/joex/emptytrash/EmptyTrashTask.scala b/modules/joex/src/main/scala/docspell/joex/emptytrash/EmptyTrashTask.scala index 5b1cbd3f..28a834c7 100644 --- a/modules/joex/src/main/scala/docspell/joex/emptytrash/EmptyTrashTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/emptytrash/EmptyTrashTask.scala @@ -12,7 +12,7 @@ import fs2.Stream import docspell.backend.ops.{OItem, OItemSearch} import docspell.common._ -import docspell.joex.scheduler._ +import docspell.scheduler._ import docspell.store.records.RItem import docspell.store.usertask.UserTask diff --git a/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala b/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala index 67be2980..2f201363 100644 --- a/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala @@ -12,7 +12,7 @@ import cats.implicits._ import docspell.common.FileCopyTaskArgs.Selection import docspell.common.{FileCopyTaskArgs, Ident} import docspell.joex.Config -import docspell.joex.scheduler.{JobTaskResultEncoder, Task} +import docspell.scheduler.{JobTaskResultEncoder, Task} import docspell.logging.Logger import docspell.store.file.{BinnyUtils, FileRepository, FileRepositoryConfig} import binny.CopyTool.Counter diff --git a/modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala b/modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala index 0b64f131..abd4d01b 100644 --- a/modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala @@ -12,7 +12,7 @@ import cats.implicits._ import docspell.backend.ops.OFileRepository import docspell.backend.ops.OFileRepository.IntegrityResult import docspell.common.{FileIntegrityCheckArgs, FileKey} -import docspell.joex.scheduler.{JobTaskResultEncoder, Task} +import docspell.scheduler.{JobTaskResultEncoder, Task} import docspell.store.records.RFileMeta import io.circe.Encoder import io.circe.generic.semiauto.deriveEncoder diff --git a/modules/joex/src/main/scala/docspell/joex/fts/FtsContext.scala b/modules/joex/src/main/scala/docspell/joex/fts/FtsContext.scala index 8e7f133b..41008a1d 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/FtsContext.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/FtsContext.scala @@ -9,7 +9,7 @@ package docspell.joex.fts import docspell.backend.fulltext.CreateIndex import docspell.ftsclient.FtsClient import docspell.joex.Config -import docspell.joex.scheduler.Context +import docspell.scheduler.Context import docspell.logging.Logger import docspell.store.Store diff --git a/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala b/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala index 1184c0a3..1a0a256f 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala @@ -14,7 +14,7 @@ import docspell.backend.fulltext.CreateIndex import docspell.common._ import docspell.ftsclient._ import docspell.joex.Config -import docspell.joex.scheduler.Context +import docspell.scheduler.Context import docspell.logging.Logger object FtsWork { diff --git a/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala b/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala index dee2fc1f..707b457b 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala @@ -13,7 +13,7 @@ import docspell.backend.fulltext.CreateIndex import docspell.common._ import docspell.ftsclient._ import docspell.joex.Config -import docspell.joex.scheduler.Task +import docspell.scheduler.Task import docspell.store.records.RJob object MigrationTask { diff --git a/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala b/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala index 7c7f81e0..d9c0e832 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala @@ -14,7 +14,7 @@ import docspell.common._ import docspell.ftsclient._ import docspell.joex.Config import docspell.joex.fts.FtsWork.syntax._ -import docspell.joex.scheduler.Task +import docspell.scheduler.Task object ReIndexTask { type Args = ReIndexTaskArgs diff --git a/modules/joex/src/main/scala/docspell/joex/hk/CheckNodesTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/CheckNodesTask.scala index 09651893..aff4a5d3 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/CheckNodesTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/CheckNodesTask.scala @@ -10,7 +10,7 @@ import cats.effect._ import cats.implicits._ import docspell.common._ -import docspell.joex.scheduler.{Context, Task} +import docspell.scheduler.{Context, Task} import docspell.logging.Logger import docspell.store.records._ diff --git a/modules/joex/src/main/scala/docspell/joex/hk/CleanupInvitesTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/CleanupInvitesTask.scala index de23eb71..e0cf916c 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/CleanupInvitesTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/CleanupInvitesTask.scala @@ -10,7 +10,7 @@ import cats.effect._ import cats.implicits._ import docspell.common._ -import docspell.joex.scheduler.Task +import docspell.scheduler.Task import docspell.store.records._ object CleanupInvitesTask { diff --git a/modules/joex/src/main/scala/docspell/joex/hk/CleanupJobsTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/CleanupJobsTask.scala index 67a1e1a2..104ae306 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/CleanupJobsTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/CleanupJobsTask.scala @@ -11,7 +11,7 @@ import cats.implicits._ import fs2.Stream import docspell.common._ -import docspell.joex.scheduler.Task +import docspell.scheduler.Task import docspell.store.Store import docspell.store.records._ diff --git a/modules/joex/src/main/scala/docspell/joex/hk/CleanupRememberMeTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/CleanupRememberMeTask.scala index 120b8cc8..e1040e1c 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/CleanupRememberMeTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/CleanupRememberMeTask.scala @@ -10,7 +10,7 @@ import cats.effect._ import cats.implicits._ import docspell.common._ -import docspell.joex.scheduler.Task +import docspell.scheduler.Task import docspell.store.records._ object CleanupRememberMeTask { diff --git a/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala index c0c3d084..206b7c29 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala @@ -13,7 +13,7 @@ import docspell.backend.ops.OFileRepository import docspell.common._ import docspell.joex.Config import docspell.joex.filecopy.FileIntegrityCheckTask -import docspell.joex.scheduler.{JobTaskResultEncoder, Task} +import docspell.scheduler.{JobTaskResultEncoder, Task} import docspell.store.records._ import docspell.store.usertask.UserTaskScope diff --git a/modules/joex/src/main/scala/docspell/joex/hk/IntegrityCheckTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/IntegrityCheckTask.scala index 42bb3333..b9b32ccb 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/IntegrityCheckTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/IntegrityCheckTask.scala @@ -12,7 +12,7 @@ import cats.implicits._ import docspell.backend.ops.OFileRepository import docspell.common._ import docspell.joex.filecopy.FileIntegrityCheckTask -import docspell.joex.scheduler.Task +import docspell.scheduler.Task object IntegrityCheckTask { diff --git a/modules/joex/src/main/scala/docspell/joex/learn/LearnClassifierTask.scala b/modules/joex/src/main/scala/docspell/joex/learn/LearnClassifierTask.scala index 129afc5a..60fb01e3 100644 --- a/modules/joex/src/main/scala/docspell/joex/learn/LearnClassifierTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/learn/LearnClassifierTask.scala @@ -14,7 +14,7 @@ import docspell.analysis.TextAnalyser import docspell.backend.ops.OCollective import docspell.common._ import docspell.joex.Config -import docspell.joex.scheduler._ +import docspell.scheduler._ import docspell.logging.Logger import docspell.store.records.{RClassifierModel, RClassifierSetting} diff --git a/modules/joex/src/main/scala/docspell/joex/learn/LearnItemEntities.scala b/modules/joex/src/main/scala/docspell/joex/learn/LearnItemEntities.scala index 9fc1b502..cf8e7fe5 100644 --- a/modules/joex/src/main/scala/docspell/joex/learn/LearnItemEntities.scala +++ b/modules/joex/src/main/scala/docspell/joex/learn/LearnItemEntities.scala @@ -14,7 +14,7 @@ import fs2.Stream import docspell.analysis.TextAnalyser import docspell.analysis.classifier.TextClassifier.Data import docspell.common._ -import docspell.joex.scheduler._ +import docspell.scheduler._ object LearnItemEntities { def learnAll[F[_]: Async, A]( diff --git a/modules/joex/src/main/scala/docspell/joex/learn/LearnTags.scala b/modules/joex/src/main/scala/docspell/joex/learn/LearnTags.scala index 732a38af..f1460c19 100644 --- a/modules/joex/src/main/scala/docspell/joex/learn/LearnTags.scala +++ b/modules/joex/src/main/scala/docspell/joex/learn/LearnTags.scala @@ -12,7 +12,7 @@ import cats.implicits._ import docspell.analysis.TextAnalyser import docspell.common._ -import docspell.joex.scheduler._ +import docspell.scheduler._ import docspell.store.records.RClassifierSetting object LearnTags { diff --git a/modules/joex/src/main/scala/docspell/joex/learn/SelectItems.scala b/modules/joex/src/main/scala/docspell/joex/learn/SelectItems.scala index fa99354d..b25f28ce 100644 --- a/modules/joex/src/main/scala/docspell/joex/learn/SelectItems.scala +++ b/modules/joex/src/main/scala/docspell/joex/learn/SelectItems.scala @@ -10,7 +10,7 @@ import fs2.{Pipe, Stream} import docspell.analysis.classifier.TextClassifier.Data import docspell.common._ -import docspell.joex.scheduler.Context +import docspell.scheduler.Context import docspell.store.Store import docspell.store.qb.Batch import docspell.store.queries.{QItem, TextAndTag} diff --git a/modules/joex/src/main/scala/docspell/joex/learn/StoreClassifierModel.scala b/modules/joex/src/main/scala/docspell/joex/learn/StoreClassifierModel.scala index e0e7eabc..8c4feb9b 100644 --- a/modules/joex/src/main/scala/docspell/joex/learn/StoreClassifierModel.scala +++ b/modules/joex/src/main/scala/docspell/joex/learn/StoreClassifierModel.scala @@ -12,7 +12,7 @@ import fs2.io.file.Files import docspell.analysis.classifier.ClassifierModel import docspell.common._ -import docspell.joex.scheduler._ +import docspell.scheduler._ import docspell.logging.Logger import docspell.store.Store import docspell.store.records.RClassifierModel diff --git a/modules/joex/src/main/scala/docspell/joex/notify/PeriodicDueItemsTask.scala b/modules/joex/src/main/scala/docspell/joex/notify/PeriodicDueItemsTask.scala index 8ef5bc33..69361b69 100644 --- a/modules/joex/src/main/scala/docspell/joex/notify/PeriodicDueItemsTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/notify/PeriodicDueItemsTask.scala @@ -12,8 +12,8 @@ import cats.implicits._ import docspell.backend.ops.ONotification import docspell.common._ -import docspell.joex.scheduler.Context -import docspell.joex.scheduler.Task +import docspell.scheduler.Context +import docspell.scheduler.Task import docspell.notification.api.EventContext import docspell.notification.api.NotificationChannel import docspell.notification.api.PeriodicDueItemsArgs diff --git a/modules/joex/src/main/scala/docspell/joex/notify/PeriodicQueryTask.scala b/modules/joex/src/main/scala/docspell/joex/notify/PeriodicQueryTask.scala index 088f65ed..8ed45a86 100644 --- a/modules/joex/src/main/scala/docspell/joex/notify/PeriodicQueryTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/notify/PeriodicQueryTask.scala @@ -13,8 +13,8 @@ import cats.implicits._ import docspell.backend.ops.ONotification import docspell.common._ -import docspell.joex.scheduler.Context -import docspell.joex.scheduler.Task +import docspell.scheduler.Context +import docspell.scheduler.Task import docspell.notification.api.EventContext import docspell.notification.api.NotificationChannel import docspell.notification.api.PeriodicQueryArgs diff --git a/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala b/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala index c882d582..99b315b7 100644 --- a/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala @@ -13,8 +13,8 @@ import fs2.{Chunk, Stream} import docspell.backend.JobFactory import docspell.backend.ops.OJoex import docspell.common._ -import docspell.joex.scheduler.Context -import docspell.joex.scheduler.Task +import docspell.scheduler.Context +import docspell.scheduler.Task import docspell.store.queue.JobQueue import docspell.store.records.RAttachment import docspell.store.records.RJob diff --git a/modules/joex/src/main/scala/docspell/joex/pagecount/PageCountTask.scala b/modules/joex/src/main/scala/docspell/joex/pagecount/PageCountTask.scala index fcbaaa79..5f4e73fa 100644 --- a/modules/joex/src/main/scala/docspell/joex/pagecount/PageCountTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/pagecount/PageCountTask.scala @@ -11,8 +11,8 @@ import cats.implicits._ import docspell.common._ import docspell.joex.process.AttachmentPageCount -import docspell.joex.scheduler.Context -import docspell.joex.scheduler.Task +import docspell.scheduler.Context +import docspell.scheduler.Task import docspell.store.records.RAttachment import docspell.store.records.RAttachmentMeta diff --git a/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala b/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala index 1df9fcf4..e4def9ac 100644 --- a/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala @@ -12,7 +12,7 @@ import fs2.{Chunk, Stream} import docspell.backend.ops.OJoex import docspell.common._ -import docspell.joex.scheduler.{Context, Task} +import docspell.scheduler.{Context, Task} import docspell.store.queue.JobQueue import docspell.store.records.RAttachment import docspell.store.records._ diff --git a/modules/joex/src/main/scala/docspell/joex/pdfconv/PdfConvTask.scala b/modules/joex/src/main/scala/docspell/joex/pdfconv/PdfConvTask.scala index ca82615a..47843a0b 100644 --- a/modules/joex/src/main/scala/docspell/joex/pdfconv/PdfConvTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/pdfconv/PdfConvTask.scala @@ -16,7 +16,7 @@ import docspell.common._ import docspell.convert.ConversionResult import docspell.convert.extern.OcrMyPdf import docspell.joex.Config -import docspell.joex.scheduler.{Context, Task} +import docspell.scheduler.{Context, Task} import docspell.store.records._ import io.circe.generic.semiauto._ diff --git a/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala b/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala index 4021b67a..15616ed5 100644 --- a/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala @@ -14,8 +14,8 @@ import docspell.backend.JobFactory import docspell.backend.ops.OJoex import docspell.common.MakePreviewArgs.StoreMode import docspell.common._ -import docspell.joex.scheduler.Context -import docspell.joex.scheduler.Task +import docspell.scheduler.Context +import docspell.scheduler.Task import docspell.store.queue.JobQueue import docspell.store.records.RAttachment import docspell.store.records.RJob diff --git a/modules/joex/src/main/scala/docspell/joex/preview/MakePreviewTask.scala b/modules/joex/src/main/scala/docspell/joex/preview/MakePreviewTask.scala index 14bf18f3..d20a6c75 100644 --- a/modules/joex/src/main/scala/docspell/joex/preview/MakePreviewTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/preview/MakePreviewTask.scala @@ -13,8 +13,8 @@ import docspell.common._ import docspell.extract.pdfbox.PdfboxPreview import docspell.extract.pdfbox.PreviewConfig import docspell.joex.process.AttachmentPreview -import docspell.joex.scheduler.Context -import docspell.joex.scheduler.Task +import docspell.scheduler.Context +import docspell.scheduler.Task import docspell.store.records.RAttachment import docspell.store.records.RAttachmentPreview diff --git a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala index cce9400d..f6444311 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala @@ -15,7 +15,7 @@ import fs2.Stream import docspell.common._ import docspell.extract.pdfbox.PdfMetaData import docspell.extract.pdfbox.PdfboxExtract -import docspell.joex.scheduler._ +import docspell.scheduler._ import docspell.store.records.RAttachment import docspell.store.records._ diff --git a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala index 2a55775d..b15253da 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala @@ -15,7 +15,7 @@ import fs2.Stream import docspell.common._ import docspell.extract.pdfbox.PdfboxPreview import docspell.extract.pdfbox.PreviewConfig -import docspell.joex.scheduler._ +import docspell.scheduler._ import docspell.store.queries.QAttachment import docspell.store.records.RAttachment import docspell.store.records._ diff --git a/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala b/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala index 73754127..a7855808 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala @@ -17,7 +17,7 @@ import docspell.convert.ConversionResult.Handler import docspell.convert.SanitizeHtml import docspell.convert._ import docspell.joex.extract.JsoupSanitizer -import docspell.joex.scheduler._ +import docspell.scheduler._ import docspell.store.records._ /** Goes through all attachments and creates a PDF version of it where supported. diff --git a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala index 092c8495..56fd9b2f 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala @@ -13,7 +13,7 @@ import cats.implicits._ import fs2.Stream import docspell.common._ -import docspell.joex.scheduler.{Context, Task} +import docspell.scheduler.{Context, Task} import docspell.store.file.FileMetadata import docspell.store.queries.QItem import docspell.store.records._ diff --git a/modules/joex/src/main/scala/docspell/joex/process/CrossCheckProposals.scala b/modules/joex/src/main/scala/docspell/joex/process/CrossCheckProposals.scala index 3cd9d3ad..e13b5da8 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/CrossCheckProposals.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/CrossCheckProposals.scala @@ -12,7 +12,7 @@ import cats.effect.Sync import cats.implicits._ import docspell.common._ -import docspell.joex.scheduler.Task +import docspell.scheduler.Task import docspell.logging.Logger /** After candidates have been determined, the set is reduced by doing some cross checks. diff --git a/modules/joex/src/main/scala/docspell/joex/process/DuplicateCheck.scala b/modules/joex/src/main/scala/docspell/joex/process/DuplicateCheck.scala index 41fcd0e3..6b07e949 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/DuplicateCheck.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/DuplicateCheck.scala @@ -10,7 +10,7 @@ import cats.effect._ import cats.implicits._ import docspell.common._ -import docspell.joex.scheduler.{Context, Task} +import docspell.scheduler.{Context, Task} import docspell.store.queries.QItem import docspell.store.records.RFileMeta import docspell.store.records.RJob diff --git a/modules/joex/src/main/scala/docspell/joex/process/EvalProposals.scala b/modules/joex/src/main/scala/docspell/joex/process/EvalProposals.scala index 38c129b1..2a9092a1 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/EvalProposals.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/EvalProposals.scala @@ -12,7 +12,7 @@ import cats.effect.Sync import cats.implicits._ import docspell.common._ -import docspell.joex.scheduler.{Context, Task} +import docspell.scheduler.{Context, Task} import docspell.store.records.{RAttachmentMeta, RPerson} /** Calculate weights for candidates that adds the most likely candidate a lower number. diff --git a/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala b/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala index 17f90b59..b3fb3bbf 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala @@ -18,7 +18,7 @@ import fs2.Stream import docspell.common._ import docspell.files.Zip import docspell.joex.mail._ -import docspell.joex.scheduler._ +import docspell.scheduler._ import docspell.store.records._ import emil.Mail diff --git a/modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala b/modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala index a6898f51..a4af8e48 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala @@ -16,7 +16,7 @@ import docspell.analysis.contact._ import docspell.common.MetaProposal.Candidate import docspell.common._ import docspell.joex.Config -import docspell.joex.scheduler.{Context, Task} +import docspell.scheduler.{Context, Task} import docspell.store.records._ /** Super simple approach to find corresponding meta data to an item by looking up values diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala index 3b5ac5ab..c96c0189 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala @@ -8,7 +8,7 @@ package docspell.joex.process import docspell.common._ import docspell.joex.process.ItemData.AttachmentDates -import docspell.joex.scheduler.JobTaskResultEncoder +import docspell.scheduler.JobTaskResultEncoder import docspell.store.records.{RAttachment, RAttachmentMeta, RItem} import io.circe.syntax.EncoderOps diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala index 5da1e6d6..21627f56 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala @@ -17,7 +17,7 @@ import docspell.common.{ItemState, ProcessItemArgs} import docspell.ftsclient.FtsClient import docspell.joex.Config import docspell.joex.analysis.RegexNerFile -import docspell.joex.scheduler.Task +import docspell.scheduler.Task import docspell.store.queries.QItem import docspell.store.records.RItem diff --git a/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala b/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala index 7f5418fc..2e85e450 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala @@ -11,7 +11,7 @@ import cats.effect.Sync import cats.implicits._ import docspell.common._ -import docspell.joex.scheduler.{Context, Task} +import docspell.scheduler.{Context, Task} import docspell.store.records.RItem object LinkProposal { diff --git a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala index 735d13be..5918698c 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala @@ -15,7 +15,7 @@ import docspell.common.ProcessItemArgs import docspell.ftsclient.FtsClient import docspell.joex.Config import docspell.joex.analysis.RegexNerFile -import docspell.joex.scheduler.Task +import docspell.scheduler.Task object ProcessItem { diff --git a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala index 15aa939d..834832a5 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala @@ -16,8 +16,8 @@ import docspell.common._ import docspell.ftsclient.FtsClient import docspell.joex.Config import docspell.joex.analysis.RegexNerFile -import docspell.joex.scheduler.Context -import docspell.joex.scheduler.Task +import docspell.scheduler.Context +import docspell.scheduler.Task import docspell.store.queries.QItem import docspell.store.records.RAttachment import docspell.store.records.RAttachmentSource diff --git a/modules/joex/src/main/scala/docspell/joex/process/RemoveEmptyItem.scala b/modules/joex/src/main/scala/docspell/joex/process/RemoveEmptyItem.scala index fadcb949..3d961937 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/RemoveEmptyItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/RemoveEmptyItem.scala @@ -11,7 +11,7 @@ import cats.implicits._ import docspell.backend.ops.OItem import docspell.common._ -import docspell.joex.scheduler.Task +import docspell.scheduler.Task object RemoveEmptyItem { diff --git a/modules/joex/src/main/scala/docspell/joex/process/SaveProposals.scala b/modules/joex/src/main/scala/docspell/joex/process/SaveProposals.scala index 18cbb933..cdc8627a 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/SaveProposals.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/SaveProposals.scala @@ -10,7 +10,7 @@ import cats.effect.Sync import cats.implicits._ import docspell.common._ -import docspell.joex.scheduler.{Context, Task} +import docspell.scheduler.{Context, Task} import docspell.store.AddResult import docspell.store.records._ diff --git a/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala b/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala index 54283101..0734d294 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/SetGivenData.scala @@ -11,7 +11,7 @@ import cats.implicits._ import docspell.backend.ops.OItem import docspell.common._ -import docspell.joex.scheduler.Task +import docspell.scheduler.Task import docspell.store.UpdateResult object SetGivenData { diff --git a/modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala b/modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala index 00e410da..aced45f7 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala @@ -18,8 +18,8 @@ import docspell.joex.Config import docspell.joex.analysis.RegexNerFile import docspell.joex.learn.{ClassifierName, Classify, LearnClassifierTask} import docspell.joex.process.ItemData.AttachmentDates -import docspell.joex.scheduler.Context -import docspell.joex.scheduler.Task +import docspell.scheduler.Context +import docspell.scheduler.Task import docspell.store.records.{RAttachmentMeta, RClassifierSetting} object TextAnalysis { diff --git a/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala b/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala index 3f6be877..eaa02683 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala @@ -13,7 +13,7 @@ import cats.implicits._ import docspell.common._ import docspell.extract.{ExtractConfig, ExtractResult, Extraction} import docspell.ftsclient.{FtsClient, TextData} -import docspell.joex.scheduler.{Context, Task} +import docspell.scheduler.{Context, Task} import docspell.store.records.{RAttachment, RAttachmentMeta, RFileMeta} object TextExtraction { diff --git a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala index 869362e3..363420b0 100644 --- a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala @@ -16,7 +16,7 @@ import fs2._ import docspell.backend.ops.{OJoex, OUpload} import docspell.common._ import docspell.joex.Config -import docspell.joex.scheduler.{Context, Task} +import docspell.scheduler.{Context, Task} import docspell.logging.Logger import docspell.store.queries.QOrganization import docspell.store.records._ diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/Context.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/Context.scala deleted file mode 100644 index 2fb2a529..00000000 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/Context.scala +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.joex.scheduler - -import cats.effect._ -import cats.implicits._ -import cats.{Applicative, Functor} - -import docspell.common._ -import docspell.logging.Logger -import docspell.store.Store -import docspell.store.records.RJob - -trait Context[F[_], A] { self => - - def jobId: Ident - - def args: A - - def config: SchedulerConfig - - def logger: Logger[F] - - def setProgress(percent: Int): F[Unit] - - def store: Store[F] - - final def isLastRetry(implicit ev: Applicative[F]): F[Boolean] = - for { - current <- store.transact(RJob.getRetries(jobId)) - last = config.retries == current.getOrElse(0) - } yield last - - def map[C](f: A => C)(implicit F: Functor[F]): Context[F, C] = - new Context.ContextImpl[F, C](f(args), logger, store, config, jobId) -} - -object Context { - - def create[F[_]: Async, A]( - jobId: Ident, - arg: A, - config: SchedulerConfig, - log: Logger[F], - store: Store[F] - ): Context[F, A] = - new ContextImpl(arg, log, store, config, jobId) - - def apply[F[_]: Async, A]( - job: RJob, - arg: A, - config: SchedulerConfig, - logSink: LogSink[F], - store: Store[F] - ): F[Context[F, A]] = { - val log = docspell.logging.getLogger[F] - for { - _ <- log.trace("Creating logger for task run") - logger <- QueueLogger(job.id, job.info, config.logBufferSize, logSink) - _ <- log.trace("Logger created, instantiating context") - ctx = create[F, A](job.id, arg, config, logger, store) - } yield ctx - } - - final private class ContextImpl[F[_]: Functor, A]( - val args: A, - val logger: Logger[F], - val store: Store[F], - val config: SchedulerConfig, - val jobId: Ident - ) extends Context[F, A] { - - def setProgress(percent: Int): F[Unit] = { - val pval = math.min(100, math.max(0, percent)) - store.transact(RJob.setProgress(jobId, pval)).map(_ => ()) - } - } -} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/CountingScheme.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/CountingScheme.scala deleted file mode 100644 index 68890f57..00000000 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/CountingScheme.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.joex.scheduler - -import cats.implicits._ - -import docspell.common.Priority - -/** A counting scheme to indicate a ratio between scheduling high and low priority jobs. - * - * For example high=4, low=1 means: ”schedule 4 high priority jobs and then 1 low - * priority job“. - */ -case class CountingScheme(high: Int, low: Int, counter: Int = 0) { - - def nextPriority: (CountingScheme, Priority) = - if (counter <= 0) (increment, Priority.High) - else { - val rest = counter % (high + low) - if (rest < high) (increment, Priority.High) - else (increment, Priority.Low) - } - - def increment: CountingScheme = - copy(counter = counter + 1) -} - -object CountingScheme { - - def writeString(cs: CountingScheme): String = - s"${cs.high},${cs.low}" - - def readString(str: String): Either[String, CountingScheme] = - str.split(',') match { - case Array(h, l) => - Either.catchNonFatal(CountingScheme(h.toInt, l.toInt)).left.map(_.getMessage) - case _ => - Left(s"Invalid counting scheme: $str") - } -} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/JobTask.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/JobTask.scala deleted file mode 100644 index aec269b2..00000000 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/JobTask.scala +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.joex.scheduler - -import cats.effect.Sync -import cats.implicits._ - -import docspell.common.Ident -import docspell.common.syntax.all._ - -import io.circe.Decoder - -/** Binds a Task to a name. This is required to lookup the code based on the taskName in - * the RJob data and to execute it given the arguments that have to be read from a - * string. - * - * Since the scheduler only has a string for the task argument, this only works for Task - * impls that accept a string. There is a convenience constructor that uses circe to - * decode json into some type A. - */ -case class JobTask[F[_]]( - name: Ident, - task: Task[F, String, JobTaskResult], - onCancel: Task[F, String, Unit] -) - -object JobTask { - - def json[F[_]: Sync, A, B]( - name: Ident, - task: Task[F, A, B], - onCancel: Task[F, A, Unit] - )(implicit - D: Decoder[A], - E: JobTaskResultEncoder[B] - ): JobTask[F] = { - val convert: String => F[A] = - str => - str.parseJsonAs[A] match { - case Right(a) => a.pure[F] - case Left(ex) => - Sync[F].raiseError(new Exception(s"Cannot parse task arguments: $str", ex)) - } - - JobTask(name, task.contramap(convert).map(E.encode), onCancel.contramap(convert)) - } -} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskRegistry.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskRegistry.scala deleted file mode 100644 index 0efb37b7..00000000 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskRegistry.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.joex.scheduler - -import docspell.common.Ident - -/** This is a mapping from some identifier to a task. This is used by the scheduler to - * lookup an implementation using the taskName field of the RJob database record. - */ -final class JobTaskRegistry[F[_]](tasks: Map[Ident, JobTask[F]]) { - - def withTask(task: JobTask[F]): JobTaskRegistry[F] = - JobTaskRegistry(tasks.updated(task.name, task)) - - def find(taskName: Ident): Option[JobTask[F]] = - tasks.get(taskName) -} - -object JobTaskRegistry { - - def apply[F[_]](map: Map[Ident, JobTask[F]]): JobTaskRegistry[F] = - new JobTaskRegistry[F](map) - - def empty[F[_]]: JobTaskRegistry[F] = apply(Map.empty) - -} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskResult.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskResult.scala deleted file mode 100644 index d735565e..00000000 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskResult.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.joex.scheduler - -import io.circe.Json - -final case class JobTaskResult(message: Option[String], json: Option[Json]) { - - def withMessage(m: String): JobTaskResult = - copy(message = Some(m)) - - def withJson(json: Json): JobTaskResult = - copy(json = Some(json)) -} - -object JobTaskResult { - - val empty: JobTaskResult = JobTaskResult(None, None) - - def message(msg: String): JobTaskResult = JobTaskResult(Some(msg), None) - - def json(json: Json): JobTaskResult = JobTaskResult(None, Some(json)) -} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskResultEncoder.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskResultEncoder.scala deleted file mode 100644 index 205d01ce..00000000 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskResultEncoder.scala +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.joex.scheduler - -import docspell.joex.scheduler.JobTaskResultEncoder.instance - -import io.circe.Encoder - -trait JobTaskResultEncoder[A] { self => - def encode(a: A): JobTaskResult - - final def contramap[B](f: B => A): JobTaskResultEncoder[B] = - JobTaskResultEncoder.instance(b => self.encode(f(b))) - - final def map(f: JobTaskResult => JobTaskResult): JobTaskResultEncoder[A] = - instance(a => f(self.encode(a))) - - final def modify(f: (A, JobTaskResult) => JobTaskResult): JobTaskResultEncoder[A] = - instance(a => f(a, self.encode(a))) - - final def withMessage(f: A => String): JobTaskResultEncoder[A] = - modify((a, r) => r.withMessage(f(a))) -} - -object JobTaskResultEncoder { - - def apply[A](implicit v: JobTaskResultEncoder[A]): JobTaskResultEncoder[A] = v - - def instance[A](f: A => JobTaskResult): JobTaskResultEncoder[A] = - (a: A) => f(a) - - def fromJson[A: Encoder]: JobTaskResultEncoder[A] = - instance(a => JobTaskResult.json(Encoder[A].apply(a))) - - implicit val unitJobTaskResultEncoder: JobTaskResultEncoder[Unit] = - instance(_ => JobTaskResult.empty) - - implicit def optionJobTaskResultEncoder[A](implicit - ea: JobTaskResultEncoder[A] - ): JobTaskResultEncoder[Option[A]] = - instance { - case Some(a) => ea.encode(a) - case None => JobTaskResult.empty - } -} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/LogEvent.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/LogEvent.scala deleted file mode 100644 index 8edc966d..00000000 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/LogEvent.scala +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.joex.scheduler - -import cats.effect.Sync -import cats.implicits._ - -import docspell.common._ - -case class LogEvent( - jobId: Ident, - jobInfo: String, - time: Timestamp, - level: LogLevel, - msg: String, - ex: Option[Throwable] = None -) { - - def logLine: String = - s">>> ${time.asString} $level $jobInfo: $msg" - -} - -object LogEvent { - - def create[F[_]: Sync]( - jobId: Ident, - jobInfo: String, - level: LogLevel, - msg: String - ): F[LogEvent] = - Timestamp.current[F].map(now => LogEvent(jobId, jobInfo, now, level, msg)) - -} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/LogSink.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/LogSink.scala deleted file mode 100644 index bf01a050..00000000 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/LogSink.scala +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.joex.scheduler - -import cats.effect._ -import cats.implicits._ -import fs2.Pipe - -import docspell.common._ -import docspell.logging -import docspell.store.Store -import docspell.store.records.RJobLog - -trait LogSink[F[_]] { - - def receive: Pipe[F, LogEvent, Unit] - -} - -object LogSink { - - def apply[F[_]](sink: Pipe[F, LogEvent, Unit]): LogSink[F] = - new LogSink[F] { - val receive = sink - } - - def logInternal[F[_]: Sync](e: LogEvent): F[Unit] = { - val logger = docspell.logging.getLogger[F] - val addData: logging.LogEvent => logging.LogEvent = - _.data("jobId", e.jobId).data("jobInfo", e.jobInfo) - - e.level match { - case LogLevel.Info => - logger.infoWith(e.logLine)(addData) - case LogLevel.Debug => - logger.debugWith(e.logLine)(addData) - case LogLevel.Warn => - logger.warnWith(e.logLine)(addData) - case LogLevel.Error => - e.ex match { - case Some(exc) => - logger.errorWith(e.logLine)(addData.andThen(_.addError(exc))) - case None => - logger.errorWith(e.logLine)(addData) - } - } - } - - def printer[F[_]: Sync]: LogSink[F] = - LogSink(_.evalMap(e => logInternal(e))) - - def db[F[_]: Async](store: Store[F]): LogSink[F] = - LogSink( - _.evalMap(ev => - for { - id <- Ident.randomId[F] - joblog = RJobLog( - id, - ev.jobId, - ev.level, - ev.time, - ev.msg + ev.ex.map(th => ": " + th.getMessage).getOrElse("") - ) - _ <- logInternal(ev) - _ <- store.transact(RJobLog.insert(joblog)) - } yield () - ) - ) - - def dbAndLog[F[_]: Async](store: Store[F]): LogSink[F] = - LogSink(_.broadcastThrough(printer[F].receive, db[F](store).receive)) -} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/PeriodicScheduler.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/PeriodicScheduler.scala deleted file mode 100644 index 13f61705..00000000 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/PeriodicScheduler.scala +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.joex.scheduler - -import cats.effect._ -import fs2._ -import fs2.concurrent.SignallingRef - -import docspell.backend.ops.OJoex -import docspell.store.queue._ - -/** A periodic scheduler takes care to submit periodic tasks to the job queue. - * - * It is run in the background to regularily find a periodic task to execute. If the task - * is due, it will be submitted into the job queue where it will be picked up by the - * scheduler from some joex instance. If it is due in the future, a notification is - * scheduled to be received at that time so the task can be looked up again. - */ -trait PeriodicScheduler[F[_]] { - - def config: PeriodicSchedulerConfig - - def start: Stream[F, Nothing] - - def shutdown: F[Unit] - - def periodicAwake: F[Fiber[F, Throwable, Unit]] - - def notifyChange: F[Unit] -} - -object PeriodicScheduler { - - def create[F[_]: Async]( - cfg: PeriodicSchedulerConfig, - sch: Scheduler[F], - queue: JobQueue[F], - store: PeriodicTaskStore[F], - joex: OJoex[F] - ): Resource[F, PeriodicScheduler[F]] = - for { - waiter <- Resource.eval(SignallingRef(true)) - state <- Resource.eval(SignallingRef(PeriodicSchedulerImpl.emptyState[F])) - psch = new PeriodicSchedulerImpl[F]( - cfg, - sch, - queue, - store, - joex, - waiter, - state - ) - _ <- Resource.eval(psch.init) - } yield psch - -} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/PeriodicSchedulerConfig.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/PeriodicSchedulerConfig.scala deleted file mode 100644 index c960f4d0..00000000 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/PeriodicSchedulerConfig.scala +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.joex.scheduler - -import docspell.common._ - -case class PeriodicSchedulerConfig( - name: Ident, - wakeupPeriod: Duration -) diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/PeriodicSchedulerImpl.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/PeriodicSchedulerImpl.scala deleted file mode 100644 index 39761a74..00000000 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/PeriodicSchedulerImpl.scala +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.joex.scheduler - -import cats.effect._ -import cats.implicits._ -import fs2._ -import fs2.concurrent.SignallingRef - -import docspell.backend.ops.OJoex -import docspell.common._ -import docspell.joex.scheduler.PeriodicSchedulerImpl.State -import docspell.store.queue._ -import docspell.store.records.RPeriodicTask - -import eu.timepit.fs2cron.calev.CalevScheduler - -final class PeriodicSchedulerImpl[F[_]: Async]( - val config: PeriodicSchedulerConfig, - sch: Scheduler[F], - queue: JobQueue[F], - store: PeriodicTaskStore[F], - joex: OJoex[F], - waiter: SignallingRef[F, Boolean], - state: SignallingRef[F, State[F]] -) extends PeriodicScheduler[F] { - private[this] val logger = docspell.logging.getLogger[F] - - def start: Stream[F, Nothing] = - logger.stream.info("Starting periodic scheduler").drain ++ - mainLoop - - def shutdown: F[Unit] = - state.modify(_.requestShutdown) - - def periodicAwake: F[Fiber[F, Throwable, Unit]] = - Async[F].start( - Stream - .awakeEvery[F](config.wakeupPeriod.toScala) - .evalMap(_ => logger.debug("Periodic awake reached") *> notifyChange) - .compile - .drain - ) - - def notifyChange: F[Unit] = - waiter.update(b => !b) - - // internal - - /** On startup, get all periodic jobs from this scheduler and remove the mark, so they - * get picked up again. - */ - def init: F[Unit] = - logError("Error clearing marks")(store.clearMarks(config.name)) - - def mainLoop: Stream[F, Nothing] = { - val body: F[Boolean] = - for { - _ <- logger.debug(s"Going into main loop") - now <- Timestamp.current[F] - _ <- logger.debug(s"Looking for next periodic task") - go <- logThrow("Error getting next task")( - store - .takeNext(config.name, None) - .use { - case Marked.Found(pj) => - logger - .debug(s"Found periodic task '${pj.subject}/${pj.timer.asString}'") *> - (if (isTriggered(pj, now)) submitJob(pj) - else scheduleNotify(pj).map(_ => false)) - case Marked.NotFound => - logger.debug("No periodic task found") *> false.pure[F] - case Marked.NotMarkable => - logger.debug("Periodic job cannot be marked. Trying again.") *> true - .pure[F] - } - ) - } yield go - - Stream - .eval(state.get.map(_.shutdownRequest)) - .evalTap( - if (_) logger.info("Stopping main loop due to shutdown request.") - else ().pure[F] - ) - .flatMap(if (_) Stream.empty else Stream.eval(cancelNotify *> body)) - .flatMap { - case true => - mainLoop - case false => - logger.stream.debug(s"Waiting for notify").drain ++ - waiter.discrete.take(2).drain ++ - logger.stream.debug(s"Notify signal, going into main loop").drain ++ - mainLoop - } - } - - def isTriggered(pj: RPeriodicTask, now: Timestamp): Boolean = - pj.nextrun < now - - def submitJob(pj: RPeriodicTask): F[Boolean] = - store - .findNonFinalJob(pj.id) - .flatMap { - case Some(job) => - logger.info( - s"There is already a job with non-final state '${job.state}' in the queue" - ) *> scheduleNotify(pj) *> false.pure[F] - - case None => - logger.info(s"Submitting job for periodic task '${pj.task.id}'") *> - pj.toJob.flatMap(queue.insert) *> notifyJoex *> true.pure[F] - } - - def notifyJoex: F[Unit] = - sch.notifyChange *> joex.notifyAllNodes - - def scheduleNotify(pj: RPeriodicTask): F[Unit] = - Timestamp - .current[F] - .flatMap(now => - logger.debug( - s"Scheduling next notify for timer ${pj.timer.asString} -> ${pj.timer.nextElapse(now.toUtcDateTime)}" - ) - ) *> - Async[F] - .start( - CalevScheduler - .utc[F] - .sleep(pj.timer) - .evalMap(_ => notifyChange) - .compile - .drain - ) - .flatMap(fb => state.modify(_.setNotify(fb))) - - def cancelNotify: F[Unit] = - state - .modify(_.clearNotify) - .flatMap { - case Some(fb) => - fb.cancel - case None => - ().pure[F] - } - - private def logError(msg: => String)(fa: F[Unit]): F[Unit] = - fa.attempt.flatMap { - case Right(_) => ().pure[F] - case Left(ex) => logger.error(ex)(msg).map(_ => ()) - } - - private def logThrow[A](msg: => String)(fa: F[A]): F[A] = - fa.attempt.flatMap { - case r @ Right(_) => (r: Either[Throwable, A]).pure[F] - case l @ Left(ex) => logger.error(ex)(msg).map(_ => (l: Either[Throwable, A])) - }.rethrow -} - -object PeriodicSchedulerImpl { - def emptyState[F[_]]: State[F] = - State(false, None) - - case class State[F[_]]( - shutdownRequest: Boolean, - scheduledNotify: Option[Fiber[F, Throwable, Unit]] - ) { - def requestShutdown: (State[F], Unit) = - (copy(shutdownRequest = true), ()) - - def setNotify(fb: Fiber[F, Throwable, Unit]): (State[F], Unit) = - (copy(scheduledNotify = Some(fb)), ()) - - def clearNotify: (State[F], Option[Fiber[F, Throwable, Unit]]) = - (copy(scheduledNotify = None), scheduledNotify) - - } -} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/QueueLogger.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/QueueLogger.scala deleted file mode 100644 index 357a1e83..00000000 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/QueueLogger.scala +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.joex.scheduler - -import cats.effect._ -import cats.effect.std.Queue -import cats.implicits._ -import fs2.Stream - -import docspell.common._ -import docspell.logging -import docspell.logging.{Level, Logger} - -object QueueLogger { - - def create[F[_]: Sync]( - jobId: Ident, - jobInfo: String, - q: Queue[F, LogEvent] - ): Logger[F] = - new Logger[F] { - - def log(logEvent: logging.LogEvent) = - LogEvent - .create[F](jobId, jobInfo, level2Level(logEvent.level), logEvent.msg()) - .flatMap { ev => - val event = - logEvent.findErrors.headOption - .map(ex => ev.copy(ex = Some(ex))) - .getOrElse(ev) - - q.offer(event) - } - - def asUnsafe = Logger.off - } - - def apply[F[_]: Async]( - jobId: Ident, - jobInfo: String, - bufferSize: Int, - sink: LogSink[F] - ): F[Logger[F]] = - for { - q <- Queue.circularBuffer[F, LogEvent](bufferSize) - log = create(jobId, jobInfo, q) - _ <- Async[F].start( - Stream.fromQueueUnterminated(q).through(sink.receive).compile.drain - ) - } yield log - - private def level2Level(level: Level): LogLevel = - LogLevel.fromLevel(level) -} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/Scheduler.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/Scheduler.scala deleted file mode 100644 index 730b616b..00000000 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/Scheduler.scala +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.joex.scheduler - -import cats.effect._ -import fs2.Stream - -import docspell.common.Ident -import docspell.store.records.RJob - -trait Scheduler[F[_]] { - - def config: SchedulerConfig - - def getRunning: F[Vector[RJob]] - - def requestCancel(jobId: Ident): F[Boolean] - - def notifyChange: F[Unit] - - def start: Stream[F, Nothing] - - /** Requests to shutdown the scheduler. - * - * The scheduler will not take any new jobs from the queue. If there are still running - * jobs, it waits for them to complete. when the cancelAll flag is set to true, it - * cancels all running jobs. - * - * The returned F[Unit] can be evaluated to wait for all that to complete. - */ - def shutdown(cancelAll: Boolean): F[Unit] - - def periodicAwake: F[Fiber[F, Throwable, Unit]] -} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerBuilder.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerBuilder.scala deleted file mode 100644 index cc5cef12..00000000 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerBuilder.scala +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.joex.scheduler - -import cats.effect._ -import cats.effect.std.Semaphore -import cats.implicits._ -import fs2.concurrent.SignallingRef - -import docspell.notification.api.EventSink -import docspell.pubsub.api.PubSubT -import docspell.store.Store -import docspell.store.queue.JobQueue - -case class SchedulerBuilder[F[_]: Async]( - config: SchedulerConfig, - tasks: JobTaskRegistry[F], - store: Store[F], - queue: Resource[F, JobQueue[F]], - logSink: LogSink[F], - pubSub: PubSubT[F], - eventSink: EventSink[F] -) { - - def withConfig(cfg: SchedulerConfig): SchedulerBuilder[F] = - copy(config = cfg) - - def withTaskRegistry(reg: JobTaskRegistry[F]): SchedulerBuilder[F] = - copy(tasks = reg) - - def withTask[A](task: JobTask[F]): SchedulerBuilder[F] = - withTaskRegistry(tasks.withTask(task)) - - def withQueue(queue: Resource[F, JobQueue[F]]): SchedulerBuilder[F] = - copy(queue = queue) - - def withLogSink(sink: LogSink[F]): SchedulerBuilder[F] = - copy(logSink = sink) - - def withQueue(queue: JobQueue[F]): SchedulerBuilder[F] = - copy(queue = Resource.pure[F, JobQueue[F]](queue)) - - def withPubSub(pubSubT: PubSubT[F]): SchedulerBuilder[F] = - copy(pubSub = pubSubT) - - def withEventSink(sink: EventSink[F]): SchedulerBuilder[F] = - copy(eventSink = sink) - - def serve: Resource[F, Scheduler[F]] = - resource.evalMap(sch => Async[F].start(sch.start.compile.drain).map(_ => sch)) - - def resource: Resource[F, Scheduler[F]] = { - val scheduler: Resource[F, SchedulerImpl[F]] = for { - jq <- queue - waiter <- Resource.eval(SignallingRef(true)) - state <- Resource.eval(SignallingRef(SchedulerImpl.emptyState[F])) - perms <- Resource.eval(Semaphore(config.poolSize.toLong)) - } yield new SchedulerImpl[F]( - config, - jq, - pubSub, - eventSink, - tasks, - store, - logSink, - state, - waiter, - perms - ) - - scheduler.evalTap(_.init).map(s => s: Scheduler[F]) - } - -} - -object SchedulerBuilder { - - def apply[F[_]: Async]( - config: SchedulerConfig, - store: Store[F] - ): SchedulerBuilder[F] = - new SchedulerBuilder[F]( - config, - JobTaskRegistry.empty[F], - store, - JobQueue(store), - LogSink.db[F](store), - PubSubT.noop[F], - EventSink.silent[F] - ) - -} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerConfig.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerConfig.scala deleted file mode 100644 index fc67dbe7..00000000 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerConfig.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.joex.scheduler - -import docspell.common._ - -case class SchedulerConfig( - name: Ident, - poolSize: Int, - countingScheme: CountingScheme, - retries: Int, - retryDelay: Duration, - logBufferSize: Int, - wakeupPeriod: Duration -) - -object SchedulerConfig { - - val default = SchedulerConfig( - name = Ident.unsafe("default-scheduler"), - poolSize = 2 // math.max(2, Runtime.getRuntime.availableProcessors / 2) - , - countingScheme = CountingScheme(2, 1), - retries = 5, - retryDelay = Duration.seconds(30), - logBufferSize = 500, - wakeupPeriod = Duration.minutes(10) - ) -} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerImpl.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerImpl.scala deleted file mode 100644 index a4cc030a..00000000 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerImpl.scala +++ /dev/null @@ -1,351 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.joex.scheduler - -import cats.data.OptionT -import cats.effect._ -import cats.effect.std.Semaphore -import cats.implicits._ -import fs2.Stream -import fs2.concurrent.SignallingRef - -import docspell.backend.msg.JobDone -import docspell.common._ -import docspell.joex.scheduler.SchedulerImpl._ -import docspell.notification.api.Event -import docspell.notification.api.EventSink -import docspell.pubsub.api.PubSubT -import docspell.store.Store -import docspell.store.queries.QJob -import docspell.store.queue.JobQueue -import docspell.store.records.RJob - -import io.circe.Json - -final class SchedulerImpl[F[_]: Async]( - val config: SchedulerConfig, - queue: JobQueue[F], - pubSub: PubSubT[F], - eventSink: EventSink[F], - tasks: JobTaskRegistry[F], - store: Store[F], - logSink: LogSink[F], - state: SignallingRef[F, State[F]], - waiter: SignallingRef[F, Boolean], - permits: Semaphore[F] -) extends Scheduler[F] { - - private[this] val logger = docspell.logging.getLogger[F] - - /** On startup, get all jobs in state running from this scheduler and put them into - * waiting state, so they get picked up again. - */ - def init: F[Unit] = - QJob.runningToWaiting(config.name, store) - - def periodicAwake: F[Fiber[F, Throwable, Unit]] = - Async[F].start( - Stream - .awakeEvery[F](config.wakeupPeriod.toScala) - .evalMap(_ => logger.debug("Periodic awake reached") *> notifyChange) - .compile - .drain - ) - - def getRunning: F[Vector[RJob]] = - state.get.flatMap(s => QJob.findAll(s.getRunning, store)) - - def requestCancel(jobId: Ident): F[Boolean] = - logger.info(s"Scheduler requested to cancel job: ${jobId.id}") *> - state.get.flatMap(_.cancelRequest(jobId) match { - case Some(ct) => ct.map(_ => true) - case None => - (for { - job <- OptionT(store.transact(RJob.findByIdAndWorker(jobId, config.name))) - _ <- OptionT.liftF( - if (job.isInProgress) executeCancel(job) - else ().pure[F] - ) - } yield true) - .getOrElseF( - logger.warn(s"Job ${jobId.id} not found, cannot cancel.").map(_ => false) - ) - }) - - def notifyChange: F[Unit] = - waiter.update(b => !b) - - def shutdown(cancelAll: Boolean): F[Unit] = { - val doCancel = - state.get.flatMap(_.cancelTokens.values.toList.traverse(identity)).map(_ => ()) - - val runShutdown = - state.modify(_.requestShutdown) *> (if (cancelAll) doCancel else ().pure[F]) - - val wait = Stream - .eval(runShutdown) - .evalMap(_ => logger.info("Scheduler is shutting down now.")) - .flatMap(_ => - Stream.eval(state.get) ++ Stream - .suspend(state.discrete.takeWhile(_.getRunning.nonEmpty)) - ) - .flatMap { state => - if (state.getRunning.isEmpty) Stream.eval(logger.info("No jobs running.")) - else - Stream.eval( - logger.info(s"Waiting for ${state.getRunning.size} jobs to finish.") - ) ++ - Stream.emit(state) - } - - (wait.drain ++ Stream.emit(())).compile.lastOrError - } - - def start: Stream[F, Nothing] = - logger.stream.info("Starting scheduler").drain ++ - mainLoop - - def mainLoop: Stream[F, Nothing] = { - val body: F[Boolean] = - for { - _ <- permits.available.flatMap(a => - logger.debug(s"Try to acquire permit ($a free)") - ) - _ <- permits.acquire - _ <- logger.debug("New permit acquired") - down <- state.get.map(_.shutdownRequest) - rjob <- - if (down) - logger.info("") *> permits.release *> (None: Option[RJob]).pure[F] - else - queue.nextJob( - group => state.modify(_.nextPrio(group, config.countingScheme)), - config.name, - config.retryDelay - ) - _ <- logger.debug(s"Next job found: ${rjob.map(_.info)}") - _ <- rjob.map(execute).getOrElse(permits.release) - } yield rjob.isDefined - - Stream - .eval(state.get.map(_.shutdownRequest)) - .evalTap( - if (_) logger.info("Stopping main loop due to shutdown request.") - else ().pure[F] - ) - .flatMap(if (_) Stream.empty else Stream.eval(body)) - .flatMap { - case true => - mainLoop - case false => - logger.stream.debug(s"Waiting for notify").drain ++ - waiter.discrete.take(2).drain ++ - logger.stream.debug(s"Notify signal, going into main loop").drain ++ - mainLoop - } - } - - private def executeCancel(job: RJob): F[Unit] = { - val task = for { - jobtask <- - tasks - .find(job.task) - .toRight(s"This executor cannot run tasks with name: ${job.task}") - } yield jobtask - - task match { - case Left(err) => - logger.error(s"Unable to run cancellation task for job ${job.info}: $err") - case Right(t) => - for { - _ <- - logger.debug(s"Creating context for job ${job.info} to run cancellation $t") - ctx <- Context[F, String](job, job.args, config, logSink, store) - _ <- t.onCancel.run(ctx) - _ <- state.modify(_.markCancelled(job)) - _ <- onFinish(job, JobTaskResult.empty, JobState.Cancelled) - _ <- ctx.logger.warn("Job has been cancelled.") - _ <- logger.debug(s"Job ${job.info} has been cancelled.") - } yield () - } - } - - def execute(job: RJob): F[Unit] = { - val task = for { - jobtask <- - tasks - .find(job.task) - .toRight(s"This executor cannot run tasks with name: ${job.task}") - } yield jobtask - - task match { - case Left(err) => - logger.error(s"Unable to start a task for job ${job.info}: $err") - case Right(t) => - for { - _ <- logger.debug(s"Creating context for job ${job.info} to run $t") - ctx <- Context[F, String](job, job.args, config, logSink, store) - jot = wrapTask(job, t.task, ctx) - tok <- forkRun(job, jot.run(ctx), t.onCancel.run(ctx), ctx) - _ <- state.modify(_.addRunning(job, tok)) - } yield () - } - } - - def onFinish(job: RJob, result: JobTaskResult, finishState: JobState): F[Unit] = - for { - _ <- logger.debug(s"Job ${job.info} done $finishState. Releasing resources.") - _ <- permits.release *> permits.available.flatMap(a => - logger.debug(s"Permit released ($a free)") - ) - _ <- state.modify(_.removeRunning(job)) - _ <- QJob.setFinalState(job.id, finishState, store) - _ <- Sync[F].whenA(JobState.isDone(finishState))( - pubSub.publish1IgnoreErrors( - JobDone.topic, - JobDone(job.id, job.group, job.task, job.args, finishState) - ) - ) - _ <- Sync[F].whenA(JobState.isDone(finishState))( - eventSink.offer( - Event.JobDone( - job.id, - job.group, - job.task, - job.args, - job.state, - job.subject, - job.submitter, - result.json.getOrElse(Json.Null), - result.message - ) - ) - ) - } yield () - - def onStart(job: RJob): F[Unit] = - QJob.setRunning( - job.id, - config.name, - store - ) // also increments retries if current state=stuck - - def wrapTask( - job: RJob, - task: Task[F, String, JobTaskResult], - ctx: Context[F, String] - ): Task[F, String, Unit] = - task - .mapF(fa => onStart(job) *> logger.debug("Starting task now") *> fa) - .mapF(_.attempt.flatMap { - case Right(result) => - logger.info(s"Job execution successful: ${job.info}") - ctx.logger.info("Job execution successful") *> - (JobState.Success: JobState, result).pure[F] - case Left(ex) => - state.get.map(_.wasCancelled(job)).flatMap { - case true => - logger.error(ex)(s"Job ${job.info} execution failed (cancel = true)") - ctx.logger.error(ex)("Job execution failed (cancel = true)") *> - (JobState.Cancelled: JobState, JobTaskResult.empty).pure[F] - case false => - QJob.exceedsRetries(job.id, config.retries, store).flatMap { - case true => - logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") - ctx.logger - .error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") - .map(_ => (JobState.Failed: JobState, JobTaskResult.empty)) - case false => - logger.error(ex)(s"Job ${job.info} execution failed. Retrying later.") - ctx.logger - .error(ex)(s"Job ${job.info} execution failed. Retrying later.") - .map(_ => (JobState.Stuck: JobState, JobTaskResult.empty)) - } - } - }) - .mapF(_.attempt.flatMap { - case Right((jstate, result)) => - onFinish(job, result, jstate) - case Left(ex) => - logger.error(ex)(s"Error happened during post-processing of ${job.info}!") - // we don't know the real outcome here… - // since tasks should be idempotent, set it to stuck. if above has failed, this might fail anyways - onFinish(job, JobTaskResult.empty, JobState.Stuck) - }) - - def forkRun( - job: RJob, - code: F[Unit], - onCancel: F[Unit], - ctx: Context[F, String] - ): F[F[Unit]] = - logger.debug(s"Forking job ${job.info}") *> - Async[F] - .start(code) - .map(fiber => - logger.debug(s"Cancelling job ${job.info}") *> - fiber.cancel *> - onCancel.attempt.map { - case Right(_) => () - case Left(ex) => - logger.error(ex)(s"Task's cancelling code failed. Job ${job.info}.") - () - } *> - state.modify(_.markCancelled(job)) *> - onFinish(job, JobTaskResult.empty, JobState.Cancelled) *> - ctx.logger.warn("Job has been cancelled.") *> - logger.debug(s"Job ${job.info} has been cancelled.") - ) -} - -object SchedulerImpl { - - type CancelToken[F[_]] = F[Unit] - - def emptyState[F[_]]: State[F] = - State(Map.empty, Set.empty, Map.empty, false) - - case class State[F[_]]( - counters: Map[Ident, CountingScheme], - cancelled: Set[Ident], - cancelTokens: Map[Ident, CancelToken[F]], - shutdownRequest: Boolean - ) { - - def nextPrio(group: Ident, initial: CountingScheme): (State[F], Priority) = { - val (cs, prio) = counters.getOrElse(group, initial).nextPriority - (copy(counters = counters.updated(group, cs)), prio) - } - - def addRunning(job: RJob, token: CancelToken[F]): (State[F], Unit) = - ( - State(counters, cancelled, cancelTokens.updated(job.id, token), shutdownRequest), - () - ) - - def removeRunning(job: RJob): (State[F], Unit) = - ( - copy(cancelled = cancelled - job.id, cancelTokens = cancelTokens.removed(job.id)), - () - ) - - def markCancelled(job: RJob): (State[F], Unit) = - (copy(cancelled = cancelled + job.id), ()) - - def wasCancelled(job: RJob): Boolean = - cancelled.contains(job.id) - - def cancelRequest(id: Ident): Option[F[Unit]] = - cancelTokens.get(id) - - def getRunning: Seq[Ident] = - cancelTokens.keys.toSeq - - def requestShutdown: (State[F], Unit) = - (copy(shutdownRequest = true), ()) - } -} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/Task.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/Task.scala deleted file mode 100644 index 4ba4d84d..00000000 --- a/modules/joex/src/main/scala/docspell/joex/scheduler/Task.scala +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.joex.scheduler - -import cats._ -import cats.data.Kleisli -import cats.effect.Sync -import cats.implicits._ - -import docspell.logging.Logger - -/** The code that is executed by the scheduler */ -trait Task[F[_], A, B] { - - def run(ctx: Context[F, A]): F[B] - - def andThen[C](f: B => F[C])(implicit F: FlatMap[F]): Task[F, A, C] = - Task(Task.toKleisli(this).andThen(f)) - - def mapF[C](f: F[B] => F[C]): Task[F, A, C] = - Task(Task.toKleisli(this).mapF(f)) - - def attempt(implicit - F: ApplicativeError[F, Throwable] - ): Task[F, A, Either[Throwable, B]] = - mapF(_.attempt) - - def contramap[C](f: C => F[A])(implicit F: FlatMap[F]): Task[F, C, B] = { - ctxc: Context[F, C] => f(ctxc.args).flatMap(a => run(ctxc.map(_ => a))) - } -} - -object Task { - - def pure[F[_]: Applicative, A, B](b: B): Task[F, A, B] = - Task(_ => b.pure[F]) - - def of[F[_], A, B](b: F[B]): Task[F, A, B] = - Task(_ => b) - - def apply[F[_], A, B](f: Context[F, A] => F[B]): Task[F, A, B] = - (ctx: Context[F, A]) => f(ctx) - - def apply[F[_], A, B](k: Kleisli[F, Context[F, A], B]): Task[F, A, B] = - c => k.run(c) - - def toKleisli[F[_], A, B](t: Task[F, A, B]): Kleisli[F, Context[F, A], B] = - Kleisli(t.run) - - def setProgress[F[_]: Sync, A, B](n: Int)(data: B): Task[F, A, B] = - Task(_.setProgress(n).map(_ => data)) - - def log[F[_], A](f: Logger[F] => F[Unit]): Task[F, A, Unit] = - Task(ctx => f(ctx.logger)) - - implicit def taskMonad[F[_]: Monad, T]: Monad[Task[F, T, *]] = - new Monad[Task[F, T, *]] { - def pure[A](x: A) = Task(_ => Monad[F].pure(x)) - def flatMap[A, B](fa: Task[F, T, A])(f: A => Task[F, T, B]) = - Task(Task.toKleisli(fa).flatMap(a => Task.toKleisli(f(a)))) - - def tailRecM[A, B](a: A)(f: A => Task[F, T, Either[A, B]]) = { - val monadK = Monad[Kleisli[F, Context[F, T], *]] - val r = monadK.tailRecM(a)(x => Task.toKleisli(f(x))) - Task(r) - } - } -} diff --git a/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala b/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala index c89b9fde..7a82f65b 100644 --- a/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala @@ -11,8 +11,8 @@ import cats.effect._ import cats.implicits._ import docspell.common._ -import docspell.joex.scheduler.Context -import docspell.joex.scheduler.Task +import docspell.scheduler.Context +import docspell.scheduler.Task import docspell.store.records.RPeriodicTask import docspell.store.records.RUserEmail import docspell.store.usertask.UserTask From 83d3644b39d22058c5a7c60b7816c7f64334f70a Mon Sep 17 00:00:00 2001 From: eikek Date: Sat, 12 Mar 2022 15:31:27 +0100 Subject: [PATCH 12/16] Move job queue to scheduler-api and fix notification of periodic tasks --- build.sbt | 2 +- .../scala/docspell/backend/BackendApp.scala | 4 +-- .../scala/docspell/backend/msg/Topics.scala | 16 +++++++----- .../docspell/backend/ops/OCollective.scala | 4 +-- .../backend/ops/OFileRepository.scala | 4 +-- .../docspell/backend/ops/OFulltext.scala | 3 +-- .../scala/docspell/backend/ops/OItem.scala | 4 +-- .../scala/docspell/backend/ops/OJob.scala | 3 +-- .../scala/docspell/backend/ops/OJoex.scala | 10 ++++--- .../scala/docspell/backend/ops/OUpload.scala | 3 +-- .../docspell/backend/ops/OUserTask.scala | 12 ++++----- .../scala/docspell/joex/JoexAppImpl.scala | 26 +++++++------------ .../joex/pagecount/AllPageCountTask.scala | 5 +--- .../joex/pdfconv/ConvertAllPdfTask.scala | 4 +-- .../joex/preview/AllPreviewsTask.scala | 5 +--- .../docspell/restserver/Subscriptions.scala | 3 +-- .../scala/docspell/scheduler}/JobQueue.scala | 5 ++-- .../scheduler/PeriodicScheduler.scala | 5 ++-- .../scala/docspell/scheduler/Scheduler.scala | 4 +++ .../docspell/scheduler}/msg/CancelJob.scala | 3 +-- .../scheduler}/msg/JobQueuePublish.scala | 10 +++---- .../scheduler}/msg/JobSubmitted.scala | 4 +-- .../docspell/scheduler/msg/JobsNotify.scala | 9 +++++++ .../scheduler/msg/PeriodicTaskNotify.scala | 9 +++++++ .../docspell/scheduler/impl}/Marked.scala | 2 +- .../impl/PeriodicSchedulerBuilder.scala | 6 ++--- .../impl/PeriodicSchedulerImpl.scala | 16 ++++++++---- .../scheduler/impl}/PeriodicTaskStore.scala | 5 ++-- .../scheduler/impl/SchedulerBuilder.scala | 6 ++--- .../scheduler/impl/SchedulerImpl.scala | 17 ++++++++---- .../store/usertask/UserTaskStore.scala | 2 +- 31 files changed, 108 insertions(+), 103 deletions(-) rename modules/{store/src/main/scala/docspell/store/queue => scheduler/api/src/main/scala/docspell/scheduler}/JobQueue.scala (95%) rename modules/{backend/src/main/scala/docspell/backend => scheduler/api/src/main/scala/docspell/scheduler}/msg/CancelJob.scala (95%) rename modules/{backend/src/main/scala/docspell/backend => scheduler/api/src/main/scala/docspell/scheduler}/msg/JobQueuePublish.scala (89%) rename modules/{backend/src/main/scala/docspell/backend => scheduler/api/src/main/scala/docspell/scheduler}/msg/JobSubmitted.scala (95%) create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobsNotify.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/msg/PeriodicTaskNotify.scala rename modules/{store/src/main/scala/docspell/store/queue => scheduler/impl/src/main/scala/docspell/scheduler/impl}/Marked.scala (93%) rename modules/{store/src/main/scala/docspell/store/queue => scheduler/impl/src/main/scala/docspell/scheduler/impl}/PeriodicTaskStore.scala (97%) diff --git a/build.sbt b/build.sbt index 9eb54d39..91c21a17 100644 --- a/build.sbt +++ b/build.sbt @@ -530,7 +530,7 @@ val schedulerApi = project Dependencies.fs2Core ++ Dependencies.circeCore ) - .dependsOn(loggingApi, common, store, pubsubApi) + .dependsOn(loggingApi, common, store, notificationApi, pubsubApi) val schedulerImpl = project .in(file("modules/scheduler/impl")) diff --git a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala index f3142016..64ce4b9b 100644 --- a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala +++ b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala @@ -7,19 +7,17 @@ package docspell.backend import cats.effect._ - import docspell.backend.auth.Login import docspell.backend.fulltext.CreateIndex -import docspell.backend.msg.JobQueuePublish import docspell.backend.ops._ import docspell.backend.signup.OSignup import docspell.ftsclient.FtsClient import docspell.notification.api.{EventExchange, NotificationModule} import docspell.pubsub.api.PubSubT +import docspell.scheduler.msg.JobQueuePublish import docspell.store.Store import docspell.store.usertask.UserTaskStore import docspell.totp.Totp - import emil.Emil trait BackendApp[F[_]] { diff --git a/modules/backend/src/main/scala/docspell/backend/msg/Topics.scala b/modules/backend/src/main/scala/docspell/backend/msg/Topics.scala index bda187d1..3a887097 100644 --- a/modules/backend/src/main/scala/docspell/backend/msg/Topics.scala +++ b/modules/backend/src/main/scala/docspell/backend/msg/Topics.scala @@ -7,17 +7,19 @@ package docspell.backend.msg import cats.data.NonEmptyList -import docspell.pubsub.api.{Topic, TypedTopic} -import docspell.scheduler.msg.JobDone +import docspell.pubsub.api.TypedTopic +import docspell.scheduler.msg._ /** All topics used in Docspell. */ object Topics { - /** A generic notification to the job executors to look for new work. */ - val jobsNotify: TypedTopic[Unit] = - TypedTopic[Unit](Topic("jobs-notify")) - /** A list of all topics. It is required to list every topic in use here! */ val all: NonEmptyList[TypedTopic[_]] = - NonEmptyList.of(JobDone.topic, CancelJob.topic, jobsNotify, JobSubmitted.topic) + NonEmptyList.of( + JobDone.topic, + CancelJob.topic, + JobsNotify(), + JobSubmitted.topic, + PeriodicTaskNotify() + ) } diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala index 907bfcef..f72ba217 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala @@ -9,19 +9,17 @@ package docspell.backend.ops import cats.effect.{Async, Resource} import cats.implicits._ import fs2.Stream - import docspell.backend.JobFactory import docspell.backend.PasswordCrypt import docspell.backend.ops.OCollective._ import docspell.common._ import docspell.store.UpdateResult import docspell.store.queries.{QCollective, QUser} -import docspell.store.queue.JobQueue import docspell.store.records._ import docspell.store.usertask.{UserTask, UserTaskScope, UserTaskStore} import docspell.store.{AddResult, Store} - import com.github.eikek.calev._ +import docspell.scheduler.JobQueue trait OCollective[F[_]] { diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala index aec12823..12ea8191 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala @@ -9,14 +9,12 @@ package docspell.backend.ops import cats.data.OptionT import cats.effect._ import cats.implicits._ - import docspell.backend.JobFactory import docspell.backend.ops.OFileRepository.IntegrityResult import docspell.common._ +import docspell.scheduler.JobQueue import docspell.store.Store -import docspell.store.queue.JobQueue import docspell.store.records.RJob - import scodec.bits.ByteVector trait OFileRepository[F[_]] { diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala index 9d057f1c..88b9c3e3 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala @@ -10,15 +10,14 @@ import cats.data.NonEmptyList import cats.effect._ import cats.implicits._ import fs2.Stream - import docspell.backend.JobFactory import docspell.backend.ops.OItemSearch._ import docspell.common._ import docspell.ftsclient._ import docspell.query.ItemQuery._ import docspell.query.ItemQueryDsl._ +import docspell.scheduler.JobQueue import docspell.store.queries.{QFolder, QItem, SelectedItem} -import docspell.store.queue.JobQueue import docspell.store.records.RJob import docspell.store.{Store, qb} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala index f625cdfe..41a48e7a 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala @@ -9,7 +9,6 @@ package docspell.backend.ops import cats.data.{NonEmptyList => Nel, OptionT} import cats.effect.{Async, Resource} import cats.implicits._ - import docspell.backend.AttachedEvent import docspell.backend.JobFactory import docspell.backend.fulltext.CreateIndex @@ -18,11 +17,10 @@ import docspell.common._ import docspell.ftsclient.FtsClient import docspell.logging.Logger import docspell.notification.api.Event +import docspell.scheduler.JobQueue import docspell.store.queries.{QAttachment, QItem, QMoveAttachment} -import docspell.store.queue.JobQueue import docspell.store.records._ import docspell.store.{AddResult, Store, UpdateResult} - import doobie.implicits._ trait OItem[F[_]] { diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala b/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala index b4b0b7ae..ad1cc8c8 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala @@ -9,11 +9,10 @@ package docspell.backend.ops import cats.data.OptionT import cats.effect._ import cats.implicits._ - -import docspell.backend.msg.JobDone import docspell.backend.ops.OJob.{CollectiveQueueState, JobCancelResult} import docspell.common._ import docspell.pubsub.api.PubSubT +import docspell.scheduler.msg.JobDone import docspell.store.Store import docspell.store.UpdateResult import docspell.store.queries.QJob diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala b/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala index e51e8bd6..f82285d9 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala @@ -9,15 +9,16 @@ package docspell.backend.ops import cats.Applicative import cats.effect._ import cats.implicits._ - -import docspell.backend.msg.{CancelJob, Topics} import docspell.common.Ident import docspell.pubsub.api.PubSubT +import docspell.scheduler.msg.{CancelJob, JobsNotify, PeriodicTaskNotify} trait OJoex[F[_]] { def notifyAllNodes: F[Unit] + def notifyPeriodicTasks: F[Unit] + def cancelJob(job: Ident, worker: Ident): F[Unit] } @@ -26,7 +27,10 @@ object OJoex { Resource.pure[F, OJoex[F]](new OJoex[F] { def notifyAllNodes: F[Unit] = - pubSub.publish1IgnoreErrors(Topics.jobsNotify, ()).as(()) + pubSub.publish1IgnoreErrors(JobsNotify(), ()).void + + def notifyPeriodicTasks: F[Unit] = + pubSub.publish1IgnoreErrors(PeriodicTaskNotify(), ()).void def cancelJob(job: Ident, worker: Ident): F[Unit] = pubSub.publish1IgnoreErrors(CancelJob.topic, CancelJob(job, worker)).as(()) diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala index dbda65b2..4e1e2dd4 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala @@ -11,11 +11,10 @@ import cats.data.{EitherT, OptionT} import cats.effect._ import cats.implicits._ import fs2.Stream - import docspell.backend.JobFactory import docspell.common._ +import docspell.scheduler.JobQueue import docspell.store.Store -import docspell.store.queue.JobQueue import docspell.store.records._ trait OUpload[F[_]] { diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala b/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala index 8f8d0ab3..3c5ade36 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala @@ -10,14 +10,12 @@ import cats.data.{NonEmptyList, OptionT} import cats.effect._ import cats.implicits._ import fs2.Stream - import docspell.common._ import docspell.notification.api.{ChannelRef, PeriodicDueItemsArgs, PeriodicQueryArgs} +import docspell.scheduler.JobQueue import docspell.store.Store -import docspell.store.queue.JobQueue import docspell.store.records.RNotificationChannel import docspell.store.usertask._ - import io.circe.Encoder trait OUserTask[F[_]] { @@ -98,7 +96,7 @@ object OUserTask { ptask <- task.encode.toPeriodicTask(scope, subject) job <- ptask.toJob _ <- queue.insert(job) - _ <- joex.notifyAllNodes + _ <- joex.notifyPeriodicTasks } yield () def getScanMailbox(scope: UserTaskScope): Stream[F, UserTask[ScanMailboxArgs]] = @@ -124,7 +122,7 @@ object OUserTask { ): F[Unit] = for { _ <- taskStore.updateTask[ScanMailboxArgs](scope, subject, task) - _ <- joex.notifyAllNodes + _ <- joex.notifyPeriodicTasks } yield () def getNotifyDueItems( @@ -153,7 +151,7 @@ object OUserTask { ): F[Unit] = for { _ <- taskStore.updateTask[PeriodicDueItemsArgs](scope, subject, task) - _ <- joex.notifyAllNodes + _ <- joex.notifyPeriodicTasks } yield () def getPeriodicQuery(scope: UserTaskScope): Stream[F, UserTask[PeriodicQueryArgs]] = @@ -180,7 +178,7 @@ object OUserTask { ): F[Unit] = for { _ <- taskStore.updateTask[PeriodicQueryArgs](scope, subject, task) - _ <- joex.notifyAllNodes + _ <- joex.notifyPeriodicTasks } yield () // When retrieving arguments containing channel references, we must update diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala index 7ba81536..3cb650c0 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala @@ -12,7 +12,6 @@ import fs2.concurrent.SignallingRef import docspell.analysis.TextAnalyser import docspell.backend.MailAddressCodec import docspell.backend.fulltext.CreateIndex -import docspell.backend.msg.{CancelJob, JobQueuePublish, Topics} import docspell.backend.ops._ import docspell.common._ import docspell.ftsclient.FtsClient @@ -32,13 +31,17 @@ import docspell.joex.process.ItemHandler import docspell.joex.process.ReProcessItem import docspell.joex.scanmailbox._ import docspell.scheduler._ -import docspell.scheduler.impl.{PeriodicSchedulerBuilder, SchedulerBuilder} +import docspell.scheduler.impl.{ + PeriodicSchedulerBuilder, + PeriodicTaskStore, + SchedulerBuilder +} import docspell.joex.updatecheck._ import docspell.notification.api.NotificationModule import docspell.notification.impl.NotificationModuleImpl import docspell.pubsub.api.{PubSub, PubSubT} +import docspell.scheduler.msg.JobQueuePublish import docspell.store.Store -import docspell.store.queue._ import docspell.store.records.{REmptyTrashSetting, RJobLog} import docspell.store.usertask.UserTaskScope import docspell.store.usertask.UserTaskStore @@ -49,7 +52,6 @@ final class JoexAppImpl[F[_]: Async]( cfg: Config, store: Store[F], queue: JobQueue[F], - pubSubT: PubSubT[F], pstore: PeriodicTaskStore[F], termSignal: SignallingRef[F, Boolean], notificationMod: NotificationModule[F], @@ -67,20 +69,11 @@ final class JoexAppImpl[F[_]: Async]( _ <- Async[F].start(eventConsume) _ <- scheduler.periodicAwake _ <- periodicScheduler.periodicAwake - _ <- subscriptions + _ <- scheduler.startSubscriptions + _ <- periodicScheduler.startSubscriptions } yield () } - def subscriptions = - for { - _ <- Async[F].start(pubSubT.subscribeSink(Topics.jobsNotify) { _ => - scheduler.notifyChange - }) - _ <- Async[F].start(pubSubT.subscribeSink(CancelJob.topic) { msg => - scheduler.requestCancel(msg.body.jobId).as(()) - }) - } yield () - def findLogs(jobId: Ident): F[Vector[RJobLog]] = store.transact(RJobLog.findLogs(jobId)) @@ -300,13 +293,12 @@ object JoexAppImpl extends MailAddressCodec { sch, queue, pstore, - joex.notifyAllNodes + pubSubT ) app = new JoexAppImpl( cfg, store, queue, - pubSubT, pstore, termSignal, notificationMod, diff --git a/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala b/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala index 99b315b7..0fb9e299 100644 --- a/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala @@ -9,13 +9,10 @@ package docspell.joex.pagecount import cats.effect._ import cats.implicits._ import fs2.{Chunk, Stream} - import docspell.backend.JobFactory import docspell.backend.ops.OJoex import docspell.common._ -import docspell.scheduler.Context -import docspell.scheduler.Task -import docspell.store.queue.JobQueue +import docspell.scheduler.{Context, JobQueue, Task} import docspell.store.records.RAttachment import docspell.store.records.RJob diff --git a/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala b/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala index e4def9ac..a0865156 100644 --- a/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala @@ -9,11 +9,9 @@ package docspell.joex.pdfconv import cats.effect._ import cats.implicits._ import fs2.{Chunk, Stream} - import docspell.backend.ops.OJoex import docspell.common._ -import docspell.scheduler.{Context, Task} -import docspell.store.queue.JobQueue +import docspell.scheduler.{Context, JobQueue, Task} import docspell.store.records.RAttachment import docspell.store.records._ diff --git a/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala b/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala index 15616ed5..849b4f7a 100644 --- a/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala @@ -9,14 +9,11 @@ package docspell.joex.preview import cats.effect._ import cats.implicits._ import fs2.{Chunk, Stream} - import docspell.backend.JobFactory import docspell.backend.ops.OJoex import docspell.common.MakePreviewArgs.StoreMode import docspell.common._ -import docspell.scheduler.Context -import docspell.scheduler.Task -import docspell.store.queue.JobQueue +import docspell.scheduler.{Context, JobQueue, Task} import docspell.store.records.RAttachment import docspell.store.records.RJob diff --git a/modules/restserver/src/main/scala/docspell/restserver/Subscriptions.scala b/modules/restserver/src/main/scala/docspell/restserver/Subscriptions.scala index faecac11..e1cafa20 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/Subscriptions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/Subscriptions.scala @@ -9,10 +9,9 @@ package docspell.restserver import cats.effect.Async import fs2.Stream import fs2.concurrent.Topic - -import docspell.backend.msg.{JobDone, JobSubmitted} import docspell.pubsub.api.PubSubT import docspell.restserver.ws.OutputEvent +import docspell.scheduler.msg.{JobDone, JobSubmitted} /** Subscribes to those events from docspell that are forwarded to the websocket endpoints */ diff --git a/modules/store/src/main/scala/docspell/store/queue/JobQueue.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobQueue.scala similarity index 95% rename from modules/store/src/main/scala/docspell/store/queue/JobQueue.scala rename to modules/scheduler/api/src/main/scala/docspell/scheduler/JobQueue.scala index 9b777086..dd927bf9 100644 --- a/modules/store/src/main/scala/docspell/store/queue/JobQueue.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobQueue.scala @@ -4,11 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -package docspell.store.queue +package docspell.scheduler import cats.effect._ import cats.implicits._ - import docspell.common._ import docspell.store.Store import docspell.store.queries.QJob @@ -40,7 +39,7 @@ trait JobQueue[F[_]] { } object JobQueue { - def apply[F[_]: Async](store: Store[F]): Resource[F, JobQueue[F]] = + private[scheduler] def create[F[_]: Async](store: Store[F]): Resource[F, JobQueue[F]] = Resource.pure[F, JobQueue[F]](new JobQueue[F] { private[this] val logger = docspell.logging.getLogger[F] diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/PeriodicScheduler.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/PeriodicScheduler.scala index 284a9465..3a633a88 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/PeriodicScheduler.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/PeriodicScheduler.scala @@ -27,6 +27,7 @@ trait PeriodicScheduler[F[_]] { def periodicAwake: F[Fiber[F, Throwable, Unit]] def notifyChange: F[Unit] -} -object PeriodicScheduler {} + /** Starts listening for notify messages in the background. */ + def startSubscriptions: F[Unit] +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/Scheduler.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/Scheduler.scala index 720214c7..0e44244b 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/Scheduler.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/Scheduler.scala @@ -22,6 +22,10 @@ trait Scheduler[F[_]] { def notifyChange: F[Unit] + /** Starts reacting on notify and cancel messages. */ + def startSubscriptions: F[Unit] + + /** Starts the schedulers main loop. */ def start: Stream[F, Nothing] /** Requests to shutdown the scheduler. diff --git a/modules/backend/src/main/scala/docspell/backend/msg/CancelJob.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/CancelJob.scala similarity index 95% rename from modules/backend/src/main/scala/docspell/backend/msg/CancelJob.scala rename to modules/scheduler/api/src/main/scala/docspell/scheduler/msg/CancelJob.scala index 677f1fba..3ec0c891 100644 --- a/modules/backend/src/main/scala/docspell/backend/msg/CancelJob.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/CancelJob.scala @@ -4,11 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -package docspell.backend.msg +package docspell.scheduler.msg import docspell.common._ import docspell.pubsub.api.{Topic, TypedTopic} - import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} import io.circe.{Decoder, Encoder} diff --git a/modules/backend/src/main/scala/docspell/backend/msg/JobQueuePublish.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobQueuePublish.scala similarity index 89% rename from modules/backend/src/main/scala/docspell/backend/msg/JobQueuePublish.scala rename to modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobQueuePublish.scala index d4011943..7c867a22 100644 --- a/modules/backend/src/main/scala/docspell/backend/msg/JobQueuePublish.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobQueuePublish.scala @@ -4,17 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -package docspell.backend.msg +package docspell.scheduler.msg import cats.effect._ import cats.implicits._ - import docspell.common.{Duration, Ident, Priority} -import docspell.notification.api.Event -import docspell.notification.api.EventSink +import docspell.notification.api.{Event, EventSink} import docspell.pubsub.api.PubSubT +import docspell.scheduler.JobQueue import docspell.store.Store -import docspell.store.queue.JobQueue import docspell.store.records.RJob final class JobQueuePublish[F[_]: Sync]( @@ -76,5 +74,5 @@ object JobQueuePublish { pubSub: PubSubT[F], eventSink: EventSink[F] ): Resource[F, JobQueue[F]] = - JobQueue(store).map(q => new JobQueuePublish[F](q, pubSub, eventSink)) + JobQueue.create(store).map(q => new JobQueuePublish[F](q, pubSub, eventSink)) } diff --git a/modules/backend/src/main/scala/docspell/backend/msg/JobSubmitted.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobSubmitted.scala similarity index 95% rename from modules/backend/src/main/scala/docspell/backend/msg/JobSubmitted.scala rename to modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobSubmitted.scala index 2cecdde8..c6ef40b2 100644 --- a/modules/backend/src/main/scala/docspell/backend/msg/JobSubmitted.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobSubmitted.scala @@ -4,10 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -package docspell.backend.msg +package docspell.scheduler.msg + import docspell.common._ import docspell.pubsub.api.{Topic, TypedTopic} - import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} import io.circe.{Decoder, Encoder} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobsNotify.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobsNotify.scala new file mode 100644 index 00000000..8bc9fc8d --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobsNotify.scala @@ -0,0 +1,9 @@ +package docspell.scheduler.msg + +import docspell.pubsub.api.{Topic, TypedTopic} + +/** A generic notification to the job executors to look for new work. */ +object JobsNotify { + def apply(): TypedTopic[Unit] = + TypedTopic[Unit](Topic("jobs-notify")) +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/PeriodicTaskNotify.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/PeriodicTaskNotify.scala new file mode 100644 index 00000000..d8f5157e --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/PeriodicTaskNotify.scala @@ -0,0 +1,9 @@ +package docspell.scheduler.msg + +import docspell.pubsub.api.{Topic, TypedTopic} + +/** A generic notification to the periodic task scheduler to look for new work. */ +object PeriodicTaskNotify { + def apply(): TypedTopic[Unit] = + TypedTopic[Unit](Topic("periodic-task-notify")) +} diff --git a/modules/store/src/main/scala/docspell/store/queue/Marked.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/Marked.scala similarity index 93% rename from modules/store/src/main/scala/docspell/store/queue/Marked.scala rename to modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/Marked.scala index af7f32db..27e9eb7d 100644 --- a/modules/store/src/main/scala/docspell/store/queue/Marked.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/Marked.scala @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -package docspell.store.queue +package docspell.scheduler.impl sealed trait Marked[+A] {} diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerBuilder.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerBuilder.scala index 5fbaada2..f4d1d7ec 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerBuilder.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerBuilder.scala @@ -1,8 +1,8 @@ package docspell.scheduler.impl import cats.effect._ +import docspell.pubsub.api.PubSubT import docspell.scheduler._ -import docspell.store.queue.{JobQueue, PeriodicTaskStore} import fs2.concurrent.SignallingRef object PeriodicSchedulerBuilder { @@ -12,7 +12,7 @@ object PeriodicSchedulerBuilder { sch: Scheduler[F], queue: JobQueue[F], store: PeriodicTaskStore[F], - notifyJoex: F[Unit] + pubsub: PubSubT[F] ): Resource[F, PeriodicScheduler[F]] = for { waiter <- Resource.eval(SignallingRef(true)) @@ -22,7 +22,7 @@ object PeriodicSchedulerBuilder { sch, queue, store, - notifyJoex, + pubsub, waiter, state ) diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerImpl.scala index efdefeaf..fd1a3e0d 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerImpl.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerImpl.scala @@ -10,13 +10,12 @@ import cats.effect._ import cats.implicits._ import fs2._ import fs2.concurrent.SignallingRef - import docspell.common._ +import docspell.pubsub.api.PubSubT import docspell.scheduler._ import docspell.scheduler.impl.PeriodicSchedulerImpl.State -import docspell.store.queue._ +import docspell.scheduler.msg.{JobsNotify, PeriodicTaskNotify} import docspell.store.records.RPeriodicTask - import eu.timepit.fs2cron.calev.CalevScheduler final class PeriodicSchedulerImpl[F[_]: Async]( @@ -24,7 +23,7 @@ final class PeriodicSchedulerImpl[F[_]: Async]( sch: Scheduler[F], queue: JobQueue[F], store: PeriodicTaskStore[F], - joexNotifyAll: F[Unit], + pubSub: PubSubT[F], waiter: SignallingRef[F, Boolean], state: SignallingRef[F, State[F]] ) extends PeriodicScheduler[F] { @@ -49,6 +48,13 @@ final class PeriodicSchedulerImpl[F[_]: Async]( def notifyChange: F[Unit] = waiter.update(b => !b) + def startSubscriptions: F[Unit] = + for { + _ <- Async[F].start(pubSub.subscribeSink(PeriodicTaskNotify()) { _ => + logger.info("Notify periodic scheduler from message") *> notifyChange + }) + } yield () + // internal /** On startup, get all periodic jobs from this scheduler and remove the mark, so they @@ -117,7 +123,7 @@ final class PeriodicSchedulerImpl[F[_]: Async]( } def notifyJoex: F[Unit] = - sch.notifyChange *> joexNotifyAll + sch.notifyChange *> pubSub.publish1IgnoreErrors(JobsNotify(), ()).void def scheduleNotify(pj: RPeriodicTask): F[Unit] = Timestamp diff --git a/modules/store/src/main/scala/docspell/store/queue/PeriodicTaskStore.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicTaskStore.scala similarity index 97% rename from modules/store/src/main/scala/docspell/store/queue/PeriodicTaskStore.scala rename to modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicTaskStore.scala index f1fad91f..54846b1f 100644 --- a/modules/store/src/main/scala/docspell/store/queue/PeriodicTaskStore.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicTaskStore.scala @@ -4,11 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -package docspell.store.queue +package docspell.scheduler.impl import cats.effect._ import cats.implicits._ - import docspell.common._ import docspell.store.queries.QPeriodicTask import docspell.store.records._ @@ -57,7 +56,7 @@ object PeriodicTaskStore { case false => Marked.notMarkable } case None => - Marked.notFound.pure[F] + Marked.notFound[RPeriodicTask].pure[F] } Resource.make(chooseNext) { diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerBuilder.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerBuilder.scala index 4e51231b..6cd3cc57 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerBuilder.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerBuilder.scala @@ -10,12 +10,10 @@ import cats.effect._ import cats.effect.std.Semaphore import cats.implicits._ import fs2.concurrent.SignallingRef - -import docspell.scheduler._ +import docspell.scheduler.{JobQueue, _} import docspell.notification.api.EventSink import docspell.pubsub.api.PubSubT import docspell.store.Store -import docspell.store.queue.JobQueue case class SchedulerBuilder[F[_]: Async]( config: SchedulerConfig, @@ -88,7 +86,7 @@ object SchedulerBuilder { config, JobTaskRegistry.empty[F], store, - JobQueue(store), + JobQueue.create(store), LogSink.db[F](store), PubSubT.noop[F], EventSink.silent[F] diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala index d0de5991..859cbde2 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala @@ -12,19 +12,16 @@ import cats.effect.std.Semaphore import cats.implicits._ import fs2.Stream import fs2.concurrent.SignallingRef - -import docspell.scheduler.msg.JobDone +import docspell.scheduler.msg.{CancelJob, JobDone, JobsNotify} import docspell.common._ -import docspell.scheduler._ +import docspell.scheduler.{JobQueue, _} import docspell.scheduler.impl.SchedulerImpl._ import docspell.notification.api.Event import docspell.notification.api.EventSink import docspell.pubsub.api.PubSubT import docspell.store.Store import docspell.store.queries.QJob -import docspell.store.queue.JobQueue import docspell.store.records.RJob - import io.circe.Json final class SchedulerImpl[F[_]: Async]( @@ -42,6 +39,16 @@ final class SchedulerImpl[F[_]: Async]( private[this] val logger = docspell.logging.getLogger[F] + def startSubscriptions = + for { + _ <- Async[F].start(pubSub.subscribeSink(JobsNotify()) { _ => + notifyChange + }) + _ <- Async[F].start(pubSub.subscribeSink(CancelJob.topic) { msg => + requestCancel(msg.body.jobId).void + }) + } yield () + /** On startup, get all jobs in state running from this scheduler and put them into * waiting state, so they get picked up again. */ diff --git a/modules/store/src/main/scala/docspell/store/usertask/UserTaskStore.scala b/modules/store/src/main/scala/docspell/store/usertask/UserTaskStore.scala index 96a08351..e19fa035 100644 --- a/modules/store/src/main/scala/docspell/store/usertask/UserTaskStore.scala +++ b/modules/store/src/main/scala/docspell/store/usertask/UserTaskStore.scala @@ -89,7 +89,7 @@ trait UserTaskStore[F[_]] { implicit E: Encoder[A] ): F[UserTask[String]] - /** Delete all tasks of the given user that have name `name'. */ + /** Delete all tasks of the given user that have name `name`. */ def deleteAll(scope: UserTaskScope, name: Ident): F[Int] } From 69765f05ff7887059a162bc20c02387af1046fab Mon Sep 17 00:00:00 2001 From: eikek Date: Sat, 12 Mar 2022 15:45:21 +0100 Subject: [PATCH 13/16] Move user task to scheduler module --- .../scala/docspell/backend/BackendApp.scala | 2 +- .../docspell/backend/ops/OCollective.scala | 2 +- .../docspell/backend/ops/OUserTask.scala | 2 +- .../scala/docspell/joex/JoexAppImpl.scala | 3 +- .../joex/emptytrash/EmptyTrashTask.scala | 4 +- .../docspell/joex/hk/HouseKeepingTask.scala | 6 +- .../joex/updatecheck/UpdateCheckTask.scala | 5 +- .../routes/NotifyDueItemsRoutes.scala | 4 +- .../routes/PeriodicQueryRoutes.scala | 4 +- .../restserver/routes/ScanMailboxRoutes.scala | 4 +- .../scheduler/usertask}/QUserTask.scala | 83 +++++++++++++++---- .../scheduler}/usertask/UserTask.scala | 11 +-- .../scheduler}/usertask/UserTaskScope.scala | 2 +- .../scheduler}/usertask/UserTaskStore.scala | 7 +- .../store/records/RPeriodicTask.scala | 64 -------------- 15 files changed, 87 insertions(+), 116 deletions(-) rename modules/{store/src/main/scala/docspell/store/queries => scheduler/api/src/main/scala/docspell/scheduler/usertask}/QUserTask.scala (58%) rename modules/{store/src/main/scala/docspell/store => scheduler/api/src/main/scala/docspell/scheduler}/usertask/UserTask.scala (92%) rename modules/{store/src/main/scala/docspell/store => scheduler/api/src/main/scala/docspell/scheduler}/usertask/UserTaskScope.scala (97%) rename modules/{store/src/main/scala/docspell/store => scheduler/api/src/main/scala/docspell/scheduler}/usertask/UserTaskStore.scala (98%) diff --git a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala index 64ce4b9b..c4758559 100644 --- a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala +++ b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala @@ -15,8 +15,8 @@ import docspell.ftsclient.FtsClient import docspell.notification.api.{EventExchange, NotificationModule} import docspell.pubsub.api.PubSubT import docspell.scheduler.msg.JobQueuePublish +import docspell.scheduler.usertask.UserTaskStore import docspell.store.Store -import docspell.store.usertask.UserTaskStore import docspell.totp.Totp import emil.Emil diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala index f72ba217..ada2bb52 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala @@ -16,10 +16,10 @@ import docspell.common._ import docspell.store.UpdateResult import docspell.store.queries.{QCollective, QUser} import docspell.store.records._ -import docspell.store.usertask.{UserTask, UserTaskScope, UserTaskStore} import docspell.store.{AddResult, Store} import com.github.eikek.calev._ import docspell.scheduler.JobQueue +import docspell.scheduler.usertask.{UserTask, UserTaskScope, UserTaskStore} trait OCollective[F[_]] { diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala b/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala index 3c5ade36..87b333e4 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala @@ -13,9 +13,9 @@ import fs2.Stream import docspell.common._ import docspell.notification.api.{ChannelRef, PeriodicDueItemsArgs, PeriodicQueryArgs} import docspell.scheduler.JobQueue +import docspell.scheduler.usertask.{UserTask, UserTaskScope, UserTaskStore} import docspell.store.Store import docspell.store.records.RNotificationChannel -import docspell.store.usertask._ import io.circe.Encoder trait OUserTask[F[_]] { diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala index 3cb650c0..fd357ebf 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala @@ -41,10 +41,9 @@ import docspell.notification.api.NotificationModule import docspell.notification.impl.NotificationModuleImpl import docspell.pubsub.api.{PubSub, PubSubT} import docspell.scheduler.msg.JobQueuePublish +import docspell.scheduler.usertask.{UserTaskScope, UserTaskStore} import docspell.store.Store import docspell.store.records.{REmptyTrashSetting, RJobLog} -import docspell.store.usertask.UserTaskScope -import docspell.store.usertask.UserTaskStore import emil.javamail._ import org.http4s.client.Client diff --git a/modules/joex/src/main/scala/docspell/joex/emptytrash/EmptyTrashTask.scala b/modules/joex/src/main/scala/docspell/joex/emptytrash/EmptyTrashTask.scala index 28a834c7..04cfc337 100644 --- a/modules/joex/src/main/scala/docspell/joex/emptytrash/EmptyTrashTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/emptytrash/EmptyTrashTask.scala @@ -9,14 +9,12 @@ package docspell.joex.emptytrash import cats.effect._ import cats.implicits._ import fs2.Stream - import docspell.backend.ops.{OItem, OItemSearch} import docspell.common._ import docspell.scheduler._ import docspell.store.records.RItem -import docspell.store.usertask.UserTask - import com.github.eikek.calev.CalEvent +import docspell.scheduler.usertask.UserTask object EmptyTrashTask { type Args = EmptyTrashArgs diff --git a/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala index 206b7c29..8515eae6 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala @@ -8,16 +8,14 @@ package docspell.joex.hk import cats.effect._ import cats.implicits._ - import docspell.backend.ops.OFileRepository import docspell.common._ import docspell.joex.Config import docspell.joex.filecopy.FileIntegrityCheckTask import docspell.scheduler.{JobTaskResultEncoder, Task} import docspell.store.records._ -import docspell.store.usertask.UserTaskScope - import com.github.eikek.calev._ +import docspell.scheduler.usertask.{QUserTask, UserTaskScope} import io.circe.Encoder import io.circe.generic.semiauto.deriveEncoder @@ -48,7 +46,7 @@ object HouseKeepingTask { Task.log[F, Unit](_.warn("Cancelling house-keeping task")) def periodicTask[F[_]: Sync](ce: CalEvent): F[RPeriodicTask] = - RPeriodicTask + QUserTask .createJson( true, UserTaskScope(DocspellSystem.taskGroup), diff --git a/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala b/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala index 7a82f65b..94db11b5 100644 --- a/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala @@ -9,15 +9,12 @@ package docspell.joex.updatecheck import cats.data.OptionT import cats.effect._ import cats.implicits._ - import docspell.common._ import docspell.scheduler.Context import docspell.scheduler.Task +import docspell.scheduler.usertask.{UserTask, UserTaskScope} import docspell.store.records.RPeriodicTask import docspell.store.records.RUserEmail -import docspell.store.usertask.UserTask -import docspell.store.usertask.UserTaskScope - import emil._ object UpdateCheckTask { diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/NotifyDueItemsRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/NotifyDueItemsRoutes.scala index eae528e2..04248885 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/NotifyDueItemsRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/NotifyDueItemsRoutes.scala @@ -9,7 +9,6 @@ package docspell.restserver.routes import cats.data.OptionT import cats.effect._ import cats.implicits._ - import docspell.backend.BackendApp import docspell.backend.MailAddressCodec import docspell.backend.auth.AuthToken @@ -19,8 +18,7 @@ import docspell.restapi.model._ import docspell.restserver.Config import docspell.restserver.conv.{Conversions, NonEmptyListSupport} import docspell.restserver.http4s.ClientRequestInfo -import docspell.store.usertask._ - +import docspell.scheduler.usertask._ import org.http4s._ import org.http4s.circe.CirceEntityDecoder._ import org.http4s.circe.CirceEntityEncoder._ diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/PeriodicQueryRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/PeriodicQueryRoutes.scala index 496b069e..f0866e76 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/PeriodicQueryRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/PeriodicQueryRoutes.scala @@ -9,7 +9,6 @@ package docspell.restserver.routes import cats.data.{NonEmptyList, OptionT} import cats.effect._ import cats.implicits._ - import docspell.backend.BackendApp import docspell.backend.MailAddressCodec import docspell.backend.auth.AuthToken @@ -20,8 +19,7 @@ import docspell.restapi.model._ import docspell.restserver.Config import docspell.restserver.conv.{Conversions, NonEmptyListSupport} import docspell.restserver.http4s.ClientRequestInfo -import docspell.store.usertask._ - +import docspell.scheduler.usertask._ import org.http4s._ import org.http4s.circe.CirceEntityDecoder._ import org.http4s.circe.CirceEntityEncoder._ diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala index 3912d872..430baae4 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala @@ -9,14 +9,12 @@ package docspell.restserver.routes import cats.data.OptionT import cats.effect._ import cats.implicits._ - import docspell.backend.BackendApp import docspell.backend.auth.AuthToken import docspell.common._ import docspell.restapi.model._ import docspell.restserver.conv.Conversions -import docspell.store.usertask._ - +import docspell.scheduler.usertask._ import org.http4s._ import org.http4s.circe.CirceEntityDecoder._ import org.http4s.circe.CirceEntityEncoder._ diff --git a/modules/store/src/main/scala/docspell/store/queries/QUserTask.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/QUserTask.scala similarity index 58% rename from modules/store/src/main/scala/docspell/store/queries/QUserTask.scala rename to modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/QUserTask.scala index fdf1f220..f724dfe9 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QUserTask.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/QUserTask.scala @@ -1,20 +1,15 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.store.queries - -import fs2._ +package docspell.scheduler.usertask +import cats.implicits._ +import cats.effect.Sync +import com.github.eikek.calev.CalEvent import docspell.common._ +import docspell.store.qb.DML import docspell.store.qb.DSL._ -import docspell.store.qb._ -import docspell.store.records._ -import docspell.store.usertask.{UserTask, UserTaskScope} - -import doobie._ +import docspell.store.records.RPeriodicTask +import doobie.ConnectionIO +import fs2.Stream +import io.circe.Encoder object QUserTask { private val RT = RPeriodicTask.T @@ -101,4 +96,64 @@ object QUserTask { def makeUserTask(r: RPeriodicTask): UserTask[String] = UserTask(r.id, r.task, r.enabled, r.timer, r.summary, r.args) + def create[F[_]: Sync]( + enabled: Boolean, + scope: UserTaskScope, + task: Ident, + args: String, + subject: String, + priority: Priority, + timer: CalEvent, + summary: Option[String] + ): F[RPeriodicTask] = + Ident + .randomId[F] + .flatMap(id => + Timestamp + .current[F] + .map { now => + RPeriodicTask( + id, + enabled, + task, + scope.collective, + args, + subject, + scope.fold(_.user, identity), + priority, + None, + None, + timer, + timer + .nextElapse(now.atZone(Timestamp.UTC)) + .map(_.toInstant) + .map(Timestamp.apply) + .getOrElse(Timestamp.Epoch), + now, + summary + ) + } + ) + + def createJson[F[_]: Sync, A]( + enabled: Boolean, + scope: UserTaskScope, + task: Ident, + args: A, + subject: String, + priority: Priority, + timer: CalEvent, + summary: Option[String] + )(implicit E: Encoder[A]): F[RPeriodicTask] = + create[F]( + enabled, + scope, + task, + E(args).noSpaces, + subject, + priority, + timer, + summary + ) + } diff --git a/modules/store/src/main/scala/docspell/store/usertask/UserTask.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTask.scala similarity index 92% rename from modules/store/src/main/scala/docspell/store/usertask/UserTask.scala rename to modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTask.scala index 13685aca..1d2dcd65 100644 --- a/modules/store/src/main/scala/docspell/store/usertask/UserTask.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTask.scala @@ -4,18 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -package docspell.store.usertask +package docspell.scheduler.usertask import cats.effect._ import cats.implicits._ - +import com.github.eikek.calev.CalEvent import docspell.common._ import docspell.common.syntax.all._ import docspell.store.records.RPeriodicTask - -import com.github.eikek.calev.CalEvent -import io.circe.Decoder -import io.circe.Encoder +import io.circe.{Decoder, Encoder} case class UserTask[A]( id: Ident, @@ -51,7 +48,7 @@ object UserTask { scope: UserTaskScope, subject: Option[String] ): F[RPeriodicTask] = - RPeriodicTask + QUserTask .create[F]( ut.enabled, scope, diff --git a/modules/store/src/main/scala/docspell/store/usertask/UserTaskScope.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskScope.scala similarity index 97% rename from modules/store/src/main/scala/docspell/store/usertask/UserTaskScope.scala rename to modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskScope.scala index 6b269d1d..45ba0311 100644 --- a/modules/store/src/main/scala/docspell/store/usertask/UserTaskScope.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskScope.scala @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -package docspell.store.usertask +package docspell.scheduler.usertask import docspell.common._ diff --git a/modules/store/src/main/scala/docspell/store/usertask/UserTaskStore.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskStore.scala similarity index 98% rename from modules/store/src/main/scala/docspell/store/usertask/UserTaskStore.scala rename to modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskStore.scala index e19fa035..0570531d 100644 --- a/modules/store/src/main/scala/docspell/store/usertask/UserTaskStore.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskStore.scala @@ -4,17 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -package docspell.store.usertask +package docspell.scheduler.usertask import cats.data.OptionT import cats.effect._ import cats.implicits._ -import fs2.Stream - import docspell.common._ -import docspell.store.queries.QUserTask import docspell.store.{AddResult, Store} - +import fs2.Stream import io.circe._ /** User tasks are `RPeriodicTask`s that can be managed by the user. The user can change diff --git a/modules/store/src/main/scala/docspell/store/records/RPeriodicTask.scala b/modules/store/src/main/scala/docspell/store/records/RPeriodicTask.scala index d0e27f07..2f298dff 100644 --- a/modules/store/src/main/scala/docspell/store/records/RPeriodicTask.scala +++ b/modules/store/src/main/scala/docspell/store/records/RPeriodicTask.scala @@ -9,16 +9,12 @@ package docspell.store.records import cats.data.NonEmptyList import cats.effect._ import cats.implicits._ - import docspell.common._ import docspell.store.qb.DSL._ import docspell.store.qb._ -import docspell.store.usertask.UserTaskScope - import com.github.eikek.calev.CalEvent import doobie._ import doobie.implicits._ -import io.circe.Encoder /** A periodic task is a special job description, that shares a few properties of a * `RJob`. It must provide all information to create a `RJob` value eventually. @@ -65,66 +61,6 @@ case class RPeriodicTask( object RPeriodicTask { - def create[F[_]: Sync]( - enabled: Boolean, - scope: UserTaskScope, - task: Ident, - args: String, - subject: String, - priority: Priority, - timer: CalEvent, - summary: Option[String] - ): F[RPeriodicTask] = - Ident - .randomId[F] - .flatMap(id => - Timestamp - .current[F] - .map { now => - RPeriodicTask( - id, - enabled, - task, - scope.collective, - args, - subject, - scope.fold(_.user, identity), - priority, - None, - None, - timer, - timer - .nextElapse(now.atZone(Timestamp.UTC)) - .map(_.toInstant) - .map(Timestamp.apply) - .getOrElse(Timestamp.Epoch), - now, - summary - ) - } - ) - - def createJson[F[_]: Sync, A]( - enabled: Boolean, - scope: UserTaskScope, - task: Ident, - args: A, - subject: String, - priority: Priority, - timer: CalEvent, - summary: Option[String] - )(implicit E: Encoder[A]): F[RPeriodicTask] = - create[F]( - enabled, - scope, - task, - E(args).noSpaces, - subject, - priority, - timer, - summary - ) - final case class Table(alias: Option[String]) extends TableDef { val tableName = "periodic_task" From 3a05dc56cbc61113caa06dd4c65cf712a1388c0d Mon Sep 17 00:00:00 2001 From: eikek Date: Sun, 13 Mar 2022 14:27:06 +0100 Subject: [PATCH 14/16] Refactor scheduler into api / impl --- build.sbt | 5 +- .../scala/docspell/backend/BackendApp.scala | 33 +- .../scala/docspell/backend/JobFactory.scala | 285 ++++++---------- .../docspell/backend/ops/OCollective.scala | 22 +- .../backend/ops/OFileRepository.scala | 26 +- .../docspell/backend/ops/OFulltext.scala | 12 +- .../scala/docspell/backend/ops/OItem.scala | 17 +- .../scala/docspell/backend/ops/OUpload.scala | 14 +- .../docspell/backend/ops/OUserTask.scala | 8 +- .../docspell/common/syntax/StringSyntax.scala | 9 +- .../scala/docspell/joex/JoexAppImpl.scala | 320 +++++++++--------- .../docspell/joex/fts/MigrationTask.scala | 31 +- .../docspell/joex/hk/HouseKeepingTask.scala | 25 +- .../joex/pagecount/AllPageCountTask.scala | 44 ++- .../joex/pdfconv/ConvertAllPdfTask.scala | 24 +- .../joex/preview/AllPreviewsTask.scala | 24 +- .../docspell/joex/process/ItemHandler.scala | 2 +- .../docspell/joex/process/ReProcessItem.scala | 2 +- .../joex/updatecheck/UpdateCheckTask.scala | 12 +- .../docspell/restserver/RestAppImpl.scala | 9 +- .../scala/docspell/scheduler/Context.scala | 57 +--- .../main/scala/docspell/scheduler/Job.scala | 36 ++ .../scala/docspell/scheduler/JobQueue.scala | 97 ------ .../scala/docspell/scheduler/JobStore.scala | 21 ++ .../docspell/scheduler/JobStoreModule.scala | 9 + .../scala/docspell/scheduler/JobTask.scala | 2 +- .../scheduler/PeriodicSchedulerConfig.scala | 5 + .../docspell/scheduler/SchedulerConfig.scala | 9 +- .../docspell/scheduler/SchedulerModule.scala | 6 + .../scheduler/usertask/UserTask.scala | 38 +-- .../scheduler/usertask/UserTaskScope.scala | 5 +- .../scheduler/usertask/UserTaskStore.scala | 102 +----- .../scheduler/CountingSchemeSpec.scala | 3 +- .../docspell/scheduler/impl/ContextImpl.scala | 61 ++++ .../docspell/scheduler/impl/JobQueue.scala | 37 ++ .../scheduler/impl/JobStoreImpl.scala | 87 +++++ .../impl/JobStoreModuleBuilder.scala | 50 +++ .../scheduler/impl/JobStorePublish.scala} | 40 ++- .../docspell/scheduler/impl}/LogSink.scala | 6 +- .../impl/PeriodicSchedulerBuilder.scala | 6 +- .../impl/PeriodicSchedulerImpl.scala | 6 +- .../scheduler/impl/PeriodicTaskStore.scala | 30 +- .../docspell/scheduler/impl}/QUserTask.scala | 39 ++- .../scheduler/impl}/QueueLogger.scala | 18 +- .../scheduler/impl/SchedulerBuilder.scala | 25 +- .../scheduler/impl/SchedulerImpl.scala | 6 +- .../impl/SchedulerModuleBuilder.scala | 68 ++++ .../scheduler/impl/UserTaskStoreImpl.scala | 117 +++++++ .../scala/docspell/store/records/RJob.scala | 31 +- .../docspell/store/queries/QJobTest.scala | 2 +- 50 files changed, 1076 insertions(+), 867 deletions(-) create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/Job.scala delete mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/JobQueue.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/JobStore.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/JobStoreModule.scala create mode 100644 modules/scheduler/api/src/main/scala/docspell/scheduler/SchedulerModule.scala rename modules/{joex/src/test/scala/docspell/joex => scheduler/api/src/test/scala/docspell}/scheduler/CountingSchemeSpec.scala (92%) create mode 100644 modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala create mode 100644 modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobQueue.scala create mode 100644 modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobStoreImpl.scala create mode 100644 modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobStoreModuleBuilder.scala rename modules/scheduler/{api/src/main/scala/docspell/scheduler/msg/JobQueuePublish.scala => impl/src/main/scala/docspell/scheduler/impl/JobStorePublish.scala} (60%) rename modules/scheduler/{api/src/main/scala/docspell/scheduler => impl/src/main/scala/docspell/scheduler/impl}/LogSink.scala (96%) rename modules/scheduler/{api/src/main/scala/docspell/scheduler/usertask => impl/src/main/scala/docspell/scheduler/impl}/QUserTask.scala (79%) rename modules/scheduler/{api/src/main/scala/docspell/scheduler => impl/src/main/scala/docspell/scheduler/impl}/QueueLogger.scala (85%) create mode 100644 modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerModuleBuilder.scala create mode 100644 modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/UserTaskStoreImpl.scala diff --git a/build.sbt b/build.sbt index 91c21a17..d8ffd775 100644 --- a/build.sbt +++ b/build.sbt @@ -530,7 +530,7 @@ val schedulerApi = project Dependencies.fs2Core ++ Dependencies.circeCore ) - .dependsOn(loggingApi, common, store, notificationApi, pubsubApi) + .dependsOn(loggingApi, common, store, pubsubApi) val schedulerImpl = project .in(file("modules/scheduler/impl")) @@ -837,7 +837,8 @@ val restserver = project ftssolr, oidc, pubsubNaive, - notificationImpl + notificationImpl, + schedulerImpl ) // --- Website Documentation diff --git a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala index c4758559..ebe361ac 100644 --- a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala +++ b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala @@ -14,8 +14,7 @@ import docspell.backend.signup.OSignup import docspell.ftsclient.FtsClient import docspell.notification.api.{EventExchange, NotificationModule} import docspell.pubsub.api.PubSubT -import docspell.scheduler.msg.JobQueuePublish -import docspell.scheduler.usertask.UserTaskStore +import docspell.scheduler.JobStoreModule import docspell.store.Store import docspell.totp.Totp import emil.Emil @@ -58,29 +57,43 @@ object BackendApp { javaEmil: Emil[F], ftsClient: FtsClient[F], pubSubT: PubSubT[F], + schedulerModule: JobStoreModule[F], notificationMod: NotificationModule[F] ): Resource[F, BackendApp[F]] = for { - utStore <- UserTaskStore(store) - queue <- JobQueuePublish(store, pubSubT, notificationMod) totpImpl <- OTotp(store, Totp.default) loginImpl <- Login[F](store, Totp.default) signupImpl <- OSignup[F](store) joexImpl <- OJoex(pubSubT) - collImpl <- OCollective[F](store, utStore, queue, joexImpl) + collImpl <- OCollective[F]( + store, + schedulerModule.userTasks, + schedulerModule.jobs, + joexImpl + ) sourceImpl <- OSource[F](store) tagImpl <- OTag[F](store) equipImpl <- OEquipment[F](store) orgImpl <- OOrganization(store) - uploadImpl <- OUpload(store, queue, joexImpl) + uploadImpl <- OUpload(store, schedulerModule.jobs, joexImpl) nodeImpl <- ONode(store) jobImpl <- OJob(store, joexImpl, pubSubT) createIndex <- CreateIndex.resource(ftsClient, store) - itemImpl <- OItem(store, ftsClient, createIndex, queue, joexImpl) + itemImpl <- OItem(store, ftsClient, createIndex, schedulerModule.jobs, joexImpl) itemSearchImpl <- OItemSearch(store) - fulltextImpl <- OFulltext(itemSearchImpl, ftsClient, store, queue, joexImpl) + fulltextImpl <- OFulltext( + itemSearchImpl, + ftsClient, + store, + schedulerModule.jobs, + joexImpl + ) mailImpl <- OMail(store, javaEmil) - userTaskImpl <- OUserTask(utStore, store, queue, joexImpl) + userTaskImpl <- OUserTask( + schedulerModule.userTasks, + store, + joexImpl + ) folderImpl <- OFolder(store) customFieldsImpl <- OCustomFields(store) simpleSearchImpl = OSimpleSearch(fulltextImpl, itemSearchImpl) @@ -90,7 +103,7 @@ object BackendApp { ) notifyImpl <- ONotification(store, notificationMod) bookmarksImpl <- OQueryBookmarks(store) - fileRepoImpl <- OFileRepository(store, queue, joexImpl) + fileRepoImpl <- OFileRepository(store, schedulerModule.jobs, joexImpl) } yield new BackendApp[F] { val pubSub = pubSubT val login = loginImpl diff --git a/modules/backend/src/main/scala/docspell/backend/JobFactory.scala b/modules/backend/src/main/scala/docspell/backend/JobFactory.scala index f41911f4..d18356fa 100644 --- a/modules/backend/src/main/scala/docspell/backend/JobFactory.scala +++ b/modules/backend/src/main/scala/docspell/backend/JobFactory.scala @@ -8,124 +8,91 @@ package docspell.backend import cats.effect._ import cats.implicits._ - import docspell.backend.MailAddressCodec import docspell.common._ import docspell.notification.api.PeriodicQueryArgs -import docspell.store.records.RJob +import docspell.scheduler.Job object JobFactory extends MailAddressCodec { def integrityCheck[F[_]: Sync]( args: FileIntegrityCheckArgs, submitter: AccountId = DocspellSystem.account - ): F[RJob] = - for { - id <- Ident.randomId[F] - now <- Timestamp.current[F] - job = RJob.newJob( - id, - FileIntegrityCheckArgs.taskName, - submitter.collective, - args, - s"Check integrity of files", - now, - submitter.user, - Priority.High, - Some(FileIntegrityCheckArgs.taskName) - ) - } yield job + ): F[Job[FileIntegrityCheckArgs]] = + Job.createNew( + FileIntegrityCheckArgs.taskName, + submitter.collective, + args, + s"Check integrity of files", + submitter.user, + Priority.High, + Some(FileIntegrityCheckArgs.taskName) + ) def fileCopy[F[_]: Sync]( args: FileCopyTaskArgs, submitter: AccountId = DocspellSystem.account - ): F[RJob] = - for { - id <- Ident.randomId[F] - now <- Timestamp.current[F] - job = RJob.newJob( - id, - FileCopyTaskArgs.taskName, - submitter.collective, - args, - s"Copying all files", - now, - submitter.user, - Priority.High, - Some(FileCopyTaskArgs.taskName) - ) - } yield job + ): F[Job[FileCopyTaskArgs]] = + Job.createNew( + FileCopyTaskArgs.taskName, + submitter.collective, + args, + "Copying all files", + submitter.user, + Priority.High, + Some(FileCopyTaskArgs.taskName) + ) - def periodicQuery[F[_]: Sync](args: PeriodicQueryArgs, submitter: AccountId): F[RJob] = - for { - id <- Ident.randomId[F] - now <- Timestamp.current[F] - job = RJob.newJob( - id, - PeriodicQueryArgs.taskName, - submitter.collective, - args, - s"Running periodic query, notify via ${args.channels.map(_.channelType)}", - now, - submitter.user, - Priority.Low, - None - ) - } yield job + def periodicQuery[F[_]: Sync]( + args: PeriodicQueryArgs, + submitter: AccountId + ): F[Job[PeriodicQueryArgs]] = + Job.createNew( + PeriodicQueryArgs.taskName, + submitter.collective, + args, + s"Running periodic query, notify via ${args.channels.map(_.channelType)}", + submitter.user, + Priority.Low, + None + ) def makePageCount[F[_]: Sync]( args: MakePageCountArgs, account: Option[AccountId] - ): F[RJob] = - for { - id <- Ident.randomId[F] - now <- Timestamp.current[F] - job = RJob.newJob( - id, - MakePageCountArgs.taskName, - account.map(_.collective).getOrElse(DocspellSystem.taskGroup), - args, - s"Find page-count metadata for ${args.attachment.id}", - now, - account.map(_.user).getOrElse(DocspellSystem.user), - Priority.Low, - Some(MakePageCountArgs.taskName / args.attachment) - ) - } yield job + ): F[Job[MakePageCountArgs]] = + Job.createNew( + MakePageCountArgs.taskName, + account.map(_.collective).getOrElse(DocspellSystem.taskGroup), + args, + s"Find page-count metadata for ${args.attachment.id}", + account.map(_.user).getOrElse(DocspellSystem.user), + Priority.Low, + Some(MakePageCountArgs.taskName / args.attachment) + ) def makePreview[F[_]: Sync]( args: MakePreviewArgs, account: Option[AccountId] - ): F[RJob] = - for { - id <- Ident.randomId[F] - now <- Timestamp.current[F] - job = RJob.newJob( - id, - MakePreviewArgs.taskName, - account.map(_.collective).getOrElse(DocspellSystem.taskGroup), - args, - s"Generate preview image", - now, - account.map(_.user).getOrElse(DocspellSystem.user), - Priority.Low, - Some(MakePreviewArgs.taskName / args.attachment) - ) - } yield job + ): F[Job[MakePreviewArgs]] = + Job.createNew( + MakePreviewArgs.taskName, + account.map(_.collective).getOrElse(DocspellSystem.taskGroup), + args, + s"Generate preview image", + account.map(_.user).getOrElse(DocspellSystem.user), + Priority.Low, + Some(MakePreviewArgs.taskName / args.attachment) + ) def allPreviews[F[_]: Sync]( args: AllPreviewsArgs, submitter: Option[Ident] - ): F[RJob] = - for { - id <- Ident.randomId[F] - now <- Timestamp.current[F] - } yield RJob.newJob( - id, + ): F[Job[AllPreviewsArgs]] = + Job.createNew( AllPreviewsArgs.taskName, args.collective.getOrElse(DocspellSystem.taskGroup), args, "Create preview images", - now, submitter.getOrElse(DocspellSystem.user), Priority.Low, Some(DocspellSystem.allPreviewTaskTracker) @@ -135,127 +102,91 @@ object JobFactory extends MailAddressCodec { collective: Option[Ident], submitter: Option[Ident], prio: Priority - ): F[RJob] = - for { - id <- Ident.randomId[F] - now <- Timestamp.current[F] - job = RJob.newJob( - id, - ConvertAllPdfArgs.taskName, - collective.getOrElse(DocspellSystem.taskGroup), - ConvertAllPdfArgs(collective), - s"Convert all pdfs not yet converted", - now, - submitter.getOrElse(DocspellSystem.user), - prio, - collective - .map(c => c / ConvertAllPdfArgs.taskName) - .orElse(ConvertAllPdfArgs.taskName.some) - ) - } yield job + ): F[Job[ConvertAllPdfArgs]] = + Job.createNew( + ConvertAllPdfArgs.taskName, + collective.getOrElse(DocspellSystem.taskGroup), + ConvertAllPdfArgs(collective), + s"Convert all pdfs not yet converted", + submitter.getOrElse(DocspellSystem.user), + prio, + collective + .map(c => c / ConvertAllPdfArgs.taskName) + .orElse(ConvertAllPdfArgs.taskName.some) + ) def reprocessItem[F[_]: Sync]( args: ReProcessItemArgs, account: AccountId, prio: Priority - ): F[RJob] = - for { - id <- Ident.randomId[F] - now <- Timestamp.current[F] - job = RJob.newJob( - id, - ReProcessItemArgs.taskName, - account.collective, - args, - s"Re-process files of item ${args.itemId.id}", - now, - account.user, - prio, - Some(ReProcessItemArgs.taskName / args.itemId) - ) - } yield job + ): F[Job[ReProcessItemArgs]] = + Job.createNew( + ReProcessItemArgs.taskName, + account.collective, + args, + s"Re-process files of item ${args.itemId.id}", + account.user, + prio, + Some(ReProcessItemArgs.taskName / args.itemId) + ) def processItem[F[_]: Sync]( args: ProcessItemArgs, account: AccountId, prio: Priority, tracker: Option[Ident] - ): F[RJob] = - for { - id <- Ident.randomId[F] - now <- Timestamp.current[F] - job = RJob.newJob( - id, - ProcessItemArgs.taskName, - account.collective, - args, - args.makeSubject, - now, - account.user, - prio, - tracker - ) - } yield job + ): F[Job[ProcessItemArgs]] = + Job.createNew( + ProcessItemArgs.taskName, + account.collective, + args, + args.makeSubject, + account.user, + prio, + tracker + ) def processItems[F[_]: Sync]( args: Vector[ProcessItemArgs], account: AccountId, prio: Priority, tracker: Option[Ident] - ): F[Vector[RJob]] = { - def create(now: Timestamp, arg: ProcessItemArgs): F[RJob] = - Ident - .randomId[F] - .map(id => - RJob.newJob( - id, - ProcessItemArgs.taskName, - account.collective, - arg, - arg.makeSubject, - now, - account.user, - prio, - tracker - ) - ) + ): F[Vector[Job[ProcessItemArgs]]] = { + def create(arg: ProcessItemArgs): F[Job[ProcessItemArgs]] = + Job.createNew( + ProcessItemArgs.taskName, + account.collective, + arg, + arg.makeSubject, + account.user, + prio, + tracker + ) - for { - now <- Timestamp.current[F] - jobs <- args.traverse(a => create(now, a)) - } yield jobs + args.traverse(create) } - def reIndexAll[F[_]: Sync]: F[RJob] = - for { - id <- Ident.randomId[F] - now <- Timestamp.current[F] - } yield RJob.newJob( - id, + def reIndexAll[F[_]: Sync]: F[Job[ReIndexTaskArgs]] = + Job.createNew( ReIndexTaskArgs.taskName, DocspellSystem.taskGroup, ReIndexTaskArgs(None), - s"Recreate full-text index", - now, + "Recreate full-text index", DocspellSystem.taskGroup, Priority.Low, Some(DocspellSystem.migrationTaskTracker) ) - def reIndex[F[_]: Sync](account: AccountId): F[RJob] = - for { - id <- Ident.randomId[F] - now <- Timestamp.current[F] - args = ReIndexTaskArgs(Some(account.collective)) - } yield RJob.newJob( - id, + def reIndex[F[_]: Sync](account: AccountId): F[Job[ReIndexTaskArgs]] = { + val args = ReIndexTaskArgs(Some(account.collective)) + Job.createNew( ReIndexTaskArgs.taskName, account.collective, args, - s"Recreate full-text index", - now, + "Recreate full-text index", account.user, Priority.Low, Some(ReIndexTaskArgs.tracker(args)) ) + } } diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala index ada2bb52..09f66ef4 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala @@ -18,7 +18,7 @@ import docspell.store.queries.{QCollective, QUser} import docspell.store.records._ import docspell.store.{AddResult, Store} import com.github.eikek.calev._ -import docspell.scheduler.JobQueue +import docspell.scheduler.JobStore import docspell.scheduler.usertask.{UserTask, UserTaskScope, UserTaskStore} trait OCollective[F[_]] { @@ -131,7 +131,7 @@ object OCollective { def apply[F[_]: Async]( store: Store[F], uts: UserTaskStore[F], - queue: JobQueue[F], + jobStore: JobStore[F], joex: OJoex[F] ): Resource[F, OCollective[F]] = Resource.pure[F, OCollective[F]](new OCollective[F] { @@ -194,32 +194,32 @@ object OCollective { for { id <- Ident.randomId[F] args = LearnClassifierArgs(collective) - ut <- UserTask( + ut = UserTask( id, LearnClassifierArgs.taskName, true, CalEvent(WeekdayComponent.All, DateEvent.All, TimeEvent.All), None, args - ).encode.toPeriodicTask(UserTaskScope(collective), args.makeSubject.some) - job <- ut.toJob - _ <- queue.insert(job) + ) + _ <- uts + .updateOneTask(UserTaskScope(collective), args.makeSubject.some, ut) _ <- joex.notifyAllNodes } yield () def startEmptyTrash(args: EmptyTrashArgs): F[Unit] = for { id <- Ident.randomId[F] - ut <- UserTask( + ut = UserTask( id, EmptyTrashArgs.taskName, true, CalEvent(WeekdayComponent.All, DateEvent.All, TimeEvent.All), None, args - ).encode.toPeriodicTask(UserTaskScope(args.collective), args.makeSubject.some) - job <- ut.toJob - _ <- queue.insert(job) + ) + _ <- uts + .updateOneTask(UserTaskScope(args.collective), args.makeSubject.some, ut) _ <- joex.notifyAllNodes } yield () @@ -319,7 +319,7 @@ object OCollective { AllPreviewsArgs(Some(account.collective), storeMode), Some(account.user) ) - _ <- queue.insertIfNew(job) + _ <- jobStore.insertIfNew(job.encode) _ <- if (notifyJoex) joex.notifyAllNodes else ().pure[F] } yield UpdateResult.success diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala index 12ea8191..b46c075f 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala @@ -12,17 +12,22 @@ import cats.implicits._ import docspell.backend.JobFactory import docspell.backend.ops.OFileRepository.IntegrityResult import docspell.common._ -import docspell.scheduler.JobQueue +import docspell.scheduler.{Job, JobStore} import docspell.store.Store -import docspell.store.records.RJob import scodec.bits.ByteVector trait OFileRepository[F[_]] { /** Inserts the job or return None if such a job already is running. */ - def cloneFileRepository(args: FileCopyTaskArgs, notifyJoex: Boolean): F[Option[RJob]] + def cloneFileRepository( + args: FileCopyTaskArgs, + notifyJoex: Boolean + ): F[Option[Job[FileCopyTaskArgs]]] - def checkIntegrityAll(part: FileKeyPart, notifyJoex: Boolean): F[Option[RJob]] + def checkIntegrityAll( + part: FileKeyPart, + notifyJoex: Boolean + ): F[Option[Job[FileIntegrityCheckArgs]]] def checkIntegrity(key: FileKey, hash: Option[ByteVector]): F[Option[IntegrityResult]] } @@ -33,7 +38,7 @@ object OFileRepository { def apply[F[_]: Async]( store: Store[F], - queue: JobQueue[F], + jobStore: JobStore[F], joex: OJoex[F] ): Resource[F, OFileRepository[F]] = Resource.pure(new OFileRepository[F] { @@ -42,17 +47,20 @@ object OFileRepository { def cloneFileRepository( args: FileCopyTaskArgs, notifyJoex: Boolean - ): F[Option[RJob]] = + ): F[Option[Job[FileCopyTaskArgs]]] = for { job <- JobFactory.fileCopy(args) - flag <- queue.insertIfNew(job) + flag <- jobStore.insertIfNew(job.encode) _ <- if (notifyJoex) joex.notifyAllNodes else ().pure[F] } yield Option.when(flag)(job) - def checkIntegrityAll(part: FileKeyPart, notifyJoex: Boolean): F[Option[RJob]] = + def checkIntegrityAll( + part: FileKeyPart, + notifyJoex: Boolean + ): F[Option[Job[FileIntegrityCheckArgs]]] = for { job <- JobFactory.integrityCheck(FileIntegrityCheckArgs(part)) - flag <- queue.insertIfNew(job) + flag <- jobStore.insertIfNew(job.encode) _ <- if (notifyJoex) joex.notifyAllNodes else ().pure[F] } yield Option.when(flag)(job) diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala index 88b9c3e3..8fcdb9cd 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala @@ -16,7 +16,7 @@ import docspell.common._ import docspell.ftsclient._ import docspell.query.ItemQuery._ import docspell.query.ItemQueryDsl._ -import docspell.scheduler.JobQueue +import docspell.scheduler.JobStore import docspell.store.queries.{QFolder, QItem, SelectedItem} import docspell.store.records.RJob import docspell.store.{Store, qb} @@ -80,7 +80,7 @@ object OFulltext { itemSearch: OItemSearch[F], fts: FtsClient[F], store: Store[F], - queue: JobQueue[F], + jobStore: JobStore[F], joex: OJoex[F] ): Resource[F, OFulltext[F]] = Resource.pure[F, OFulltext[F]](new OFulltext[F] { @@ -89,7 +89,7 @@ object OFulltext { for { _ <- logger.info(s"Re-index all.") job <- JobFactory.reIndexAll[F] - _ <- queue.insertIfNew(job) *> joex.notifyAllNodes + _ <- jobStore.insertIfNew(job.encode) *> joex.notifyAllNodes } yield () def reindexCollective(account: AccountId): F[Unit] = @@ -101,7 +101,7 @@ object OFulltext { job <- JobFactory.reIndex(account) _ <- if (exist.isDefined) ().pure[F] - else queue.insertIfNew(job) *> joex.notifyAllNodes + else jobStore.insertIfNew(job.encode) *> joex.notifyAllNodes } yield () def findIndexOnly(maxNoteLen: Int)( @@ -323,9 +323,7 @@ object OFulltext { def apply[A](implicit ev: ItemId[A]): ItemId[A] = ev def from[A](f: A => Ident): ItemId[A] = - new ItemId[A] { - def itemId(a: A) = f(a) - } + (a: A) => f(a) implicit val listItemId: ItemId[ListItem] = ItemId.from(_.id) diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala index 41a48e7a..3a8b08ea 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala @@ -17,7 +17,7 @@ import docspell.common._ import docspell.ftsclient.FtsClient import docspell.logging.Logger import docspell.notification.api.Event -import docspell.scheduler.JobQueue +import docspell.scheduler.JobStore import docspell.store.queries.{QAttachment, QItem, QMoveAttachment} import docspell.store.records._ import docspell.store.{AddResult, Store, UpdateResult} @@ -226,7 +226,7 @@ object OItem { store: Store[F], fts: FtsClient[F], createIndex: CreateIndex[F], - queue: JobQueue[F], + jobStore: JobStore[F], joex: OJoex[F] ): Resource[F, OItem[F]] = for { @@ -286,7 +286,7 @@ object OItem { ) ev = Event.TagsChanged.partial( itemIds, - added.toList.flatten.map(_.id).toList, + added.toList.flatten.map(_.id), Nil ) } yield AttachedEvent(UpdateResult.success)(ev)) @@ -761,7 +761,7 @@ object OItem { job <- OptionT.liftF( JobFactory.reprocessItem[F](args, account, Priority.Low) ) - _ <- OptionT.liftF(queue.insertIfNew(job)) + _ <- OptionT.liftF(jobStore.insertIfNew(job.encode)) _ <- OptionT.liftF(if (notifyJoex) joex.notifyAllNodes else ().pure[F]) } yield UpdateResult.success).getOrElse(UpdateResult.notFound) @@ -775,7 +775,8 @@ object OItem { jobs <- items .map(item => ReProcessItemArgs(item, Nil)) .traverse(arg => JobFactory.reprocessItem[F](arg, account, Priority.Low)) - _ <- queue.insertAllIfNew(jobs) + .map(_.map(_.encode)) + _ <- jobStore.insertAllIfNew(jobs) _ <- if (notifyJoex) joex.notifyAllNodes else ().pure[F] } yield items.size) @@ -786,7 +787,7 @@ object OItem { ): F[UpdateResult] = for { job <- JobFactory.convertAllPdfs[F](collective, submitter, Priority.Low) - _ <- queue.insertIfNew(job) + _ <- jobStore.insertIfNew(job.encode) _ <- if (notifyJoex) joex.notifyAllNodes else ().pure[F] } yield UpdateResult.success @@ -797,7 +798,7 @@ object OItem { ): F[UpdateResult] = for { job <- JobFactory.makePreview[F](args, account.some) - _ <- queue.insertIfNew(job) + _ <- jobStore.insertIfNew(job.encode) _ <- if (notifyJoex) joex.notifyAllNodes else ().pure[F] } yield UpdateResult.success @@ -807,7 +808,7 @@ object OItem { ): F[UpdateResult] = for { job <- JobFactory.allPreviews[F](AllPreviewsArgs(None, storeMode), None) - _ <- queue.insertIfNew(job) + _ <- jobStore.insertIfNew(job.encode) _ <- if (notifyJoex) joex.notifyAllNodes else ().pure[F] } yield UpdateResult.success diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala index 4e1e2dd4..674aa5a4 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala @@ -13,7 +13,7 @@ import cats.implicits._ import fs2.Stream import docspell.backend.JobFactory import docspell.common._ -import docspell.scheduler.JobQueue +import docspell.scheduler.{Job, JobStore} import docspell.store.Store import docspell.store.records._ @@ -107,7 +107,7 @@ object OUpload { def apply[F[_]: Sync]( store: Store[F], - queue: JobQueue[F], + jobStore: JobStore[F], joex: OJoex[F] ): Resource[F, OUpload[F]] = Resource.pure[F, OUpload[F]](new OUpload[F] { @@ -186,10 +186,10 @@ object OUpload { private def submitJobs( notifyJoex: Boolean - )(jobs: Vector[RJob]): F[OUpload.UploadResult] = + )(jobs: Vector[Job[String]]): F[OUpload.UploadResult] = for { _ <- logger.debug(s"Storing jobs: $jobs") - _ <- queue.insertAll(jobs) + _ <- jobStore.insertAll(jobs) _ <- if (notifyJoex) joex.notifyAllNodes else ().pure[F] } yield UploadResult.Success @@ -243,7 +243,9 @@ object OUpload { account: AccountId, prio: Priority, tracker: Option[Ident] - ): F[Vector[RJob]] = - JobFactory.processItems[F](args, account, prio, tracker) + ): F[Vector[Job[String]]] = + JobFactory + .processItems[F](args, account, prio, tracker) + .map(_.map(_.encode)) }) } diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala b/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala index 87b333e4..f1171bd7 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala @@ -12,7 +12,6 @@ import cats.implicits._ import fs2.Stream import docspell.common._ import docspell.notification.api.{ChannelRef, PeriodicDueItemsArgs, PeriodicQueryArgs} -import docspell.scheduler.JobQueue import docspell.scheduler.usertask.{UserTask, UserTaskScope, UserTaskStore} import docspell.store.Store import docspell.store.records.RNotificationChannel @@ -84,7 +83,6 @@ object OUserTask { def apply[F[_]: Async]( taskStore: UserTaskStore[F], store: Store[F], - queue: JobQueue[F], joex: OJoex[F] ): Resource[F, OUserTask[F]] = Resource.pure[F, OUserTask[F]](new OUserTask[F] { @@ -93,10 +91,8 @@ object OUserTask { implicit E: Encoder[A] ): F[Unit] = for { - ptask <- task.encode.toPeriodicTask(scope, subject) - job <- ptask.toJob - _ <- queue.insert(job) - _ <- joex.notifyPeriodicTasks + _ <- taskStore.executeNow(scope, subject, task) + _ <- joex.notifyAllNodes } yield () def getScanMailbox(scope: UserTaskScope): Stream[F, UserTask[ScanMailboxArgs]] = diff --git a/modules/common/src/main/scala/docspell/common/syntax/StringSyntax.scala b/modules/common/src/main/scala/docspell/common/syntax/StringSyntax.scala index 40d5bf80..5e0038da 100644 --- a/modules/common/src/main/scala/docspell/common/syntax/StringSyntax.scala +++ b/modules/common/src/main/scala/docspell/common/syntax/StringSyntax.scala @@ -6,10 +6,8 @@ package docspell.common.syntax -import cats.implicits._ - import io.circe.Decoder -import io.circe.parser._ +import io.circe.parser trait StringSyntax { implicit class EvenMoreStringOps(s: String) { @@ -18,9 +16,8 @@ trait StringSyntax { def parseJsonAs[A](implicit d: Decoder[A]): Either[Throwable, A] = for { - json <- parse(s).leftMap(_.underlying) - value <- json.as[A] - } yield value + json <- parser.decode[A](s) + } yield json } } diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala index fd357ebf..ec9d5682 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala @@ -31,16 +31,11 @@ import docspell.joex.process.ItemHandler import docspell.joex.process.ReProcessItem import docspell.joex.scanmailbox._ import docspell.scheduler._ -import docspell.scheduler.impl.{ - PeriodicSchedulerBuilder, - PeriodicTaskStore, - SchedulerBuilder -} +import docspell.scheduler.impl.{JobStoreModuleBuilder, SchedulerModuleBuilder} import docspell.joex.updatecheck._ import docspell.notification.api.NotificationModule import docspell.notification.impl.NotificationModuleImpl import docspell.pubsub.api.{PubSub, PubSubT} -import docspell.scheduler.msg.JobQueuePublish import docspell.scheduler.usertask.{UserTaskScope, UserTaskStore} import docspell.store.Store import docspell.store.records.{REmptyTrashSetting, RJobLog} @@ -50,8 +45,8 @@ import org.http4s.client.Client final class JoexAppImpl[F[_]: Async]( cfg: Config, store: Store[F], - queue: JobQueue[F], - pstore: PeriodicTaskStore[F], + uts: UserTaskStore[F], + jobStore: JobStore[F], termSignal: SignallingRef[F, Boolean], notificationMod: NotificationModule[F], val scheduler: Scheduler[F], @@ -82,32 +77,30 @@ final class JoexAppImpl[F[_]: Async]( private def scheduleBackgroundTasks: F[Unit] = HouseKeepingTask .periodicTask[F](cfg.houseKeeping.schedule) - .flatMap(pstore.insert) *> + .flatMap(t => uts.updateTask(UserTaskScope.system, t.summary, t)) *> scheduleEmptyTrashTasks *> UpdateCheckTask .periodicTask(cfg.updateCheck) - .flatMap(pstore.insert) *> - MigrationTask.job.flatMap(queue.insertIfNew) *> + .flatMap(t => uts.updateTask(UserTaskScope.system, t.summary, t)) *> + MigrationTask.job.flatMap(jobStore.insertIfNew) *> AllPreviewsTask .job(MakePreviewArgs.StoreMode.WhenMissing, None) - .flatMap(queue.insertIfNew) *> - AllPageCountTask.job.flatMap(queue.insertIfNew).as(()) + .flatMap(jobStore.insertIfNew) *> + AllPageCountTask.job.flatMap(jobStore.insertIfNew).void private def scheduleEmptyTrashTasks: F[Unit] = store .transact( REmptyTrashSetting.findForAllCollectives(OCollective.EmptyTrash.default, 50) ) - .evalMap(es => - UserTaskStore(store).use { uts => - val args = EmptyTrashArgs(es.cid, es.minAge) - uts.updateOneTask( - UserTaskScope(args.collective), - args.makeSubject.some, - EmptyTrashTask.userTask(args, es.schedule) - ) - } - ) + .evalMap { es => + val args = EmptyTrashArgs(es.cid, es.minAge) + uts.updateOneTask( + UserTaskScope(args.collective), + args.makeSubject.some, + EmptyTrashTask.userTask(args, es.schedule) + ) + } .compile .drain @@ -123,186 +116,186 @@ object JoexAppImpl extends MailAddressCodec { pubSub: PubSub[F] ): Resource[F, JoexApp[F]] = for { - pstore <- PeriodicTaskStore.create(store) - joexLogger = docspell.logging.getLogger[F](s"joex-${cfg.appId.id}") + joexLogger <- Resource.pure(docspell.logging.getLogger[F](s"joex-${cfg.appId.id}")) pubSubT = PubSubT(pubSub, joexLogger) javaEmil = JavaMailEmil(Settings.defaultSettings.copy(debug = cfg.mailDebug)) notificationMod <- Resource.eval( NotificationModuleImpl[F](store, javaEmil, httpClient, 200) ) - queue <- JobQueuePublish(store, pubSubT, notificationMod) + + jobStoreModule = JobStoreModuleBuilder(store) + .withPubsub(pubSubT) + .withEventSink(notificationMod) + .build + joex <- OJoex(pubSubT) - upload <- OUpload(store, queue, joex) + upload <- OUpload(store, jobStoreModule.jobs, joex) fts <- createFtsClient(cfg)(httpClient) createIndex <- CreateIndex.resource(fts, store) - itemOps <- OItem(store, fts, createIndex, queue, joex) + itemOps <- OItem(store, fts, createIndex, jobStoreModule.jobs, joex) itemSearchOps <- OItemSearch(store) analyser <- TextAnalyser.create[F](cfg.textAnalysis.textAnalysisConfig) regexNer <- RegexNerFile(cfg.textAnalysis.regexNerFileConfig, store) updateCheck <- UpdateCheck.resource(httpClient) notification <- ONotification(store, notificationMod) - fileRepo <- OFileRepository(store, queue, joex) - sch <- SchedulerBuilder(cfg.scheduler, store) - .withQueue(queue) - .withPubSub(pubSubT) - .withEventSink(notificationMod) - .withTask( - JobTask.json( - ProcessItemArgs.taskName, - ItemHandler.newItem[F](cfg, itemOps, fts, analyser, regexNer), - ItemHandler.onCancel[F] + fileRepo <- OFileRepository(store, jobStoreModule.jobs, joex) + + schedulerModule <- SchedulerModuleBuilder(jobStoreModule) + .withSchedulerConfig(cfg.scheduler) + .withPeriodicSchedulerConfig(cfg.periodicScheduler) + .withTaskRegistry(JobTaskRegistry + .empty[F] + .withTask( + JobTask.json( + ProcessItemArgs.taskName, + ItemHandler.newItem[F](cfg, itemOps, fts, analyser, regexNer), + ItemHandler.onCancel[F] + ) ) - ) - .withTask( - JobTask.json( - ReProcessItemArgs.taskName, - ReProcessItem[F](cfg, fts, itemOps, analyser, regexNer), - ReProcessItem.onCancel[F] + .withTask( + JobTask.json( + ReProcessItemArgs.taskName, + ReProcessItem[F](cfg, fts, itemOps, analyser, regexNer), + ReProcessItem.onCancel[F] + ) ) - ) - .withTask( - JobTask.json( - ScanMailboxArgs.taskName, - ScanMailboxTask[F](cfg.userTasks.scanMailbox, javaEmil, upload, joex), - ScanMailboxTask.onCancel[F] + .withTask( + JobTask.json( + ScanMailboxArgs.taskName, + ScanMailboxTask[F](cfg.userTasks.scanMailbox, javaEmil, upload, joex), + ScanMailboxTask.onCancel[F] + ) ) - ) - .withTask( - JobTask.json( - MigrationTask.taskName, - MigrationTask[F](cfg.fullTextSearch, fts, createIndex), - MigrationTask.onCancel[F] + .withTask( + JobTask.json( + MigrationTask.taskName, + MigrationTask[F](cfg.fullTextSearch, fts, createIndex), + MigrationTask.onCancel[F] + ) ) - ) - .withTask( - JobTask.json( - ReIndexTask.taskName, - ReIndexTask[F](cfg.fullTextSearch, fts, createIndex), - ReIndexTask.onCancel[F] + .withTask( + JobTask.json( + ReIndexTask.taskName, + ReIndexTask[F](cfg.fullTextSearch, fts, createIndex), + ReIndexTask.onCancel[F] + ) ) - ) - .withTask( - JobTask.json( - HouseKeepingTask.taskName, - HouseKeepingTask[F](cfg, fileRepo), - HouseKeepingTask.onCancel[F] + .withTask( + JobTask.json( + HouseKeepingTask.taskName, + HouseKeepingTask[F](cfg, fileRepo), + HouseKeepingTask.onCancel[F] + ) ) - ) - .withTask( - JobTask.json( - PdfConvTask.taskName, - PdfConvTask[F](cfg), - PdfConvTask.onCancel[F] + .withTask( + JobTask.json( + PdfConvTask.taskName, + PdfConvTask[F](cfg), + PdfConvTask.onCancel[F] + ) ) - ) - .withTask( - JobTask.json( - ConvertAllPdfArgs.taskName, - ConvertAllPdfTask[F](queue, joex), - ConvertAllPdfTask.onCancel[F] + .withTask( + JobTask.json( + ConvertAllPdfArgs.taskName, + ConvertAllPdfTask[F](jobStoreModule.jobs, joex), + ConvertAllPdfTask.onCancel[F] + ) ) - ) - .withTask( - JobTask.json( - LearnClassifierArgs.taskName, - LearnClassifierTask[F](cfg.textAnalysis, analyser), - LearnClassifierTask.onCancel[F] + .withTask( + JobTask.json( + LearnClassifierArgs.taskName, + LearnClassifierTask[F](cfg.textAnalysis, analyser), + LearnClassifierTask.onCancel[F] + ) ) - ) - .withTask( - JobTask.json( - MakePreviewArgs.taskName, - MakePreviewTask[F](cfg.extraction.preview), - MakePreviewTask.onCancel[F] + .withTask( + JobTask.json( + MakePreviewArgs.taskName, + MakePreviewTask[F](cfg.extraction.preview), + MakePreviewTask.onCancel[F] + ) ) - ) - .withTask( - JobTask.json( - AllPreviewsArgs.taskName, - AllPreviewsTask[F](queue, joex), - AllPreviewsTask.onCancel[F] + .withTask( + JobTask.json( + AllPreviewsArgs.taskName, + AllPreviewsTask[F](jobStoreModule.jobs, joex), + AllPreviewsTask.onCancel[F] + ) ) - ) - .withTask( - JobTask.json( - MakePageCountArgs.taskName, - MakePageCountTask[F](), - MakePageCountTask.onCancel[F] + .withTask( + JobTask.json( + MakePageCountArgs.taskName, + MakePageCountTask[F](), + MakePageCountTask.onCancel[F] + ) ) - ) - .withTask( - JobTask.json( - AllPageCountTask.taskName, - AllPageCountTask[F](queue, joex), - AllPageCountTask.onCancel[F] + .withTask( + JobTask.json( + AllPageCountTask.taskName, + AllPageCountTask[F](jobStoreModule.jobs, joex), + AllPageCountTask.onCancel[F] + ) ) - ) - .withTask( - JobTask.json( - EmptyTrashArgs.taskName, - EmptyTrashTask[F](itemOps, itemSearchOps), - EmptyTrashTask.onCancel[F] + .withTask( + JobTask.json( + EmptyTrashArgs.taskName, + EmptyTrashTask[F](itemOps, itemSearchOps), + EmptyTrashTask.onCancel[F] + ) ) - ) - .withTask( - JobTask.json( - UpdateCheckTask.taskName, - UpdateCheckTask[F]( - cfg.updateCheck, - cfg.sendMail, - javaEmil, - updateCheck, - ThisVersion.default - ), - UpdateCheckTask.onCancel[F] + .withTask( + JobTask.json( + UpdateCheckTask.taskName, + UpdateCheckTask[F]( + cfg.updateCheck, + cfg.sendMail, + javaEmil, + updateCheck, + ThisVersion.default + ), + UpdateCheckTask.onCancel[F] + ) ) - ) - .withTask( - JobTask.json( - PeriodicQueryTask.taskName, - PeriodicQueryTask[F](notification), - PeriodicQueryTask.onCancel[F] + .withTask( + JobTask.json( + PeriodicQueryTask.taskName, + PeriodicQueryTask[F](notification), + PeriodicQueryTask.onCancel[F] + ) ) - ) - .withTask( - JobTask.json( - PeriodicDueItemsTask.taskName, - PeriodicDueItemsTask[F](notification), - PeriodicDueItemsTask.onCancel[F] + .withTask( + JobTask.json( + PeriodicDueItemsTask.taskName, + PeriodicDueItemsTask[F](notification), + PeriodicDueItemsTask.onCancel[F] + ) ) - ) - .withTask( - JobTask.json( - FileCopyTaskArgs.taskName, - FileCopyTask[F](cfg), - FileCopyTask.onCancel[F] + .withTask( + JobTask.json( + FileCopyTaskArgs.taskName, + FileCopyTask[F](cfg), + FileCopyTask.onCancel[F] + ) ) - ) - .withTask( - JobTask.json( - FileIntegrityCheckArgs.taskName, - FileIntegrityCheckTask[F](fileRepo), - FileIntegrityCheckTask.onCancel[F] + .withTask( + JobTask.json( + FileIntegrityCheckArgs.taskName, + FileIntegrityCheckTask[F](fileRepo), + FileIntegrityCheckTask.onCancel[F] + ) ) ) .resource - psch <- PeriodicSchedulerBuilder.build( - cfg.periodicScheduler, - sch, - queue, - pstore, - pubSubT - ) app = new JoexAppImpl( cfg, store, - queue, - pstore, + jobStoreModule.userTasks, + jobStoreModule.jobs, termSignal, notificationMod, - sch, - psch + schedulerModule.scheduler, + schedulerModule.periodicScheduler ) appR <- Resource.make(app.init.map(_ => app))(_.initShutdown) } yield appR @@ -312,4 +305,5 @@ object JoexAppImpl extends MailAddressCodec { )(client: Client[F]): Resource[F, FtsClient[F]] = if (cfg.fullTextSearch.enabled) SolrFtsClient(cfg.fullTextSearch.solr, client) else Resource.pure[F, FtsClient[F]](FtsClient.none[F]) + } diff --git a/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala b/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala index 707b457b..bd0e6fb9 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala @@ -8,13 +8,11 @@ package docspell.joex.fts import cats.effect._ import cats.implicits._ - import docspell.backend.fulltext.CreateIndex import docspell.common._ import docspell.ftsclient._ import docspell.joex.Config -import docspell.scheduler.Task -import docspell.store.records.RJob +import docspell.scheduler.{Job, Task} object MigrationTask { val taskName = Ident.unsafe("full-text-index") @@ -38,21 +36,18 @@ object MigrationTask { def onCancel[F[_]]: Task[F, Unit, Unit] = Task.log[F, Unit](_.warn("Cancelling full-text-index task")) - def job[F[_]: Sync]: F[RJob] = - for { - id <- Ident.randomId[F] - now <- Timestamp.current[F] - } yield RJob.newJob( - id, - taskName, - DocspellSystem.taskGroup, - (), - "Create full-text index", - now, - DocspellSystem.taskGroup, - Priority.Low, - Some(DocspellSystem.migrationTaskTracker) - ) + def job[F[_]: Sync]: F[Job[String]] = + Job + .createNew( + taskName, + DocspellSystem.taskGroup, + (), + "Create full-text index", + DocspellSystem.taskGroup, + Priority.Low, + Some(DocspellSystem.migrationTaskTracker) + ) + .map(_.encode) def migrationTasks[F[_]: Async](fts: FtsClient[F]): F[List[Migration[F]]] = fts.initialize.map(_.map(fm => Migration.from(fm))) diff --git a/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala index 8515eae6..ff025d31 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala @@ -13,9 +13,8 @@ import docspell.common._ import docspell.joex.Config import docspell.joex.filecopy.FileIntegrityCheckTask import docspell.scheduler.{JobTaskResultEncoder, Task} -import docspell.store.records._ import com.github.eikek.calev._ -import docspell.scheduler.usertask.{QUserTask, UserTaskScope} +import docspell.scheduler.usertask.UserTask import io.circe.Encoder import io.circe.generic.semiauto.deriveEncoder @@ -45,19 +44,15 @@ object HouseKeepingTask { def onCancel[F[_]]: Task[F, Unit, Unit] = Task.log[F, Unit](_.warn("Cancelling house-keeping task")) - def periodicTask[F[_]: Sync](ce: CalEvent): F[RPeriodicTask] = - QUserTask - .createJson( - true, - UserTaskScope(DocspellSystem.taskGroup), - taskName, - (), - "Docspell house-keeping", - Priority.Low, - ce, - None - ) - .map(_.copy(id = periodicId)) + def periodicTask[F[_]: Sync](ce: CalEvent): F[UserTask[Unit]] = + UserTask( + periodicId, + taskName, + true, + ce, + "Docspell house-keeping".some, + () + ).pure[F] case class Result( checkNodes: CleanupResult, diff --git a/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala b/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala index 0fb9e299..5b8448d9 100644 --- a/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala @@ -12,20 +12,19 @@ import fs2.{Chunk, Stream} import docspell.backend.JobFactory import docspell.backend.ops.OJoex import docspell.common._ -import docspell.scheduler.{Context, JobQueue, Task} +import docspell.scheduler.{Context, Job, JobStore, Task} import docspell.store.records.RAttachment -import docspell.store.records.RJob object AllPageCountTask { val taskName = Ident.unsafe("all-page-count") type Args = Unit - def apply[F[_]: Sync](queue: JobQueue[F], joex: OJoex[F]): Task[F, Args, Unit] = + def apply[F[_]: Sync](jobStore: JobStore[F], joex: OJoex[F]): Task[F, Args, Unit] = Task { ctx => for { _ <- ctx.logger.info("Generating previews for attachments") - n <- submitConversionJobs(ctx, queue) + n <- submitConversionJobs(ctx, jobStore) _ <- ctx.logger.info(s"Submitted $n jobs") _ <- joex.notifyAllNodes } yield () @@ -36,14 +35,14 @@ object AllPageCountTask { def submitConversionJobs[F[_]: Sync]( ctx: Context[F, Args], - queue: JobQueue[F] + jobStore: JobStore[F] ): F[Int] = ctx.store .transact(findAttachments) .chunks .flatMap(createJobs[F]) .chunks - .evalMap(jobs => queue.insertAllIfNew(jobs.toVector).map(_ => jobs.size)) + .evalMap(jobs => jobStore.insertAllIfNew(jobs.toVector).map(_ => jobs.size)) .evalTap(n => ctx.logger.debug(s"Submitted $n jobs …")) .compile .foldMonoid @@ -51,28 +50,25 @@ object AllPageCountTask { private def findAttachments[F[_]] = RAttachment.findAllWithoutPageCount(50) - private def createJobs[F[_]: Sync](ras: Chunk[RAttachment]): Stream[F, RJob] = { - def mkJob(ra: RAttachment): F[RJob] = + private def createJobs[F[_]: Sync](ras: Chunk[RAttachment]): Stream[F, Job[String]] = { + def mkJob(ra: RAttachment): F[Job[MakePageCountArgs]] = JobFactory.makePageCount(MakePageCountArgs(ra.id), None) val jobs = ras.traverse(mkJob) - Stream.evalUnChunk(jobs) + Stream.evalUnChunk(jobs).map(_.encode) } - def job[F[_]: Sync]: F[RJob] = - for { - id <- Ident.randomId[F] - now <- Timestamp.current[F] - } yield RJob.newJob( - id, - AllPageCountTask.taskName, - DocspellSystem.taskGroup, - (), - "Create all page-counts", - now, - DocspellSystem.taskGroup, - Priority.Low, - Some(DocspellSystem.allPageCountTaskTracker) - ) + def job[F[_]: Sync]: F[Job[String]] = + Job + .createNew( + AllPageCountTask.taskName, + DocspellSystem.taskGroup, + (), + "Create all page-counts", + DocspellSystem.taskGroup, + Priority.Low, + Some(DocspellSystem.allPageCountTaskTracker) + ) + .map(_.encode) } diff --git a/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala b/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala index a0865156..f1b30c5a 100644 --- a/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala @@ -11,9 +11,8 @@ import cats.implicits._ import fs2.{Chunk, Stream} import docspell.backend.ops.OJoex import docspell.common._ -import docspell.scheduler.{Context, JobQueue, Task} +import docspell.scheduler.{Context, Job, JobStore, Task} import docspell.store.records.RAttachment -import docspell.store.records._ /* A task to find all non-converted pdf files (of a collective, or * all) and converting them using ocrmypdf by submitting a job for @@ -22,11 +21,11 @@ import docspell.store.records._ object ConvertAllPdfTask { type Args = ConvertAllPdfArgs - def apply[F[_]: Sync](queue: JobQueue[F], joex: OJoex[F]): Task[F, Args, Unit] = + def apply[F[_]: Sync](jobStore: JobStore[F], joex: OJoex[F]): Task[F, Args, Unit] = Task { ctx => for { _ <- ctx.logger.info("Converting pdfs using ocrmypdf") - n <- submitConversionJobs(ctx, queue) + n <- submitConversionJobs(ctx, jobStore) _ <- ctx.logger.info(s"Submitted $n file conversion jobs") _ <- joex.notifyAllNodes } yield () @@ -37,40 +36,35 @@ object ConvertAllPdfTask { def submitConversionJobs[F[_]: Sync]( ctx: Context[F, Args], - queue: JobQueue[F] + jobStore: JobStore[F] ): F[Int] = ctx.store .transact(RAttachment.findNonConvertedPdf(ctx.args.collective, 50)) .chunks .flatMap(createJobs[F](ctx)) .chunks - .evalMap(jobs => queue.insertAllIfNew(jobs.toVector).map(_ => jobs.size)) + .evalMap(jobs => jobStore.insertAllIfNew(jobs.toVector).map(_ => jobs.size)) .evalTap(n => ctx.logger.debug(s"Submitted $n jobs …")) .compile .foldMonoid private def createJobs[F[_]: Sync]( ctx: Context[F, Args] - )(ras: Chunk[RAttachment]): Stream[F, RJob] = { + )(ras: Chunk[RAttachment]): Stream[F, Job[String]] = { val collectiveOrSystem = ctx.args.collective.getOrElse(DocspellSystem.taskGroup) - def mkJob(ra: RAttachment): F[RJob] = - for { - id <- Ident.randomId[F] - now <- Timestamp.current[F] - } yield RJob.newJob( - id, + def mkJob(ra: RAttachment): F[Job[PdfConvTask.Args]] = + Job.createNew( PdfConvTask.taskName, collectiveOrSystem, PdfConvTask.Args(ra.id), s"Convert pdf ${ra.id.id}/${ra.name.getOrElse("-")}", - now, collectiveOrSystem, Priority.Low, Some(PdfConvTask.taskName / ra.id) ) val jobs = ras.traverse(mkJob) - Stream.evalUnChunk(jobs) + Stream.evalUnChunk(jobs).map(_.encode) } } diff --git a/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala b/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala index 849b4f7a..100ace8f 100644 --- a/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala @@ -13,19 +13,18 @@ import docspell.backend.JobFactory import docspell.backend.ops.OJoex import docspell.common.MakePreviewArgs.StoreMode import docspell.common._ -import docspell.scheduler.{Context, JobQueue, Task} +import docspell.scheduler.{Context, Job, JobStore, Task} import docspell.store.records.RAttachment -import docspell.store.records.RJob object AllPreviewsTask { type Args = AllPreviewsArgs - def apply[F[_]: Sync](queue: JobQueue[F], joex: OJoex[F]): Task[F, Args, Unit] = + def apply[F[_]: Sync](jobStore: JobStore[F], joex: OJoex[F]): Task[F, Args, Unit] = Task { ctx => for { _ <- ctx.logger.info("Generating previews for attachments") - n <- submitConversionJobs(ctx, queue) + n <- submitConversionJobs(ctx, jobStore) _ <- ctx.logger.info(s"Submitted $n jobs") _ <- joex.notifyAllNodes } yield () @@ -36,14 +35,16 @@ object AllPreviewsTask { def submitConversionJobs[F[_]: Sync]( ctx: Context[F, Args], - queue: JobQueue[F] + jobStore: JobStore[F] ): F[Int] = ctx.store .transact(findAttachments(ctx)) .chunks .flatMap(createJobs[F](ctx)) .chunks - .evalMap(jobs => queue.insertAllIfNew(jobs.toVector).map(_ => jobs.size)) + .evalMap(jobs => + jobStore.insertAllIfNew(jobs.map(_.encode).toVector).map(_ => jobs.size) + ) .evalTap(n => ctx.logger.debug(s"Submitted $n jobs …")) .compile .foldMonoid @@ -58,13 +59,13 @@ object AllPreviewsTask { private def createJobs[F[_]: Sync]( ctx: Context[F, Args] - )(ras: Chunk[RAttachment]): Stream[F, RJob] = { + )(ras: Chunk[RAttachment]): Stream[F, Job[MakePreviewArgs]] = { val collectiveOrSystem = { val cid = ctx.args.collective.getOrElse(DocspellSystem.taskGroup) AccountId(cid, DocspellSystem.user) } - def mkJob(ra: RAttachment): F[RJob] = + def mkJob(ra: RAttachment): F[Job[MakePreviewArgs]] = JobFactory.makePreview( MakePreviewArgs(ra.id, ctx.args.storeMode), collectiveOrSystem.some @@ -74,7 +75,10 @@ object AllPreviewsTask { Stream.evalUnChunk(jobs) } - def job[F[_]: Sync](storeMode: MakePreviewArgs.StoreMode, cid: Option[Ident]): F[RJob] = - JobFactory.allPreviews(AllPreviewsArgs(cid, storeMode), None) + def job[F[_]: Sync]( + storeMode: MakePreviewArgs.StoreMode, + cid: Option[Ident] + ): F[Job[String]] = + JobFactory.allPreviews(AllPreviewsArgs(cid, storeMode), None).map(_.encode) } diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala index 21627f56..b3a5ae7b 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala @@ -65,7 +65,7 @@ object ItemHandler { .map(_ => data) ) - def isLastRetry[F[_]: Sync]: Task[F, Args, Boolean] = + def isLastRetry[F[_]]: Task[F, Args, Boolean] = Task(_.isLastRetry) def safeProcess[F[_]: Async]( diff --git a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala index 834832a5..97091b75 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala @@ -141,7 +141,7 @@ object ReProcessItem { lang1.orElse(lang2).getOrElse(Language.German) } - def isLastRetry[F[_]: Sync]: Task[F, Args, Boolean] = + def isLastRetry[F[_]]: Task[F, Args, Boolean] = Task(_.isLastRetry) def safeProcess[F[_]: Async]( diff --git a/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala b/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala index 94db11b5..6da9e345 100644 --- a/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala @@ -12,8 +12,7 @@ import cats.implicits._ import docspell.common._ import docspell.scheduler.Context import docspell.scheduler.Task -import docspell.scheduler.usertask.{UserTask, UserTaskScope} -import docspell.store.records.RPeriodicTask +import docspell.scheduler.usertask.UserTask import docspell.store.records.RUserEmail import emil._ @@ -25,18 +24,15 @@ object UpdateCheckTask { def onCancel[F[_]]: Task[F, Args, Unit] = Task.log(_.warn("Cancelling update-check task")) - def periodicTask[F[_]: Sync](cfg: UpdateCheckConfig): F[RPeriodicTask] = + def periodicTask[F[_]: Sync](cfg: UpdateCheckConfig): F[UserTask[Unit]] = UserTask( Ident.unsafe("docspell-update-check"), taskName, cfg.enabled, cfg.schedule, - None, + "Docspell Update Check".some, () - ).encode.toPeriodicTask( - UserTaskScope(cfg.senderAccount.collective), - "Docspell Update Check".some - ) + ).pure[F] def apply[F[_]: Async]( cfg: UpdateCheckConfig, diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala index 7ed9d2dc..0ff2957b 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala @@ -9,7 +9,6 @@ package docspell.restserver import cats.effect._ import fs2.Stream import fs2.concurrent.Topic - import docspell.backend.BackendApp import docspell.backend.auth.{AuthToken, ShareToken} import docspell.ftsclient.FtsClient @@ -23,8 +22,8 @@ import docspell.restserver.http4s.EnvMiddleware import docspell.restserver.routes._ import docspell.restserver.webapp.{TemplateRoutes, Templates, WebjarRoutes} import docspell.restserver.ws.{OutputEvent, WebSocketRoutes} +import docspell.scheduler.impl.JobStoreModuleBuilder import docspell.store.Store - import emil.javamail.JavaMailEmil import org.http4s.HttpRoutes import org.http4s.client.Client @@ -167,8 +166,12 @@ object RestAppImpl { notificationMod <- Resource.eval( NotificationModuleImpl[F](store, javaEmil, httpClient, 200) ) + schedulerMod = JobStoreModuleBuilder(store) + .withPubsub(pubSubT) + .withEventSink(notificationMod) + .build backend <- BackendApp - .create[F](store, javaEmil, ftsClient, pubSubT, notificationMod) + .create[F](store, javaEmil, ftsClient, pubSubT, schedulerMod, notificationMod) app = new RestAppImpl[F]( cfg, diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala index d5380f94..abefafa5 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala @@ -6,14 +6,9 @@ package docspell.scheduler -import cats.effect._ -import cats.implicits._ -import cats.{Applicative, Functor} - import docspell.common._ import docspell.logging.Logger import docspell.store.Store -import docspell.store.records.RJob trait Context[F[_], A] { self => @@ -29,54 +24,8 @@ trait Context[F[_], A] { self => def store: Store[F] - final def isLastRetry(implicit ev: Applicative[F]): F[Boolean] = - for { - current <- store.transact(RJob.getRetries(jobId)) - last = config.retries == current.getOrElse(0) - } yield last + def isLastRetry: F[Boolean] + + def map[C](f: A => C): Context[F, C] - def map[C](f: A => C)(implicit F: Functor[F]): Context[F, C] = - new Context.ContextImpl[F, C](f(args), logger, store, config, jobId) -} - -object Context { - - def create[F[_]: Async, A]( - jobId: Ident, - arg: A, - config: SchedulerConfig, - log: Logger[F], - store: Store[F] - ): Context[F, A] = - new ContextImpl(arg, log, store, config, jobId) - - def apply[F[_]: Async, A]( - job: RJob, - arg: A, - config: SchedulerConfig, - logSink: LogSink[F], - store: Store[F] - ): F[Context[F, A]] = { - val log = docspell.logging.getLogger[F] - for { - _ <- log.trace("Creating logger for task run") - logger <- QueueLogger(job.id, job.info, config.logBufferSize, logSink) - _ <- log.trace("Logger created, instantiating context") - ctx = create[F, A](job.id, arg, config, logger, store) - } yield ctx - } - - final private class ContextImpl[F[_]: Functor, A]( - val args: A, - val logger: Logger[F], - val store: Store[F], - val config: SchedulerConfig, - val jobId: Ident - ) extends Context[F, A] { - - def setProgress(percent: Int): F[Unit] = { - val pval = math.min(100, math.max(0, percent)) - store.transact(RJob.setProgress(jobId, pval)).map(_ => ()) - } - } } diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/Job.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/Job.scala new file mode 100644 index 00000000..9dc08b5c --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/Job.scala @@ -0,0 +1,36 @@ +package docspell.scheduler + +import cats.effect.Sync +import cats.syntax.functor._ +import docspell.common._ +import io.circe.Encoder + +final case class Job[A]( + id: Ident, + task: Ident, + group: Ident, + args: A, + subject: String, + submitter: Ident, + priority: Priority, + tracker: Option[Ident] +) { + + def encode(implicit E: Encoder[A]): Job[String] = + Job(id, task, group, E.apply(args).noSpaces, subject, submitter, priority, tracker) +} + +object Job { + def createNew[F[_]: Sync, A]( + task: Ident, + group: Ident, + args: A, + subject: String, + submitter: Ident, + priority: Priority, + tracker: Option[Ident] + ): F[Job[A]] = + Ident.randomId[F].map { id => + Job(id, task, group, args, subject, submitter, priority, tracker) + } +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/JobQueue.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobQueue.scala deleted file mode 100644 index dd927bf9..00000000 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/JobQueue.scala +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -package docspell.scheduler - -import cats.effect._ -import cats.implicits._ -import docspell.common._ -import docspell.store.Store -import docspell.store.queries.QJob -import docspell.store.records.RJob - -trait JobQueue[F[_]] { - - /** Inserts the job into the queue to get picked up as soon as possible. The job must - * have a new unique id. - */ - def insert(job: RJob): F[Unit] - - /** Inserts the job into the queue only, if there is no job with the same tracker-id - * running at the moment. The job id must be a new unique id. - * - * If the job has no tracker defined, it is simply inserted. - */ - def insertIfNew(job: RJob): F[Boolean] - - def insertAll(jobs: Seq[RJob]): F[List[Boolean]] - - def insertAllIfNew(jobs: Seq[RJob]): F[List[Boolean]] - - def nextJob( - prio: Ident => F[Priority], - worker: Ident, - retryPause: Duration - ): F[Option[RJob]] -} - -object JobQueue { - private[scheduler] def create[F[_]: Async](store: Store[F]): Resource[F, JobQueue[F]] = - Resource.pure[F, JobQueue[F]](new JobQueue[F] { - private[this] val logger = docspell.logging.getLogger[F] - - def nextJob( - prio: Ident => F[Priority], - worker: Ident, - retryPause: Duration - ): F[Option[RJob]] = - logger - .trace("Select next job") *> QJob.takeNextJob(store)(prio, worker, retryPause) - - def insert(job: RJob): F[Unit] = - store - .transact(RJob.insert(job)) - .flatMap { n => - if (n != 1) - Async[F] - .raiseError(new Exception(s"Inserting job failed. Update count: $n")) - else ().pure[F] - } - - def insertIfNew(job: RJob): F[Boolean] = - for { - rj <- job.tracker match { - case Some(tid) => - store.transact(RJob.findNonFinalByTracker(tid)) - case None => - None.pure[F] - } - ret <- - if (rj.isDefined) false.pure[F] - else insert(job).as(true) - } yield ret - - def insertAll(jobs: Seq[RJob]): F[List[Boolean]] = - jobs.toList - .traverse(j => insert(j).attempt) - .flatMap(_.traverse { - case Right(()) => true.pure[F] - case Left(ex) => - logger.error(ex)("Could not insert job. Skipping it.").as(false) - - }) - - def insertAllIfNew(jobs: Seq[RJob]): F[List[Boolean]] = - jobs.toList - .traverse(j => insertIfNew(j).attempt) - .flatMap(_.traverse { - case Right(true) => true.pure[F] - case Right(false) => false.pure[F] - case Left(ex) => - logger.error(ex)("Could not insert job. Skipping it.").as(false) - }) - }) -} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/JobStore.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobStore.scala new file mode 100644 index 00000000..8ca887f9 --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobStore.scala @@ -0,0 +1,21 @@ +package docspell.scheduler + +trait JobStore[F[_]] { + + /** Inserts the job into the queue to get picked up as soon as possible. The job must + * have a new unique id. + */ + def insert(job: Job[String]): F[Unit] + + /** Inserts the job into the queue only, if there is no job with the same tracker-id + * running at the moment. The job id must be a new unique id. + * + * If the job has no tracker defined, it is simply inserted. + */ + def insertIfNew(job: Job[String]): F[Boolean] + + def insertAll(jobs: Seq[Job[String]]): F[List[Boolean]] + + def insertAllIfNew(jobs: Seq[Job[String]]): F[List[Boolean]] + +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/JobStoreModule.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobStoreModule.scala new file mode 100644 index 00000000..2d170127 --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobStoreModule.scala @@ -0,0 +1,9 @@ +package docspell.scheduler + +import docspell.scheduler.usertask.UserTaskStore + +trait JobStoreModule[F[_]] { + + def userTasks: UserTaskStore[F] + def jobs: JobStore[F] +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/JobTask.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobTask.scala index e4c5074f..a7198378 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/JobTask.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobTask.scala @@ -43,7 +43,7 @@ object JobTask { str.parseJsonAs[A] match { case Right(a) => a.pure[F] case Left(ex) => - Sync[F].raiseError(new Exception(s"Cannot parse task arguments: $str", ex)) + Sync[F].raiseError(new Exception(s"Cannot parse task arguments: '$str'", ex)) } JobTask(name, task.contramap(convert).map(E.encode), onCancel.contramap(convert)) diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/PeriodicSchedulerConfig.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/PeriodicSchedulerConfig.scala index 7445e9e3..6fa82069 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/PeriodicSchedulerConfig.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/PeriodicSchedulerConfig.scala @@ -12,3 +12,8 @@ case class PeriodicSchedulerConfig( name: Ident, wakeupPeriod: Duration ) + +object PeriodicSchedulerConfig { + def default(id: Ident): PeriodicSchedulerConfig = + PeriodicSchedulerConfig(id, Duration.minutes(10)) +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/SchedulerConfig.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/SchedulerConfig.scala index fbbc4f3c..25fad892 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/SchedulerConfig.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/SchedulerConfig.scala @@ -20,11 +20,10 @@ case class SchedulerConfig( object SchedulerConfig { - val default = SchedulerConfig( - name = Ident.unsafe("default-scheduler"), - poolSize = 2 // math.max(2, Runtime.getRuntime.availableProcessors / 2) - , - countingScheme = CountingScheme(2, 1), + def default(id: Ident) = SchedulerConfig( + name = id, + poolSize = 1, + countingScheme = CountingScheme(3, 1), retries = 5, retryDelay = Duration.seconds(30), logBufferSize = 500, diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/SchedulerModule.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/SchedulerModule.scala new file mode 100644 index 00000000..d422f200 --- /dev/null +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/SchedulerModule.scala @@ -0,0 +1,6 @@ +package docspell.scheduler + +trait SchedulerModule[F[_]] { + def scheduler: Scheduler[F] + def periodicScheduler: PeriodicScheduler[F] +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTask.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTask.scala index 1d2dcd65..f700da5c 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTask.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTask.scala @@ -6,15 +6,11 @@ package docspell.scheduler.usertask -import cats.effect._ -import cats.implicits._ import com.github.eikek.calev.CalEvent import docspell.common._ -import docspell.common.syntax.all._ -import docspell.store.records.RPeriodicTask -import io.circe.{Decoder, Encoder} +import io.circe.Encoder -case class UserTask[A]( +final case class UserTask[A]( id: Ident, name: Ident, enabled: Boolean, @@ -32,33 +28,3 @@ case class UserTask[A]( def mapArgs[B](f: A => B): UserTask[B] = withArgs(f(args)) } - -object UserTask { - - implicit final class UserTaskCodec(ut: UserTask[String]) { - - def decode[A](implicit D: Decoder[A]): Either[String, UserTask[A]] = - ut.args - .parseJsonAs[A] - .left - .map(_.getMessage) - .map(a => ut.copy(args = a)) - - def toPeriodicTask[F[_]: Sync]( - scope: UserTaskScope, - subject: Option[String] - ): F[RPeriodicTask] = - QUserTask - .create[F]( - ut.enabled, - scope, - ut.name, - ut.args, - subject.getOrElse(s"${scope.fold(_.user.id, _.id)}: ${ut.name.id}"), - Priority.Low, - ut.timer, - ut.summary - ) - .map(r => r.copy(id = ut.id)) - } -} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskScope.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskScope.scala index 45ba0311..236c7ee6 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskScope.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskScope.scala @@ -20,7 +20,7 @@ sealed trait UserTaskScope { self: Product => /** Maps to the account or uses the collective for both parts if the scope is collective * wide. */ - private[usertask] def toAccountId: AccountId = + private[scheduler] def toAccountId: AccountId = AccountId(collective, fold(_.user, identity)) } @@ -49,4 +49,7 @@ object UserTaskScope { def apply(collective: Ident): UserTaskScope = UserTaskScope.collective(collective) + + def system: UserTaskScope = + collective(DocspellSystem.taskGroup) } diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskStore.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskStore.scala index 0570531d..2a3dcf0e 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskStore.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskStore.scala @@ -7,10 +7,7 @@ package docspell.scheduler.usertask import cats.data.OptionT -import cats.effect._ -import cats.implicits._ import docspell.common._ -import docspell.store.{AddResult, Store} import fs2.Stream import io.circe._ @@ -88,96 +85,11 @@ trait UserTaskStore[F[_]] { /** Delete all tasks of the given user that have name `name`. */ def deleteAll(scope: UserTaskScope, name: Ident): F[Int] -} - -object UserTaskStore { - - def apply[F[_]: Async](store: Store[F]): Resource[F, UserTaskStore[F]] = - Resource.pure[F, UserTaskStore[F]](new UserTaskStore[F] { - - def getAll(scope: UserTaskScope): Stream[F, UserTask[String]] = - store.transact(QUserTask.findAll(scope.toAccountId)) - - def getByNameRaw(scope: UserTaskScope, name: Ident): Stream[F, UserTask[String]] = - store.transact(QUserTask.findByName(scope.toAccountId, name)) - - def getByIdRaw(scope: UserTaskScope, id: Ident): OptionT[F, UserTask[String]] = - OptionT(store.transact(QUserTask.findById(scope.toAccountId, id))) - - def getByName[A](scope: UserTaskScope, name: Ident)(implicit - D: Decoder[A] - ): Stream[F, UserTask[A]] = - getByNameRaw(scope, name).flatMap(_.decode match { - case Right(ua) => Stream.emit(ua) - case Left(err) => Stream.raiseError[F](new Exception(err)) - }) - - def updateTask[A](scope: UserTaskScope, subject: Option[String], ut: UserTask[A])( - implicit E: Encoder[A] - ): F[Int] = { - val exists = QUserTask.exists(ut.id) - val insert = QUserTask.insert(scope, subject, ut.encode) - store.add(insert, exists).flatMap { - case AddResult.Success => - 1.pure[F] - case AddResult.EntityExists(_) => - store.transact(QUserTask.update(scope, subject, ut.encode)) - case AddResult.Failure(ex) => - Async[F].raiseError(ex) - } - } - - def deleteTask(scope: UserTaskScope, id: Ident): F[Int] = - store.transact(QUserTask.delete(scope.toAccountId, id)) - - def getOneByNameRaw( - scope: UserTaskScope, - name: Ident - ): OptionT[F, UserTask[String]] = - OptionT( - getByNameRaw(scope, name) - .take(2) - .compile - .toList - .flatMap { - case Nil => (None: Option[UserTask[String]]).pure[F] - case ut :: Nil => ut.some.pure[F] - case _ => Async[F].raiseError(new Exception("More than one result found")) - } - ) - - def getOneByName[A](scope: UserTaskScope, name: Ident)(implicit - D: Decoder[A] - ): OptionT[F, UserTask[A]] = - getOneByNameRaw(scope, name) - .semiflatMap(_.decode match { - case Right(ua) => ua.pure[F] - case Left(err) => Async[F].raiseError(new Exception(err)) - }) - - def updateOneTask[A]( - scope: UserTaskScope, - subject: Option[String], - ut: UserTask[A] - )(implicit - E: Encoder[A] - ): F[UserTask[String]] = - getByNameRaw(scope, ut.name).compile.toList.flatMap { - case a :: rest => - val task = ut.copy(id = a.id).encode - for { - _ <- store.transact(QUserTask.update(scope, subject, task)) - _ <- store.transact( - rest.traverse(t => QUserTask.delete(scope.toAccountId, t.id)) - ) - } yield task - case Nil => - val task = ut.encode - store.transact(QUserTask.insert(scope, subject, task)).map(_ => task) - } - - def deleteAll(scope: UserTaskScope, name: Ident): F[Int] = - store.transact(QUserTask.deleteAll(scope.toAccountId, name)) - }) - + + /** Discards the schedule and immediately submits the task to the job executor's queue. + * It will not update the corresponding periodic task. + */ + def executeNow[A](scope: UserTaskScope, subject: Option[String], task: UserTask[A])( + implicit E: Encoder[A] + ): F[Unit] } diff --git a/modules/joex/src/test/scala/docspell/joex/scheduler/CountingSchemeSpec.scala b/modules/scheduler/api/src/test/scala/docspell/scheduler/CountingSchemeSpec.scala similarity index 92% rename from modules/joex/src/test/scala/docspell/joex/scheduler/CountingSchemeSpec.scala rename to modules/scheduler/api/src/test/scala/docspell/scheduler/CountingSchemeSpec.scala index cf819f2f..1339d220 100644 --- a/modules/joex/src/test/scala/docspell/joex/scheduler/CountingSchemeSpec.scala +++ b/modules/scheduler/api/src/test/scala/docspell/scheduler/CountingSchemeSpec.scala @@ -4,10 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -package docspell.joex.scheduler +package docspell.scheduler import docspell.common.Priority - import munit._ class CountingSchemeSpec extends FunSuite { diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala new file mode 100644 index 00000000..45647f03 --- /dev/null +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala @@ -0,0 +1,61 @@ +package docspell.scheduler.impl + +import cats._ +import cats.syntax.all._ +import cats.effect._ + +import docspell.common._ +import docspell.logging.Logger +import docspell.scheduler._ +import docspell.store.Store +import docspell.store.records.RJob + +class ContextImpl[F[_]: Functor, A]( + val args: A, + val logger: Logger[F], + val store: Store[F], + val config: SchedulerConfig, + val jobId: Ident +) extends Context[F, A] { + + def setProgress(percent: Int): F[Unit] = { + val pval = math.min(100, math.max(0, percent)) + store.transact(RJob.setProgress(jobId, pval)).map(_ => ()) + } + + def isLastRetry: F[Boolean] = + for { + current <- store.transact(RJob.getRetries(jobId)) + last = config.retries == current.getOrElse(0) + } yield last + + def map[C](f: A => C) = + new ContextImpl[F, C](f(args), logger, store, config, jobId) +} + +object ContextImpl { + def create[F[_]: Async, A]( + jobId: Ident, + arg: A, + config: SchedulerConfig, + log: Logger[F], + store: Store[F] + ): Context[F, A] = + new ContextImpl(arg, log, store, config, jobId) + + def apply[F[_]: Async, A]( + job: RJob, + arg: A, + config: SchedulerConfig, + logSink: LogSink[F], + store: Store[F] + ): F[Context[F, A]] = { + val log = docspell.logging.getLogger[F] + for { + _ <- log.trace("Creating logger for task run") + logger <- QueueLogger(job.id, job.info, config.logBufferSize, logSink) + _ <- log.trace("Logger created, instantiating context") + ctx = create[F, A](job.id, arg, config, logger, store) + } yield ctx + } +} diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobQueue.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobQueue.scala new file mode 100644 index 00000000..0c190723 --- /dev/null +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobQueue.scala @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.scheduler.impl + +import cats.effect._ +import cats.implicits._ +import docspell.common._ +import docspell.store.Store +import docspell.store.queries.QJob +import docspell.store.records.RJob + +trait JobQueue[F[_]] { + def nextJob( + prio: Ident => F[Priority], + worker: Ident, + retryPause: Duration + ): F[Option[RJob]] +} + +object JobQueue { + private[scheduler] def apply[F[_]: Async](store: Store[F]): JobQueue[F] = + new JobQueue[F] { + private[this] val logger = docspell.logging.getLogger[F] + + def nextJob( + prio: Ident => F[Priority], + worker: Ident, + retryPause: Duration + ): F[Option[RJob]] = + logger + .trace("Select next job") *> QJob.takeNextJob(store)(prio, worker, retryPause) + } +} diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobStoreImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobStoreImpl.scala new file mode 100644 index 00000000..f2507301 --- /dev/null +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobStoreImpl.scala @@ -0,0 +1,87 @@ +package docspell.scheduler.impl + +import cats.effect.Sync +import cats.syntax.all._ + +import docspell.common.Timestamp +import docspell.scheduler._ + +import docspell.store.Store +import docspell.store.records.RJob + +final class JobStoreImpl[F[_]: Sync](store: Store[F]) extends JobStore[F] { + private[this] val logger = docspell.logging.getLogger[F] + + def insert(job: Job[String]): F[Unit] = + for { + now <- Timestamp.current[F] + _ <- insert0(job, now) + } yield () + + def insert0(job: Job[String], submitted: Timestamp): F[Unit] = + store + .transact(RJob.insert(toRecord(job, submitted))) + .flatMap { n => + if (n != 1) + Sync[F] + .raiseError(new Exception(s"Inserting job failed. Update count: $n")) + else ().pure[F] + } + + def insertIfNew(job: Job[String]): F[Boolean] = + Timestamp.current[F].flatMap(now => insertIfNew0(job, now)) + + def insertIfNew0(job: Job[String], submitted: Timestamp): F[Boolean] = + for { + rj <- job.tracker match { + case Some(tid) => + store.transact(RJob.findNonFinalByTracker(tid)) + case None => + None.pure[F] + } + ret <- + if (rj.isDefined) false.pure[F] + else insert0(job, submitted).as(true) + } yield ret + + def insertAll(jobs: Seq[Job[String]]): F[List[Boolean]] = + Timestamp.current[F].flatMap { now => + jobs.toList + .traverse(j => insert0(j, now).attempt) + .flatMap(_.traverse { + case Right(()) => true.pure[F] + case Left(ex) => + logger.error(ex)("Could not insert job. Skipping it.").as(false) + }) + } + + def insertAllIfNew(jobs: Seq[Job[String]]) = + Timestamp.current[F].flatMap { now => + jobs.toList + .traverse(j => insertIfNew0(j, now).attempt) + .flatMap(_.traverse { + case Right(true) => true.pure[F] + case Right(false) => false.pure[F] + case Left(ex) => + logger.error(ex)("Could not insert job. Skipping it.").as(false) + }) + } + + def toRecord(job: Job[String], timestamp: Timestamp): RJob = + RJob.newJob( + job.id, + job.task, + job.group, + job.args, + job.subject, + timestamp, + job.submitter, + job.priority, + job.tracker + ) +} + +object JobStoreImpl { + def apply[F[_]: Sync](store: Store[F]): JobStore[F] = + new JobStoreImpl[F](store) +} diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobStoreModuleBuilder.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobStoreModuleBuilder.scala new file mode 100644 index 00000000..e5762935 --- /dev/null +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobStoreModuleBuilder.scala @@ -0,0 +1,50 @@ +package docspell.scheduler.impl + +import cats.effect.Async + +import docspell.notification.api.EventSink +import docspell.pubsub.api.PubSubT +import docspell.scheduler._ +import docspell.scheduler.usertask.UserTaskStore +import docspell.store.Store + +case class JobStoreModuleBuilder[F[_]: Async]( + store: Store[F], + pubsub: PubSubT[F], + eventSink: EventSink[F] +) { + def withPubsub(ps: PubSubT[F]): JobStoreModuleBuilder[F] = + copy(pubsub = ps) + + def withEventSink(es: EventSink[F]): JobStoreModuleBuilder[F] = + copy(eventSink = es) + + def build: JobStoreModuleBuilder.Module[F] = { + val jobStore = JobStorePublish(store, pubsub, eventSink) + val periodicTaskStore = PeriodicTaskStore(store, jobStore) + val userTaskStore = UserTaskStoreImpl(store, periodicTaskStore) + new JobStoreModuleBuilder.Module( + userTaskStore, + periodicTaskStore, + jobStore, + store, + eventSink, + pubsub + ) + } +} + +object JobStoreModuleBuilder { + + def apply[F[_]: Async](store: Store[F]): JobStoreModuleBuilder[F] = + JobStoreModuleBuilder(store, PubSubT.noop[F], EventSink.silent[F]) + + final class Module[F[_]]( + val userTasks: UserTaskStore[F], + val periodicTaskStore: PeriodicTaskStore[F], + val jobs: JobStore[F], + val store: Store[F], + val eventSink: EventSink[F], + val pubSubT: PubSubT[F] + ) extends JobStoreModule[F] {} +} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobQueuePublish.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobStorePublish.scala similarity index 60% rename from modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobQueuePublish.scala rename to modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobStorePublish.scala index 7c867a22..f9ced949 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobQueuePublish.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobStorePublish.scala @@ -4,51 +4,52 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -package docspell.scheduler.msg +package docspell.scheduler.impl import cats.effect._ import cats.implicits._ -import docspell.common.{Duration, Ident, Priority} + +import docspell.common.JobState import docspell.notification.api.{Event, EventSink} import docspell.pubsub.api.PubSubT -import docspell.scheduler.JobQueue +import docspell.scheduler._ +import docspell.scheduler.msg.JobSubmitted import docspell.store.Store -import docspell.store.records.RJob -final class JobQueuePublish[F[_]: Sync]( - delegate: JobQueue[F], +final class JobStorePublish[F[_]: Sync]( + delegate: JobStore[F], pubsub: PubSubT[F], eventSink: EventSink[F] -) extends JobQueue[F] { +) extends JobStore[F] { - private def msg(job: RJob): JobSubmitted = + private def msg(job: Job[String]): JobSubmitted = JobSubmitted(job.id, job.group, job.task, job.args) - private def event(job: RJob): Event.JobSubmitted = + private def event(job: Job[String]): Event.JobSubmitted = Event.JobSubmitted( job.id, job.group, job.task, job.args, - job.state, + JobState.waiting, job.subject, job.submitter ) - private def publish(job: RJob): F[Unit] = + private def publish(job: Job[String]): F[Unit] = pubsub.publish1(JobSubmitted.topic, msg(job)).as(()) *> eventSink.offer(event(job)) - def insert(job: RJob) = + def insert(job: Job[String]) = delegate.insert(job).flatTap(_ => publish(job)) - def insertIfNew(job: RJob) = + def insertIfNew(job: Job[String]) = delegate.insertIfNew(job).flatTap { case true => publish(job) case false => ().pure[F] } - def insertAll(jobs: Seq[RJob]) = + def insertAll(jobs: Seq[Job[String]]) = delegate.insertAll(jobs).flatTap { results => results.zip(jobs).traverse { case (res, job) => if (res) publish(job) @@ -56,23 +57,20 @@ final class JobQueuePublish[F[_]: Sync]( } } - def insertAllIfNew(jobs: Seq[RJob]) = + def insertAllIfNew(jobs: Seq[Job[String]]) = delegate.insertAllIfNew(jobs).flatTap { results => results.zip(jobs).traverse { case (res, job) => if (res) publish(job) else ().pure[F] } } - - def nextJob(prio: Ident => F[Priority], worker: Ident, retryPause: Duration) = - delegate.nextJob(prio, worker, retryPause) } -object JobQueuePublish { +object JobStorePublish { def apply[F[_]: Async]( store: Store[F], pubSub: PubSubT[F], eventSink: EventSink[F] - ): Resource[F, JobQueue[F]] = - JobQueue.create(store).map(q => new JobQueuePublish[F](q, pubSub, eventSink)) + ): JobStore[F] = + new JobStorePublish[F](JobStoreImpl(store), pubSub, eventSink) } diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/LogSink.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/LogSink.scala similarity index 96% rename from modules/scheduler/api/src/main/scala/docspell/scheduler/LogSink.scala rename to modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/LogSink.scala index 6bf34f05..8094b90e 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/LogSink.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/LogSink.scala @@ -4,16 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -package docspell.scheduler +package docspell.scheduler.impl import cats.effect._ import cats.implicits._ -import fs2.Pipe - import docspell.common._ import docspell.logging +import docspell.scheduler.LogEvent import docspell.store.Store import docspell.store.records.RJobLog +import fs2.Pipe trait LogSink[F[_]] { diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerBuilder.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerBuilder.scala index f4d1d7ec..a3566f13 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerBuilder.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerBuilder.scala @@ -7,10 +7,8 @@ import fs2.concurrent.SignallingRef object PeriodicSchedulerBuilder { - def build[F[_]: Async]( + def resource[F[_]: Async]( cfg: PeriodicSchedulerConfig, - sch: Scheduler[F], - queue: JobQueue[F], store: PeriodicTaskStore[F], pubsub: PubSubT[F] ): Resource[F, PeriodicScheduler[F]] = @@ -19,8 +17,6 @@ object PeriodicSchedulerBuilder { state <- Resource.eval(SignallingRef(PeriodicSchedulerImpl.emptyState[F])) psch = new PeriodicSchedulerImpl[F]( cfg, - sch, - queue, store, pubsub, waiter, diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerImpl.scala index fd1a3e0d..43d89324 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerImpl.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerImpl.scala @@ -20,8 +20,6 @@ import eu.timepit.fs2cron.calev.CalevScheduler final class PeriodicSchedulerImpl[F[_]: Async]( val config: PeriodicSchedulerConfig, - sch: Scheduler[F], - queue: JobQueue[F], store: PeriodicTaskStore[F], pubSub: PubSubT[F], waiter: SignallingRef[F, Boolean], @@ -119,11 +117,11 @@ final class PeriodicSchedulerImpl[F[_]: Async]( case None => logger.info(s"Submitting job for periodic task '${pj.task.id}'") *> - pj.toJob.flatMap(queue.insert) *> notifyJoex *> true.pure[F] + store.submit(pj) *> notifyJoex *> true.pure[F] } def notifyJoex: F[Unit] = - sch.notifyChange *> pubSub.publish1IgnoreErrors(JobsNotify(), ()).void + pubSub.publish1IgnoreErrors(JobsNotify(), ()).void def scheduleNotify(pj: RPeriodicTask): F[Unit] = Timestamp diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicTaskStore.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicTaskStore.scala index 54846b1f..1f1ecb52 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicTaskStore.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicTaskStore.scala @@ -9,6 +9,7 @@ package docspell.scheduler.impl import cats.effect._ import cats.implicits._ import docspell.common._ +import docspell.scheduler.{Job, JobStore} import docspell.store.queries.QPeriodicTask import docspell.store.records._ import docspell.store.{AddResult, Store} @@ -37,12 +38,18 @@ trait PeriodicTaskStore[F[_]] { /** Find all joex nodes as registered in the database. */ def findJoexNodes: F[Vector[RNode]] + + /** Creates a job from the given task and submits it into the job queue */ + def submit(task: RPeriodicTask): F[Unit] } object PeriodicTaskStore { - def create[F[_]: Sync](store: Store[F]): Resource[F, PeriodicTaskStore[F]] = - Resource.pure[F, PeriodicTaskStore[F]](new PeriodicTaskStore[F] { + def apply[F[_]: Sync]( + store: Store[F], + jobStore: JobStore[F] + ): PeriodicTaskStore[F] = + new PeriodicTaskStore[F] { private[this] val logger = docspell.logging.getLogger[F] def takeNext( worker: Ident, @@ -116,5 +123,22 @@ object PeriodicTaskStore { def findJoexNodes: F[Vector[RNode]] = store.transact(RNode.findAll(NodeType.Joex)) - }) + def submit(task: RPeriodicTask) = + makeJob(task).flatMap(jobStore.insert) + + def makeJob(rt: RPeriodicTask): F[Job[String]] = + Ident.randomId[F].map { id => + Job( + id, + rt.task, + rt.group, + rt.args, + rt.subject, + rt.submitter, + rt.priority, + Some(id) + ) + } + + } } diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/QUserTask.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QUserTask.scala similarity index 79% rename from modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/QUserTask.scala rename to modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QUserTask.scala index f724dfe9..a1ea1632 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/QUserTask.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QUserTask.scala @@ -1,19 +1,48 @@ -package docspell.scheduler.usertask +package docspell.scheduler.impl -import cats.implicits._ import cats.effect.Sync +import cats.syntax.all._ import com.github.eikek.calev.CalEvent -import docspell.common._ +import docspell.common.{AccountId, Ident, Priority, Timestamp} +import docspell.scheduler.usertask.{UserTask, UserTaskScope} import docspell.store.qb.DML import docspell.store.qb.DSL._ import docspell.store.records.RPeriodicTask -import doobie.ConnectionIO import fs2.Stream -import io.circe.Encoder +import io.circe.{Decoder, Encoder} +import doobie._ object QUserTask { private val RT = RPeriodicTask.T + implicit final class UserTaskCodec(ut: UserTask[String]) { + import docspell.common.syntax.all._ + + def decode[A](implicit D: Decoder[A]): Either[String, UserTask[A]] = + ut.args + .parseJsonAs[A] + .left + .map(_.getMessage) + .map(a => ut.copy(args = a)) + + def toPeriodicTask[F[_]: Sync]( + scope: UserTaskScope, + subject: Option[String] + ): F[RPeriodicTask] = + QUserTask + .create[F]( + ut.enabled, + scope, + ut.name, + ut.args, + subject.getOrElse(s"${scope.fold(_.user.id, _.id)}: ${ut.name.id}"), + Priority.Low, + ut.timer, + ut.summary + ) + .map(r => r.copy(id = ut.id)) + } + def findAll(account: AccountId): Stream[ConnectionIO, UserTask[String]] = run( select(RT.all), diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/QueueLogger.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QueueLogger.scala similarity index 85% rename from modules/scheduler/api/src/main/scala/docspell/scheduler/QueueLogger.scala rename to modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QueueLogger.scala index b0f17d23..08a176ec 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/QueueLogger.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QueueLogger.scala @@ -1,19 +1,13 @@ -/* - * Copyright 2020 Eike K. & Contributors - * - * SPDX-License-Identifier: AGPL-3.0-or-later - */ +package docspell.scheduler.impl -package docspell.scheduler - -import cats.effect._ +import cats.syntax.all._ import cats.effect.std.Queue -import cats.implicits._ -import fs2.Stream - -import docspell.common._ +import cats.effect.{Async, Sync} +import docspell.common.{Ident, LogLevel} import docspell.logging import docspell.logging.{Level, Logger} +import docspell.scheduler.LogEvent +import fs2.Stream object QueueLogger { diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerBuilder.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerBuilder.scala index 6cd3cc57..e2cf3286 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerBuilder.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerBuilder.scala @@ -10,7 +10,7 @@ import cats.effect._ import cats.effect.std.Semaphore import cats.implicits._ import fs2.concurrent.SignallingRef -import docspell.scheduler.{JobQueue, _} +import docspell.scheduler._ import docspell.notification.api.EventSink import docspell.pubsub.api.PubSubT import docspell.store.Store @@ -19,7 +19,7 @@ case class SchedulerBuilder[F[_]: Async]( config: SchedulerConfig, tasks: JobTaskRegistry[F], store: Store[F], - queue: Resource[F, JobQueue[F]], + queue: JobQueue[F], logSink: LogSink[F], pubSub: PubSubT[F], eventSink: EventSink[F] @@ -34,14 +34,11 @@ case class SchedulerBuilder[F[_]: Async]( def withTask[A](task: JobTask[F]): SchedulerBuilder[F] = withTaskRegistry(tasks.withTask(task)) - def withQueue(queue: Resource[F, JobQueue[F]]): SchedulerBuilder[F] = - copy(queue = queue) - def withLogSink(sink: LogSink[F]): SchedulerBuilder[F] = copy(logSink = sink) def withQueue(queue: JobQueue[F]): SchedulerBuilder[F] = - copy(queue = Resource.pure[F, JobQueue[F]](queue)) + copy(queue = queue) def withPubSub(pubSubT: PubSubT[F]): SchedulerBuilder[F] = copy(pubSub = pubSubT) @@ -53,14 +50,13 @@ case class SchedulerBuilder[F[_]: Async]( resource.evalMap(sch => Async[F].start(sch.start.compile.drain).map(_ => sch)) def resource: Resource[F, Scheduler[F]] = { - val scheduler: Resource[F, SchedulerImpl[F]] = for { - jq <- queue - waiter <- Resource.eval(SignallingRef(true)) - state <- Resource.eval(SignallingRef(SchedulerImpl.emptyState[F])) - perms <- Resource.eval(Semaphore(config.poolSize.toLong)) + val scheduler: F[SchedulerImpl[F]] = for { + waiter <- SignallingRef(true) + state <- SignallingRef(SchedulerImpl.emptyState[F]) + perms <- Semaphore(config.poolSize.toLong) } yield new SchedulerImpl[F]( config, - jq, + queue, pubSub, eventSink, tasks, @@ -71,7 +67,7 @@ case class SchedulerBuilder[F[_]: Async]( perms ) - scheduler.evalTap(_.init).map(s => s: Scheduler[F]) + Resource.eval(scheduler.flatTap(_.init)).map(s => s: Scheduler[F]) } } @@ -86,10 +82,9 @@ object SchedulerBuilder { config, JobTaskRegistry.empty[F], store, - JobQueue.create(store), + JobQueue(store), LogSink.db[F](store), PubSubT.noop[F], EventSink.silent[F] ) - } diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala index 859cbde2..4aad8901 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala @@ -14,7 +14,7 @@ import fs2.Stream import fs2.concurrent.SignallingRef import docspell.scheduler.msg.{CancelJob, JobDone, JobsNotify} import docspell.common._ -import docspell.scheduler.{JobQueue, _} +import docspell.scheduler._ import docspell.scheduler.impl.SchedulerImpl._ import docspell.notification.api.Event import docspell.notification.api.EventSink @@ -172,7 +172,7 @@ final class SchedulerImpl[F[_]: Async]( for { _ <- logger.debug(s"Creating context for job ${job.info} to run cancellation $t") - ctx <- Context[F, String](job, job.args, config, logSink, store) + ctx <- ContextImpl[F, String](job, job.args, config, logSink, store) _ <- t.onCancel.run(ctx) _ <- state.modify(_.markCancelled(job)) _ <- onFinish(job, JobTaskResult.empty, JobState.Cancelled) @@ -196,7 +196,7 @@ final class SchedulerImpl[F[_]: Async]( case Right(t) => for { _ <- logger.debug(s"Creating context for job ${job.info} to run $t") - ctx <- Context[F, String](job, job.args, config, logSink, store) + ctx <- ContextImpl[F, String](job, job.args, config, logSink, store) jot = wrapTask(job, t.task, ctx) tok <- forkRun(job, jot.run(ctx), t.onCancel.run(ctx), ctx) _ <- state.modify(_.addRunning(job, tok)) diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerModuleBuilder.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerModuleBuilder.scala new file mode 100644 index 00000000..09e6c6a1 --- /dev/null +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerModuleBuilder.scala @@ -0,0 +1,68 @@ +package docspell.scheduler.impl + +import cats.effect._ +import docspell.common.Ident +import docspell.scheduler.{ + JobTaskRegistry, + PeriodicSchedulerConfig, + SchedulerConfig, + SchedulerModule +} + +case class SchedulerModuleBuilder[F[_]: Async] private ( + periodicSchedulerConfig: PeriodicSchedulerConfig, + schedulerBuilder: SchedulerBuilder[F], + jobStoreModule: JobStoreModuleBuilder.Module[F] +) { + + private def configureScheduler( + f: SchedulerBuilder[F] => SchedulerBuilder[F] + ): SchedulerModuleBuilder[F] = + copy(schedulerBuilder = f(schedulerBuilder)) + + def withTaskRegistry(reg: JobTaskRegistry[F]): SchedulerModuleBuilder[F] = + configureScheduler(_.withTaskRegistry(reg)) + + def withSchedulerConfig(cfg: SchedulerConfig): SchedulerModuleBuilder[F] = + configureScheduler(_.withConfig(cfg)) + + def withPeriodicSchedulerConfig( + cfg: PeriodicSchedulerConfig + ): SchedulerModuleBuilder[F] = + copy(periodicSchedulerConfig = cfg) + + def resource: Resource[F, SchedulerModule[F]] = { + val queue = JobQueue(jobStoreModule.store) + for { + schedulerR <- schedulerBuilder + .withPubSub(jobStoreModule.pubSubT) + .withEventSink(jobStoreModule.eventSink) + .withQueue(queue) + .resource + + periodicTaskSchedulerR <- + PeriodicSchedulerBuilder.resource( + periodicSchedulerConfig, + jobStoreModule.periodicTaskStore, + jobStoreModule.pubSubT + ) + } yield new SchedulerModule[F] { + val scheduler = schedulerR + val periodicScheduler = periodicTaskSchedulerR + } + } +} + +object SchedulerModuleBuilder { + + def apply[F[_]: Async]( + jobStoreModule: JobStoreModuleBuilder.Module[F] + ): SchedulerModuleBuilder[F] = { + val id = Ident.unsafe("default-node-id") + new SchedulerModuleBuilder( + PeriodicSchedulerConfig.default(id), + SchedulerBuilder(SchedulerConfig.default(id), jobStoreModule.store), + jobStoreModule + ) + } +} diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/UserTaskStoreImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/UserTaskStoreImpl.scala new file mode 100644 index 00000000..2f90617e --- /dev/null +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/UserTaskStoreImpl.scala @@ -0,0 +1,117 @@ +package docspell.scheduler.impl + +import cats.effect._ +import docspell.scheduler.usertask.UserTaskStore +import cats.data.OptionT +import cats.implicits._ +import docspell.common._ +import docspell.scheduler.usertask._ +import docspell.store.{AddResult, Store} +import fs2.Stream +import io.circe._ +import QUserTask.UserTaskCodec + +final class UserTaskStoreImpl[F[_]: Sync]( + store: Store[F], + periodicTaskStore: PeriodicTaskStore[F] +) extends UserTaskStore[F] { + def getAll(scope: UserTaskScope): Stream[F, UserTask[String]] = + store.transact(QUserTask.findAll(scope.toAccountId)) + + def getByNameRaw(scope: UserTaskScope, name: Ident): Stream[F, UserTask[String]] = + store.transact(QUserTask.findByName(scope.toAccountId, name)) + + def getByIdRaw(scope: UserTaskScope, id: Ident): OptionT[F, UserTask[String]] = + OptionT(store.transact(QUserTask.findById(scope.toAccountId, id))) + + def getByName[A](scope: UserTaskScope, name: Ident)(implicit + D: Decoder[A] + ): Stream[F, UserTask[A]] = + getByNameRaw(scope, name).flatMap(_.decode match { + case Right(ua) => Stream.emit(ua) + case Left(err) => Stream.raiseError[F](new Exception(err)) + }) + + def updateTask[A](scope: UserTaskScope, subject: Option[String], ut: UserTask[A])( + implicit E: Encoder[A] + ): F[Int] = { + val exists = QUserTask.exists(ut.id) + val insert = QUserTask.insert(scope, subject, ut.encode) + store.add(insert, exists).flatMap { + case AddResult.Success => + 1.pure[F] + case AddResult.EntityExists(_) => + store.transact(QUserTask.update(scope, subject, ut.encode)) + case AddResult.Failure(ex) => + Sync[F].raiseError(ex) + } + } + + def deleteTask(scope: UserTaskScope, id: Ident): F[Int] = + store.transact(QUserTask.delete(scope.toAccountId, id)) + + def getOneByNameRaw( + scope: UserTaskScope, + name: Ident + ): OptionT[F, UserTask[String]] = + OptionT( + getByNameRaw(scope, name) + .take(2) + .compile + .toList + .flatMap { + case Nil => (None: Option[UserTask[String]]).pure[F] + case ut :: Nil => ut.some.pure[F] + case _ => Sync[F].raiseError(new Exception("More than one result found")) + } + ) + + def getOneByName[A](scope: UserTaskScope, name: Ident)(implicit + D: Decoder[A] + ): OptionT[F, UserTask[A]] = + getOneByNameRaw(scope, name) + .semiflatMap(_.decode match { + case Right(ua) => ua.pure[F] + case Left(err) => Sync[F].raiseError(new Exception(err)) + }) + + def updateOneTask[A]( + scope: UserTaskScope, + subject: Option[String], + ut: UserTask[A] + )(implicit + E: Encoder[A] + ): F[UserTask[String]] = + getByNameRaw(scope, ut.name).compile.toList.flatMap { + case a :: rest => + val task = ut.copy(id = a.id).encode + for { + _ <- store.transact(QUserTask.update(scope, subject, task)) + _ <- store.transact( + rest.traverse(t => QUserTask.delete(scope.toAccountId, t.id)) + ) + } yield task + case Nil => + val task = ut.encode + store.transact(QUserTask.insert(scope, subject, task)).map(_ => task) + } + + def deleteAll(scope: UserTaskScope, name: Ident): F[Int] = + store.transact(QUserTask.deleteAll(scope.toAccountId, name)) + + def executeNow[A](scope: UserTaskScope, subject: Option[String], task: UserTask[A])( + implicit E: Encoder[A] + ): F[Unit] = + for { + ptask <- task.encode.toPeriodicTask(scope, subject) + _ <- periodicTaskStore.submit(ptask) + } yield () +} + +object UserTaskStoreImpl { + def apply[F[_]: Sync]( + store: Store[F], + periodicTaskStore: PeriodicTaskStore[F] + ): UserTaskStore[F] = + new UserTaskStoreImpl[F](store, periodicTaskStore) +} diff --git a/modules/store/src/main/scala/docspell/store/records/RJob.scala b/modules/store/src/main/scala/docspell/store/records/RJob.scala index 89852623..4804232b 100644 --- a/modules/store/src/main/scala/docspell/store/records/RJob.scala +++ b/modules/store/src/main/scala/docspell/store/records/RJob.scala @@ -48,22 +48,22 @@ case class RJob( object RJob { - def newJob[A]( + def newJob( id: Ident, task: Ident, group: Ident, - args: A, + args: String, subject: String, submitted: Timestamp, submitter: Ident, priority: Priority, tracker: Option[Ident] - )(implicit E: Encoder[A]): RJob = + ): RJob = RJob( id, task, group, - E(args).noSpaces, + args, subject, submitted, submitter, @@ -77,6 +77,29 @@ object RJob { None ) + def fromJson[A]( + id: Ident, + task: Ident, + group: Ident, + args: A, + subject: String, + submitted: Timestamp, + submitter: Ident, + priority: Priority, + tracker: Option[Ident] + )(implicit E: Encoder[A]): RJob = + newJob( + id, + task, + group, + E(args).noSpaces, + subject, + submitted, + submitter, + priority, + tracker + ) + final case class Table(alias: Option[String]) extends TableDef { val tableName = "job" diff --git a/modules/store/src/test/scala/docspell/store/queries/QJobTest.scala b/modules/store/src/test/scala/docspell/store/queries/QJobTest.scala index 8c60f240..adf6facd 100644 --- a/modules/store/src/test/scala/docspell/store/queries/QJobTest.scala +++ b/modules/store/src/test/scala/docspell/store/queries/QJobTest.scala @@ -30,7 +30,7 @@ class QJobTest extends CatsEffectSuite with StoreFixture with TestLoggingConfig private val group2 = Ident.unsafe("group2") def createJob(group: Ident): RJob = - RJob.newJob[Unit]( + RJob.fromJson[Unit]( Ident.unsafe(s"job-${c.incrementAndGet()}"), Ident.unsafe("task"), group, From 90e52efbffb6233ea1fda80c777689b04997c5ec Mon Sep 17 00:00:00 2001 From: eikek Date: Sun, 13 Mar 2022 14:25:09 +0100 Subject: [PATCH 15/16] Remove store dependency from scheduler api --- build.sbt | 4 +- .../scala/docspell/joex/JoexAppImpl.scala | 37 +++++----- .../docspell/joex/filecopy/FileCopyTask.scala | 7 +- .../filecopy/FileIntegrityCheckTask.scala | 5 +- .../scala/docspell/joex/fts/FtsContext.scala | 14 +--- .../scala/docspell/joex/fts/FtsWork.scala | 5 +- .../docspell/joex/fts/MigrationTask.scala | 4 +- .../scala/docspell/joex/fts/ReIndexTask.scala | 5 +- .../docspell/joex/hk/CheckNodesTask.scala | 30 ++++---- .../docspell/joex/hk/CleanupInvitesTask.scala | 7 +- .../docspell/joex/hk/CleanupJobsTask.scala | 5 +- .../joex/hk/CleanupRememberMeTask.scala | 7 +- .../docspell/joex/hk/HouseKeepingTask.scala | 12 ++-- .../docspell/joex/hk/IntegrityCheckTask.scala | 5 +- .../joex/learn/LearnClassifierTask.scala | 32 +++++---- .../joex/learn/LearnItemEntities.scala | 36 ++++++---- .../scala/docspell/joex/learn/LearnTags.scala | 15 ++-- .../docspell/joex/learn/SelectItems.scala | 12 +--- .../joex/learn/StoreClassifierModel.scala | 14 +--- .../scala/docspell/joex/mail/ReadMail.scala | 2 +- .../joex/notify/PeriodicDueItemsTask.scala | 18 +++-- .../joex/notify/PeriodicQueryTask.scala | 28 +++++--- .../joex/pagecount/AllPageCountTask.scala | 12 +++- ...ountTask.scala => MakePageCountTask.scala} | 19 +++--- .../joex/pdfconv/ConvertAllPdfTask.scala | 12 +++- .../docspell/joex/pdfconv/PdfConvTask.scala | 36 +++++----- .../joex/preview/AllPreviewsTask.scala | 12 +++- .../joex/preview/MakePreviewTask.scala | 20 +++--- .../joex/process/AttachmentPageCount.scala | 28 ++++---- .../joex/process/AttachmentPreview.scala | 29 ++++---- .../docspell/joex/process/ConvertPdf.scala | 52 ++++++++------ .../docspell/joex/process/CreateItem.scala | 38 ++++++----- .../joex/process/CrossCheckProposals.scala | 10 +-- .../joex/process/DuplicateCheck.scala | 33 +++++---- .../docspell/joex/process/EvalProposals.scala | 17 ++--- .../joex/process/ExtractArchive.scala | 68 +++++++++++-------- .../docspell/joex/process/FindProposal.scala | 64 +++++++++-------- .../docspell/joex/process/ItemHandler.scala | 67 +++++++++--------- .../docspell/joex/process/LinkProposal.scala | 38 ++++++----- .../docspell/joex/process/ProcessItem.scala | 40 ++++++----- .../docspell/joex/process/ReProcessItem.scala | 37 +++++----- .../docspell/joex/process/SaveProposals.scala | 21 +++--- .../docspell/joex/process/TextAnalysis.scala | 28 +++++--- .../joex/process/TextExtraction.scala | 32 +++++---- .../docspell/joex/routes/JoexRoutes.scala | 16 +++-- .../joex/scanmailbox/ScanMailboxTask.scala | 27 +++++--- .../joex/updatecheck/UpdateCheckTask.scala | 9 +-- .../scala/docspell/scheduler/Context.scala | 3 - .../scala/docspell/scheduler/Scheduler.scala | 3 +- .../docspell/scheduler/impl/ContextImpl.scala | 2 +- .../scheduler/impl/SchedulerImpl.scala | 19 +++++- 51 files changed, 614 insertions(+), 482 deletions(-) rename modules/joex/src/main/scala/docspell/joex/pagecount/{PageCountTask.scala => MakePageCountTask.scala} (71%) diff --git a/build.sbt b/build.sbt index d8ffd775..7f395a82 100644 --- a/build.sbt +++ b/build.sbt @@ -530,7 +530,7 @@ val schedulerApi = project Dependencies.fs2Core ++ Dependencies.circeCore ) - .dependsOn(loggingApi, common, store, pubsubApi) + .dependsOn(loggingApi, common, pubsubApi) val schedulerImpl = project .in(file("modules/scheduler/impl")) @@ -540,7 +540,7 @@ val schedulerImpl = project .settings( name := "docspell-scheduler-impl" ) - .dependsOn(schedulerApi, notificationApi, pubsubApi) + .dependsOn(store, schedulerApi, notificationApi, pubsubApi) val extract = project .in(file("modules/extract")) diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala index ec9d5682..df0f53f0 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala @@ -149,91 +149,91 @@ object JoexAppImpl extends MailAddressCodec { .withTask( JobTask.json( ProcessItemArgs.taskName, - ItemHandler.newItem[F](cfg, itemOps, fts, analyser, regexNer), - ItemHandler.onCancel[F] + ItemHandler.newItem[F](cfg,store, itemOps, fts, analyser, regexNer), + ItemHandler.onCancel[F](store) ) ) .withTask( JobTask.json( ReProcessItemArgs.taskName, - ReProcessItem[F](cfg, fts, itemOps, analyser, regexNer), + ReProcessItem[F](cfg, fts, itemOps, analyser, regexNer, store), ReProcessItem.onCancel[F] ) ) .withTask( JobTask.json( ScanMailboxArgs.taskName, - ScanMailboxTask[F](cfg.userTasks.scanMailbox, javaEmil, upload, joex), + ScanMailboxTask[F](cfg.userTasks.scanMailbox, store, javaEmil, upload, joex), ScanMailboxTask.onCancel[F] ) ) .withTask( JobTask.json( MigrationTask.taskName, - MigrationTask[F](cfg.fullTextSearch, fts, createIndex), + MigrationTask[F](cfg.fullTextSearch, store, fts, createIndex), MigrationTask.onCancel[F] ) ) .withTask( JobTask.json( ReIndexTask.taskName, - ReIndexTask[F](cfg.fullTextSearch, fts, createIndex), + ReIndexTask[F](cfg.fullTextSearch, store, fts, createIndex), ReIndexTask.onCancel[F] ) ) .withTask( JobTask.json( HouseKeepingTask.taskName, - HouseKeepingTask[F](cfg, fileRepo), + HouseKeepingTask[F](cfg, store, fileRepo), HouseKeepingTask.onCancel[F] ) ) .withTask( JobTask.json( PdfConvTask.taskName, - PdfConvTask[F](cfg), + PdfConvTask[F](cfg, store), PdfConvTask.onCancel[F] ) ) .withTask( JobTask.json( ConvertAllPdfArgs.taskName, - ConvertAllPdfTask[F](jobStoreModule.jobs, joex), + ConvertAllPdfTask[F](jobStoreModule.jobs, joex, store), ConvertAllPdfTask.onCancel[F] ) ) .withTask( JobTask.json( LearnClassifierArgs.taskName, - LearnClassifierTask[F](cfg.textAnalysis, analyser), + LearnClassifierTask[F](cfg.textAnalysis, store, analyser), LearnClassifierTask.onCancel[F] ) ) .withTask( JobTask.json( MakePreviewArgs.taskName, - MakePreviewTask[F](cfg.extraction.preview), + MakePreviewTask[F](cfg.extraction.preview, store), MakePreviewTask.onCancel[F] ) ) .withTask( JobTask.json( AllPreviewsArgs.taskName, - AllPreviewsTask[F](jobStoreModule.jobs, joex), + AllPreviewsTask[F](jobStoreModule.jobs, joex, store), AllPreviewsTask.onCancel[F] ) ) .withTask( JobTask.json( MakePageCountArgs.taskName, - MakePageCountTask[F](), + MakePageCountTask[F](store), MakePageCountTask.onCancel[F] ) ) .withTask( JobTask.json( AllPageCountTask.taskName, - AllPageCountTask[F](jobStoreModule.jobs, joex), + AllPageCountTask[F](store, jobStoreModule.jobs, joex), AllPageCountTask.onCancel[F] ) ) @@ -250,6 +250,7 @@ object JoexAppImpl extends MailAddressCodec { UpdateCheckTask[F]( cfg.updateCheck, cfg.sendMail, + store, javaEmil, updateCheck, ThisVersion.default @@ -260,28 +261,28 @@ object JoexAppImpl extends MailAddressCodec { .withTask( JobTask.json( PeriodicQueryTask.taskName, - PeriodicQueryTask[F](notification), + PeriodicQueryTask[F](store, notification), PeriodicQueryTask.onCancel[F] ) ) .withTask( JobTask.json( PeriodicDueItemsTask.taskName, - PeriodicDueItemsTask[F](notification), + PeriodicDueItemsTask[F](store, notification), PeriodicDueItemsTask.onCancel[F] ) ) .withTask( JobTask.json( FileCopyTaskArgs.taskName, - FileCopyTask[F](cfg), + FileCopyTask[F](cfg, store), FileCopyTask.onCancel[F] ) ) .withTask( JobTask.json( FileIntegrityCheckArgs.taskName, - FileIntegrityCheckTask[F](fileRepo), + FileIntegrityCheckTask[F](fileRepo, store), FileIntegrityCheckTask.onCancel[F] ) ) diff --git a/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala b/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala index 2f201363..8c89a157 100644 --- a/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala @@ -17,6 +17,7 @@ import docspell.logging.Logger import docspell.store.file.{BinnyUtils, FileRepository, FileRepositoryConfig} import binny.CopyTool.Counter import binny.{BinaryId, BinaryStore, CopyTool} +import docspell.store.Store import io.circe.generic.semiauto.deriveCodec import io.circe.{Codec, Decoder, Encoder} @@ -69,7 +70,7 @@ object FileCopyTask { def onCancel[F[_]]: Task[F, Args, Unit] = Task.log(_.warn(s"Cancelling ${FileCopyTaskArgs.taskName.id} task")) - def apply[F[_]: Async](cfg: Config): Task[F, Args, CopyResult] = + def apply[F[_]: Async](cfg: Config, store: Store[F]): Task[F, Args, CopyResult] = Task { ctx => val src = ctx.args.from .map(id => @@ -93,8 +94,8 @@ object FileCopyTask { .fromList(targets.filter(_ != srcConfig)) .toRight(CopyResult.noTargetStore) - srcRepo = ctx.store.createFileRepository(srcConfig, true) - targetRepos = trgConfig.map(ctx.store.createFileRepository(_, false)) + srcRepo = store.createFileRepository(srcConfig, true) + targetRepos = trgConfig.map(store.createFileRepository(_, false)) } yield (srcRepo, targetRepos) data match { diff --git a/modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala b/modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala index abd4d01b..1d407453 100644 --- a/modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala @@ -13,6 +13,7 @@ import docspell.backend.ops.OFileRepository import docspell.backend.ops.OFileRepository.IntegrityResult import docspell.common.{FileIntegrityCheckArgs, FileKey} import docspell.scheduler.{JobTaskResultEncoder, Task} +import docspell.store.Store import docspell.store.records.RFileMeta import io.circe.Encoder import io.circe.generic.semiauto.deriveEncoder @@ -54,9 +55,9 @@ object FileIntegrityCheckTask { } } - def apply[F[_]: Sync](ops: OFileRepository[F]): Task[F, Args, Result] = + def apply[F[_]: Sync](ops: OFileRepository[F], store: Store[F]): Task[F, Args, Result] = Task { ctx => - ctx.store + store .transact( RFileMeta .findAll(ctx.args.pattern, 50) diff --git a/modules/joex/src/main/scala/docspell/joex/fts/FtsContext.scala b/modules/joex/src/main/scala/docspell/joex/fts/FtsContext.scala index 41008a1d..2e046d89 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/FtsContext.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/FtsContext.scala @@ -9,25 +9,13 @@ package docspell.joex.fts import docspell.backend.fulltext.CreateIndex import docspell.ftsclient.FtsClient import docspell.joex.Config -import docspell.scheduler.Context import docspell.logging.Logger import docspell.store.Store -case class FtsContext[F[_]]( +final case class FtsContext[F[_]]( cfg: Config.FullTextSearch, store: Store[F], fulltext: CreateIndex[F], fts: FtsClient[F], logger: Logger[F] ) - -object FtsContext { - - def apply[F[_]]( - cfg: Config.FullTextSearch, - fts: FtsClient[F], - fulltext: CreateIndex[F], - ctx: Context[F, _] - ): FtsContext[F] = - FtsContext(cfg, ctx.store, fulltext, fts, ctx.logger) -} diff --git a/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala b/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala index 1a0a256f..8379098f 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala @@ -9,13 +9,13 @@ package docspell.joex.fts import cats._ import cats.data.{Kleisli, NonEmptyList} import cats.implicits._ - import docspell.backend.fulltext.CreateIndex import docspell.common._ import docspell.ftsclient._ import docspell.joex.Config import docspell.scheduler.Context import docspell.logging.Logger +import docspell.store.Store object FtsWork { import syntax._ @@ -106,10 +106,11 @@ object FtsWork { def forContext( cfg: Config.FullTextSearch, + store: Store[F], fts: FtsClient[F], fulltext: CreateIndex[F] ): Kleisli[F, Context[F, _], Unit] = - mt.local(ctx => FtsContext(cfg, fts, fulltext, ctx)) + mt.local(ctx => FtsContext(cfg, store, fulltext, fts, ctx.logger)) } } } diff --git a/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala b/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala index bd0e6fb9..6430b332 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala @@ -13,12 +13,14 @@ import docspell.common._ import docspell.ftsclient._ import docspell.joex.Config import docspell.scheduler.{Job, Task} +import docspell.store.Store object MigrationTask { val taskName = Ident.unsafe("full-text-index") def apply[F[_]: Async]( cfg: Config.FullTextSearch, + store: Store[F], fts: FtsClient[F], createIndex: CreateIndex[F] ): Task[F, Unit, Unit] = @@ -28,7 +30,7 @@ object MigrationTask { Task(ctx => for { migs <- migrationTasks[F](fts) - res <- Migration[F](cfg, fts, ctx.store, createIndex, ctx.logger).run(migs) + res <- Migration[F](cfg, fts, store, createIndex, ctx.logger).run(migs) } yield res ) ) diff --git a/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala b/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala index d9c0e832..ca1dd228 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala @@ -8,13 +8,13 @@ package docspell.joex.fts import cats.effect._ import cats.implicits._ - import docspell.backend.fulltext.CreateIndex import docspell.common._ import docspell.ftsclient._ import docspell.joex.Config import docspell.joex.fts.FtsWork.syntax._ import docspell.scheduler.Task +import docspell.store.Store object ReIndexTask { type Args = ReIndexTaskArgs @@ -24,6 +24,7 @@ object ReIndexTask { def apply[F[_]: Async]( cfg: Config.FullTextSearch, + store: Store[F], fts: FtsClient[F], fulltext: CreateIndex[F] ): Task[F, Args, Unit] = @@ -31,7 +32,7 @@ object ReIndexTask { .log[F, Args](_.info(s"Running full-text re-index now")) .flatMap(_ => Task(ctx => - clearData[F](ctx.args.collective).forContext(cfg, fts, fulltext).run(ctx) + clearData[F](ctx.args.collective).forContext(cfg, store, fts, fulltext).run(ctx) ) ) diff --git a/modules/joex/src/main/scala/docspell/joex/hk/CheckNodesTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/CheckNodesTask.scala index aff4a5d3..1d202b52 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/CheckNodesTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/CheckNodesTask.scala @@ -8,18 +8,18 @@ package docspell.joex.hk import cats.effect._ import cats.implicits._ - import docspell.common._ -import docspell.scheduler.{Context, Task} +import docspell.scheduler.Task import docspell.logging.Logger +import docspell.store.Store import docspell.store.records._ - import org.http4s.blaze.client.BlazeClientBuilder import org.http4s.client.Client object CheckNodesTask { def apply[F[_]: Async]( - cfg: HouseKeepingConfig.CheckNodes + cfg: HouseKeepingConfig.CheckNodes, + store: Store[F] ): Task[F, Unit, CleanupResult] = Task { ctx => if (cfg.enabled) @@ -27,12 +27,12 @@ object CheckNodesTask { _ <- ctx.logger.info("Check nodes reachability") ec = scala.concurrent.ExecutionContext.global _ <- BlazeClientBuilder[F].withExecutionContext(ec).resource.use { client => - checkNodes(ctx, client) + checkNodes(ctx.logger, store, client) } _ <- ctx.logger.info( s"Remove nodes not found more than ${cfg.minNotFound} times" ) - n <- removeNodes(ctx, cfg) + n <- removeNodes(store, cfg) _ <- ctx.logger.info(s"Removed $n nodes") } yield CleanupResult.of(n) else @@ -41,14 +41,18 @@ object CheckNodesTask { } - def checkNodes[F[_]: Async](ctx: Context[F, _], client: Client[F]): F[Unit] = - ctx.store + def checkNodes[F[_]: Async]( + logger: Logger[F], + store: Store[F], + client: Client[F] + ): F[Unit] = + store .transact(RNode.streamAll) .evalMap(node => - checkNode(ctx.logger, client)(node.url) + checkNode(logger, client)(node.url) .flatMap(seen => - if (seen) ctx.store.transact(RNode.resetNotFound(node.id)) - else ctx.store.transact(RNode.incrementNotFound(node.id)) + if (seen) store.transact(RNode.resetNotFound(node.id)) + else store.transact(RNode.incrementNotFound(node.id)) ) ) .compile @@ -68,9 +72,9 @@ object CheckNodesTask { } def removeNodes[F[_]]( - ctx: Context[F, _], + store: Store[F], cfg: HouseKeepingConfig.CheckNodes ): F[Int] = - ctx.store.transact(RNode.deleteNotFound(cfg.minNotFound)) + store.transact(RNode.deleteNotFound(cfg.minNotFound)) } diff --git a/modules/joex/src/main/scala/docspell/joex/hk/CleanupInvitesTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/CleanupInvitesTask.scala index e0cf916c..2dbed981 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/CleanupInvitesTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/CleanupInvitesTask.scala @@ -8,15 +8,16 @@ package docspell.joex.hk import cats.effect._ import cats.implicits._ - import docspell.common._ import docspell.scheduler.Task +import docspell.store.Store import docspell.store.records._ object CleanupInvitesTask { def apply[F[_]: Sync]( - cfg: HouseKeepingConfig.CleanupInvites + cfg: HouseKeepingConfig.CleanupInvites, + store: Store[F] ): Task[F, Unit, CleanupResult] = Task { ctx => if (cfg.enabled) @@ -24,7 +25,7 @@ object CleanupInvitesTask { now <- Timestamp.current[F] ts = now - cfg.olderThan _ <- ctx.logger.info(s"Cleanup invitations older than $ts") - n <- ctx.store.transact(RInvitation.deleteOlderThan(ts)) + n <- store.transact(RInvitation.deleteOlderThan(ts)) _ <- ctx.logger.info(s"Removed $n invitations") } yield CleanupResult.of(n) else diff --git a/modules/joex/src/main/scala/docspell/joex/hk/CleanupJobsTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/CleanupJobsTask.scala index 104ae306..0a4ee43b 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/CleanupJobsTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/CleanupJobsTask.scala @@ -18,7 +18,8 @@ import docspell.store.records._ object CleanupJobsTask { def apply[F[_]: Sync]( - cfg: HouseKeepingConfig.CleanupJobs + cfg: HouseKeepingConfig.CleanupJobs, + store: Store[F] ): Task[F, Unit, CleanupResult] = Task { ctx => if (cfg.enabled) @@ -26,7 +27,7 @@ object CleanupJobsTask { now <- Timestamp.current[F] ts = now - cfg.olderThan _ <- ctx.logger.info(s"Cleanup jobs older than $ts") - n <- deleteDoneJobs(ctx.store, ts, cfg.deleteBatch) + n <- deleteDoneJobs(store, ts, cfg.deleteBatch) _ <- ctx.logger.info(s"Removed $n jobs") } yield CleanupResult.of(n) else diff --git a/modules/joex/src/main/scala/docspell/joex/hk/CleanupRememberMeTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/CleanupRememberMeTask.scala index e1040e1c..b3d909ab 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/CleanupRememberMeTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/CleanupRememberMeTask.scala @@ -8,14 +8,15 @@ package docspell.joex.hk import cats.effect._ import cats.implicits._ - import docspell.common._ import docspell.scheduler.Task +import docspell.store.Store import docspell.store.records._ object CleanupRememberMeTask { def apply[F[_]: Sync]( - cfg: HouseKeepingConfig.CleanupRememberMe + cfg: HouseKeepingConfig.CleanupRememberMe, + store: Store[F] ): Task[F, Unit, CleanupResult] = Task { ctx => if (cfg.enabled) @@ -23,7 +24,7 @@ object CleanupRememberMeTask { now <- Timestamp.current[F] ts = now - cfg.olderThan _ <- ctx.logger.info(s"Cleanup remember-me tokens older than $ts") - n <- ctx.store.transact(RRememberMe.deleteOlderThan(ts)) + n <- store.transact(RRememberMe.deleteOlderThan(ts)) _ <- ctx.logger.info(s"Removed $n tokens") } yield CleanupResult.of(n) else diff --git a/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala index ff025d31..5e87ce17 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala @@ -15,6 +15,7 @@ import docspell.joex.filecopy.FileIntegrityCheckTask import docspell.scheduler.{JobTaskResultEncoder, Task} import com.github.eikek.calev._ import docspell.scheduler.usertask.UserTask +import docspell.store.Store import io.circe.Encoder import io.circe.generic.semiauto.deriveEncoder @@ -25,15 +26,16 @@ object HouseKeepingTask { def apply[F[_]: Async]( cfg: Config, + store: Store[F], fileRepo: OFileRepository[F] ): Task[F, Unit, Result] = { val combined = ( - CheckNodesTask(cfg.houseKeeping.checkNodes), - CleanupInvitesTask(cfg.houseKeeping.cleanupInvites), - CleanupJobsTask(cfg.houseKeeping.cleanupJobs), - CleanupRememberMeTask(cfg.houseKeeping.cleanupRememberMe), - IntegrityCheckTask(cfg.houseKeeping.integrityCheck, fileRepo) + CheckNodesTask(cfg.houseKeeping.checkNodes, store), + CleanupInvitesTask(cfg.houseKeeping.cleanupInvites, store), + CleanupJobsTask(cfg.houseKeeping.cleanupJobs, store), + CleanupRememberMeTask(cfg.houseKeeping.cleanupRememberMe, store), + IntegrityCheckTask(cfg.houseKeeping.integrityCheck, store, fileRepo) ).mapN(Result.apply) Task diff --git a/modules/joex/src/main/scala/docspell/joex/hk/IntegrityCheckTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/IntegrityCheckTask.scala index b9b32ccb..06fca014 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/IntegrityCheckTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/IntegrityCheckTask.scala @@ -8,21 +8,22 @@ package docspell.joex.hk import cats.effect._ import cats.implicits._ - import docspell.backend.ops.OFileRepository import docspell.common._ import docspell.joex.filecopy.FileIntegrityCheckTask import docspell.scheduler.Task +import docspell.store.Store object IntegrityCheckTask { def apply[F[_]: Sync]( cfg: HouseKeepingConfig.IntegrityCheck, + store: Store[F], fileRepo: OFileRepository[F] ): Task[F, Unit, FileIntegrityCheckTask.Result] = Task { ctx => if (cfg.enabled) - FileIntegrityCheckTask(fileRepo).run( + FileIntegrityCheckTask(fileRepo, store).run( ctx.map(_ => FileIntegrityCheckArgs(FileKeyPart.Empty)) ) else diff --git a/modules/joex/src/main/scala/docspell/joex/learn/LearnClassifierTask.scala b/modules/joex/src/main/scala/docspell/joex/learn/LearnClassifierTask.scala index 60fb01e3..ce49077f 100644 --- a/modules/joex/src/main/scala/docspell/joex/learn/LearnClassifierTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/learn/LearnClassifierTask.scala @@ -9,13 +9,13 @@ package docspell.joex.learn import cats.data.OptionT import cats.effect._ import cats.implicits._ - import docspell.analysis.TextAnalyser import docspell.backend.ops.OCollective import docspell.common._ import docspell.joex.Config import docspell.scheduler._ import docspell.logging.Logger +import docspell.store.Store import docspell.store.records.{RClassifierModel, RClassifierSetting} object LearnClassifierTask { @@ -29,14 +29,16 @@ object LearnClassifierTask { def apply[F[_]: Async]( cfg: Config.TextAnalysis, + store: Store[F], analyser: TextAnalyser[F] ): Task[F, Args, Unit] = - learnTags(cfg, analyser) - .flatMap(_ => learnItemEntities(cfg, analyser)) + learnTags(cfg, store, analyser) + .flatMap(_ => learnItemEntities(cfg, store, analyser)) .flatMap(_ => Task(_ => Sync[F].delay(System.gc()))) private def learnItemEntities[F[_]: Async]( cfg: Config.TextAnalysis, + store: Store[F], analyser: TextAnalyser[F] ): Task[F, Args, Unit] = Task { ctx => @@ -44,6 +46,7 @@ object LearnClassifierTask { LearnItemEntities .learnAll( analyser, + store, ctx.args.collective, cfg.classification.itemCount, cfg.maxLength @@ -54,16 +57,17 @@ object LearnClassifierTask { private def learnTags[F[_]: Async]( cfg: Config.TextAnalysis, + store: Store[F], analyser: TextAnalyser[F] ): Task[F, Args, Unit] = Task { ctx => val learnTags = for { - sett <- findActiveSettings[F](ctx, cfg) + sett <- findActiveSettings[F](ctx, store, cfg) maxItems = cfg.classification.itemCountOrWhenLower(sett.itemCount) _ <- OptionT.liftF( LearnTags - .learnAllTagCategories(analyser)( + .learnAllTagCategories(analyser, store)( ctx.args.collective, maxItems, cfg.maxLength @@ -74,34 +78,38 @@ object LearnClassifierTask { // learn classifier models from active tag categories learnTags.getOrElseF(logInactiveWarning(ctx.logger)) *> // delete classifier model files for categories that have been removed - clearObsoleteTagModels(ctx) *> + clearObsoleteTagModels(ctx, store) *> // when tags are deleted, categories may get removed. fix the json array - ctx.store + store .transact(RClassifierSetting.fixCategoryList(ctx.args.collective)) .map(_ => ()) } - private def clearObsoleteTagModels[F[_]: Sync](ctx: Context[F, Args]): F[Unit] = + private def clearObsoleteTagModels[F[_]: Sync]( + ctx: Context[F, Args], + store: Store[F] + ): F[Unit] = for { - list <- ctx.store.transact( + list <- store.transact( ClassifierName.findOrphanTagModels(ctx.args.collective) ) _ <- ctx.logger.info( s"Found ${list.size} obsolete model files that are deleted now." ) - n <- ctx.store.transact(RClassifierModel.deleteAll(list.map(_.id))) + n <- store.transact(RClassifierModel.deleteAll(list.map(_.id))) _ <- list .map(_.fileId) - .traverse(id => ctx.store.fileRepo.delete(id)) + .traverse(id => store.fileRepo.delete(id)) _ <- ctx.logger.debug(s"Deleted $n model files.") } yield () private def findActiveSettings[F[_]: Sync]( ctx: Context[F, Args], + store: Store[F], cfg: Config.TextAnalysis ): OptionT[F, OCollective.Classifier] = if (cfg.classification.enabled) - OptionT(ctx.store.transact(RClassifierSetting.findById(ctx.args.collective))) + OptionT(store.transact(RClassifierSetting.findById(ctx.args.collective))) .filter(_.autoTagEnabled) .map(OCollective.Classifier.fromRecord) else diff --git a/modules/joex/src/main/scala/docspell/joex/learn/LearnItemEntities.scala b/modules/joex/src/main/scala/docspell/joex/learn/LearnItemEntities.scala index cf8e7fe5..7d59afef 100644 --- a/modules/joex/src/main/scala/docspell/joex/learn/LearnItemEntities.scala +++ b/modules/joex/src/main/scala/docspell/joex/learn/LearnItemEntities.scala @@ -10,76 +10,84 @@ import cats.data.Kleisli import cats.effect._ import cats.implicits._ import fs2.Stream - import docspell.analysis.TextAnalyser import docspell.analysis.classifier.TextClassifier.Data import docspell.common._ import docspell.scheduler._ +import docspell.store.Store object LearnItemEntities { def learnAll[F[_]: Async, A]( analyser: TextAnalyser[F], + store: Store[F], collective: Ident, maxItems: Int, maxTextLen: Int ): Task[F, A, Unit] = - learnCorrOrg[F, A](analyser, collective, maxItems, maxTextLen) - .flatMap(_ => learnCorrPerson[F, A](analyser, collective, maxItems, maxTextLen)) - .flatMap(_ => learnConcPerson(analyser, collective, maxItems, maxTextLen)) - .flatMap(_ => learnConcEquip(analyser, collective, maxItems, maxTextLen)) + learnCorrOrg[F, A](analyser, store, collective, maxItems, maxTextLen) + .flatMap(_ => + learnCorrPerson[F, A](analyser, store, collective, maxItems, maxTextLen) + ) + .flatMap(_ => learnConcPerson(analyser, store, collective, maxItems, maxTextLen)) + .flatMap(_ => learnConcEquip(analyser, store, collective, maxItems, maxTextLen)) def learnCorrOrg[F[_]: Async, A]( analyser: TextAnalyser[F], + store: Store[F], collective: Ident, maxItems: Int, maxTextLen: Int ): Task[F, A, Unit] = - learn(analyser, collective)( + learn(store, analyser, collective)( ClassifierName.correspondentOrg, - ctx => SelectItems.forCorrOrg(ctx.store, collective, maxItems, maxTextLen) + _ => SelectItems.forCorrOrg(store, collective, maxItems, maxTextLen) ) def learnCorrPerson[F[_]: Async, A]( analyser: TextAnalyser[F], + store: Store[F], collective: Ident, maxItems: Int, maxTextLen: Int ): Task[F, A, Unit] = - learn(analyser, collective)( + learn(store, analyser, collective)( ClassifierName.correspondentPerson, - ctx => SelectItems.forCorrPerson(ctx.store, collective, maxItems, maxTextLen) + _ => SelectItems.forCorrPerson(store, collective, maxItems, maxTextLen) ) def learnConcPerson[F[_]: Async, A]( analyser: TextAnalyser[F], + store: Store[F], collective: Ident, maxItems: Int, maxTextLen: Int ): Task[F, A, Unit] = - learn(analyser, collective)( + learn(store, analyser, collective)( ClassifierName.concernedPerson, - ctx => SelectItems.forConcPerson(ctx.store, collective, maxItems, maxTextLen) + _ => SelectItems.forConcPerson(store, collective, maxItems, maxTextLen) ) def learnConcEquip[F[_]: Async, A]( analyser: TextAnalyser[F], + store: Store[F], collective: Ident, maxItems: Int, maxTextLen: Int ): Task[F, A, Unit] = - learn(analyser, collective)( + learn(store, analyser, collective)( ClassifierName.concernedEquip, - ctx => SelectItems.forConcEquip(ctx.store, collective, maxItems, maxTextLen) + _ => SelectItems.forConcEquip(store, collective, maxItems, maxTextLen) ) private def learn[F[_]: Async, A]( + store: Store[F], analyser: TextAnalyser[F], collective: Ident )(cname: ClassifierName, data: Context[F, _] => Stream[F, Data]): Task[F, A, Unit] = Task { ctx => ctx.logger.info(s"Learn classifier ${cname.name}") *> analyser.classifier.trainClassifier(ctx.logger, data(ctx))( - Kleisli(StoreClassifierModel.handleModel(ctx, collective, cname)) + Kleisli(StoreClassifierModel.handleModel(store, ctx.logger, collective, cname)) ) } } diff --git a/modules/joex/src/main/scala/docspell/joex/learn/LearnTags.scala b/modules/joex/src/main/scala/docspell/joex/learn/LearnTags.scala index f1460c19..54eb5a88 100644 --- a/modules/joex/src/main/scala/docspell/joex/learn/LearnTags.scala +++ b/modules/joex/src/main/scala/docspell/joex/learn/LearnTags.scala @@ -9,16 +9,17 @@ package docspell.joex.learn import cats.data.Kleisli import cats.effect._ import cats.implicits._ - import docspell.analysis.TextAnalyser import docspell.common._ import docspell.scheduler._ +import docspell.store.Store import docspell.store.records.RClassifierSetting object LearnTags { def learnTagCategory[F[_]: Async, A]( analyser: TextAnalyser[F], + store: Store[F], collective: Ident, maxItems: Int, maxTextLen: Int @@ -26,12 +27,14 @@ object LearnTags { category: String ): Task[F, A, Unit] = Task { ctx => - val data = SelectItems.forCategory(ctx, collective)(maxItems, category, maxTextLen) + val data = + SelectItems.forCategory(store, collective)(maxItems, category, maxTextLen) ctx.logger.info(s"Learn classifier for tag category: $category") *> analyser.classifier.trainClassifier(ctx.logger, data)( Kleisli( StoreClassifierModel.handleModel( - ctx, + store, + ctx.logger, collective, ClassifierName.tagCategory(category) ) @@ -39,15 +42,15 @@ object LearnTags { ) } - def learnAllTagCategories[F[_]: Async, A](analyser: TextAnalyser[F])( + def learnAllTagCategories[F[_]: Async, A](analyser: TextAnalyser[F], store: Store[F])( collective: Ident, maxItems: Int, maxTextLen: Int ): Task[F, A, Unit] = Task { ctx => for { - cats <- ctx.store.transact(RClassifierSetting.getActiveCategories(collective)) - task = learnTagCategory[F, A](analyser, collective, maxItems, maxTextLen) _ + cats <- store.transact(RClassifierSetting.getActiveCategories(collective)) + task = learnTagCategory[F, A](analyser, store, collective, maxItems, maxTextLen) _ _ <- cats.map(task).traverse(_.run(ctx)) } yield () } diff --git a/modules/joex/src/main/scala/docspell/joex/learn/SelectItems.scala b/modules/joex/src/main/scala/docspell/joex/learn/SelectItems.scala index b25f28ce..a54d7f7e 100644 --- a/modules/joex/src/main/scala/docspell/joex/learn/SelectItems.scala +++ b/modules/joex/src/main/scala/docspell/joex/learn/SelectItems.scala @@ -10,7 +10,6 @@ import fs2.{Pipe, Stream} import docspell.analysis.classifier.TextClassifier.Data import docspell.common._ -import docspell.scheduler.Context import docspell.store.Store import docspell.store.qb.Batch import docspell.store.queries.{QItem, TextAndTag} @@ -21,16 +20,7 @@ object SelectItems { val pageSep = LearnClassifierTask.pageSep val noClass = LearnClassifierTask.noClass - def forCategory[F[_]](ctx: Context[F, _], collective: Ident)( - maxItems: Int, - category: String, - maxTextLen: Int - ): Stream[F, Data] = - forCategory(ctx.store, collective, maxItems, category, maxTextLen) - - def forCategory[F[_]]( - store: Store[F], - collective: Ident, + def forCategory[F[_]](store: Store[F], collective: Ident)( maxItems: Int, category: String, maxTextLen: Int diff --git a/modules/joex/src/main/scala/docspell/joex/learn/StoreClassifierModel.scala b/modules/joex/src/main/scala/docspell/joex/learn/StoreClassifierModel.scala index 8c4feb9b..9d5aafe9 100644 --- a/modules/joex/src/main/scala/docspell/joex/learn/StoreClassifierModel.scala +++ b/modules/joex/src/main/scala/docspell/joex/learn/StoreClassifierModel.scala @@ -12,7 +12,6 @@ import fs2.io.file.Files import docspell.analysis.classifier.ClassifierModel import docspell.common._ -import docspell.scheduler._ import docspell.logging.Logger import docspell.store.Store import docspell.store.records.RClassifierModel @@ -20,21 +19,12 @@ import docspell.store.records.RClassifierModel object StoreClassifierModel { def handleModel[F[_]: Async]( - ctx: Context[F, _], + store: Store[F], + logger: Logger[F], collective: Ident, modelName: ClassifierName )( trainedModel: ClassifierModel - ): F[Unit] = - handleModel(ctx.store, ctx.logger)(collective, modelName, trainedModel) - - def handleModel[F[_]: Async]( - store: Store[F], - logger: Logger[F] - )( - collective: Ident, - modelName: ClassifierName, - trainedModel: ClassifierModel ): F[Unit] = for { oldFile <- store.transact( diff --git a/modules/joex/src/main/scala/docspell/joex/mail/ReadMail.scala b/modules/joex/src/main/scala/docspell/joex/mail/ReadMail.scala index 9b2db148..66450864 100644 --- a/modules/joex/src/main/scala/docspell/joex/mail/ReadMail.scala +++ b/modules/joex/src/main/scala/docspell/joex/mail/ReadMail.scala @@ -70,7 +70,7 @@ object ReadMail { HtmlBodyViewConfig.default.copy( textToHtml = MarkdownBody.makeHtml(markdownCfg) ) - ).map(makeHtmlBinary[F] _).map(b => Some(b)) + ).map(makeHtmlBinary[F]).map(b => Some(b)) } for { diff --git a/modules/joex/src/main/scala/docspell/joex/notify/PeriodicDueItemsTask.scala b/modules/joex/src/main/scala/docspell/joex/notify/PeriodicDueItemsTask.scala index 69361b69..1969a1aa 100644 --- a/modules/joex/src/main/scala/docspell/joex/notify/PeriodicDueItemsTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/notify/PeriodicDueItemsTask.scala @@ -9,7 +9,6 @@ package docspell.joex.notify import cats.data.NonEmptyList import cats.effect._ import cats.implicits._ - import docspell.backend.ops.ONotification import docspell.common._ import docspell.scheduler.Context @@ -20,6 +19,7 @@ import docspell.notification.api.PeriodicDueItemsArgs import docspell.query.Date import docspell.query.ItemQuery._ import docspell.query.ItemQueryDsl._ +import docspell.store.Store import docspell.store.qb.Batch import docspell.store.queries.ListItem import docspell.store.queries.{QItem, Query} @@ -32,11 +32,14 @@ object PeriodicDueItemsTask { def onCancel[F[_]]: Task[F, Args, Unit] = Task.log(_.warn(s"Cancelling ${taskName.id} task")) - def apply[F[_]: Sync](notificationOps: ONotification[F]): Task[F, Args, Unit] = + def apply[F[_]: Sync]( + store: Store[F], + notificationOps: ONotification[F] + ): Task[F, Args, Unit] = Task { ctx => val limit = 7 Timestamp.current[F].flatMap { now => - withItems(ctx, limit, now) { items => + withItems(ctx, store, limit, now) { items => withEventContext(ctx, items, limit, now) { eventCtx => withChannel(ctx, notificationOps) { channels => notificationOps.sendMessage(ctx.logger, eventCtx, channels) @@ -51,7 +54,12 @@ object PeriodicDueItemsTask { ): F[Unit] = TaskOperations.withChannel(ctx.logger, ctx.args.channels, ctx.args.account, ops)(cont) - def withItems[F[_]: Sync](ctx: Context[F, Args], limit: Int, now: Timestamp)( + def withItems[F[_]: Sync]( + ctx: Context[F, Args], + store: Store[F], + limit: Int, + now: Timestamp + )( cont: Vector[ListItem] => F[Unit] ): F[Unit] = { val rightDate = Date((now + Duration.days(ctx.args.remindDays.toLong)).toMillis) @@ -77,7 +85,7 @@ object PeriodicDueItemsTask { for { res <- - ctx.store + store .transact( QItem .findItems(q, now.toUtcDate, 0, Batch.limit(limit)) diff --git a/modules/joex/src/main/scala/docspell/joex/notify/PeriodicQueryTask.scala b/modules/joex/src/main/scala/docspell/joex/notify/PeriodicQueryTask.scala index 8ed45a86..3b08edca 100644 --- a/modules/joex/src/main/scala/docspell/joex/notify/PeriodicQueryTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/notify/PeriodicQueryTask.scala @@ -10,7 +10,6 @@ import cats.data.OptionT import cats.data.{NonEmptyList => Nel} import cats.effect._ import cats.implicits._ - import docspell.backend.ops.ONotification import docspell.common._ import docspell.scheduler.Context @@ -22,6 +21,7 @@ import docspell.query.ItemQuery import docspell.query.ItemQuery.Expr import docspell.query.ItemQuery.Expr.AndExpr import docspell.query.ItemQueryParser +import docspell.store.Store import docspell.store.qb.Batch import docspell.store.queries.ListItem import docspell.store.queries.{QItem, Query} @@ -36,11 +36,14 @@ object PeriodicQueryTask { def onCancel[F[_]]: Task[F, Args, Unit] = Task.log(_.warn(s"Cancelling ${taskName.id} task")) - def apply[F[_]: Sync](notificationOps: ONotification[F]): Task[F, Args, Unit] = + def apply[F[_]: Sync]( + store: Store[F], + notificationOps: ONotification[F] + ): Task[F, Args, Unit] = Task { ctx => val limit = 7 Timestamp.current[F].flatMap { now => - withItems(ctx, limit, now) { items => + withItems(ctx, store, limit, now) { items => withEventContext(ctx, items, limit, now) { eventCtx => withChannel(ctx, notificationOps) { channels => notificationOps.sendMessage(ctx.logger, eventCtx, channels) @@ -58,9 +61,11 @@ object PeriodicQueryTask { private def queryString(q: ItemQuery.Expr) = ItemQueryParser.asString(q) - def withQuery[F[_]: Sync](ctx: Context[F, Args])(cont: Query => F[Unit]): F[Unit] = { + def withQuery[F[_]: Sync](ctx: Context[F, Args], store: Store[F])( + cont: Query => F[Unit] + ): F[Unit] = { def fromBookmark(id: String) = - ctx.store + store .transact(RQueryBookmark.findByNameOrId(ctx.args.account, id)) .map(_.map(_.query)) .flatTap(q => @@ -68,7 +73,7 @@ object PeriodicQueryTask { ) def fromShare(id: String) = - ctx.store + store .transact(RShare.findOneByCollective(ctx.args.account.collective, Some(true), id)) .map(_.map(_.query)) .flatTap(q => @@ -120,11 +125,16 @@ object PeriodicQueryTask { } } - def withItems[F[_]: Sync](ctx: Context[F, Args], limit: Int, now: Timestamp)( + def withItems[F[_]: Sync]( + ctx: Context[F, Args], + store: Store[F], + limit: Int, + now: Timestamp + )( cont: Vector[ListItem] => F[Unit] ): F[Unit] = - withQuery(ctx) { query => - val items = ctx.store + withQuery(ctx, store) { query => + val items = store .transact(QItem.findItems(query, now.toUtcDate, 0, Batch.limit(limit))) .compile .to(Vector) diff --git a/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala b/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala index 5b8448d9..6ff6b578 100644 --- a/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala @@ -13,6 +13,7 @@ import docspell.backend.JobFactory import docspell.backend.ops.OJoex import docspell.common._ import docspell.scheduler.{Context, Job, JobStore, Task} +import docspell.store.Store import docspell.store.records.RAttachment object AllPageCountTask { @@ -20,11 +21,15 @@ object AllPageCountTask { val taskName = Ident.unsafe("all-page-count") type Args = Unit - def apply[F[_]: Sync](jobStore: JobStore[F], joex: OJoex[F]): Task[F, Args, Unit] = + def apply[F[_]: Sync]( + store: Store[F], + jobStore: JobStore[F], + joex: OJoex[F] + ): Task[F, Args, Unit] = Task { ctx => for { _ <- ctx.logger.info("Generating previews for attachments") - n <- submitConversionJobs(ctx, jobStore) + n <- submitConversionJobs(ctx, store, jobStore) _ <- ctx.logger.info(s"Submitted $n jobs") _ <- joex.notifyAllNodes } yield () @@ -35,9 +40,10 @@ object AllPageCountTask { def submitConversionJobs[F[_]: Sync]( ctx: Context[F, Args], + store: Store[F], jobStore: JobStore[F] ): F[Int] = - ctx.store + store .transact(findAttachments) .chunks .flatMap(createJobs[F]) diff --git a/modules/joex/src/main/scala/docspell/joex/pagecount/PageCountTask.scala b/modules/joex/src/main/scala/docspell/joex/pagecount/MakePageCountTask.scala similarity index 71% rename from modules/joex/src/main/scala/docspell/joex/pagecount/PageCountTask.scala rename to modules/joex/src/main/scala/docspell/joex/pagecount/MakePageCountTask.scala index 5f4e73fa..bcb5fd77 100644 --- a/modules/joex/src/main/scala/docspell/joex/pagecount/PageCountTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/pagecount/MakePageCountTask.scala @@ -8,11 +8,11 @@ package docspell.joex.pagecount import cats.effect._ import cats.implicits._ - import docspell.common._ import docspell.joex.process.AttachmentPageCount import docspell.scheduler.Context import docspell.scheduler.Task +import docspell.store.Store import docspell.store.records.RAttachment import docspell.store.records.RAttachmentMeta @@ -20,10 +20,10 @@ object MakePageCountTask { type Args = MakePageCountArgs - def apply[F[_]: Sync](): Task[F, Args, Unit] = + def apply[F[_]: Sync](store: Store[F]): Task[F, Args, Unit] = Task { ctx => for { - exists <- pageCountExists(ctx) + exists <- pageCountExists(ctx, store) _ <- if (exists) ctx.logger.info( @@ -32,7 +32,7 @@ object MakePageCountTask { else ctx.logger.info( s"Reading page-count for attachment ${ctx.args.attachment}" - ) *> generatePageCount(ctx) + ) *> generatePageCount(ctx, store) } yield () } @@ -40,19 +40,20 @@ object MakePageCountTask { Task.log(_.warn("Cancelling make-page-count task")) private def generatePageCount[F[_]: Sync]( - ctx: Context[F, Args] + ctx: Context[F, Args], + store: Store[F] ): F[Unit] = for { - ra <- ctx.store.transact(RAttachment.findById(ctx.args.attachment)) + ra <- store.transact(RAttachment.findById(ctx.args.attachment)) _ <- ra - .map(AttachmentPageCount.createPageCount(ctx)) + .map(AttachmentPageCount.createPageCount(ctx, store)) .getOrElse( ctx.logger.warn(s"No attachment found with id: ${ctx.args.attachment}") ) } yield () - private def pageCountExists[F[_]](ctx: Context[F, Args]): F[Boolean] = - ctx.store.transact( + private def pageCountExists[F[_]](ctx: Context[F, Args], store: Store[F]): F[Boolean] = + store.transact( RAttachmentMeta .findPageCountById(ctx.args.attachment) .map(_.exists(_ > 0)) diff --git a/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala b/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala index f1b30c5a..47b66323 100644 --- a/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala @@ -12,6 +12,7 @@ import fs2.{Chunk, Stream} import docspell.backend.ops.OJoex import docspell.common._ import docspell.scheduler.{Context, Job, JobStore, Task} +import docspell.store.Store import docspell.store.records.RAttachment /* A task to find all non-converted pdf files (of a collective, or @@ -21,11 +22,15 @@ import docspell.store.records.RAttachment object ConvertAllPdfTask { type Args = ConvertAllPdfArgs - def apply[F[_]: Sync](jobStore: JobStore[F], joex: OJoex[F]): Task[F, Args, Unit] = + def apply[F[_]: Sync]( + jobStore: JobStore[F], + joex: OJoex[F], + store: Store[F] + ): Task[F, Args, Unit] = Task { ctx => for { _ <- ctx.logger.info("Converting pdfs using ocrmypdf") - n <- submitConversionJobs(ctx, jobStore) + n <- submitConversionJobs(ctx, store, jobStore) _ <- ctx.logger.info(s"Submitted $n file conversion jobs") _ <- joex.notifyAllNodes } yield () @@ -36,9 +41,10 @@ object ConvertAllPdfTask { def submitConversionJobs[F[_]: Sync]( ctx: Context[F, Args], + store: Store[F], jobStore: JobStore[F] ): F[Int] = - ctx.store + store .transact(RAttachment.findNonConvertedPdf(ctx.args.collective, 50)) .chunks .flatMap(createJobs[F](ctx)) diff --git a/modules/joex/src/main/scala/docspell/joex/pdfconv/PdfConvTask.scala b/modules/joex/src/main/scala/docspell/joex/pdfconv/PdfConvTask.scala index 47843a0b..7bf6fd5c 100644 --- a/modules/joex/src/main/scala/docspell/joex/pdfconv/PdfConvTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/pdfconv/PdfConvTask.scala @@ -11,14 +11,13 @@ import cats.data.OptionT import cats.effect._ import cats.implicits._ import fs2.Stream - import docspell.common._ import docspell.convert.ConversionResult import docspell.convert.extern.OcrMyPdf import docspell.joex.Config import docspell.scheduler.{Context, Task} +import docspell.store.Store import docspell.store.records._ - import io.circe.generic.semiauto._ import io.circe.{Decoder, Encoder} @@ -36,12 +35,12 @@ object PdfConvTask { val taskName = Ident.unsafe("pdf-files-migration") - def apply[F[_]: Async](cfg: Config): Task[F, Args, Unit] = + def apply[F[_]: Async](cfg: Config, store: Store[F]): Task[F, Args, Unit] = Task { ctx => for { _ <- ctx.logger.info(s"Converting pdf file ${ctx.args} using ocrmypdf") - meta <- checkInputs(cfg, ctx) - _ <- meta.traverse(fm => convert(cfg, ctx, fm)) + meta <- checkInputs(cfg, ctx, store) + _ <- meta.traverse(fm => convert(cfg, ctx, store, fm)) } yield () } @@ -53,19 +52,20 @@ object PdfConvTask { // check if file exists and if it is pdf and if source id is the same and if ocrmypdf is enabled def checkInputs[F[_]: Sync]( cfg: Config, - ctx: Context[F, Args] + ctx: Context[F, Args], + store: Store[F] ): F[Option[RFileMeta]] = { val none: Option[RFileMeta] = None val checkSameFiles = (for { - ra <- OptionT(ctx.store.transact(RAttachment.findById(ctx.args.attachId))) + ra <- OptionT(store.transact(RAttachment.findById(ctx.args.attachId))) isSame <- OptionT.liftF( - ctx.store.transact(RAttachmentSource.isSameFile(ra.id, ra.fileId)) + store.transact(RAttachmentSource.isSameFile(ra.id, ra.fileId)) ) } yield isSame).getOrElse(false) val existsPdf = for { - meta <- ctx.store.transact(RAttachment.findMeta(ctx.args.attachId)) + meta <- store.transact(RAttachment.findMeta(ctx.args.attachId)) res = meta.filter(_.mimetype.matches(MimeType.pdf)) _ <- if (res.isEmpty) @@ -90,18 +90,19 @@ object PdfConvTask { def convert[F[_]: Async]( cfg: Config, ctx: Context[F, Args], + store: Store[F], in: RFileMeta ): F[Unit] = { - val fs = ctx.store.fileRepo + val fs = store.fileRepo val data = fs.getBytes(in.id) val storeResult: ConversionResult.Handler[F, Unit] = Kleisli { case ConversionResult.SuccessPdf(file) => - storeToAttachment(ctx, in, file) + storeToAttachment(ctx, store, in, file) case ConversionResult.SuccessPdfTxt(file, _) => - storeToAttachment(ctx, in, file) + storeToAttachment(ctx, store, in, file) case ConversionResult.UnsupportedFormat(mime) => ctx.logger.warn( @@ -124,19 +125,20 @@ object PdfConvTask { )(data, storeResult) for { - lang <- getLanguage(ctx) + lang <- getLanguage(ctx, store) _ <- ocrMyPdf(lang) } yield () } - def getLanguage[F[_]: Sync](ctx: Context[F, Args]): F[Language] = + def getLanguage[F[_]: Sync](ctx: Context[F, Args], store: Store[F]): F[Language] = (for { - coll <- OptionT(ctx.store.transact(RCollective.findByAttachment(ctx.args.attachId))) + coll <- OptionT(store.transact(RCollective.findByAttachment(ctx.args.attachId))) lang = coll.language } yield lang).getOrElse(Language.German) def storeToAttachment[F[_]: Sync]( ctx: Context[F, Args], + store: Store[F], meta: RFileMeta, newFile: Stream[F, Byte] ): F[Unit] = { @@ -146,10 +148,10 @@ object PdfConvTask { for { fid <- newFile - .through(ctx.store.fileRepo.save(collective, cat, mimeHint)) + .through(store.fileRepo.save(collective, cat, mimeHint)) .compile .lastOrError - _ <- ctx.store.transact(RAttachment.updateFileId(ctx.args.attachId, fid)) + _ <- store.transact(RAttachment.updateFileId(ctx.args.attachId, fid)) } yield () } } diff --git a/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala b/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala index 100ace8f..dcf66d56 100644 --- a/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala @@ -14,17 +14,22 @@ import docspell.backend.ops.OJoex import docspell.common.MakePreviewArgs.StoreMode import docspell.common._ import docspell.scheduler.{Context, Job, JobStore, Task} +import docspell.store.Store import docspell.store.records.RAttachment object AllPreviewsTask { type Args = AllPreviewsArgs - def apply[F[_]: Sync](jobStore: JobStore[F], joex: OJoex[F]): Task[F, Args, Unit] = + def apply[F[_]: Sync]( + jobStore: JobStore[F], + joex: OJoex[F], + store: Store[F] + ): Task[F, Args, Unit] = Task { ctx => for { _ <- ctx.logger.info("Generating previews for attachments") - n <- submitConversionJobs(ctx, jobStore) + n <- submitConversionJobs(ctx, store, jobStore) _ <- ctx.logger.info(s"Submitted $n jobs") _ <- joex.notifyAllNodes } yield () @@ -35,9 +40,10 @@ object AllPreviewsTask { def submitConversionJobs[F[_]: Sync]( ctx: Context[F, Args], + store: Store[F], jobStore: JobStore[F] ): F[Int] = - ctx.store + store .transact(findAttachments(ctx)) .chunks .flatMap(createJobs[F](ctx)) diff --git a/modules/joex/src/main/scala/docspell/joex/preview/MakePreviewTask.scala b/modules/joex/src/main/scala/docspell/joex/preview/MakePreviewTask.scala index d20a6c75..afd541ea 100644 --- a/modules/joex/src/main/scala/docspell/joex/preview/MakePreviewTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/preview/MakePreviewTask.scala @@ -8,13 +8,13 @@ package docspell.joex.preview import cats.effect._ import cats.implicits._ - import docspell.common._ import docspell.extract.pdfbox.PdfboxPreview import docspell.extract.pdfbox.PreviewConfig import docspell.joex.process.AttachmentPreview import docspell.scheduler.Context import docspell.scheduler.Task +import docspell.store.Store import docspell.store.records.RAttachment import docspell.store.records.RAttachmentPreview @@ -22,10 +22,10 @@ object MakePreviewTask { type Args = MakePreviewArgs - def apply[F[_]: Sync](pcfg: PreviewConfig): Task[F, Args, Unit] = + def apply[F[_]: Sync](pcfg: PreviewConfig, store: Store[F]): Task[F, Args, Unit] = Task { ctx => for { - exists <- previewExists(ctx) + exists <- previewExists(ctx, store) preview <- PdfboxPreview(pcfg) _ <- if (exists) @@ -35,7 +35,7 @@ object MakePreviewTask { else ctx.logger.info( s"Generating preview image for attachment ${ctx.args.attachment}" - ) *> generatePreview(ctx, preview) + ) *> generatePreview(ctx, store, preview) } yield () } @@ -44,20 +44,24 @@ object MakePreviewTask { private def generatePreview[F[_]: Sync]( ctx: Context[F, Args], + store: Store[F], preview: PdfboxPreview[F] ): F[Unit] = for { - ra <- ctx.store.transact(RAttachment.findById(ctx.args.attachment)) + ra <- store.transact(RAttachment.findById(ctx.args.attachment)) _ <- ra - .map(AttachmentPreview.createPreview(ctx, preview)) + .map(AttachmentPreview.createPreview(ctx, store, preview)) .getOrElse( ctx.logger.error(s"No attachment found with id: ${ctx.args.attachment}") ) } yield () - private def previewExists[F[_]: Sync](ctx: Context[F, Args]): F[Boolean] = + private def previewExists[F[_]: Sync]( + ctx: Context[F, Args], + store: Store[F] + ): F[Boolean] = if (ctx.args.store == MakePreviewArgs.StoreMode.WhenMissing) - ctx.store.transact( + store.transact( RAttachmentPreview.findById(ctx.args.attachment).map(_.isDefined) ) else diff --git a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala index f6444311..1e216897 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala @@ -11,11 +11,11 @@ import cats.data.OptionT import cats.effect._ import cats.implicits._ import fs2.Stream - import docspell.common._ import docspell.extract.pdfbox.PdfMetaData import docspell.extract.pdfbox.PdfboxExtract import docspell.scheduler._ +import docspell.store.Store import docspell.store.records.RAttachment import docspell.store.records._ @@ -24,7 +24,7 @@ import docspell.store.records._ */ object AttachmentPageCount { - def apply[F[_]: Sync]()( + def apply[F[_]: Sync](store: Store[F])( item: ItemData ): Task[F, ProcessItemArgs, ItemData] = Task { ctx => @@ -33,7 +33,7 @@ object AttachmentPageCount { s"Retrieving page count for ${item.attachments.size} files…" ) _ <- item.attachments - .traverse(createPageCount(ctx)) + .traverse(createPageCount(ctx, store)) .attempt .flatMap { case Right(_) => ().pure[F] @@ -46,14 +46,15 @@ object AttachmentPageCount { } def createPageCount[F[_]: Sync]( - ctx: Context[F, _] + ctx: Context[F, _], + store: Store[F] )(ra: RAttachment): F[Option[PdfMetaData]] = - findMime[F](ctx)(ra).flatMap { + findMime[F](store)(ra).flatMap { case MimeType.PdfMatch(_) => - PdfboxExtract.getMetaData(loadFile(ctx)(ra)).flatMap { + PdfboxExtract.getMetaData(loadFile(store)(ra)).flatMap { case Right(md) => ctx.logger.debug(s"Found number of pages: ${md.pageCount}") *> - updatePageCount(ctx, md, ra).map(_.some) + updatePageCount(ctx, store, md, ra).map(_.some) case Left(ex) => ctx.logger.warn(s"Error obtaining pages count: ${ex.getMessage}") *> (None: Option[PdfMetaData]).pure[F] @@ -66,6 +67,7 @@ object AttachmentPageCount { private def updatePageCount[F[_]: Sync]( ctx: Context[F, _], + store: Store[F], md: PdfMetaData, ra: RAttachment ): F[PdfMetaData] = @@ -73,12 +75,12 @@ object AttachmentPageCount { _ <- ctx.logger.debug( s"Update attachment ${ra.id.id} with page count ${md.pageCount.some}" ) - n <- ctx.store.transact(RAttachmentMeta.updatePageCount(ra.id, md.pageCount.some)) + n <- store.transact(RAttachmentMeta.updatePageCount(ra.id, md.pageCount.some)) m <- if (n == 0) ctx.logger.warn( s"No attachmentmeta record exists for ${ra.id.id}. Creating new." - ) *> ctx.store.transact( + ) *> store.transact( RAttachmentMeta.insert( RAttachmentMeta( ra.id, @@ -94,11 +96,11 @@ object AttachmentPageCount { _ <- ctx.logger.debug(s"Stored page count (${n + m}).") } yield md - def findMime[F[_]: Functor](ctx: Context[F, _])(ra: RAttachment): F[MimeType] = - OptionT(ctx.store.transact(RFileMeta.findById(ra.fileId))) + def findMime[F[_]: Functor](store: Store[F])(ra: RAttachment): F[MimeType] = + OptionT(store.transact(RFileMeta.findById(ra.fileId))) .map(_.mimetype) .getOrElse(MimeType.octetStream) - def loadFile[F[_]](ctx: Context[F, _])(ra: RAttachment): Stream[F, Byte] = - ctx.store.fileRepo.getBytes(ra.fileId) + def loadFile[F[_]](store: Store[F])(ra: RAttachment): Stream[F, Byte] = + store.fileRepo.getBytes(ra.fileId) } diff --git a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala index b15253da..db7c5f14 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala @@ -11,11 +11,11 @@ import cats.data.OptionT import cats.effect._ import cats.implicits._ import fs2.Stream - import docspell.common._ import docspell.extract.pdfbox.PdfboxPreview import docspell.extract.pdfbox.PreviewConfig import docspell.scheduler._ +import docspell.store.Store import docspell.store.queries.QAttachment import docspell.store.records.RAttachment import docspell.store.records._ @@ -26,7 +26,7 @@ import docspell.store.records._ */ object AttachmentPreview { - def apply[F[_]: Sync](pcfg: PreviewConfig)( + def apply[F[_]: Sync](pcfg: PreviewConfig, store: Store[F])( item: ItemData ): Task[F, ProcessItemArgs, ItemData] = Task { ctx => @@ -36,7 +36,7 @@ object AttachmentPreview { ) preview <- PdfboxPreview(pcfg) _ <- item.attachments - .traverse(createPreview(ctx, preview)) + .traverse(createPreview(ctx, store, preview)) .attempt .flatMap { case Right(_) => ().pure[F] @@ -50,16 +50,17 @@ object AttachmentPreview { def createPreview[F[_]: Sync]( ctx: Context[F, _], + store: Store[F], preview: PdfboxPreview[F] )( ra: RAttachment ): F[Option[RAttachmentPreview]] = - findMime[F](ctx)(ra).flatMap { + findMime[F](store)(ra).flatMap { case MimeType.PdfMatch(_) => - preview.previewPNG(loadFile(ctx)(ra)).flatMap { + preview.previewPNG(loadFile(store)(ra)).flatMap { case Some(out) => ctx.logger.debug("Preview generated, saving to database…") *> - createRecord(ctx, ra.fileId.collective, out, ra).map(_.some) + createRecord(store, ra.fileId.collective, out, ra).map(_.some) case None => ctx.logger .info(s"Preview could not be generated. Maybe the pdf has no pages?") *> @@ -72,7 +73,7 @@ object AttachmentPreview { } private def createRecord[F[_]: Sync]( - ctx: Context[F, _], + store: Store[F], collective: Ident, png: Stream[F, Byte], ra: RAttachment @@ -83,7 +84,7 @@ object AttachmentPreview { for { fileId <- png .through( - ctx.store.fileRepo.save( + store.fileRepo.save( collective, FileCategory.PreviewImage, MimeTypeHint(name.map(_.fullName), Some("image/png")) @@ -93,16 +94,16 @@ object AttachmentPreview { .lastOrError now <- Timestamp.current[F] rp = RAttachmentPreview(ra.id, fileId, name.map(_.fullName), now) - _ <- QAttachment.deletePreview(ctx.store)(ra.id) - _ <- ctx.store.transact(RAttachmentPreview.insert(rp)) + _ <- QAttachment.deletePreview(store)(ra.id) + _ <- store.transact(RAttachmentPreview.insert(rp)) } yield rp } - def findMime[F[_]: Functor](ctx: Context[F, _])(ra: RAttachment): F[MimeType] = - OptionT(ctx.store.transact(RFileMeta.findById(ra.fileId))) + def findMime[F[_]: Functor](store: Store[F])(ra: RAttachment): F[MimeType] = + OptionT(store.transact(RFileMeta.findById(ra.fileId))) .map(_.mimetype) .getOrElse(MimeType.octetStream) - def loadFile[F[_]](ctx: Context[F, _])(ra: RAttachment): Stream[F, Byte] = - ctx.store.fileRepo.getBytes(ra.fileId) + def loadFile[F[_]](store: Store[F])(ra: RAttachment): Stream[F, Byte] = + store.fileRepo.getBytes(ra.fileId) } diff --git a/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala b/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala index a7855808..1d4b9196 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala @@ -11,13 +11,13 @@ import cats.data.{Kleisli, OptionT} import cats.effect._ import cats.implicits._ import fs2.Stream - import docspell.common._ import docspell.convert.ConversionResult.Handler import docspell.convert.SanitizeHtml import docspell.convert._ import docspell.joex.extract.JsoupSanitizer import docspell.scheduler._ +import docspell.store.Store import docspell.store.records._ /** Goes through all attachments and creates a PDF version of it where supported. @@ -36,21 +36,22 @@ object ConvertPdf { def apply[F[_]: Async]( cfg: ConvertConfig, + store: Store[F], item: ItemData ): Task[F, Args, ItemData] = Task { ctx => def convert(ra: RAttachment): F[(RAttachment, Option[RAttachmentMeta])] = - isConverted(ctx)(ra).flatMap { + isConverted(store)(ra).flatMap { case true if ctx.args.isNormalProcessing => ctx.logger.info( s"Conversion to pdf already done for attachment ${ra.name}." ) *> - ctx.store + store .transact(RAttachmentMeta.findById(ra.id)) .map(rmOpt => (ra, rmOpt)) case _ => - findMime(ctx)(ra).flatMap(m => - convertSafe(cfg, JsoupSanitizer.clean, ctx, item)(ra, m) + findMime(store)(ra).flatMap(m => + convertSafe(cfg, JsoupSanitizer.clean, ctx, store, item)(ra, m) ) } @@ -62,13 +63,15 @@ object ConvertPdf { } - def isConverted[F[_]](ctx: Context[F, Args])( + def isConverted[F[_]](store: Store[F])( ra: RAttachment ): F[Boolean] = - ctx.store.transact(RAttachmentSource.isConverted(ra.id)) + store.transact(RAttachmentSource.isConverted(ra.id)) - def findMime[F[_]: Functor](ctx: Context[F, _])(ra: RAttachment): F[MimeType] = - OptionT(ctx.store.transact(RFileMeta.findById(ra.fileId))) + def findMime[F[_]: Functor](store: Store[F])( + ra: RAttachment + ): F[MimeType] = + OptionT(store.transact(RFileMeta.findById(ra.fileId))) .map(_.mimetype) .getOrElse(MimeType.octetStream) @@ -76,14 +79,15 @@ object ConvertPdf { cfg: ConvertConfig, sanitizeHtml: SanitizeHtml, ctx: Context[F, Args], + store: Store[F], item: ItemData )(ra: RAttachment, mime: MimeType): F[(RAttachment, Option[RAttachmentMeta])] = - loadCollectivePasswords(ctx).flatMap(collPass => + loadCollectivePasswords(ctx, store).flatMap(collPass => Conversion.create[F](cfg, sanitizeHtml, collPass, ctx.logger).use { conv => mime match { case mt => - val data = ctx.store.fileRepo.getBytes(ra.fileId) - val handler = conversionHandler[F](ctx, cfg, ra, item) + val data = store.fileRepo.getBytes(ra.fileId) + val handler = conversionHandler[F](ctx, store, cfg, ra, item) ctx.logger .info(s"Converting file ${ra.name} (${mime.asString}) into a PDF") *> conv.toPDF(DataType(mt), ctx.args.meta.language, handler)( @@ -94,14 +98,16 @@ object ConvertPdf { ) private def loadCollectivePasswords[F[_]: Async]( - ctx: Context[F, Args] + ctx: Context[F, Args], + store: Store[F] ): F[List[Password]] = - ctx.store + store .transact(RCollectivePassword.findAll(ctx.args.meta.collective)) .map(_.map(_.password).distinct) private def conversionHandler[F[_]: Sync]( ctx: Context[F, Args], + store: Store[F], cfg: ConvertConfig, ra: RAttachment, item: ItemData @@ -109,12 +115,12 @@ object ConvertPdf { Kleisli { case ConversionResult.SuccessPdf(pdf) => ctx.logger.info(s"Conversion to pdf successful. Saving file.") *> - storePDF(ctx, cfg, ra, pdf) + storePDF(ctx, store, cfg, ra, pdf) .map(r => (r, None)) case ConversionResult.SuccessPdfTxt(pdf, txt) => ctx.logger.info(s"Conversion to pdf+txt successful. Saving file.") *> - storePDF(ctx, cfg, ra, pdf) + storePDF(ctx, store, cfg, ra, pdf) .flatMap(r => txt.map(t => ( @@ -148,6 +154,7 @@ object ConvertPdf { private def storePDF[F[_]: Sync]( ctx: Context[F, Args], + store: Store[F], cfg: ConvertConfig, ra: RAttachment, pdf: Stream[F, Byte] @@ -162,7 +169,7 @@ object ConvertPdf { pdf .through( - ctx.store.fileRepo.save( + store.fileRepo.save( ctx.args.meta.collective, FileCategory.AttachmentConvert, MimeTypeHint(hint.filename, hint.advertised) @@ -170,32 +177,33 @@ object ConvertPdf { ) .compile .lastOrError - .flatMap(fmId => updateAttachment[F](ctx, ra, fmId, newName).map(_ => fmId)) + .flatMap(fmId => updateAttachment[F](ctx, store, ra, fmId, newName).map(_ => fmId)) .map(fmId => ra.copy(fileId = fmId, name = newName)) } private def updateAttachment[F[_]: Sync]( ctx: Context[F, _], + store: Store[F], ra: RAttachment, fmId: FileKey, newName: Option[String] ): F[Unit] = for { - oldFile <- ctx.store.transact(RAttachment.findById(ra.id)) + oldFile <- store.transact(RAttachment.findById(ra.id)) _ <- - ctx.store + store .transact(RAttachment.updateFileIdAndName(ra.id, fmId, newName)) _ <- oldFile match { case Some(raPrev) => for { sameFile <- - ctx.store + store .transact(RAttachmentSource.isSameFile(ra.id, raPrev.fileId)) _ <- if (sameFile) ().pure[F] else ctx.logger.info("Deleting previous attachment file") *> - ctx.store.fileRepo + store.fileRepo .delete(raPrev.fileId) .attempt .flatMap { diff --git a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala index 56fd9b2f..7bd299bb 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala @@ -11,9 +11,9 @@ import cats.data.OptionT import cats.effect.Sync import cats.implicits._ import fs2.Stream - import docspell.common._ import docspell.scheduler.{Context, Task} +import docspell.store.Store import docspell.store.file.FileMetadata import docspell.store.queries.QItem import docspell.store.records._ @@ -21,13 +21,13 @@ import docspell.store.records._ /** Task that creates the item. */ object CreateItem { - def apply[F[_]: Sync]: Task[F, ProcessItemArgs, ItemData] = - findExisting[F].flatMap { + def apply[F[_]: Sync](store: Store[F]): Task[F, ProcessItemArgs, ItemData] = + findExisting[F](store).flatMap { case Some(ri) => Task.pure(ri) - case None => createNew[F] + case None => createNew[F](store) } - def createNew[F[_]: Sync]: Task[F, ProcessItemArgs, ItemData] = + def createNew[F[_]: Sync](store: Store[F]): Task[F, ProcessItemArgs, ItemData] = Task { ctx => def isValidFile(fm: FileMetadata) = ctx.args.meta.validFileTypes.isEmpty || @@ -36,11 +36,11 @@ object CreateItem { def fileMetas(itemId: Ident, now: Timestamp) = Stream - .eval(ctx.store.transact(RAttachment.nextPosition(itemId))) + .eval(store.transact(RAttachment.nextPosition(itemId))) .flatMap { offset => Stream .emits(ctx.args.files) - .evalMap(f => ctx.store.fileRepo.findMeta(f.fileMetaId).map(fm => (f, fm))) + .evalMap(f => store.fileRepo.findMeta(f.fileMetaId).map(fm => (f, fm))) .collect { case (f, Some(fm)) if isValidFile(fm) => f } .zipWithIndex .evalMap { case (f, index) => @@ -67,11 +67,11 @@ object CreateItem { (for { _ <- OptionT.liftF( ctx.logger.info( - s"Loading item with id ${id.id} to ammend" + s"Loading item with id ${id.id} to amend" ) ) item <- OptionT( - ctx.store + store .transact(RItem.findByIdAndCollective(id, ctx.args.meta.collective)) ) } yield (1, item)) @@ -88,7 +88,7 @@ object CreateItem { ctx.args.meta.direction.getOrElse(Direction.Incoming), ItemState.Premature ) - n <- ctx.store.transact(RItem.insert(item)) + n <- store.transact(RItem.insert(item)) } yield (n, item) } @@ -98,7 +98,7 @@ object CreateItem { _ <- if (it._1 != 1) storeItemError[F](ctx) else ().pure[F] now <- Timestamp.current[F] fm <- fileMetas(it._2.id, now) - k <- fm.traverse(insertAttachment(ctx)) + k <- fm.traverse(insertAttachment(store)) _ <- logDifferences(ctx, fm, k.sum) dur <- time _ <- ctx.logger.info(s"Creating item finished in ${dur.formatExact}") @@ -115,25 +115,27 @@ object CreateItem { ) } - def insertAttachment[F[_]](ctx: Context[F, _])(ra: RAttachment): F[Int] = { + def insertAttachment[F[_]](store: Store[F])(ra: RAttachment): F[Int] = { val rs = RAttachmentSource.of(ra) - ctx.store.transact(for { + store.transact(for { n <- RAttachment.insert(ra) _ <- RAttachmentSource.insert(rs) } yield n) } - private def findExisting[F[_]: Sync]: Task[F, ProcessItemArgs, Option[ItemData]] = + private def findExisting[F[_]: Sync]( + store: Store[F] + ): Task[F, ProcessItemArgs, Option[ItemData]] = Task { ctx => val states = ItemState.invalidStates val fileMetaIds = ctx.args.files.map(_.fileMetaId).toSet for { - cand <- ctx.store.transact(QItem.findByFileIds(fileMetaIds.toSeq, states)) + cand <- store.transact(QItem.findByFileIds(fileMetaIds.toSeq, states)) _ <- if (cand.nonEmpty) ctx.logger.warn(s"Found ${cand.size} existing item with these files.") else ().pure[F] - ht <- cand.drop(1).traverse(ri => QItem.delete(ctx.store)(ri.id, ri.cid)) + ht <- cand.drop(1).traverse(ri => QItem.delete(store)(ri.id, ri.cid)) _ <- if (ht.sum > 0) ctx.logger.warn(s"Removed ${ht.sum} items with same attachments") @@ -144,7 +146,7 @@ object CreateItem { OptionT( // load attachments but only those mentioned in the task's arguments cand.headOption.traverse(ri => - ctx.store + store .transact(RAttachment.findByItemCollectiveSource(ri.id, ri.cid, fids)) .flatTap(ats => ctx.logger.debug( @@ -156,7 +158,7 @@ object CreateItem { ) .getOrElse(Vector.empty) orig <- rms.traverse(a => - ctx.store.transact(RAttachmentSource.findById(a.id)).map(s => (a, s)) + store.transact(RAttachmentSource.findById(a.id)).map(s => (a, s)) ) origMap = orig diff --git a/modules/joex/src/main/scala/docspell/joex/process/CrossCheckProposals.scala b/modules/joex/src/main/scala/docspell/joex/process/CrossCheckProposals.scala index e13b5da8..74066434 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/CrossCheckProposals.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/CrossCheckProposals.scala @@ -10,10 +10,10 @@ import cats.data.NonEmptyList import cats.data.OptionT import cats.effect.Sync import cats.implicits._ - import docspell.common._ import docspell.scheduler.Task import docspell.logging.Logger +import docspell.store.Store /** After candidates have been determined, the set is reduced by doing some cross checks. * For example: if a organization is suggested as correspondent, the correspondent person @@ -22,13 +22,15 @@ import docspell.logging.Logger */ object CrossCheckProposals { - def apply[F[_]: Sync](data: ItemData): Task[F, ProcessItemArgs, ItemData] = + def apply[F[_]: Sync]( + store: Store[F] + )(data: ItemData): Task[F, ProcessItemArgs, ItemData] = Task { ctx => val proposals = data.finalProposals val corrOrg = proposals.find(MetaProposalType.CorrOrg) (for { orgRef <- OptionT.fromOption[F](corrOrg) - persRefs <- OptionT.liftF(EvalProposals.findOrganizationRelation(data, ctx)) + persRefs <- OptionT.liftF(EvalProposals.findOrganizationRelation(data, store)) clProps <- OptionT.liftF( personOrgCheck[F](ctx.logger, data.classifyProposals, persRefs)(orgRef) ) @@ -53,7 +55,7 @@ object CrossCheckProposals { mpl.find(MetaProposalType.CorrPerson) match { case Some(ppl) => val list = ppl.values.filter(c => - persRefs.get(c.ref.id).exists(_.organization == Some(orgId)) + persRefs.get(c.ref.id).exists(_.organization.contains(orgId)) ) if (ppl.values.toList == list) mpl.pure[F] diff --git a/modules/joex/src/main/scala/docspell/joex/process/DuplicateCheck.scala b/modules/joex/src/main/scala/docspell/joex/process/DuplicateCheck.scala index 6b07e949..44b01592 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/DuplicateCheck.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/DuplicateCheck.scala @@ -8,58 +8,63 @@ package docspell.joex.process import cats.effect._ import cats.implicits._ - import docspell.common._ import docspell.scheduler.{Context, Task} +import docspell.store.Store import docspell.store.queries.QItem import docspell.store.records.RFileMeta import docspell.store.records.RJob - import doobie._ object DuplicateCheck { type Args = ProcessItemArgs - def apply[F[_]: Sync]: Task[F, Args, Args] = + def apply[F[_]: Sync](store: Store[F]): Task[F, Args, Args] = Task { ctx => if (ctx.args.meta.skipDuplicate) for { - retries <- getRetryCount(ctx) + retries <- getRetryCount(ctx, store) res <- if (retries == 0) - ctx.logger.debug("Checking for duplicate files") *> removeDuplicates(ctx) + ctx.logger + .debug("Checking for duplicate files") *> removeDuplicates(ctx, store) else ctx.args.pure[F] } yield res else ctx.logger.debug("Not checking for duplicates") *> ctx.args.pure[F] } - def removeDuplicates[F[_]: Sync](ctx: Context[F, Args]): F[ProcessItemArgs] = + def removeDuplicates[F[_]: Sync]( + ctx: Context[F, Args], + store: Store[F] + ): F[ProcessItemArgs] = for { - fileMetas <- findDuplicates(ctx) - _ <- fileMetas.traverse(deleteDuplicate(ctx)) + fileMetas <- findDuplicates(ctx, store) + _ <- fileMetas.traverse(deleteDuplicate(ctx, store)) ids = fileMetas.filter(_.exists).map(_.fm.id).toSet } yield ctx.args.copy(files = ctx.args.files.filterNot(f => ids.contains(f.fileMetaId)) ) - private def getRetryCount[F[_]: Sync](ctx: Context[F, Args]): F[Int] = - ctx.store.transact(RJob.getRetries(ctx.jobId)).map(_.getOrElse(0)) + private def getRetryCount[F[_]: Sync](ctx: Context[F, _], store: Store[F]): F[Int] = + store.transact(RJob.getRetries(ctx.jobId)).map(_.getOrElse(0)) private def deleteDuplicate[F[_]: Sync]( - ctx: Context[F, Args] + ctx: Context[F, Args], + store: Store[F] )(fd: FileMetaDupes): F[Unit] = { val fname = ctx.args.files.find(_.fileMetaId == fd.fm.id).flatMap(_.name) if (fd.exists) ctx.logger - .info(s"Deleting duplicate file $fname!") *> ctx.store.fileRepo + .info(s"Deleting duplicate file $fname!") *> store.fileRepo .delete(fd.fm.id) else ().pure[F] } private def findDuplicates[F[_]]( - ctx: Context[F, Args] + ctx: Context[F, Args], + store: Store[F] ): F[Vector[FileMetaDupes]] = - ctx.store.transact(for { + store.transact(for { fileMetas <- RFileMeta.findByIds(ctx.args.files.map(_.fileMetaId)) dupes <- fileMetas.traverse(checkDuplicate(ctx)) } yield dupes) diff --git a/modules/joex/src/main/scala/docspell/joex/process/EvalProposals.scala b/modules/joex/src/main/scala/docspell/joex/process/EvalProposals.scala index 2a9092a1..327c0d08 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/EvalProposals.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/EvalProposals.scala @@ -7,30 +7,31 @@ package docspell.joex.process import java.time.{LocalDate, Period} - import cats.effect.Sync import cats.implicits._ - import docspell.common._ -import docspell.scheduler.{Context, Task} +import docspell.scheduler.Task +import docspell.store.Store import docspell.store.records.{RAttachmentMeta, RPerson} /** Calculate weights for candidates that adds the most likely candidate a lower number. */ object EvalProposals { - def apply[F[_]: Sync](data: ItemData): Task[F, ProcessItemArgs, ItemData] = - Task { ctx => + def apply[F[_]: Sync]( + store: Store[F] + )(data: ItemData): Task[F, ProcessItemArgs, ItemData] = + Task { _ => for { now <- Timestamp.current[F] - personRefs <- findOrganizationRelation[F](data, ctx) + personRefs <- findOrganizationRelation[F](data, store) metas = data.metas.map(calcCandidateWeight(now.toUtcDate, personRefs)) } yield data.copy(metas = metas) } def findOrganizationRelation[F[_]: Sync]( data: ItemData, - ctx: Context[F, _] + store: Store[F] ): F[Map[Ident, PersonRef]] = { val corrPersIds = data.metas .map(_.proposals) @@ -38,7 +39,7 @@ object EvalProposals { .flatMap(_.find(MetaProposalType.CorrPerson)) .flatMap(_.values.toList.map(_.ref.id)) .toSet - ctx.store + store .transact(RPerson.findOrganization(corrPersIds)) .map(_.map(p => (p.id, p)).toMap) } diff --git a/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala b/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala index b3fb3bbf..c4b02630 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala @@ -14,13 +14,12 @@ import cats.implicits._ import cats.kernel.Monoid import cats.kernel.Order import fs2.Stream - import docspell.common._ import docspell.files.Zip import docspell.joex.mail._ import docspell.scheduler._ +import docspell.store.Store import docspell.store.records._ - import emil.Mail /** Goes through all attachments and extracts archive files, like zip files. The process @@ -34,39 +33,41 @@ import emil.Mail object ExtractArchive { type Args = ProcessItemArgs - def apply[F[_]: Async]( + def apply[F[_]: Async](store: Store[F])( item: ItemData ): Task[F, Args, ItemData] = - multiPass(item, None).map(_._2) + multiPass(store, item, None).map(_._2) def multiPass[F[_]: Async]( + store: Store[F], item: ItemData, archive: Option[RAttachmentArchive] ): Task[F, Args, (Option[RAttachmentArchive], ItemData)] = - singlePass(item, archive).flatMap { t => + singlePass(store, item, archive).flatMap { t => if (t._1.isEmpty) Task.pure(t) - else multiPass(t._2, t._1) + else multiPass(store, t._2, t._1) } def singlePass[F[_]: Async]( + store: Store[F], item: ItemData, archive: Option[RAttachmentArchive] ): Task[F, Args, (Option[RAttachmentArchive], ItemData)] = Task { ctx => def extract(ra: RAttachment, pos: Int): F[Extracted] = - findMime(ctx)(ra).flatMap(m => extractSafe(ctx, archive)(ra, pos, m)) + findMime(store)(ra).flatMap(m => extractSafe(ctx, store, archive)(ra, pos, m)) for { - lastPos <- ctx.store.transact(RAttachment.nextPosition(item.item.id)) + lastPos <- store.transact(RAttachment.nextPosition(item.item.id)) extracts <- item.attachments.zipWithIndex .traverse(t => extract(t._1, lastPos + t._2)) .map(Monoid[Extracted].combineAll) .map(fixPositions) nra = extracts.files - _ <- extracts.files.traverse(storeAttachment(ctx)) + _ <- extracts.files.traverse(storeAttachment(store)) naa = extracts.archives - _ <- naa.traverse(storeArchive(ctx)) + _ <- naa.traverse(storeArchive(store)) } yield naa.headOption -> item.copy( attachments = nra, originFile = item.originFile ++ nra.map(a => a.id -> a.fileId).toMap, @@ -83,25 +84,26 @@ object ExtractArchive { if (extract.archives.isEmpty) extract else extract.updatePositions - def findMime[F[_]: Functor](ctx: Context[F, _])(ra: RAttachment): F[MimeType] = - OptionT(ctx.store.transact(RFileMeta.findById(ra.fileId))) + def findMime[F[_]: Functor](store: Store[F])(ra: RAttachment): F[MimeType] = + OptionT(store.transact(RFileMeta.findById(ra.fileId))) .map(_.mimetype) .getOrElse(MimeType.octetStream) def extractSafe[F[_]: Async]( ctx: Context[F, Args], + store: Store[F], archive: Option[RAttachmentArchive] )(ra: RAttachment, pos: Int, mime: MimeType): F[Extracted] = mime match { case MimeType.ZipMatch(_) if ra.name.exists(_.toLowerCase.endsWith(".zip")) => ctx.logger.info(s"Extracting zip archive ${ra.name.getOrElse("")}.") *> - extractZip(ctx, archive)(ra, pos) - .flatMap(cleanupParents(ctx, ra, archive)) + extractZip(ctx, store, archive)(ra, pos) + .flatMap(cleanupParents(ctx, store, ra, archive)) case MimeType.EmailMatch(_) => ctx.logger.info(s"Reading e-mail ${ra.name.getOrElse("")}") *> - extractMail(ctx, archive)(ra, pos) - .flatMap(cleanupParents(ctx, ra, archive)) + extractMail(ctx, store, archive)(ra, pos) + .flatMap(cleanupParents(ctx, store, ra, archive)) case _ => ctx.logger.debug(s"Not an archive: ${mime.asString}") *> @@ -110,6 +112,7 @@ object ExtractArchive { def cleanupParents[F[_]: Sync]( ctx: Context[F, _], + store: Store[F], ra: RAttachment, archive: Option[RAttachmentArchive] )(extracted: Extracted): F[Extracted] = @@ -119,30 +122,31 @@ object ExtractArchive { _ <- ctx.logger.debug( s"Extracted inner attachment ${ra.name}. Remove it completely." ) - _ <- ctx.store.transact(RAttachmentArchive.delete(ra.id)) - _ <- ctx.store.transact(RAttachment.delete(ra.id)) - _ <- ctx.store.fileRepo.delete(ra.fileId) + _ <- store.transact(RAttachmentArchive.delete(ra.id)) + _ <- store.transact(RAttachment.delete(ra.id)) + _ <- store.fileRepo.delete(ra.fileId) } yield extracted case None => for { _ <- ctx.logger.debug( s"Extracted attachment ${ra.name}. Remove it from the item." ) - _ <- ctx.store.transact(RAttachment.delete(ra.id)) + _ <- store.transact(RAttachment.delete(ra.id)) } yield extracted.copy(files = extracted.files.filter(_.id != ra.id)) } def extractZip[F[_]: Async]( ctx: Context[F, Args], + store: Store[F], archive: Option[RAttachmentArchive] )(ra: RAttachment, pos: Int): F[Extracted] = { - val zipData = ctx.store.fileRepo.getBytes(ra.fileId) + val zipData = store.fileRepo.getBytes(ra.fileId) val glob = ctx.args.meta.fileFilter.getOrElse(Glob.all) ctx.logger.debug(s"Filtering zip entries with '${glob.asString}'") *> zipData .through(Zip.unzipP[F](8192, glob)) .zipWithIndex - .flatMap(handleEntry(ctx, ra, pos, archive, None)) + .flatMap(handleEntry(ctx, store, ra, pos, archive, None)) .foldMonoid .compile .lastOrError @@ -150,9 +154,10 @@ object ExtractArchive { def extractMail[F[_]: Async]( ctx: Context[F, Args], + store: Store[F], archive: Option[RAttachmentArchive] )(ra: RAttachment, pos: Int): F[Extracted] = { - val email: Stream[F, Byte] = ctx.store.fileRepo.getBytes(ra.fileId) + val email: Stream[F, Byte] = store.fileRepo.getBytes(ra.fileId) val glob = ctx.args.meta.fileFilter.getOrElse(Glob.all) val attachOnly = ctx.args.meta.attachmentsOnly.getOrElse(false) @@ -170,7 +175,9 @@ object ExtractArchive { ReadMail .mailToEntries(ctx.logger, glob, attachOnly)(mail) .zipWithIndex - .flatMap(handleEntry(ctx, ra, pos, archive, mId)) ++ Stream.eval(givenMeta) + .flatMap(handleEntry(ctx, store, ra, pos, archive, mId)) ++ Stream.eval( + givenMeta + ) } .foldMonoid .compile @@ -185,6 +192,7 @@ object ExtractArchive { def handleEntry[F[_]: Sync]( ctx: Context[F, Args], + store: Store[F], ra: RAttachment, pos: Int, archive: Option[RAttachmentArchive], @@ -195,7 +203,7 @@ object ExtractArchive { val (entry, subPos) = tentry val mimeHint = MimeTypeHint.filename(entry.name).withAdvertised(entry.mime.asString) val fileId = entry.data.through( - ctx.store.fileRepo + store.fileRepo .save(ctx.args.meta.collective, FileCategory.AttachmentSource, mimeHint) ) @@ -217,16 +225,16 @@ object ExtractArchive { } - def storeAttachment[F[_]: Sync](ctx: Context[F, _])(ra: RAttachment): F[Int] = { - val insert = CreateItem.insertAttachment(ctx)(ra) + def storeAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] = { + val insert = CreateItem.insertAttachment(store)(ra) for { - n1 <- ctx.store.transact(RAttachment.updatePosition(ra.id, ra.position)) + n1 <- store.transact(RAttachment.updatePosition(ra.id, ra.position)) n2 <- if (n1 > 0) 0.pure[F] else insert } yield n1 + n2 } - def storeArchive[F[_]](ctx: Context[F, _])(aa: RAttachmentArchive): F[Int] = - ctx.store.transact(RAttachmentArchive.insert(aa)) + def storeArchive[F[_]](store: Store[F])(aa: RAttachmentArchive): F[Int] = + store.transact(RAttachmentArchive.insert(aa)) case class Extracted( files: Vector[RAttachment], diff --git a/modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala b/modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala index a4af8e48..80beb187 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala @@ -7,16 +7,15 @@ package docspell.joex.process import java.time.ZoneId - import cats.effect.Sync import cats.implicits._ import cats.{Applicative, FlatMap} - import docspell.analysis.contact._ import docspell.common.MetaProposal.Candidate import docspell.common._ import docspell.joex.Config import docspell.scheduler.{Context, Task} +import docspell.store.Store import docspell.store.records._ /** Super simple approach to find corresponding meta data to an item by looking up values @@ -26,7 +25,8 @@ object FindProposal { type Args = ProcessItemArgs def apply[F[_]: Sync]( - cfg: Config.TextAnalysis + cfg: Config.TextAnalysis, + store: Store[F] )(data: ItemData): Task[F, Args, ItemData] = Task { ctx => val rmas = data.metas.map(rm => rm.copy(nerlabels = removeDuplicates(rm.nerlabels))) @@ -34,15 +34,16 @@ object FindProposal { _ <- ctx.logger.info("Starting find-proposal") rmv <- rmas .traverse(rm => - processAttachment(cfg, rm, data.findDates(rm), ctx) + processAttachment(cfg, rm, data.findDates(rm), ctx, store) .map(ml => rm.copy(proposals = ml)) ) - clp <- lookupClassifierProposals(ctx, data.classifyProposals) + clp <- lookupClassifierProposals(ctx, store, data.classifyProposals) } yield data.copy(metas = rmv, classifyProposals = clp) } def lookupClassifierProposals[F[_]: Sync]( ctx: Context[F, Args], + store: Store[F], mpList: MetaProposalList ): F[MetaProposalList] = { val coll = ctx.args.meta.collective @@ -50,7 +51,7 @@ object FindProposal { def lookup(mp: MetaProposal): F[Option[IdRef]] = mp.proposalType match { case MetaProposalType.CorrOrg => - ctx.store + store .transact( ROrganization .findLike(coll, mp.values.head.ref.name.toLowerCase, OrgUse.notDisabled) @@ -60,7 +61,7 @@ object FindProposal { ctx.logger.debug(s"Found classifier organization for $mp: $oref") ) case MetaProposalType.CorrPerson => - ctx.store + store .transact( RPerson .findLike( @@ -74,7 +75,7 @@ object FindProposal { ctx.logger.debug(s"Found classifier corr-person for $mp: $oref") ) case MetaProposalType.ConcPerson => - ctx.store + store .transact( RPerson .findLike( @@ -88,7 +89,7 @@ object FindProposal { ctx.logger.debug(s"Found classifier conc-person for $mp: $oref") ) case MetaProposalType.ConcEquip => - ctx.store + store .transact( REquipment .findLike( @@ -123,9 +124,10 @@ object FindProposal { cfg: Config.TextAnalysis, rm: RAttachmentMeta, rd: Vector[NerDateLabel], - ctx: Context[F, ProcessItemArgs] + ctx: Context[F, Args], + store: Store[F] ): F[MetaProposalList] = { - val finder = Finder.searchExact(ctx).next(Finder.searchFuzzy(ctx)) + val finder = Finder.searchExact(ctx, store).next(Finder.searchFuzzy(ctx, store)) List(finder.find(rm.nerlabels), makeDateProposal(cfg, rd)) .traverse(identity) .map(MetaProposalList.flatten) @@ -215,19 +217,24 @@ object FindProposal { def unit[F[_]: Applicative](value: MetaProposalList): Finder[F] = _ => value.pure[F] - def searchExact[F[_]: Sync](ctx: Context[F, ProcessItemArgs]): Finder[F] = + def searchExact[F[_]: Sync](ctx: Context[F, Args], store: Store[F]): Finder[F] = labels => - labels.toList.traverse(nl => search(nl, true, ctx)).map(MetaProposalList.flatten) + labels.toList + .traverse(nl => search(nl, true, ctx, store)) + .map(MetaProposalList.flatten) - def searchFuzzy[F[_]: Sync](ctx: Context[F, ProcessItemArgs]): Finder[F] = + def searchFuzzy[F[_]: Sync](ctx: Context[F, Args], store: Store[F]): Finder[F] = labels => - labels.toList.traverse(nl => search(nl, false, ctx)).map(MetaProposalList.flatten) + labels.toList + .traverse(nl => search(nl, false, ctx, store)) + .map(MetaProposalList.flatten) } private def search[F[_]: Sync]( nt: NerLabel, exact: Boolean, - ctx: Context[F, ProcessItemArgs] + ctx: Context[F, ProcessItemArgs], + store: Store[F] ): F[MetaProposalList] = { val value = if (exact) normalizeSearchValue(nt.label) @@ -243,7 +250,7 @@ object FindProposal { nt.tag match { case NerTag.Organization => ctx.logger.debug(s"Looking for organizations: $value") *> - ctx.store + store .transact( ROrganization .findLike(ctx.args.meta.collective, value, OrgUse.notDisabled) @@ -251,20 +258,20 @@ object FindProposal { .map(MetaProposalList.from(MetaProposalType.CorrOrg, nt)) case NerTag.Person => - val s1 = ctx.store + val s1 = store .transact( RPerson .findLike(ctx.args.meta.collective, value, PersonUse.concerningAndBoth) ) .map(MetaProposalList.from(MetaProposalType.ConcPerson, nt)) - val s2 = ctx.store + val s2 = store .transact( RPerson .findLike(ctx.args.meta.collective, value, PersonUse.correspondentAndBoth) ) .map(MetaProposalList.from(MetaProposalType.CorrPerson, nt)) val s3 = - ctx.store + store .transact( ROrganization .findLike(ctx.args.meta.collective, value, OrgUse.notDisabled) @@ -283,7 +290,7 @@ object FindProposal { case NerTag.Misc => ctx.logger.debug(s"Looking for equipments: $value") *> - ctx.store + store .transact( REquipment .findLike(ctx.args.meta.collective, value, EquipmentUse.notDisabled) @@ -291,7 +298,7 @@ object FindProposal { .map(MetaProposalList.from(MetaProposalType.ConcEquip, nt)) case NerTag.Email => - searchContact(nt, ContactKind.Email, value, ctx) + searchContact(nt, ContactKind.Email, value, ctx, store) case NerTag.Website => if (!exact) { @@ -301,9 +308,9 @@ object FindProposal { .map(_.toPrimaryDomain.asString) .map(s => s"%$s%") .getOrElse(value) - searchContact(nt, ContactKind.Website, searchString, ctx) + searchContact(nt, ContactKind.Website, searchString, ctx, store) } else - searchContact(nt, ContactKind.Website, value, ctx) + searchContact(nt, ContactKind.Website, value, ctx, store) case NerTag.Date => // There is no database search required for this tag @@ -315,18 +322,19 @@ object FindProposal { nt: NerLabel, kind: ContactKind, value: String, - ctx: Context[F, ProcessItemArgs] + ctx: Context[F, ProcessItemArgs], + store: Store[F] ): F[MetaProposalList] = { - val orgs = ctx.store + val orgs = store .transact(ROrganization.findLike(ctx.args.meta.collective, kind, value)) .map(MetaProposalList.from(MetaProposalType.CorrOrg, nt)) - val corrP = ctx.store + val corrP = store .transact( RPerson .findLike(ctx.args.meta.collective, kind, value, PersonUse.correspondentAndBoth) ) .map(MetaProposalList.from(MetaProposalType.CorrPerson, nt)) - val concP = ctx.store + val concP = store .transact( RPerson .findLike(ctx.args.meta.collective, kind, value, PersonUse.concerningAndBoth) diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala index b3a5ae7b..3122da9f 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala @@ -10,7 +10,6 @@ import cats.data.OptionT import cats.effect._ import cats.implicits._ import fs2.Stream - import docspell.analysis.TextAnalyser import docspell.backend.ops.OItem import docspell.common.{ItemState, ProcessItemArgs} @@ -18,49 +17,51 @@ import docspell.ftsclient.FtsClient import docspell.joex.Config import docspell.joex.analysis.RegexNerFile import docspell.scheduler.Task +import docspell.store.Store import docspell.store.queries.QItem import docspell.store.records.RItem object ItemHandler { type Args = ProcessItemArgs - def onCancel[F[_]: Sync]: Task[F, Args, Unit] = + def onCancel[F[_]: Sync](store: Store[F]): Task[F, Args, Unit] = logWarn[F]("Now cancelling.").flatMap(_ => - markItemCreated.flatMap { + markItemCreated(store).flatMap { case true => Task.pure(()) case false => - deleteByFileIds[F].flatMap(_ => deleteFiles) + deleteByFileIds[F](store).flatMap(_ => deleteFiles(store)) } ) def newItem[F[_]: Async]( cfg: Config, + store: Store[F], itemOps: OItem[F], fts: FtsClient[F], analyser: TextAnalyser[F], regexNer: RegexNerFile[F] ): Task[F, Args, Option[ItemData]] = logBeginning[F].flatMap(_ => - DuplicateCheck[F] + DuplicateCheck[F](store) .flatMap(args => if (args.files.isEmpty) logNoFiles[F].map(_ => None) else { val create: Task[F, Args, ItemData] = - CreateItem[F].contramap(_ => args.pure[F]) + CreateItem[F](store).contramap(_ => args.pure[F]) create - .flatMap(itemStateTask(ItemState.Processing)) - .flatMap(safeProcess[F](cfg, itemOps, fts, analyser, regexNer)) + .flatMap(itemStateTask(store, ItemState.Processing)) + .flatMap(safeProcess[F](cfg, store, itemOps, fts, analyser, regexNer)) .map(_.some) } ) ) - def itemStateTask[F[_]: Sync, A]( - state: ItemState - )(data: ItemData): Task[F, A, ItemData] = - Task(ctx => - ctx.store + def itemStateTask[F[_]: Sync, A](store: Store[F], state: ItemState)( + data: ItemData + ): Task[F, A, ItemData] = + Task(_ => + store .transact(RItem.updateState(data.item.id, state, ItemState.invalidStates)) .map(_ => data) ) @@ -70,6 +71,7 @@ object ItemHandler { def safeProcess[F[_]: Async]( cfg: Config, + store: Store[F], itemOps: OItem[F], fts: FtsClient[F], analyser: TextAnalyser[F], @@ -77,30 +79,31 @@ object ItemHandler { )(data: ItemData): Task[F, Args, ItemData] = isLastRetry[F].flatMap { case true => - ProcessItem[F](cfg, itemOps, fts, analyser, regexNer)(data).attempt.flatMap { - case Right(d) => - Task.pure(d) - case Left(ex) => - logWarn[F]( - "Processing failed on last retry. Creating item but without proposals." - ).flatMap(_ => itemStateTask(ItemState.Created)(data)) - .andThen(_ => Sync[F].raiseError(ex)) - } + ProcessItem[F](cfg, itemOps, fts, analyser, regexNer, store)(data).attempt + .flatMap { + case Right(d) => + Task.pure(d) + case Left(ex) => + logWarn[F]( + "Processing failed on last retry. Creating item but without proposals." + ).flatMap(_ => itemStateTask(store, ItemState.Created)(data)) + .andThen(_ => Sync[F].raiseError(ex)) + } case false => - ProcessItem[F](cfg, itemOps, fts, analyser, regexNer)(data) - .flatMap(itemStateTask(ItemState.Created)) + ProcessItem[F](cfg, itemOps, fts, analyser, regexNer, store)(data) + .flatMap(itemStateTask(store, ItemState.Created)) } - private def markItemCreated[F[_]: Sync]: Task[F, Args, Boolean] = + private def markItemCreated[F[_]: Sync](store: Store[F]): Task[F, Args, Boolean] = Task { ctx => val fileMetaIds = ctx.args.files.map(_.fileMetaId).toSet (for { - item <- OptionT(ctx.store.transact(QItem.findOneByFileIds(fileMetaIds.toSeq))) + item <- OptionT(store.transact(QItem.findOneByFileIds(fileMetaIds.toSeq))) _ <- OptionT.liftF( ctx.logger.info("Processing cancelled. Marking item as created anyways.") ) _ <- OptionT.liftF( - ctx.store + store .transact( RItem.updateState(item.id, ItemState.Created, ItemState.invalidStates) ) @@ -111,11 +114,11 @@ object ItemHandler { ) } - private def deleteByFileIds[F[_]: Sync]: Task[F, Args, Unit] = + private def deleteByFileIds[F[_]: Sync](store: Store[F]): Task[F, Args, Unit] = Task { ctx => val states = ItemState.invalidStates for { - items <- ctx.store.transact( + items <- store.transact( QItem.findByFileIds(ctx.args.files.map(_.fileMetaId), states) ) _ <- @@ -124,16 +127,16 @@ object ItemHandler { ctx.logger.info( s"No items found for file ids ${ctx.args.files.map(_.fileMetaId)}" ) - _ <- items.traverse(i => QItem.delete(ctx.store)(i.id, ctx.args.meta.collective)) + _ <- items.traverse(i => QItem.delete(store)(i.id, ctx.args.meta.collective)) } yield () } - private def deleteFiles[F[_]: Sync]: Task[F, Args, Unit] = + private def deleteFiles[F[_]: Sync](store: Store[F]): Task[F, Args, Unit] = Task(ctx => ctx.logger.info("Deleting input files …") *> Stream .emits(ctx.args.files.map(_.fileMetaId)) - .evalMap(id => ctx.store.fileRepo.delete(id).attempt) + .evalMap(id => store.fileRepo.delete(id).attempt) .compile .drain ) diff --git a/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala b/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala index 2e85e450..71598b64 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala @@ -9,22 +9,26 @@ package docspell.joex.process import cats.data.NonEmptyList import cats.effect.Sync import cats.implicits._ - import docspell.common._ import docspell.scheduler.{Context, Task} +import docspell.store.Store import docspell.store.records.RItem object LinkProposal { - def onlyNew[F[_]: Sync](data: ItemData): Task[F, ProcessItemArgs, ItemData] = + def onlyNew[F[_]: Sync]( + store: Store[F] + )(data: ItemData): Task[F, ProcessItemArgs, ItemData] = if (data.item.state.isValid) Task .log[F, ProcessItemArgs](_.debug(s"Not linking proposals on existing item")) .map(_ => data) else - LinkProposal[F](data) + LinkProposal[F](store)(data) - def apply[F[_]: Sync](data: ItemData): Task[F, ProcessItemArgs, ItemData] = + def apply[F[_]: Sync]( + store: Store[F] + )(data: ItemData): Task[F, ProcessItemArgs, ItemData] = if (data.item.state == ItemState.Confirmed) Task .log[F, ProcessItemArgs](_.debug(s"Not linking proposals on confirmed item")) @@ -35,7 +39,7 @@ object LinkProposal { ctx.logger.info(s"Starting linking proposals") *> MetaProposalType.all - .traverse(applyValue(data, proposals, ctx)) + .traverse(applyValue(data, proposals, ctx, store)) .map(result => ctx.logger.info(s"Results from proposal processing: $result")) .map(_ => data) } @@ -43,7 +47,8 @@ object LinkProposal { def applyValue[F[_]: Sync]( data: ItemData, proposalList: MetaProposalList, - ctx: Context[F, ProcessItemArgs] + ctx: Context[F, ProcessItemArgs], + store: Store[F] )(mpt: MetaProposalType): F[Result] = data.givenMeta.find(mpt).orElse(proposalList.find(mpt)) match { case None => @@ -51,29 +56,30 @@ object LinkProposal { Result.noneFound(mpt).pure[F] case Some(a) if a.isSingleValue => ctx.logger.info(s"Found one candidate for ${a.proposalType}") *> - setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).map(_ => - Result.single(mpt) + setItemMeta(data.item.id, ctx, store, a.proposalType, a.values.head.ref.id).map( + _ => Result.single(mpt) ) case Some(a) => val ids = a.values.map(_.ref.id.id) ctx.logger.info( s"Found many (${a.size}, $ids) candidates for ${a.proposalType}. Setting first." ) *> - setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id).map(_ => - Result.multiple(mpt) + setItemMeta(data.item.id, ctx, store, a.proposalType, a.values.head.ref.id).map( + _ => Result.multiple(mpt) ) } def setItemMeta[F[_]: Sync]( itemId: Ident, ctx: Context[F, ProcessItemArgs], + store: Store[F], mpt: MetaProposalType, value: Ident ): F[Int] = mpt match { case MetaProposalType.CorrOrg => ctx.logger.debug(s"Updating item organization with: ${value.id}") *> - ctx.store.transact( + store.transact( RItem.updateCorrOrg( NonEmptyList.of(itemId), ctx.args.meta.collective, @@ -82,7 +88,7 @@ object LinkProposal { ) case MetaProposalType.ConcPerson => ctx.logger.debug(s"Updating item concerning person with: $value") *> - ctx.store.transact( + store.transact( RItem.updateConcPerson( NonEmptyList.of(itemId), ctx.args.meta.collective, @@ -91,7 +97,7 @@ object LinkProposal { ) case MetaProposalType.CorrPerson => ctx.logger.debug(s"Updating item correspondent person with: $value") *> - ctx.store.transact( + store.transact( RItem.updateCorrPerson( NonEmptyList.of(itemId), ctx.args.meta.collective, @@ -100,7 +106,7 @@ object LinkProposal { ) case MetaProposalType.ConcEquip => ctx.logger.debug(s"Updating item concerning equipment with: $value") *> - ctx.store.transact( + store.transact( RItem.updateConcEquip( NonEmptyList.of(itemId), ctx.args.meta.collective, @@ -112,7 +118,7 @@ object LinkProposal { case Some(ld) => val ts = Timestamp.from(ld.atStartOfDay(Timestamp.UTC)) ctx.logger.debug(s"Updating item date ${value.id}") *> - ctx.store.transact( + store.transact( RItem.updateDate( NonEmptyList.of(itemId), ctx.args.meta.collective, @@ -128,7 +134,7 @@ object LinkProposal { case Some(ld) => val ts = Timestamp.from(ld.atStartOfDay(Timestamp.UTC)) ctx.logger.debug(s"Updating item due-date suggestion ${value.id}") *> - ctx.store.transact( + store.transact( RItem.updateDueDate( NonEmptyList.of(itemId), ctx.args.meta.collective, diff --git a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala index 5918698c..819be5c7 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala @@ -8,7 +8,6 @@ package docspell.joex.process import cats.effect._ import cats.implicits._ - import docspell.analysis.TextAnalyser import docspell.backend.ops.OItem import docspell.common.ProcessItemArgs @@ -16,6 +15,7 @@ import docspell.ftsclient.FtsClient import docspell.joex.Config import docspell.joex.analysis.RegexNerFile import docspell.scheduler.Task +import docspell.store.Store object ProcessItem { @@ -24,12 +24,13 @@ object ProcessItem { itemOps: OItem[F], fts: FtsClient[F], analyser: TextAnalyser[F], - regexNer: RegexNerFile[F] + regexNer: RegexNerFile[F], + store: Store[F] )(item: ItemData): Task[F, ProcessItemArgs, ItemData] = - ExtractArchive(item) + ExtractArchive(store)(item) .flatMap(Task.setProgress(20)) - .flatMap(processAttachments0(cfg, fts, analyser, regexNer, (40, 60, 80))) - .flatMap(LinkProposal.onlyNew[F]) + .flatMap(processAttachments0(cfg, fts, analyser, regexNer, store, (40, 60, 80))) + .flatMap(LinkProposal.onlyNew[F](store)) .flatMap(SetGivenData.onlyNew[F](itemOps)) .flatMap(Task.setProgress(99)) .flatMap(RemoveEmptyItem(itemOps)) @@ -38,34 +39,37 @@ object ProcessItem { cfg: Config, fts: FtsClient[F], analyser: TextAnalyser[F], - regexNer: RegexNerFile[F] + regexNer: RegexNerFile[F], + store: Store[F] )(item: ItemData): Task[F, ProcessItemArgs, ItemData] = - processAttachments0[F](cfg, fts, analyser, regexNer, (30, 60, 90))(item) + processAttachments0[F](cfg, fts, analyser, regexNer, store, (30, 60, 90))(item) def analysisOnly[F[_]: Async]( cfg: Config, analyser: TextAnalyser[F], - regexNer: RegexNerFile[F] + regexNer: RegexNerFile[F], + store: Store[F] )(item: ItemData): Task[F, ProcessItemArgs, ItemData] = - TextAnalysis[F](cfg.textAnalysis, analyser, regexNer)(item) - .flatMap(FindProposal[F](cfg.textAnalysis)) - .flatMap(EvalProposals[F]) - .flatMap(CrossCheckProposals[F]) - .flatMap(SaveProposals[F]) + TextAnalysis[F](cfg.textAnalysis, analyser, regexNer, store)(item) + .flatMap(FindProposal[F](cfg.textAnalysis, store)) + .flatMap(EvalProposals[F](store)) + .flatMap(CrossCheckProposals[F](store)) + .flatMap(SaveProposals[F](store)) private def processAttachments0[F[_]: Async]( cfg: Config, fts: FtsClient[F], analyser: TextAnalyser[F], regexNer: RegexNerFile[F], + store: Store[F], progress: (Int, Int, Int) )(item: ItemData): Task[F, ProcessItemArgs, ItemData] = - ConvertPdf(cfg.convert, item) + ConvertPdf(cfg.convert, store, item) .flatMap(Task.setProgress(progress._1)) - .flatMap(TextExtraction(cfg.extraction, fts)) - .flatMap(AttachmentPreview(cfg.extraction.preview)) - .flatMap(AttachmentPageCount()) + .flatMap(TextExtraction(cfg.extraction, fts, store)) + .flatMap(AttachmentPreview(cfg.extraction.preview, store)) + .flatMap(AttachmentPageCount(store)) .flatMap(Task.setProgress(progress._2)) - .flatMap(analysisOnly[F](cfg, analyser, regexNer)) + .flatMap(analysisOnly[F](cfg, analyser, regexNer, store)) .flatMap(Task.setProgress(progress._3)) } diff --git a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala index 97091b75..cc840710 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala @@ -9,7 +9,6 @@ package docspell.joex.process import cats.data.OptionT import cats.effect._ import cats.implicits._ - import docspell.analysis.TextAnalyser import docspell.backend.ops.OItem import docspell.common._ @@ -18,6 +17,7 @@ import docspell.joex.Config import docspell.joex.analysis.RegexNerFile import docspell.scheduler.Context import docspell.scheduler.Task +import docspell.store.Store import docspell.store.queries.QItem import docspell.store.records.RAttachment import docspell.store.records.RAttachmentSource @@ -32,13 +32,14 @@ object ReProcessItem { fts: FtsClient[F], itemOps: OItem[F], analyser: TextAnalyser[F], - regexNer: RegexNerFile[F] + regexNer: RegexNerFile[F], + store: Store[F] ): Task[F, Args, Unit] = Task .log[F, Args](_.info("===== Start reprocessing ======")) .flatMap(_ => - loadItem[F] - .flatMap(safeProcess[F](cfg, fts, itemOps, analyser, regexNer)) + loadItem[F](store) + .flatMap(safeProcess[F](cfg, fts, itemOps, analyser, regexNer, store)) .map(_ => ()) ) @@ -53,13 +54,13 @@ object ReProcessItem { else ra => selection.contains(ra.id) } - def loadItem[F[_]: Sync]: Task[F, Args, ItemData] = + def loadItem[F[_]: Sync](store: Store[F]): Task[F, Args, ItemData] = Task { ctx => (for { - item <- OptionT(ctx.store.transact(RItem.findById(ctx.args.itemId))) - attach <- OptionT.liftF(ctx.store.transact(RAttachment.findByItem(item.id))) + item <- OptionT(store.transact(RItem.findById(ctx.args.itemId))) + attach <- OptionT.liftF(store.transact(RAttachment.findByItem(item.id))) asrc <- - OptionT.liftF(ctx.store.transact(RAttachmentSource.findByItem(ctx.args.itemId))) + OptionT.liftF(store.transact(RAttachmentSource.findByItem(ctx.args.itemId))) asrcMap = asrc.map(s => s.id -> s).toMap // copy the original files over to attachments to run the default processing task // the processing doesn't touch the original files, only RAttachments @@ -97,6 +98,7 @@ object ReProcessItem { itemOps: OItem[F], analyser: TextAnalyser[F], regexNer: RegexNerFile[F], + store: Store[F], data: ItemData ): Task[F, Args, ItemData] = { @@ -121,21 +123,21 @@ object ReProcessItem { Nil ).pure[F] - getLanguage[F].flatMap { lang => + getLanguage[F](store).flatMap { lang => ProcessItem - .processAttachments[F](cfg, fts, analyser, regexNer)(data) - .flatMap(LinkProposal[F]) + .processAttachments[F](cfg, fts, analyser, regexNer, store)(data) + .flatMap(LinkProposal[F](store)) .flatMap(SetGivenData[F](itemOps)) .contramap[Args](convertArgs(lang)) } } - def getLanguage[F[_]: Sync]: Task[F, Args, Language] = + def getLanguage[F[_]: Sync](store: Store[F]): Task[F, Args, Language] = Task { ctx => val lang1 = OptionT( - ctx.store.transact(QItem.getItemLanguage(ctx.args.itemId)).map(_.headOption) + store.transact(QItem.getItemLanguage(ctx.args.itemId)).map(_.headOption) ) - val lang2 = OptionT(ctx.store.transact(RCollective.findByItem(ctx.args.itemId))) + val lang2 = OptionT(store.transact(RCollective.findByItem(ctx.args.itemId))) .map(_.language) lang1.orElse(lang2).getOrElse(Language.German) @@ -149,11 +151,12 @@ object ReProcessItem { fts: FtsClient[F], itemOps: OItem[F], analyser: TextAnalyser[F], - regexNer: RegexNerFile[F] + regexNer: RegexNerFile[F], + store: Store[F] )(data: ItemData): Task[F, Args, ItemData] = isLastRetry[F].flatMap { case true => - processFiles[F](cfg, fts, itemOps, analyser, regexNer, data).attempt + processFiles[F](cfg, fts, itemOps, analyser, regexNer, store, data).attempt .flatMap { case Right(d) => Task.pure(d) @@ -163,7 +166,7 @@ object ReProcessItem { ).andThen(_ => Sync[F].raiseError(ex)) } case false => - processFiles[F](cfg, fts, itemOps, analyser, regexNer, data) + processFiles[F](cfg, fts, itemOps, analyser, regexNer, store, data) } private def logWarn[F[_]](msg: => String): Task[F, Args, Unit] = diff --git a/modules/joex/src/main/scala/docspell/joex/process/SaveProposals.scala b/modules/joex/src/main/scala/docspell/joex/process/SaveProposals.scala index cdc8627a..fdb0c7a7 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/SaveProposals.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/SaveProposals.scala @@ -8,17 +8,16 @@ package docspell.joex.process import cats.effect.Sync import cats.implicits._ - import docspell.common._ import docspell.scheduler.{Context, Task} -import docspell.store.AddResult +import docspell.store.{AddResult, Store} import docspell.store.records._ /** Saves the proposals in the database */ object SaveProposals { type Args = ProcessItemArgs - def apply[F[_]: Sync](data: ItemData): Task[F, Args, ItemData] = + def apply[F[_]: Sync](store: Store[F])(data: ItemData): Task[F, Args, ItemData] = Task { ctx => for { _ <- ctx.logger.info("Storing proposals") @@ -26,20 +25,24 @@ object SaveProposals { .traverse(rm => ctx.logger.debug( s"Storing attachment proposals: ${rm.proposals}" - ) *> ctx.store.transact(RAttachmentMeta.updateProposals(rm.id, rm.proposals)) + ) *> store.transact(RAttachmentMeta.updateProposals(rm.id, rm.proposals)) ) _ <- if (data.classifyProposals.isEmpty && data.classifyTags.isEmpty) 0.pure[F] - else saveItemProposal(ctx, data) + else saveItemProposal(ctx, store, data) } yield data } - def saveItemProposal[F[_]: Sync](ctx: Context[F, Args], data: ItemData): F[Unit] = { + def saveItemProposal[F[_]: Sync]( + ctx: Context[F, Args], + store: Store[F], + data: ItemData + ): F[Unit] = { def upsert(v: RItemProposal): F[Int] = - ctx.store.add(RItemProposal.insert(v), RItemProposal.exists(v.itemId)).flatMap { + store.add(RItemProposal.insert(v), RItemProposal.exists(v.itemId)).flatMap { case AddResult.Success => 1.pure[F] case AddResult.EntityExists(_) => - ctx.store.transact(RItemProposal.update(v)) + store.transact(RItemProposal.update(v)) case AddResult.Failure(ex) => ctx.logger.warn(s"Could not store item proposals: ${ex.getMessage}") *> 0 .pure[F] @@ -47,7 +50,7 @@ object SaveProposals { for { _ <- ctx.logger.debug(s"Storing classifier proposals: ${data.classifyProposals}") - tags <- ctx.store.transact( + tags <- store.transact( RTag.findAllByNameOrId(data.classifyTags, ctx.args.meta.collective) ) tagRefs = tags.map(t => IdRef(t.tagId, t.name)) diff --git a/modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala b/modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala index aced45f7..774040dd 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala @@ -9,7 +9,6 @@ package docspell.joex.process import cats.Traverse import cats.effect._ import cats.implicits._ - import docspell.analysis.classifier.TextClassifier import docspell.analysis.{NlpSettings, TextAnalyser} import docspell.common.MetaProposal.Candidate @@ -20,6 +19,7 @@ import docspell.joex.learn.{ClassifierName, Classify, LearnClassifierTask} import docspell.joex.process.ItemData.AttachmentDates import docspell.scheduler.Context import docspell.scheduler.Task +import docspell.store.Store import docspell.store.records.{RAttachmentMeta, RClassifierSetting} object TextAnalysis { @@ -28,7 +28,8 @@ object TextAnalysis { def apply[F[_]: Async]( cfg: Config.TextAnalysis, analyser: TextAnalyser[F], - nerFile: RegexNerFile[F] + nerFile: RegexNerFile[F], + store: Store[F] )(item: ItemData): Task[F, Args, ItemData] = Task { ctx => for { @@ -41,18 +42,19 @@ object TextAnalysis { ) _ <- ctx.logger.debug(s"Storing tags: ${t.map(_._1.copy(content = None))}") _ <- t.traverse(m => - ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels)) + store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels)) ) v = t.toVector - autoTagEnabled <- getActiveAutoTag(ctx, cfg) + autoTagEnabled <- getActiveAutoTag(ctx, store, cfg) tag <- - if (autoTagEnabled) predictTags(ctx, cfg, item.metas, analyser.classifier) + if (autoTagEnabled) + predictTags(ctx, store, cfg, item.metas, analyser.classifier) else List.empty[String].pure[F] classProposals <- if (cfg.classification.enabled) - predictItemEntities(ctx, cfg, item.metas, analyser.classifier) + predictItemEntities(ctx, store, cfg, item.metas, analyser.classifier) else MetaProposalList.empty.pure[F] e <- s @@ -86,16 +88,17 @@ object TextAnalysis { def predictTags[F[_]: Async]( ctx: Context[F, Args], + store: Store[F], cfg: Config.TextAnalysis, metas: Vector[RAttachmentMeta], classifier: TextClassifier[F] ): F[List[String]] = { val text = metas.flatMap(_.content).mkString(LearnClassifierTask.pageSep) val classifyWith: ClassifierName => F[Option[String]] = - makeClassify(ctx, cfg, classifier)(text) + makeClassify(ctx, store, cfg, classifier)(text) for { - names <- ctx.store.transact( + names <- store.transact( ClassifierName.findTagClassifiers(ctx.args.meta.collective) ) _ <- ctx.logger.debug(s"Guessing tags for ${names.size} categories") @@ -105,6 +108,7 @@ object TextAnalysis { def predictItemEntities[F[_]: Async]( ctx: Context[F, Args], + store: Store[F], cfg: Config.TextAnalysis, metas: Vector[RAttachmentMeta], classifier: TextClassifier[F] @@ -116,7 +120,7 @@ object TextAnalysis { mtype: MetaProposalType ): F[Option[MetaProposal]] = for { - label <- makeClassify(ctx, cfg, classifier)(text).apply(cname) + label <- makeClassify(ctx, store, cfg, classifier)(text).apply(cname) } yield label.map(str => MetaProposal(mtype, Candidate(IdRef(Ident.unsafe(""), str), Set.empty)) ) @@ -136,13 +140,14 @@ object TextAnalysis { private def makeClassify[F[_]: Async]( ctx: Context[F, Args], + store: Store[F], cfg: Config.TextAnalysis, classifier: TextClassifier[F] )(text: String): ClassifierName => F[Option[String]] = Classify[F]( ctx.logger, cfg.workingDir, - ctx.store, + store, classifier, ctx.args.meta.collective, text @@ -150,10 +155,11 @@ object TextAnalysis { private def getActiveAutoTag[F[_]: Sync]( ctx: Context[F, Args], + store: Store[F], cfg: Config.TextAnalysis ): F[Boolean] = if (cfg.classification.enabled) - ctx.store + store .transact(RClassifierSetting.findById(ctx.args.meta.collective)) .map(_.exists(_.autoTagEnabled)) .flatTap(enabled => diff --git a/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala b/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala index eaa02683..757ee54f 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala @@ -9,16 +9,16 @@ package docspell.joex.process import cats.data.OptionT import cats.effect._ import cats.implicits._ - import docspell.common._ import docspell.extract.{ExtractConfig, ExtractResult, Extraction} import docspell.ftsclient.{FtsClient, TextData} import docspell.scheduler.{Context, Task} +import docspell.store.Store import docspell.store.records.{RAttachment, RAttachmentMeta, RFileMeta} object TextExtraction { - def apply[F[_]: Async](cfg: ExtractConfig, fts: FtsClient[F])( + def apply[F[_]: Async](cfg: ExtractConfig, fts: FtsClient[F], store: Store[F])( item: ItemData ): Task[F, ProcessItemArgs, ItemData] = Task { ctx => @@ -30,6 +30,7 @@ object TextExtraction { txt <- item.attachments.traverse( extractTextIfEmpty( ctx, + store, cfg, ctx.args.meta.language, ctx.args.meta.collective, @@ -38,7 +39,7 @@ object TextExtraction { ) _ <- ctx.logger.debug("Storing extracted texts …") _ <- - txt.toList.traverse(res => ctx.store.transact(RAttachmentMeta.upsert(res.am))) + txt.toList.traverse(res => store.transact(RAttachmentMeta.upsert(res.am))) _ <- ctx.logger.debug(s"Extracted text stored.") idxItem = TextData.item( item.item.id, @@ -65,6 +66,7 @@ object TextExtraction { def extractTextIfEmpty[F[_]: Async]( ctx: Context[F, ProcessItemArgs], + store: Store[F], cfg: ExtractConfig, lang: Language, collective: Ident, @@ -91,13 +93,14 @@ object TextExtraction { ctx.logger.info("TextExtraction skipped, since text is already available.") *> makeTextData((rm, Nil)).pure[F] case _ => - extractTextToMeta[F](ctx, cfg, lang, item)(ra) + extractTextToMeta[F](ctx, store, cfg, lang, item)(ra) .map(makeTextData) } } def extractTextToMeta[F[_]: Async]( ctx: Context[F, _], + store: Store[F], cfg: ExtractConfig, lang: Language, item: ItemData @@ -105,8 +108,8 @@ object TextExtraction { for { _ <- ctx.logger.debug(s"Extracting text for attachment ${stripAttachmentName(ra)}") dst <- Duration.stopTime[F] - fids <- filesToExtract(ctx)(item, ra) - res <- extractTextFallback(ctx, cfg, ra, lang)(fids) + fids <- filesToExtract(store)(item, ra) + res <- extractTextFallback(ctx, store, cfg, ra, lang)(fids) meta = item.changeMeta( ra.id, lang, @@ -123,14 +126,14 @@ object TextExtraction { } yield (meta, tags) def extractText[F[_]: Sync]( - ctx: Context[F, _], + store: Store[F], extr: Extraction[F], lang: Language )(fileId: FileKey): F[ExtractResult] = { - val data = ctx.store.fileRepo.getBytes(fileId) + val data = store.fileRepo.getBytes(fileId) def findMime: F[MimeType] = - OptionT(ctx.store.fileRepo.findMeta(fileId)) + OptionT(store.fileRepo.findMeta(fileId)) .map(_.mimetype) .getOrElse(MimeType.octetStream) @@ -140,6 +143,7 @@ object TextExtraction { private def extractTextFallback[F[_]: Async]( ctx: Context[F, _], + store: Store[F], cfg: ExtractConfig, ra: RAttachment, lang: Language @@ -151,7 +155,7 @@ object TextExtraction { case id :: rest => val extr = Extraction.create[F](ctx.logger, cfg) - extractText[F](ctx, extr, lang)(id) + extractText[F](store, extr, lang)(id) .flatMap { case res @ ExtractResult.Success(_, _) => res.some.pure[F] @@ -161,12 +165,12 @@ object TextExtraction { .warn( s"Cannot extract text from file ${stripAttachmentName(ra)}: unsupported format ${mt.asString}. Try with converted file." ) - .flatMap(_ => extractTextFallback[F](ctx, cfg, ra, lang)(rest)) + .flatMap(_ => extractTextFallback[F](ctx, store, cfg, ra, lang)(rest)) case ExtractResult.Failure(ex) => ctx.logger .warn(s"Cannot extract text: ${ex.getMessage}. Try with converted file") - .flatMap(_ => extractTextFallback[F](ctx, cfg, ra, lang)(rest)) + .flatMap(_ => extractTextFallback[F](ctx, store, cfg, ra, lang)(rest)) } } @@ -176,13 +180,13 @@ object TextExtraction { * If the source file is a PDF, then use the converted file. This may then already * contain the text if ocrmypdf is enabled. If it is disabled, both files are the same. */ - private def filesToExtract[F[_]: Sync](ctx: Context[F, _])( + private def filesToExtract[F[_]: Sync](store: Store[F])( item: ItemData, ra: RAttachment ): F[List[FileKey]] = item.originFile.get(ra.id) match { case Some(sid) => - ctx.store.transact(RFileMeta.findMime(sid)).map { + store.transact(RFileMeta.findMime(sid)).map { case Some(MimeType.PdfMatch(_)) => List(ra.fileId) case _ => diff --git a/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala b/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala index 1a582b98..6810f3d9 100644 --- a/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala +++ b/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala @@ -12,7 +12,7 @@ import cats.implicits._ import docspell.common.{Duration, Ident, Timestamp} import docspell.joex.JoexApp import docspell.joexapi.model._ -import docspell.store.records.{RJob, RJobLog} +import docspell.store.records.RJobLog import org.http4s.HttpRoutes import org.http4s.circe.CirceEntityEncoder._ @@ -67,17 +67,19 @@ object JoexRoutes { } } - def mkJob(j: RJob): Job = + // TODO !! + + def mkJob(j: docspell.scheduler.Job[String]): Job = Job( j.id, j.subject, - j.submitted, + Timestamp.Epoch, j.priority, - j.retries, - j.progress, - j.started.getOrElse(Timestamp.Epoch) + -1, + -1, + Timestamp.Epoch ) - def mkJobLog(j: RJob, jl: Vector[RJobLog]): JobAndLog = + def mkJobLog(j: docspell.scheduler.Job[String], jl: Vector[RJobLog]): JobAndLog = JobAndLog(mkJob(j), jl.map(r => JobLogEvent(r.created, r.level, r.message)).toList) } diff --git a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala index 363420b0..5a91003b 100644 --- a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala @@ -12,7 +12,6 @@ import cats.data.OptionT import cats.effect._ import cats.implicits._ import fs2._ - import docspell.backend.ops.{OJoex, OUpload} import docspell.common._ import docspell.joex.Config @@ -20,8 +19,8 @@ import docspell.scheduler.{Context, Task} import docspell.logging.Logger import docspell.store.queries.QOrganization import docspell.store.records._ - import _root_.io.circe.syntax._ +import docspell.store.Store import emil.SearchQuery.{All, ReceivedDate} import emil.javamail.syntax._ import emil.{MimeType => _, _} @@ -32,6 +31,7 @@ object ScanMailboxTask { def apply[F[_]: Sync]( cfg: Config.ScanMailbox, + store: Store[F], emil: Emil[F], upload: OUpload[F], joex: OJoex[F] @@ -42,22 +42,22 @@ object ScanMailboxTask { s"=== Start importing mails for user ${ctx.args.account.user.id}" ) _ <- ctx.logger.debug(s"Settings: ${ctx.args.asJson.noSpaces}") - mailCfg <- getMailSettings(ctx) + mailCfg <- getMailSettings(ctx, store) folders = ctx.args.folders.mkString(", ") userId = ctx.args.account.user imapConn = ctx.args.imapConnection _ <- ctx.logger.info( s"Reading mails for user ${userId.id} from ${imapConn.id}/$folders" ) - _ <- importMails(cfg, mailCfg, emil, upload, joex, ctx) + _ <- importMails(cfg, mailCfg, emil, upload, joex, ctx, store) } yield () } def onCancel[F[_]]: Task[F, ScanMailboxArgs, Unit] = Task.log(_.warn("Cancelling scan-mailbox task")) - def getMailSettings[F[_]: Sync](ctx: Context[F, Args]): F[RUserImap] = - ctx.store + def getMailSettings[F[_]: Sync](ctx: Context[F, Args], store: Store[F]): F[RUserImap] = + store .transact(RUserImap.getByName(ctx.args.account, ctx.args.imapConnection)) .flatMap { case Some(c) => c.pure[F] @@ -75,10 +75,11 @@ object ScanMailboxTask { theEmil: Emil[F], upload: OUpload[F], joex: OJoex[F], - ctx: Context[F, Args] + ctx: Context[F, Args], + store: Store[F] ): F[Unit] = { val mailer = theEmil(mailCfg.toMailConfig) - val impl = new Impl[F](cfg, ctx) + val impl = new Impl[F](cfg, ctx, store) val inFolders = ctx.args.folders.take(cfg.maxFolders) val getInitialInput = @@ -142,7 +143,11 @@ object ScanMailboxTask { ScanResult(List(folder -> left), processed) } - final private class Impl[F[_]: Sync](cfg: Config.ScanMailbox, ctx: Context[F, Args]) { + final private class Impl[F[_]: Sync]( + cfg: Config.ScanMailbox, + ctx: Context[F, Args], + store: Store[F] + ) { private def logOp[C](f: Logger[F] => F[Unit]): MailOp[F, C, Unit] = MailOp(_ => f(ctx.logger)) @@ -213,7 +218,7 @@ object ScanMailboxTask { NonEmptyList.fromFoldable(headers.flatMap(_.mh.messageId)) match { case Some(nl) => for { - archives <- ctx.store.transact( + archives <- store.transact( RAttachmentArchive .findByMessageIdAndCollective(nl, ctx.args.account.collective) ) @@ -237,7 +242,7 @@ object ScanMailboxTask { for { from <- OptionT.fromOption[F](mh.from) _ <- OptionT( - ctx.store.transact( + store.transact( QOrganization .findPersonByContact( ctx.args.account.collective, diff --git a/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala b/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala index 6da9e345..128bc76b 100644 --- a/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala @@ -10,9 +10,9 @@ import cats.data.OptionT import cats.effect._ import cats.implicits._ import docspell.common._ -import docspell.scheduler.Context import docspell.scheduler.Task import docspell.scheduler.usertask.UserTask +import docspell.store.Store import docspell.store.records.RUserEmail import emil._ @@ -37,6 +37,7 @@ object UpdateCheckTask { def apply[F[_]: Async]( cfg: UpdateCheckConfig, sendCfg: MailSendConfig, + store: Store[F], emil: Emil[F], updateCheck: UpdateCheck[F], thisVersion: ThisVersion @@ -50,7 +51,7 @@ object UpdateCheckTask { _ <- ctx.logger.debug( s"Get SMTP connection for ${cfg.senderAccount.asString} and ${cfg.smtpId}" ) - smtpCfg <- findConnection(ctx, cfg) + smtpCfg <- findConnection(store, cfg) _ <- ctx.logger.debug("Checking for latest release at GitHub") latest <- updateCheck.latestRelease _ <- ctx.logger.debug(s"Got latest release: $latest.") @@ -77,10 +78,10 @@ object UpdateCheckTask { Task.pure(()) def findConnection[F[_]: Sync]( - ctx: Context[F, _], + store: Store[F], cfg: UpdateCheckConfig ): F[RUserEmail] = - OptionT(ctx.store.transact(RUserEmail.getByName(cfg.senderAccount, cfg.smtpId))) + OptionT(store.transact(RUserEmail.getByName(cfg.senderAccount, cfg.smtpId))) .getOrElseF( Sync[F].raiseError( new Exception( diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala index abefafa5..0594dcf8 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/Context.scala @@ -8,7 +8,6 @@ package docspell.scheduler import docspell.common._ import docspell.logging.Logger -import docspell.store.Store trait Context[F[_], A] { self => @@ -22,8 +21,6 @@ trait Context[F[_], A] { self => def setProgress(percent: Int): F[Unit] - def store: Store[F] - def isLastRetry: F[Boolean] def map[C](f: A => C): Context[F, C] diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/Scheduler.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/Scheduler.scala index 0e44244b..e3f1193e 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/Scheduler.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/Scheduler.scala @@ -10,13 +10,12 @@ import cats.effect._ import fs2.Stream import docspell.common.Ident -import docspell.store.records.RJob trait Scheduler[F[_]] { def config: SchedulerConfig - def getRunning: F[Vector[RJob]] + def getRunning: F[Vector[Job[String]]] def requestCancel(jobId: Ident): F[Boolean] diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala index 45647f03..69d2b34b 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala @@ -13,7 +13,7 @@ import docspell.store.records.RJob class ContextImpl[F[_]: Functor, A]( val args: A, val logger: Logger[F], - val store: Store[F], + store: Store[F], val config: SchedulerConfig, val jobId: Ident ) extends Context[F, A] { diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala index 4aad8901..0f063599 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala @@ -64,8 +64,23 @@ final class SchedulerImpl[F[_]: Async]( .drain ) - def getRunning: F[Vector[RJob]] = - state.get.flatMap(s => QJob.findAll(s.getRunning, store)) + def getRunning: F[Vector[Job[String]]] = + state.get + .flatMap(s => QJob.findAll(s.getRunning, store)) + .map( + _.map(rj => + Job( + rj.id, + rj.task, + rj.group, + rj.args, + rj.subject, + rj.submitter, + rj.priority, + rj.tracker + ) + ) + ) def requestCancel(jobId: Ident): F[Boolean] = logger.info(s"Scheduler requested to cancel job: ${jobId.id}") *> From 4c220879db2f5186e677857182507ae4c9e1d102 Mon Sep 17 00:00:00 2001 From: eikek Date: Sun, 13 Mar 2022 15:05:29 +0100 Subject: [PATCH 16/16] Reformat code --- .../scala/docspell/backend/BackendApp.scala | 2 + .../scala/docspell/backend/JobFactory.scala | 1 + .../scala/docspell/backend/msg/Topics.scala | 1 + .../docspell/backend/ops/OCollective.scala | 6 +- .../backend/ops/OFileRepository.scala | 2 + .../docspell/backend/ops/OFulltext.scala | 1 + .../scala/docspell/backend/ops/OItem.scala | 2 + .../scala/docspell/backend/ops/OJob.scala | 1 + .../scala/docspell/backend/ops/OJoex.scala | 1 + .../scala/docspell/backend/ops/OUpload.scala | 1 + .../docspell/backend/ops/OUserTask.scala | 2 + .../src/main/scala/docspell/joex/Config.scala | 2 +- .../scala/docspell/joex/JoexAppImpl.scala | 191 ++------------ .../main/scala/docspell/joex/JoexTasks.scala | 248 ++++++++++++++++++ .../joex/emptytrash/EmptyTrashTask.scala | 6 +- .../docspell/joex/filecopy/FileCopyTask.scala | 6 +- .../filecopy/FileIntegrityCheckTask.scala | 2 + .../scala/docspell/joex/fts/FtsWork.scala | 3 +- .../docspell/joex/fts/MigrationTask.scala | 1 + .../scala/docspell/joex/fts/ReIndexTask.scala | 1 + .../docspell/joex/hk/CheckNodesTask.scala | 4 +- .../docspell/joex/hk/CleanupInvitesTask.scala | 1 + .../joex/hk/CleanupRememberMeTask.scala | 1 + .../docspell/joex/hk/HouseKeepingTask.scala | 6 +- .../docspell/joex/hk/IntegrityCheckTask.scala | 1 + .../joex/learn/LearnClassifierTask.scala | 3 +- .../joex/learn/LearnItemEntities.scala | 1 + .../scala/docspell/joex/learn/LearnTags.scala | 1 + .../joex/notify/PeriodicDueItemsTask.scala | 5 +- .../joex/notify/PeriodicQueryTask.scala | 5 +- .../joex/pagecount/AllPageCountTask.scala | 3 +- .../joex/pagecount/MakePageCountTask.scala | 1 + .../joex/pdfconv/ConvertAllPdfTask.scala | 3 +- .../docspell/joex/pdfconv/PdfConvTask.scala | 2 + .../joex/preview/AllPreviewsTask.scala | 3 +- .../joex/preview/MakePreviewTask.scala | 1 + .../joex/process/AttachmentPageCount.scala | 1 + .../joex/process/AttachmentPreview.scala | 1 + .../docspell/joex/process/ConvertPdf.scala | 1 + .../docspell/joex/process/CreateItem.scala | 1 + .../joex/process/CrossCheckProposals.scala | 3 +- .../joex/process/DuplicateCheck.scala | 2 + .../docspell/joex/process/EvalProposals.scala | 2 + .../joex/process/ExtractArchive.scala | 2 + .../docspell/joex/process/FindProposal.scala | 2 + .../docspell/joex/process/ItemHandler.scala | 1 + .../docspell/joex/process/LinkProposal.scala | 1 + .../docspell/joex/process/ProcessItem.scala | 1 + .../docspell/joex/process/ReProcessItem.scala | 1 + .../docspell/joex/process/SaveProposals.scala | 3 +- .../docspell/joex/process/TextAnalysis.scala | 1 + .../joex/process/TextExtraction.scala | 1 + .../joex/scanmailbox/ScanMailboxTask.scala | 6 +- .../joex/updatecheck/UpdateCheckTask.scala | 2 + .../docspell/restserver/RestAppImpl.scala | 2 + .../docspell/restserver/Subscriptions.scala | 1 + .../routes/NotifyDueItemsRoutes.scala | 2 + .../routes/PeriodicQueryRoutes.scala | 2 + .../restserver/routes/ScanMailboxRoutes.scala | 2 + .../main/scala/docspell/scheduler/Job.scala | 8 + .../scala/docspell/scheduler/JobStore.scala | 6 + .../docspell/scheduler/JobStoreModule.scala | 6 + .../docspell/scheduler/SchedulerModule.scala | 6 + .../docspell/scheduler/msg/CancelJob.scala | 1 + .../docspell/scheduler/msg/JobSubmitted.scala | 1 + .../docspell/scheduler/msg/JobsNotify.scala | 6 + .../scheduler/msg/PeriodicTaskNotify.scala | 6 + .../scheduler/usertask/UserTask.scala | 3 +- .../scheduler/usertask/UserTaskStore.scala | 4 +- .../scheduler/CountingSchemeSpec.scala | 1 + .../docspell/scheduler/impl/ContextImpl.scala | 8 +- .../docspell/scheduler/impl/JobQueue.scala | 1 + .../scheduler/impl/JobStoreImpl.scala | 7 +- .../impl/JobStoreModuleBuilder.scala | 6 + .../docspell/scheduler/impl/LogSink.scala | 3 +- .../impl/PeriodicSchedulerBuilder.scala | 9 +- .../impl/PeriodicSchedulerImpl.scala | 2 + .../scheduler/impl/PeriodicTaskStore.scala | 1 + .../docspell/scheduler/impl/QUserTask.scala | 16 +- .../docspell/scheduler/impl/QueueLogger.scala | 11 +- .../scheduler/impl/SchedulerBuilder.scala | 3 +- .../scheduler/impl/SchedulerImpl.scala | 8 +- .../impl/SchedulerModuleBuilder.scala | 14 +- .../scheduler/impl/UserTaskStoreImpl.scala | 16 +- .../store/records/RPeriodicTask.scala | 2 + 85 files changed, 492 insertions(+), 225 deletions(-) create mode 100644 modules/joex/src/main/scala/docspell/joex/JoexTasks.scala diff --git a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala index ebe361ac..5d5dc532 100644 --- a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala +++ b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala @@ -7,6 +7,7 @@ package docspell.backend import cats.effect._ + import docspell.backend.auth.Login import docspell.backend.fulltext.CreateIndex import docspell.backend.ops._ @@ -17,6 +18,7 @@ import docspell.pubsub.api.PubSubT import docspell.scheduler.JobStoreModule import docspell.store.Store import docspell.totp.Totp + import emil.Emil trait BackendApp[F[_]] { diff --git a/modules/backend/src/main/scala/docspell/backend/JobFactory.scala b/modules/backend/src/main/scala/docspell/backend/JobFactory.scala index d18356fa..9a79f104 100644 --- a/modules/backend/src/main/scala/docspell/backend/JobFactory.scala +++ b/modules/backend/src/main/scala/docspell/backend/JobFactory.scala @@ -8,6 +8,7 @@ package docspell.backend import cats.effect._ import cats.implicits._ + import docspell.backend.MailAddressCodec import docspell.common._ import docspell.notification.api.PeriodicQueryArgs diff --git a/modules/backend/src/main/scala/docspell/backend/msg/Topics.scala b/modules/backend/src/main/scala/docspell/backend/msg/Topics.scala index 3a887097..63d8c401 100644 --- a/modules/backend/src/main/scala/docspell/backend/msg/Topics.scala +++ b/modules/backend/src/main/scala/docspell/backend/msg/Topics.scala @@ -7,6 +7,7 @@ package docspell.backend.msg import cats.data.NonEmptyList + import docspell.pubsub.api.TypedTopic import docspell.scheduler.msg._ diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala index 09f66ef4..e59ddb9a 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala @@ -9,17 +9,19 @@ package docspell.backend.ops import cats.effect.{Async, Resource} import cats.implicits._ import fs2.Stream + import docspell.backend.JobFactory import docspell.backend.PasswordCrypt import docspell.backend.ops.OCollective._ import docspell.common._ +import docspell.scheduler.JobStore +import docspell.scheduler.usertask.{UserTask, UserTaskScope, UserTaskStore} import docspell.store.UpdateResult import docspell.store.queries.{QCollective, QUser} import docspell.store.records._ import docspell.store.{AddResult, Store} + import com.github.eikek.calev._ -import docspell.scheduler.JobStore -import docspell.scheduler.usertask.{UserTask, UserTaskScope, UserTaskStore} trait OCollective[F[_]] { diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala index b46c075f..0e2d9cdb 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFileRepository.scala @@ -9,11 +9,13 @@ package docspell.backend.ops import cats.data.OptionT import cats.effect._ import cats.implicits._ + import docspell.backend.JobFactory import docspell.backend.ops.OFileRepository.IntegrityResult import docspell.common._ import docspell.scheduler.{Job, JobStore} import docspell.store.Store + import scodec.bits.ByteVector trait OFileRepository[F[_]] { diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala index 8fcdb9cd..39452ea5 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala @@ -10,6 +10,7 @@ import cats.data.NonEmptyList import cats.effect._ import cats.implicits._ import fs2.Stream + import docspell.backend.JobFactory import docspell.backend.ops.OItemSearch._ import docspell.common._ diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala index 3a8b08ea..04643348 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala @@ -9,6 +9,7 @@ package docspell.backend.ops import cats.data.{NonEmptyList => Nel, OptionT} import cats.effect.{Async, Resource} import cats.implicits._ + import docspell.backend.AttachedEvent import docspell.backend.JobFactory import docspell.backend.fulltext.CreateIndex @@ -21,6 +22,7 @@ import docspell.scheduler.JobStore import docspell.store.queries.{QAttachment, QItem, QMoveAttachment} import docspell.store.records._ import docspell.store.{AddResult, Store, UpdateResult} + import doobie.implicits._ trait OItem[F[_]] { diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala b/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala index ad1cc8c8..e0c14552 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala @@ -9,6 +9,7 @@ package docspell.backend.ops import cats.data.OptionT import cats.effect._ import cats.implicits._ + import docspell.backend.ops.OJob.{CollectiveQueueState, JobCancelResult} import docspell.common._ import docspell.pubsub.api.PubSubT diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala b/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala index f82285d9..9f83d46c 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala @@ -9,6 +9,7 @@ package docspell.backend.ops import cats.Applicative import cats.effect._ import cats.implicits._ + import docspell.common.Ident import docspell.pubsub.api.PubSubT import docspell.scheduler.msg.{CancelJob, JobsNotify, PeriodicTaskNotify} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala index 674aa5a4..ca523cf8 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala @@ -11,6 +11,7 @@ import cats.data.{EitherT, OptionT} import cats.effect._ import cats.implicits._ import fs2.Stream + import docspell.backend.JobFactory import docspell.common._ import docspell.scheduler.{Job, JobStore} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala b/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala index f1171bd7..7fc80b7d 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OUserTask.scala @@ -10,11 +10,13 @@ import cats.data.{NonEmptyList, OptionT} import cats.effect._ import cats.implicits._ import fs2.Stream + import docspell.common._ import docspell.notification.api.{ChannelRef, PeriodicDueItemsArgs, PeriodicQueryArgs} import docspell.scheduler.usertask.{UserTask, UserTaskScope, UserTaskStore} import docspell.store.Store import docspell.store.records.RNotificationChannel + import io.circe.Encoder trait OUserTask[F[_]] { diff --git a/modules/joex/src/main/scala/docspell/joex/Config.scala b/modules/joex/src/main/scala/docspell/joex/Config.scala index 0d0945d6..3418a56d 100644 --- a/modules/joex/src/main/scala/docspell/joex/Config.scala +++ b/modules/joex/src/main/scala/docspell/joex/Config.scala @@ -19,10 +19,10 @@ import docspell.ftssolr.SolrConfig import docspell.joex.analysis.RegexNerFile import docspell.joex.hk.HouseKeepingConfig import docspell.joex.routes.InternalHeader -import docspell.scheduler.{PeriodicSchedulerConfig, SchedulerConfig} import docspell.joex.updatecheck.UpdateCheckConfig import docspell.logging.LogConfig import docspell.pubsub.naive.PubSubConfig +import docspell.scheduler.{PeriodicSchedulerConfig, SchedulerConfig} import docspell.store.JdbcConfig case class Config( diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala index df0f53f0..5d8b6a40 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala @@ -9,36 +9,25 @@ package docspell.joex import cats.effect._ import cats.implicits._ import fs2.concurrent.SignallingRef -import docspell.analysis.TextAnalyser + import docspell.backend.MailAddressCodec -import docspell.backend.fulltext.CreateIndex import docspell.backend.ops._ import docspell.common._ -import docspell.ftsclient.FtsClient -import docspell.ftssolr.SolrFtsClient -import docspell.joex.analysis.RegexNerFile import docspell.joex.emptytrash._ -import docspell.joex.filecopy.{FileCopyTask, FileIntegrityCheckTask} -import docspell.joex.fts.{MigrationTask, ReIndexTask} +import docspell.joex.fts.MigrationTask import docspell.joex.hk._ -import docspell.joex.learn.LearnClassifierTask -import docspell.joex.notify._ import docspell.joex.pagecount._ -import docspell.joex.pdfconv.ConvertAllPdfTask -import docspell.joex.pdfconv.PdfConvTask import docspell.joex.preview._ -import docspell.joex.process.ItemHandler -import docspell.joex.process.ReProcessItem -import docspell.joex.scanmailbox._ -import docspell.scheduler._ -import docspell.scheduler.impl.{JobStoreModuleBuilder, SchedulerModuleBuilder} import docspell.joex.updatecheck._ import docspell.notification.api.NotificationModule import docspell.notification.impl.NotificationModuleImpl import docspell.pubsub.api.{PubSub, PubSubT} +import docspell.scheduler._ +import docspell.scheduler.impl.{JobStoreModuleBuilder, SchedulerModuleBuilder} import docspell.scheduler.usertask.{UserTaskScope, UserTaskStore} import docspell.store.Store import docspell.store.records.{REmptyTrashSetting, RJobLog} + import emil.javamail._ import org.http4s.client.Client @@ -129,165 +118,21 @@ object JoexAppImpl extends MailAddressCodec { .withEventSink(notificationMod) .build - joex <- OJoex(pubSubT) - upload <- OUpload(store, jobStoreModule.jobs, joex) - fts <- createFtsClient(cfg)(httpClient) - createIndex <- CreateIndex.resource(fts, store) - itemOps <- OItem(store, fts, createIndex, jobStoreModule.jobs, joex) - itemSearchOps <- OItemSearch(store) - analyser <- TextAnalyser.create[F](cfg.textAnalysis.textAnalysisConfig) - regexNer <- RegexNerFile(cfg.textAnalysis.regexNerFileConfig, store) - updateCheck <- UpdateCheck.resource(httpClient) - notification <- ONotification(store, notificationMod) - fileRepo <- OFileRepository(store, jobStoreModule.jobs, joex) + tasks <- JoexTasks.resource( + cfg, + jobStoreModule, + httpClient, + pubSubT, + notificationMod, + javaEmil + ) schedulerModule <- SchedulerModuleBuilder(jobStoreModule) .withSchedulerConfig(cfg.scheduler) .withPeriodicSchedulerConfig(cfg.periodicScheduler) - .withTaskRegistry(JobTaskRegistry - .empty[F] - .withTask( - JobTask.json( - ProcessItemArgs.taskName, - ItemHandler.newItem[F](cfg,store, itemOps, fts, analyser, regexNer), - ItemHandler.onCancel[F](store) - ) - ) - .withTask( - JobTask.json( - ReProcessItemArgs.taskName, - ReProcessItem[F](cfg, fts, itemOps, analyser, regexNer, store), - ReProcessItem.onCancel[F] - ) - ) - .withTask( - JobTask.json( - ScanMailboxArgs.taskName, - ScanMailboxTask[F](cfg.userTasks.scanMailbox, store, javaEmil, upload, joex), - ScanMailboxTask.onCancel[F] - ) - ) - .withTask( - JobTask.json( - MigrationTask.taskName, - MigrationTask[F](cfg.fullTextSearch, store, fts, createIndex), - MigrationTask.onCancel[F] - ) - ) - .withTask( - JobTask.json( - ReIndexTask.taskName, - ReIndexTask[F](cfg.fullTextSearch, store, fts, createIndex), - ReIndexTask.onCancel[F] - ) - ) - .withTask( - JobTask.json( - HouseKeepingTask.taskName, - HouseKeepingTask[F](cfg, store, fileRepo), - HouseKeepingTask.onCancel[F] - ) - ) - .withTask( - JobTask.json( - PdfConvTask.taskName, - PdfConvTask[F](cfg, store), - PdfConvTask.onCancel[F] - ) - ) - .withTask( - JobTask.json( - ConvertAllPdfArgs.taskName, - ConvertAllPdfTask[F](jobStoreModule.jobs, joex, store), - ConvertAllPdfTask.onCancel[F] - ) - ) - .withTask( - JobTask.json( - LearnClassifierArgs.taskName, - LearnClassifierTask[F](cfg.textAnalysis, store, analyser), - LearnClassifierTask.onCancel[F] - ) - ) - .withTask( - JobTask.json( - MakePreviewArgs.taskName, - MakePreviewTask[F](cfg.extraction.preview, store), - MakePreviewTask.onCancel[F] - ) - ) - .withTask( - JobTask.json( - AllPreviewsArgs.taskName, - AllPreviewsTask[F](jobStoreModule.jobs, joex, store), - AllPreviewsTask.onCancel[F] - ) - ) - .withTask( - JobTask.json( - MakePageCountArgs.taskName, - MakePageCountTask[F](store), - MakePageCountTask.onCancel[F] - ) - ) - .withTask( - JobTask.json( - AllPageCountTask.taskName, - AllPageCountTask[F](store, jobStoreModule.jobs, joex), - AllPageCountTask.onCancel[F] - ) - ) - .withTask( - JobTask.json( - EmptyTrashArgs.taskName, - EmptyTrashTask[F](itemOps, itemSearchOps), - EmptyTrashTask.onCancel[F] - ) - ) - .withTask( - JobTask.json( - UpdateCheckTask.taskName, - UpdateCheckTask[F]( - cfg.updateCheck, - cfg.sendMail, - store, - javaEmil, - updateCheck, - ThisVersion.default - ), - UpdateCheckTask.onCancel[F] - ) - ) - .withTask( - JobTask.json( - PeriodicQueryTask.taskName, - PeriodicQueryTask[F](store, notification), - PeriodicQueryTask.onCancel[F] - ) - ) - .withTask( - JobTask.json( - PeriodicDueItemsTask.taskName, - PeriodicDueItemsTask[F](store, notification), - PeriodicDueItemsTask.onCancel[F] - ) - ) - .withTask( - JobTask.json( - FileCopyTaskArgs.taskName, - FileCopyTask[F](cfg, store), - FileCopyTask.onCancel[F] - ) - ) - .withTask( - JobTask.json( - FileIntegrityCheckArgs.taskName, - FileIntegrityCheckTask[F](fileRepo, store), - FileIntegrityCheckTask.onCancel[F] - ) - ) - ) + .withTaskRegistry(tasks.get) .resource + app = new JoexAppImpl( cfg, store, @@ -301,10 +146,4 @@ object JoexAppImpl extends MailAddressCodec { appR <- Resource.make(app.init.map(_ => app))(_.initShutdown) } yield appR - private def createFtsClient[F[_]: Async]( - cfg: Config - )(client: Client[F]): Resource[F, FtsClient[F]] = - if (cfg.fullTextSearch.enabled) SolrFtsClient(cfg.fullTextSearch.solr, client) - else Resource.pure[F, FtsClient[F]](FtsClient.none[F]) - } diff --git a/modules/joex/src/main/scala/docspell/joex/JoexTasks.scala b/modules/joex/src/main/scala/docspell/joex/JoexTasks.scala new file mode 100644 index 00000000..8d2f0a1f --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/JoexTasks.scala @@ -0,0 +1,248 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package docspell.joex + +import cats.effect.{Async, Resource} + +import docspell.analysis.TextAnalyser +import docspell.backend.fulltext.CreateIndex +import docspell.backend.ops._ +import docspell.common._ +import docspell.ftsclient.FtsClient +import docspell.ftssolr.SolrFtsClient +import docspell.joex.analysis.RegexNerFile +import docspell.joex.emptytrash.EmptyTrashTask +import docspell.joex.filecopy.{FileCopyTask, FileIntegrityCheckTask} +import docspell.joex.fts.{MigrationTask, ReIndexTask} +import docspell.joex.hk.HouseKeepingTask +import docspell.joex.learn.LearnClassifierTask +import docspell.joex.notify.{PeriodicDueItemsTask, PeriodicQueryTask} +import docspell.joex.pagecount.{AllPageCountTask, MakePageCountTask} +import docspell.joex.pdfconv.{ConvertAllPdfTask, PdfConvTask} +import docspell.joex.preview.{AllPreviewsTask, MakePreviewTask} +import docspell.joex.process.{ItemHandler, ReProcessItem} +import docspell.joex.scanmailbox.ScanMailboxTask +import docspell.joex.updatecheck.{ThisVersion, UpdateCheck, UpdateCheckTask} +import docspell.notification.api.NotificationModule +import docspell.pubsub.api.PubSubT +import docspell.scheduler.impl.JobStoreModuleBuilder +import docspell.scheduler.{JobStoreModule, JobTask, JobTaskRegistry} +import docspell.store.Store + +import emil.Emil +import org.http4s.client.Client + +final class JoexTasks[F[_]: Async]( + cfg: Config, + store: Store[F], + itemOps: OItem[F], + fts: FtsClient[F], + analyser: TextAnalyser[F], + regexNer: RegexNerFile[F], + updateCheck: UpdateCheck[F], + notification: ONotification[F], + fileRepo: OFileRepository[F], + javaEmil: Emil[F], + jobStoreModule: JobStoreModule[F], + upload: OUpload[F], + createIndex: CreateIndex[F], + joex: OJoex[F], + itemSearch: OItemSearch[F] +) { + + def get: JobTaskRegistry[F] = + JobTaskRegistry + .empty[F] + .withTask( + JobTask.json( + ProcessItemArgs.taskName, + ItemHandler.newItem[F](cfg, store, itemOps, fts, analyser, regexNer), + ItemHandler.onCancel[F](store) + ) + ) + .withTask( + JobTask.json( + ReProcessItemArgs.taskName, + ReProcessItem[F](cfg, fts, itemOps, analyser, regexNer, store), + ReProcessItem.onCancel[F] + ) + ) + .withTask( + JobTask.json( + ScanMailboxArgs.taskName, + ScanMailboxTask[F](cfg.userTasks.scanMailbox, store, javaEmil, upload, joex), + ScanMailboxTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + MigrationTask.taskName, + MigrationTask[F](cfg.fullTextSearch, store, fts, createIndex), + MigrationTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + ReIndexTask.taskName, + ReIndexTask[F](cfg.fullTextSearch, store, fts, createIndex), + ReIndexTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + HouseKeepingTask.taskName, + HouseKeepingTask[F](cfg, store, fileRepo), + HouseKeepingTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + PdfConvTask.taskName, + PdfConvTask[F](cfg, store), + PdfConvTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + ConvertAllPdfArgs.taskName, + ConvertAllPdfTask[F](jobStoreModule.jobs, joex, store), + ConvertAllPdfTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + LearnClassifierArgs.taskName, + LearnClassifierTask[F](cfg.textAnalysis, store, analyser), + LearnClassifierTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + MakePreviewArgs.taskName, + MakePreviewTask[F](cfg.extraction.preview, store), + MakePreviewTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + AllPreviewsArgs.taskName, + AllPreviewsTask[F](jobStoreModule.jobs, joex, store), + AllPreviewsTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + MakePageCountArgs.taskName, + MakePageCountTask[F](store), + MakePageCountTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + AllPageCountTask.taskName, + AllPageCountTask[F](store, jobStoreModule.jobs, joex), + AllPageCountTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + EmptyTrashArgs.taskName, + EmptyTrashTask[F](itemOps, itemSearch), + EmptyTrashTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + UpdateCheckTask.taskName, + UpdateCheckTask[F]( + cfg.updateCheck, + cfg.sendMail, + store, + javaEmil, + updateCheck, + ThisVersion.default + ), + UpdateCheckTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + PeriodicQueryTask.taskName, + PeriodicQueryTask[F](store, notification), + PeriodicQueryTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + PeriodicDueItemsTask.taskName, + PeriodicDueItemsTask[F](store, notification), + PeriodicDueItemsTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + FileCopyTaskArgs.taskName, + FileCopyTask[F](cfg, store), + FileCopyTask.onCancel[F] + ) + ) + .withTask( + JobTask.json( + FileIntegrityCheckArgs.taskName, + FileIntegrityCheckTask[F](fileRepo, store), + FileIntegrityCheckTask.onCancel[F] + ) + ) +} + +object JoexTasks { + + def resource[F[_]: Async]( + cfg: Config, + jobStoreModule: JobStoreModuleBuilder.Module[F], + httpClient: Client[F], + pubSub: PubSubT[F], + notificationModule: NotificationModule[F], + emailService: Emil[F] + ): Resource[F, JoexTasks[F]] = + for { + joex <- OJoex(pubSub) + store = jobStoreModule.store + upload <- OUpload(store, jobStoreModule.jobs, joex) + fts <- createFtsClient(cfg)(httpClient) + createIndex <- CreateIndex.resource(fts, store) + itemOps <- OItem(store, fts, createIndex, jobStoreModule.jobs, joex) + itemSearchOps <- OItemSearch(store) + analyser <- TextAnalyser.create[F](cfg.textAnalysis.textAnalysisConfig) + regexNer <- RegexNerFile(cfg.textAnalysis.regexNerFileConfig, store) + updateCheck <- UpdateCheck.resource(httpClient) + notification <- ONotification(store, notificationModule) + fileRepo <- OFileRepository(store, jobStoreModule.jobs, joex) + } yield new JoexTasks[F]( + cfg, + store, + itemOps, + fts, + analyser, + regexNer, + updateCheck, + notification, + fileRepo, + emailService, + jobStoreModule, + upload, + createIndex, + joex, + itemSearchOps + ) + + private def createFtsClient[F[_]: Async]( + cfg: Config + )(client: Client[F]): Resource[F, FtsClient[F]] = + if (cfg.fullTextSearch.enabled) SolrFtsClient(cfg.fullTextSearch.solr, client) + else Resource.pure[F, FtsClient[F]](FtsClient.none[F]) +} diff --git a/modules/joex/src/main/scala/docspell/joex/emptytrash/EmptyTrashTask.scala b/modules/joex/src/main/scala/docspell/joex/emptytrash/EmptyTrashTask.scala index 04cfc337..3d50a0ca 100644 --- a/modules/joex/src/main/scala/docspell/joex/emptytrash/EmptyTrashTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/emptytrash/EmptyTrashTask.scala @@ -9,12 +9,14 @@ package docspell.joex.emptytrash import cats.effect._ import cats.implicits._ import fs2.Stream + import docspell.backend.ops.{OItem, OItemSearch} import docspell.common._ import docspell.scheduler._ -import docspell.store.records.RItem -import com.github.eikek.calev.CalEvent import docspell.scheduler.usertask.UserTask +import docspell.store.records.RItem + +import com.github.eikek.calev.CalEvent object EmptyTrashTask { type Args = EmptyTrashArgs diff --git a/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala b/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala index 8c89a157..630857fd 100644 --- a/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala @@ -9,15 +9,17 @@ package docspell.joex.filecopy import cats.data.NonEmptyList import cats.effect._ import cats.implicits._ + import docspell.common.FileCopyTaskArgs.Selection import docspell.common.{FileCopyTaskArgs, Ident} import docspell.joex.Config -import docspell.scheduler.{JobTaskResultEncoder, Task} import docspell.logging.Logger +import docspell.scheduler.{JobTaskResultEncoder, Task} +import docspell.store.Store import docspell.store.file.{BinnyUtils, FileRepository, FileRepositoryConfig} + import binny.CopyTool.Counter import binny.{BinaryId, BinaryStore, CopyTool} -import docspell.store.Store import io.circe.generic.semiauto.deriveCodec import io.circe.{Codec, Decoder, Encoder} diff --git a/modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala b/modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala index 1d407453..4c1b1751 100644 --- a/modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/filecopy/FileIntegrityCheckTask.scala @@ -9,12 +9,14 @@ package docspell.joex.filecopy import cats.Monoid import cats.effect._ import cats.implicits._ + import docspell.backend.ops.OFileRepository import docspell.backend.ops.OFileRepository.IntegrityResult import docspell.common.{FileIntegrityCheckArgs, FileKey} import docspell.scheduler.{JobTaskResultEncoder, Task} import docspell.store.Store import docspell.store.records.RFileMeta + import io.circe.Encoder import io.circe.generic.semiauto.deriveEncoder diff --git a/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala b/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala index 8379098f..3f5ed2a6 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/FtsWork.scala @@ -9,12 +9,13 @@ package docspell.joex.fts import cats._ import cats.data.{Kleisli, NonEmptyList} import cats.implicits._ + import docspell.backend.fulltext.CreateIndex import docspell.common._ import docspell.ftsclient._ import docspell.joex.Config -import docspell.scheduler.Context import docspell.logging.Logger +import docspell.scheduler.Context import docspell.store.Store object FtsWork { diff --git a/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala b/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala index 6430b332..ddb44a21 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/MigrationTask.scala @@ -8,6 +8,7 @@ package docspell.joex.fts import cats.effect._ import cats.implicits._ + import docspell.backend.fulltext.CreateIndex import docspell.common._ import docspell.ftsclient._ diff --git a/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala b/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala index ca1dd228..b457d2fb 100644 --- a/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/fts/ReIndexTask.scala @@ -8,6 +8,7 @@ package docspell.joex.fts import cats.effect._ import cats.implicits._ + import docspell.backend.fulltext.CreateIndex import docspell.common._ import docspell.ftsclient._ diff --git a/modules/joex/src/main/scala/docspell/joex/hk/CheckNodesTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/CheckNodesTask.scala index 1d202b52..68256465 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/CheckNodesTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/CheckNodesTask.scala @@ -8,11 +8,13 @@ package docspell.joex.hk import cats.effect._ import cats.implicits._ + import docspell.common._ -import docspell.scheduler.Task import docspell.logging.Logger +import docspell.scheduler.Task import docspell.store.Store import docspell.store.records._ + import org.http4s.blaze.client.BlazeClientBuilder import org.http4s.client.Client diff --git a/modules/joex/src/main/scala/docspell/joex/hk/CleanupInvitesTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/CleanupInvitesTask.scala index 2dbed981..864a73d7 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/CleanupInvitesTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/CleanupInvitesTask.scala @@ -8,6 +8,7 @@ package docspell.joex.hk import cats.effect._ import cats.implicits._ + import docspell.common._ import docspell.scheduler.Task import docspell.store.Store diff --git a/modules/joex/src/main/scala/docspell/joex/hk/CleanupRememberMeTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/CleanupRememberMeTask.scala index b3d909ab..574abfc7 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/CleanupRememberMeTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/CleanupRememberMeTask.scala @@ -8,6 +8,7 @@ package docspell.joex.hk import cats.effect._ import cats.implicits._ + import docspell.common._ import docspell.scheduler.Task import docspell.store.Store diff --git a/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala index 5e87ce17..0da26529 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala @@ -8,14 +8,16 @@ package docspell.joex.hk import cats.effect._ import cats.implicits._ + import docspell.backend.ops.OFileRepository import docspell.common._ import docspell.joex.Config import docspell.joex.filecopy.FileIntegrityCheckTask -import docspell.scheduler.{JobTaskResultEncoder, Task} -import com.github.eikek.calev._ import docspell.scheduler.usertask.UserTask +import docspell.scheduler.{JobTaskResultEncoder, Task} import docspell.store.Store + +import com.github.eikek.calev._ import io.circe.Encoder import io.circe.generic.semiauto.deriveEncoder diff --git a/modules/joex/src/main/scala/docspell/joex/hk/IntegrityCheckTask.scala b/modules/joex/src/main/scala/docspell/joex/hk/IntegrityCheckTask.scala index 06fca014..90a3a073 100644 --- a/modules/joex/src/main/scala/docspell/joex/hk/IntegrityCheckTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/hk/IntegrityCheckTask.scala @@ -8,6 +8,7 @@ package docspell.joex.hk import cats.effect._ import cats.implicits._ + import docspell.backend.ops.OFileRepository import docspell.common._ import docspell.joex.filecopy.FileIntegrityCheckTask diff --git a/modules/joex/src/main/scala/docspell/joex/learn/LearnClassifierTask.scala b/modules/joex/src/main/scala/docspell/joex/learn/LearnClassifierTask.scala index ce49077f..0de696c9 100644 --- a/modules/joex/src/main/scala/docspell/joex/learn/LearnClassifierTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/learn/LearnClassifierTask.scala @@ -9,12 +9,13 @@ package docspell.joex.learn import cats.data.OptionT import cats.effect._ import cats.implicits._ + import docspell.analysis.TextAnalyser import docspell.backend.ops.OCollective import docspell.common._ import docspell.joex.Config -import docspell.scheduler._ import docspell.logging.Logger +import docspell.scheduler._ import docspell.store.Store import docspell.store.records.{RClassifierModel, RClassifierSetting} diff --git a/modules/joex/src/main/scala/docspell/joex/learn/LearnItemEntities.scala b/modules/joex/src/main/scala/docspell/joex/learn/LearnItemEntities.scala index 7d59afef..19048abd 100644 --- a/modules/joex/src/main/scala/docspell/joex/learn/LearnItemEntities.scala +++ b/modules/joex/src/main/scala/docspell/joex/learn/LearnItemEntities.scala @@ -10,6 +10,7 @@ import cats.data.Kleisli import cats.effect._ import cats.implicits._ import fs2.Stream + import docspell.analysis.TextAnalyser import docspell.analysis.classifier.TextClassifier.Data import docspell.common._ diff --git a/modules/joex/src/main/scala/docspell/joex/learn/LearnTags.scala b/modules/joex/src/main/scala/docspell/joex/learn/LearnTags.scala index 54eb5a88..e80fe83c 100644 --- a/modules/joex/src/main/scala/docspell/joex/learn/LearnTags.scala +++ b/modules/joex/src/main/scala/docspell/joex/learn/LearnTags.scala @@ -9,6 +9,7 @@ package docspell.joex.learn import cats.data.Kleisli import cats.effect._ import cats.implicits._ + import docspell.analysis.TextAnalyser import docspell.common._ import docspell.scheduler._ diff --git a/modules/joex/src/main/scala/docspell/joex/notify/PeriodicDueItemsTask.scala b/modules/joex/src/main/scala/docspell/joex/notify/PeriodicDueItemsTask.scala index 1969a1aa..94db119b 100644 --- a/modules/joex/src/main/scala/docspell/joex/notify/PeriodicDueItemsTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/notify/PeriodicDueItemsTask.scala @@ -9,16 +9,17 @@ package docspell.joex.notify import cats.data.NonEmptyList import cats.effect._ import cats.implicits._ + import docspell.backend.ops.ONotification import docspell.common._ -import docspell.scheduler.Context -import docspell.scheduler.Task import docspell.notification.api.EventContext import docspell.notification.api.NotificationChannel import docspell.notification.api.PeriodicDueItemsArgs import docspell.query.Date import docspell.query.ItemQuery._ import docspell.query.ItemQueryDsl._ +import docspell.scheduler.Context +import docspell.scheduler.Task import docspell.store.Store import docspell.store.qb.Batch import docspell.store.queries.ListItem diff --git a/modules/joex/src/main/scala/docspell/joex/notify/PeriodicQueryTask.scala b/modules/joex/src/main/scala/docspell/joex/notify/PeriodicQueryTask.scala index 3b08edca..af1242cd 100644 --- a/modules/joex/src/main/scala/docspell/joex/notify/PeriodicQueryTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/notify/PeriodicQueryTask.scala @@ -10,10 +10,9 @@ import cats.data.OptionT import cats.data.{NonEmptyList => Nel} import cats.effect._ import cats.implicits._ + import docspell.backend.ops.ONotification import docspell.common._ -import docspell.scheduler.Context -import docspell.scheduler.Task import docspell.notification.api.EventContext import docspell.notification.api.NotificationChannel import docspell.notification.api.PeriodicQueryArgs @@ -21,6 +20,8 @@ import docspell.query.ItemQuery import docspell.query.ItemQuery.Expr import docspell.query.ItemQuery.Expr.AndExpr import docspell.query.ItemQueryParser +import docspell.scheduler.Context +import docspell.scheduler.Task import docspell.store.Store import docspell.store.qb.Batch import docspell.store.queries.ListItem diff --git a/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala b/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala index 6ff6b578..f6f91346 100644 --- a/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/pagecount/AllPageCountTask.scala @@ -9,10 +9,11 @@ package docspell.joex.pagecount import cats.effect._ import cats.implicits._ import fs2.{Chunk, Stream} + import docspell.backend.JobFactory import docspell.backend.ops.OJoex import docspell.common._ -import docspell.scheduler.{Context, Job, JobStore, Task} +import docspell.scheduler._ import docspell.store.Store import docspell.store.records.RAttachment diff --git a/modules/joex/src/main/scala/docspell/joex/pagecount/MakePageCountTask.scala b/modules/joex/src/main/scala/docspell/joex/pagecount/MakePageCountTask.scala index bcb5fd77..2769ac00 100644 --- a/modules/joex/src/main/scala/docspell/joex/pagecount/MakePageCountTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/pagecount/MakePageCountTask.scala @@ -8,6 +8,7 @@ package docspell.joex.pagecount import cats.effect._ import cats.implicits._ + import docspell.common._ import docspell.joex.process.AttachmentPageCount import docspell.scheduler.Context diff --git a/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala b/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala index 47b66323..84d3687d 100644 --- a/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/pdfconv/ConvertAllPdfTask.scala @@ -9,9 +9,10 @@ package docspell.joex.pdfconv import cats.effect._ import cats.implicits._ import fs2.{Chunk, Stream} + import docspell.backend.ops.OJoex import docspell.common._ -import docspell.scheduler.{Context, Job, JobStore, Task} +import docspell.scheduler._ import docspell.store.Store import docspell.store.records.RAttachment diff --git a/modules/joex/src/main/scala/docspell/joex/pdfconv/PdfConvTask.scala b/modules/joex/src/main/scala/docspell/joex/pdfconv/PdfConvTask.scala index 7bf6fd5c..a1ce38fb 100644 --- a/modules/joex/src/main/scala/docspell/joex/pdfconv/PdfConvTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/pdfconv/PdfConvTask.scala @@ -11,6 +11,7 @@ import cats.data.OptionT import cats.effect._ import cats.implicits._ import fs2.Stream + import docspell.common._ import docspell.convert.ConversionResult import docspell.convert.extern.OcrMyPdf @@ -18,6 +19,7 @@ import docspell.joex.Config import docspell.scheduler.{Context, Task} import docspell.store.Store import docspell.store.records._ + import io.circe.generic.semiauto._ import io.circe.{Decoder, Encoder} diff --git a/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala b/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala index dcf66d56..b0988305 100644 --- a/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/preview/AllPreviewsTask.scala @@ -9,11 +9,12 @@ package docspell.joex.preview import cats.effect._ import cats.implicits._ import fs2.{Chunk, Stream} + import docspell.backend.JobFactory import docspell.backend.ops.OJoex import docspell.common.MakePreviewArgs.StoreMode import docspell.common._ -import docspell.scheduler.{Context, Job, JobStore, Task} +import docspell.scheduler._ import docspell.store.Store import docspell.store.records.RAttachment diff --git a/modules/joex/src/main/scala/docspell/joex/preview/MakePreviewTask.scala b/modules/joex/src/main/scala/docspell/joex/preview/MakePreviewTask.scala index afd541ea..ce807205 100644 --- a/modules/joex/src/main/scala/docspell/joex/preview/MakePreviewTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/preview/MakePreviewTask.scala @@ -8,6 +8,7 @@ package docspell.joex.preview import cats.effect._ import cats.implicits._ + import docspell.common._ import docspell.extract.pdfbox.PdfboxPreview import docspell.extract.pdfbox.PreviewConfig diff --git a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala index 1e216897..7c2880aa 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPageCount.scala @@ -11,6 +11,7 @@ import cats.data.OptionT import cats.effect._ import cats.implicits._ import fs2.Stream + import docspell.common._ import docspell.extract.pdfbox.PdfMetaData import docspell.extract.pdfbox.PdfboxExtract diff --git a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala index db7c5f14..31c1e007 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/AttachmentPreview.scala @@ -11,6 +11,7 @@ import cats.data.OptionT import cats.effect._ import cats.implicits._ import fs2.Stream + import docspell.common._ import docspell.extract.pdfbox.PdfboxPreview import docspell.extract.pdfbox.PreviewConfig diff --git a/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala b/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala index 1d4b9196..648d6b29 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ConvertPdf.scala @@ -11,6 +11,7 @@ import cats.data.{Kleisli, OptionT} import cats.effect._ import cats.implicits._ import fs2.Stream + import docspell.common._ import docspell.convert.ConversionResult.Handler import docspell.convert.SanitizeHtml diff --git a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala index 7bd299bb..b5ae352b 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala @@ -11,6 +11,7 @@ import cats.data.OptionT import cats.effect.Sync import cats.implicits._ import fs2.Stream + import docspell.common._ import docspell.scheduler.{Context, Task} import docspell.store.Store diff --git a/modules/joex/src/main/scala/docspell/joex/process/CrossCheckProposals.scala b/modules/joex/src/main/scala/docspell/joex/process/CrossCheckProposals.scala index 74066434..58f24854 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/CrossCheckProposals.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/CrossCheckProposals.scala @@ -10,9 +10,10 @@ import cats.data.NonEmptyList import cats.data.OptionT import cats.effect.Sync import cats.implicits._ + import docspell.common._ -import docspell.scheduler.Task import docspell.logging.Logger +import docspell.scheduler.Task import docspell.store.Store /** After candidates have been determined, the set is reduced by doing some cross checks. diff --git a/modules/joex/src/main/scala/docspell/joex/process/DuplicateCheck.scala b/modules/joex/src/main/scala/docspell/joex/process/DuplicateCheck.scala index 44b01592..ed0035b3 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/DuplicateCheck.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/DuplicateCheck.scala @@ -8,12 +8,14 @@ package docspell.joex.process import cats.effect._ import cats.implicits._ + import docspell.common._ import docspell.scheduler.{Context, Task} import docspell.store.Store import docspell.store.queries.QItem import docspell.store.records.RFileMeta import docspell.store.records.RJob + import doobie._ object DuplicateCheck { diff --git a/modules/joex/src/main/scala/docspell/joex/process/EvalProposals.scala b/modules/joex/src/main/scala/docspell/joex/process/EvalProposals.scala index 327c0d08..521aeb5b 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/EvalProposals.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/EvalProposals.scala @@ -7,8 +7,10 @@ package docspell.joex.process import java.time.{LocalDate, Period} + import cats.effect.Sync import cats.implicits._ + import docspell.common._ import docspell.scheduler.Task import docspell.store.Store diff --git a/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala b/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala index c4b02630..5e2d86b0 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ExtractArchive.scala @@ -14,12 +14,14 @@ import cats.implicits._ import cats.kernel.Monoid import cats.kernel.Order import fs2.Stream + import docspell.common._ import docspell.files.Zip import docspell.joex.mail._ import docspell.scheduler._ import docspell.store.Store import docspell.store.records._ + import emil.Mail /** Goes through all attachments and extracts archive files, like zip files. The process diff --git a/modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala b/modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala index 80beb187..b5cf948a 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala @@ -7,9 +7,11 @@ package docspell.joex.process import java.time.ZoneId + import cats.effect.Sync import cats.implicits._ import cats.{Applicative, FlatMap} + import docspell.analysis.contact._ import docspell.common.MetaProposal.Candidate import docspell.common._ diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala index 3122da9f..8d59a969 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala @@ -10,6 +10,7 @@ import cats.data.OptionT import cats.effect._ import cats.implicits._ import fs2.Stream + import docspell.analysis.TextAnalyser import docspell.backend.ops.OItem import docspell.common.{ItemState, ProcessItemArgs} diff --git a/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala b/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala index 71598b64..a656c230 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala @@ -9,6 +9,7 @@ package docspell.joex.process import cats.data.NonEmptyList import cats.effect.Sync import cats.implicits._ + import docspell.common._ import docspell.scheduler.{Context, Task} import docspell.store.Store diff --git a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala index 819be5c7..6087b37f 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala @@ -8,6 +8,7 @@ package docspell.joex.process import cats.effect._ import cats.implicits._ + import docspell.analysis.TextAnalyser import docspell.backend.ops.OItem import docspell.common.ProcessItemArgs diff --git a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala index cc840710..1863d2ef 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala @@ -9,6 +9,7 @@ package docspell.joex.process import cats.data.OptionT import cats.effect._ import cats.implicits._ + import docspell.analysis.TextAnalyser import docspell.backend.ops.OItem import docspell.common._ diff --git a/modules/joex/src/main/scala/docspell/joex/process/SaveProposals.scala b/modules/joex/src/main/scala/docspell/joex/process/SaveProposals.scala index fdb0c7a7..fc17a54f 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/SaveProposals.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/SaveProposals.scala @@ -8,10 +8,11 @@ package docspell.joex.process import cats.effect.Sync import cats.implicits._ + import docspell.common._ import docspell.scheduler.{Context, Task} -import docspell.store.{AddResult, Store} import docspell.store.records._ +import docspell.store.{AddResult, Store} /** Saves the proposals in the database */ object SaveProposals { diff --git a/modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala b/modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala index 774040dd..3a8ab6aa 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala @@ -9,6 +9,7 @@ package docspell.joex.process import cats.Traverse import cats.effect._ import cats.implicits._ + import docspell.analysis.classifier.TextClassifier import docspell.analysis.{NlpSettings, TextAnalyser} import docspell.common.MetaProposal.Candidate diff --git a/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala b/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala index 757ee54f..ee6b8939 100644 --- a/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala +++ b/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala @@ -9,6 +9,7 @@ package docspell.joex.process import cats.data.OptionT import cats.effect._ import cats.implicits._ + import docspell.common._ import docspell.extract.{ExtractConfig, ExtractResult, Extraction} import docspell.ftsclient.{FtsClient, TextData} diff --git a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala index 5a91003b..039dee08 100644 --- a/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala @@ -12,15 +12,17 @@ import cats.data.OptionT import cats.effect._ import cats.implicits._ import fs2._ + import docspell.backend.ops.{OJoex, OUpload} import docspell.common._ import docspell.joex.Config -import docspell.scheduler.{Context, Task} import docspell.logging.Logger +import docspell.scheduler.{Context, Task} +import docspell.store.Store import docspell.store.queries.QOrganization import docspell.store.records._ + import _root_.io.circe.syntax._ -import docspell.store.Store import emil.SearchQuery.{All, ReceivedDate} import emil.javamail.syntax._ import emil.{MimeType => _, _} diff --git a/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala b/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala index 128bc76b..39dfbbbc 100644 --- a/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/updatecheck/UpdateCheckTask.scala @@ -9,11 +9,13 @@ package docspell.joex.updatecheck import cats.data.OptionT import cats.effect._ import cats.implicits._ + import docspell.common._ import docspell.scheduler.Task import docspell.scheduler.usertask.UserTask import docspell.store.Store import docspell.store.records.RUserEmail + import emil._ object UpdateCheckTask { diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala index 0ff2957b..6b0f87b8 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala @@ -9,6 +9,7 @@ package docspell.restserver import cats.effect._ import fs2.Stream import fs2.concurrent.Topic + import docspell.backend.BackendApp import docspell.backend.auth.{AuthToken, ShareToken} import docspell.ftsclient.FtsClient @@ -24,6 +25,7 @@ import docspell.restserver.webapp.{TemplateRoutes, Templates, WebjarRoutes} import docspell.restserver.ws.{OutputEvent, WebSocketRoutes} import docspell.scheduler.impl.JobStoreModuleBuilder import docspell.store.Store + import emil.javamail.JavaMailEmil import org.http4s.HttpRoutes import org.http4s.client.Client diff --git a/modules/restserver/src/main/scala/docspell/restserver/Subscriptions.scala b/modules/restserver/src/main/scala/docspell/restserver/Subscriptions.scala index e1cafa20..8ffa7045 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/Subscriptions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/Subscriptions.scala @@ -9,6 +9,7 @@ package docspell.restserver import cats.effect.Async import fs2.Stream import fs2.concurrent.Topic + import docspell.pubsub.api.PubSubT import docspell.restserver.ws.OutputEvent import docspell.scheduler.msg.{JobDone, JobSubmitted} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/NotifyDueItemsRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/NotifyDueItemsRoutes.scala index 04248885..dc180ba0 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/NotifyDueItemsRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/NotifyDueItemsRoutes.scala @@ -9,6 +9,7 @@ package docspell.restserver.routes import cats.data.OptionT import cats.effect._ import cats.implicits._ + import docspell.backend.BackendApp import docspell.backend.MailAddressCodec import docspell.backend.auth.AuthToken @@ -19,6 +20,7 @@ import docspell.restserver.Config import docspell.restserver.conv.{Conversions, NonEmptyListSupport} import docspell.restserver.http4s.ClientRequestInfo import docspell.scheduler.usertask._ + import org.http4s._ import org.http4s.circe.CirceEntityDecoder._ import org.http4s.circe.CirceEntityEncoder._ diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/PeriodicQueryRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/PeriodicQueryRoutes.scala index f0866e76..0a300130 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/PeriodicQueryRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/PeriodicQueryRoutes.scala @@ -9,6 +9,7 @@ package docspell.restserver.routes import cats.data.{NonEmptyList, OptionT} import cats.effect._ import cats.implicits._ + import docspell.backend.BackendApp import docspell.backend.MailAddressCodec import docspell.backend.auth.AuthToken @@ -20,6 +21,7 @@ import docspell.restserver.Config import docspell.restserver.conv.{Conversions, NonEmptyListSupport} import docspell.restserver.http4s.ClientRequestInfo import docspell.scheduler.usertask._ + import org.http4s._ import org.http4s.circe.CirceEntityDecoder._ import org.http4s.circe.CirceEntityEncoder._ diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala index 430baae4..df271180 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ScanMailboxRoutes.scala @@ -9,12 +9,14 @@ package docspell.restserver.routes import cats.data.OptionT import cats.effect._ import cats.implicits._ + import docspell.backend.BackendApp import docspell.backend.auth.AuthToken import docspell.common._ import docspell.restapi.model._ import docspell.restserver.conv.Conversions import docspell.scheduler.usertask._ + import org.http4s._ import org.http4s.circe.CirceEntityDecoder._ import org.http4s.circe.CirceEntityEncoder._ diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/Job.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/Job.scala index 9dc08b5c..a6f4fc82 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/Job.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/Job.scala @@ -1,8 +1,16 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + package docspell.scheduler import cats.effect.Sync import cats.syntax.functor._ + import docspell.common._ + import io.circe.Encoder final case class Job[A]( diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/JobStore.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobStore.scala index 8ca887f9..de9eee9d 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/JobStore.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobStore.scala @@ -1,3 +1,9 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + package docspell.scheduler trait JobStore[F[_]] { diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/JobStoreModule.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobStoreModule.scala index 2d170127..729bf645 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/JobStoreModule.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/JobStoreModule.scala @@ -1,3 +1,9 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + package docspell.scheduler import docspell.scheduler.usertask.UserTaskStore diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/SchedulerModule.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/SchedulerModule.scala index d422f200..a8e5f53e 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/SchedulerModule.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/SchedulerModule.scala @@ -1,3 +1,9 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + package docspell.scheduler trait SchedulerModule[F[_]] { diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/CancelJob.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/CancelJob.scala index 3ec0c891..1c9bae36 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/CancelJob.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/CancelJob.scala @@ -8,6 +8,7 @@ package docspell.scheduler.msg import docspell.common._ import docspell.pubsub.api.{Topic, TypedTopic} + import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} import io.circe.{Decoder, Encoder} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobSubmitted.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobSubmitted.scala index c6ef40b2..c798b9d7 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobSubmitted.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobSubmitted.scala @@ -8,6 +8,7 @@ package docspell.scheduler.msg import docspell.common._ import docspell.pubsub.api.{Topic, TypedTopic} + import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} import io.circe.{Decoder, Encoder} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobsNotify.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobsNotify.scala index 8bc9fc8d..1f4331d2 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobsNotify.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/JobsNotify.scala @@ -1,3 +1,9 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + package docspell.scheduler.msg import docspell.pubsub.api.{Topic, TypedTopic} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/PeriodicTaskNotify.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/PeriodicTaskNotify.scala index d8f5157e..2ff4e055 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/PeriodicTaskNotify.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/msg/PeriodicTaskNotify.scala @@ -1,3 +1,9 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + package docspell.scheduler.msg import docspell.pubsub.api.{Topic, TypedTopic} diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTask.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTask.scala index f700da5c..b799a20c 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTask.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTask.scala @@ -6,8 +6,9 @@ package docspell.scheduler.usertask -import com.github.eikek.calev.CalEvent import docspell.common._ + +import com.github.eikek.calev.CalEvent import io.circe.Encoder final case class UserTask[A]( diff --git a/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskStore.scala b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskStore.scala index 2a3dcf0e..8d8b594d 100644 --- a/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskStore.scala +++ b/modules/scheduler/api/src/main/scala/docspell/scheduler/usertask/UserTaskStore.scala @@ -7,8 +7,10 @@ package docspell.scheduler.usertask import cats.data.OptionT -import docspell.common._ import fs2.Stream + +import docspell.common._ + import io.circe._ /** User tasks are `RPeriodicTask`s that can be managed by the user. The user can change diff --git a/modules/scheduler/api/src/test/scala/docspell/scheduler/CountingSchemeSpec.scala b/modules/scheduler/api/src/test/scala/docspell/scheduler/CountingSchemeSpec.scala index 1339d220..e396127e 100644 --- a/modules/scheduler/api/src/test/scala/docspell/scheduler/CountingSchemeSpec.scala +++ b/modules/scheduler/api/src/test/scala/docspell/scheduler/CountingSchemeSpec.scala @@ -7,6 +7,7 @@ package docspell.scheduler import docspell.common.Priority + import munit._ class CountingSchemeSpec extends FunSuite { diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala index 69d2b34b..d801a169 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/ContextImpl.scala @@ -1,8 +1,14 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + package docspell.scheduler.impl import cats._ -import cats.syntax.all._ import cats.effect._ +import cats.syntax.all._ import docspell.common._ import docspell.logging.Logger diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobQueue.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobQueue.scala index 0c190723..5317b7dc 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobQueue.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobQueue.scala @@ -8,6 +8,7 @@ package docspell.scheduler.impl import cats.effect._ import cats.implicits._ + import docspell.common._ import docspell.store.Store import docspell.store.queries.QJob diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobStoreImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobStoreImpl.scala index f2507301..38b0b067 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobStoreImpl.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobStoreImpl.scala @@ -1,3 +1,9 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + package docspell.scheduler.impl import cats.effect.Sync @@ -5,7 +11,6 @@ import cats.syntax.all._ import docspell.common.Timestamp import docspell.scheduler._ - import docspell.store.Store import docspell.store.records.RJob diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobStoreModuleBuilder.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobStoreModuleBuilder.scala index e5762935..0cab5e30 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobStoreModuleBuilder.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/JobStoreModuleBuilder.scala @@ -1,3 +1,9 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + package docspell.scheduler.impl import cats.effect.Async diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/LogSink.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/LogSink.scala index 8094b90e..c59b45ce 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/LogSink.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/LogSink.scala @@ -8,12 +8,13 @@ package docspell.scheduler.impl import cats.effect._ import cats.implicits._ +import fs2.Pipe + import docspell.common._ import docspell.logging import docspell.scheduler.LogEvent import docspell.store.Store import docspell.store.records.RJobLog -import fs2.Pipe trait LogSink[F[_]] { diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerBuilder.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerBuilder.scala index a3566f13..0933105e 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerBuilder.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerBuilder.scala @@ -1,9 +1,16 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + package docspell.scheduler.impl import cats.effect._ +import fs2.concurrent.SignallingRef + import docspell.pubsub.api.PubSubT import docspell.scheduler._ -import fs2.concurrent.SignallingRef object PeriodicSchedulerBuilder { diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerImpl.scala index 43d89324..2e7ac515 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerImpl.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicSchedulerImpl.scala @@ -10,12 +10,14 @@ import cats.effect._ import cats.implicits._ import fs2._ import fs2.concurrent.SignallingRef + import docspell.common._ import docspell.pubsub.api.PubSubT import docspell.scheduler._ import docspell.scheduler.impl.PeriodicSchedulerImpl.State import docspell.scheduler.msg.{JobsNotify, PeriodicTaskNotify} import docspell.store.records.RPeriodicTask + import eu.timepit.fs2cron.calev.CalevScheduler final class PeriodicSchedulerImpl[F[_]: Async]( diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicTaskStore.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicTaskStore.scala index 1f1ecb52..bd922cb2 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicTaskStore.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/PeriodicTaskStore.scala @@ -8,6 +8,7 @@ package docspell.scheduler.impl import cats.effect._ import cats.implicits._ + import docspell.common._ import docspell.scheduler.{Job, JobStore} import docspell.store.queries.QPeriodicTask diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QUserTask.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QUserTask.scala index a1ea1632..058ed421 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QUserTask.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QUserTask.scala @@ -1,16 +1,24 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + package docspell.scheduler.impl import cats.effect.Sync import cats.syntax.all._ -import com.github.eikek.calev.CalEvent -import docspell.common.{AccountId, Ident, Priority, Timestamp} +import fs2.Stream + +import docspell.common._ import docspell.scheduler.usertask.{UserTask, UserTaskScope} import docspell.store.qb.DML import docspell.store.qb.DSL._ import docspell.store.records.RPeriodicTask -import fs2.Stream -import io.circe.{Decoder, Encoder} + +import com.github.eikek.calev.CalEvent import doobie._ +import io.circe.{Decoder, Encoder} object QUserTask { private val RT = RPeriodicTask.T diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QueueLogger.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QueueLogger.scala index 08a176ec..8a4a0824 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QueueLogger.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/QueueLogger.scala @@ -1,13 +1,20 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + package docspell.scheduler.impl -import cats.syntax.all._ import cats.effect.std.Queue import cats.effect.{Async, Sync} +import cats.syntax.all._ +import fs2.Stream + import docspell.common.{Ident, LogLevel} import docspell.logging import docspell.logging.{Level, Logger} import docspell.scheduler.LogEvent -import fs2.Stream object QueueLogger { diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerBuilder.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerBuilder.scala index e2cf3286..6a6e0c2a 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerBuilder.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerBuilder.scala @@ -10,9 +10,10 @@ import cats.effect._ import cats.effect.std.Semaphore import cats.implicits._ import fs2.concurrent.SignallingRef -import docspell.scheduler._ + import docspell.notification.api.EventSink import docspell.pubsub.api.PubSubT +import docspell.scheduler._ import docspell.store.Store case class SchedulerBuilder[F[_]: Async]( diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala index 0f063599..955a3b59 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerImpl.scala @@ -12,16 +12,18 @@ import cats.effect.std.Semaphore import cats.implicits._ import fs2.Stream import fs2.concurrent.SignallingRef -import docspell.scheduler.msg.{CancelJob, JobDone, JobsNotify} + import docspell.common._ -import docspell.scheduler._ -import docspell.scheduler.impl.SchedulerImpl._ import docspell.notification.api.Event import docspell.notification.api.EventSink import docspell.pubsub.api.PubSubT +import docspell.scheduler._ +import docspell.scheduler.impl.SchedulerImpl._ +import docspell.scheduler.msg.{CancelJob, JobDone, JobsNotify} import docspell.store.Store import docspell.store.queries.QJob import docspell.store.records.RJob + import io.circe.Json final class SchedulerImpl[F[_]: Async]( diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerModuleBuilder.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerModuleBuilder.scala index 09e6c6a1..e8d68d77 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerModuleBuilder.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/SchedulerModuleBuilder.scala @@ -1,13 +1,15 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + package docspell.scheduler.impl import cats.effect._ + import docspell.common.Ident -import docspell.scheduler.{ - JobTaskRegistry, - PeriodicSchedulerConfig, - SchedulerConfig, - SchedulerModule -} +import docspell.scheduler._ case class SchedulerModuleBuilder[F[_]: Async] private ( periodicSchedulerConfig: PeriodicSchedulerConfig, diff --git a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/UserTaskStoreImpl.scala b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/UserTaskStoreImpl.scala index 2f90617e..21d14b75 100644 --- a/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/UserTaskStoreImpl.scala +++ b/modules/scheduler/impl/src/main/scala/docspell/scheduler/impl/UserTaskStoreImpl.scala @@ -1,15 +1,23 @@ +/* + * Copyright 2020 Eike K. & Contributors + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + package docspell.scheduler.impl -import cats.effect._ -import docspell.scheduler.usertask.UserTaskStore import cats.data.OptionT +import cats.effect._ import cats.implicits._ +import fs2.Stream + import docspell.common._ +import docspell.scheduler.impl.QUserTask.UserTaskCodec +import docspell.scheduler.usertask.UserTaskStore import docspell.scheduler.usertask._ import docspell.store.{AddResult, Store} -import fs2.Stream + import io.circe._ -import QUserTask.UserTaskCodec final class UserTaskStoreImpl[F[_]: Sync]( store: Store[F], diff --git a/modules/store/src/main/scala/docspell/store/records/RPeriodicTask.scala b/modules/store/src/main/scala/docspell/store/records/RPeriodicTask.scala index 2f298dff..5be5ca08 100644 --- a/modules/store/src/main/scala/docspell/store/records/RPeriodicTask.scala +++ b/modules/store/src/main/scala/docspell/store/records/RPeriodicTask.scala @@ -9,9 +9,11 @@ package docspell.store.records import cats.data.NonEmptyList import cats.effect._ import cats.implicits._ + import docspell.common._ import docspell.store.qb.DSL._ import docspell.store.qb._ + import com.github.eikek.calev.CalEvent import doobie._ import doobie.implicits._
nlp.modeEnglishGermanFrench
nlp.modeEnglishGermanFrench