mirror of
https://github.com/TheAnachronism/docspell.git
synced 2025-06-02 13:32:51 +00:00
Add a folder-id to item processing
This allows to define a folder when uploading files. All generated items are associated to this folder on creation.
This commit is contained in:
parent
ec7f027b4e
commit
5b01c93711
@ -58,6 +58,7 @@ object OUpload {
|
||||
case class UploadMeta(
|
||||
direction: Option[Direction],
|
||||
sourceAbbrev: String,
|
||||
folderId: Option[Ident],
|
||||
validFileTypes: Seq[MimeType]
|
||||
)
|
||||
|
||||
@ -123,6 +124,7 @@ object OUpload {
|
||||
lang.getOrElse(Language.German),
|
||||
data.meta.direction,
|
||||
data.meta.sourceAbbrev,
|
||||
data.meta.folderId,
|
||||
data.meta.validFileTypes
|
||||
)
|
||||
args =
|
||||
@ -147,7 +149,10 @@ object OUpload {
|
||||
(for {
|
||||
src <- OptionT(store.transact(RSource.find(sourceId)))
|
||||
updata = data.copy(
|
||||
meta = data.meta.copy(sourceAbbrev = src.abbrev),
|
||||
meta = data.meta.copy(
|
||||
sourceAbbrev = src.abbrev,
|
||||
folderId = data.meta.folderId.orElse(src.folderId)
|
||||
),
|
||||
priority = src.priority
|
||||
)
|
||||
accId = AccountId(src.cid, src.sid)
|
||||
|
@ -36,6 +36,7 @@ object ProcessItemArgs {
|
||||
language: Language,
|
||||
direction: Option[Direction],
|
||||
sourceAbbrev: String,
|
||||
folderId: Option[Ident],
|
||||
validFileTypes: Seq[MimeType]
|
||||
)
|
||||
|
||||
|
@ -27,7 +27,9 @@ case class ScanMailboxArgs(
|
||||
// delete the after submitting (only if targetFolder is None)
|
||||
deleteMail: Boolean,
|
||||
// set the direction when submitting
|
||||
direction: Option[Direction]
|
||||
direction: Option[Direction],
|
||||
// set a folder for items
|
||||
itemFolder: Option[Ident]
|
||||
)
|
||||
|
||||
object ScanMailboxArgs {
|
||||
|
@ -84,6 +84,7 @@ object JoexAppImpl {
|
||||
joex <- OJoex(client, store)
|
||||
upload <- OUpload(store, queue, cfg.files, joex)
|
||||
fts <- createFtsClient(cfg)(httpClient)
|
||||
itemOps <- OItem(store, fts)
|
||||
javaEmil =
|
||||
JavaMailEmil(blocker, Settings.defaultSettings.copy(debug = cfg.mailDebug))
|
||||
sch <- SchedulerBuilder(cfg.scheduler, blocker, store)
|
||||
@ -91,7 +92,7 @@ object JoexAppImpl {
|
||||
.withTask(
|
||||
JobTask.json(
|
||||
ProcessItemArgs.taskName,
|
||||
ItemHandler.newItem[F](cfg, fts),
|
||||
ItemHandler.newItem[F](cfg, itemOps, fts),
|
||||
ItemHandler.onCancel[F]
|
||||
)
|
||||
)
|
||||
|
@ -5,6 +5,7 @@ import cats.effect._
|
||||
import cats.implicits._
|
||||
import fs2.Stream
|
||||
|
||||
import docspell.backend.ops.OItem
|
||||
import docspell.common.{ItemState, ProcessItemArgs}
|
||||
import docspell.ftsclient.FtsClient
|
||||
import docspell.joex.Config
|
||||
@ -27,11 +28,12 @@ object ItemHandler {
|
||||
|
||||
def newItem[F[_]: ConcurrentEffect: ContextShift](
|
||||
cfg: Config,
|
||||
itemOps: OItem[F],
|
||||
fts: FtsClient[F]
|
||||
): Task[F, Args, Unit] =
|
||||
CreateItem[F]
|
||||
.flatMap(itemStateTask(ItemState.Processing))
|
||||
.flatMap(safeProcess[F](cfg, fts))
|
||||
.flatMap(safeProcess[F](cfg, itemOps, fts))
|
||||
.map(_ => ())
|
||||
|
||||
def itemStateTask[F[_]: Sync, A](
|
||||
@ -48,11 +50,12 @@ object ItemHandler {
|
||||
|
||||
def safeProcess[F[_]: ConcurrentEffect: ContextShift](
|
||||
cfg: Config,
|
||||
itemOps: OItem[F],
|
||||
fts: FtsClient[F]
|
||||
)(data: ItemData): Task[F, Args, ItemData] =
|
||||
isLastRetry[F].flatMap {
|
||||
case true =>
|
||||
ProcessItem[F](cfg, fts)(data).attempt.flatMap({
|
||||
ProcessItem[F](cfg, itemOps, fts)(data).attempt.flatMap({
|
||||
case Right(d) =>
|
||||
Task.pure(d)
|
||||
case Left(ex) =>
|
||||
@ -62,7 +65,7 @@ object ItemHandler {
|
||||
.andThen(_ => Sync[F].raiseError(ex))
|
||||
})
|
||||
case false =>
|
||||
ProcessItem[F](cfg, fts)(data).flatMap(itemStateTask(ItemState.Created))
|
||||
ProcessItem[F](cfg, itemOps, fts)(data).flatMap(itemStateTask(ItemState.Created))
|
||||
}
|
||||
|
||||
private def markItemCreated[F[_]: Sync]: Task[F, Args, Boolean] =
|
||||
|
@ -2,6 +2,7 @@ package docspell.joex.process
|
||||
|
||||
import cats.effect._
|
||||
|
||||
import docspell.backend.ops.OItem
|
||||
import docspell.common.ProcessItemArgs
|
||||
import docspell.ftsclient.FtsClient
|
||||
import docspell.joex.Config
|
||||
@ -11,6 +12,7 @@ object ProcessItem {
|
||||
|
||||
def apply[F[_]: ConcurrentEffect: ContextShift](
|
||||
cfg: Config,
|
||||
itemOps: OItem[F],
|
||||
fts: FtsClient[F]
|
||||
)(item: ItemData): Task[F, ProcessItemArgs, ItemData] =
|
||||
ExtractArchive(item)
|
||||
@ -22,6 +24,7 @@ object ProcessItem {
|
||||
.flatMap(analysisOnly[F](cfg))
|
||||
.flatMap(Task.setProgress(80))
|
||||
.flatMap(LinkProposal[F])
|
||||
.flatMap(SetGivenData[F](itemOps))
|
||||
.flatMap(Task.setProgress(99))
|
||||
|
||||
def analysisOnly[F[_]: Sync](
|
||||
|
@ -0,0 +1,35 @@
|
||||
package docspell.joex.process
|
||||
|
||||
import cats.effect._
|
||||
import cats.implicits._
|
||||
|
||||
import docspell.backend.ops.OItem
|
||||
import docspell.common._
|
||||
import docspell.joex.scheduler.Task
|
||||
|
||||
object SetGivenData {
|
||||
|
||||
def apply[F[_]: Sync](
|
||||
ops: OItem[F]
|
||||
)(data: ItemData): Task[F, ProcessItemArgs, ItemData] =
|
||||
if (data.item.state.isValid)
|
||||
Task
|
||||
.log[F, ProcessItemArgs](_.debug(s"Not setting data on existing item"))
|
||||
.map(_ => data)
|
||||
else
|
||||
Task { ctx =>
|
||||
val itemId = data.item.id
|
||||
val folderId = ctx.args.meta.folderId
|
||||
val collective = ctx.args.meta.collective
|
||||
for {
|
||||
_ <- ctx.logger.info("Starting setting given data")
|
||||
_ <- ctx.logger.debug(s"Set item folder: '${folderId.map(_.id)}'")
|
||||
e <- ops.setFolder(itemId, folderId, collective).attempt
|
||||
_ <- e.fold(
|
||||
ex => ctx.logger.warn(s"Error setting folder: ${ex.getMessage}"),
|
||||
_ => ().pure[F]
|
||||
)
|
||||
} yield data
|
||||
}
|
||||
|
||||
}
|
@ -143,7 +143,7 @@ object ScanMailboxTask {
|
||||
folder <- requireFolder(a)(name)
|
||||
search <- searchMails(a)(folder)
|
||||
headers <- Kleisli.liftF(filterMessageIds(search.mails))
|
||||
_ <- headers.traverse(handleOne(a, upload))
|
||||
_ <- headers.traverse(handleOne(ctx.args, a, upload))
|
||||
} yield ScanResult(name, search.mails.size, search.count - search.mails.size)
|
||||
|
||||
def requireFolder[C](a: Access[F, C])(name: String): MailOp[F, C, MailFolder] =
|
||||
@ -239,7 +239,9 @@ object ScanMailboxTask {
|
||||
MailOp.pure(())
|
||||
}
|
||||
|
||||
def submitMail(upload: OUpload[F])(mail: Mail[F]): F[OUpload.UploadResult] = {
|
||||
def submitMail(upload: OUpload[F], args: Args)(
|
||||
mail: Mail[F]
|
||||
): F[OUpload.UploadResult] = {
|
||||
val file = OUpload.File(
|
||||
Some(mail.header.subject + ".eml"),
|
||||
Some(MimeType.emls.head),
|
||||
@ -251,6 +253,7 @@ object ScanMailboxTask {
|
||||
meta = OUpload.UploadMeta(
|
||||
Some(dir),
|
||||
s"mailbox-${ctx.args.account.user.id}",
|
||||
args.itemFolder,
|
||||
Seq.empty
|
||||
)
|
||||
data = OUpload.UploadData(
|
||||
@ -264,14 +267,14 @@ object ScanMailboxTask {
|
||||
} yield res
|
||||
}
|
||||
|
||||
def handleOne[C](a: Access[F, C], upload: OUpload[F])(
|
||||
def handleOne[C](args: Args, a: Access[F, C], upload: OUpload[F])(
|
||||
mh: MailHeader
|
||||
): MailOp[F, C, Unit] =
|
||||
for {
|
||||
mail <- a.loadMail(mh)
|
||||
res <- mail match {
|
||||
case Some(m) =>
|
||||
Kleisli.liftF(submitMail(upload)(m).attempt)
|
||||
Kleisli.liftF(submitMail(upload, args)(m).attempt)
|
||||
case None =>
|
||||
MailOp.pure[F, C, Either[Throwable, OUpload.UploadResult]](
|
||||
Either.left(new Exception(s"Mail not found"))
|
||||
|
@ -144,6 +144,7 @@ structure:
|
||||
```
|
||||
{ multiple: Bool
|
||||
, direction: Maybe String
|
||||
, folder: Maybe String
|
||||
}
|
||||
```
|
||||
|
||||
@ -156,6 +157,11 @@ Furthermore, the direction of the document (one of `incoming` or
|
||||
`outgoing`) can be given. It is optional, it can be left out or
|
||||
`null`.
|
||||
|
||||
A `folder` id can be specified. Each item created by this request will
|
||||
be placed into this folder. Errors are logged (for example, the folder
|
||||
may have been deleted before the task is executed) and the item is
|
||||
then not put into any folder.
|
||||
|
||||
This kind of request is very common and most programming languages
|
||||
have support for this. For example, here is another curl command
|
||||
uploading two files with meta data:
|
||||
|
@ -2694,6 +2694,13 @@ components:
|
||||
The direction to apply to items resulting from importing
|
||||
mails. If not set, the value is guessed based on the from
|
||||
and to mail headers and your address book.
|
||||
itemFolder:
|
||||
type: string
|
||||
format: ident
|
||||
description: |
|
||||
The folder id that is applied to items resulting from
|
||||
importing mails. If the folder id is not valid when the
|
||||
task executes, items have no folder set.
|
||||
ImapSettingsList:
|
||||
description: |
|
||||
A list of user email settings.
|
||||
@ -3437,9 +3444,15 @@ components:
|
||||
Meta information for an item upload. The user can specify some
|
||||
structured information with a binary file.
|
||||
|
||||
Additional metadata is not required. However, you have to
|
||||
specifiy whether the corresponding files should become one
|
||||
single item or if an item is created for each file.
|
||||
Additional metadata is not required. However, if there is some
|
||||
specified, you have to specifiy whether the corresponding
|
||||
files should become one single item or if an item is created
|
||||
for each file.
|
||||
|
||||
A direction can be given, `Incoming` is used if not specified.
|
||||
|
||||
A folderId can be given, the item is placed into this folder
|
||||
after creation.
|
||||
required:
|
||||
- multiple
|
||||
properties:
|
||||
@ -3449,6 +3462,9 @@ components:
|
||||
direction:
|
||||
type: string
|
||||
format: direction
|
||||
folder:
|
||||
type: string
|
||||
format: ident
|
||||
Collective:
|
||||
description: |
|
||||
Information about a collective.
|
||||
@ -3519,6 +3535,9 @@ components:
|
||||
priority:
|
||||
type: string
|
||||
format: priority
|
||||
folder:
|
||||
type: string
|
||||
format: ident
|
||||
created:
|
||||
description: DateTime
|
||||
type: integer
|
||||
|
@ -287,9 +287,11 @@ trait Conversions {
|
||||
.find(_.name.exists(_.equalsIgnoreCase("meta")))
|
||||
.map(p => parseMeta(p.body))
|
||||
.map(fm =>
|
||||
fm.map(m => (m.multiple, UploadMeta(m.direction, "webapp", validFileTypes)))
|
||||
fm.map(m =>
|
||||
(m.multiple, UploadMeta(m.direction, "webapp", m.folder, validFileTypes))
|
||||
)
|
||||
)
|
||||
.getOrElse((true, UploadMeta(None, "webapp", validFileTypes)).pure[F])
|
||||
.getOrElse((true, UploadMeta(None, "webapp", None, validFileTypes)).pure[F])
|
||||
|
||||
val files = mp.parts
|
||||
.filter(p => p.name.forall(s => !s.equalsIgnoreCase("meta")))
|
||||
@ -491,12 +493,21 @@ trait Conversions {
|
||||
// sources
|
||||
|
||||
def mkSource(s: RSource): Source =
|
||||
Source(s.sid, s.abbrev, s.description, s.counter, s.enabled, s.priority, s.created)
|
||||
Source(
|
||||
s.sid,
|
||||
s.abbrev,
|
||||
s.description,
|
||||
s.counter,
|
||||
s.enabled,
|
||||
s.priority,
|
||||
s.folderId,
|
||||
s.created
|
||||
)
|
||||
|
||||
def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] =
|
||||
timeId.map({
|
||||
case (id, now) =>
|
||||
RSource(id, cid, s.abbrev, s.description, 0, s.enabled, s.priority, now)
|
||||
RSource(id, cid, s.abbrev, s.description, 0, s.enabled, s.priority, now, s.folder)
|
||||
})
|
||||
|
||||
def changeSource[F[_]: Sync](s: Source, coll: Ident): RSource =
|
||||
@ -508,7 +519,8 @@ trait Conversions {
|
||||
s.counter,
|
||||
s.enabled,
|
||||
s.priority,
|
||||
s.created
|
||||
s.created,
|
||||
s.folder
|
||||
)
|
||||
|
||||
// equipment
|
||||
|
@ -112,7 +112,8 @@ object ScanMailboxRoutes {
|
||||
settings.receivedSinceHours.map(_.toLong).map(Duration.hours),
|
||||
settings.targetFolder,
|
||||
settings.deleteMail,
|
||||
settings.direction
|
||||
settings.direction,
|
||||
settings.itemFolder
|
||||
)
|
||||
)
|
||||
)
|
||||
@ -139,6 +140,7 @@ object ScanMailboxRoutes {
|
||||
task.args.receivedSince.map(_.hours.toInt),
|
||||
task.args.targetFolder,
|
||||
task.args.deleteMail,
|
||||
task.args.direction
|
||||
task.args.direction,
|
||||
task.args.itemFolder
|
||||
)
|
||||
}
|
||||
|
@ -50,6 +50,7 @@ object QFolder {
|
||||
def tryDelete =
|
||||
for {
|
||||
_ <- RItem.removeFolder(id)
|
||||
_ <- RSource.removeFolder(id)
|
||||
_ <- RFolderMember.deleteAll(id)
|
||||
_ <- RFolder.delete(id)
|
||||
} yield FolderChangeResult.success
|
||||
|
@ -15,7 +15,8 @@ case class RSource(
|
||||
counter: Int,
|
||||
enabled: Boolean,
|
||||
priority: Priority,
|
||||
created: Timestamp
|
||||
created: Timestamp,
|
||||
folderId: Option[Ident]
|
||||
) {}
|
||||
|
||||
object RSource {
|
||||
@ -32,8 +33,10 @@ object RSource {
|
||||
val enabled = Column("enabled")
|
||||
val priority = Column("priority")
|
||||
val created = Column("created")
|
||||
val folder = Column("folder_id")
|
||||
|
||||
val all = List(sid, cid, abbrev, description, counter, enabled, priority, created)
|
||||
val all =
|
||||
List(sid, cid, abbrev, description, counter, enabled, priority, created, folder)
|
||||
}
|
||||
|
||||
import Columns._
|
||||
@ -42,7 +45,7 @@ object RSource {
|
||||
val sql = insertRow(
|
||||
table,
|
||||
all,
|
||||
fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created}"
|
||||
fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created},${v.folderId}"
|
||||
)
|
||||
sql.update.run
|
||||
}
|
||||
@ -56,7 +59,8 @@ object RSource {
|
||||
abbrev.setTo(v.abbrev),
|
||||
description.setTo(v.description),
|
||||
enabled.setTo(v.enabled),
|
||||
priority.setTo(v.priority)
|
||||
priority.setTo(v.priority),
|
||||
folder.setTo(v.folderId)
|
||||
)
|
||||
)
|
||||
sql.update.run
|
||||
@ -97,4 +101,9 @@ object RSource {
|
||||
|
||||
def delete(sourceId: Ident, coll: Ident): ConnectionIO[Int] =
|
||||
deleteFrom(table, and(sid.is(sourceId), cid.is(coll))).update.run
|
||||
|
||||
def removeFolder(folderId: Ident): ConnectionIO[Int] = {
|
||||
val empty: Option[Ident] = None
|
||||
updateRow(table, folder.is(folderId), folder.setTo(empty)).update.run
|
||||
}
|
||||
}
|
||||
|
Loading…
x
Reference in New Issue
Block a user