Add attachments-only filter to uploads

When uploading a file which is an e-mail, this option allows to skip
the mail body when the file is being processed.
This commit is contained in:
eikek
2021-08-21 13:37:17 +02:00
parent bb8a6c054b
commit 751fa3da5a
11 changed files with 84 additions and 31 deletions

View File

@ -23,9 +23,12 @@ object ReadMail {
def readBytesP[F[_]: Async](
logger: Logger[F],
glob: Glob
glob: Glob,
attachmentsOnly: Boolean
): Pipe[F, Byte, Binary[F]] =
_.through(bytesToMail(logger)).flatMap(mailToEntries[F](logger, glob))
_.through(bytesToMail(logger)).flatMap(
mailToEntries[F](logger, glob, attachmentsOnly)
)
def bytesToMail[F[_]: Sync](logger: Logger[F]): Pipe[F, Byte, Mail[F]] =
s =>
@ -34,10 +37,30 @@ object ReadMail {
def mailToEntries[F[_]: Async](
logger: Logger[F],
glob: Glob
glob: Glob,
attachmentsOnly: Boolean
)(mail: Mail[F]): Stream[F, Binary[F]] =
Stream.eval(
logger.debug(
s"E-mail has ${mail.attachments.size} attachments and ${bodyType(mail.body)}"
)
) >>
(makeBodyEntry(logger, glob, attachmentsOnly)(mail) ++
Stream
.eval(TnefExtract.replace(mail))
.flatMap(m => Stream.emits(m.attachments.all))
.filter(a => a.filename.exists(glob.matches(caseSensitive = false)))
.map(a =>
Binary(a.filename.getOrElse("noname"), a.mimeType.toLocal, a.content)
))
private def makeBodyEntry[F[_]: Async](
logger: Logger[F],
glob: Glob,
attachmentsOnly: Boolean
)(mail: Mail[F]): Stream[F, Binary[F]] = {
val bodyEntry: F[Option[Binary[F]]] =
if (mail.body.isEmpty) (None: Option[Binary[F]]).pure[F]
if (mail.body.isEmpty || attachmentsOnly) (None: Option[Binary[F]]).pure[F]
else {
val markdownCfg = MarkdownConfig.defaultConfig
HtmlBodyView(
@ -49,22 +72,14 @@ object ReadMail {
).map(makeHtmlBinary[F] _).map(b => Some(b))
}
Stream.eval(
logger.debug(
s"E-mail has ${mail.attachments.size} attachments and ${bodyType(mail.body)}"
)
) >>
(Stream
.eval(bodyEntry)
.flatMap(e => Stream.emits(e.toSeq))
.filter(a => glob.matches(caseSensitive = false)(a.name)) ++
for {
_ <- Stream.eval(logger.debug(s"Import attachments only: $attachmentsOnly"))
bin <-
Stream
.eval(TnefExtract.replace(mail))
.flatMap(m => Stream.emits(m.attachments.all))
.filter(a => a.filename.exists(glob.matches(caseSensitive = false)))
.map(a =>
Binary(a.filename.getOrElse("noname"), a.mimeType.toLocal, a.content)
))
.eval(bodyEntry)
.flatMap(e => Stream.emits(e.toSeq))
.filter(a => glob.matches(caseSensitive = false)(a.name))
} yield bin
}
private def makeHtmlBinary[F[_]](cnt: BodyContent): Binary[F] =

View File

@ -161,7 +161,8 @@ object ExtractArchive {
.unNoneTerminate
.through(ctx.store.bitpeace.fetchData2(RangeDef.all))
val glob = ctx.args.meta.fileFilter.getOrElse(Glob.all)
val glob = ctx.args.meta.fileFilter.getOrElse(Glob.all)
val attachOnly = ctx.args.meta.attachmentsOnly.getOrElse(false)
ctx.logger.debug(s"Filtering email attachments with '${glob.asString}'") *>
email
.through(ReadMail.bytesToMail[F](ctx.logger))
@ -174,7 +175,7 @@ object ExtractArchive {
} yield s
ReadMail
.mailToEntries(ctx.logger, glob)(mail)
.mailToEntries(ctx.logger, glob, attachOnly)(mail)
.zipWithIndex
.flatMap(handleEntry(ctx, ra, pos, archive, mId)) ++ Stream.eval(givenMeta)
}

View File

@ -114,7 +114,8 @@ object ReProcessItem {
false,
None,
None,
true
true,
None // attachOnly (not used when reprocessing attachments)
),
Nil
).pure[F]

View File

@ -300,7 +300,8 @@ object ScanMailboxTask {
true,
args.fileFilter.getOrElse(Glob.all),
args.tags.getOrElse(Nil),
args.language
args.language,
args.attachmentsOnly
)
data = OUpload.UploadData(
multiple = false,