diff --git a/modules/backend/src/main/scala/docspell/backend/item/Merge.scala b/modules/backend/src/main/scala/docspell/backend/item/Merge.scala new file mode 100644 index 00000000..06885313 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/item/Merge.scala @@ -0,0 +1,202 @@ +/* + * Copyright 2020 Docspell Contributors + * + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +package docspell.backend.item + +import cats.data.EitherT +import cats.data.NonEmptyList +import cats.effect._ +import cats.implicits._ + +import docspell.backend.fulltext.CreateIndex +import docspell.backend.ops.OItem +import docspell.common._ +import docspell.store.Store +import docspell.store.queries.QCustomField +import docspell.store.queries.QCustomField.FieldValue +import docspell.store.records.RAttachment +import docspell.store.records.RCustomField +import docspell.store.records.RItem +import docspell.store.records.RTagItem + +trait Merge[F[_]] { + def merge(items: NonEmptyList[Ident], collective: Ident): F[Merge.Result[RItem]] +} + +object Merge { + + type Result[A] = Either[Error, A] + sealed trait Error + object Error { + final case object NoItems extends Error + def noItems: Error = NoItems + + } + + def apply[F[_]: Async]( + logger: Logger[F], + store: Store[F], + itemOps: OItem[F], + createIndex: CreateIndex[F] + ): Merge[F] = + new Merge[F] { + def merge(givenIds: NonEmptyList[Ident], collective: Ident): F[Result[RItem]] = + (for { + items <- loadItems(givenIds, collective) + ids = items.map(_.id) + target = moveMainData(items) + _ <- EitherT.right[Error](store.transact(RItem.updateAll(target))) + _ <- EitherT.right[Error](moveTags(ids)) + _ <- EitherT.right[Error](moveCustomFields(ids)) + _ <- EitherT.right[Error](moveAttachments(ids)) + _ <- EitherT.right[Error]( + createIndex + .reIndexData(logger, collective.some, NonEmptyList.one(ids.head).some, 50) + ) + _ <- EitherT.right[Error]( + NonEmptyList.fromList(items.tail.map(_.id)) match { + case Some(nel) => itemOps.setDeletedState(nel, collective) + case None => 0.pure[F] + } + ) + } yield target).value + + def loadItems( + items: NonEmptyList[Ident], + collective: Ident + ): EitherT[F, Error, NonEmptyList[RItem]] = { + val loaded = + store + .transact( + items.toList.traverse(id => RItem.findByIdAndCollective(id, collective)) + ) + .map(_.flatten) + .map(NonEmptyList.fromList) + EitherT.fromOptionF(loaded, Error.NoItems) + } + + def moveAttachments(items: NonEmptyList[Ident]): F[Int] = { + val target = items.head + for { + attachs <- store.transact(items.tail.traverse(id => RAttachment.findByItem(id))) + attachFlat = attachs.flatMap(_.toList) + n <- attachFlat.traverse(a => + store.transact(RAttachment.updateItemId(a.id, target)) + ) + } yield n.sum + } + + def moveTags(items: NonEmptyList[Ident]): F[Int] = { + val target = items.head + items.tail + .traverse(id => store.transact(RTagItem.moveTags(id, target))) + .map(_.sum) + } + + def moveMainData(items: NonEmptyList[RItem]): RItem = + items.tail.foldLeft(items.head)(combine) + + def moveCustomFields(items: NonEmptyList[Ident]): F[Unit] = + for { + values <- store.transact(QCustomField.findAllValues(items)) + byField = values.groupBy(_.field.name) + newValues = mergeFields(items.head, byField) + _ <- newValues.traverse(fv => + store.transact(RCustomField.setValue(fv.field, items.head, fv.value)) + ) + } yield () + } + + private def mergeFields( + targetId: Ident, + byField: Map[Ident, List[FieldValue]] + ): List[FieldValue] = + byField + .filter(kv => kv._1 != targetId || kv._2.size > 1) + .values + .flatMap(NonEmptyList.fromList) + .map { nel => + if (nel.tail.isEmpty) nel.head + else mergeFieldSameName(nel) + } + .toList + + private def mergeFieldSameName(fields: NonEmptyList[FieldValue]): FieldValue = + fields.head.field.ftype match { + case CustomFieldType.Bool => fields.head + case CustomFieldType.Date => fields.head + case CustomFieldType.Money => + val amount = + fields.toList + .flatMap(fv => CustomFieldType.Money.parseValue(fv.value).toOption) + .toList + .sum + fields.head.copy(value = CustomFieldType.Money.valueString(amount)) + + case CustomFieldType.Numeric => + val amount = + fields.toList + .flatMap(fv => CustomFieldType.Numeric.parseValue(fv.value).toOption) + .toList + .sum + fields.head.copy(value = CustomFieldType.Numeric.valueString(amount)) + + case CustomFieldType.Text => + val text = fields.toList + .map(fv => CustomFieldType.Text.parseValue(fv.value).toOption) + .mkString(", ") + fields.head.copy(value = CustomFieldType.Text.valueString(text)) + } + + private def combine(target: RItem, source: RItem): RItem = + MoveProp + .all( + MoveProp.whenNotExists(_.itemDate)((i, v) => i.copy(itemDate = v)), + MoveProp.whenNotExists(_.corrPerson)((i, v) => i.copy(corrPerson = v)), + MoveProp.whenNotExists(_.concPerson)((i, v) => i.copy(concPerson = v)), + MoveProp.whenNotExists(_.concEquipment)((i, v) => i.copy(concEquipment = v)), + MoveProp.whenNotExists(_.dueDate)((i, v) => i.copy(dueDate = v)), + MoveProp.whenNotExists(_.folderId)((i, v) => i.copy(folderId = v)), + MoveProp.concat(_.notes)((i, v) => i.copy(notes = v)) + ) + .move(target, source) + + trait MoveProp { + def move(target: RItem, source: RItem): RItem + } + object MoveProp { + def whenNotExists[A]( + get: RItem => Option[A] + )(set: (RItem, Option[A]) => RItem): MoveProp = + new MoveProp { + def move(target: RItem, source: RItem): RItem = + get(target) match { + case Some(_) => target + case None => set(target, get(source)) + } + } + + def concat( + get: RItem => Option[String] + )(set: (RItem, Option[String]) => RItem): MoveProp = + new MoveProp { + def move(target: RItem, source: RItem): RItem = + (get(target), get(source)) match { + case (Some(st), Some(ss)) => set(target, Some(st + "\n\n" + ss)) + case (Some(_), None) => target + case (None, src) => set(target, src) + } + } + + def all(props: MoveProp*): MoveProp = + new MoveProp { + def move(target: RItem, source: RItem): RItem = + props.foldLeft(target) { (el, move) => + move.move(el, source) + } + } + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OCustomFields.scala b/modules/backend/src/main/scala/docspell/backend/ops/OCustomFields.scala index c4f9af78..c864b098 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OCustomFields.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OCustomFields.scala @@ -13,6 +13,7 @@ import cats.effect._ import cats.implicits._ import docspell.backend.ops.OCustomFields.CustomFieldData +import docspell.backend.ops.OCustomFields.FieldValue import docspell.backend.ops.OCustomFields.NewCustomField import docspell.backend.ops.OCustomFields.RemoveValue import docspell.backend.ops.OCustomFields.SetValue @@ -53,6 +54,9 @@ trait OCustomFields[F[_]] { /** Deletes a value for a given field an item. */ def deleteValue(in: RemoveValue): F[UpdateResult] + + /** Finds all values to the given items */ + def findAllValues(itemIds: NonEmptyList[Ident]): F[List[FieldValue]] } object OCustomFields { @@ -60,6 +64,9 @@ object OCustomFields { type CustomFieldData = QCustomField.CustomFieldData val CustomFieldData = QCustomField.CustomFieldData + type FieldValue = QCustomField.FieldValue + val FieldValue = QCustomField.FieldValue + case class NewCustomField( name: Ident, label: Option[String], @@ -100,6 +107,9 @@ object OCustomFields { private[this] val logger = Logger.log4s[ConnectionIO](getLogger) + def findAllValues(itemIds: NonEmptyList[Ident]): F[List[FieldValue]] = + store.transact(QCustomField.findAllValues(itemIds)) + def findAll(coll: Ident, nameQuery: Option[String]): F[Vector[CustomFieldData]] = store.transact( QCustomField.findAllLike( diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala index eb82a641..4cf31aee 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala @@ -12,6 +12,7 @@ import cats.implicits._ import docspell.backend.JobFactory import docspell.backend.fulltext.CreateIndex +import docspell.backend.item.Merge import docspell.common._ import docspell.ftsclient.FtsClient import docspell.store.queries.{QAttachment, QItem, QMoveAttachment} @@ -206,6 +207,14 @@ trait OItem[F[_]] { storeMode: MakePreviewArgs.StoreMode, notifyJoex: Boolean ): F[UpdateResult] + + /** Merges a list of items into one item. The remaining items are deleted. + */ + def merge( + logger: Logger[F], + items: NonEmptyList[Ident], + collective: Ident + ): F[UpdateResult] } object OItem { @@ -223,6 +232,18 @@ object OItem { oequip <- OEquipment(store) logger <- Resource.pure[F, Logger[F]](Logger.log4s(getLogger)) oitem <- Resource.pure[F, OItem[F]](new OItem[F] { + + def merge( + logger: Logger[F], + items: NonEmptyList[Ident], + collective: Ident + ): F[UpdateResult] = + Merge(logger, store, this, createIndex).merge(items, collective).attempt.map { + case Right(Right(_)) => UpdateResult.success + case Right(Left(Merge.Error.NoItems)) => UpdateResult.NotFound + case Left(ex) => UpdateResult.failure(ex) + } + def moveAttachmentBefore( itemId: Ident, source: Ident, diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index a46a60f9..2cf107c0 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -2332,6 +2332,34 @@ paths: $ref: "#/components/schemas/BasicResult" + /sec/items/merge: + post: + operationId: "sec-items-merge" + tags: + - Item (Multi Edit) + summary: Merge multiple items into one. + description: | + A list of items is merged into one item by copying all + metadata into the first item in the list. Metadata is only + written, if there is no value present. So the order of items + in the list matters - the first item with a correspondent or + folder will win. For metadata that allow multiple values, like + tags or custom fields the values are combined. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/IdList" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/items/deleteAll: post: operationId: "sec-items-delete-all" diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemMultiRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemMultiRoutes.scala index 771734ec..77af7851 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemMultiRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemMultiRoutes.scala @@ -12,7 +12,7 @@ import cats.implicits._ import docspell.backend.BackendApp import docspell.backend.auth.AuthToken import docspell.backend.ops.OCustomFields.{RemoveValue, SetValue} -import docspell.common.ItemState +import docspell.common._ import docspell.restapi.model._ import docspell.restserver.conv.{Conversions, MultiIdSupport} @@ -20,8 +20,10 @@ import org.http4s.HttpRoutes import org.http4s.circe.CirceEntityDecoder._ import org.http4s.circe.CirceEntityEncoder._ import org.http4s.dsl.Http4sDsl +import org.log4s.getLogger object ItemMultiRoutes extends MultiIdSupport { + private[this] val log4sLogger = getLogger def apply[F[_]: Async]( backend: BackendApp[F], @@ -217,6 +219,14 @@ object ItemMultiRoutes extends MultiIdSupport { resp <- Ok(Conversions.basicResult(res, "Custom fields removed.")) } yield resp + case req @ POST -> Root / "merge" => + for { + json <- req.as[IdList] + items <- readIds[F](json.ids) + logger = Logger.log4s(log4sLogger) + res <- backend.item.merge(logger, items, user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Items merged")) + } yield resp } } diff --git a/modules/store/src/main/scala/docspell/store/queries/QCustomField.scala b/modules/store/src/main/scala/docspell/store/queries/QCustomField.scala index c944cddf..b2ce9836 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QCustomField.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QCustomField.scala @@ -6,6 +6,7 @@ package docspell.store.queries +import cats.data.{NonEmptyList => Nel} import cats.implicits._ import docspell.common._ @@ -19,7 +20,7 @@ object QCustomField { private val f = RCustomField.as("f") private val v = RCustomFieldValue.as("v") - case class CustomFieldData(field: RCustomField, usageCount: Int) + final case class CustomFieldData(field: RCustomField, usageCount: Int) def findAllLike( coll: Ident, @@ -47,4 +48,24 @@ object QCustomField { GroupBy(f.all) ) } + + final case class FieldValue( + field: RCustomField, + itemId: Ident, + value: String + ) + + def findAllValues(itemIds: Nel[Ident]): ConnectionIO[List[FieldValue]] = { + val cf = RCustomField.as("cf") + val cv = RCustomFieldValue.as("cv") + + run( + select(cf.all, Nel.of(cv.itemId, cv.value)), + from(cv).innerJoin(cf, cv.field === cf.id), + cv.itemId.in(itemIds) + ) + .query[FieldValue] + .to[List] + } + } diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala index 4384e67d..f08ff74f 100644 --- a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala +++ b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala @@ -97,6 +97,9 @@ object RAttachment { DML.set(T.fileId.setTo(fId)) ) + def updateItemId(attachId: Ident, itemId: Ident): ConnectionIO[Int] = + DML.update(T, T.id === attachId, DML.set(T.itemId.setTo(itemId))) + def updatePosition(attachId: Ident, pos: Int): ConnectionIO[Int] = DML.update(T, T.id === attachId, DML.set(T.position.setTo(pos))) diff --git a/modules/store/src/main/scala/docspell/store/records/RItem.scala b/modules/store/src/main/scala/docspell/store/records/RItem.scala index 22f3e60c..1be29fa9 100644 --- a/modules/store/src/main/scala/docspell/store/records/RItem.scala +++ b/modules/store/src/main/scala/docspell/store/records/RItem.scala @@ -130,6 +130,30 @@ object RItem { def getCollective(itemId: Ident): ConnectionIO[Option[Ident]] = Select(T.cid.s, from(T), T.id === itemId).build.query[Ident].option + def updateAll(item: RItem): ConnectionIO[Int] = + for { + t <- currentTime + n <- DML.update( + T, + T.id === item.id, + DML.set( + T.name.setTo(item.name), + T.itemDate.setTo(item.itemDate), + T.source.setTo(item.source), + T.incoming.setTo(item.direction), + T.corrOrg.setTo(item.corrOrg), + T.corrPerson.setTo(item.corrPerson), + T.concPerson.setTo(item.concPerson), + T.concEquipment.setTo(item.concEquipment), + T.inReplyTo.setTo(item.inReplyTo), + T.dueDate.setTo(item.dueDate), + T.notes.setTo(item.notes), + T.folder.setTo(item.folderId), + T.updated.setTo(t) + ) + ) + } yield n + def updateState( itemId: Ident, itemState: ItemState, diff --git a/modules/store/src/main/scala/docspell/store/records/RTagItem.scala b/modules/store/src/main/scala/docspell/store/records/RTagItem.scala index 6b8a6ae5..95c8d441 100644 --- a/modules/store/src/main/scala/docspell/store/records/RTagItem.scala +++ b/modules/store/src/main/scala/docspell/store/records/RTagItem.scala @@ -34,6 +34,9 @@ object RTagItem { def insert(v: RTagItem): ConnectionIO[Int] = DML.insert(T, T.all, fr"${v.tagItemId},${v.itemId},${v.tagId}") + def moveTags(from: Ident, to: Ident): ConnectionIO[Int] = + DML.update(T, T.itemId === from, DML.set(T.itemId.setTo(to))) + def deleteItemTags(item: Ident): ConnectionIO[Int] = DML.delete(T, T.itemId === item)