Implement item merge

2025-11-03 18:00:11 +00:00 · 2021-08-16 11:55:47 +02:00
parent 22d331f082
commit 85085ec173
9 changed files with 324 additions and 2 deletions
--- a/modules/backend/src/main/scala/docspell/backend/item/Merge.scala
+++ b/modules/backend/src/main/scala/docspell/backend/item/Merge.scala
@@ -0,0 +1,202 @@
+/*
+ * Copyright 2020 Docspell Contributors
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+package docspell.backend.item
+
+import cats.data.EitherT
+import cats.data.NonEmptyList
+import cats.effect._
+import cats.implicits._
+
+import docspell.backend.fulltext.CreateIndex
+import docspell.backend.ops.OItem
+import docspell.common._
+import docspell.store.Store
+import docspell.store.queries.QCustomField
+import docspell.store.queries.QCustomField.FieldValue
+import docspell.store.records.RAttachment
+import docspell.store.records.RCustomField
+import docspell.store.records.RItem
+import docspell.store.records.RTagItem
+
+trait Merge[F[_]] {
+  def merge(items: NonEmptyList[Ident], collective: Ident): F[Merge.Result[RItem]]
+}
+
+object Merge {
+
+  type Result[A] = Either[Error, A]
+  sealed trait Error
+  object Error {
+    final case object NoItems extends Error
+    def noItems: Error = NoItems
+
+  }
+
+  def apply[F[_]: Async](
+      logger: Logger[F],
+      store: Store[F],
+      itemOps: OItem[F],
+      createIndex: CreateIndex[F]
+  ): Merge[F] =
+    new Merge[F] {
+      def merge(givenIds: NonEmptyList[Ident], collective: Ident): F[Result[RItem]] =
+        (for {
+          items <- loadItems(givenIds, collective)
+          ids    = items.map(_.id)
+          target = moveMainData(items)
+          _ <- EitherT.right[Error](store.transact(RItem.updateAll(target)))
+          _ <- EitherT.right[Error](moveTags(ids))
+          _ <- EitherT.right[Error](moveCustomFields(ids))
+          _ <- EitherT.right[Error](moveAttachments(ids))
+          _ <- EitherT.right[Error](
+            createIndex
+              .reIndexData(logger, collective.some, NonEmptyList.one(ids.head).some, 50)
+          )
+          _ <- EitherT.right[Error](
+            NonEmptyList.fromList(items.tail.map(_.id)) match {
+              case Some(nel) => itemOps.setDeletedState(nel, collective)
+              case None      => 0.pure[F]
+            }
+          )
+        } yield target).value
+
+      def loadItems(
+          items: NonEmptyList[Ident],
+          collective: Ident
+      ): EitherT[F, Error, NonEmptyList[RItem]] = {
+        val loaded =
+          store
+            .transact(
+              items.toList.traverse(id => RItem.findByIdAndCollective(id, collective))
+            )
+            .map(_.flatten)
+            .map(NonEmptyList.fromList)
+        EitherT.fromOptionF(loaded, Error.NoItems)
+      }
+
+      def moveAttachments(items: NonEmptyList[Ident]): F[Int] = {
+        val target = items.head
+        for {
+          attachs <- store.transact(items.tail.traverse(id => RAttachment.findByItem(id)))
+          attachFlat = attachs.flatMap(_.toList)
+          n <- attachFlat.traverse(a =>
+            store.transact(RAttachment.updateItemId(a.id, target))
+          )
+        } yield n.sum
+      }
+
+      def moveTags(items: NonEmptyList[Ident]): F[Int] = {
+        val target = items.head
+        items.tail
+          .traverse(id => store.transact(RTagItem.moveTags(id, target)))
+          .map(_.sum)
+      }
+
+      def moveMainData(items: NonEmptyList[RItem]): RItem =
+        items.tail.foldLeft(items.head)(combine)
+
+      def moveCustomFields(items: NonEmptyList[Ident]): F[Unit] =
+        for {
+          values <- store.transact(QCustomField.findAllValues(items))
+          byField   = values.groupBy(_.field.name)
+          newValues = mergeFields(items.head, byField)
+          _ <- newValues.traverse(fv =>
+            store.transact(RCustomField.setValue(fv.field, items.head, fv.value))
+          )
+        } yield ()
+    }
+
+  private def mergeFields(
+      targetId: Ident,
+      byField: Map[Ident, List[FieldValue]]
+  ): List[FieldValue] =
+    byField
+      .filter(kv => kv._1 != targetId || kv._2.size > 1)
+      .values
+      .flatMap(NonEmptyList.fromList)
+      .map { nel =>
+        if (nel.tail.isEmpty) nel.head
+        else mergeFieldSameName(nel)
+      }
+      .toList
+
+  private def mergeFieldSameName(fields: NonEmptyList[FieldValue]): FieldValue =
+    fields.head.field.ftype match {
+      case CustomFieldType.Bool => fields.head
+      case CustomFieldType.Date => fields.head
+      case CustomFieldType.Money =>
+        val amount =
+          fields.toList
+            .flatMap(fv => CustomFieldType.Money.parseValue(fv.value).toOption)
+            .toList
+            .sum
+        fields.head.copy(value = CustomFieldType.Money.valueString(amount))
+
+      case CustomFieldType.Numeric =>
+        val amount =
+          fields.toList
+            .flatMap(fv => CustomFieldType.Numeric.parseValue(fv.value).toOption)
+            .toList
+            .sum
+        fields.head.copy(value = CustomFieldType.Numeric.valueString(amount))
+
+      case CustomFieldType.Text =>
+        val text = fields.toList
+          .map(fv => CustomFieldType.Text.parseValue(fv.value).toOption)
+          .mkString(", ")
+        fields.head.copy(value = CustomFieldType.Text.valueString(text))
+    }
+
+  private def combine(target: RItem, source: RItem): RItem =
+    MoveProp
+      .all(
+        MoveProp.whenNotExists(_.itemDate)((i, v) => i.copy(itemDate = v)),
+        MoveProp.whenNotExists(_.corrPerson)((i, v) => i.copy(corrPerson = v)),
+        MoveProp.whenNotExists(_.concPerson)((i, v) => i.copy(concPerson = v)),
+        MoveProp.whenNotExists(_.concEquipment)((i, v) => i.copy(concEquipment = v)),
+        MoveProp.whenNotExists(_.dueDate)((i, v) => i.copy(dueDate = v)),
+        MoveProp.whenNotExists(_.folderId)((i, v) => i.copy(folderId = v)),
+        MoveProp.concat(_.notes)((i, v) => i.copy(notes = v))
+      )
+      .move(target, source)
+
+  trait MoveProp {
+    def move(target: RItem, source: RItem): RItem
+  }
+  object MoveProp {
+    def whenNotExists[A](
+        get: RItem => Option[A]
+    )(set: (RItem, Option[A]) => RItem): MoveProp =
+      new MoveProp {
+        def move(target: RItem, source: RItem): RItem =
+          get(target) match {
+            case Some(_) => target
+            case None    => set(target, get(source))
+          }
+      }
+
+    def concat(
+        get: RItem => Option[String]
+    )(set: (RItem, Option[String]) => RItem): MoveProp =
+      new MoveProp {
+        def move(target: RItem, source: RItem): RItem =
+          (get(target), get(source)) match {
+            case (Some(st), Some(ss)) => set(target, Some(st + "\n\n" + ss))
+            case (Some(_), None)      => target
+            case (None, src)          => set(target, src)
+          }
+      }
+
+    def all(props: MoveProp*): MoveProp =
+      new MoveProp {
+        def move(target: RItem, source: RItem): RItem =
+          props.foldLeft(target) { (el, move) =>
+            move.move(el, source)
+          }
+      }
+  }
+}
--- a/modules/backend/src/main/scala/docspell/backend/ops/OCustomFields.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OCustomFields.scala
@@ -13,6 +13,7 @@ import cats.effect._
 import cats.implicits._

 import docspell.backend.ops.OCustomFields.CustomFieldData
+import docspell.backend.ops.OCustomFields.FieldValue
 import docspell.backend.ops.OCustomFields.NewCustomField
 import docspell.backend.ops.OCustomFields.RemoveValue
 import docspell.backend.ops.OCustomFields.SetValue
@@ -53,6 +54,9 @@ trait OCustomFields[F[_]] {

  /** Deletes a value for a given field an item. */
  def deleteValue(in: RemoveValue): F[UpdateResult]
+
+  /** Finds all values to the given items */
+  def findAllValues(itemIds: NonEmptyList[Ident]): F[List[FieldValue]]
 }

 object OCustomFields {
@@ -60,6 +64,9 @@ object OCustomFields {
  type CustomFieldData = QCustomField.CustomFieldData
  val CustomFieldData = QCustomField.CustomFieldData

+  type FieldValue = QCustomField.FieldValue
+  val FieldValue = QCustomField.FieldValue
+
  case class NewCustomField(
      name: Ident,
      label: Option[String],
@@ -100,6 +107,9 @@ object OCustomFields {

      private[this] val logger = Logger.log4s[ConnectionIO](getLogger)

+      def findAllValues(itemIds: NonEmptyList[Ident]): F[List[FieldValue]] =
+        store.transact(QCustomField.findAllValues(itemIds))
+
      def findAll(coll: Ident, nameQuery: Option[String]): F[Vector[CustomFieldData]] =
        store.transact(
          QCustomField.findAllLike(
--- a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala
+++ b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala
@@ -12,6 +12,7 @@ import cats.implicits._

 import docspell.backend.JobFactory
 import docspell.backend.fulltext.CreateIndex
+import docspell.backend.item.Merge
 import docspell.common._
 import docspell.ftsclient.FtsClient
 import docspell.store.queries.{QAttachment, QItem, QMoveAttachment}
@@ -206,6 +207,14 @@ trait OItem[F[_]] {
      storeMode: MakePreviewArgs.StoreMode,
      notifyJoex: Boolean
  ): F[UpdateResult]
+
+  /** Merges a list of items into one item. The remaining items are deleted.
+    */
+  def merge(
+      logger: Logger[F],
+      items: NonEmptyList[Ident],
+      collective: Ident
+  ): F[UpdateResult]
 }

 object OItem {
@@ -223,6 +232,18 @@ object OItem {
      oequip <- OEquipment(store)
      logger <- Resource.pure[F, Logger[F]](Logger.log4s(getLogger))
      oitem <- Resource.pure[F, OItem[F]](new OItem[F] {
+
+        def merge(
+            logger: Logger[F],
+            items: NonEmptyList[Ident],
+            collective: Ident
+        ): F[UpdateResult] =
+          Merge(logger, store, this, createIndex).merge(items, collective).attempt.map {
+            case Right(Right(_))                  => UpdateResult.success
+            case Right(Left(Merge.Error.NoItems)) => UpdateResult.NotFound
+            case Left(ex)                         => UpdateResult.failure(ex)
+          }
+
        def moveAttachmentBefore(
            itemId: Ident,
            source: Ident,
--- a/modules/restapi/src/main/resources/docspell-openapi.yml
+++ b/modules/restapi/src/main/resources/docspell-openapi.yml
@@ -2332,6 +2332,34 @@ paths:
                $ref: "#/components/schemas/BasicResult"


+  /sec/items/merge:
+    post:
+      operationId: "sec-items-merge"
+      tags:
+        - Item (Multi Edit)
+      summary: Merge multiple items into one.
+      description: |
+        A list of items is merged into one item by copying all
+        metadata into the first item in the list. Metadata is only
+        written, if there is no value present. So the order of items
+        in the list matters - the first item with a correspondent or
+        folder will win. For metadata that allow multiple values, like
+        tags or custom fields the values are combined.
+      security:
+        - authTokenHeader: []
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/IdList"
+      responses:
+        200:
+          description: Ok
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/BasicResult"
+
  /sec/items/deleteAll:
    post:
      operationId: "sec-items-delete-all"
--- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemMultiRoutes.scala
+++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemMultiRoutes.scala
@@ -12,7 +12,7 @@ import cats.implicits._
 import docspell.backend.BackendApp
 import docspell.backend.auth.AuthToken
 import docspell.backend.ops.OCustomFields.{RemoveValue, SetValue}
-import docspell.common.ItemState
+import docspell.common._
 import docspell.restapi.model._
 import docspell.restserver.conv.{Conversions, MultiIdSupport}

@@ -20,8 +20,10 @@ import org.http4s.HttpRoutes
 import org.http4s.circe.CirceEntityDecoder._
 import org.http4s.circe.CirceEntityEncoder._
 import org.http4s.dsl.Http4sDsl
+import org.log4s.getLogger

 object ItemMultiRoutes extends MultiIdSupport {
+  private[this] val log4sLogger = getLogger

  def apply[F[_]: Async](
      backend: BackendApp[F],
@@ -217,6 +219,14 @@ object ItemMultiRoutes extends MultiIdSupport {
          resp <- Ok(Conversions.basicResult(res, "Custom fields removed."))
        } yield resp

+      case req @ POST -> Root / "merge" =>
+        for {
+          json  <- req.as[IdList]
+          items <- readIds[F](json.ids)
+          logger = Logger.log4s(log4sLogger)
+          res  <- backend.item.merge(logger, items, user.account.collective)
+          resp <- Ok(Conversions.basicResult(res, "Items merged"))
+        } yield resp
    }
  }

--- a/modules/store/src/main/scala/docspell/store/queries/QCustomField.scala
+++ b/modules/store/src/main/scala/docspell/store/queries/QCustomField.scala
@@ -6,6 +6,7 @@

 package docspell.store.queries

+import cats.data.{NonEmptyList => Nel}
 import cats.implicits._

 import docspell.common._
@@ -19,7 +20,7 @@ object QCustomField {
  private val f = RCustomField.as("f")
  private val v = RCustomFieldValue.as("v")

-  case class CustomFieldData(field: RCustomField, usageCount: Int)
+  final case class CustomFieldData(field: RCustomField, usageCount: Int)

  def findAllLike(
      coll: Ident,
@@ -47,4 +48,24 @@ object QCustomField {
      GroupBy(f.all)
    )
  }
+
+  final case class FieldValue(
+      field: RCustomField,
+      itemId: Ident,
+      value: String
+  )
+
+  def findAllValues(itemIds: Nel[Ident]): ConnectionIO[List[FieldValue]] = {
+    val cf = RCustomField.as("cf")
+    val cv = RCustomFieldValue.as("cv")
+
+    run(
+      select(cf.all, Nel.of(cv.itemId, cv.value)),
+      from(cv).innerJoin(cf, cv.field === cf.id),
+      cv.itemId.in(itemIds)
+    )
+      .query[FieldValue]
+      .to[List]
+  }
+
 }
--- a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala
+++ b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala
@@ -97,6 +97,9 @@ object RAttachment {
      DML.set(T.fileId.setTo(fId))
    )

+  def updateItemId(attachId: Ident, itemId: Ident): ConnectionIO[Int] =
+    DML.update(T, T.id === attachId, DML.set(T.itemId.setTo(itemId)))
+
  def updatePosition(attachId: Ident, pos: Int): ConnectionIO[Int] =
    DML.update(T, T.id === attachId, DML.set(T.position.setTo(pos)))

--- a/modules/store/src/main/scala/docspell/store/records/RItem.scala
+++ b/modules/store/src/main/scala/docspell/store/records/RItem.scala
@@ -130,6 +130,30 @@ object RItem {
  def getCollective(itemId: Ident): ConnectionIO[Option[Ident]] =
    Select(T.cid.s, from(T), T.id === itemId).build.query[Ident].option

+  def updateAll(item: RItem): ConnectionIO[Int] =
+    for {
+      t <- currentTime
+      n <- DML.update(
+        T,
+        T.id === item.id,
+        DML.set(
+          T.name.setTo(item.name),
+          T.itemDate.setTo(item.itemDate),
+          T.source.setTo(item.source),
+          T.incoming.setTo(item.direction),
+          T.corrOrg.setTo(item.corrOrg),
+          T.corrPerson.setTo(item.corrPerson),
+          T.concPerson.setTo(item.concPerson),
+          T.concEquipment.setTo(item.concEquipment),
+          T.inReplyTo.setTo(item.inReplyTo),
+          T.dueDate.setTo(item.dueDate),
+          T.notes.setTo(item.notes),
+          T.folder.setTo(item.folderId),
+          T.updated.setTo(t)
+        )
+      )
+    } yield n
+
  def updateState(
      itemId: Ident,
      itemState: ItemState,
--- a/modules/store/src/main/scala/docspell/store/records/RTagItem.scala
+++ b/modules/store/src/main/scala/docspell/store/records/RTagItem.scala
@@ -34,6 +34,9 @@ object RTagItem {
  def insert(v: RTagItem): ConnectionIO[Int] =
    DML.insert(T, T.all, fr"${v.tagItemId},${v.itemId},${v.tagId}")

+  def moveTags(from: Ident, to: Ident): ConnectionIO[Int] =
+    DML.update(T, T.itemId === from, DML.set(T.itemId.setTo(to)))
+
  def deleteItemTags(item: Ident): ConnectionIO[Int] =
    DML.delete(T, T.itemId === item)