TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-11-04 12:30:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add endpoint to search for items and return their tags

Browse Source 
This is a more expensive query, since the tags must be resolved per
item. This is now implemented by doing additional queries while
caching each resolved tag.
This commit is contained in:
Eike Kettner
2020-06-07 15:18:28 +02:00
parent 0382ff2308
commit 1d2a6e6caa
6 changed files with 117 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
modules/backend/src/main/scala/docspell/backend/ops/OItem.scala
View File

@@ -15,6 +15,7 @@ import OItem.{
Batch,
ItemData,
ListItem,
ListItemWithTags,
Query
}
import bitpeace.{FileMeta, RangeDef}
@@ -27,6 +28,9 @@ trait OItem[F[_]] {
def findItems(q: Query, batch: Batch): F[Vector[ListItem]]
/** Same as `findItems` but does more queries per item to find all tags. */
def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]]
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]]
def findAttachmentSource(
@@ -91,6 +95,9 @@ object OItem {
type ListItem = QItem.ListItem
val ListItem = QItem.ListItem
type ListItemWithTags = QItem.ListItemWithTags
val ListItemWithTags = QItem.ListItemWithTags
type ItemData = QItem.ItemData
val ItemData = QItem.ItemData
@@ -148,6 +155,12 @@ object OItem {
.compile
.toVector
def findItemsWithTags(q: Query, batch: Batch): F[Vector[ListItemWithTags]] =
store
.transact(QItem.findItemsWithTags(q, batch).take(batch.limit.toLong))
.compile
.toVector
def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] =
store
.transact(RAttachment.findByIdAndCollective(id, collective))

31
modules/restapi/src/main/resources/docspell-openapi.yml
View File

@@ -987,7 +987,31 @@ paths:
summary: Search for items.
description: |
Search for items given a search form. The results are grouped
by month by default.
by month by default. Tags are *not* resolved! The results will
always contain an empty list for item tags. Use
`/searchWithTags` to also retrieve all tags of an item.
security:
- authTokenHeader: []
requestBody:
content:
application/json:
schema:
$ref: "#/components/schemas/ItemSearch"
responses:
200:
description: Ok
content:
application/json:
schema:
$ref: "#/components/schemas/ItemLightList"
/sec/item/searchWithTags:
post:
tags: [ Item ]
summary: Search for items.
description: |
Search for items given a search form. The results are grouped
by month by default. For each item, its tags are also
returned. This uses more queries and is therefore slower.
security:
- authTokenHeader: []
requestBody:
@@ -3188,6 +3212,7 @@ components:
- date
- source
- fileCount
- tags
properties:
id:
type: string
@@ -3221,6 +3246,10 @@ components:
fileCount:
type: integer
format: int32
tags:
type: array
items:
$ref: "#/components/schemas/Tag"
IdName:
description: |
The identifier and a human readable name of some entity.

17
modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala
View File

@@ -135,6 +135,17 @@ trait Conversions {
ItemLightList(gs)
}
def mkItemListWithTags(v: Vector[OItem.ListItemWithTags]): ItemLightList = {
val groups = v.groupBy(ti => ti.item.date.toUtcDate.toString.substring(0, 7))
def mkGroup(g: (String, Vector[OItem.ListItemWithTags])): ItemLightGroup =
ItemLightGroup(g._1, g._2.map(mkItemLightWithTags).toList)
val gs =
groups.map(mkGroup _).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0)
ItemLightList(gs)
}
def mkItemLight(i: OItem.ListItem): ItemLight =
ItemLight(
i.id,
@@ -148,9 +159,13 @@ trait Conversions {
i.corrPerson.map(mkIdName),
i.concPerson.map(mkIdName),
i.concEquip.map(mkIdName),
i.fileCount
i.fileCount,
Nil
)
def mkItemLightWithTags(i: OItem.ListItemWithTags): ItemLight =
mkItemLight(i.item).copy(tags = i.tags.map(mkTag))
// job
def mkJobQueueState(state: OJob.CollectiveQueueState): JobQueueState = {
def desc(f: JobDetail => Option[Timestamp])(j1: JobDetail, j2: JobDetail): Boolean = {

13
modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala
View File

@@ -36,6 +36,19 @@ object ItemRoutes {
resp <- Ok(Conversions.mkItemList(items))
} yield resp
case req @ POST -> Root / "searchWithTags" =>
for {
mask <- req.as[ItemSearch]
_ <- logger.ftrace(s"Got search mask: $mask")
query = Conversions.mkQuery(mask, user.account.collective)
_ <- logger.ftrace(s"Running query: $query")
items <- backend.item.findItemsWithTags(
query,
Batch(mask.offset, mask.limit).restrictLimitTo(500)
)
resp <- Ok(Conversions.mkItemListWithTags(items))
} yield resp
case GET -> Root / Ident(id) =>
for {
item <- backend.item.findItem(id, user.account.collective)

54
modules/store/src/main/scala/docspell/store/queries/QItem.scala
View File

@@ -4,6 +4,7 @@ import bitpeace.FileMeta
import cats.effect.Sync
import cats.data.OptionT
import cats.implicits._
import cats.effect.concurrent.Ref
import fs2.Stream
import doobie._
import doobie.implicits._
@@ -255,18 +256,10 @@ object QItem {
) ++
fr"SELECT DISTINCT" ++ finalCols ++ fr" FROM items i" ++
fr"LEFT JOIN attachs a ON" ++ IC.id.prefix("i").is(AC.itemId.prefix("a")) ++
fr"LEFT JOIN persons p0 ON" ++ IC.corrPerson
.prefix("i")
.is(PC.pid.prefix("p0")) ++ // i.corrperson = p0.pid" ++
fr"LEFT JOIN orgs o0 ON" ++ IC.corrOrg
.prefix("i")
.is(OC.oid.prefix("o0")) ++ // i.corrorg = o0.oid" ++
fr"LEFT JOIN persons p1 ON" ++ IC.concPerson
.prefix("i")
.is(PC.pid.prefix("p1")) ++ // i.concperson = p1.pid" ++
fr"LEFT JOIN equips e1 ON" ++ IC.concEquipment
.prefix("i")
.is(EC.eid.prefix("e1")) // i.concequipment = e1.eid"
fr"LEFT JOIN persons p0 ON" ++ IC.corrPerson.prefix("i").is(PC.pid.prefix("p0")) ++
fr"LEFT JOIN orgs o0 ON" ++ IC.corrOrg.prefix("i").is(OC.oid.prefix("o0")) ++
fr"LEFT JOIN persons p1 ON" ++ IC.concPerson.prefix("i").is(PC.pid.prefix("p1")) ++
fr"LEFT JOIN equips e1 ON" ++ IC.concEquipment.prefix("i").is(EC.eid.prefix("e1"))
// inclusive tags are AND-ed
val tagSelectsIncl = q.tagsInclude
@@ -339,6 +332,43 @@ object QItem {
frag.query[ListItem].stream
}
case class ListItemWithTags(item: ListItem, tags: List[RTag])
/** Same as `findItems` but resolves the tags for each item. Note that
* this is implemented by running an additional query per item.
*/
def findItemsWithTags(
q: Query,
batch: Batch
): Stream[ConnectionIO, ListItemWithTags] = {
def findTag(
cache: Ref[ConnectionIO, Map[Ident, RTag]],
tagItem: RTagItem
): ConnectionIO[Option[RTag]] =
for {
cc <- cache.get
fromCache = cc.get(tagItem.tagId)
orFromDB <-
if (fromCache.isDefined) fromCache.pure[ConnectionIO]
else RTag.findById(tagItem.tagId)
_ <-
if (fromCache.isDefined) ().pure[ConnectionIO]
else
orFromDB match {
case Some(t) => cache.update(tmap => tmap.updated(t.tagId, t))
case None => ().pure[ConnectionIO]
}
} yield orFromDB
for {
resolvedTags <- Stream.eval(Ref.of[ConnectionIO, Map[Ident, RTag]](Map.empty))
item <- findItems(q, batch)
tagItems <- Stream.eval(RTagItem.findByItem(item.id))
tags <- Stream.eval(tagItems.traverse(ti => findTag(resolvedTags, ti)))
ftags = tags.flatten.filter(t => t.collective == q.collective)
} yield ListItemWithTags(item, ftags.toList.sortBy(_.name))
}
def delete[F[_]: Sync](store: Store[F])(itemId: Ident, collective: Ident): F[Int] =
for {
tn <- store.transact(RTagItem.deleteItemTags(itemId))

3
modules/store/src/main/scala/docspell/store/records/RTagItem.scala
View File

@@ -38,4 +38,7 @@ object RTagItem {
tagFrag = tagValues.map(v => fr"${v.tagItemId},${v.itemId},${v.tagId}")
ins <- insertRows(table, all, tagFrag).update.run
} yield ins
def findByItem(item: Ident): ConnectionIO[Vector[RTagItem]] =
selectSimple(all, table, itemId.is(item)).query[RTagItem].to[Vector]
}