From cc38b850a67a0b44b9f24e52b1ff2d82c9a70dae Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sat, 27 Mar 2021 22:03:43 +0100 Subject: [PATCH] Remove deprecated search routes and some refactoring --- .../docspell/backend/ops/OFulltext.scala | 25 ++- .../docspell/backend/ops/OSimpleSearch.scala | 4 +- .../joex/notify/NotifyDueItemsTask.scala | 27 ++- .../main/scala/docspell/query/ItemQuery.scala | 2 + .../scala/docspell/query/ItemQueryDsl.scala | 76 +++++++ .../docspell/query/internal/ExprUtil.scala | 3 + .../src/main/resources/docspell-openapi.yml | 188 ------------------ .../restserver/conv/Conversions.scala | 30 --- .../restserver/routes/ItemRoutes.scala | 136 ------------- .../qb/generator/ItemQueryGenerator.scala | 3 + .../scala/docspell/store/queries/QItem.scala | 91 ++------- .../scala/docspell/store/queries/Query.scala | 65 +----- modules/webapp/src/main/elm/Api.elm | 1 - .../src/main/elm/Comp/NotificationForm.elm | 14 +- .../src/main/elm/Comp/NotificationManage.elm | 1 + tools/export-files/export-files.sh | 8 +- 16 files changed, 167 insertions(+), 507 deletions(-) create mode 100644 modules/query/shared/src/main/scala/docspell/query/ItemQueryDsl.scala diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala index 0dd2348c..73b9a015 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OFulltext.scala @@ -1,5 +1,6 @@ package docspell.backend.ops +import cats.data.NonEmptyList import cats.effect._ import cats.implicits._ import fs2.Stream @@ -9,6 +10,8 @@ import docspell.backend.ops.OItemSearch._ import docspell.common._ import docspell.common.syntax.all._ import docspell.ftsclient._ +import docspell.query.ItemQuery._ +import docspell.query.ItemQueryDsl._ import docspell.store.queries.{QFolder, QItem, SelectedItem} import docspell.store.queue.JobQueue import docspell.store.records.RJob @@ -125,12 +128,18 @@ object OFulltext { .map(_.minBy(-_.score)) .map(r => SelectedItem(r.itemId, r.score)) .toSet + now <- Timestamp.current[F] itemsWithTags <- store .transact( QItem.findItemsWithTags( account.collective, - QItem.findSelectedItems(Query.empty(account), maxNoteLen, select) + QItem.findSelectedItems( + Query.all(account), + now.toUtcDate, + maxNoteLen, + select + ) ) ) .take(batch.limit.toLong) @@ -165,7 +174,13 @@ object OFulltext { .flatMap(r => Stream.emits(r.results.map(_.itemId))) .compile .to(Set) - q = Query.empty(account).withFix(_.copy(itemIds = itemIds.some)) + itemIdsQuery = NonEmptyList + .fromList(itemIds.toList) + .map(ids => Attr.ItemId.in(ids.map(_.id))) + .getOrElse(Attr.ItemId.notExists) + q = Query + .all(account) + .withFix(_.copy(query = itemIdsQuery.some)) res <- store.transact(QItem.searchStats(now.toUtcDate)(q)) } yield res } @@ -221,7 +236,11 @@ object OFulltext { .flatMap(r => Stream.emits(r.results.map(_.itemId))) .compile .to(Set) - qnext = q.withFix(_.copy(itemIds = items.some)) + itemIdsQuery = NonEmptyList + .fromList(items.toList) + .map(ids => Attr.ItemId.in(ids.map(_.id))) + .getOrElse(Attr.ItemId.notExists) + qnext = q.withFix(_.copy(query = itemIdsQuery.some)) now <- Timestamp.current[F] res <- store.transact(QItem.searchStats(now.toUtcDate)(qnext)) } yield res diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala b/modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala index 1c5e54df..7ca0337e 100644 --- a/modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala +++ b/modules/backend/src/main/scala/docspell/backend/ops/OSimpleSearch.scala @@ -129,7 +129,7 @@ object OSimpleSearch { def makeQuery(iq: ItemQuery): F[StringSearchResult[Items]] = iq.findFulltext match { case FulltextExtract.Result.Success(expr, ftq) => - search(settings)(Query(fix, Query.QueryExpr(iq.copy(expr = expr))), ftq) + search(settings)(Query(fix, Query.QueryExpr(expr.some)), ftq) .map(StringSearchResult.Success.apply) case other: FulltextExtract.FailureResult => StringSearchResult.fulltextMismatch[Items](other).pure[F] @@ -152,7 +152,7 @@ object OSimpleSearch { def makeQuery(iq: ItemQuery): F[StringSearchResult[SearchSummary]] = iq.findFulltext match { case FulltextExtract.Result.Success(expr, ftq) => - searchSummary(useFTS)(Query(fix, Query.QueryExpr(iq.copy(expr = expr))), ftq) + searchSummary(useFTS)(Query(fix, Query.QueryExpr(expr.some)), ftq) .map(StringSearchResult.Success.apply) case other: FulltextExtract.FailureResult => StringSearchResult.fulltextMismatch[SearchSummary](other).pure[F] diff --git a/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala b/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala index 4ce26507..ff0e8f5e 100644 --- a/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala +++ b/modules/joex/src/main/scala/docspell/joex/notify/NotifyDueItemsTask.scala @@ -1,6 +1,6 @@ package docspell.joex.notify -import cats.data.OptionT +import cats.data.{NonEmptyList => Nel, OptionT} import cats.effect._ import cats.implicits._ @@ -10,6 +10,9 @@ import docspell.joex.mail.EmilHeader import docspell.joex.scheduler.{Context, Task} import docspell.store.queries.QItem import docspell.store.records._ +import docspell.query.Date +import docspell.query.ItemQuery._ +import docspell.query.ItemQueryDsl._ import emil._ import emil.builder._ @@ -69,18 +72,24 @@ object NotifyDueItemsTask { def findItems[F[_]: Sync](ctx: Context[F, Args]): F[Vector[ListItem]] = for { now <- Timestamp.current[F] + rightDate = Date((now + Duration.days(ctx.args.remindDays.toLong)).toMillis) q = Query - .empty(ctx.args.account) + .all(ctx.args.account) .withOrder(orderAsc = _.dueDate) + .withFix(_.copy(query = Expr.ValidItemStates.some)) .withCond(_ => - Query.QueryForm.empty.copy( - states = ItemState.validStates.toList, - tagsInclude = ctx.args.tagsInclude, - tagsExclude = ctx.args.tagsExclude, - dueDateFrom = - ctx.args.daysBack.map(back => now - Duration.days(back.toLong)), - dueDateTo = Some(now + Duration.days(ctx.args.remindDays.toLong)) + Query.QueryExpr( + Attr.DueDate <= rightDate &&? + ctx.args.daysBack.map(back => + Attr.DueDate >= Date((now - Duration.days(back.toLong)).toMillis) + ) &&? + Nel + .fromList(ctx.args.tagsInclude) + .map(ids => Q.tagIdsEq(ids.map(_.id))) &&? + Nel + .fromList(ctx.args.tagsExclude) + .map(ids => Q.tagIdsIn(ids.map(_.id)).negate) ) ) res <- diff --git a/modules/query/shared/src/main/scala/docspell/query/ItemQuery.scala b/modules/query/shared/src/main/scala/docspell/query/ItemQuery.scala index e2b7ef06..2bd1f410 100644 --- a/modules/query/shared/src/main/scala/docspell/query/ItemQuery.scala +++ b/modules/query/shared/src/main/scala/docspell/query/ItemQuery.scala @@ -119,6 +119,8 @@ object ItemQuery { final case class ChecksumMatch(checksum: String) extends Expr final case class AttachId(id: String) extends Expr + case object ValidItemStates extends Expr + // things that can be expressed with terms above sealed trait MacroExpr extends Expr { def body: Expr diff --git a/modules/query/shared/src/main/scala/docspell/query/ItemQueryDsl.scala b/modules/query/shared/src/main/scala/docspell/query/ItemQueryDsl.scala new file mode 100644 index 00000000..a016de90 --- /dev/null +++ b/modules/query/shared/src/main/scala/docspell/query/ItemQueryDsl.scala @@ -0,0 +1,76 @@ +package docspell.query + +import cats.data.NonEmptyList + +import docspell.query.ItemQuery._ +import docspell.query.internal.ExprUtil + +object ItemQueryDsl { + + implicit final class StringAttrDsl(val attr: Attr.StringAttr) extends AnyVal { + def ===(value: String): Expr = + Expr.SimpleExpr(Operator.Eq, Property(attr, value)) + + def <=(value: String): Expr = + Expr.SimpleExpr(Operator.Lte, Property(attr, value)) + def >=(value: String): Expr = + Expr.SimpleExpr(Operator.Gte, Property(attr, value)) + + def <(value: String): Expr = + Expr.SimpleExpr(Operator.Lt, Property(attr, value)) + def >(value: String): Expr = + Expr.SimpleExpr(Operator.Gt, Property(attr, value)) + + def in(values: NonEmptyList[String]): Expr = + Expr.InExpr(attr, values) + + def exists: Expr = + Expr.Exists(attr) + + def notExists: Expr = + Expr.NotExpr(exists) + } + + implicit final class DateAttrDsl(val attr: Attr.DateAttr) extends AnyVal { + def <=(value: Date): Expr = + Expr.SimpleExpr(Operator.Lte, Property(attr, value)) + + def >=(value: Date): Expr = + Expr.SimpleExpr(Operator.Gte, Property(attr, value)) + } + + implicit final class ExprDsl(val expr: Expr) extends AnyVal { + def &&(other: Expr): Expr = + ExprUtil.reduce(Expr.and(expr, other)) + + def ||(other: Expr): Expr = + ExprUtil.reduce(Expr.or(expr, other)) + + def &&?(other: Option[Expr]): Expr = + other.map(e => &&(e)).getOrElse(expr) + + def ||?(other: Option[Expr]): Expr = + other.map(e => ||(e)).getOrElse(expr) + + def negate: Expr = + ExprUtil.reduce(Expr.NotExpr(expr)) + + def unary_! : Expr = + negate + } + + object Q { + def tagIdsIn(values: NonEmptyList[String]): Expr = + Expr.TagIdsMatch(TagOperator.AnyMatch, values) + + def tagIdsEq(values: NonEmptyList[String]): Expr = + Expr.TagIdsMatch(TagOperator.AllMatch, values) + + def tagsIn(values: NonEmptyList[String]): Expr = + Expr.TagsMatch(TagOperator.AnyMatch, values) + + def tagsEq(values: NonEmptyList[String]): Expr = + Expr.TagsMatch(TagOperator.AllMatch, values) + + } +} diff --git a/modules/query/shared/src/main/scala/docspell/query/internal/ExprUtil.scala b/modules/query/shared/src/main/scala/docspell/query/internal/ExprUtil.scala index df81983f..8f7b6c2c 100644 --- a/modules/query/shared/src/main/scala/docspell/query/internal/ExprUtil.scala +++ b/modules/query/shared/src/main/scala/docspell/query/internal/ExprUtil.scala @@ -69,6 +69,9 @@ object ExprUtil { expr case AttachId(_) => expr + + case ValidItemStates => + expr } private def spliceAnd(nodes: Nel[Expr]): Nel[Expr] = diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 3b18422c..5d8d4bf6 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -1309,73 +1309,6 @@ paths: schema: $ref: "#/components/schemas/BasicResult" - - /sec/item/searchForm: - post: - tags: [ Item Search ] - summary: Search for items. - deprecated: true - description: | - Search for items given a search form. The results are grouped - by month and are sorted by item date (newest first). Tags and - attachments are *not* resolved. The results will always - contain an empty list for item tags and attachments. Use - `/searchFormWithTags` to also retrieve all tags and a list of - attachments of an item. - - The `fulltext` field can be used to restrict the results by - using full-text search in the documents contents. - - The customfields used in the search query are allowed to be - specified by either field id or field name. The values may - contain the wildcard `*` at beginning or end. - - **NOTE** This is deprecated in favor for using a search query. - security: - - authTokenHeader: [] - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/ItemSearch" - responses: - 200: - description: Ok - content: - application/json: - schema: - $ref: "#/components/schemas/ItemLightList" - /sec/item/searchFormWithTags: - post: - tags: [ Item Search ] - summary: Search for items. - deprecated: true - description: | - Search for items given a search form. The results are grouped - by month by default. For each item, its tags and attachments - are also returned. This uses more queries and is therefore - slower, but returns all tags to an item as well as their - attachments with some minor details. - - The `fulltext` field can be used to restrict the results by - using full-text search in the documents contents. - - **NOTE** This is deprecated in favor for using search query. - security: - - authTokenHeader: [] - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/ItemSearch" - responses: - 200: - description: Ok - content: - application/json: - schema: - $ref: "#/components/schemas/ItemLightList" - /sec/item/search: get: tags: [ Item Search ] @@ -1457,29 +1390,6 @@ paths: schema: $ref: "#/components/schemas/ItemLightList" - /sec/item/searchFormStats: - post: - tags: [ Item Search ] - summary: Get basic statistics about the data of a search. - deprecated: true - description: | - Takes a search query and returns a summary about the results. - - **NOTE** This is deprecated in favor of using a search query. - security: - - authTokenHeader: [] - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/ItemSearch" - responses: - 200: - description: Ok - content: - application/json: - schema: - $ref: "#/components/schemas/SearchStats" /sec/item/searchStats: post: tags: [ Item Search ] @@ -5268,104 +5178,6 @@ components: type: array items: $ref: "#/components/schemas/ItemLight" - ItemSearch: - description: | - A structure for a search form. - required: - - tagsInclude - - tagsExclude - - tagCategoriesInclude - - tagCategoriesExclude - - inbox - - offset - - limit - - customValues - properties: - tagsInclude: - type: array - items: - type: string - format: ident - tagsExclude: - type: array - items: - type: string - format: ident - tagCategoriesInclude: - type: array - items: - type: string - tagCategoriesExclude: - type: array - items: - type: string - inbox: - type: boolean - offset: - type: integer - format: int32 - limit: - type: integer - format: int32 - description: | - The maximum number of results to return. Note that this - limit is a soft limit, there is some hard limit on the - server, too. - direction: - type: string - format: direction - enum: - - incoming - - outgoing - name: - type: string - description: | - Search in item names. - allNames: - type: string - description: | - Search in item names, correspondents, concerned entities - and notes. - fullText: - type: string - description: | - A query searching the contents of documents. If only this - field is set, then a fulltext-only search is done. - corrOrg: - type: string - format: ident - corrPerson: - type: string - format: ident - concPerson: - type: string - format: ident - concEquip: - type: string - format: ident - folder: - type: string - format: ident - dateFrom: - type: integer - format: date-time - dateUntil: - type: integer - format: date-time - dueDateFrom: - type: integer - format: date-time - dueDateUntil: - type: integer - format: date-time - itemSubset: - $ref: "#/components/schemas/IdList" - customValues: - type: array - items: - $ref: "#/components/schemas/CustomFieldValue" - source: - type: string ItemLight: description: | An item with only a few important properties. diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala index b826488c..ed4d5f0e 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -143,36 +143,6 @@ trait Conversions { // item list - def mkQuery(m: ItemSearch, account: AccountId): OItemSearch.Query = - OItemSearch.Query( - OItemSearch.Query.Fix(account, None, None), - OItemSearch.Query.QueryForm( - m.name, - if (m.inbox) Seq(ItemState.Created) - else ItemState.validStates.toList, - m.direction, - m.corrPerson, - m.corrOrg, - m.concPerson, - m.concEquip, - m.folder, - m.tagsInclude.map(Ident.unsafe), - m.tagsExclude.map(Ident.unsafe), - m.tagCategoriesInclude, - m.tagCategoriesExclude, - m.dateFrom, - m.dateUntil, - m.dueDateFrom, - m.dueDateUntil, - m.allNames, - m.itemSubset - .map(_.ids.flatMap(i => Ident.fromString(i).toOption).toSet) - .filter(_.nonEmpty), - m.customValues.map(mkCustomValue), - m.source - ) - ) - def mkCustomValue(v: CustomFieldValue): OItemSearch.CustomValue = OItemSearch.CustomValue(v.field, v.value) diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala index 2e7fbbb5..35fe9320 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala @@ -1,6 +1,5 @@ package docspell.restserver.routes -import cats.Monoid import cats.data.NonEmptyList import cats.effect._ import cats.implicits._ @@ -102,85 +101,6 @@ object ItemRoutes { ) } yield resp - //DEPRECATED - case req @ POST -> Root / "searchForm" => - for { - mask <- req.as[ItemSearch] - _ <- logger.ftrace(s"Got search mask: $mask") - query = Conversions.mkQuery(mask, user.account) - _ <- logger.ftrace(s"Running query: $query") - resp <- mask match { - case SearchFulltextOnly(ftq) if cfg.fullTextSearch.enabled => - val ftsIn = OFulltext.FtsInput(ftq.query) - for { - items <- backend.fulltext.findIndexOnly(cfg.maxNoteLength)( - ftsIn, - user.account, - Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize) - ) - ok <- Ok(Conversions.mkItemListWithTagsFtsPlain(items)) - } yield ok - - case SearchWithFulltext(fq) if cfg.fullTextSearch.enabled => - for { - items <- backend.fulltext.findItems(cfg.maxNoteLength)( - query, - OFulltext.FtsInput(fq), - Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize) - ) - ok <- Ok(Conversions.mkItemListFts(items)) - } yield ok - - case _ => - for { - items <- backend.itemSearch.findItems(cfg.maxNoteLength)( - query, - Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize) - ) - ok <- Ok(Conversions.mkItemList(items)) - } yield ok - } - } yield resp - - //DEPRECATED - case req @ POST -> Root / "searchFormWithTags" => - for { - mask <- req.as[ItemSearch] - _ <- logger.ftrace(s"Got search mask: $mask") - query = Conversions.mkQuery(mask, user.account) - _ <- logger.ftrace(s"Running query: $query") - resp <- mask match { - case SearchFulltextOnly(ftq) if cfg.fullTextSearch.enabled => - val ftsIn = OFulltext.FtsInput(ftq.query) - for { - items <- backend.fulltext.findIndexOnly(cfg.maxNoteLength)( - ftsIn, - user.account, - Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize) - ) - ok <- Ok(Conversions.mkItemListWithTagsFtsPlain(items)) - } yield ok - - case SearchWithFulltext(fq) if cfg.fullTextSearch.enabled => - for { - items <- backend.fulltext.findItemsWithTags(cfg.maxNoteLength)( - query, - OFulltext.FtsInput(fq), - Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize) - ) - ok <- Ok(Conversions.mkItemListWithTagsFts(items)) - } yield ok - case _ => - for { - items <- backend.itemSearch.findItemsWithTags(cfg.maxNoteLength)( - query, - Batch(mask.offset, mask.limit).restrictLimitTo(cfg.maxItemPageSize) - ) - ok <- Ok(Conversions.mkItemListWithTags(items)) - } yield ok - } - } yield resp - case req @ POST -> Root / "searchIndex" => for { mask <- req.as[ItemQuery] @@ -204,26 +124,6 @@ object ItemRoutes { } } yield resp - //DEPRECATED - case req @ POST -> Root / "searchFormStats" => - for { - mask <- req.as[ItemSearch] - query = Conversions.mkQuery(mask, user.account) - stats <- mask match { - case SearchFulltextOnly(ftq) if cfg.fullTextSearch.enabled => - logger.finfo(s"Make index only summary: $ftq") *> - backend.fulltext.findIndexOnlySummary( - user.account, - OFulltext.FtsInput(ftq.query) - ) - case SearchWithFulltext(fq) if cfg.fullTextSearch.enabled => - backend.fulltext.findItemsSummary(query, OFulltext.FtsInput(fq)) - case _ => - backend.itemSearch.findItemsSummary(query) - } - resp <- Ok(Conversions.mkSearchStats(stats)) - } yield resp - case GET -> Root / Ident(id) => for { item <- backend.itemSearch.findItem(id, user.account.collective) @@ -560,40 +460,4 @@ object ItemRoutes { def notEmpty: Option[String] = opt.map(_.trim).filter(_.nonEmpty) } - - object SearchFulltextOnly { - implicit private val identMonoid: Monoid[Ident] = - Monoid.instance(Ident.unsafe(""), _ / _) - - implicit private val timestampMonoid: Monoid[Timestamp] = - Monoid.instance(Timestamp.Epoch, (a, _) => a) - - implicit private val directionMonoid: Monoid[Direction] = - Monoid.instance(Direction.Incoming, (a, _) => a) - - implicit private val idListMonoid: Monoid[IdList] = - Monoid.instance(IdList(Nil), (a, b) => IdList(a.ids ++ b.ids)) - - implicit private val boolMonoid: Monoid[Boolean] = - Monoid.instance(false, _ || _) - - private val itemSearchMonoid: Monoid[ItemSearch] = - cats.derived.semiauto.monoid - - def unapply(m: ItemSearch): Option[ItemQuery] = - m.fullText match { - case Some(fq) => - val me = m.copy(fullText = None, offset = 0, limit = 0) - if (itemSearchMonoid.empty == me) - Some(ItemQuery(m.offset.some, m.limit.some, Some(false), fq)) - else None - case _ => - None - } - } - - object SearchWithFulltext { - def unapply(m: ItemSearch): Option[String] = - m.fullText - } } diff --git a/modules/store/src/main/scala/docspell/store/qb/generator/ItemQueryGenerator.scala b/modules/store/src/main/scala/docspell/store/qb/generator/ItemQueryGenerator.scala index 6a721270..203c4a0d 100644 --- a/modules/store/src/main/scala/docspell/store/qb/generator/ItemQueryGenerator.scala +++ b/modules/store/src/main/scala/docspell/store/qb/generator/ItemQueryGenerator.scala @@ -117,6 +117,9 @@ object ItemQueryGenerator { if (flag) tables.item.state === ItemState.created else tables.item.state === ItemState.confirmed + case Expr.ValidItemStates => + tables.item.state.in(ItemState.validStates) + case Expr.TagIdsMatch(op, tags) => val ids = tags.toList.flatMap(s => Ident.fromString(s).toOption) Nel diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala index c1ee5f2c..637a6890 100644 --- a/modules/store/src/main/scala/docspell/store/queries/QItem.scala +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -98,30 +98,7 @@ object QItem { cv.itemId === itemId ).build.query[ItemFieldValue].to[Vector] - private def findCustomFieldValuesForColl( - coll: Ident, - values: Seq[CustomValue] - ): Option[Select] = { - val cf = RCustomField.as("cf") - val cv = RCustomFieldValue.as("cv") - - def singleSelect(v: CustomValue) = - Select( - cv.itemId.s, - from(cv).innerJoin(cf, cv.field === cf.id), - where( - cf.cid === coll && - (cf.name === v.field || cf.id === v.field) && - cv.value.like(QueryWildcard(v.value.toLowerCase)) - ) - ) - - Nel - .fromList(values.toList) - .map(nel => intersect(nel.map(singleSelect))) - } - - private def findItemsBase(q: Query.Fix, noteMaxLen: Int): Select = { + private def findItemsBase(q: Query.Fix, today: LocalDate, noteMaxLen: Int): Select = { val attachs = AttachCountTable("cta") val coll = q.account.collective @@ -169,9 +146,7 @@ object QItem { .leftJoin(pers1, pers1.pid === i.concPerson && pers1.cid === coll) .leftJoin(equip, equip.eid === i.concEquipment && equip.cid === coll), where( - i.cid === coll &&? q.itemIds.map(s => - Nel.fromList(s.toList).map(nel => i.id.in(nel)).getOrElse(i.id.isNull) - ) + i.cid === coll &&? q.query.map(qs => queryCondFromExpr(today, coll, qs)) && or( i.folder.isNull, i.folder.in(QFolder.findMemberFolderIds(q.account)) @@ -184,54 +159,17 @@ object QItem { ) } - def queryCondFromForm(coll: Ident, q: Query.QueryForm): Condition = - Condition.unit &&? - q.direction.map(d => i.incoming === d) &&? - q.name.map(n => i.name.like(QueryWildcard.lower(n))) &&? - Nel.fromList(q.states.toList).map(nel => i.state.in(nel)) &&? - q.allNames - .map(QueryWildcard.lower) - .map(n => - org.name.like(n) || - pers0.name.like(n) || - pers1.name.like(n) || - equip.name.like(n) || - i.name.like(n) || - i.notes.like(n) - ) &&? - q.corrPerson.map(p => pers0.pid === p) &&? - q.corrOrg.map(o => org.oid === o) &&? - q.concPerson.map(p => pers1.pid === p) &&? - q.concEquip.map(e => equip.eid === e) &&? - q.folder.map(fid => f.id === fid) &&? - q.dateFrom.map(d => coalesce(i.itemDate.s, i.created.s) >= d) &&? - q.dateTo.map(d => coalesce(i.itemDate.s, i.created.s) <= d) &&? - q.dueDateFrom.map(d => i.dueDate > d) &&? - q.dueDateTo.map(d => i.dueDate < d) &&? - q.source.map(n => i.source.like(QueryWildcard.lower(n))) &&? - q.itemIds.map(s => - Nel.fromList(s.toList).map(nel => i.id.in(nel)).getOrElse(i.id.isNull) - ) &&? - TagItemName - .itemsWithAllTagAndCategory(q.tagsInclude, q.tagCategoryIncl) - .map(subsel => i.id.in(subsel)) &&? - TagItemName - .itemsWithEitherTagOrCategory(q.tagsExclude, q.tagCategoryExcl) - .map(subsel => i.id.notIn(subsel)) &&? - findCustomFieldValuesForColl(coll, q.customValues) - .map(itemIds => i.id.in(itemIds)) - - def queryCondFromExpr(today: LocalDate, coll: Ident, q: ItemQuery): Condition = { + def queryCondFromExpr(today: LocalDate, coll: Ident, q: ItemQuery.Expr): Condition = { val tables = Tables(i, org, pers0, pers1, equip, f, a, m, AttachCountTable("cta")) - ItemQueryGenerator.fromExpr(today, tables, coll)(q.expr) + ItemQueryGenerator.fromExpr(today, tables, coll)(q) } def queryCondition(today: LocalDate, coll: Ident, cond: Query.QueryCond): Condition = cond match { - case fm: Query.QueryForm => - queryCondFromForm(coll, fm) - case expr: Query.QueryExpr => - queryCondFromExpr(today, coll, expr.q) + case Query.QueryExpr(Some(expr)) => + queryCondFromExpr(today, coll, expr) + case Query.QueryExpr(None) => + Condition.unit } def findItems( @@ -240,7 +178,7 @@ object QItem { maxNoteLen: Int, batch: Batch ): Stream[ConnectionIO, ListItem] = { - val sql = findItemsBase(q.fix, maxNoteLen) + val sql = findItemsBase(q.fix, today, maxNoteLen) .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond)) .limit(batch) .build @@ -263,7 +201,7 @@ object QItem { .innerJoin(i, i.id === ti.itemId) val tagCloud = - findItemsBase(q.fix, 0).unwrap + findItemsBase(q.fix, today, 0).unwrap .withSelect(select(tag.all).append(count(i.id).as("num"))) .changeFrom(_.prepend(tagFrom)) .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond)) @@ -281,7 +219,7 @@ object QItem { } def searchCountSummary(today: LocalDate)(q: Query): ConnectionIO[Int] = - findItemsBase(q.fix, 0).unwrap + findItemsBase(q.fix, today, 0).unwrap .withSelect(Nel.of(count(i.id).as("num"))) .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond)) .build @@ -290,7 +228,7 @@ object QItem { def searchFolderSummary(today: LocalDate)(q: Query): ConnectionIO[List[FolderCount]] = { val fu = RUser.as("fu") - findItemsBase(q.fix, 0).unwrap + findItemsBase(q.fix, today, 0).unwrap .withSelect(select(f.id, f.name, f.owner, fu.login).append(count(i.id).as("num"))) .changeFrom(_.innerJoin(fu, fu.uid === f.owner)) .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond)) @@ -307,7 +245,7 @@ object QItem { .innerJoin(i, i.id === cv.itemId) val base = - findItemsBase(q.fix, 0).unwrap + findItemsBase(q.fix, today, 0).unwrap .changeFrom(_.prepend(fieldJoin)) .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond)) .groupBy(GroupBy(cf.all)) @@ -359,6 +297,7 @@ object QItem { def findSelectedItems( q: Query, + today: LocalDate, maxNoteLen: Int, items: Set[SelectedItem] ): Stream[ConnectionIO, ListItem] = @@ -386,7 +325,7 @@ object QItem { ) ) - val from = findItemsBase(q.fix, maxNoteLen) + val from = findItemsBase(q.fix, today, maxNoteLen) .appendCte(cte) .appendSelect(Tids.weight.s) .changeFrom(_.innerJoin(Tids, Tids.itemId === i.id)) diff --git a/modules/store/src/main/scala/docspell/store/queries/Query.scala b/modules/store/src/main/scala/docspell/store/queries/Query.scala index 6be04a1e..5ebf6a21 100644 --- a/modules/store/src/main/scala/docspell/store/queries/Query.scala +++ b/modules/store/src/main/scala/docspell/store/queries/Query.scala @@ -26,12 +26,12 @@ object Query { case class Fix( account: AccountId, - itemIds: Option[Set[Ident]], + query: Option[ItemQuery.Expr], orderAsc: Option[RItem.Table => Column[_]] ) { def isEmpty: Boolean = - itemIds.isEmpty + query.isEmpty } sealed trait QueryCond { @@ -41,64 +41,17 @@ object Query { !isEmpty } - case class QueryForm( - name: Option[String], - states: Seq[ItemState], - direction: Option[Direction], - corrPerson: Option[Ident], - corrOrg: Option[Ident], - concPerson: Option[Ident], - concEquip: Option[Ident], - folder: Option[Ident], - tagsInclude: List[Ident], - tagsExclude: List[Ident], - tagCategoryIncl: List[String], - tagCategoryExcl: List[String], - dateFrom: Option[Timestamp], - dateTo: Option[Timestamp], - dueDateFrom: Option[Timestamp], - dueDateTo: Option[Timestamp], - allNames: Option[String], - itemIds: Option[Set[Ident]], - customValues: Seq[CustomValue], - source: Option[String] - ) extends QueryCond { - + case class QueryExpr(q: Option[ItemQuery.Expr]) extends QueryCond { def isEmpty: Boolean = - this == QueryForm.empty - } - object QueryForm { - val empty = - QueryForm( - None, - Seq.empty, - None, - None, - None, - None, - None, - None, - Nil, - Nil, - Nil, - Nil, - None, - None, - None, - None, - None, - None, - Seq.empty, - None - ) + q.isEmpty } - case class QueryExpr(q: ItemQuery) extends QueryCond { - def isEmpty: Boolean = - q.expr == ItemQuery.all.expr + object QueryExpr { + def apply(q: ItemQuery.Expr): QueryExpr = + QueryExpr(Some(q)) } - def empty(account: AccountId): Query = - Query(Fix(account, None, None), QueryForm.empty) + def all(account: AccountId): Query = + Query(Fix(account, None, None), QueryExpr(None)) } diff --git a/modules/webapp/src/main/elm/Api.elm b/modules/webapp/src/main/elm/Api.elm index b7869df7..968f8041 100644 --- a/modules/webapp/src/main/elm/Api.elm +++ b/modules/webapp/src/main/elm/Api.elm @@ -162,7 +162,6 @@ import Api.Model.ItemInsights exposing (ItemInsights) import Api.Model.ItemLightList exposing (ItemLightList) import Api.Model.ItemProposals exposing (ItemProposals) import Api.Model.ItemQuery exposing (ItemQuery) -import Api.Model.ItemSearch exposing (ItemSearch) import Api.Model.ItemUploadMeta exposing (ItemUploadMeta) import Api.Model.ItemsAndDate exposing (ItemsAndDate) import Api.Model.ItemsAndDirection exposing (ItemsAndDirection) diff --git a/modules/webapp/src/main/elm/Comp/NotificationForm.elm b/modules/webapp/src/main/elm/Comp/NotificationForm.elm index 81ebb11c..eec3e045 100644 --- a/modules/webapp/src/main/elm/Comp/NotificationForm.elm +++ b/modules/webapp/src/main/elm/Comp/NotificationForm.elm @@ -473,6 +473,14 @@ view2 extraClasses settings model = let dimmerSettings = Comp.YesNoDimmer.defaultSettings2 "Really delete this notification task?" + + startOnceBtn = + MB.SecondaryButton + { tagger = StartOnce + , label = "Start Once" + , title = "Start this task now" + , icon = Just "fa fa-play" + } in div [ class "flex flex-col md:relative" @@ -501,7 +509,8 @@ view2 extraClasses settings model = ] , end = if model.settings.id /= "" then - [ MB.DeleteButton + [ startOnceBtn + , MB.DeleteButton { tagger = RequestDelete , label = "Delete" , title = "Delete this task" @@ -510,7 +519,8 @@ view2 extraClasses settings model = ] else - [] + [ startOnceBtn + ] , rootClasses = "mb-4" } , div diff --git a/modules/webapp/src/main/elm/Comp/NotificationManage.elm b/modules/webapp/src/main/elm/Comp/NotificationManage.elm index eb88db3f..e1b2467a 100644 --- a/modules/webapp/src/main/elm/Comp/NotificationManage.elm +++ b/modules/webapp/src/main/elm/Comp/NotificationManage.elm @@ -221,6 +221,7 @@ view2 settings model = , ( S.successMessage, Maybe.map .success model.result == Just True ) , ( "hidden", model.result == Nothing ) ] + , class "mb-2" ] [ Maybe.map .message model.result |> Maybe.withDefault "" diff --git a/tools/export-files/export-files.sh b/tools/export-files/export-files.sh index 374faf9c..b11817eb 100755 --- a/tools/export-files/export-files.sh +++ b/tools/export-files/export-files.sh @@ -70,7 +70,7 @@ fi set -o errexit -o pipefail -o noclobber -o nounset LOGIN_URL="$BASE_URL/api/v1/open/auth/login" -SEARCH_URL="$BASE_URL/api/v1/sec/item/searchWithTags" +SEARCH_URL="$BASE_URL/api/v1/sec/item/search" INSIGHT_URL="$BASE_URL/api/v1/sec/collective/insights" DETAIL_URL="$BASE_URL/api/v1/sec/item" ATTACH_URL="$BASE_URL/api/v1/sec/attachment" @@ -108,11 +108,11 @@ mcurl() { errout "Login to Docspell." errout "Using url: $BASE_URL" -if [ -z "$DS_USER" ]; then +if [ -z "${DS_USER:-}" ]; then errout -n "Account: " read DS_USER fi -if [ -z "$DS_PASS" ]; then +if [ -z "${DS_PASS:-}" ]; then errout -n "Password: " read -s DS_PASS fi @@ -152,7 +152,7 @@ listItems() { OFFSET="${1:-0}" LIMIT="${2:-50}" errout "Get next items with offset=$OFFSET, limit=$LIMIT" - REQ="{\"offset\":$OFFSET, \"limit\":$LIMIT, \"tagsInclude\":[],\"tagsExclude\":[],\"tagCategoriesInclude\":[], \"tagCategoriesExclude\":[],\"customValues\":[],\"inbox\":false}" + REQ="{\"offset\":$OFFSET, \"limit\":$LIMIT, \"withDetails\":true, \"query\":\"\"}" mcurl -XPOST -H 'ContentType: application/json' -d "$REQ" "$SEARCH_URL" | "$JQ_CMD" -r '.groups[].items[]|.id' }