Fix obvious things and add search summary

modules/backend/src/main/scala/docspell/backend/ops/search/OSearch.scala

@@ -10,7 +10,7 @@ import java.time.LocalDate
 
 import cats.effect._
 import cats.syntax.all._
-import cats.~>
+import cats.{Functor, ~>}
 import fs2.Stream
 
 import docspell.backend.ops.OItemSearch.{ListItemWithTags, SearchSummary}
@@ -52,9 +52,23 @@ trait OSearch[F[_]] {
       fulltextQuery: Option[String]
   ): F[Vector[ListItemWithTags]]
 
+  /** Selects either `search` or `searchWithDetails`. For the former the items are filled
+    * with empty details.
+    */
+  final def searchSelect(
+      withDetails: Boolean,
+      maxNoteLen: Int,
+      today: LocalDate,
+      batch: Batch
+  )(
+      q: Query,
+      fulltextQuery: Option[String]
+  )(implicit F: Functor[F]): F[Vector[ListItemWithTags]] =
+    if (withDetails) searchWithDetails(maxNoteLen, today, batch)(q, fulltextQuery)
+    else search(maxNoteLen, today, batch)(q, fulltextQuery).map(_.map(_.toWithTags))
+
   /** Run multiple database calls with the give query to collect a summary. */
   def searchSummary(
-      mode: SearchMode,
       today: LocalDate
   )(q: Query, fulltextQuery: Option[String]): F[SearchSummary]
 
@@ -80,41 +94,49 @@ object OSearch {
           accountId: AccountId,
           mode: SearchMode,
           qs: String
-      ): QueryParseResult =
+      ): QueryParseResult = {
-        ItemQueryParser.parse(qs) match {
+        val validItemQuery =
-          case Right(iq) =>
+          mode match {
-            val validItemQuery =
+            case SearchMode.Trashed => ItemQuery.Expr.Trashed
-              mode match {
+            case SearchMode.Normal  => ItemQuery.Expr.ValidItemStates
-                case SearchMode.Trashed => ItemQuery.Expr.Trashed
+            case SearchMode.All     => ItemQuery.Expr.ValidItemsOrTrashed
-                case SearchMode.Normal  => ItemQuery.Expr.ValidItemStates
+          }
-                case SearchMode.All     => ItemQuery.Expr.ValidItemsOrTrashed
+
+        if (qs.trim.isEmpty) {
+          val qf = Query.Fix(accountId, Some(validItemQuery), None)
+          val qq = Query.QueryExpr(None)
+          val q = Query(qf, qq)
+          QueryParseResult.Success(q, None)
+        } else
+          ItemQueryParser.parse(qs) match {
+            case Right(iq) =>
+              FulltextExtract.findFulltext(iq.expr) match {
+                case FulltextExtract.Result.SuccessNoFulltext(expr) =>
+                  val qf = Query.Fix(accountId, Some(validItemQuery), None)
+                  val qq = Query.QueryExpr(expr)
+                  val q = Query(qf, qq)
+                  QueryParseResult.Success(q, None)
+
+                case FulltextExtract.Result.SuccessNoExpr(fts) =>
+                  val qf = Query.Fix(accountId, Some(validItemQuery), Option(_.byScore))
+                  val qq = Query.QueryExpr(None)
+                  val q = Query(qf, qq)
+                  QueryParseResult.Success(q, Some(fts))
+
+                case FulltextExtract.Result.SuccessBoth(expr, fts) =>
+                  val qf = Query.Fix(accountId, Some(validItemQuery), None)
+                  val qq = Query.QueryExpr(expr)
+                  val q = Query(qf, qq)
+                  QueryParseResult.Success(q, Some(fts))
+
+                case f: FulltextExtract.FailureResult =>
+                  QueryParseResult.FulltextMismatch(f)
               }
-            FulltextExtract.findFulltext(iq.expr) match {
-              case FulltextExtract.Result.SuccessNoFulltext(expr) =>
-                val qf = Query.Fix(accountId, Some(validItemQuery), None)
-                val qq = Query.QueryExpr(expr)
-                val q = Query(qf, qq)
-                QueryParseResult.Success(q, None)
 
-              case FulltextExtract.Result.SuccessNoExpr(fts) =>
+            case Left(err) =>
-                val qf = Query.Fix(accountId, Some(validItemQuery), Option(_.byScore))
+              QueryParseResult.ParseFailed(err).cast
-                val qq = Query.QueryExpr(None)
+          }
-                val q = Query(qf, qq)
+      }
-                QueryParseResult.Success(q, Some(fts))
-
-              case FulltextExtract.Result.SuccessBoth(expr, fts) =>
-                val qf = Query.Fix(accountId, Some(validItemQuery), None)
-                val qq = Query.QueryExpr(expr)
-                val q = Query(qf, qq)
-                QueryParseResult.Success(q, Some(fts))
-
-              case f: FulltextExtract.FailureResult =>
-                QueryParseResult.FulltextMismatch(f)
-            }
-
-          case Left(err) =>
-            QueryParseResult.ParseFailed(err).cast
-        }
 
       def search(maxNoteLen: Int, today: LocalDate, batch: Batch)(
           q: Query,
@@ -167,7 +189,6 @@ object OSearch {
         } yield resolved
 
       def searchSummary(
-          mode: SearchMode,
           today: LocalDate
       )(q: Query, fulltextQuery: Option[String]): F[SearchSummary] =
         fulltextQuery match {
@@ -194,7 +215,7 @@ object OSearch {
         store
           .transact(QFolder.getMemberFolders(account))
           .map(folders =>
-            FtsQuery(ftq, account.collective, batch.limit, batch.offset)
+            FtsQuery(ftq, account.collective, batch.limit, 0)
               .withFolders(folders)
           )
 

modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala

@@ -297,7 +297,7 @@ trait Conversions {
         relatedItems = i.relatedItems
       )
 
-  private def mkAttachmentLight(qa: QAttachmentLight): AttachmentLight =
+  def mkAttachmentLight(qa: QAttachmentLight): AttachmentLight =
     AttachmentLight(qa.id, qa.position, qa.name, qa.pageCount)
 
   def mkItemLightWithTags(i: OFulltext.FtsItemWithTags): ItemLight = {

modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala

@@ -41,10 +41,11 @@ object ItemRoutes {
       user: AuthToken
   ): HttpRoutes[F] = {
     val logger = docspell.logging.getLogger[F]
+    val searchPart = ItemSearchPart[F](backend, cfg, user)
     val dsl = new Http4sDsl[F] {}
     import dsl._
 
-    ItemSearchPart(backend, cfg, user) <+>
+    searchPart.routes <+>
       HttpRoutes.of {
         case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset(
               offset

modules/restserver/src/main/scala/docspell/restserver/routes/ItemSearchPart.scala

@@ -6,50 +6,206 @@
 
 package docspell.restserver.routes
 
-import cats.effect.Async
+import java.time.LocalDate
+
+import cats.effect._
+import cats.syntax.all._
 
 import docspell.backend.BackendApp
 import docspell.backend.auth.AuthToken
+import docspell.backend.ops.search.QueryParseResult
+import docspell.common.{SearchMode, Timestamp}
+import docspell.query.FulltextExtract
+import docspell.restapi.model._
 import docspell.restserver.Config
+import docspell.restserver.conv.Conversions
 import docspell.restserver.http4s.{QueryParam => QP}
+import docspell.store.qb.Batch
+import docspell.store.queries.ListItemWithTags
 
-import org.http4s.HttpRoutes
+import org.http4s.circe.CirceEntityCodec._
 import org.http4s.dsl.Http4sDsl
+import org.http4s.{HttpRoutes, Response}
+
+final class ItemSearchPart[F[_]: Async](
+    backend: BackendApp[F],
+    cfg: Config,
+    authToken: AuthToken
+) extends Http4sDsl[F] {
+
+  private[this] val logger = docspell.logging.getLogger[F]
+
+  def routes: HttpRoutes[F] =
+    if (!cfg.featureSearch2) HttpRoutes.empty
+    else
+      HttpRoutes.of {
+        case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset(
+              offset
+            ) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) =>
+          val userQuery =
+            ItemQuery(offset, limit, detailFlag, searchMode, q.getOrElse(""))
+
+          Timestamp
+            .current[F]
+            .map(_.toUtcDate)
+            .flatMap(search(userQuery, _))
+
+        case req @ POST -> Root / "search" =>
+          for {
+            userQuery <- req.as[ItemQuery]
+            today <- Timestamp.current[F]
+            resp <- search(userQuery, today.toUtcDate)
+          } yield resp
+
+        case GET -> Root / "searchStats" :? QP.Query(q) :? QP.SearchKind(searchMode) =>
+          val userQuery = ItemQuery(None, None, None, searchMode, q.getOrElse(""))
+          Timestamp
+            .current[F]
+            .map(_.toUtcDate)
+            .flatMap(searchStats(userQuery, _))
+
+        case req @ POST -> Root / "searchStats" =>
+          for {
+            userQuery <- req.as[ItemQuery]
+            today <- Timestamp.current[F].map(_.toUtcDate)
+            resp <- searchStats(userQuery, today)
+          } yield resp
+      }
+
+  def searchStats(userQuery: ItemQuery, today: LocalDate): F[Response[F]] = {
+    val mode = userQuery.searchMode.getOrElse(SearchMode.Normal)
+    parsedQuery(userQuery, mode)
+      .fold(
+        identity,
+        res =>
+          for {
+            summary <- backend.search.searchSummary(today)(res.q, res.ftq)
+            resp <- Ok(Conversions.mkSearchStats(summary))
+          } yield resp
+      )
+  }
+
+  def search(userQuery: ItemQuery, today: LocalDate): F[Response[F]] = {
+    val details = userQuery.withDetails.getOrElse(false)
+    val batch =
+      Batch(userQuery.offset.getOrElse(0), userQuery.limit.getOrElse(cfg.maxItemPageSize))
+        .restrictLimitTo(cfg.maxItemPageSize)
+    val limitCapped = userQuery.limit.exists(_ > cfg.maxItemPageSize)
+    val mode = userQuery.searchMode.getOrElse(SearchMode.Normal)
+
+    parsedQuery(userQuery, mode)
+      .fold(
+        identity,
+        res =>
+          for {
+            items <- backend.search
+              .searchSelect(details, cfg.maxNoteLength, today, batch)(
+                res.q,
+                res.ftq
+              )
+
+            // order is always by date unless q is empty and ftq is not
+            // TODO this is not obvious from the types and an impl detail.
+            ftsOrder = res.q.cond.isEmpty && res.ftq.isDefined
+
+            resp <- Ok(convert(items, batch, limitCapped, ftsOrder))
+          } yield resp
+      )
+  }
+
+  def parsedQuery(
+      userQuery: ItemQuery,
+      mode: SearchMode
+  ): Either[F[Response[F]], QueryParseResult.Success] =
+    backend.search.parseQueryString(authToken.account, mode, userQuery.query) match {
+      case s: QueryParseResult.Success =>
+        Right(s)
+
+      case QueryParseResult.ParseFailed(err) =>
+        Left(BadRequest(BasicResult(false, s"Invalid query: $err")))
+
+      case QueryParseResult.FulltextMismatch(FulltextExtract.Result.TooMany) =>
+        Left(
+          BadRequest(
+            BasicResult(false, "Only one fulltext search expression is allowed.")
+          )
+        )
+      case QueryParseResult.FulltextMismatch(
+            FulltextExtract.Result.UnsupportedPosition
+          ) =>
+        Left(
+          BadRequest(
+            BasicResult(
+              false,
+              "A fulltext search may only appear in the root and expression."
+            )
+          )
+        )
+    }
+
+  def convert(
+      items: Vector[ListItemWithTags],
+      batch: Batch,
+      capped: Boolean,
+      ftsOrder: Boolean
+  ): ItemLightList =
+    if (ftsOrder)
+      ItemLightList(
+        List(ItemLightGroup("Results", items.map(convertItem).toList)),
+        batch.limit,
+        batch.offset,
+        capped
+      )
+    else {
+      val groups = items.groupBy(ti => ti.item.date.toUtcDate.toString.substring(0, 7))
+
+      def mkGroup(g: (String, Vector[ListItemWithTags])): ItemLightGroup =
+        ItemLightGroup(g._1, g._2.map(convertItem).toList)
+
+      val gs =
+        groups.map(mkGroup).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0)
+
+      ItemLightList(gs, batch.limit, batch.offset, capped)
+    }
+
+  def convertItem(item: ListItemWithTags): ItemLight =
+    ItemLight(
+      id = item.item.id,
+      name = item.item.name,
+      state = item.item.state,
+      date = item.item.date,
+      dueDate = item.item.dueDate,
+      source = item.item.source,
+      direction = item.item.direction.name.some,
+      corrOrg = item.item.corrOrg.map(Conversions.mkIdName),
+      corrPerson = item.item.corrPerson.map(Conversions.mkIdName),
+      concPerson = item.item.concPerson.map(Conversions.mkIdName),
+      concEquipment = item.item.concEquip.map(Conversions.mkIdName),
+      folder = item.item.folder.map(Conversions.mkIdName),
+      attachments = item.attachments.map(Conversions.mkAttachmentLight),
+      tags = item.tags.map(Conversions.mkTag),
+      customfields = item.customfields.map(Conversions.mkItemFieldValue),
+      relatedItems = item.relatedItems,
+      notes = item.item.notes,
+      highlighting = item.item.decodeContext match {
+        case Some(Right(hlctx)) =>
+          hlctx.map(c => HighlightEntry(c.name, c.context))
+        case Some(Left(err)) =>
+          logger.asUnsafe.error(
+            s"Internal error: cannot decode highlight context '${item.item.context}': $err"
+          )
+          Nil
+        case None =>
+          Nil
+      }
+    )
+}
 
-@annotation.nowarn
 object ItemSearchPart {
-
   def apply[F[_]: Async](
       backend: BackendApp[F],
       cfg: Config,
-      authToken: AuthToken
+      token: AuthToken
-  ): HttpRoutes[F] =
+  ): ItemSearchPart[F] =
-    if (cfg.featureSearch2) routes(backend, cfg, authToken)
+    new ItemSearchPart[F](backend, cfg, token)
-    else HttpRoutes.empty
-
-  def routes[F[_]: Async](
-      backend: BackendApp[F],
-      cfg: Config,
-      authToken: AuthToken
-  ): HttpRoutes[F] = {
-    val logger = docspell.logging.getLogger[F]
-    val dsl = new Http4sDsl[F] {}
-    import dsl._
-
-    HttpRoutes.of {
-      case GET -> Root / "search" :? QP.Query(q) :? QP.Limit(limit) :? QP.Offset(
-            offset
-          ) :? QP.WithDetails(detailFlag) :? QP.SearchKind(searchMode) =>
-        ???
-
-      case req @ POST -> Root / "search" =>
-        ???
-
-      case GET -> Root / "searchStats" :? QP.Query(q) :? QP.SearchKind(searchMode) =>
-        ???
-
-      case req @ POST -> Root / "searchStats" =>
-        ???
-    }
-  }
 }

modules/store/src/main/scala/docspell/store/queries/FtsSupport.scala

@@ -23,7 +23,7 @@ trait FtsSupport {
           val tt = cteTable(ftst)
           select
             .appendCte(ftst.distinctCteSimple(tt.tableName))
-            .changeFrom(_.innerJoin(tt, itemTable.id === tt.id))
+            .changeFrom(_.prepend(from(itemTable).innerJoin(tt, itemTable.id === tt.id)))
         case None =>
           select
       }
@@ -37,7 +37,19 @@ trait FtsSupport {
           val tt = cteTable(ftst)
           select
             .appendCte(ftst.distinctCte(tt.tableName))
-            .changeFrom(_.innerJoin(tt, itemTable.id === tt.id))
+            .changeFrom(_.prepend(from(itemTable).innerJoin(tt, itemTable.id === tt.id)))
+        case None =>
+          select
+      }
+
+    def ftsCondition(
+        itemTable: RItem.Table,
+        ftsTable: Option[TempFtsTable.Table]
+    ): Select =
+      ftsTable match {
+        case Some(ftst) =>
+          val ftsIds = Select(ftst.id.s, from(ftst)).distinct
+          select.changeWhere(c => c && itemTable.id.in(ftsIds))
         case None =>
           select
       }

modules/store/src/main/scala/docspell/store/queries/ListItem.scala

@@ -29,7 +29,7 @@ case class ListItem(
 
   def decodeContext: Option[Either[String, List[ContextEntry]]] =
     context.map(_.trim).filter(_.nonEmpty).map { str =>
-      // This is a bit… well. The common denominator for the dbms used is string aggregation
+      // This is a bit…. The common denominator for the dbms used is string aggregation
       // when combining multiple matches. So the `ContextEntry` objects are concatenated and
       // separated by comma. TemplateFtsTable ensures than the single entries are all json
       // objects.
@@ -40,4 +40,7 @@ case class ListItem(
         .map(_.getMessage)
         .map(_.flatten)
     }
+
+  def toWithTags: ListItemWithTags =
+    ListItemWithTags(this, Nil, Nil, Nil, Nil)
 }

modules/store/src/main/scala/docspell/store/queries/QItem.scala

@@ -475,9 +475,9 @@ object QItem extends FtsSupport {
 
     val base =
       findItemsBase(q.fix, today, 0, None).unwrap
-        .joinFtsIdOnly(i, ftsTable)
         .changeFrom(_.prepend(fieldJoin))
         .changeWhere(c => c && queryCondition(today, q.fix.account.collective, q.cond))
+        .ftsCondition(i, ftsTable)
         .groupBy(GroupBy(cf.all))
 
     val basicFields = Nel.of(