Setup solr schema and index all data using a system task

The task runs on application start. It sets the schema using solr's
schema api and then indexes all data in the database. Each step is
memorized so that it is not executed again on subsequent starts.

modules/fts-solr/src/main/scala/docspell/ftssolr/Fields.scala

@@ -0,0 +1,19 @@
+package docspell.ftssolr
+
+object Fields {
+  val discriminator = "discriminator"
+  val id = "id"
+  val itemId = "itemId"
+  val collectiveId = "collectiveId"
+
+  object Attachment {
+    val attachmentId = "attachmentId"
+    val attachmentName = "attachmentName"
+    val content = "content"
+  }
+
+  object Item {
+    val itemName = "itemName"
+    val itemNotes = "itemNotes"
+  }
+}

modules/fts-solr/src/main/scala/docspell/ftssolr/JsonCodec.scala

@@ -3,30 +3,32 @@ package docspell.ftssolr
 import docspell.common._
 import docspell.ftsclient._
 import io.circe._
+import Fields.{Item, Attachment}
 
 trait JsonCodec {
 
   implicit def attachmentEncoder: Encoder[TextData.Attachment] =
     new Encoder[TextData.Attachment] {
       final def apply(td: TextData.Attachment): Json = Json.obj(
-        ("id", Ident.encodeIdent(td.id)),
-        ("item", Ident.encodeIdent(td.item)),
-        ("collective", Ident.encodeIdent(td.collective)),
-        ("attachmentName", Json.fromString(td.name.getOrElse(""))),
-        ("content", Json.fromString(td.text.getOrElse(""))),
-        ("discriminator", Json.fromString("attachment"))
+        (Fields.id, Ident.encodeIdent(td.id)),
+        (Fields.itemId, Ident.encodeIdent(td.item)),
+        (Fields.collectiveId, Ident.encodeIdent(td.collective)),
+        (Attachment.attachmentId, Ident.encodeIdent(td.attachId)),
+        (Attachment.attachmentName, Json.fromString(td.name.getOrElse(""))),
+        (Attachment.content, Json.fromString(td.text.getOrElse(""))),
+        (Fields.discriminator, Json.fromString("attachment"))
       )
     }
 
   implicit def itemEncoder: Encoder[TextData.Item] =
     new Encoder[TextData.Item] {
       final def apply(td: TextData.Item): Json = Json.obj(
-        ("id", Ident.encodeIdent(td.id)),
-        ("item", Ident.encodeIdent(td.item)),
-        ("collective", Ident.encodeIdent(td.collective)),
-        ("itemName", Json.fromString(td.name.getOrElse(""))),
-        ("itemNotes", Json.fromString(td.notes.getOrElse(""))),
-        ("discriminator", Json.fromString("item"))
+        (Fields.id, Ident.encodeIdent(td.id)),
+        (Fields.itemId, Ident.encodeIdent(td.item)),
+        (Fields.collectiveId, Ident.encodeIdent(td.collective)),
+        (Item.itemName, Json.fromString(td.name.getOrElse(""))),
+        (Item.itemNotes, Json.fromString(td.notes.getOrElse(""))),
+        (Fields.discriminator, Json.fromString("item"))
       )
     }
 

modules/fts-solr/src/main/scala/docspell/ftssolr/SolrConfig.scala

@@ -2,6 +2,6 @@ package docspell.ftssolr
 
 import docspell.common._
 
-final case class SolrConfig(url: LenientUri)
+final case class SolrConfig(url: LenientUri, commitWithin: Int)
 
 object SolrConfig {}

modules/fts-solr/src/main/scala/docspell/ftssolr/SolrFtsClient.scala

@@ -11,11 +11,12 @@ import docspell.ftsclient._
 import docspell.ftsclient.FtsBasicResult._
 
 final class SolrFtsClient[F[_]: Effect](
-    solrUpdate: SolrUpdate[F]
+    solrUpdate: SolrUpdate[F],
+    solrSetup: SolrSetup[F]
 ) extends FtsClient[F] {
 
   def initialize: F[Unit] =
-    ().pure[F]
+    solrSetup.setupSchema
 
   def searchBasic(q: FtsQuery): Stream[F, FtsBasicResult] =
     Stream.emits(
@@ -52,7 +53,7 @@ object SolrFtsClient {
       httpClient: Client[F]
   ): Resource[F, FtsClient[F]] =
     Resource.pure[F, FtsClient[F]](
-      new SolrFtsClient(SolrUpdate(cfg, httpClient))
+      new SolrFtsClient(SolrUpdate(cfg, httpClient), SolrSetup(cfg, httpClient))
     )
 
 }

modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala

@@ -0,0 +1,102 @@
+package docspell.ftssolr
+
+import cats.effect._
+import org.http4s._
+import cats.implicits._
+import org.http4s.client.Client
+import org.http4s.circe._
+import org.http4s.client.dsl.Http4sClientDsl
+import org.log4s.getLogger
+import _root_.io.circe.syntax._
+import _root_.io.circe._
+import _root_.io.circe.generic.semiauto._
+
+import Fields.{Attachment, Item}
+
+trait SolrSetup[F[_]] {
+
+  def setupSchema: F[Unit]
+
+}
+
+object SolrSetup {
+  private[this] val logger = getLogger
+
+  def apply[F[_]: ConcurrentEffect](cfg: SolrConfig, client: Client[F]): SolrSetup[F] = {
+    val dsl = new Http4sClientDsl[F] {}
+    import dsl._
+
+    new SolrSetup[F] {
+      val url = (Uri.unsafeFromString(cfg.url.asString) / "schema")
+        .withQueryParam("commitWithin", cfg.commitWithin.toString)
+
+      def setupSchema: F[Unit] = {
+        val cmds0 =
+          List(
+            Fields.id,
+            Fields.itemId,
+            Fields.collectiveId,
+            Fields.discriminator,
+            Attachment.attachmentId
+          )
+            .traverse(addStringField)
+        val cmds1 = List(
+          Attachment.attachmentName,
+          Attachment.content,
+          Item.itemName,
+          Item.itemNotes
+        )
+          .traverse(addTextField)
+
+        cmds0 *> cmds1 *> ().pure[F]
+      }
+
+      private def run(cmd: Json): F[Unit] = {
+        val req = Method.POST(cmd, url)
+        logger.debug(s"Running request $req: ${cmd.noSpaces}")
+        client.expect[String](req).map(r => logger.debug(s"Response: $r"))
+      }
+
+      private def addStringField(name: String): F[Unit] =
+        run(DeleteField.command(DeleteField(name))).attempt *>
+          run(AddField.command(AddField.string(name)))
+
+      private def addTextField(name: String): F[Unit] =
+        run(DeleteField.command(DeleteField(name))).attempt *>
+          run(AddField.command(AddField.text(name)))
+
+    }
+  }
+
+  // Schema Commands
+
+  case class AddField(
+      name: String,
+      `type`: String,
+      stored: Boolean,
+      indexed: Boolean,
+      multiValued: Boolean
+  )
+  object AddField {
+    implicit val encoder: Encoder[AddField] =
+      deriveEncoder[AddField]
+
+    def command(body: AddField): Json =
+      Map("add-field" -> body.asJson).asJson
+
+    def string(name: String): AddField =
+      AddField(name, "string", true, true, false)
+
+    def text(name: String): AddField =
+      AddField(name, "text_general", true, true, false)
+  }
+
+  case class DeleteField(name: String)
+  object DeleteField {
+    implicit val encoder: Encoder[DeleteField] =
+      deriveEncoder[DeleteField]
+
+    def command(body: DeleteField): Json =
+      Map("delete-field" -> body.asJson).asJson
+  }
+}

modules/fts-solr/src/main/scala/docspell/ftssolr/SolrUpdate.scala

@@ -29,7 +29,7 @@ object SolrUpdate {
 
     new SolrUpdate[F] {
       val url = (Uri.unsafeFromString(cfg.url.asString) / "update")
-        .withQueryParam("commitWithin", "1000")
+        .withQueryParam("commitWithin", cfg.commitWithin.toString)
         .withQueryParam("overwrite", "true")
         .withQueryParam("wt", "json")