From 383614f908a6334be43979c4f7ba74df9d87bda0 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sat, 20 Jun 2020 23:37:47 +0200 Subject: [PATCH] Allow updating single fields in solr --- .../main/scala/docspell/ftssolr/Field.scala | 29 +++++++----- .../scala/docspell/ftssolr/JsonCodec.scala | 46 +++++++++++++++---- .../scala/docspell/ftssolr/SetFields.scala | 5 ++ .../docspell/ftssolr/SolrFtsClient.scala | 2 +- .../scala/docspell/ftssolr/SolrSetup.scala | 4 +- .../scala/docspell/ftssolr/SolrUpdate.scala | 23 +++++++--- 6 files changed, 80 insertions(+), 29 deletions(-) create mode 100644 modules/fts-solr/src/main/scala/docspell/ftssolr/SetFields.scala diff --git a/modules/fts-solr/src/main/scala/docspell/ftssolr/Field.scala b/modules/fts-solr/src/main/scala/docspell/ftssolr/Field.scala index 1bf78304..256844ee 100644 --- a/modules/fts-solr/src/main/scala/docspell/ftssolr/Field.scala +++ b/modules/fts-solr/src/main/scala/docspell/ftssolr/Field.scala @@ -1,6 +1,7 @@ package docspell.ftssolr import io.circe._ +import docspell.common._ final class Field(val name: String) extends AnyVal { @@ -12,19 +13,25 @@ object Field { def apply(name: String): Field = new Field(name) - - val id = Field("id") - val itemId = Field("itemId") - val collectiveId = Field("collectiveId") - val attachmentId = Field("attachmentId") - val discriminator = Field("discriminator") + val id = Field("id") + val itemId = Field("itemId") + val collectiveId = Field("collectiveId") + val attachmentId = Field("attachmentId") + val discriminator = Field("discriminator") val attachmentName = Field("attachmentName") - val content = Field("content") - val content_de = Field("content_de") - val content_en = Field("content_en") - val itemName = Field("itemName") - val itemNotes = Field("itemNotes") + val content = Field("content") + val content_de = Field("content_de") + val content_en = Field("content_en") + val itemName = Field("itemName") + val itemNotes = Field("itemNotes") + def contentField(lang: Language): Field = + lang match { + case Language.German => + Field.content_de + case Language.English => + Field.content_en + } implicit val jsonEncoder: Encoder[Field] = Encoder.encodeString.contramap(_.name) diff --git a/modules/fts-solr/src/main/scala/docspell/ftssolr/JsonCodec.scala b/modules/fts-solr/src/main/scala/docspell/ftssolr/JsonCodec.scala index 42a0bd5c..c03db843 100644 --- a/modules/fts-solr/src/main/scala/docspell/ftssolr/JsonCodec.scala +++ b/modules/fts-solr/src/main/scala/docspell/ftssolr/JsonCodec.scala @@ -1,6 +1,7 @@ package docspell.ftssolr import io.circe._ +import io.circe.syntax._ import docspell.common._ import docspell.ftsclient._ @@ -12,15 +13,7 @@ trait JsonCodec { new Encoder[TextData.Attachment] { final def apply(td: TextData.Attachment): Json = { val cnt = - ( - td.lang match { - case Language.German => - Field.content_de.name - case Language.English => - Field.content_en.name - }, - Json.fromString(td.text.getOrElse("")) - ) + (Field.contentField(td.lang).name, Json.fromString(td.text.getOrElse(""))) Json.fromFields( cnt :: List( @@ -100,6 +93,41 @@ trait JsonCodec { new KeyDecoder[Ident] { override def apply(ident: String): Option[Ident] = Ident(ident).toOption } + + def setAttachmentEncoder(implicit + enc: Encoder[Ident] + ): Encoder[TextData.Attachment] = + new Encoder[TextData.Attachment] { + final def apply(td: TextData.Attachment): Json = { + val setter = List( + td.name.map(n => (Field.attachmentName.name, Map("set" -> n.asJson).asJson)), + td.text.map(txt => + (Field.contentField(td.lang).name, Map("set" -> txt.asJson).asJson) + ) + ).flatten + Json.fromFields( + (Field.id.name, enc(td.id)) :: setter + ) + } + } + + def setItemEncoder(implicit enc: Encoder[Ident]): Encoder[TextData.Item] = + new Encoder[TextData.Item] { + final def apply(td: TextData.Item): Json = { + val setter = List( + td.name.map(n => (Field.itemName.name, Map("set" -> n.asJson).asJson)), + td.notes.map(n => (Field.itemNotes.name, Map("set" -> n.asJson).asJson)) + ).flatten + + Json.fromFields( + (Field.id.name, enc(td.id)) :: setter + ) + } + } + + implicit def textDataEncoder: Encoder[SetFields] = + Encoder(_.td.fold(setAttachmentEncoder.apply, setItemEncoder.apply)) + } object JsonCodec extends JsonCodec diff --git a/modules/fts-solr/src/main/scala/docspell/ftssolr/SetFields.scala b/modules/fts-solr/src/main/scala/docspell/ftssolr/SetFields.scala new file mode 100644 index 00000000..af9b370e --- /dev/null +++ b/modules/fts-solr/src/main/scala/docspell/ftssolr/SetFields.scala @@ -0,0 +1,5 @@ +package docspell.ftssolr + +import docspell.ftsclient._ + +final case class SetFields(td: TextData) diff --git a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrFtsClient.scala b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrFtsClient.scala index 75e14ca8..c4142a0f 100644 --- a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrFtsClient.scala +++ b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrFtsClient.scala @@ -24,7 +24,7 @@ final class SolrFtsClient[F[_]: Effect]( (for { _ <- Stream.eval(logger.debug("Inserting data into index")) chunks <- data.chunks - res <- Stream.eval(solrUpdate.many(chunks.toList).attempt) + res <- Stream.eval(solrUpdate.add(chunks.toList).attempt) _ <- res match { case Right(()) => Stream.emit(()) case Left(ex) => diff --git a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala index 275f61d8..d07cacf0 100644 --- a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala +++ b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala @@ -82,7 +82,9 @@ object SolrSetup { } } - // Schema Commands + // Schema Commands: The structure is for conveniently creating the + // solr json. All fields must be stored, because of highlighting and + // single-updates only work when all fields are stored. case class AddField( name: Field, diff --git a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrUpdate.scala b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrUpdate.scala index 59c21eba..c8de4d09 100644 --- a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrUpdate.scala +++ b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrUpdate.scala @@ -14,9 +14,9 @@ import JsonCodec._ trait SolrUpdate[F[_]] { - def single(td: TextData): F[Unit] + def add(tds: List[TextData]): F[Unit] - def many(tds: List[TextData]): F[Unit] + def update(tds: List[TextData]): F[Unit] } @@ -33,15 +33,24 @@ object SolrUpdate { .withQueryParam("overwrite", "true") .withQueryParam("wt", "json") - def single(td: TextData): F[Unit] = { - val req = Method.POST(td.asJson, url) - client.expect[String](req).map(r => logger.debug(s"Req: $req Response: $r")) - } - def many(tds: List[TextData]): F[Unit] = { + def add(tds: List[TextData]): F[Unit] = { val req = Method.POST(tds.asJson, url) client.expect[String](req).map(r => logger.debug(s"Req: $req Response: $r")) } + + def update(tds: List[TextData]): F[Unit] = { + val req = Method.POST(tds.filter(minOneChange).map(SetFields).asJson, url) + client.expect[String](req).map(r => logger.debug(s"Req: $req Response: $r")) + } + + private val minOneChange: TextData => Boolean = + _ match { + case td: TextData.Attachment => + td.name.isDefined || td.text.isDefined + case td: TextData.Item => + td.name.isDefined || td.notes.isDefined + } } } }