TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-06-22 02:18:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add language to schema, extend fts-client

Browse Source
This commit is contained in:
Eike Kettner
2020-06-20 22:27:26 +02:00
parent 3576c45d1a
commit 1f4ff0d4c4
13 changed files with 145 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
modules/fts-solr/src/main/scala/docspell/ftssolr/Field.scala
View File

@ -13,13 +13,15 @@ object Field {
new Field(name)
val discriminator = Field("discriminator")
val id = Field("id")
val itemId = Field("itemId")
val collectiveId = Field("collectiveId")
val attachmentId = Field("attachmentId")
val discriminator = Field("discriminator")
val attachmentName = Field("attachmentName")
val content = Field("content")
val content_de = Field("content_de")
val content_en = Field("content_en")
val itemName = Field("itemName")
val itemNotes = Field("itemNotes")

47
modules/fts-solr/src/main/scala/docspell/ftssolr/JsonCodec.scala
View File

@ -1,6 +1,5 @@
package docspell.ftssolr
//import cats.implicits._
import io.circe._
import docspell.common._
import docspell.ftsclient._
@ -11,16 +10,30 @@ trait JsonCodec {
enc: Encoder[Ident]
): Encoder[TextData.Attachment] =
new Encoder[TextData.Attachment] {
final def apply(td: TextData.Attachment): Json =
Json.obj(
(Field.id.name, enc(td.id)),
(Field.itemId.name, enc(td.item)),
(Field.collectiveId.name, enc(td.collective)),
(Field.attachmentId.name, enc(td.attachId)),
(Field.attachmentName.name, Json.fromString(td.name.getOrElse(""))),
(Field.content.name, Json.fromString(td.text.getOrElse(""))),
(Field.discriminator.name, Json.fromString("attachment"))
final def apply(td: TextData.Attachment): Json = {
val cnt =
(
td.lang match {
case Language.German =>
Field.content_de.name
case Language.English =>
Field.content_en.name
},
Json.fromString(td.text.getOrElse(""))
)
Json.fromFields(
cnt :: List(
(Field.id.name, enc(td.id)),
(Field.itemId.name, enc(td.item)),
(Field.collectiveId.name, enc(td.collective)),
(Field.attachmentId.name, enc(td.attachId)),
(Field.attachmentName.name, Json.fromString(td.name.getOrElse(""))),
(Field.discriminator.name, Json.fromString("attachment"))
)
)
}
}
implicit def itemEncoder(implicit enc: Encoder[Ident]): Encoder[TextData.Item] =
@ -46,13 +59,13 @@ trait JsonCodec {
new Decoder[FtsResult] {
final def apply(c: HCursor): Decoder.Result[FtsResult] =
for {
qtime <- c.downField("responseHeader").get[Duration]("QTime")
count <- c.downField("response").get[Int]("numFound")
maxScore <- c.downField("response").get[Double]("maxScore")
results <- c.downField("response").get[List[FtsResult.ItemMatch]]("docs")
highligh <- c.get[Map[Ident, Map[String, List[String]]]]("highlighting")
highline = highligh.map(kv => kv._1 -> kv._2.values.flatten.toList)
} yield FtsResult(qtime, count, maxScore, highline, results)
qtime <- c.downField("responseHeader").get[Duration]("QTime")
count <- c.downField("response").get[Int]("numFound")
maxScore <- c.downField("response").get[Double]("maxScore")
results <- c.downField("response").get[List[FtsResult.ItemMatch]]("docs")
highlightng <- c.get[Map[Ident, Map[String, List[String]]]]("highlighting")
highlight = highlightng.map(kv => kv._1 -> kv._2.values.flatten.toList)
} yield FtsResult(qtime, count, maxScore, highlight, results)
}
implicit def decodeItemMatch: Decoder[FtsResult.ItemMatch] =

6
modules/fts-solr/src/main/scala/docspell/ftssolr/SolrFtsClient.scala
View File

@ -17,8 +17,8 @@ final class SolrFtsClient[F[_]: Effect](
def initialize: F[Unit] =
solrSetup.setupSchema
def searchBasic(q: FtsQuery): Stream[F, FtsResult] =
Stream.eval(solrQuery.query(q))
def search(q: FtsQuery): F[FtsResult] =
solrQuery.query(q)
def indexData(logger: Logger[F], data: Stream[F, TextData]): F[Unit] =
(for {
@ -32,6 +32,8 @@ final class SolrFtsClient[F[_]: Effect](
}
} yield ()).compile.drain
def updateIndex(logger: Logger[F], data: Stream[F, TextData]): F[Unit] = ???
}
object SolrFtsClient {

9
modules/fts-solr/src/main/scala/docspell/ftssolr/SolrQuery.scala
View File

@ -18,7 +18,14 @@ trait SolrQuery[F[_]] {
def query(q: FtsQuery): F[FtsResult] = {
val fq = QueryData(
List(Field.content, Field.itemName, Field.itemNotes, Field.attachmentName),
List(
Field.content,
Field.content_de,
Field.content_en,
Field.itemName,
Field.itemNotes,
Field.attachmentName
),
List(
Field.id,
Field.itemId,

34
modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala
View File

@ -10,6 +10,7 @@ import org.log4s.getLogger
import _root_.io.circe.syntax._
import _root_.io.circe._
import _root_.io.circe.generic.semiauto._
import docspell.common._
trait SolrSetup[F[_]] {
@ -44,9 +45,16 @@ object SolrSetup {
Field.itemName,
Field.itemNotes
)
.traverse(addTextField)
.traverse(addTextField(None))
cmds0 *> cmds1 *> ().pure[F]
val cntLang = Language.all.traverse {
case l @ Language.German =>
addTextField(l.some)(Field.content_de)
case l @ Language.English =>
addTextField(l.some)(Field.content_en)
}
cmds0 *> cmds1 *> cntLang *> ().pure[F]
}
private def run(cmd: Json): F[Unit] = {
@ -59,10 +67,18 @@ object SolrSetup {
run(DeleteField.command(DeleteField(field))).attempt *>
run(AddField.command(AddField.string(field)))
private def addTextField(field: Field): F[Unit] =
run(DeleteField.command(DeleteField(field))).attempt *>
run(AddField.command(AddField.text(field)))
private def addTextField(lang: Option[Language])(field: Field): F[Unit] =
lang match {
case None =>
run(DeleteField.command(DeleteField(field))).attempt *>
run(AddField.command(AddField.text(field)))
case Some(Language.German) =>
run(DeleteField.command(DeleteField(field))).attempt *>
run(AddField.command(AddField.textDE(field)))
case Some(Language.English) =>
run(DeleteField.command(DeleteField(field))).attempt *>
run(AddField.command(AddField.textEN(field)))
}
}
}
@ -87,6 +103,12 @@ object SolrSetup {
def text(field: Field): AddField =
AddField(field, "text_general", true, true, false)
def textDE(field: Field): AddField =
AddField(field, "text_de", true, true, false)
def textEN(field: Field): AddField =
AddField(field, "text_en", true, true, false)
}
case class DeleteField(name: Field)

6
modules/fts-solr/src/main/scala/docspell/ftssolr/SolrUpdate.scala
View File

@ -35,14 +35,12 @@ object SolrUpdate {
def single(td: TextData): F[Unit] = {
val req = Method.POST(td.asJson, url)
logger.debug(s"Running request $req")
client.expect[String](req).map(r => logger.debug(s"Response: $r"))
client.expect[String](req).map(r => logger.debug(s"Req: $req Response: $r"))
}
def many(tds: List[TextData]): F[Unit] = {
val req = Method.POST(tds.asJson, url)
logger.debug(s"Running request $req")
client.expect[String](req).map(r => logger.debug(s"Response: $r"))
client.expect[String](req).map(r => logger.debug(s"Req: $req Response: $r"))
}
}
}