add item cache

extract_items.py

@@ -0,0 +1,56 @@
+import json
+import requests
+
+# NOTE: it does not cover all items, just workaround to extract most gear pieces from patches
+MIN_ILVL = 580
+MAX_ILVL = 605
+
+TOME = (
+    'radiant',
+)
+SAVAGE = (
+    'asphodelos',
+)
+
+
+payload = {
+    'queries': [
+        {
+            'slots': []
+        },
+        {
+            'jobs': [],
+            'minItemLevel': 580,
+            'maxItemLevel': 605
+        }
+    ],
+    'existing': []
+}
+# it does not support application/json
+r = requests.post('https://ffxiv.ariyala.com/items.app', data=json.dumps(payload))
+r.raise_for_status()
+
+result = []
+
+for item in r.json():
+    item_id = item['itemID']
+    source_dict = item['source']
+    name = item['name']['en']
+    if 'crafting' in source_dict:
+        source = 'Crafted'
+    elif 'gathering' in source_dict:
+        continue  # some random shit
+    elif 'purchase' in source_dict:
+        if any(tome in name.lower() for tome in TOME):
+            source = 'Tome'
+        elif any(savage in name.lower() for savage in SAVAGE):
+            source = 'Savage'
+        else:
+            source = None
+            continue
+    else:
+        raise RuntimeError(f'Unknown source {source_dict}')
+    result.append({'id': item_id, 'source': source, 'name': name})
+
+output = {'cached-items': result}
+print(json.dumps(output, indent=4, sort_keys=True))

src/main/resources/reference.conf

@@ -1,5 +1,8 @@
 me.arcanis.ffxivbis {
+
   bis-provider {
+    include "item_data.json"
+
     # xivapi base url, string, required
     xivapi-url = "https://xivapi.com"
     # xivapi developer key, string, optional

src/main/scala/me/arcanis/ffxivbis/http/RootEndpoint.scala

@@ -49,28 +49,26 @@ class RootEndpoint(system: ActorSystem[Nothing], storage: ActorRef[Message], pro
 
   def route: Route =
     withHttpLog {
-      apiRoute ~ htmlRoute ~ swagger.routes ~ swaggerUIRoute
+      ignoreTrailingSlash {
+        apiRoute ~ htmlRoute ~ swagger.routes ~ swaggerUIRoute
+      }
     }
 
   private def apiRoute: Route =
-    ignoreTrailingSlash {
+    pathPrefix("api") {
-      pathPrefix("api") {
+      pathPrefix(Segment) {
-        pathPrefix(Segment) {
+        case "v1" => rootApiV1Endpoint.route
-          case "v1" => rootApiV1Endpoint.route
+        case _ => reject
-          case _ => reject
-        }
       }
     }
 
   private def htmlRoute: Route =
-    ignoreTrailingSlash {
+    pathPrefix("static") {
-      pathPrefix("static") {
+      getFromResourceDirectory("static")
-        getFromResourceDirectory("static")
+    } ~ rootView.route
-      } ~ rootView.route
-    }
 
   private def swaggerUIRoute: Route =
     path("swagger") {
-      getFromResource("swagger/index.html")
+      getFromResource("html/swagger.html")
-    } ~ getFromResourceDirectory("swagger")
+    }
 }

src/main/scala/me/arcanis/ffxivbis/service/bis/XivApi.scala

@@ -13,6 +13,7 @@ import me.arcanis.ffxivbis.models.PieceType
 import spray.json._
 
 import scala.concurrent.{ExecutionContext, Future}
+import scala.jdk.CollectionConverters._
 import scala.util.Try
 
 trait XivApi extends RequestExecutor {
@@ -21,7 +22,26 @@ trait XivApi extends RequestExecutor {
   private val xivapiUrl = config.getString("me.arcanis.ffxivbis.bis-provider.xivapi-url")
   private val xivapiKey = Try(config.getString("me.arcanis.ffxivbis.bis-provider.xivapi-key")).toOption
 
+  private val preloadedItems: Map[Long, PieceType.PieceType] =
+    config
+      .getConfigList("me.arcanis.ffxivbis.bis-provider.cached-items")
+      .asScala
+      .map { item =>
+        item.getLong("id") -> PieceType.withName(item.getString("source"))
+      }
+      .toMap
+
   def getPieceType(itemIds: Seq[Long]): Future[Map[Long, PieceType.PieceType]] = {
+    val (local, remote) = itemIds.foldLeft((Map.empty[Long, PieceType.PieceType], Seq.empty[Long])) {
+      case ((l, r), id) =>
+        if (preloadedItems.contains(id)) (l.updated(id, preloadedItems(id)), r)
+        else (l, r :+ id)
+    }
+    if (remote.isEmpty) Future.successful(local)
+    else remotePieceType(remote).map(_ ++ local)
+  }
+
+  private def remotePieceType(itemIds: Seq[Long]): Future[Map[Long, PieceType.PieceType]] = {
     val uriForItems = Uri(xivapiUrl)
       .withPath(Uri.Path / "item")
       .withQuery(
@@ -108,7 +128,7 @@ object XivApi {
         val pieceType =
           if (index == "crafted" && shopId == -1L) PieceType.Crafted
           else
-            Try(shopMap(shopId).fields(s"ItemCost$index").asJsObject).toOption
+            Try(shopMap(shopId).fields(s"ItemCost$index").asJsObject)
               .getOrElse(throw new Exception(s"${shopMap(shopId).fields(s"ItemCost$index")}, $index"))
               .getFields("IsUnique", "StackSize") match {
               case Seq(JsNumber(isUnique), JsNumber(stackSize)) =>