etro support

src/main/resources/reference.conf

@@ -1,7 +1,7 @@
 me.arcanis.ffxivbis {
-  ariyala {
-    # ariyala base url, string, required
-    ariyala-url = "https://ffxiv.ariyala.com"
+  bis-provider {
+    # either use ariyala or etro
+    use-etro = no
     # xivapi base url, string, required
     xivapi-url = "https://xivapi.com"
     # xivapi developer key, string, optional

src/main/scala/me/arcanis/ffxivbis/Application.scala

@@ -13,8 +13,9 @@ import akka.http.scaladsl.Http
 import akka.stream.ActorMaterializer
 import com.typesafe.scalalogging.StrictLogging
 import me.arcanis.ffxivbis.http.RootEndpoint
+import me.arcanis.ffxivbis.service.bis.BisProvider
 import me.arcanis.ffxivbis.service.impl.DatabaseImpl
-import me.arcanis.ffxivbis.service.{Ariyala, PartyService}
+import me.arcanis.ffxivbis.service.PartyService
 import me.arcanis.ffxivbis.storage.Migration
 
 import scala.concurrent.duration.Duration
@@ -33,10 +34,12 @@ class Application extends Actor with StrictLogging {
 
   Migration(config).onComplete {
     case Success(_) =>
-      val ariyala = context.system.actorOf(Ariyala.props, "ariyala")
+      val useEtro = config.getBoolean("me.arcanis.ffxivbis.bis-provider.use-etro")
+
+      val bisProvider = context.system.actorOf(BisProvider.props(useEtro), "bis-provider")
       val storage = context.system.actorOf(DatabaseImpl.props, "storage")
       val party = context.system.actorOf(PartyService.props(storage), "party")
-      val http = new RootEndpoint(context.system, party, ariyala)
+      val http = new RootEndpoint(context.system, party, bisProvider)
 
       logger.info(s"start server at $host:$port")
       val bind = Http()(context.system).bindAndHandle(http.route, host, port)

src/main/scala/me/arcanis/ffxivbis/http/BiSHelper.scala

@@ -17,7 +17,7 @@ import me.arcanis.ffxivbis.service.impl.DatabaseBiSHandler
 
 import scala.concurrent.{ExecutionContext, Future}
 
-trait BiSHelper extends AriyalaHelper {
+trait BiSHelper extends BisProviderHelper {
 
   def storage: ActorRef
 

src/main/scala/me/arcanis/ffxivbis/http/BisProviderHelper.scala

@@ -12,15 +12,15 @@ import akka.actor.ActorRef
 import akka.pattern.ask
 import akka.util.Timeout
 import me.arcanis.ffxivbis.models.{BiS, Job}
-import me.arcanis.ffxivbis.service.Ariyala
+import me.arcanis.ffxivbis.service.bis.BisProvider
 
 import scala.concurrent.{ExecutionContext, Future}
 
-trait AriyalaHelper {
+trait BisProviderHelper {
 
   def ariyala: ActorRef
 
   def downloadBiS(link: String, job: Job.Job)
                  (implicit executionContext: ExecutionContext, timeout: Timeout): Future[BiS] =
-    (ariyala ? Ariyala.GetBiS(link, job)).mapTo[BiS]
+    (ariyala ? BisProvider.GetBiS(link, job)).mapTo[BiS]
 }

src/main/scala/me/arcanis/ffxivbis/http/PlayerHelper.scala

@@ -17,7 +17,7 @@ import me.arcanis.ffxivbis.service.impl.{DatabaseBiSHandler, DatabasePartyHandle
 
 import scala.concurrent.{ExecutionContext, Future}
 
-trait PlayerHelper extends AriyalaHelper {
+trait PlayerHelper extends BisProviderHelper {
 
   def storage: ActorRef
 

src/main/scala/me/arcanis/ffxivbis/models/Piece.scala

@@ -106,7 +106,7 @@ object Piece {
       case "feet" => Feet(pieceType, job)
       case "ears" => Ears(pieceType, job)
       case "neck" => Neck(pieceType, job)
-      case "wrist" => Wrist(pieceType, job)
+      case "wrist" | "wrists" => Wrist(pieceType, job)
       case ring @ ("ring" | "left ring" | "right ring") => Ring(pieceType, job, ring)
       case "accessory upgrade" => AccessoryUpgrade
       case "body upgrade" => BodyUpgrade

src/main/scala/me/arcanis/ffxivbis/service/Ariyala.scala

@@ -1,204 +0,0 @@
-/*
- * Copyright (c) 2019 Evgeniy Alekseev.
- *
- * This file is part of ffxivbis
- * (see https://github.com/arcan1s/ffxivbis).
- *
- * License: 3-clause BSD, see https://opensource.org/licenses/BSD-3-Clause
- */
-package me.arcanis.ffxivbis.service
-
-import java.nio.file.Paths
-
-import akka.actor.{Actor, Props}
-import akka.http.scaladsl.Http
-import akka.http.scaladsl.model._
-import akka.pattern.pipe
-import akka.stream.ActorMaterializer
-import akka.stream.scaladsl.{Keep, Sink}
-import akka.util.ByteString
-import com.typesafe.scalalogging.StrictLogging
-import me.arcanis.ffxivbis.models.{BiS, Job, Piece, PieceType}
-import spray.json._
-
-import scala.concurrent.{ExecutionContext, Future}
-import scala.util.Try
-
-class Ariyala extends Actor with StrictLogging {
-  import Ariyala._
-
-  private val settings = context.system.settings.config
-  private val ariyalaUrl = settings.getString("me.arcanis.ffxivbis.ariyala.ariyala-url")
-  private val xivapiUrl = settings.getString("me.arcanis.ffxivbis.ariyala.xivapi-url")
-  private val xivapiKey = Try(settings.getString("me.arcanis.ffxivbis.ariyala.xivapi-key")).toOption
-
-  private val http = Http()(context.system)
-  implicit private val materializer: ActorMaterializer = ActorMaterializer()
-  implicit private val executionContext: ExecutionContext =
-    context.system.dispatchers.lookup("me.arcanis.ffxivbis.default-dispatcher")
-
-  override def receive: Receive = {
-    case GetBiS(link, job) =>
-      val client = sender()
-      get(link, job).map(BiS(_)).pipeTo(client)
-  }
-
-  override def postStop(): Unit = {
-    http.shutdownAllConnectionPools()
-    super.postStop()
-  }
-
-  private def get(link: String, job: Job.Job): Future[Seq[Piece]] = {
-    val id = Paths.get(link).normalize.getFileName.toString
-    val uri = Uri(ariyalaUrl)
-      .withPath(Uri.Path / "store.app")
-      .withQuery(Uri.Query(Map("identifier" -> id)))
-
-    sendRequest(uri, Ariyala.parseAriyalaJsonToPieces(job, getPieceType))
-  }
-
-  private def getPieceType(itemIds: Seq[Long]): Future[Map[Long, PieceType.PieceType]] = {
-    val uriForItems = Uri(xivapiUrl)
-      .withPath(Uri.Path / "item")
-      .withQuery(Uri.Query(Map(
-        "columns" -> Seq("ID", "GameContentLinks").mkString(","),
-        "ids" -> itemIds.mkString(","),
-        "private_key" -> xivapiKey.getOrElse("")
-      )))
-
-    sendRequest(uriForItems, Ariyala.parseXivapiJsonToShop).flatMap { shops =>
-      val shopIds = shops.values.map(_._2).toSet
-      val columns = shops.values.map(pair => s"ItemCost${pair._1}").toSet
-      val uriForShops = Uri(xivapiUrl)
-        .withPath(Uri.Path / "specialshop")
-        .withQuery(Uri.Query(Map(
-          "columns" -> (columns + "ID").mkString(","),
-          "ids" -> shopIds.mkString(","),
-          "private_key" -> xivapiKey.getOrElse("")
-        )))
-
-      sendRequest(uriForShops, Ariyala.parseXivapiJsonToType(shops))
-    }
-  }
-
-  private def sendRequest[T](uri: Uri, parser: JsObject => Future[T]): Future[T] =
-    http.singleRequest(HttpRequest(uri = uri)).map {
-      case HttpResponse(status, _, entity, _) if status.isSuccess() =>
-        entity.dataBytes
-          .fold(ByteString.empty)(_ ++ _)
-          .map(_.utf8String)
-          .map(result => parser(result.parseJson.asJsObject))
-          .toMat(Sink.head)(Keep.right)
-          .run().flatten
-      case other => Future.failed(new Error(s"Invalid response from server $other"))
-    }.flatten
-}
-
-object Ariyala {
-  def props: Props = Props(new Ariyala)
-
-  case class GetBiS(link: String, job: Job.Job)
-
-  private def parseAriyalaJson(job: Job.Job)(js: JsObject)
-                              (implicit executionContext: ExecutionContext): Future[Map[String, Long]] =
-    Future {
-      val apiJob = js.fields.get("content") match {
-        case Some(JsString(value)) => value
-        case other => throw deserializationError(s"Invalid job name $other")
-      }
-      js.fields.get("datasets") match {
-        case Some(datasets: JsObject) =>
-          val fields = datasets.fields
-          fields.getOrElse(apiJob, fields(job.toString)).asJsObject
-            .fields("normal").asJsObject
-            .fields("items").asJsObject
-            .fields.foldLeft(Map.empty[String, Long]) {
-            case (acc, (key, JsNumber(id))) => remapKey(key).map(k => acc + (k -> id.toLong)).getOrElse(acc)
-            case (acc, _) => acc
-          }
-        case other => throw deserializationError(s"Invalid json $other")
-      }
-    }
-
-  private def parseAriyalaJsonToPieces(job: Job.Job, pieceTypes: Seq[Long] => Future[Map[Long, PieceType.PieceType]])
-                                      (js: JsObject)
-                                      (implicit executionContext: ExecutionContext): Future[Seq[Piece]] =
-    parseAriyalaJson(job)(js).flatMap { pieces =>
-      pieceTypes(pieces.values.toSeq).map { types =>
-        pieces.view.mapValues(types).map {
-          case (piece, pieceType) => Piece(piece, pieceType, job)
-        }.toSeq
-      }
-    }
-
-  private def parseXivapiJsonToShop(js: JsObject)
-                                   (implicit executionContext: ExecutionContext): Future[Map[Long, (String, Long)]] = {
-    def extractTraderId(js: JsObject) = {
-      js.fields
-        .get("Recipe").map(_ => "crafted" -> -1L) // you can craft this item
-        .orElse {  // lets try shop items
-          js.fields("SpecialShop").asJsObject
-            .fields.collectFirst {
-            case (shopName, JsArray(array)) if shopName.startsWith("ItemReceive") =>
-              val shopId = array.head match {
-                case JsNumber(id) => id.toLong
-                case other => throw deserializationError(s"Could not parse $other")
-              }
-              shopName.replace("ItemReceive", "") -> shopId
-          }
-        }.getOrElse(throw deserializationError(s"Could not parse $js"))
-    }
-
-    Future {
-      js.fields("Results") match {
-        case array: JsArray =>
-          array.elements.map(_.asJsObject.getFields("ID", "GameContentLinks") match {
-            case Seq(JsNumber(id), shop) => id.toLong -> extractTraderId(shop.asJsObject)
-            case other => throw deserializationError(s"Could not parse $other")
-          }).toMap
-        case other => throw deserializationError(s"Could not parse $other")
-      }
-    }
-  }
-
-  private def parseXivapiJsonToType(shops: Map[Long, (String, Long)])(js: JsObject)
-                                   (implicit executionContext: ExecutionContext): Future[Map[Long, PieceType.PieceType]] =
-    Future {
-      val shopMap = js.fields("Results") match {
-        case array: JsArray =>
-          array.elements.map { shop =>
-            shop.asJsObject.fields("ID") match {
-              case JsNumber(id) => id.toLong -> shop.asJsObject
-              case other => throw deserializationError(s"Could not parse $other")
-            }
-          }.toMap
-        case other => throw deserializationError(s"Could not parse $other")
-      }
-
-      shops.map { case (itemId, (index, shopId)) =>
-        val pieceType =
-          if (index == "crafted" && shopId == -1) PieceType.Crafted
-          else {
-            Try(shopMap(shopId).fields(s"ItemCost$index").asJsObject)
-              .toOption
-              .getOrElse(throw new Exception(s"${shopMap(shopId).fields(s"ItemCost$index")}, $index"))
-              .getFields("IsUnique", "StackSize") match {
-              case Seq(JsNumber(isUnique), JsNumber(stackSize)) =>
-                if (isUnique == 1 || stackSize.toLong == 2000) PieceType.Tome // either upgraded gear or tomes found
-                else PieceType.Savage
-              case other => throw deserializationError(s"Could not parse $other")
-            }
-          }
-        itemId -> pieceType
-      }
-    }
-
-  private def remapKey(key: String): Option[String] = key match {
-    case "mainhand" => Some("weapon")
-    case "chest" => Some("body")
-    case "ringLeft" => Some("left ring")
-    case "ringRight" => Some("right ring")
-    case "head" | "hands" | "waist" | "legs" | "feet" | "ears" | "neck" | "wrist" => Some(key)
-    case _ => None
-  }
-}

src/main/scala/me/arcanis/ffxivbis/service/bis/Ariyala.scala

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2019 Evgeniy Alekseev.
+ *
+ * This file is part of ffxivbis
+ * (see https://github.com/arcan1s/ffxivbis).
+ *
+ * License: 3-clause BSD, see https://opensource.org/licenses/BSD-3-Clause
+ */
+package me.arcanis.ffxivbis.service.bis
+
+import akka.http.scaladsl.model.Uri
+import me.arcanis.ffxivbis.models.Job
+import spray.json.{JsNumber, JsObject, JsString, deserializationError}
+
+import scala.concurrent.{ExecutionContext, Future}
+
+trait Ariyala {
+
+  def idParser(job: Job.Job, js: JsObject)
+              (implicit executionContext: ExecutionContext): Future[Map[String, Long]] =
+    Future {
+      val apiJob = js.fields.get("content") match {
+        case Some(JsString(value)) => value
+        case other => throw deserializationError(s"Invalid job name $other")
+      }
+      js.fields.get("datasets") match {
+        case Some(datasets: JsObject) =>
+          val fields = datasets.fields
+          fields.getOrElse(apiJob, fields(job.toString)).asJsObject
+            .fields("normal").asJsObject
+            .fields("items").asJsObject
+            .fields.foldLeft(Map.empty[String, Long]) {
+            case (acc, (key, JsNumber(id))) => BisProvider.remapKey(key).map(k => acc + (k -> id.toLong)).getOrElse(acc)
+            case (acc, _) => acc
+          }
+        case other => throw deserializationError(s"Invalid json $other")
+      }
+    }
+
+  def uri(root: Uri, id: String): Uri =
+    root
+      .withPath(Uri.Path / "store.app")
+      .withQuery(Uri.Query(Map("identifier" -> id)))
+}

src/main/scala/me/arcanis/ffxivbis/service/bis/BisProvider.scala

@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2019 Evgeniy Alekseev.
+ *
+ * This file is part of ffxivbis
+ * (see https://github.com/arcan1s/ffxivbis).
+ *
+ * License: 3-clause BSD, see https://opensource.org/licenses/BSD-3-Clause
+ */
+package me.arcanis.ffxivbis.service.bis
+
+import java.nio.file.Paths
+
+import akka.actor.{Actor, Props}
+import akka.http.scaladsl.model._
+import akka.pattern.pipe
+import com.typesafe.scalalogging.StrictLogging
+import me.arcanis.ffxivbis.models.{BiS, Job, Piece, PieceType}
+import spray.json._
+
+import scala.concurrent.{ExecutionContext, Future}
+
+abstract class BisProvider extends Actor with XivApi with StrictLogging {
+
+  def idParser(job: Job.Job, js: JsObject)
+              (implicit executionContext: ExecutionContext): Future[Map[String, Long]]
+  def uri(root: Uri, id: String): Uri
+
+  override def receive: Receive = {
+    case BisProvider.GetBiS(link, job) =>
+      val client = sender()
+      get(link, job).map(BiS(_)).pipeTo(client)
+  }
+
+  override def postStop(): Unit = {
+    shutdown()
+    super.postStop()
+  }
+
+  private def get(link: String, job: Job.Job): Future[Seq[Piece]] = {
+    val url = Uri(link)
+    val id = Paths.get(link).normalize.getFileName.toString
+
+    sendRequest(uri(url, id), BisProvider.parseBisJsonToPieces(job, idParser, getPieceType))
+  }
+}
+
+object BisProvider {
+
+  def props(useEtro: Boolean): Props =
+    if (useEtro) Props(new BisProvider with Etro)
+    else Props(new BisProvider with Ariyala)
+
+  case class GetBiS(link: String, job: Job.Job)
+
+  private def parseBisJsonToPieces(job: Job.Job,
+                                   idParser: (Job.Job, JsObject) => Future[Map[String, Long]],
+                                   pieceTypes: Seq[Long] => Future[Map[Long, PieceType.PieceType]])
+                                   (js: JsObject)
+                                   (implicit executionContext: ExecutionContext): Future[Seq[Piece]] =
+    idParser(job, js).flatMap { pieces =>
+      pieceTypes(pieces.values.toSeq).map { types =>
+        pieces.view.mapValues(types).map {
+          case (piece, pieceType) => Piece(piece, pieceType, job)
+        }.toSeq
+      }
+    }
+
+  def remapKey(key: String): Option[String] = key match {
+    case "mainhand" => Some("weapon")
+    case "chest" => Some("body")
+    case "ringLeft" | "fingerL" => Some("left ring")
+    case "ringRight" | "fingerR" => Some("right ring")
+    case "weapon" | "head" | "body" | "hands" | "waist" | "legs" | "feet" | "ears" | "neck" | "wrist" | "wrists" => Some(key)
+    case _ => None
+  }
+}

src/main/scala/me/arcanis/ffxivbis/service/bis/Etro.scala

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2019 Evgeniy Alekseev.
+ *
+ * This file is part of ffxivbis
+ * (see https://github.com/arcan1s/ffxivbis).
+ *
+ * License: 3-clause BSD, see https://opensource.org/licenses/BSD-3-Clause
+ */
+package me.arcanis.ffxivbis.service.bis
+
+import akka.http.scaladsl.model.Uri
+import me.arcanis.ffxivbis.models.Job
+import spray.json.{JsNumber, JsObject}
+
+import scala.concurrent.{ExecutionContext, Future}
+
+trait Etro {
+
+  def idParser(job: Job.Job, js: JsObject)
+              (implicit executionContext: ExecutionContext): Future[Map[String, Long]] =
+    Future {
+      js.fields.foldLeft(Map.empty[String, Long]) {
+        case (acc, (key, JsNumber(id))) => BisProvider.remapKey(key).map(k => acc + (k -> id.toLong)).getOrElse(acc)
+        case (acc, _) => acc
+      }
+    }
+
+  def uri(root: Uri, id: String): Uri =
+    root.withPath(Uri.Path / "api" / "gearsets" / id)
+}

src/main/scala/me/arcanis/ffxivbis/service/bis/RequestExecutor.scala

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2019 Evgeniy Alekseev.
+ *
+ * This file is part of ffxivbis
+ * (see https://github.com/arcan1s/ffxivbis).
+ *
+ * License: 3-clause BSD, see https://opensource.org/licenses/BSD-3-Clause
+ */
+package me.arcanis.ffxivbis.service.bis
+
+import akka.actor.ActorContext
+import akka.http.scaladsl.Http
+import akka.http.scaladsl.model.headers.Location
+import akka.http.scaladsl.model.{HttpRequest, HttpResponse, Uri}
+import akka.stream.ActorMaterializer
+import akka.stream.scaladsl.{Keep, Sink}
+import akka.util.ByteString
+import spray.json._
+
+import scala.concurrent.{ExecutionContext, Future}
+
+trait RequestExecutor {
+
+  def context: ActorContext
+
+  private val http = Http()(context.system)
+  implicit val materializer: ActorMaterializer = ActorMaterializer()(context)
+  implicit val executionContext: ExecutionContext =
+    context.system.dispatchers.lookup("me.arcanis.ffxivbis.default-dispatcher")
+
+  def sendRequest[T](uri: Uri, parser: JsObject => Future[T]): Future[T] =
+    http.singleRequest(HttpRequest(uri = uri)).map {
+      case r: HttpResponse if r.status.isRedirection() =>
+        val location = r.header[Location].get.uri
+        sendRequest(uri.withPath(location.path), parser)
+      case HttpResponse(status, _, entity, _) if status.isSuccess() =>
+        entity.dataBytes
+          .fold(ByteString.empty)(_ ++ _)
+          .map(_.utf8String)
+          .map(result => parser(result.parseJson.asJsObject))
+          .toMat(Sink.head)(Keep.right)
+          .run().flatten
+      case other => Future.failed(new Error(s"Invalid response from server $other"))
+    }.flatten
+
+  def shutdown(): Unit = http.shutdownAllConnectionPools()
+}

src/main/scala/me/arcanis/ffxivbis/service/bis/XivApi.scala

@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2019 Evgeniy Alekseev.
+ *
+ * This file is part of ffxivbis
+ * (see https://github.com/arcan1s/ffxivbis).
+ *
+ * License: 3-clause BSD, see https://opensource.org/licenses/BSD-3-Clause
+ */
+package me.arcanis.ffxivbis.service.bis
+
+import akka.http.scaladsl.model.Uri
+import me.arcanis.ffxivbis.models.PieceType
+import spray.json._
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.Try
+
+trait XivApi extends RequestExecutor {
+
+  private val config = context.system.settings.config
+  private val xivapiUrl = config.getString("me.arcanis.ffxivbis.bis-provider.xivapi-url")
+  private val xivapiKey = Try(config.getString("me.arcanis.ffxivbis.bis-provider.xivapi-key")).toOption
+
+  def getPieceType(itemIds: Seq[Long]): Future[Map[Long, PieceType.PieceType]] = {
+    val uriForItems = Uri(xivapiUrl)
+      .withPath(Uri.Path / "item")
+      .withQuery(Uri.Query(Map(
+        "columns" -> Seq("ID", "GameContentLinks").mkString(","),
+        "ids" -> itemIds.mkString(","),
+        "private_key" -> xivapiKey.getOrElse("")
+      )))
+
+    sendRequest(uriForItems, XivApi.parseXivapiJsonToShop).flatMap { shops =>
+      val shopIds = shops.values.map(_._2).toSet
+      val columns = shops.values.map(pair => s"ItemCost${pair._1}").toSet
+      val uriForShops = Uri(xivapiUrl)
+        .withPath(Uri.Path / "specialshop")
+        .withQuery(Uri.Query(Map(
+          "columns" -> (columns + "ID").mkString(","),
+          "ids" -> shopIds.mkString(","),
+          "private_key" -> xivapiKey.getOrElse("")
+        )))
+
+      sendRequest(uriForShops, XivApi.parseXivapiJsonToType(shops))
+    }
+  }
+}
+
+object XivApi {
+
+  private def parseXivapiJsonToShop(js: JsObject)
+                                   (implicit executionContext: ExecutionContext): Future[Map[Long, (String, Long)]] = {
+    def extractTraderId(js: JsObject) = {
+      js.fields
+        .get("Recipe").map(_ => "crafted" -> -1L) // you can craft this item
+        .orElse {  // lets try shop items
+          js.fields("SpecialShop").asJsObject
+            .fields.collectFirst {
+            case (shopName, JsArray(array)) if shopName.startsWith("ItemReceive") =>
+              val shopId = array.head match {
+                case JsNumber(id) => id.toLong
+                case other => throw deserializationError(s"Could not parse $other")
+              }
+              shopName.replace("ItemReceive", "") -> shopId
+          }
+        }.getOrElse(throw deserializationError(s"Could not parse $js"))
+    }
+
+    Future {
+      js.fields("Results") match {
+        case array: JsArray =>
+          array.elements.map(_.asJsObject.getFields("ID", "GameContentLinks") match {
+            case Seq(JsNumber(id), shop) => id.toLong -> extractTraderId(shop.asJsObject)
+            case other => throw deserializationError(s"Could not parse $other")
+          }).toMap
+        case other => throw deserializationError(s"Could not parse $other")
+      }
+    }
+  }
+
+  private def parseXivapiJsonToType(shops: Map[Long, (String, Long)])(js: JsObject)
+                                   (implicit executionContext: ExecutionContext): Future[Map[Long, PieceType.PieceType]] =
+    Future {
+      val shopMap = js.fields("Results") match {
+        case array: JsArray =>
+          array.elements.map { shop =>
+            shop.asJsObject.fields("ID") match {
+              case JsNumber(id) => id.toLong -> shop.asJsObject
+              case other => throw deserializationError(s"Could not parse $other")
+            }
+          }.toMap
+        case other => throw deserializationError(s"Could not parse $other")
+      }
+
+      shops.map { case (itemId, (index, shopId)) =>
+        val pieceType =
+          if (index == "crafted" && shopId == -1) PieceType.Crafted
+          else {
+            Try(shopMap(shopId).fields(s"ItemCost$index").asJsObject)
+              .toOption
+              .getOrElse(throw new Exception(s"${shopMap(shopId).fields(s"ItemCost$index")}, $index"))
+              .getFields("IsUnique", "StackSize") match {
+              case Seq(JsNumber(isUnique), JsNumber(stackSize)) =>
+                if (isUnique == 1 || stackSize.toLong == 2000) PieceType.Tome // either upgraded gear or tomes found
+                else PieceType.Savage
+              case other => throw deserializationError(s"Could not parse $other")
+            }
+          }
+        itemId -> pieceType
+      }
+    }
+}

src/test/scala/me/arcanis/ffxivbis/Fixtures.scala

@@ -22,6 +22,7 @@ object Fixtures {
 
   lazy val link: String = "https://ffxiv.ariyala.com/19V5R"
   lazy val link2: String = "https://ffxiv.ariyala.com/1A0WM"
+  lazy val link3: String = "https://etro.gg/gearset/26a67536-b4ce-4adc-a46a-f70e348bb138"
 
   lazy val lootWeapon: Piece = Weapon(pieceType = PieceType.Tome, Job.AnyJob)
   lazy val lootBody: Piece = Body(pieceType = PieceType.Savage, Job.AnyJob)

src/test/scala/me/arcanis/ffxivbis/http/api/v1/BiSEndpointTest.scala

@@ -11,7 +11,8 @@ import com.typesafe.config.Config
 import me.arcanis.ffxivbis.{Fixtures, Settings}
 import me.arcanis.ffxivbis.http.api.v1.json._
 import me.arcanis.ffxivbis.models.BiS
-import me.arcanis.ffxivbis.service.{Ariyala, PartyService, impl}
+import me.arcanis.ffxivbis.service.bis.BisProvider
+import me.arcanis.ffxivbis.service.{PartyService, impl}
 import me.arcanis.ffxivbis.storage.Migration
 import org.scalatest.{Matchers, WordSpec}
 
@@ -30,9 +31,9 @@ class BiSEndpointTest extends WordSpec
   implicit private val routeTimeout: RouteTestTimeout = RouteTestTimeout(timeout)
 
   private val storage: ActorRef = system.actorOf(impl.DatabaseImpl.props)
-  private val ariyala: ActorRef = system.actorOf(Ariyala.props)
+  private val provider: ActorRef = system.actorOf(BisProvider.props(false))
   private val party: ActorRef = system.actorOf(PartyService.props(storage))
-  private val route: Route = new BiSEndpoint(party, ariyala)(timeout).route
+  private val route: Route = new BiSEndpoint(party, provider)(timeout).route
 
   override def testConfig: Config = Settings.withRandomDatabase
 

src/test/scala/me/arcanis/ffxivbis/http/api/v1/PartyEndpointTest.scala

@@ -11,7 +11,8 @@ import com.typesafe.config.Config
 import me.arcanis.ffxivbis.{Fixtures, Settings}
 import me.arcanis.ffxivbis.http.api.v1.json._
 import me.arcanis.ffxivbis.models.PartyDescription
-import me.arcanis.ffxivbis.service.{Ariyala, impl}
+import me.arcanis.ffxivbis.service.bis.BisProvider
+import me.arcanis.ffxivbis.service.impl
 import me.arcanis.ffxivbis.storage.Migration
 import org.scalatest.{Matchers, WordSpec}
 
@@ -29,8 +30,8 @@ class PartyEndpointTest extends WordSpec
   implicit private val routeTimeout: RouteTestTimeout = RouteTestTimeout(timeout)
 
   private val storage: ActorRef = system.actorOf(impl.DatabaseImpl.props)
-  private val ariyala: ActorRef = system.actorOf(Ariyala.props)
-  private val route: Route = new PartyEndpoint(storage, ariyala)(timeout).route
+  private val provider: ActorRef = system.actorOf(BisProvider.props(false))
+  private val route: Route = new PartyEndpoint(storage, provider)(timeout).route
 
   override def testConfig: Config = Settings.withRandomDatabase
 

src/test/scala/me/arcanis/ffxivbis/http/api/v1/PlayerEndpointTest.scala

@@ -10,7 +10,8 @@ import akka.testkit.TestKit
 import com.typesafe.config.Config
 import me.arcanis.ffxivbis.{Fixtures, Settings}
 import me.arcanis.ffxivbis.http.api.v1.json._
-import me.arcanis.ffxivbis.service.{Ariyala, PartyService, impl}
+import me.arcanis.ffxivbis.service.bis.BisProvider
+import me.arcanis.ffxivbis.service.{PartyService, impl}
 import me.arcanis.ffxivbis.storage.Migration
 import org.scalatest.{Matchers, WordSpec}
 
@@ -29,9 +30,9 @@ class PlayerEndpointTest extends WordSpec
   implicit private val routeTimeout: RouteTestTimeout = RouteTestTimeout(timeout)
 
   private val storage: ActorRef = system.actorOf(impl.DatabaseImpl.props)
-  private val ariyala: ActorRef = system.actorOf(Ariyala.props)
+  private val provider: ActorRef = system.actorOf(BisProvider.props(false))
   private val party: ActorRef = system.actorOf(PartyService.props(storage))
-  private val route: Route = new PlayerEndpoint(party, ariyala)(timeout).route
+  private val route: Route = new PlayerEndpoint(party, provider)(timeout).route
 
   override def testConfig: Config = Settings.withRandomDatabase
 

src/test/scala/me/arcanis/ffxivbis/service/LootSelectorTest.scala

@@ -5,6 +5,7 @@ import akka.pattern.ask
 import akka.testkit.{ImplicitSender, TestKit}
 import me.arcanis.ffxivbis.{Fixtures, Settings}
 import me.arcanis.ffxivbis.models._
+import me.arcanis.ffxivbis.service.bis.BisProvider
 import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike}
 
 import scala.concurrent.Await
@@ -22,16 +23,16 @@ class LootSelectorTest extends TestKit(ActorSystem("lootselector"))
   private val timeout: FiniteDuration = 60 seconds
 
   override def beforeAll(): Unit = {
-    val ariyala = system.actorOf(Ariyala.props)
+    val provider = system.actorOf(BisProvider.props(false))
 
-    val dncSet = Await.result((ariyala ? Ariyala.GetBiS(Fixtures.link, Job.DNC) )(timeout).mapTo[BiS], timeout)
+    val dncSet = Await.result((provider ? BisProvider.GetBiS(Fixtures.link, Job.DNC) )(timeout).mapTo[BiS], timeout)
     dnc = dnc.withBiS(Some(dncSet))
 
-    val drgSet = Await.result((ariyala ? Ariyala.GetBiS(Fixtures.link2, Job.DRG) )(timeout).mapTo[BiS], timeout)
+    val drgSet = Await.result((provider ? BisProvider.GetBiS(Fixtures.link2, Job.DRG) )(timeout).mapTo[BiS], timeout)
     drg = drg.withBiS(Some(drgSet))
 
     default = default.withPlayer(dnc).withPlayer(drg)
-    system.stop(ariyala)
+    system.stop(provider)
   }
 
   "loot selector" must {

src/test/scala/me/arcanis/ffxivbis/service/bis/BisProviderTest.scala

@@ -1,4 +1,4 @@
-package me.arcanis.ffxivbis.service
+package me.arcanis.ffxivbis.service.bis
 
 import akka.actor.ActorSystem
 import akka.testkit.{ImplicitSender, TestKit}
@@ -9,7 +9,7 @@ import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike}
 import scala.concurrent.duration._
 import scala.language.postfixOps
 
-class AriyalaTest extends TestKit(ActorSystem("ariyala"))
+class BisProviderTest extends TestKit(ActorSystem("bis-provider"))
   with ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll {
 
   private val timeout: FiniteDuration = 60 seconds
@@ -18,9 +18,15 @@ class AriyalaTest extends TestKit(ActorSystem("ariyala"))
 
   "ariyala actor" must {
 
-    "get best in slot set" in {
-      val ariyala = system.actorOf(Ariyala.props)
-      ariyala ! Ariyala.GetBiS(Fixtures.link, Job.DNC)
+    "get best in slot set (ariyala)" in {
+      val provider = system.actorOf(BisProvider.props(false))
+      provider ! BisProvider.GetBiS(Fixtures.link, Job.DNC)
+      expectMsg(timeout, Fixtures.bis)
+    }
+
+    "get best in slot set (etro)" in {
+      val provider = system.actorOf(BisProvider.props(true))
+      provider ! BisProvider.GetBiS(Fixtures.link3, Job.DNC)
       expectMsg(timeout, Fixtures.bis)
     }