Add item db for 1.20.3 to builder3

builder3.html

@@ -990,7 +990,7 @@
         <script type="text/javascript" src="skillpoints.js"></script>
         <script type="text/javascript" src="damage_calc3.js"></script>
         <script type="text/javascript" src="display.js"></script>
-        <script type="text/javascript" src="load.js"></script>
+        <script type="text/javascript" src="load3.js"></script>
         <script type="text/javascript" src="load_ing.js"></script>  
         <script type="text/javascript" src="custom.js"></script> 
         <script type="text/javascript" src="customizer.js"></script>         

load3.js

@@ -0,0 +1,239 @@
+const DB_VERSION = 44;
+// @See https://github.com/mdn/learning-area/blob/master/javascript/apis/client-side-storage/indexeddb/video-store/index.jsA
+
+let db;
+let reload = false;
+let load_complete = false;
+let items;
+let sets;
+let itemMap;
+let idMap;
+let redirectMap;
+let itemLists = new Map();
+/*
+ * Load item set from local DB. Calls init() on success.
+ */
+async function load_local(init_func) {
+    let get_tx = db.transaction(['item_db', 'set_db'], 'readonly');
+    let sets_store = get_tx.objectStore('set_db');
+    let get_store = get_tx.objectStore('item_db');
+    let request = get_store.getAll();
+    request.onerror = function(event) {
+        console.log("Could not read local item db...");
+    }
+    request.onsuccess = function(event) {
+        console.log("Successfully read local item db.");
+        items = request.result;
+        //console.log(items);
+        let request2 = sets_store.openCursor();
+
+        sets = {};
+        request2.onerror = function(event) {
+            console.log("Could not read local set db...");
+        }
+
+        request2.onsuccess = function(event) {
+            let cursor = event.target.result;
+            if (cursor) {
+                sets[cursor.primaryKey] = cursor.value;
+                cursor.continue();
+            }
+            else {
+                console.log("Successfully read local set db.");
+                //console.log(sets);
+                init_maps();
+                init_func();
+                load_complete = true;
+            }
+        }
+    }
+    await get_tx.complete;
+    db.close();
+}
+
+/*
+ * Clean bad item data. For now just assigns display name if it isn't already assigned.
+ */
+function clean_item(item) {
+    if (item.remapID === undefined) {
+        if (item.displayName === undefined) {
+            item.displayName = item.name;
+        }
+        item.skillpoints = [item.str, item.dex, item.int, item.def, item.agi];
+        item.has_negstat = item.str < 0 || item.dex < 0 || item.int < 0 || item.def < 0 || item.agi < 0;
+        item.reqs = [item.strReq, item.dexReq, item.intReq, item.defReq, item.agiReq];
+        if (item.slots === undefined) {
+            item.slots = 0
+        }
+    }
+}
+
+/*
+ * Load item set from remote DB (aka a big json file). Calls init() on success.
+ */
+async function load(init_func) {
+
+    let getUrl = window.location;
+    let baseUrl = getUrl.protocol + "//" + getUrl.host + "/";// + getUrl.pathname.split('/')[1];
+    // "Random" string to prevent caching!
+    let url = baseUrl + "/compress3.json?"+new Date();
+    let result = await (await fetch(url)).json();
+    items = result.items;
+    sets = result.sets;
+    
+
+
+//    let clear_tx = db.transaction(['item_db', 'set_db'], 'readwrite');
+//    let clear_items = clear_tx.objectStore('item_db');
+//    let clear_sets = clear_tx.objectStore('item_db');
+//
+//    await clear_items.clear();
+//    await clear_sets.clear();
+//    await clear_tx.complete;
+
+    let add_tx = db.transaction(['item_db', 'set_db'], 'readwrite');
+    add_tx.onabort = function(e) {
+        console.log(e);
+        console.log("Not enough space...");
+    };
+    let items_store = add_tx.objectStore('item_db');
+    let add_promises = [];
+    for (const item of items) {
+        clean_item(item);
+        let req = items_store.add(item, item.name);
+        req.onerror = function() {
+            console.log("ADD ITEM ERROR? " + item.name);
+        };
+        add_promises.push(req);
+    }
+    let sets_store = add_tx.objectStore('set_db');
+    for (const set in sets) {
+        add_promises.push(sets_store.add(sets[set], set));
+    }
+    add_promises.push(add_tx.complete);
+    Promise.all(add_promises).then((values) => {
+        init_maps();
+        init_func();
+        load_complete = true;
+    });
+    // DB not closed? idfk man
+}
+
+function load_init(init_func) {
+    if (load_complete) {
+        console.log("Item db already loaded, skipping load sequence");
+        init_func();
+        return;
+    }
+    let request = window.indexedDB.open('item_db', DB_VERSION);
+
+    request.onerror = function() {
+        console.log("DB failed to open...");
+    };
+
+    request.onsuccess = function() {
+        db = request.result;
+        if (!reload) {
+            console.log("Using stored data...")
+            load_local(init_func);
+        }
+        else {
+            console.log("Using new data...")
+            load(init_func);
+        }
+    }
+
+    request.onupgradeneeded = function(e) {
+        reload = true;
+
+        let db = e.target.result;
+        
+        try {
+            db.deleteObjectStore('item_db');
+        }
+        catch (error) {
+            console.log("Could not delete item DB. This is probably fine");
+        }
+        try {
+            db.deleteObjectStore('set_db');
+        }
+        catch (error) {
+            console.log("Could not delete set DB. This is probably fine");
+        }
+
+        db.createObjectStore('item_db');
+        db.createObjectStore('set_db');
+
+        console.log("DB setup complete...");
+    }
+}
+
+function init_maps() {
+    //warp
+    itemMap = new Map();
+    /* Mapping from item names to set names. */
+    idMap = new Map();
+    redirectMap = new Map();
+    for (const it of itemTypes) {
+        itemLists.set(it, []);
+    }
+
+    let noneItems = [
+        ["armor", "helmet", "No Helmet"],
+        ["armor", "chestplate", "No Chestplate"],
+        ["armor", "leggings", "No Leggings"],
+        ["armor", "boots", "No Boots"],
+        ["accessory", "ring", "No Ring 1"],
+        ["accessory", "ring", "No Ring 2"],
+        ["accessory", "bracelet", "No Bracelet"],
+        ["accessory", "necklace", "No Necklace"],
+        ["weapon", "dagger", "No Weapon"],
+    ];
+    for (let i = 0; i < 9; i++) {
+        let item = Object();
+        item.slots = 0;
+        item.category = noneItems[i][0];
+        item.type = noneItems[i][1];
+        item.name = noneItems[i][2];
+        item.displayName = item.name;
+        item.set = null;
+        item.quest = null;
+        item.skillpoints = [0, 0, 0, 0, 0];
+        item.has_negstat = false;
+        item.reqs = [0, 0, 0, 0, 0];
+        item.fixID = true;
+        item.tier = "Normal";//do not get rid of this @hpp
+        item.id = 10000 + i;
+        item.nDam = "0-0";
+        item.eDam = "0-0";
+        item.tDam = "0-0";
+        item.wDam = "0-0";
+        item.fDam = "0-0";
+        item.aDam = "0-0";
+        item.str = 0,
+        item.dex = 0,
+        item.int = 0,
+        item.def = 0,
+        item.agi = 0,
+
+        noneItems[i] = item;
+    }
+    items = items.concat(noneItems);
+    //console.log(items);
+    for (const item of items) {
+        if (item.remapID === undefined) {
+            itemLists.get(item.type).push(item.displayName);
+            itemMap.set(item.displayName, item);
+            if (noneItems.includes(item)) {
+                idMap.set(item.id, "");
+            }
+            else {
+                idMap.set(item.id, item.displayName);
+            }
+        }
+        else {
+            redirectMap.set(item.id, item.remapID);
+        }
+    }
+    console.log(itemMap);
+}

py_script/parse_log.py

@@ -0,0 +1,231 @@
+import json
+import difflib
+
+reqs = ["dexReq", "strReq", "defReq", "agiReq", "intReq"]
+
+attack_speeds = ["SUPER_SLOW", "VERY_SLOW", "SLOW", "NORMAL", "FAST", "VERY_FAST", "SUPER_FAST"]
+
+baseDamageMultiplier = [ 0.51, 0.83, 1.5, 2.05, 2.5, 3.1, 4.3 ];
+
+# tefaw
+powders = [ [4.5, 6.5, 8.5,  9,    10.5, 12.5, "T"],
+            [4.5, 6.5, 8,    8.5,  10,   12, "E"],
+            [3.5, 6,   7,    7.5,  9,    11, "F"],
+            [4,   6.5, 7.5,  8,    9.5,  11, "A"],
+            [3.5, 5,   6.5,  7,    8.5,  10, "W"]]
+
+elements_damages = {
+        "neutral": "nDam",
+        "earth": "eDam",
+        "thunder": "tDam",
+        "water": "wDam",
+        "fire": "fDam",
+        "air": "aDam"
+    }
+
+def getDisplayName(item):
+    if "displayName" in item:
+        return item["displayName"]
+    return item["name"]
+
+with open("clean.json", "r") as infile:
+    all_json = json.load(infile)
+    old_items = all_json["items"]
+    old_item_map = dict()
+    for item in old_items:
+        name = getDisplayName(item)
+        max_i = 0
+        for i, req in enumerate(reqs):
+            if req in item and item[req] > 0:
+                max_i = i
+                break
+        item["max_req_ind"] = max_i
+        old_item_map[name.lower()] = item
+        
+    
+items = []
+state = 0
+
+def file_iter(infile):
+    for line in infile:
+        yield line
+
+with open("log.txt", "r") as infile:
+    fiter = file_iter(infile)
+    while True:
+        line = next(fiter, None)
+        if line is None:
+            break
+        line = line.strip()
+        if line.lower() in old_item_map:
+            item_name = line
+            item = old_item_map[line.lower()]
+            item["changed"] = True
+            line = next(fiter, None).strip()
+            if line:
+                print("AAAAA ERROR")
+                print(line)
+                assert(False)
+            # Skip original damage
+            line = next(fiter, None).strip()
+            while line:
+                line = next(fiter, None).strip()
+
+            line = next(fiter, None).strip()
+            used_elements = []
+            while line:
+                parts = line.split(" ")
+                if len(parts) == 3:
+                    # Attack speed
+                    speeds = difflib.get_close_matches(parts[2], attack_speeds, n=1)
+                    if len(speeds) != 1:
+                        print(f"ERROR UNRECOGNIZED ATKSPD FOR [{item_name}]: {parts[2]}")
+                    speed = speeds[0]
+                    oldSpeed = item["atkSpd"]
+                    if oldSpeed != speed:
+                        print(f"ATTACK SPEED CHANGE FOR [{item_name}]: {oldSpeed} -> {speed}")
+                        item["atkSpd"] = speed
+                elif len(parts) > 3:
+                    print("ERROR MORE THAN 3 PARTS: " + line)
+                damage = parts[0].strip("*")
+                element = parts[1].lower().strip("*")
+                if element == "all":
+                    assert(not used_elements)
+                    for e, v in elements_damages.items():
+                        used_elements.append(e)
+                        item[v] = damage
+                elif element == "elemental":
+                    for e, v in elements_damages.items():
+                        if e not in used_elements:
+                            used_elements.append(e)
+                            item[v] = damage
+                else:
+                    elements = element.split("/")
+                    if len(elements) == 0:
+                        used_elements.append(element)
+                        item[elements_damages[element]] = damage
+                    else:
+                        for element in elements:
+                            used_elements.append(element)
+                            item[elements_damages[element]] = damage
+                line = next(fiter, None)
+                if line is None:
+                    break
+                line = line.strip("\n *")
+
+damage_translate = {
+        "damage": "nDam",
+        "fireDamage": "fDam",
+        "waterDamage": "wDam",
+        "airDamage": "aDam",
+        "thunderDamage": "tDam",
+        "earthDamage": "eDam",
+    }
+with open("spears.txt", "r") as infile:
+    fiter = file_iter(infile)
+    while True:
+        line = next(fiter, None)
+        if line is None:
+            break
+        line = line.strip()
+        if line.lower() in old_item_map:
+            item_name = line
+            item = old_item_map[line.lower()]
+            item["changed"] = True
+            line = next(fiter, None).strip()
+            if line:
+                print("AAAAA ERROR")
+                print(line)
+                assert(False)
+            line = next(fiter, None).strip()
+            while line:
+                parts = line.split(":")
+                if len(parts) != 2:
+                    print("???? " + line)
+                    break
+
+                if parts[0] == "attackSpeed":
+                    new_speed_str = parts[1].split("->")[-1].strip()
+                    speeds = difflib.get_close_matches(new_speed_str, attack_speeds, n=1)
+                    if len(speeds) != 1:
+                        print(f"ERROR UNRECOGNIZED ATKSPD FOR [{item_name}]: {new_speed_str}")
+                    speed = speeds[0]
+                    oldSpeed = item["atkSpd"]
+                    if oldSpeed != speed:
+                        print(f"ATTACK SPEED CHANGE FOR [{item_name}]: {oldSpeed} -> {speed}")
+                        item["atkSpd"] = speed
+                else:
+                    damage_target = damage_translate[parts[0]]
+                    damage_val = parts[1].split("->")[-1].strip()
+                    item[damage_target] = damage_val
+                line = next(fiter, None)
+                if line is None:
+                    break
+                line = line.strip()
+
+
+item_dps_by_type = {"dagger": [], "bow": [], "spear": [], "relik": [], "wand": []}
+
+for i, item in enumerate(old_items):
+    if "changed" in item:
+        damage_total = 0
+        for v in elements_damages.values():
+            if v in item:
+                dam = item[v].split("-")
+                damage_total += int(dam[0]) + int(dam[1])
+        damage_total /= 2
+        idx = 5
+        if item["lvl"] < 80:
+            idx = 4
+
+        if "slots" not in item:
+            item["slots"] = 0
+        damage_total += item["slots"] * powders[item["max_req_ind"]][idx]
+        base_dps = damage_total * baseDamageMultiplier[attack_speeds.index(item["atkSpd"])]
+        lst = item_dps_by_type[item["type"]]
+        lst.append((-base_dps, i, powders[item["max_req_ind"]][6], item))
+        del item["changed"]
+    del item["max_req_ind"]
+
+with open("raw.txt", "r") as infile:
+    fiter = file_iter(infile)
+    while True:
+        line = next(fiter, None)
+        if line is None:
+            break
+        line = line.strip(" \n:")
+        if line.lower() in old_item_map:
+            item_name = line
+            item = old_item_map[line.lower()]
+            line = next(fiter, None).strip()
+            if line:
+                print("AAAAA ERROR")
+                print(line)
+                assert(False)
+            line = next(fiter, None).strip()
+            while line:
+                parts = line.split(":")
+                if len(parts) != 2:
+                    print("???? " + line)
+                    break
+
+                if "spell" in parts[0].lower():
+                    item["sdRaw"] = int(parts[1].split("->")[-1].strip())
+                elif parts[0] == "damageBonusRaw":
+                    item["mdRaw"] = int(parts[1].split("->")[-1].strip())
+                else:
+                    print("!!!!"+line)
+                line = next(fiter, None)
+                if line is None:
+                    break
+                line = line.strip()
+
+for k, v in item_dps_by_type.items():
+    print(f"\n{k}:")
+    v.sort()
+    for i in range(50):
+        dps, _, p, item = v[i]
+        name = getDisplayName(item)
+        print(f"{name}: {-dps:.2f}, {p}")
+
+json.dump(all_json, open("nerf_items.json", "w"), indent=2)

py_script/transform_combine.py

@@ -14,6 +14,7 @@ with open("dump.json", "r") as infile:
     data = json.loads(infile.read())
 
 items = data["items"]
+if "request" in data:
     del data["request"]
 
 import os

py_script/transform_merge.py

@@ -0,0 +1,216 @@
+"""
+
+NOTE!!!!!!!
+
+DEMON TIDE 1.20 IS HARD CODED!
+
+AMBIVALENCE IS REMOVED!
+
+"""
+
+import json
+
+with open("dump.json", "r") as infile:
+    data = json.load(infile)
+
+with open("updated.json", "r") as oldfile:
+    old_data = json.load(oldfile)
+
+items = data["items"]
+old_items = old_data["items"]
+if "request" in data:
+    del data["request"]
+
+# import os
+# sets = dict()
+# for filename in os.listdir('sets'):
+#     if "json" not in filename:
+#         continue
+#     set_name = filename[1:].split(".")[0].replace("+", " ").replace("%27", "'")
+#     with open("sets/"+filename) as set_info:
+#         set_obj = json.load(set_info)
+#         for item in set_obj["items"]:
+#             item_set_map[item] = set_name
+#         sets[set_name] = set_obj
+# 
+# data["sets"] = sets
+data["sets"] = old_data["sets"]
+item_set_map = dict()
+for set_name, set_data in data["sets"].items():
+    for item_name in set_data["items"]:
+        item_set_map[item_name] = set_name
+
+must_mappings = [
+    "strength",
+    "dexterity",
+    "intelligence",
+    "agility",
+    "defense",
+    "strengthPoints",
+    "dexterityPoints",
+    "intelligencePoints",
+    "agilityPoints",
+    "defensePoints",
+]
+
+translate_mappings = {
+    #"name": "name",
+    #"displayName": "displayName",
+    #"tier": "tier",
+    #"set": "set",
+    "sockets": "slots",
+    #"type": "type",
+    #"armorType": "armorType", (deleted)
+    "armorColor": "color", #(deleted)
+    "addedLore": "lore", #(deleted)
+    #"material": "material", (deleted)
+    "dropType": "drop",
+    #"quest": "quest",
+    "restrictions": "restrict",
+    "damage": "nDam",
+    "fireDamage": "fDam",
+    "waterDamage": "wDam",
+    "airDamage": "aDam",
+    "thunderDamage": "tDam",
+    "earthDamage": "eDam",
+    "attackSpeed": "atkSpd",
+    "health": "hp",
+    "fireDefense": "fDef",
+    "waterDefense": "wDef",
+    "airDefense": "aDef",
+    "thunderDefense": "tDef",
+    "earthDefense": "eDef",
+    "level": "lvl",
+    "classRequirement": "classReq",
+    "strength": "strReq",
+    "dexterity": "dexReq",
+    "intelligence": "intReq",
+    "agility": "agiReq",
+    "defense": "defReq",
+    "healthRegen": "hprPct",
+    "manaRegen": "mr",
+    "spellDamage": "sdPct",
+    "damageBonus": "mdPct",
+    "lifeSteal": "ls",
+    "manaSteal": "ms",
+    "xpBonus": "xpb",
+    "lootBonus": "lb",
+    "reflection": "ref",
+    "strengthPoints": "str",
+    "dexterityPoints": "dex",
+    "intelligencePoints": "int",
+    "agilityPoints": "agi",
+    "defensePoints": "def",
+    #"thorns": "thorns",
+    "exploding": "expd",
+    "speed": "spd",
+    "attackSpeedBonus": "atkTier",
+    #"poison": "poison",
+    "healthBonus": "hpBonus",
+    "soulPoints": "spRegen",
+    "emeraldStealing": "eSteal",
+    "healthRegenRaw": "hprRaw",
+    "spellDamageRaw": "sdRaw",
+    "damageBonusRaw": "mdRaw",
+    "bonusFireDamage": "fDamPct",
+    "bonusWaterDamage": "wDamPct",
+    "bonusAirDamage": "aDamPct",
+    "bonusThunderDamage": "tDamPct",
+    "bonusEarthDamage": "eDamPct",
+    "bonusFireDefense": "fDefPct",
+    "bonusWaterDefense": "wDefPct",
+    "bonusAirDefense": "aDefPct",
+    "bonusThunderDefense": "tDefPct",
+    "bonusEarthDefense": "eDefPct",
+    "accessoryType": "type",
+    "identified": "fixID",
+    #"skin": "skin",
+    #"category": "category",
+
+    "spellCostPct1": "spPct1",
+    "spellCostRaw1": "spRaw1",
+    "spellCostPct2": "spPct2",
+    "spellCostRaw2": "spRaw2",
+    "spellCostPct3": "spPct3",
+    "spellCostRaw3": "spRaw3",
+    "spellCostPct4": "spPct4",
+    "spellCostRaw4": "spRaw4",
+
+    "rainbowSpellDamageRaw": "rainbowRaw",
+    #"sprint": "sprint",
+    "sprintRegen": "sprintReg",
+    "jumpHeight": "jh",
+    "lootQuality": "lq",
+
+    "gatherXpBonus": "gXp",
+    "gatherSpeed": "gSpd",
+}
+
+delete_keys = [
+    #"addedLore",
+    #"skin",
+    #"armorType",
+    #"armorColor",
+    #"material"
+]
+
+id_map = {item["name"]: item["id"] for item in old_items}
+used_ids = set([v for k, v in id_map.items()])
+max_id = 0
+
+known_item_names = set()
+
+for item in items:
+    known_item_names.add(item["name"])
+
+old_items_map = dict()
+unchanged_items = []
+remap_items = []
+for item in old_items:
+    if "remapID" in item:
+        remap_items.append(item)
+    elif item["name"] not in known_item_names:
+        unchanged_items.append(item)
+    old_items_map[item["name"]] = item
+
+for item in items:
+    for key in delete_keys:
+        if key in item:
+            del item[key]
+    
+    for k in list(item.keys()):
+        if (item[k] == 0 or item[k] is None) and not k in must_mappings:
+            del item[k]
+
+    for k, v in translate_mappings.items():
+        if k in item:
+            item[v] = item[k]
+            del item[k]
+
+    if not (item["name"] in id_map):
+        while max_id in used_ids:
+            max_id += 1
+        used_ids.add(max_id)
+        id_map[item["name"]] = max_id
+        print(f'New item: {item["name"]} (id: {max_id})')
+    item["id"] = id_map[item["name"]]
+
+    item["type"] = item["type"].lower()
+    if "displayName" in item:
+        item_name = item["displayName"]
+    else:
+        item_name = item["name"]
+    if item_name in item_set_map:
+        item["set"] = item_set_map[item_name]
+        if item["name"] in old_items_map:
+            old_item = old_items_map[item["name"]]
+            if "hideSet" in old_item:
+                item["hideSet"] = old_item["hideSet"]
+
+items.extend(unchanged_items)
+items.extend(remap_items)
+
+with open("clean.json", "w") as outfile:
+    json.dump(data, outfile, indent=2)
+with open("compress.json", "w") as outfile:
+    json.dump(data, outfile)

py_script/transform_preserve.py

@@ -18,6 +18,7 @@ with open("updated.json", "r") as oldfile:
 
 items = data["items"]
 old_items = old_data["items"]
+if "request" in data:
     del data["request"]
 
 # import os