Merge branch 'dev' of https://github.com/hppeng-wynn/hppeng-wynn.github.io into dev

py_script/json_diff.py

@@ -1,66 +1,154 @@
-import sys
+"""Json diff checker for manual testing."""
+
+import argparse
 import json
-f1 = sys.argv[1]
+import sys
-f2 = sys.argv[2]
 
-json1 = json.load(open(f1))
+from recordclass import recordclass
-json2 = json.load(open(f2))
 
-def shorten(v):
+JSONDiffReporter = recordclass('JSONDiffReporter', 'val_diff len_diff type_diff path_diff get_key')
-    if len(v) > 100:
-        return v[:100] + "..."
-    return v
-def is_basic(t):
-    return t is int or t is str or t is float or t is bool or t is list
 
-def custom_input():
+def shorten(val):
-    if custom_input.alive:
+    """Utility for printing large functions, auto shorten"""
+    if len(val) > 100:
+        return val[:100] + "..."
+    return val
+
+def is_basic(val):
+    """Check if the given value is a "primitive" type in json (not object)."""
+    return val is int or val is str or val is float or val is bool or val is list
+
+def __custom_input(_):
+    """Read from stdin unless user has cancelled"""
+    if __custom_input.alive:
         try:
             return input()
-        except:
+        except KeyboardInterrupt:
-            custom_input.alive = False
+            __custom_input.alive = False
     return ""
-custom_input.alive = True
+__custom_input.alive = True
 
-def list_diff(list1, list2, path):
+def __print_val_diff(val1, val2, path):
+    print(f"{path}: Value difference")
+    print(f"    Left:  {shorten(str(val1))}")
+    print(f"    Right: {shorten(str(val2))}")
+
+def __print_len_diff(val1, val2, path):
+    print(f"{path}: Length difference")
+    print(f"    Left  (length {len(val1)}): {shorten(str(val1))}")
+    print(f"    Right (length {len(val2)}): {shorten(str(val2))}")
+
+def __print_type_diff(type1, type2, path):
+    print(f"{path}: Type difference [{str(type1)} != {str(type2)}]")
+
+#def __print_path_diff(left, right, key, path, side):
+def __print_path_diff(_1, _2, key, path, side):
+    if side:
+        print(f"{path}.{key}: Contained in right but not left")
+    else:
+        print(f"{path}.{key}: Contained in left but not right")
+    print(f"    Value: {shorten(str(key))}")
+
+# Default diff reporter (just prints everything)
+JSON_DIFF_PRINTER = JSONDiffReporter(
+    __print_val_diff,
+    __print_len_diff,
+    __print_type_diff,
+    __print_path_diff,
+    __custom_input
+)
+
+def __val_diff(val1, val2, path):
+    errmsg = (f"{path}: Value difference\n"
+            + f"    Left:  {shorten(str(val1))}\n"
+            + f"    Right: {shorten(str(val2))}")
+    raise ValueError(errmsg)
+
+def __len_diff(val1, val2, path):
+    errmsg = (f"{path}: Length difference\n"
+            + f"    Left  (length {len(val1)}): {shorten(str(val1))}\n"
+            + f"    Right (length {len(val2)}): {shorten(str(val2))}")
+    raise ValueError(errmsg)
+
+def __type_diff(type1, type2, path):
+    raise TypeError(f"{path}: Type difference [{str(type1)} != {str(type2)}]")
+
+#def __print_path_diff(left, right, key, path, side):
+def __path_diff(_1, _2, key, path, side):
+    if side:
+        errmsg = f"{path}.{key}: Contained in right but not left\n"
+    else:
+        errmsg = f"{path}.{key}: Contained in left but not right\n"
+    errmsg += f"    Value: {shorten(str(key))}"
+    raise AttributeError(errmsg)
+
+def get_test_diff_handler(get_key):
+    """Make a JSON diff handler that throws errors on failure.
+
+    :param: get_key: key getter func
+    """
+    return JSONDiffReporter(__val_diff, __len_diff, __type_diff, __path_diff, get_key)
+
+def list_diff(reporter, list1, list2, path) -> bool:
+    """Compute list difference between two object lists (compare by key)"""
     print(f"Encountered object list {path}, enter match key: ", end="", file=sys.stderr)
-    key = custom_input()
+    key = reporter.get_key(path)
-    if (key == ""):
+    if key == "":
         if list1 != list2:
-            print(f"{path}.{k}: Value difference")
+            reporter.val_diff(list1, list2, path)
-            print(f"    Left: {shorten(str(v))}")
-            print(f"    Right: {shorten(str(obj))}")
     else:
         left = {x[key]: x for x in list1}
         right = {x[key]: x for x in list2}
-        object_diff(left, right, path)
+        object_diff(reporter, left, right, path)
 
-def object_diff(obj1, obj2, path):
+def object_diff(reporter, obj1, obj2, path) -> bool:
-    for (k, v) in obj1.items():
+    """Compute object difference between two objects... kinda"""
+    for (k, val) in obj1.items():
         if k in obj2:
             obj = obj2[k]
-            type1 = type(v)
+            type1 = type(val)
             type2 = type(obj)
             if type1 != type2:
-                print(f"{path}.{k}: Type difference [{str(type1)} != {str(type2)}]")
+                reporter.type_diff(type1, type2, f"{path}.{k}")
-            elif type1 is list and type2 is list and not is_basic(type(v[0])):
+            elif type1 is list:
-                list_diff(v, obj, path+"."+str(k))
+                if len(val) != len(obj):
-            elif (type1 is list and is_basic(type(v[0]))) or is_basic(type1) or v is None or obj2 is None:
+                    reporter.len_diff(val, obj, f"{path}.{k}")
-                if v != obj:
+                elif len(val) == 0:
-                    print(f"{path}.{k}: Value difference")
+                    continue
-                    print(f"    Left: {shorten(str(v))}")
+                elif is_basic(type(val[0])):
-                    print(f"    Right: {shorten(str(obj))}")
+                    if val != obj:
+                        reporter.val_diff(val, obj, f"{path}.{k}")
+                    continue
+                list_diff(reporter, val, obj, path+"."+k)
+            elif is_basic(type1) or val is None or obj2 is None:
+                if val != obj:
+                    reporter.val_diff(val, obj, f"{path}.{k}")
             else:
-                object_diff(v, obj, path+"."+str(k))
+                object_diff(reporter, val, obj, f"{path}.{k}")
-        else:
+            continue
-            print(f"{path}.{k}: Contained in left but not right")
+        reporter.path_diff(obj1, obj2, k, path, True)
-            print(f"    Value: {shorten(str(v))}")
+    for k in obj2:
-    for (k, v) in obj2.items():
         if k not in obj1:
-            print(f"{path}.{k}: Contained in right but not left")
+            reporter.path_diff(obj1, obj2, k, path, True)
-            print(f"    Value: {shorten(str(v))}")
 
-if type(json1) is list and type(json2) is list:
+def json_diff(json1, json2, reporter=JSON_DIFF_PRINTER) -> bool:
-    list_diff(json1, json2, "$")
+    """Run the json diff tool on two json objects."""
-else:
+    if isinstance(json1, list) and isinstance(json2, list):
-    object_diff(json1, json2, "$")
+        return list_diff(reporter, json1, json2, "$")
+    return object_diff(reporter, json1, json2, "$")
+
+if __name__ == "__main__":
+    argparser = argparse.ArgumentParser(description="JSON diff utility")
+    argparser.add_argument('file1', action='store', type=str,
+        help="First file to compare"
+    )
+    argparser.add_argument('file2', action='store', type=str,
+        help="Second file to compare"
+    )
+
+    args = argparser.parse_args()
+    with open(args.file1, 'r', encoding="utf-8") as file1:
+        json1 = json.load(file1)
+    with open(args.file2, 'r', encoding="utf-8") as file2:
+        json2 = json.load(file2)
+    json_diff(json1, json2)

py_script/transform_merge.py

@@ -209,7 +209,13 @@ for item in items:
 
 items.extend(unchanged_items)
 items.extend(remap_items)
-
+with open("id_map.json","w") as id_mapfile:
+    print("{", file=id_mapfile)
+    outputs = []
+    for v, k in sorted((v, k) for k, v in id_map.items()):
+        outputs.append(f' "{k}": {v}')
+    print(',\n'.join(outputs), file=id_mapfile)
+    print("}", file=id_mapfile)
 with open("clean.json", "w") as outfile:
     json.dump(data, outfile, indent=2)
 with open("compress.json", "w") as outfile:

py_script/update_merge.py

@@ -1,80 +0,0 @@
-import json
-
-with open("clean.json") as infile:
-    olds = json.load(infile)
-
-items = olds["items"]
-
-item_oldnames_map = dict()
-item_newnames_map = dict()
-
-VERSION_STR = " (1.20)"
-
-max_old_id = 0
-
-for item in items:
-    item_id = item["id"]
-    if "displayName" in item:
-        displayName = item["displayName"]
-    else:
-        displayName = item["name"]
-    item_name = displayName.replace(VERSION_STR, "")
-    if item_id > 10000:
-        map_name = item["name"].replace(VERSION_STR, "")
-        item_newnames_map[map_name] = item
-        item["displayName"] = item_name
-    else:
-        item_oldnames_map[item_name] = item
-        if item_id > max_old_id:
-            max_old_id = item_id
-
-dummy_items = []
-
-for (name, item) in item_newnames_map.items():
-    if name in item_oldnames_map:
-        old_item = item_oldnames_map[name]
-        if "displayName" in item:
-            displayName = item["displayName"].replace(VERSION_STR, "")
-        else:
-            displayName = name
-        save_old = ["id","set","quest","drop","restrict", "name"]
-        old_mappings = { k: old_item[k] for k in save_old if k in old_item }
-        old_item.clear()
-
-        if "restrict" in item:
-            del item["restrict"]
-
-        for k in item:
-            old_item[k] = item[k]
-        for k in old_mappings:
-            old_item[k] = old_mappings[k]
-        save_id = item["id"]
-        item.clear()
-        item["id"] = save_id
-        item["name"] = str(save_id)
-        item["remapID"] = old_item["id"]
-    else:
-        if "restrict" in item:
-            in_str = input(name + " restriction: ").strip()
-            if in_str:
-                item["restrict"] = in_str
-            else:
-                del item["restrict"]
-        item["name"] = name
-        dummy_item = dict()
-        dummy_item["id"] = item["id"]
-        max_old_id += 1
-        item["id"] = max_old_id
-        dummy_item["remapID"] = item["id"]
-        dummy_items.append(dummy_item)
-        
-items.extend(dummy_items)
-
-sets = olds["sets"]
-
-data = dict()
-data["items"] = items
-data["sets"] = sets
-
-with open("updated.json", "w") as outfile:
-    json.dump(data, outfile, indent=2)