From 9489a950cb48d883af648bd152b919b813238180 Mon Sep 17 00:00:00 2001 From: black-sliver <59490463+black-sliver@users.noreply.github.com> Date: Fri, 27 Feb 2026 00:53:34 +0100 Subject: [PATCH 01/15] MultiServer, customserver: move data package handling Create a new class that handles conversion worlds+embedded -> context data. Create a derived class that uses static_server_data+pony instead. There is also a not very efficient feature to deduplicate strings (may need perf testing). By moving code around, we can simplify a lot of the world loading. Where code lines were touched, some typing and some reformatting was added. The back compat for GetDataPackage without games was finally dropped. This was done as a cleanup because the refactoring touched those lines anyway. Also reworked the per-context dicts and the RoomInfo to hopefully be more efficient by ignoring unused games. (Generating the list of used games was required for the new code anyway.) Side effect of the MultiServer cache: we now load worlds lazily (but still all at once) and don't modify the games package in place. If needed we create copies. This almost gets us to the point where MultiServer doesn't need worlds - it still needs them for the forbidden items. There is a bonus optimization that deduplicates strings in name_groups that may have bad performance and may need some perf testing if we run into issues. --- MultiServer.py | 183 +++++++++--------- WebHostLib/customserver.py | 141 +++++++------- apmw/__init__.py | 0 apmw/multiserver/__init__.py | 0 apmw/multiserver/gamespackage/__init__.py | 0 apmw/multiserver/gamespackage/cache.py | 110 +++++++++++ apmw/webhost/__init__.py | 0 apmw/webhost/customserver/__init__.py | 0 .../customserver/gamespackage/__init__.py | 0 .../customserver/gamespackage/cache.py | 32 +++ 10 files changed, 301 insertions(+), 165 deletions(-) create mode 100644 apmw/__init__.py create mode 100644 apmw/multiserver/__init__.py create mode 100644 apmw/multiserver/gamespackage/__init__.py create mode 100644 apmw/multiserver/gamespackage/cache.py create mode 100644 apmw/webhost/__init__.py create mode 100644 apmw/webhost/customserver/__init__.py create mode 100644 apmw/webhost/customserver/gamespackage/__init__.py create mode 100644 apmw/webhost/customserver/gamespackage/cache.py diff --git a/MultiServer.py b/MultiServer.py index 52c80c5540..ac3c9c18c1 100644 --- a/MultiServer.py +++ b/MultiServer.py @@ -43,8 +43,9 @@ import NetUtils import Utils from Utils import version_tuple, restricted_loads, Version, async_start, get_intended_text from NetUtils import Endpoint, ClientStatus, NetworkItem, decode, encode, NetworkPlayer, Permission, NetworkSlot, \ - SlotType, LocationStore, MultiData, Hint, HintStatus + SlotType, LocationStore, MultiData, Hint, HintStatus, GamesPackage from BaseClasses import ItemClassification +from apmw.multiserver.gamespackage.cache import GamesPackageCache min_client_version = Version(0, 5, 0) @@ -240,21 +241,38 @@ class Context: slot_info: typing.Dict[int, NetworkSlot] generator_version = Version(0, 0, 0) checksums: typing.Dict[str, str] + played_games: set[str] item_names: typing.Dict[str, typing.Dict[int, str]] - item_name_groups: typing.Dict[str, typing.Dict[str, typing.Set[str]]] + item_name_groups: typing.Dict[str, typing.Dict[str, list[str]]] location_names: typing.Dict[str, typing.Dict[int, str]] - location_name_groups: typing.Dict[str, typing.Dict[str, typing.Set[str]]] + location_name_groups: typing.Dict[str, typing.Dict[str, list[str]]] all_item_and_group_names: typing.Dict[str, typing.Set[str]] all_location_and_group_names: typing.Dict[str, typing.Set[str]] non_hintable_names: typing.Dict[str, typing.AbstractSet[str]] spheres: typing.List[typing.Dict[int, typing.Set[int]]] """ each sphere is { player: { location_id, ... } } """ + games_package_cache: GamesPackageCache logger: logging.Logger - def __init__(self, host: str, port: int, server_password: str, password: str, location_check_points: int, - hint_cost: int, item_cheat: bool, release_mode: str = "disabled", collect_mode="disabled", - countdown_mode: str = "auto", remaining_mode: str = "disabled", auto_shutdown: typing.SupportsFloat = 0, - compatibility: int = 2, log_network: bool = False, logger: logging.Logger = logging.getLogger()): + def __init__( + self, + host: str, + port: int, + server_password: str, + password: str, + location_check_points: int, + hint_cost: int, + item_cheat: bool, + release_mode: str = "disabled", + collect_mode="disabled", + countdown_mode: str = "auto", + remaining_mode: str = "disabled", + auto_shutdown: typing.SupportsFloat = 0, + compatibility: int = 2, + log_network: bool = False, + games_package_cache: GamesPackageCache | None = None, + logger: logging.Logger = logging.getLogger() + ) -> None: self.logger = logger super(Context, self).__init__() self.slot_info = {} @@ -305,6 +323,7 @@ class Context: self.save_dirty = False self.tags = ['AP'] self.games: typing.Dict[int, str] = {} + self.played_games = set() self.minimum_client_versions: typing.Dict[int, Version] = {} self.seed_name = "" self.groups = {} @@ -314,9 +333,10 @@ class Context: self.stored_data_notification_clients = collections.defaultdict(weakref.WeakSet) self.read_data = {} self.spheres = [] + self.games_package_cache = games_package_cache or GamesPackageCache() # init empty to satisfy linter, I suppose - self.gamespackage = {} + self.reduced_games_package = {} self.checksums = {} self.item_name_groups = {} self.location_name_groups = {} @@ -328,50 +348,11 @@ class Context: lambda: Utils.KeyedDefaultDict(lambda code: f'Unknown location (ID:{code})')) self.non_hintable_names = collections.defaultdict(frozenset) - self._load_game_data() - - # Data package retrieval - def _load_game_data(self): - import worlds - self.gamespackage = worlds.network_data_package["games"] - - self.item_name_groups = {world_name: world.item_name_groups for world_name, world in - worlds.AutoWorldRegister.world_types.items()} - self.location_name_groups = {world_name: world.location_name_groups for world_name, world in - worlds.AutoWorldRegister.world_types.items()} - for world_name, world in worlds.AutoWorldRegister.world_types.items(): - self.non_hintable_names[world_name] = world.hint_blacklist - - for game_package in self.gamespackage.values(): - # remove groups from data sent to clients - del game_package["item_name_groups"] - del game_package["location_name_groups"] - - def _init_game_data(self): - for game_name, game_package in self.gamespackage.items(): - if "checksum" in game_package: - self.checksums[game_name] = game_package["checksum"] - for item_name, item_id in game_package["item_name_to_id"].items(): - self.item_names[game_name][item_id] = item_name - for location_name, location_id in game_package["location_name_to_id"].items(): - self.location_names[game_name][location_id] = location_name - self.all_item_and_group_names[game_name] = \ - set(game_package["item_name_to_id"]) | set(self.item_name_groups[game_name]) - self.all_location_and_group_names[game_name] = \ - set(game_package["location_name_to_id"]) | set(self.location_name_groups.get(game_name, [])) - - archipelago_item_names = self.item_names["Archipelago"] - archipelago_location_names = self.location_names["Archipelago"] - for game in [game_name for game_name in self.gamespackage if game_name != "Archipelago"]: - # Add Archipelago items and locations to each data package. - self.item_names[game].update(archipelago_item_names) - self.location_names[game].update(archipelago_location_names) - def item_names_for_game(self, game: str) -> typing.Optional[typing.Dict[str, int]]: - return self.gamespackage[game]["item_name_to_id"] if game in self.gamespackage else None + return self.reduced_games_package[game]["item_name_to_id"] if game in self.reduced_games_package else None def location_names_for_game(self, game: str) -> typing.Optional[typing.Dict[str, int]]: - return self.gamespackage[game]["location_name_to_id"] if game in self.gamespackage else None + return self.reduced_games_package[game]["location_name_to_id"] if game in self.reduced_games_package else None # General networking async def send_msgs(self, endpoint: Endpoint, msgs: typing.Iterable[dict]) -> bool: @@ -481,19 +462,17 @@ class Context: with open(multidatapath, 'rb') as f: data = f.read() - self._load(self.decompress(data), {}, use_embedded_server_options) + self._load(self.decompress(data), use_embedded_server_options) self.data_filename = multidatapath @staticmethod - def decompress(data: bytes) -> dict: + def decompress(data: bytes) -> typing.Any: format_version = data[0] if format_version > 3: raise Utils.VersionException("Incompatible multidata.") return restricted_loads(zlib.decompress(data[1:])) - def _load(self, decoded_obj: MultiData, game_data_packages: typing.Dict[str, typing.Any], - use_embedded_server_options: bool): - + def _load(self, decoded_obj: MultiData, use_embedded_server_options: bool) -> None: self.read_data = {} # there might be a better place to put this. self.read_data["race_mode"] = lambda: decoded_obj.get("race_mode", 0) @@ -513,6 +492,7 @@ class Context: self.slot_info = decoded_obj["slot_info"] self.games = {slot: slot_info.game for slot, slot_info in self.slot_info.items()} + self.played_games = {"Archipelago"} | {self.games[x] for x in range(1, len(self.games) + 1)} self.groups = {slot: set(slot_info.group_members) for slot, slot_info in self.slot_info.items() if slot_info.type == SlotType.group} @@ -557,18 +537,11 @@ class Context: server_options = decoded_obj.get("server_options", {}) self._set_options(server_options) - # embedded data package - for game_name, data in decoded_obj.get("datapackage", {}).items(): - if game_name in game_data_packages: - data = game_data_packages[game_name] - self.logger.info(f"Loading embedded data package for game {game_name}") - self.gamespackage[game_name] = data - self.item_name_groups[game_name] = data["item_name_groups"] - if "location_name_groups" in data: - self.location_name_groups[game_name] = data["location_name_groups"] - del data["location_name_groups"] - del data["item_name_groups"] # remove from data package, but keep in self.item_name_groups + # load and apply world data and (embedded) data package + self._load_world_data() + self._load_data_package(decoded_obj.get("datapackage", {})) self._init_game_data() + for game_name, data in self.item_name_groups.items(): self.read_data[f"item_name_groups_{game_name}"] = lambda lgame=game_name: self.item_name_groups[lgame] for game_name, data in self.location_name_groups.items(): @@ -577,6 +550,55 @@ class Context: # sorted access spheres self.spheres = decoded_obj.get("spheres", []) + def _load_world_data(self) -> None: + import worlds + + for world_name, world in worlds.AutoWorldRegister.world_types.items(): + # TODO: move hint_blacklist into GamesPackage? + self.non_hintable_names[world_name] = world.hint_blacklist + + def _load_data_package(self, data_package: dict[str, GamesPackage]) -> None: + """Populates reduced_games_package, item_name_groups, location_name_groups from static data and data_package""" + # NOTE: for worlds loaded from db, only checksum is set in GamesPackage, but this is handled by cache + for game_name in sorted(self.played_games): + if game_name in data_package: + self.logger.info(f"Loading embedded data package for game {game_name}") + data = self.games_package_cache.get(game_name, data_package[game_name]) + else: + # NOTE: we still allow uploading a game without datapackage. Once that is changed, we could drop this. + data = self.games_package_cache.get_static(game_name) + ( + self.reduced_games_package[game_name], + self.item_name_groups[game_name], + self.location_name_groups[game_name], + ) = data + + del self.games_package_cache # Not used past this point. Free memory. + + def _init_game_data(self) -> None: + """Update internal values from previously loaded data packages""" + for game_name, game_package in self.reduced_games_package.items(): + if game_name not in self.played_games: + continue + if "checksum" in game_package: + self.checksums[game_name] = game_package["checksum"] + # NOTE: we could save more memory by moving the stuff below to data package cache as well + for item_name, item_id in game_package["item_name_to_id"].items(): + self.item_names[game_name][item_id] = item_name + for location_name, location_id in game_package["location_name_to_id"].items(): + self.location_names[game_name][location_id] = location_name + self.all_item_and_group_names[game_name] = \ + set(game_package["item_name_to_id"]) | set(self.item_name_groups[game_name]) + self.all_location_and_group_names[game_name] = \ + set(game_package["location_name_to_id"]) | set(self.location_name_groups.get(game_name, [])) + + archipelago_item_names = self.item_names["Archipelago"] + archipelago_location_names = self.location_names["Archipelago"] + for game in [game_name for game_name in self.reduced_games_package if game_name != "Archipelago"]: + # Add Archipelago items and locations to each data package. + self.item_names[game].update(archipelago_item_names) + self.location_names[game].update(archipelago_location_names) + # saving def save(self, now=False) -> bool: @@ -917,12 +939,10 @@ async def server(websocket: "ServerConnection", path: str = "/", ctx: Context = async def on_client_connected(ctx: Context, client: Client): - games = {ctx.games[x] for x in range(1, len(ctx.games) + 1)} - games.add("Archipelago") await ctx.send_msgs(client, [{ 'cmd': 'RoomInfo', 'password': bool(ctx.password), - 'games': games, + 'games': sorted(ctx.played_games), # tags are for additional features in the communication. # Name them by feature or fork, as you feel is appropriate. 'tags': ctx.tags, @@ -931,8 +951,7 @@ async def on_client_connected(ctx: Context, client: Client): 'permissions': get_permissions(ctx), 'hint_cost': ctx.hint_cost, 'location_check_points': ctx.location_check_points, - 'datapackage_checksums': {game: game_data["checksum"] for game, game_data - in ctx.gamespackage.items() if game in games and "checksum" in game_data}, + 'datapackage_checksums': ctx.checksums, 'seed_name': ctx.seed_name, 'time': time.time(), }]) @@ -1931,25 +1950,11 @@ async def process_client_cmd(ctx: Context, client: Client, args: dict): await ctx.send_msgs(client, reply) elif cmd == "GetDataPackage": - exclusions = args.get("exclusions", []) - if "games" in args: - games = {name: game_data for name, game_data in ctx.gamespackage.items() - if name in set(args.get("games", []))} - await ctx.send_msgs(client, [{"cmd": "DataPackage", - "data": {"games": games}}]) - # TODO: remove exclusions behaviour around 0.5.0 - elif exclusions: - exclusions = set(exclusions) - games = {name: game_data for name, game_data in ctx.gamespackage.items() - if name not in exclusions} - - package = {"games": games} - await ctx.send_msgs(client, [{"cmd": "DataPackage", - "data": package}]) - - else: - await ctx.send_msgs(client, [{"cmd": "DataPackage", - "data": {"games": ctx.gamespackage}}]) + games = { + name: game_data for name, game_data in ctx.reduced_games_package.items() + if name in set(args.get("games", [])) + } + await ctx.send_msgs(client, [{"cmd": "DataPackage", "data": {"games": games}}]) elif client.auth: if cmd == "ConnectUpdate": diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index e353cf2ab2..060afafd37 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -13,6 +13,7 @@ import threading import time import typing import sys +from asyncio import AbstractEventLoop import websockets from pony.orm import commit, db_session, select @@ -24,8 +25,10 @@ from MultiServer import ( server_per_message_deflate_factory, ) from Utils import restricted_loads, cache_argsless +from NetUtils import GamesPackage +from apmw.webhost.customserver.gamespackage.cache import DBGamesPackageCache from .locker import Locker -from .models import Command, GameDataPackage, Room, db +from .models import Command, Room, db class CustomClientMessageProcessor(ClientMessageProcessor): @@ -62,18 +65,39 @@ class DBCommandProcessor(ServerCommandProcessor): class WebHostContext(Context): room_id: int + video: dict[tuple[int, int], tuple[str, str]] + main_loop: AbstractEventLoop + static_server_data: StaticServerData - def __init__(self, static_server_data: dict, logger: logging.Logger): + def __init__( + self, + static_server_data: StaticServerData, + games_package_cache: DBGamesPackageCache, + logger: logging.Logger, + ) -> None: # static server data is used during _load_game_data to load required data, # without needing to import worlds system, which takes quite a bit of memory - self.static_server_data = static_server_data - super(WebHostContext, self).__init__("", 0, "", "", 1, - 40, True, "enabled", "enabled", - "enabled", 0, 2, logger=logger) - del self.static_server_data - self.main_loop = asyncio.get_running_loop() - self.video = {} + super(WebHostContext, self).__init__( + "", + 0, + "", + "", + 1, + 40, + True, + "enabled", + "enabled", + "enabled", + 0, + 2, + games_package_cache=games_package_cache, + logger=logger, + ) self.tags = ["AP", "WebHost"] + self.video = {} + self.main_loop = asyncio.get_running_loop() + self.static_server_data = static_server_data + self.games_package_cache = games_package_cache def __del__(self): try: @@ -83,12 +107,6 @@ class WebHostContext(Context): except ImportError: self.logger.debug("Context destroyed") - def _load_game_data(self): - for key, value in self.static_server_data.items(): - # NOTE: attributes are mutable and shared, so they will have to be copied before being modified - setattr(self, key, value) - self.non_hintable_names = collections.defaultdict(frozenset, self.non_hintable_names) - async def listen_to_db_commands(self): cmdprocessor = DBCommandProcessor(self) @@ -118,42 +136,14 @@ class WebHostContext(Context): self.port = get_random_port() multidata = self.decompress(room.seed.multidata) - game_data_packages = {} + return self._load(multidata, True) - static_gamespackage = self.gamespackage # this is shared across all rooms - static_item_name_groups = self.item_name_groups - static_location_name_groups = self.location_name_groups - self.gamespackage = {"Archipelago": static_gamespackage.get("Archipelago", {})} # this may be modified by _load - self.item_name_groups = {"Archipelago": static_item_name_groups.get("Archipelago", {})} - self.location_name_groups = {"Archipelago": static_location_name_groups.get("Archipelago", {})} - missing_checksum = False - - for game in list(multidata.get("datapackage", {})): - game_data = multidata["datapackage"][game] - if "checksum" in game_data: - if static_gamespackage.get(game, {}).get("checksum") == game_data["checksum"]: - # non-custom. remove from multidata and use static data - # games package could be dropped from static data once all rooms embed data package - del multidata["datapackage"][game] - else: - row = GameDataPackage.get(checksum=game_data["checksum"]) - if row: # None if rolled on >= 0.3.9 but uploaded to <= 0.3.8. multidata should be complete - game_data_packages[game] = restricted_loads(row.data) - continue - else: - self.logger.warning(f"Did not find game_data_package for {game}: {game_data['checksum']}") - else: - missing_checksum = True # Game rolled on old AP and will load data package from multidata - self.gamespackage[game] = static_gamespackage.get(game, {}) - self.item_name_groups[game] = static_item_name_groups.get(game, {}) - self.location_name_groups[game] = static_location_name_groups.get(game, {}) - - if not game_data_packages and not missing_checksum: - # all static -> use the static dicts directly - self.gamespackage = static_gamespackage - self.item_name_groups = static_item_name_groups - self.location_name_groups = static_location_name_groups - return self._load(multidata, game_data_packages, True) + def _load_world_data(self): + # Use static_server_data, but skip static data package since that is in cache anyway. + # Also NOT importing worlds here! + # FIXME: does this copy the non_hintable_names (also for games not part of the room)? + self.non_hintable_names = collections.defaultdict(frozenset, self.static_server_data["non_hintable_names"]) + del self.static_server_data # Not used past this point. Free memory. def init_save(self, enabled: bool = True): self.saving = enabled @@ -185,34 +175,23 @@ def get_random_port(): return random.randint(49152, 65535) +class StaticServerData(typing.TypedDict, total=True): + non_hintable_names: dict[str, typing.AbstractSet[str]] + games_package: dict[str, GamesPackage] + + @cache_argsless -def get_static_server_data() -> dict: +def get_static_server_data() -> StaticServerData: import worlds - data = { + + return { "non_hintable_names": { world_name: world.hint_blacklist for world_name, world in worlds.AutoWorldRegister.world_types.items() }, - "gamespackage": { - world_name: { - key: value - for key, value in game_package.items() - if key not in ("item_name_groups", "location_name_groups") - } - for world_name, game_package in worlds.network_data_package["games"].items() - }, - "item_name_groups": { - world_name: world.item_name_groups - for world_name, world in worlds.AutoWorldRegister.world_types.items() - }, - "location_name_groups": { - world_name: world.location_name_groups - for world_name, world in worlds.AutoWorldRegister.world_types.items() - }, + "games_package": worlds.network_data_package["games"] } - return data - def set_up_logging(room_id) -> logging.Logger: import os @@ -245,9 +224,18 @@ def tear_down_logging(room_id): del logging.Logger.manager.loggerDict[logger_name] -def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, - cert_file: typing.Optional[str], cert_key_file: typing.Optional[str], - host: str, rooms_to_run: multiprocessing.Queue, rooms_shutting_down: multiprocessing.Queue): +def run_server_process( + name: str, + ponyconfig: dict[str, typing.Any], + static_server_data: StaticServerData, + cert_file: typing.Optional[str], + cert_key_file: typing.Optional[str], + host: str, + rooms_to_run: multiprocessing.Queue, + rooms_shutting_down: multiprocessing.Queue +) -> None: + import gc + from setproctitle import setproctitle setproctitle(name) @@ -263,6 +251,9 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, resource.setrlimit(resource.RLIMIT_NOFILE, (file_limit, file_limit)) del resource, file_limit + # prime the data package cache with static data + games_package_cache = DBGamesPackageCache(static_server_data["games_package"]) + # establish DB connection for multidata and multisave db.bind(**ponyconfig) db.generate_mapping(check_tables=False) @@ -270,8 +261,6 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, if "worlds" in sys.modules: raise Exception("Worlds system should not be loaded in the custom server.") - import gc - if not cert_file: def get_ssl_context(): return None @@ -296,7 +285,7 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, with Locker(f"RoomLocker {room_id}"): try: logger = set_up_logging(room_id) - ctx = WebHostContext(static_server_data, logger) + ctx = WebHostContext(static_server_data, games_package_cache, logger) ctx.load(room_id) ctx.init_save() assert ctx.server is None diff --git a/apmw/__init__.py b/apmw/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apmw/multiserver/__init__.py b/apmw/multiserver/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apmw/multiserver/gamespackage/__init__.py b/apmw/multiserver/gamespackage/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apmw/multiserver/gamespackage/cache.py b/apmw/multiserver/gamespackage/cache.py new file mode 100644 index 0000000000..1a4dff9f81 --- /dev/null +++ b/apmw/multiserver/gamespackage/cache.py @@ -0,0 +1,110 @@ +import typing as t +from weakref import WeakValueDictionary + +from NetUtils import GamesPackage, DataPackage + +GameAndChecksum = tuple[str, str | None] +ItemNameGroups = dict[str, list[str]] +LocationNameGroups = dict[str, list[str]] + + +K = t.TypeVar("K") +V = t.TypeVar("V") + + +class DictLike(dict[K, V]): + __slots__ = ("__weakref__",) + + +class GamesPackageCache: + # NOTE: this uses 3 separate collections because unpacking the get() result would end the container lifetime + _reduced_games_packages: WeakValueDictionary[GameAndChecksum, GamesPackage] + """Does not include item_name_groups nor location_name_groups""" + _item_name_groups: WeakValueDictionary[GameAndChecksum, dict[str, list[str]]] + _location_name_groups: WeakValueDictionary[GameAndChecksum, dict[str, list[str]]] + + def __init__(self) -> None: + self._reduced_games_packages = WeakValueDictionary() + self._item_name_groups = WeakValueDictionary() + self._location_name_groups = WeakValueDictionary() + + def _cached_item_name(self, key: GameAndChecksum, item_name: str) -> str: + """Returns a reference to an already-stored copy of item_name, or item_name""" + # TODO: there gotta be a better way, but maybe only in a C module? + for cached_item_name in self._reduced_games_packages[key].keys(): + if cached_item_name == item_name: + return cached_item_name + return item_name + + def _cached_location_name(self, key: GameAndChecksum, location_name: str) -> str: + """Returns a reference to an already-stored copy of location_name, or location_name""" + # TODO: as above + for cached_item_name in self._reduced_games_packages[key].keys(): + if cached_item_name == location_name: + return cached_item_name + return location_name + + def _get( + self, + cache_key: GameAndChecksum, + ) -> tuple[GamesPackage | None, ItemNameGroups | None, LocationNameGroups | None]: + if cache_key[1] is None: + return None, None, None + return ( + self._reduced_games_packages.get(cache_key, None), + self._item_name_groups.get(cache_key, None), + self._location_name_groups.get(cache_key, None), + ) + + def get( + self, + game: str, + full_games_package: GamesPackage, + ) -> tuple[GamesPackage, ItemNameGroups, LocationNameGroups]: + """Loads and caches embedded data package provided by multidata""" + cache_key = (game, full_games_package.get("checksum", None)) + cached_reduced_games_package, cached_item_name_groups, cached_location_name_groups = self._get(cache_key) + + if cached_reduced_games_package is None: + cached_reduced_games_package = t.cast( + t.Any, + DictLike( + { + "item_name_to_id": full_games_package["item_name_to_id"], + "location_name_to_id": full_games_package["location_name_to_id"], + "checksum": full_games_package["checksum"], + } + ), + ) + if cache_key[1] is not None: # only cache if checksum is available + self._reduced_games_packages[cache_key] = cached_reduced_games_package + + if cached_item_name_groups is None: + cached_item_name_groups = DictLike( + { + group_name: [self._cached_item_name(cache_key, item_name) for item_name in group_items] + for group_name, group_items in full_games_package["item_name_groups"].items() + } + ) + if cache_key[1] is not None: # only cache if checksum is available + self._item_name_groups[cache_key] = cached_item_name_groups + + if cached_location_name_groups is None: + cached_location_name_groups = DictLike( + { + group_name: [ + self._cached_location_name(cache_key, location_name) for location_name in group_locations + ] + for group_name, group_locations in full_games_package.get("location_name_groups", {}).items() + } + ) + if cache_key[1] is not None: # only cache if checksum is available + self._location_name_groups[cache_key] = cached_location_name_groups + + return cached_reduced_games_package, cached_item_name_groups, cached_location_name_groups + + def get_static(self, game: str) -> tuple[GamesPackage, ItemNameGroups, LocationNameGroups]: + """Loads legacy data package from installed worlds""" + import worlds + + return self.get(game, worlds.network_data_package["games"][game]) diff --git a/apmw/webhost/__init__.py b/apmw/webhost/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apmw/webhost/customserver/__init__.py b/apmw/webhost/customserver/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apmw/webhost/customserver/gamespackage/__init__.py b/apmw/webhost/customserver/gamespackage/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apmw/webhost/customserver/gamespackage/cache.py b/apmw/webhost/customserver/gamespackage/cache.py new file mode 100644 index 0000000000..8e3f7c83e0 --- /dev/null +++ b/apmw/webhost/customserver/gamespackage/cache.py @@ -0,0 +1,32 @@ +import typing as t + +from NetUtils import GamesPackage +from Utils import restricted_loads +from WebHostLib.models import GameDataPackage +from apmw.multiserver.gamespackage.cache import GamesPackageCache, ItemNameGroups, LocationNameGroups + + +class DBGamesPackageCache(GamesPackageCache): + _static: dict[str, tuple[GamesPackage, ItemNameGroups, LocationNameGroups]] + + def __init__(self, static_games_package: dict[str, GamesPackage]) -> None: + super().__init__() + self._static = {game: super().get(game, games_package) for game, games_package in static_games_package.items()} + + def get( + self, + game: str, + full_games_package: GamesPackage, + ) -> tuple[GamesPackage, ItemNameGroups, LocationNameGroups]: + # for games started on webhost, full_games_package is likely unpopulated and only has the checksum field + cache_key = (game, full_games_package.get("checksum", None)) + cached = self._get(cache_key) + if any(value is None for value in cached): + row = GameDataPackage.get(checksum=full_games_package["checksum"]) + if row: # None if rolled on >= 0.3.9 but uploaded to <= 0.3.8 ... + return super().get(game, restricted_loads(row.data)) + return super().get(game, full_games_package) # ... in which case full_games_package should be populated + return t.cast(tuple[GamesPackage, ItemNameGroups, LocationNameGroups], cached) + + def get_static(self, game: str) -> tuple[GamesPackage, ItemNameGroups, LocationNameGroups]: + return self._static[game] From 59051cda245e08af3865126c255f38d178d74a24 Mon Sep 17 00:00:00 2001 From: black-sliver <59490463+black-sliver@users.noreply.github.com> Date: Fri, 27 Feb 2026 01:23:28 +0100 Subject: [PATCH 02/15] MultiServer: fix string deduplication in data package cache --- apmw/multiserver/gamespackage/cache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apmw/multiserver/gamespackage/cache.py b/apmw/multiserver/gamespackage/cache.py index 1a4dff9f81..1ce9bffafb 100644 --- a/apmw/multiserver/gamespackage/cache.py +++ b/apmw/multiserver/gamespackage/cache.py @@ -31,7 +31,7 @@ class GamesPackageCache: def _cached_item_name(self, key: GameAndChecksum, item_name: str) -> str: """Returns a reference to an already-stored copy of item_name, or item_name""" # TODO: there gotta be a better way, but maybe only in a C module? - for cached_item_name in self._reduced_games_packages[key].keys(): + for cached_item_name in self._reduced_games_packages[key]["item_name_to_id"].keys(): if cached_item_name == item_name: return cached_item_name return item_name @@ -39,9 +39,9 @@ class GamesPackageCache: def _cached_location_name(self, key: GameAndChecksum, location_name: str) -> str: """Returns a reference to an already-stored copy of location_name, or location_name""" # TODO: as above - for cached_item_name in self._reduced_games_packages[key].keys(): - if cached_item_name == location_name: - return cached_item_name + for cached_location_name in self._reduced_games_packages[key]["location_name_to_id"].keys(): + if cached_location_name == location_name: + return cached_location_name return location_name def _get( From 9c00b546dd42598aaeead40162c96ab8d332c31a Mon Sep 17 00:00:00 2001 From: black-sliver <59490463+black-sliver@users.noreply.github.com> Date: Fri, 27 Feb 2026 01:24:52 +0100 Subject: [PATCH 03/15] MultiServer, customserver: minor formatting fixes --- MultiServer.py | 2 +- WebHostLib/customserver.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MultiServer.py b/MultiServer.py index ac3c9c18c1..c109206a19 100644 --- a/MultiServer.py +++ b/MultiServer.py @@ -271,7 +271,7 @@ class Context: compatibility: int = 2, log_network: bool = False, games_package_cache: GamesPackageCache | None = None, - logger: logging.Logger = logging.getLogger() + logger: logging.Logger = logging.getLogger(), ) -> None: self.logger = logger super(Context, self).__init__() diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 060afafd37..5402607d81 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -232,7 +232,7 @@ def run_server_process( cert_key_file: typing.Optional[str], host: str, rooms_to_run: multiprocessing.Queue, - rooms_shutting_down: multiprocessing.Queue + rooms_shutting_down: multiprocessing.Queue, ) -> None: import gc From f4ec11990067a2f578ac337a99f11200831b9cfb Mon Sep 17 00:00:00 2001 From: black-sliver <59490463+black-sliver@users.noreply.github.com> Date: Fri, 27 Feb 2026 01:32:56 +0100 Subject: [PATCH 04/15] MultiServer: fix data package cache for missing checksum case --- apmw/multiserver/gamespackage/cache.py | 43 ++++++++++++++------------ 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/apmw/multiserver/gamespackage/cache.py b/apmw/multiserver/gamespackage/cache.py index 1ce9bffafb..feb8ab5d1c 100644 --- a/apmw/multiserver/gamespackage/cache.py +++ b/apmw/multiserver/gamespackage/cache.py @@ -16,6 +16,24 @@ class DictLike(dict[K, V]): __slots__ = ("__weakref__",) +def _cached_item_name(games_package: GamesPackage, item_name: str) -> str: + """Returns a reference to an already-stored copy of item_name, or item_name""" + # TODO: there gotta be a better way, but maybe only in a C module? + for cached_item_name in games_package["item_name_to_id"].keys(): + if cached_item_name == item_name: + return cached_item_name + return item_name + + +def _cached_location_name(games_package: GamesPackage, location_name: str) -> str: + """Returns a reference to an already-stored copy of location_name, or location_name""" + # TODO: as above + for cached_location_name in games_package["location_name_to_id"].keys(): + if cached_location_name == location_name: + return cached_location_name + return location_name + + class GamesPackageCache: # NOTE: this uses 3 separate collections because unpacking the get() result would end the container lifetime _reduced_games_packages: WeakValueDictionary[GameAndChecksum, GamesPackage] @@ -28,22 +46,6 @@ class GamesPackageCache: self._item_name_groups = WeakValueDictionary() self._location_name_groups = WeakValueDictionary() - def _cached_item_name(self, key: GameAndChecksum, item_name: str) -> str: - """Returns a reference to an already-stored copy of item_name, or item_name""" - # TODO: there gotta be a better way, but maybe only in a C module? - for cached_item_name in self._reduced_games_packages[key]["item_name_to_id"].keys(): - if cached_item_name == item_name: - return cached_item_name - return item_name - - def _cached_location_name(self, key: GameAndChecksum, location_name: str) -> str: - """Returns a reference to an already-stored copy of location_name, or location_name""" - # TODO: as above - for cached_location_name in self._reduced_games_packages[key]["location_name_to_id"].keys(): - if cached_location_name == location_name: - return cached_location_name - return location_name - def _get( self, cache_key: GameAndChecksum, @@ -72,7 +74,7 @@ class GamesPackageCache: { "item_name_to_id": full_games_package["item_name_to_id"], "location_name_to_id": full_games_package["location_name_to_id"], - "checksum": full_games_package["checksum"], + "checksum": full_games_package.get("checksum", None), } ), ) @@ -82,7 +84,9 @@ class GamesPackageCache: if cached_item_name_groups is None: cached_item_name_groups = DictLike( { - group_name: [self._cached_item_name(cache_key, item_name) for item_name in group_items] + group_name: [ + _cached_item_name(cached_reduced_games_package, item_name) for item_name in group_items + ] for group_name, group_items in full_games_package["item_name_groups"].items() } ) @@ -93,7 +97,8 @@ class GamesPackageCache: cached_location_name_groups = DictLike( { group_name: [ - self._cached_location_name(cache_key, location_name) for location_name in group_locations + _cached_location_name(cached_reduced_games_package, location_name) + for location_name in group_locations ] for group_name, group_locations in full_games_package.get("location_name_groups", {}).items() } From 1cdd6570687a7c87ca553e6fbf7b02b58b4e511e Mon Sep 17 00:00:00 2001 From: black-sliver <59490463+black-sliver@users.noreply.github.com> Date: Sat, 28 Feb 2026 15:34:05 +0100 Subject: [PATCH 05/15] MultiServer: improve string deduplication in games package cache --- apmw/multiserver/gamespackage/cache.py | 31 +++++--------------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/apmw/multiserver/gamespackage/cache.py b/apmw/multiserver/gamespackage/cache.py index feb8ab5d1c..373acadea6 100644 --- a/apmw/multiserver/gamespackage/cache.py +++ b/apmw/multiserver/gamespackage/cache.py @@ -16,24 +16,6 @@ class DictLike(dict[K, V]): __slots__ = ("__weakref__",) -def _cached_item_name(games_package: GamesPackage, item_name: str) -> str: - """Returns a reference to an already-stored copy of item_name, or item_name""" - # TODO: there gotta be a better way, but maybe only in a C module? - for cached_item_name in games_package["item_name_to_id"].keys(): - if cached_item_name == item_name: - return cached_item_name - return item_name - - -def _cached_location_name(games_package: GamesPackage, location_name: str) -> str: - """Returns a reference to an already-stored copy of location_name, or location_name""" - # TODO: as above - for cached_location_name in games_package["location_name_to_id"].keys(): - if cached_location_name == location_name: - return cached_location_name - return location_name - - class GamesPackageCache: # NOTE: this uses 3 separate collections because unpacking the get() result would end the container lifetime _reduced_games_packages: WeakValueDictionary[GameAndChecksum, GamesPackage] @@ -82,11 +64,11 @@ class GamesPackageCache: self._reduced_games_packages[cache_key] = cached_reduced_games_package if cached_item_name_groups is None: + # optimize strings to be references instead of copies + item_names = {name: name for name in cached_reduced_games_package["item_name_to_id"].keys()} cached_item_name_groups = DictLike( { - group_name: [ - _cached_item_name(cached_reduced_games_package, item_name) for item_name in group_items - ] + group_name: [item_names.get(item_name, item_name) for item_name in group_items] for group_name, group_items in full_games_package["item_name_groups"].items() } ) @@ -94,12 +76,11 @@ class GamesPackageCache: self._item_name_groups[cache_key] = cached_item_name_groups if cached_location_name_groups is None: + # optimize strings to be references instead of copies + location_names = {name: name for name in cached_reduced_games_package["location_name_to_id"].keys()} cached_location_name_groups = DictLike( { - group_name: [ - _cached_location_name(cached_reduced_games_package, location_name) - for location_name in group_locations - ] + group_name: [location_names.get(location_name, location_name) for location_name in group_locations] for group_name, group_locations in full_games_package.get("location_name_groups", {}).items() } ) From 4a355f35859ed2bda0e455d541f58164bd42da52 Mon Sep 17 00:00:00 2001 From: black-sliver <59490463+black-sliver@users.noreply.github.com> Date: Sat, 28 Feb 2026 15:34:58 +0100 Subject: [PATCH 06/15] customserver: handle missing checksum in datapackage cache --- apmw/webhost/customserver/gamespackage/cache.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apmw/webhost/customserver/gamespackage/cache.py b/apmw/webhost/customserver/gamespackage/cache.py index 8e3f7c83e0..dcf150cdec 100644 --- a/apmw/webhost/customserver/gamespackage/cache.py +++ b/apmw/webhost/customserver/gamespackage/cache.py @@ -22,6 +22,8 @@ class DBGamesPackageCache(GamesPackageCache): cache_key = (game, full_games_package.get("checksum", None)) cached = self._get(cache_key) if any(value is None for value in cached): + if "checksum" not in full_games_package: + return super().get(game, full_games_package) # predates checksum, assume fully populated row = GameDataPackage.get(checksum=full_games_package["checksum"]) if row: # None if rolled on >= 0.3.9 but uploaded to <= 0.3.8 ... return super().get(game, restricted_loads(row.data)) From 63bc205dab372fa713c23acccd6a437d54ebeb85 Mon Sep 17 00:00:00 2001 From: black-sliver <59490463+black-sliver@users.noreply.github.com> Date: Sat, 28 Feb 2026 15:35:24 +0100 Subject: [PATCH 07/15] customserver: typing cleanup in games package cache --- apmw/webhost/customserver/gamespackage/cache.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apmw/webhost/customserver/gamespackage/cache.py b/apmw/webhost/customserver/gamespackage/cache.py index dcf150cdec..dcb11c5cbf 100644 --- a/apmw/webhost/customserver/gamespackage/cache.py +++ b/apmw/webhost/customserver/gamespackage/cache.py @@ -1,5 +1,3 @@ -import typing as t - from NetUtils import GamesPackage from Utils import restricted_loads from WebHostLib.models import GameDataPackage @@ -28,7 +26,7 @@ class DBGamesPackageCache(GamesPackageCache): if row: # None if rolled on >= 0.3.9 but uploaded to <= 0.3.8 ... return super().get(game, restricted_loads(row.data)) return super().get(game, full_games_package) # ... in which case full_games_package should be populated - return t.cast(tuple[GamesPackage, ItemNameGroups, LocationNameGroups], cached) + return cached # type: ignore # mypy doesn't understand any value is None def get_static(self, game: str) -> tuple[GamesPackage, ItemNameGroups, LocationNameGroups]: return self._static[game] From 6f7fce9c73390a05dd665ef0f15725ef85448f69 Mon Sep 17 00:00:00 2001 From: black-sliver <59490463+black-sliver@users.noreply.github.com> Date: Sat, 28 Feb 2026 15:36:17 +0100 Subject: [PATCH 08/15] Test, MultiServer, customser: add tests for games package cache --- test/multiserver/__init__.py | 0 test/multiserver/test_gamespackage_cache.py | 129 +++++++++++++++++ test/webhost/customserver/__init__.py | 0 .../customserver/test_gamespackage_cache.py | 134 ++++++++++++++++++ 4 files changed, 263 insertions(+) create mode 100644 test/multiserver/__init__.py create mode 100644 test/multiserver/test_gamespackage_cache.py create mode 100644 test/webhost/customserver/__init__.py create mode 100644 test/webhost/customserver/test_gamespackage_cache.py diff --git a/test/multiserver/__init__.py b/test/multiserver/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/multiserver/test_gamespackage_cache.py b/test/multiserver/test_gamespackage_cache.py new file mode 100644 index 0000000000..f59486f84e --- /dev/null +++ b/test/multiserver/test_gamespackage_cache.py @@ -0,0 +1,129 @@ +import typing as t +from copy import deepcopy +from unittest import TestCase + +import NetUtils +from NetUtils import GamesPackage +from apmw.multiserver.gamespackage.cache import GamesPackageCache + + +class GamesPackageCacheTest(TestCase): + cache: GamesPackageCache + any_game: t.ClassVar[str] = "APQuest" + example_games_package: GamesPackage = { + "item_name_to_id": {"Item 1": 1}, + "item_name_groups": {"Everything": ["Item 1"]}, + "location_name_to_id": {"Location 1": 1}, + "location_name_groups": {"Everywhere": ["Location 1"]}, + "checksum": "1234", + } + + def setUp(self) -> None: + self.cache = GamesPackageCache() + + def test_get_static_is_same(self) -> None: + """Tests that get_static returns the same objects twice""" + reduced_games_package1, item_name_groups1, location_name_groups1 = self.cache.get_static(self.any_game) + reduced_games_package2, item_name_groups2, location_name_groups2 = self.cache.get_static(self.any_game) + self.assertIs(reduced_games_package1, reduced_games_package2) + self.assertIs(item_name_groups1, item_name_groups2) + self.assertIs(location_name_groups1, location_name_groups2) + + def test_get_static_data_format(self) -> None: + """Tests that get_static returns data in the correct format""" + reduced_games_package, item_name_groups, location_name_groups = self.cache.get_static(self.any_game) + self.assertTrue(reduced_games_package["checksum"]) + self.assertTrue(reduced_games_package["item_name_to_id"]) + self.assertTrue(reduced_games_package["location_name_to_id"]) + self.assertNotIn("item_name_groups", reduced_games_package) + self.assertNotIn("location_name_groups", reduced_games_package) + self.assertTrue(item_name_groups["Everything"]) + self.assertTrue(location_name_groups["Everywhere"]) + + def test_get_static_is_serializable(self) -> None: + """Tests that get_static returns data that can be serialized""" + NetUtils.encode(self.cache.get_static(self.any_game)) + + def test_get_static_missing_raises(self) -> None: + """Tests that get_static raises KeyError if the world is missing""" + with self.assertRaises(KeyError): + _ = self.cache.get_static("Does not exist") + + def test_eviction(self) -> None: + """Tests that unused items get evicted from cache""" + game_name = "Test" + before_add = len(self.cache._reduced_games_packages) + data = self.cache.get(game_name, self.example_games_package) + self.assertTrue(data) + self.assertEqual(before_add + 1, len(self.cache._reduced_games_packages)) + + del data + if len(self.cache._reduced_games_packages) != before_add: # gc.collect() may not even be required + import gc + + gc.collect() + + self.assertEqual(before_add, len(self.cache._reduced_games_packages)) + + def test_get_required_field(self) -> None: + """Tests that missing required field raises a KeyError""" + for field in ("item_name_to_id", "location_name_to_id", "item_name_groups"): + with self.subTest(field=field): + games_package = deepcopy(self.example_games_package) + del games_package[field] # type: ignore + with self.assertRaises(KeyError): + _ = self.cache.get(self.any_game, games_package) + + def test_get_optional_properties(self) -> None: + """Tests that missing optional field works""" + for field in ("checksum", "location_name_groups"): + with self.subTest(field=field): + games_package = deepcopy(self.example_games_package) + del games_package[field] # type: ignore + _, item_name_groups, location_name_groups = self.cache.get(self.any_game, games_package) + self.assertTrue(item_name_groups) + self.assertEqual(field != "location_name_groups", bool(location_name_groups)) + + def test_item_name_deduplication(self) -> None: + n = 1 + s1 = f"Item {n}" + s2 = f"Item {n}" + # check if the deduplication is actually gonna do anything + self.assertIsNot(s1, s2) + self.assertEqual(s1, s2) + # do the thing + game_name = "Test" + games_package: GamesPackage = { + "item_name_to_id": {s1: n}, + "item_name_groups": {"Everything": [s2]}, + "location_name_to_id": {}, + "location_name_groups": {}, + "checksum": "1234", + } + reduced_games_package, item_name_groups, location_name_groups = self.cache.get(game_name, games_package) + self.assertIs( + next(iter(reduced_games_package["item_name_to_id"].keys())), + item_name_groups["Everything"][0], + ) + + def test_location_name_deduplication(self) -> None: + n = 1 + s1 = f"Location {n}" + s2 = f"Location {n}" + # check if the deduplication is actually gonna do anything + self.assertIsNot(s1, s2) + self.assertEqual(s1, s2) + # do the thing + game_name = "Test" + games_package: GamesPackage = { + "item_name_to_id": {}, + "item_name_groups": {}, + "location_name_to_id": {s1: n}, + "location_name_groups": {"Everywhere": [s2]}, + "checksum": "1234", + } + reduced_games_package, item_name_groups, location_name_groups = self.cache.get(game_name, games_package) + self.assertIs( + next(iter(reduced_games_package["location_name_to_id"].keys())), + location_name_groups["Everywhere"][0], + ) diff --git a/test/webhost/customserver/__init__.py b/test/webhost/customserver/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/webhost/customserver/test_gamespackage_cache.py b/test/webhost/customserver/test_gamespackage_cache.py new file mode 100644 index 0000000000..01c99e3c69 --- /dev/null +++ b/test/webhost/customserver/test_gamespackage_cache.py @@ -0,0 +1,134 @@ +import typing as t +from copy import deepcopy + +import Utils +import apmw.webhost.customserver.gamespackage.cache +from NetUtils import GamesPackage +from apmw.webhost.customserver.gamespackage.cache import DBGamesPackageCache +from test.multiserver.test_gamespackage_cache import GamesPackageCacheTest + + +class FakeGameDataPackage: + _rows: "t.ClassVar[dict[str, FakeGameDataPackage]]" = {} + data: bytes + + @classmethod + def get(cls, checksum: str) -> "FakeGameDataPackage | None": + return cls._rows.get(checksum, None) + + @classmethod + def add(cls, checksum: str, full_games_package: GamesPackage) -> None: + row = FakeGameDataPackage() + row.data = Utils.restricted_dumps(full_games_package) + cls._rows[checksum] = row + + +class DBGamesPackageCacheTest(GamesPackageCacheTest): + cache: DBGamesPackageCache + any_game: t.ClassVar[str] = "My Game" + static_data: t.ClassVar[dict[str, GamesPackage]] = { # noqa: pycharm doesn't understand this + "My Game": { + "item_name_to_id": {"Item 1": 1}, + "location_name_to_id": {"Location 1": 1}, + "item_name_groups": {"Everything": ["Item 1"]}, + "location_name_groups": {"Everywhere": ["Location 1"]}, + "checksum": "2345", + } + } + orig_db_type: t.Any + + def setUp(self) -> None: + self.orig_db_type = apmw.webhost.customserver.gamespackage.cache.GameDataPackage # type: ignore[attr-defined] + self.cache = DBGamesPackageCache(self.static_data) + apmw.webhost.customserver.gamespackage.cache.GameDataPackage = FakeGameDataPackage # type: ignore + + def tearDown(self) -> None: + apmw.webhost.customserver.gamespackage.cache.GameDataPackage = self.orig_db_type # type: ignore[attr-defined] + + def assert_conversion( + self, + full_games_package: GamesPackage, + reduced_games_package: dict[str, t.Any], + item_name_groups: dict[str, t.Any], + location_name_groups: dict[str, t.Any], + ) -> None: + for key in ("item_name_to_id", "location_name_to_id", "checksum"): + if key in full_games_package: + self.assertEqual(reduced_games_package[key], full_games_package[key]) # noqa: pycharm + self.assertEqual(item_name_groups, full_games_package["item_name_groups"]) + self.assertEqual(location_name_groups, full_games_package["location_name_groups"]) + + def assert_static_conversion( + self, + full_games_package: GamesPackage, + reduced_games_package: dict[str, t.Any], + item_name_groups: dict[str, t.Any], + location_name_groups: dict[str, t.Any], + ) -> None: + self.assert_conversion(full_games_package, reduced_games_package, item_name_groups, location_name_groups) + for key in ("item_name_to_id", "location_name_to_id", "checksum"): + self.assertIs(reduced_games_package[key], full_games_package[key]) # noqa: pycharm + + def test_get_static_contents(self) -> None: + """Tests that get_static returns the correct data""" + reduced_games_package, item_name_groups, location_name_groups = self.cache.get_static(self.any_game) + for key in ("item_name_to_id", "location_name_to_id", "checksum"): + self.assertIs(reduced_games_package[key], self.static_data[self.any_game][key]) # noqa: pycharm + self.assertEqual(item_name_groups, self.static_data[self.any_game]["item_name_groups"]) + self.assertEqual(location_name_groups, self.static_data[self.any_game]["location_name_groups"]) + + def test_static_not_evicted(self) -> None: + """Tests that static data is not evicted from cache during gc""" + import gc + + game_name = next(iter(self.static_data.keys())) + ids = [id(o) for o in self.cache.get_static(game_name)] + gc.collect() + self.assertEqual(ids, [id(o) for o in self.cache.get_static(game_name)]) + + def test_get_is_static(self) -> None: + """Tests that a get with correct checksum return the static items""" + # NOTE: this is only true for the DB cache, not the "regular" one, since we want to avoid loading worlds there + cks: GamesPackage = {"checksum": self.static_data[self.any_game]["checksum"]} # noqa: pycharm doesn't like this + reduced_games_package1, item_name_groups1, location_name_groups1 = self.cache.get(self.any_game, cks) + reduced_games_package2, item_name_groups2, location_name_groups2 = self.cache.get_static(self.any_game) + self.assertIs(reduced_games_package1, reduced_games_package2) + self.assertEqual(location_name_groups1, location_name_groups2) + self.assertEqual(item_name_groups1, item_name_groups2) + + def test_get_from_db(self) -> None: + """Tests that a get with only checksum will load the full data from db and is cached""" + game_name = "Another Game" + full_games_package = deepcopy(self.static_data[self.any_game]) + full_games_package["checksum"] = "3456" + cks: GamesPackage = {"checksum": full_games_package["checksum"]} # noqa: pycharm doesn't like this + FakeGameDataPackage.add(full_games_package["checksum"], full_games_package) + before_add = len(self.cache._reduced_games_packages) + data = self.cache.get(game_name, cks) + self.assert_conversion(full_games_package, *data) # type: ignore + self.assertEqual(before_add + 1, len(self.cache._reduced_games_packages)) + + def test_get_missing_from_db_uses_full_games_package(self) -> None: + """Tests that a get with full data (missing from db) will use the full data and is cached""" + game_name = "Yet Another Game" + full_games_package = deepcopy(self.static_data[self.any_game]) + full_games_package["checksum"] = "4567" + before_add = len(self.cache._reduced_games_packages) + data = self.cache.get(game_name, full_games_package) + self.assert_conversion(full_games_package, *data) # type: ignore + self.assertEqual(before_add + 1, len(self.cache._reduced_games_packages)) + + def test_get_without_checksum_uses_full_games_package(self) -> None: + """Tests that a get with full data and no checksum will use the full data and is not cached""" + game_name = "Yet Another Game" + full_games_package = deepcopy(self.static_data[self.any_game]) + del full_games_package["checksum"] + before_add = len(self.cache._reduced_games_packages) + data = self.cache.get(game_name, full_games_package) + self.assert_conversion(full_games_package, *data) # type: ignore + self.assertEqual(before_add, len(self.cache._reduced_games_packages)) + + def test_get_missing_from_db_raises(self) -> None: + """Tests that a get that requires a row to exist raise an exception if it doesn't""" + with self.assertRaises(Exception): + _ = self.cache.get("Does not exist", {"checksum": "0000"}) From aad980a3a2e5c65f699f928ae801c192ab796cc2 Mon Sep 17 00:00:00 2001 From: black-sliver <59490463+black-sliver@users.noreply.github.com> Date: Sat, 28 Feb 2026 19:19:27 +0100 Subject: [PATCH 09/15] Test, MultiServer: reorder imports Hopefully this fixes the random test failures with pytest-xdist --- apmw/multiserver/gamespackage/cache.py | 2 +- apmw/webhost/customserver/gamespackage/cache.py | 3 ++- test/webhost/customserver/test_gamespackage_cache.py | 10 ++++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/apmw/multiserver/gamespackage/cache.py b/apmw/multiserver/gamespackage/cache.py index 373acadea6..c90200c744 100644 --- a/apmw/multiserver/gamespackage/cache.py +++ b/apmw/multiserver/gamespackage/cache.py @@ -1,7 +1,7 @@ import typing as t from weakref import WeakValueDictionary -from NetUtils import GamesPackage, DataPackage +from NetUtils import GamesPackage GameAndChecksum = tuple[str, str | None] ItemNameGroups = dict[str, list[str]] diff --git a/apmw/webhost/customserver/gamespackage/cache.py b/apmw/webhost/customserver/gamespackage/cache.py index dcb11c5cbf..3224ae649c 100644 --- a/apmw/webhost/customserver/gamespackage/cache.py +++ b/apmw/webhost/customserver/gamespackage/cache.py @@ -1,8 +1,9 @@ from NetUtils import GamesPackage from Utils import restricted_loads -from WebHostLib.models import GameDataPackage from apmw.multiserver.gamespackage.cache import GamesPackageCache, ItemNameGroups, LocationNameGroups +from WebHostLib.models import GameDataPackage + class DBGamesPackageCache(GamesPackageCache): _static: dict[str, tuple[GamesPackage, ItemNameGroups, LocationNameGroups]] diff --git a/test/webhost/customserver/test_gamespackage_cache.py b/test/webhost/customserver/test_gamespackage_cache.py index 01c99e3c69..16f74791f8 100644 --- a/test/webhost/customserver/test_gamespackage_cache.py +++ b/test/webhost/customserver/test_gamespackage_cache.py @@ -1,12 +1,14 @@ import typing as t from copy import deepcopy -import Utils -import apmw.webhost.customserver.gamespackage.cache -from NetUtils import GamesPackage -from apmw.webhost.customserver.gamespackage.cache import DBGamesPackageCache from test.multiserver.test_gamespackage_cache import GamesPackageCacheTest +import Utils +from NetUtils import GamesPackage + +import apmw.webhost.customserver.gamespackage.cache +from apmw.webhost.customserver.gamespackage.cache import DBGamesPackageCache + class FakeGameDataPackage: _rows: "t.ClassVar[dict[str, FakeGameDataPackage]]" = {} From a294e1cdc904bfb61575c99964b6563c43650b9d Mon Sep 17 00:00:00 2001 From: black-sliver <59490463+black-sliver@users.noreply.github.com> Date: Sat, 28 Feb 2026 19:45:15 +0100 Subject: [PATCH 10/15] customserver: make WebHost import lazy in games package cache and fix test --- .../customserver/gamespackage/cache.py | 8 +++++--- .../customserver/test_gamespackage_cache.py | 20 ++++++++++++------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/apmw/webhost/customserver/gamespackage/cache.py b/apmw/webhost/customserver/gamespackage/cache.py index 3224ae649c..887d128ddd 100644 --- a/apmw/webhost/customserver/gamespackage/cache.py +++ b/apmw/webhost/customserver/gamespackage/cache.py @@ -2,8 +2,6 @@ from NetUtils import GamesPackage from Utils import restricted_loads from apmw.multiserver.gamespackage.cache import GamesPackageCache, ItemNameGroups, LocationNameGroups -from WebHostLib.models import GameDataPackage - class DBGamesPackageCache(GamesPackageCache): _static: dict[str, tuple[GamesPackage, ItemNameGroups, LocationNameGroups]] @@ -22,11 +20,15 @@ class DBGamesPackageCache(GamesPackageCache): cached = self._get(cache_key) if any(value is None for value in cached): if "checksum" not in full_games_package: - return super().get(game, full_games_package) # predates checksum, assume fully populated + return super().get(game, full_games_package) # no checksum, assume fully populated + + from WebHostLib.models import GameDataPackage + row = GameDataPackage.get(checksum=full_games_package["checksum"]) if row: # None if rolled on >= 0.3.9 but uploaded to <= 0.3.8 ... return super().get(game, restricted_loads(row.data)) return super().get(game, full_games_package) # ... in which case full_games_package should be populated + return cached # type: ignore # mypy doesn't understand any value is None def get_static(self, game: str) -> tuple[GamesPackage, ItemNameGroups, LocationNameGroups]: diff --git a/test/webhost/customserver/test_gamespackage_cache.py b/test/webhost/customserver/test_gamespackage_cache.py index 16f74791f8..57baab7513 100644 --- a/test/webhost/customserver/test_gamespackage_cache.py +++ b/test/webhost/customserver/test_gamespackage_cache.py @@ -5,8 +5,6 @@ from test.multiserver.test_gamespackage_cache import GamesPackageCacheTest import Utils from NetUtils import GamesPackage - -import apmw.webhost.customserver.gamespackage.cache from apmw.webhost.customserver.gamespackage.cache import DBGamesPackageCache @@ -37,15 +35,23 @@ class DBGamesPackageCacheTest(GamesPackageCacheTest): "checksum": "2345", } } - orig_db_type: t.Any + orig_db_type: t.ClassVar[type] + + @classmethod + def setUpClass(cls) -> None: + import WebHostLib.models + + cls.orig_db_type = WebHostLib.models.GameDataPackage + WebHostLib.models.GameDataPackage = FakeGameDataPackage # type: ignore def setUp(self) -> None: - self.orig_db_type = apmw.webhost.customserver.gamespackage.cache.GameDataPackage # type: ignore[attr-defined] self.cache = DBGamesPackageCache(self.static_data) - apmw.webhost.customserver.gamespackage.cache.GameDataPackage = FakeGameDataPackage # type: ignore - def tearDown(self) -> None: - apmw.webhost.customserver.gamespackage.cache.GameDataPackage = self.orig_db_type # type: ignore[attr-defined] + @classmethod + def tearDownClass(cls) -> None: + import WebHostLib.models + + WebHostLib.models.GameDataPackage = cls.orig_db_type # type: ignore def assert_conversion( self, From 1346a89a4aecb1ba6826b8e7e5c08cb7029cd0fc Mon Sep 17 00:00:00 2001 From: black-sliver <59490463+black-sliver@users.noreply.github.com> Date: Sat, 28 Feb 2026 20:33:27 +0100 Subject: [PATCH 11/15] customserver: games package cache: fix py3.11 compat --- apmw/webhost/customserver/gamespackage/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apmw/webhost/customserver/gamespackage/cache.py b/apmw/webhost/customserver/gamespackage/cache.py index 887d128ddd..928b10ba28 100644 --- a/apmw/webhost/customserver/gamespackage/cache.py +++ b/apmw/webhost/customserver/gamespackage/cache.py @@ -8,7 +8,7 @@ class DBGamesPackageCache(GamesPackageCache): def __init__(self, static_games_package: dict[str, GamesPackage]) -> None: super().__init__() - self._static = {game: super().get(game, games_package) for game, games_package in static_games_package.items()} + self._static = {game: GamesPackageCache.get(self, game, games_package) for game, games_package in static_games_package.items()} def get( self, From 9996c12ef95ad2caec815867b30801e918104c61 Mon Sep 17 00:00:00 2001 From: black-sliver <59490463+black-sliver@users.noreply.github.com> Date: Sat, 28 Feb 2026 20:33:50 +0100 Subject: [PATCH 12/15] MultiServer, customserver: cache: typing improvements --- apmw/webhost/customserver/gamespackage/cache.py | 6 +++++- test/multiserver/test_gamespackage_cache.py | 3 +++ test/webhost/customserver/test_gamespackage_cache.py | 5 +++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/apmw/webhost/customserver/gamespackage/cache.py b/apmw/webhost/customserver/gamespackage/cache.py index 928b10ba28..970de515ea 100644 --- a/apmw/webhost/customserver/gamespackage/cache.py +++ b/apmw/webhost/customserver/gamespackage/cache.py @@ -1,3 +1,5 @@ +from typing_extensions import override + from NetUtils import GamesPackage from Utils import restricted_loads from apmw.multiserver.gamespackage.cache import GamesPackageCache, ItemNameGroups, LocationNameGroups @@ -10,6 +12,7 @@ class DBGamesPackageCache(GamesPackageCache): super().__init__() self._static = {game: GamesPackageCache.get(self, game, games_package) for game, games_package in static_games_package.items()} + @override def get( self, game: str, @@ -24,12 +27,13 @@ class DBGamesPackageCache(GamesPackageCache): from WebHostLib.models import GameDataPackage - row = GameDataPackage.get(checksum=full_games_package["checksum"]) + row: GameDataPackage | None = GameDataPackage.get(checksum=full_games_package["checksum"]) if row: # None if rolled on >= 0.3.9 but uploaded to <= 0.3.8 ... return super().get(game, restricted_loads(row.data)) return super().get(game, full_games_package) # ... in which case full_games_package should be populated return cached # type: ignore # mypy doesn't understand any value is None + @override def get_static(self, game: str) -> tuple[GamesPackage, ItemNameGroups, LocationNameGroups]: return self._static[game] diff --git a/test/multiserver/test_gamespackage_cache.py b/test/multiserver/test_gamespackage_cache.py index f59486f84e..e764a4e8e8 100644 --- a/test/multiserver/test_gamespackage_cache.py +++ b/test/multiserver/test_gamespackage_cache.py @@ -2,6 +2,8 @@ import typing as t from copy import deepcopy from unittest import TestCase +from typing_extensions import override + import NetUtils from NetUtils import GamesPackage from apmw.multiserver.gamespackage.cache import GamesPackageCache @@ -18,6 +20,7 @@ class GamesPackageCacheTest(TestCase): "checksum": "1234", } + @override def setUp(self) -> None: self.cache = GamesPackageCache() diff --git a/test/webhost/customserver/test_gamespackage_cache.py b/test/webhost/customserver/test_gamespackage_cache.py index 57baab7513..32497526d5 100644 --- a/test/webhost/customserver/test_gamespackage_cache.py +++ b/test/webhost/customserver/test_gamespackage_cache.py @@ -1,6 +1,8 @@ import typing as t from copy import deepcopy +from typing_extensions import override + from test.multiserver.test_gamespackage_cache import GamesPackageCacheTest import Utils @@ -37,6 +39,7 @@ class DBGamesPackageCacheTest(GamesPackageCacheTest): } orig_db_type: t.ClassVar[type] + @override @classmethod def setUpClass(cls) -> None: import WebHostLib.models @@ -44,9 +47,11 @@ class DBGamesPackageCacheTest(GamesPackageCacheTest): cls.orig_db_type = WebHostLib.models.GameDataPackage WebHostLib.models.GameDataPackage = FakeGameDataPackage # type: ignore + @override def setUp(self) -> None: self.cache = DBGamesPackageCache(self.static_data) + @override @classmethod def tearDownClass(cls) -> None: import WebHostLib.models From 2ec4be6f1f4e73c8ffef95b1a18d27f61f3e5532 Mon Sep 17 00:00:00 2001 From: black-sliver <59490463+black-sliver@users.noreply.github.com> Date: Sat, 28 Feb 2026 21:02:34 +0100 Subject: [PATCH 13/15] customserver: cache: reformat --- apmw/webhost/customserver/gamespackage/cache.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apmw/webhost/customserver/gamespackage/cache.py b/apmw/webhost/customserver/gamespackage/cache.py index 970de515ea..a453c1922d 100644 --- a/apmw/webhost/customserver/gamespackage/cache.py +++ b/apmw/webhost/customserver/gamespackage/cache.py @@ -10,7 +10,10 @@ class DBGamesPackageCache(GamesPackageCache): def __init__(self, static_games_package: dict[str, GamesPackage]) -> None: super().__init__() - self._static = {game: GamesPackageCache.get(self, game, games_package) for game, games_package in static_games_package.items()} + self._static = { + game: GamesPackageCache.get(self, game, games_package) + for game, games_package in static_games_package.items() + } @override def get( From 055acf4826ec86d6dd237e1780f713396ac439ec Mon Sep 17 00:00:00 2001 From: black-sliver <59490463+black-sliver@users.noreply.github.com> Date: Sat, 28 Feb 2026 23:44:05 +0100 Subject: [PATCH 14/15] Test: move customserver tests to not interfere with webhost --- test/{webhost/customserver => webhost_customserver}/__init__.py | 0 .../test_gamespackage_cache.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename test/{webhost/customserver => webhost_customserver}/__init__.py (100%) rename test/{webhost/customserver => webhost_customserver}/test_gamespackage_cache.py (100%) diff --git a/test/webhost/customserver/__init__.py b/test/webhost_customserver/__init__.py similarity index 100% rename from test/webhost/customserver/__init__.py rename to test/webhost_customserver/__init__.py diff --git a/test/webhost/customserver/test_gamespackage_cache.py b/test/webhost_customserver/test_gamespackage_cache.py similarity index 100% rename from test/webhost/customserver/test_gamespackage_cache.py rename to test/webhost_customserver/test_gamespackage_cache.py From 821645a88103a04992440ba77566834acdbf26df Mon Sep 17 00:00:00 2001 From: black-sliver <59490463+black-sliver@users.noreply.github.com> Date: Thu, 5 Mar 2026 22:17:10 +0100 Subject: [PATCH 15/15] MultiServer, customserver: cache: rename module --- MultiServer.py | 2 +- WebHostLib/customserver.py | 2 +- apmw/multiserver/gamespackage/__init__.py | 0 .../multiserver/{gamespackage/cache.py => gamespackagecache.py} | 0 apmw/webhost/customserver/gamespackage/__init__.py | 0 .../{gamespackage/cache.py => gamespackagecache.py} | 2 +- test/multiserver/test_gamespackage_cache.py | 2 +- test/webhost_customserver/test_gamespackage_cache.py | 2 +- 8 files changed, 5 insertions(+), 5 deletions(-) delete mode 100644 apmw/multiserver/gamespackage/__init__.py rename apmw/multiserver/{gamespackage/cache.py => gamespackagecache.py} (100%) delete mode 100644 apmw/webhost/customserver/gamespackage/__init__.py rename apmw/webhost/customserver/{gamespackage/cache.py => gamespackagecache.py} (94%) diff --git a/MultiServer.py b/MultiServer.py index c109206a19..eb9f592548 100644 --- a/MultiServer.py +++ b/MultiServer.py @@ -45,7 +45,7 @@ from Utils import version_tuple, restricted_loads, Version, async_start, get_int from NetUtils import Endpoint, ClientStatus, NetworkItem, decode, encode, NetworkPlayer, Permission, NetworkSlot, \ SlotType, LocationStore, MultiData, Hint, HintStatus, GamesPackage from BaseClasses import ItemClassification -from apmw.multiserver.gamespackage.cache import GamesPackageCache +from apmw.multiserver.gamespackagecache import GamesPackageCache min_client_version = Version(0, 5, 0) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 5402607d81..a8553a21d7 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -26,7 +26,7 @@ from MultiServer import ( ) from Utils import restricted_loads, cache_argsless from NetUtils import GamesPackage -from apmw.webhost.customserver.gamespackage.cache import DBGamesPackageCache +from apmw.webhost.customserver.gamespackagecache import DBGamesPackageCache from .locker import Locker from .models import Command, Room, db diff --git a/apmw/multiserver/gamespackage/__init__.py b/apmw/multiserver/gamespackage/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/apmw/multiserver/gamespackage/cache.py b/apmw/multiserver/gamespackagecache.py similarity index 100% rename from apmw/multiserver/gamespackage/cache.py rename to apmw/multiserver/gamespackagecache.py diff --git a/apmw/webhost/customserver/gamespackage/__init__.py b/apmw/webhost/customserver/gamespackage/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/apmw/webhost/customserver/gamespackage/cache.py b/apmw/webhost/customserver/gamespackagecache.py similarity index 94% rename from apmw/webhost/customserver/gamespackage/cache.py rename to apmw/webhost/customserver/gamespackagecache.py index a453c1922d..c58af4b4c6 100644 --- a/apmw/webhost/customserver/gamespackage/cache.py +++ b/apmw/webhost/customserver/gamespackagecache.py @@ -2,7 +2,7 @@ from typing_extensions import override from NetUtils import GamesPackage from Utils import restricted_loads -from apmw.multiserver.gamespackage.cache import GamesPackageCache, ItemNameGroups, LocationNameGroups +from apmw.multiserver.gamespackagecache import GamesPackageCache, ItemNameGroups, LocationNameGroups class DBGamesPackageCache(GamesPackageCache): diff --git a/test/multiserver/test_gamespackage_cache.py b/test/multiserver/test_gamespackage_cache.py index e764a4e8e8..440a46a08d 100644 --- a/test/multiserver/test_gamespackage_cache.py +++ b/test/multiserver/test_gamespackage_cache.py @@ -6,7 +6,7 @@ from typing_extensions import override import NetUtils from NetUtils import GamesPackage -from apmw.multiserver.gamespackage.cache import GamesPackageCache +from apmw.multiserver.gamespackagecache import GamesPackageCache class GamesPackageCacheTest(TestCase): diff --git a/test/webhost_customserver/test_gamespackage_cache.py b/test/webhost_customserver/test_gamespackage_cache.py index 32497526d5..58ea2e37f9 100644 --- a/test/webhost_customserver/test_gamespackage_cache.py +++ b/test/webhost_customserver/test_gamespackage_cache.py @@ -7,7 +7,7 @@ from test.multiserver.test_gamespackage_cache import GamesPackageCacheTest import Utils from NetUtils import GamesPackage -from apmw.webhost.customserver.gamespackage.cache import DBGamesPackageCache +from apmw.webhost.customserver.gamespackagecache import DBGamesPackageCache class FakeGameDataPackage: