WebHost: add document for other games and tools, wiki etc.

* WebHost: make autolauncher max room timeout configurable

* WebHost: launch rooms with assigned port first

.gitignore

@@ -45,6 +45,7 @@ EnemizerCLI/
 /SNI/
 /sni-*/
 /appimagetool*
+/VC_redist.x64.exe
 /host.yaml
 /options.yaml
 /config.yaml

CommonClient.py

@@ -773,7 +773,7 @@ class CommonContext:
         if len(parts) == 1:
             parts = title.split(', ', 1)
         if len(parts) > 1:
-            text = parts[1] + '\n\n' + text
+            text = f"{parts[1]}\n\n{text}" if text else parts[1]
             title = parts[0]
         # display error
         self._messagebox = MessageBox(title, text, error=True)
@@ -896,6 +896,8 @@ async def server_loop(ctx: CommonContext, address: typing.Optional[str] = None)
                                    "May not be running Archipelago on that address or port.")
     except websockets.InvalidURI:
         ctx.handle_connection_loss("Failed to connect to the multiworld server (invalid URI)")
+    except asyncio.TimeoutError:
+        ctx.handle_connection_loss("Failed to connect to the multiworld server. Connection timed out.")
     except OSError:
         ctx.handle_connection_loss("Failed to connect to the multiworld server")
     except Exception:

MultiServer.py

@@ -44,9 +44,8 @@ import NetUtils
 import Utils
 from Utils import version_tuple, restricted_loads, Version, async_start, get_intended_text
 from NetUtils import Endpoint, ClientStatus, NetworkItem, decode, encode, NetworkPlayer, Permission, NetworkSlot, \
-    SlotType, LocationStore, MultiData, Hint, HintStatus, GamesPackage
+    SlotType, LocationStore, MultiData, Hint, HintStatus
 from BaseClasses import ItemClassification
-from apmw.multiserver.gamespackagecache import GamesPackageCache
 
 
 min_client_version = Version(0, 5, 0)
@@ -242,38 +241,21 @@ class Context:
     slot_info: typing.Dict[int, NetworkSlot]
     generator_version = Version(0, 0, 0)
     checksums: typing.Dict[str, str]
-    played_games: set[str]
     item_names: typing.Dict[str, typing.Dict[int, str]]
-    item_name_groups: typing.Dict[str, typing.Dict[str, list[str]]]
+    item_name_groups: typing.Dict[str, typing.Dict[str, typing.Set[str]]]
     location_names: typing.Dict[str, typing.Dict[int, str]]
-    location_name_groups: typing.Dict[str, typing.Dict[str, list[str]]]
+    location_name_groups: typing.Dict[str, typing.Dict[str, typing.Set[str]]]
     all_item_and_group_names: typing.Dict[str, typing.Set[str]]
     all_location_and_group_names: typing.Dict[str, typing.Set[str]]
     non_hintable_names: typing.Dict[str, typing.AbstractSet[str]]
     spheres: typing.List[typing.Dict[int, typing.Set[int]]]
     """ each sphere is { player: { location_id, ... } } """
-    games_package_cache: GamesPackageCache
     logger: logging.Logger
 
-    def __init__(
-            self,
-            host: str,
-            port: int,
-            server_password: str,
-            password: str,
-            location_check_points: int,
-            hint_cost: int,
-            item_cheat: bool,
-            release_mode: str = "disabled",
-            collect_mode="disabled",
-            countdown_mode: str = "auto",
-            remaining_mode: str = "disabled",
-            auto_shutdown: typing.SupportsFloat = 0,
-            compatibility: int = 2,
-            log_network: bool = False,
-            games_package_cache: GamesPackageCache | None = None,
-            logger: logging.Logger = logging.getLogger(),
-    ) -> None:
+    def __init__(self, host: str, port: int, server_password: str, password: str, location_check_points: int,
+                 hint_cost: int, item_cheat: bool, release_mode: str = "disabled", collect_mode="disabled",
+                 countdown_mode: str = "auto", remaining_mode: str = "disabled", auto_shutdown: typing.SupportsFloat = 0, 
+                 compatibility: int = 2, log_network: bool = False, logger: logging.Logger = logging.getLogger()):
         self.logger = logger
         super(Context, self).__init__()
         self.slot_info = {}
@@ -324,7 +306,6 @@ class Context:
         self.save_dirty = False
         self.tags = ['AP']
         self.games: typing.Dict[int, str] = {}
-        self.played_games = set()
         self.minimum_client_versions: typing.Dict[int, Version] = {}
         self.seed_name = ""
         self.groups = {}
@@ -334,10 +315,9 @@ class Context:
         self.stored_data_notification_clients = collections.defaultdict(weakref.WeakSet)
         self.read_data = {}
         self.spheres = []
-        self.games_package_cache = games_package_cache or GamesPackageCache()
 
         # init empty to satisfy linter, I suppose
-        self.reduced_games_package = {}
+        self.gamespackage = {}
         self.checksums = {}
         self.item_name_groups = {}
         self.location_name_groups = {}
@@ -349,11 +329,50 @@ class Context:
             lambda: Utils.KeyedDefaultDict(lambda code: f'Unknown location (ID:{code})'))
         self.non_hintable_names = collections.defaultdict(frozenset)
 
+        self._load_game_data()
+
+    # Data package retrieval
+    def _load_game_data(self):
+        import worlds
+        self.gamespackage = worlds.network_data_package["games"]
+
+        self.item_name_groups = {world_name: world.item_name_groups for world_name, world in
+                                 worlds.AutoWorldRegister.world_types.items()}
+        self.location_name_groups = {world_name: world.location_name_groups for world_name, world in
+                                     worlds.AutoWorldRegister.world_types.items()}
+        for world_name, world in worlds.AutoWorldRegister.world_types.items():
+            self.non_hintable_names[world_name] = world.hint_blacklist
+
+        for game_package in self.gamespackage.values():
+            # remove groups from data sent to clients
+            del game_package["item_name_groups"]
+            del game_package["location_name_groups"]
+
+    def _init_game_data(self):
+        for game_name, game_package in self.gamespackage.items():
+            if "checksum" in game_package:
+                self.checksums[game_name] = game_package["checksum"]
+            for item_name, item_id in game_package["item_name_to_id"].items():
+                self.item_names[game_name][item_id] = item_name
+            for location_name, location_id in game_package["location_name_to_id"].items():
+                self.location_names[game_name][location_id] = location_name
+            self.all_item_and_group_names[game_name] = \
+                set(game_package["item_name_to_id"]) | set(self.item_name_groups[game_name])
+            self.all_location_and_group_names[game_name] = \
+                set(game_package["location_name_to_id"]) | set(self.location_name_groups.get(game_name, []))
+
+        archipelago_item_names = self.item_names["Archipelago"]
+        archipelago_location_names = self.location_names["Archipelago"]
+        for game in [game_name for game_name in self.gamespackage if game_name != "Archipelago"]:
+            # Add Archipelago items and locations to each data package.
+            self.item_names[game].update(archipelago_item_names)
+            self.location_names[game].update(archipelago_location_names)
+
     def item_names_for_game(self, game: str) -> typing.Optional[typing.Dict[str, int]]:
-        return self.reduced_games_package[game]["item_name_to_id"] if game in self.reduced_games_package else None
+        return self.gamespackage[game]["item_name_to_id"] if game in self.gamespackage else None
 
     def location_names_for_game(self, game: str) -> typing.Optional[typing.Dict[str, int]]:
-        return self.reduced_games_package[game]["location_name_to_id"] if game in self.reduced_games_package else None
+        return self.gamespackage[game]["location_name_to_id"] if game in self.gamespackage else None
 
     # General networking
     async def send_msgs(self, endpoint: Endpoint, msgs: typing.Iterable[dict]) -> bool:
@@ -463,17 +482,19 @@ class Context:
             with open(multidatapath, 'rb') as f:
                 data = f.read()
 
-        self._load(self.decompress(data), use_embedded_server_options)
+        self._load(self.decompress(data), {}, use_embedded_server_options)
         self.data_filename = multidatapath
 
     @staticmethod
-    def decompress(data: bytes) -> typing.Any:
+    def decompress(data: bytes) -> dict:
         format_version = data[0]
         if format_version > 3:
             raise Utils.VersionException("Incompatible multidata.")
         return restricted_loads(zlib.decompress(data[1:]))
 
-    def _load(self, decoded_obj: MultiData, use_embedded_server_options: bool) -> None:
+    def _load(self, decoded_obj: MultiData, game_data_packages: typing.Dict[str, typing.Any],
+              use_embedded_server_options: bool):
+
         self.read_data = {}
         # there might be a better place to put this.
         race_mode = decoded_obj.get("race_mode", 0)
@@ -494,7 +515,6 @@ class Context:
 
         self.slot_info = decoded_obj["slot_info"]
         self.games = {slot: slot_info.game for slot, slot_info in self.slot_info.items()}
-        self.played_games = {"Archipelago"} | {self.games[x] for x in range(1, len(self.games) + 1)}
         self.groups = {slot: set(slot_info.group_members) for slot, slot_info in self.slot_info.items()
                        if slot_info.type == SlotType.group}
 
@@ -539,11 +559,18 @@ class Context:
             server_options = decoded_obj.get("server_options", {})
             self._set_options(server_options)
 
-        # load and apply world data and (embedded) data package
-        self._load_world_data()
-        self._load_data_package(decoded_obj.get("datapackage", {}))
+        # embedded data package
+        for game_name, data in decoded_obj.get("datapackage", {}).items():
+            if game_name in game_data_packages:
+                data = game_data_packages[game_name]
+            self.logger.info(f"Loading embedded data package for game {game_name}")
+            self.gamespackage[game_name] = data
+            self.item_name_groups[game_name] = data["item_name_groups"]
+            if "location_name_groups" in data:
+                self.location_name_groups[game_name] = data["location_name_groups"]
+                del data["location_name_groups"]
+            del data["item_name_groups"]  # remove from data package, but keep in self.item_name_groups
         self._init_game_data()
-
         for game_name, data in self.item_name_groups.items():
             self.read_data[f"item_name_groups_{game_name}"] = lambda lgame=game_name: self.item_name_groups[lgame]
         for game_name, data in self.location_name_groups.items():
@@ -552,55 +579,6 @@ class Context:
         # sorted access spheres
         self.spheres = decoded_obj.get("spheres", [])
 
-    def _load_world_data(self) -> None:
-        import worlds
-
-        for world_name, world in worlds.AutoWorldRegister.world_types.items():
-            # TODO: move hint_blacklist into GamesPackage?
-            self.non_hintable_names[world_name] = world.hint_blacklist
-
-    def _load_data_package(self, data_package: dict[str, GamesPackage]) -> None:
-        """Populates reduced_games_package, item_name_groups, location_name_groups from static data and data_package"""
-        # NOTE: for worlds loaded from db, only checksum is set in GamesPackage, but this is handled by cache
-        for game_name in sorted(self.played_games):
-            if game_name in data_package:
-                self.logger.info(f"Loading embedded data package for game {game_name}")
-                data = self.games_package_cache.get(game_name, data_package[game_name])
-            else:
-                # NOTE: we still allow uploading a game without datapackage. Once that is changed, we could drop this.
-                data = self.games_package_cache.get_static(game_name)
-            (
-                self.reduced_games_package[game_name],
-                self.item_name_groups[game_name],
-                self.location_name_groups[game_name],
-            ) = data
-
-        del self.games_package_cache  # Not used past this point. Free memory.
-
-    def _init_game_data(self) -> None:
-        """Update internal values from previously loaded data packages"""
-        for game_name, game_package in self.reduced_games_package.items():
-            if game_name not in self.played_games:
-                continue
-            if "checksum" in game_package:
-                self.checksums[game_name] = game_package["checksum"]
-            # NOTE: we could save more memory by moving the stuff below to data package cache as well
-            for item_name, item_id in game_package["item_name_to_id"].items():
-                self.item_names[game_name][item_id] = item_name
-            for location_name, location_id in game_package["location_name_to_id"].items():
-                self.location_names[game_name][location_id] = location_name
-            self.all_item_and_group_names[game_name] = \
-                set(game_package["item_name_to_id"]) | set(self.item_name_groups[game_name])
-            self.all_location_and_group_names[game_name] = \
-                set(game_package["location_name_to_id"]) | set(self.location_name_groups.get(game_name, []))
-
-        archipelago_item_names = self.item_names["Archipelago"]
-        archipelago_location_names = self.location_names["Archipelago"]
-        for game in [game_name for game_name in self.reduced_games_package if game_name != "Archipelago"]:
-            # Add Archipelago items and locations to each data package.
-            self.item_names[game].update(archipelago_item_names)
-            self.location_names[game].update(archipelago_location_names)
-
     # saving
 
     def save(self, now=False) -> bool:
@@ -941,10 +919,12 @@ async def server(websocket: "ServerConnection", path: str = "/", ctx: Context =
 
 
 async def on_client_connected(ctx: Context, client: Client):
+    games = {ctx.games[x] for x in range(1, len(ctx.games) + 1)}
+    games.add("Archipelago")
     await ctx.send_msgs(client, [{
         'cmd': 'RoomInfo',
         'password': bool(ctx.password),
-        'games': sorted(ctx.played_games),
+        'games': games,
         # tags are for additional features in the communication.
         # Name them by feature or fork, as you feel is appropriate.
         'tags': ctx.tags,
@@ -953,7 +933,8 @@ async def on_client_connected(ctx: Context, client: Client):
         'permissions': get_permissions(ctx),
         'hint_cost': ctx.hint_cost,
         'location_check_points': ctx.location_check_points,
-        'datapackage_checksums': ctx.checksums,
+        'datapackage_checksums': {game: game_data["checksum"] for game, game_data
+                                  in ctx.gamespackage.items() if game in games and "checksum" in game_data},
         'seed_name': ctx.seed_name,
         'time': time.time(),
     }])
@@ -1959,11 +1940,25 @@ async def process_client_cmd(ctx: Context, client: Client, args: dict):
             await ctx.send_msgs(client, reply)
 
     elif cmd == "GetDataPackage":
-        games = {
-            name: game_data for name, game_data in ctx.reduced_games_package.items()
-            if name in set(args.get("games", []))
-        }
-        await ctx.send_msgs(client, [{"cmd": "DataPackage", "data": {"games": games}}])
+        exclusions = args.get("exclusions", [])
+        if "games" in args:
+            games = {name: game_data for name, game_data in ctx.gamespackage.items()
+                     if name in set(args.get("games", []))}
+            await ctx.send_msgs(client, [{"cmd": "DataPackage",
+                                          "data": {"games": games}}])
+        # TODO: remove exclusions behaviour around 0.5.0
+        elif exclusions:
+            exclusions = set(exclusions)
+            games = {name: game_data for name, game_data in ctx.gamespackage.items()
+                     if name not in exclusions}
+
+            package = {"games": games}
+            await ctx.send_msgs(client, [{"cmd": "DataPackage",
+                                          "data": package}])
+
+        else:
+            await ctx.send_msgs(client, [{"cmd": "DataPackage",
+                                          "data": {"games": ctx.gamespackage}}])
 
     elif client.auth:
         if cmd == "ConnectUpdate":

WebHostLib/__init__.py

@@ -42,11 +42,12 @@ app.config["SELFLAUNCH"] = True  # application process is in charge of launching
 app.config["SELFLAUNCHCERT"] = None  # can point to a SSL Certificate to encrypt Room websocket connections
 app.config["SELFLAUNCHKEY"] = None  # can point to a SSL Certificate Key to encrypt Room websocket connections
 app.config["SELFGEN"] = True  # application process is in charge of scheduling Generations.
-app.config["GAME_PORTS"] = ["49152-65535", 0]
 # at what amount of worlds should scheduling be used, instead of rolling in the web-thread
 app.config["JOB_THRESHOLD"] = 1
 # after what time in seconds should generation be aborted, freeing the queue slot. Can be set to None to disable.
 app.config["JOB_TIME"] = 600
+# maximum time in seconds since last activity for a room to be hosted
+app.config["MAX_ROOM_TIMEOUT"] = 259200
 # memory limit for generator processes in bytes
 app.config["GENERATOR_MEMORY_LIMIT"] = 4294967296
 

WebHostLib/autolauncher.py

@@ -9,7 +9,7 @@ from threading import Event, Thread
 from typing import Any
 from uuid import UUID
 
-from pony.orm import db_session, select, commit, PrimaryKey
+from pony.orm import db_session, select, commit, PrimaryKey, desc
 
 from Utils import restricted_loads, utcnow
 from .locker import Locker, AlreadyRunningException
@@ -129,7 +129,8 @@ def autohost(config: dict):
                     with db_session:
                         rooms = select(
                             room for room in Room if
-                            room.last_activity >= utcnow() - timedelta(days=3))
+                            room.last_activity >= utcnow() - timedelta(
+                                seconds=config["MAX_ROOM_TIMEOUT"])).order_by(desc(Room.last_port))
                         for room in rooms:
                             # we have to filter twice, as the per-room timeout can't currently be PonyORM transpiled.
                             if room.last_activity >= utcnow() - timedelta(seconds=room.timeout + 5):
@@ -187,7 +188,6 @@ class MultiworldInstance():
         self.cert = config["SELFLAUNCHCERT"]
         self.key = config["SELFLAUNCHKEY"]
         self.host = config["HOST_ADDRESS"]
-        self.game_ports = config["GAME_PORTS"]
         self.rooms_to_start = multiprocessing.Queue()
         self.rooms_shutting_down = multiprocessing.Queue()
         self.name = f"MultiHoster{id}"
@@ -198,7 +198,7 @@ class MultiworldInstance():
 
         process = multiprocessing.Process(group=None, target=run_server_process,
                                           args=(self.name, self.ponyconfig, get_static_server_data(),
-                                                self.cert, self.key, self.host, self.game_ports,
+                                                self.cert, self.key, self.host,
                                                 self.rooms_to_start, self.rooms_shutting_down),
                                           name=self.name)
         process.start()

WebHostLib/customserver.py

@@ -4,7 +4,6 @@ import asyncio
 import collections
 import datetime
 import functools
-import itertools
 import logging
 import multiprocessing
 import pickle
@@ -14,9 +13,7 @@ import threading
 import time
 import typing
 import sys
-from asyncio import AbstractEventLoop
 
-import psutil
 import websockets
 from pony.orm import commit, db_session, select
 
@@ -27,10 +24,8 @@ from MultiServer import (
     server_per_message_deflate_factory,
 )
 from Utils import restricted_loads, cache_argsless
-from NetUtils import GamesPackage
-from apmw.webhost.customserver.gamespackagecache import DBGamesPackageCache
 from .locker import Locker
-from .models import Command, Room, db
+from .models import Command, GameDataPackage, Room, db
 
 
 class CustomClientMessageProcessor(ClientMessageProcessor):
@@ -67,39 +62,18 @@ class DBCommandProcessor(ServerCommandProcessor):
 
 class WebHostContext(Context):
     room_id: int
-    video: dict[tuple[int, int], tuple[str, str]]
-    main_loop: AbstractEventLoop
-    static_server_data: StaticServerData
 
-    def __init__(
-            self,
-            static_server_data: StaticServerData,
-            games_package_cache: DBGamesPackageCache,
-            logger: logging.Logger,
-    ) -> None:
+    def __init__(self, static_server_data: dict, logger: logging.Logger):
         # static server data is used during _load_game_data to load required data,
         # without needing to import worlds system, which takes quite a bit of memory
-        super(WebHostContext, self).__init__(
-            "",
-            0,
-            "",
-            "",
-            1,
-            40,
-            True,
-            "enabled",
-            "enabled",
-            "enabled",
-            0,
-            2,
-            games_package_cache=games_package_cache,
-            logger=logger,
-        )
-        self.tags = ["AP", "WebHost"]
-        self.video = {}
-        self.main_loop = asyncio.get_running_loop()
         self.static_server_data = static_server_data
-        self.games_package_cache = games_package_cache
+        super(WebHostContext, self).__init__("", 0, "", "", 1,
+                                             40, True, "enabled", "enabled",
+                                             "enabled", 0, 2, logger=logger)
+        del self.static_server_data
+        self.main_loop = asyncio.get_running_loop()
+        self.video = {}
+        self.tags = ["AP", "WebHost"]
 
     def __del__(self):
         try:
@@ -109,6 +83,12 @@ class WebHostContext(Context):
         except ImportError:
             self.logger.debug("Context destroyed")
 
+    def _load_game_data(self):
+        for key, value in self.static_server_data.items():
+            # NOTE: attributes are mutable and shared, so they will have to be copied before being modified
+            setattr(self, key, value)
+        self.non_hintable_names = collections.defaultdict(frozenset, self.non_hintable_names)
+
     async def listen_to_db_commands(self):
         cmdprocessor = DBCommandProcessor(self)
 
@@ -135,17 +115,45 @@ class WebHostContext(Context):
         if room.last_port:
             self.port = room.last_port
         else:
-            self.port = 0
+            self.port = get_random_port()
 
         multidata = self.decompress(room.seed.multidata)
-        return self._load(multidata, True)
+        game_data_packages = {}
 
-    def _load_world_data(self):
-        # Use static_server_data, but skip static data package since that is in cache anyway.
-        # Also NOT importing worlds here!
-        # FIXME: does this copy the non_hintable_names (also for games not part of the room)?
-        self.non_hintable_names = collections.defaultdict(frozenset, self.static_server_data["non_hintable_names"])
-        del self.static_server_data  # Not used past this point. Free memory.
+        static_gamespackage = self.gamespackage  # this is shared across all rooms
+        static_item_name_groups = self.item_name_groups
+        static_location_name_groups = self.location_name_groups
+        self.gamespackage = {"Archipelago": static_gamespackage.get("Archipelago", {})}  # this may be modified by _load
+        self.item_name_groups = {"Archipelago": static_item_name_groups.get("Archipelago", {})}
+        self.location_name_groups = {"Archipelago": static_location_name_groups.get("Archipelago", {})}
+        missing_checksum = False
+
+        for game in list(multidata.get("datapackage", {})):
+            game_data = multidata["datapackage"][game]
+            if "checksum" in game_data:
+                if static_gamespackage.get(game, {}).get("checksum") == game_data["checksum"]:
+                    # non-custom. remove from multidata and use static data
+                    # games package could be dropped from static data once all rooms embed data package
+                    del multidata["datapackage"][game]
+                else:
+                    row = GameDataPackage.get(checksum=game_data["checksum"])
+                    if row:  # None if rolled on >= 0.3.9 but uploaded to <= 0.3.8. multidata should be complete
+                        game_data_packages[game] = restricted_loads(row.data)
+                        continue
+                    else:
+                        self.logger.warning(f"Did not find game_data_package for {game}: {game_data['checksum']}")
+            else:
+                missing_checksum = True  # Game rolled on old AP and will load data package from multidata
+            self.gamespackage[game] = static_gamespackage.get(game, {})
+            self.item_name_groups[game] = static_item_name_groups.get(game, {})
+            self.location_name_groups[game] = static_location_name_groups.get(game, {})
+
+        if not game_data_packages and not missing_checksum:
+            # all static -> use the static dicts directly
+            self.gamespackage = static_gamespackage
+            self.item_name_groups = static_item_name_groups
+            self.location_name_groups = static_location_name_groups
+        return self._load(multidata, game_data_packages, True)
 
     def init_save(self, enabled: bool = True):
         self.saving = enabled
@@ -173,117 +181,38 @@ class WebHostContext(Context):
         return d
 
 
-class GameRangePorts(typing.NamedTuple):
-    parsed_ports: list[range]
-    weights: list[int]
-    ephemeral_allowed: bool
-
-
-@functools.cache
-def parse_game_ports(game_ports: tuple[str | int, ...]) -> GameRangePorts:
-    parsed_ports: list[range] = []
-    weights: list[int] = []
-    ephemeral_allowed = False
-    total_length = 0
-
-    for item in game_ports:
-        if isinstance(item, str) and "-" in item:
-            start, end = map(int, item.split("-"))
-            x = range(start, end + 1)
-            total_length += len(x)
-            weights.append(total_length)
-            parsed_ports.append(x)
-        elif int(item) == 0:
-            ephemeral_allowed = True
-        else:
-            total_length += 1
-            weights.append(total_length)
-            num = int(item)
-            parsed_ports.append(range(num, num + 1))
-
-    return GameRangePorts(parsed_ports, weights, ephemeral_allowed)
-
-
-def weighted_random(ranges: list[range], cum_weights: list[int]) -> int:
-    [picked] = random.choices(ranges, cum_weights=cum_weights)
-    return random.randrange(picked.start, picked.stop, picked.step)
-
-
-def create_random_port_socket(game_ports: tuple[str | int, ...], host: str) -> socket.socket:
-    parsed_ports, weights, ephemeral_allowed = parse_game_ports(game_ports)
-    used_ports = get_used_ports()
-    i = 1024 if len(parsed_ports) > 0 else 0
-    while i > 0:
-        port_num = weighted_random(parsed_ports, weights)
-        if port_num in used_ports:
-            used_ports = get_used_ports()
-            continue
-
-        i -= 0
-
-        try:
-            return socket.create_server((host, port_num))
-        except OSError:
-            pass
-
-    if ephemeral_allowed:
-        return socket.create_server((host, 0))
-
-    raise OSError(98, "No available ports")
-
-
-def try_conns_per_process(p: psutil.Process) -> typing.Iterable[int]:
-    try:
-        return (c.laddr.port for c in p.net_connections("tcp4"))
-    except psutil.AccessDenied:
-        return ()
-
-
-def get_active_net_connections() -> typing.Iterable[int]:
-    # Don't even try to check if system using AIX
-    if psutil.AIX:
-        return ()
-
-    try:
-        return (c.laddr.port for c in psutil.net_connections("tcp4"))
-    # raises AccessDenied when done on macOS
-    except psutil.AccessDenied:
-        # flatten the list of iterables
-        return itertools.chain.from_iterable(map(
-            # get the net connections of the process and then map its ports
-            try_conns_per_process,
-            # this method has caching handled by psutil
-            psutil.process_iter(["net_connections"])
-        ))
-
-
-def get_used_ports():
-    last_used_ports: tuple[frozenset[int], float] | None = getattr(get_used_ports, "last", None)
-    t_hash = round(time.time() / 90)  # cache for 90 seconds
-    if last_used_ports is None or last_used_ports[1] != t_hash:
-        last_used_ports = (frozenset(get_active_net_connections()), t_hash)
-        setattr(get_used_ports, "last", last_used_ports)
-
-    return last_used_ports[0]
-
-
-class StaticServerData(typing.TypedDict, total=True):
-    non_hintable_names: dict[str, typing.AbstractSet[str]]
-    games_package: dict[str, GamesPackage]
+def get_random_port():
+    return random.randint(49152, 65535)
 
 
 @cache_argsless
-def get_static_server_data() -> StaticServerData:
+def get_static_server_data() -> dict:
     import worlds
-
-    return {
+    data = {
         "non_hintable_names": {
             world_name: world.hint_blacklist
             for world_name, world in worlds.AutoWorldRegister.world_types.items()
         },
-        "games_package": worlds.network_data_package["games"]
+        "gamespackage": {
+            world_name: {
+                key: value
+                for key, value in game_package.items()
+                if key not in ("item_name_groups", "location_name_groups")
+            }
+            for world_name, game_package in worlds.network_data_package["games"].items()
+        },
+        "item_name_groups": {
+            world_name: world.item_name_groups
+            for world_name, world in worlds.AutoWorldRegister.world_types.items()
+        },
+        "location_name_groups": {
+            world_name: world.location_name_groups
+            for world_name, world in worlds.AutoWorldRegister.world_types.items()
+        },
     }
 
+    return data
+
 
 def set_up_logging(room_id) -> logging.Logger:
     import os
@@ -316,19 +245,9 @@ def tear_down_logging(room_id):
         del logging.Logger.manager.loggerDict[logger_name]
 
 
-def run_server_process(
-        name: str,
-        ponyconfig: dict[str, typing.Any],
-        static_server_data: StaticServerData,
-        cert_file: typing.Optional[str],
-        cert_key_file: typing.Optional[str],
-        host: str,
-        game_ports: typing.Iterable[str | int],
-        rooms_to_run: multiprocessing.Queue,
-        rooms_shutting_down: multiprocessing.Queue,
-) -> None:
-    import gc
-
+def run_server_process(name: str, ponyconfig: dict, static_server_data: dict,
+                       cert_file: typing.Optional[str], cert_key_file: typing.Optional[str],
+                       host: str, rooms_to_run: multiprocessing.Queue, rooms_shutting_down: multiprocessing.Queue):
     from setproctitle import setproctitle
 
     setproctitle(name)
@@ -344,11 +263,6 @@ def run_server_process(
         resource.setrlimit(resource.RLIMIT_NOFILE, (file_limit, file_limit))
         del resource, file_limit
 
-    # prime the data package cache with static data
-    games_package_cache = DBGamesPackageCache(static_server_data["games_package"])
-    # convert to tuple because its hashable
-    game_ports = tuple(game_ports)
-
     # establish DB connection for multidata and multisave
     db.bind(**ponyconfig)
     db.generate_mapping(check_tables=False)
@@ -356,6 +270,8 @@ def run_server_process(
     if "worlds" in sys.modules:
         raise Exception("Worlds system should not be loaded in the custom server.")
 
+    import gc
+
     if not cert_file:
         def get_ssl_context():
             return None
@@ -380,30 +296,24 @@ def run_server_process(
         with Locker(f"RoomLocker {room_id}"):
             try:
                 logger = set_up_logging(room_id)
-                ctx = WebHostContext(static_server_data, games_package_cache, logger)
+                ctx = WebHostContext(static_server_data, logger)
                 ctx.load(room_id)
                 ctx.init_save()
                 assert ctx.server is None
-                if ctx.port != 0:
-                    try:
-                        ctx.server = websockets.serve(
-                            functools.partial(server, ctx=ctx),
-                            ctx.host,
-                            ctx.port,
-                            ssl=get_ssl_context(),
-                            extensions=[server_per_message_deflate_factory],
-                        )
-                        await ctx.server
-                    except OSError:
-                        ctx.port = 0
-                if ctx.port == 0:
+                try:
                     ctx.server = websockets.serve(
                         functools.partial(server, ctx=ctx),
-                        sock=create_random_port_socket(game_ports, ctx.host),
+                        ctx.host,
+                        ctx.port,
                         ssl=get_ssl_context(),
                         extensions=[server_per_message_deflate_factory],
                     )
                     await ctx.server
+                except OSError:  # likely port in use
+                    ctx.server = websockets.serve(
+                        functools.partial(server, ctx=ctx), ctx.host, 0, ssl=get_ssl_context())
+
+                    await ctx.server
                 port = 0
                 for wssocket in ctx.server.ws_server.sockets:
                     socketname = wssocket.getsockname()
@@ -478,7 +388,7 @@ def run_server_process(
 
         def run(self):
             while 1:
-                next_room = rooms_to_run.get(block=True, timeout=None)
+                next_room = rooms_to_run.get(block=True,  timeout=None)
                 gc.collect()
                 task = asyncio.run_coroutine_threadsafe(start_room(next_room), loop)
                 self._tasks.append(task)

WebHostLib/static/styles/landing.css

@@ -33,17 +33,6 @@ html{
     z-index: 10;
 }
 
-#landing-header h5 {
-    color: #ffffff;
-    font-style: italic;
-    font-size: 28px;
-    margin-top: 15px;
-    margin-bottom: -43px;
-    text-shadow: 1px 1px 7px #000000;
-    font-kerning: none;
-    z-index: 10;
-}
-
 #landing-links{
     margin-left: auto;
     margin-right: auto;

WebHostLib/templates/landing.html

@@ -11,7 +11,7 @@
     <div id="landing-wrapper">
         <div id="landing-header">
             <img id="landing-logo" src="static/static/branding/landing-logo.png" alt="Archipelago Logo" />
-            <h4>multiworld multi-game randomizer</h4><h5>beta</h5>
+            <h4>multiworld multi-game randomizer</h4>
         </div>
         <div id="landing-links">
             <a href="/games" id="far-left-button">Supported<br />Games</a>
@@ -35,8 +35,7 @@
         </div>
         <div id="landing" class="grass-island">
             <div id="landing-body">
-                <p id="first-line">Welcome to Archipelago Beta!</p>
-                <p>For the stable version, visit <a href="//archipelago.gg">Archipelago.gg</a>!</p>
+                <p id="first-line">Welcome to Archipelago!</p>
                 <p>
                     This is a cross-game modification system which randomizes different games, then uses the result to
                     build a single unified multi-player game. Items from one game may be present in another, and

WebHostLib/templates/pageWrapper.html

@@ -21,7 +21,6 @@
                 </div>
             {% endif %}
         {% endwith %}
-        <div class="user-message">This is the beta site! For the stable version, visit <a href="https://archipelago.gg">Archipelago.gg</a>!</div>
 
         {% block body %}
         {% endblock %}

apmw/multiserver/gamespackagecache.py

@@ -1,96 +0,0 @@
-import typing as t
-from weakref import WeakValueDictionary
-
-from NetUtils import GamesPackage
-
-GameAndChecksum = tuple[str, str | None]
-ItemNameGroups = dict[str, list[str]]
-LocationNameGroups = dict[str, list[str]]
-
-
-K = t.TypeVar("K")
-V = t.TypeVar("V")
-
-
-class DictLike(dict[K, V]):
-    __slots__ = ("__weakref__",)
-
-
-class GamesPackageCache:
-    # NOTE: this uses 3 separate collections because unpacking the get() result would end the container lifetime
-    _reduced_games_packages: WeakValueDictionary[GameAndChecksum, GamesPackage]
-    """Does not include item_name_groups nor location_name_groups"""
-    _item_name_groups: WeakValueDictionary[GameAndChecksum, dict[str, list[str]]]
-    _location_name_groups: WeakValueDictionary[GameAndChecksum, dict[str, list[str]]]
-
-    def __init__(self) -> None:
-        self._reduced_games_packages = WeakValueDictionary()
-        self._item_name_groups = WeakValueDictionary()
-        self._location_name_groups = WeakValueDictionary()
-
-    def _get(
-        self,
-        cache_key: GameAndChecksum,
-    ) -> tuple[GamesPackage | None, ItemNameGroups | None, LocationNameGroups | None]:
-        if cache_key[1] is None:
-            return None, None, None
-        return (
-            self._reduced_games_packages.get(cache_key, None),
-            self._item_name_groups.get(cache_key, None),
-            self._location_name_groups.get(cache_key, None),
-        )
-
-    def get(
-        self,
-        game: str,
-        full_games_package: GamesPackage,
-    ) -> tuple[GamesPackage, ItemNameGroups, LocationNameGroups]:
-        """Loads and caches embedded data package provided by multidata"""
-        cache_key = (game, full_games_package.get("checksum", None))
-        cached_reduced_games_package, cached_item_name_groups, cached_location_name_groups = self._get(cache_key)
-
-        if cached_reduced_games_package is None:
-            cached_reduced_games_package = t.cast(
-                t.Any,
-                DictLike(
-                    {
-                        "item_name_to_id": full_games_package["item_name_to_id"],
-                        "location_name_to_id": full_games_package["location_name_to_id"],
-                        "checksum": full_games_package.get("checksum", None),
-                    }
-                ),
-            )
-            if cache_key[1] is not None:  # only cache if checksum is available
-                self._reduced_games_packages[cache_key] = cached_reduced_games_package
-
-        if cached_item_name_groups is None:
-            # optimize strings to be references instead of copies
-            item_names = {name: name for name in cached_reduced_games_package["item_name_to_id"].keys()}
-            cached_item_name_groups = DictLike(
-                {
-                    group_name: [item_names.get(item_name, item_name) for item_name in group_items]
-                    for group_name, group_items in full_games_package["item_name_groups"].items()
-                }
-            )
-            if cache_key[1] is not None:  # only cache if checksum is available
-                self._item_name_groups[cache_key] = cached_item_name_groups
-
-        if cached_location_name_groups is None:
-            # optimize strings to be references instead of copies
-            location_names = {name: name for name in cached_reduced_games_package["location_name_to_id"].keys()}
-            cached_location_name_groups = DictLike(
-                {
-                    group_name: [location_names.get(location_name, location_name) for location_name in group_locations]
-                    for group_name, group_locations in full_games_package.get("location_name_groups", {}).items()
-                }
-            )
-            if cache_key[1] is not None:  # only cache if checksum is available
-                self._location_name_groups[cache_key] = cached_location_name_groups
-
-        return cached_reduced_games_package, cached_item_name_groups, cached_location_name_groups
-
-    def get_static(self, game: str) -> tuple[GamesPackage, ItemNameGroups, LocationNameGroups]:
-        """Loads legacy data package from installed worlds"""
-        import worlds
-
-        return self.get(game, worlds.network_data_package["games"][game])

apmw/webhost/customserver/gamespackagecache.py

@@ -1,42 +0,0 @@
-from typing_extensions import override
-
-from NetUtils import GamesPackage
-from Utils import restricted_loads
-from apmw.multiserver.gamespackagecache import GamesPackageCache, ItemNameGroups, LocationNameGroups
-
-
-class DBGamesPackageCache(GamesPackageCache):
-    _static: dict[str, tuple[GamesPackage, ItemNameGroups, LocationNameGroups]]
-
-    def __init__(self, static_games_package: dict[str, GamesPackage]) -> None:
-        super().__init__()
-        self._static = {
-            game: GamesPackageCache.get(self, game, games_package)
-            for game, games_package in static_games_package.items()
-        }
-
-    @override
-    def get(
-        self,
-        game: str,
-        full_games_package: GamesPackage,
-    ) -> tuple[GamesPackage, ItemNameGroups, LocationNameGroups]:
-        # for games started on webhost, full_games_package is likely unpopulated and only has the checksum field
-        cache_key = (game, full_games_package.get("checksum", None))
-        cached = self._get(cache_key)
-        if any(value is None for value in cached):
-            if "checksum" not in full_games_package:
-                return super().get(game, full_games_package)  # no checksum, assume fully populated
-
-            from WebHostLib.models import GameDataPackage
-
-            row: GameDataPackage | None = GameDataPackage.get(checksum=full_games_package["checksum"])
-            if row:  # None if rolled on >= 0.3.9 but uploaded to <= 0.3.8 ...
-                return super().get(game, restricted_loads(row.data))
-            return super().get(game, full_games_package)  # ... in which case full_games_package should be populated
-
-        return cached  # type: ignore # mypy doesn't understand any value is None
-
-    @override
-    def get_static(self, game: str) -> tuple[GamesPackage, ItemNameGroups, LocationNameGroups]:
-        return self._static[game]

docs/webhost configuration sample.yaml

@@ -17,12 +17,6 @@
 # Web hosting port
 #PORT: 80
 
-# Ports used for game hosting. Values can be specific ports, port ranges or both. Default is: [49152-65535, 0]
-# Zero means it will use a random free port if there is no port in the next 1024 randomly chosen ports from the range
-# Examples of valid values: [40000-41000, 49152-65535]
-# If ports within the range(s) are already in use, the WebHost will fallback to the default [49152-65535, 0] range.
-#GAME_PORTS: [49152-65535, 0]
-
 # Place where uploads go.
 #UPLOAD_FOLDER: uploads
 

test/multiserver/test_gamespackage_cache.py

@@ -1,132 +0,0 @@
-import typing as t
-from copy import deepcopy
-from unittest import TestCase
-
-from typing_extensions import override
-
-import NetUtils
-from NetUtils import GamesPackage
-from apmw.multiserver.gamespackagecache import GamesPackageCache
-
-
-class GamesPackageCacheTest(TestCase):
-    cache: GamesPackageCache
-    any_game: t.ClassVar[str] = "APQuest"
-    example_games_package: GamesPackage = {
-        "item_name_to_id": {"Item 1": 1},
-        "item_name_groups": {"Everything": ["Item 1"]},
-        "location_name_to_id": {"Location 1": 1},
-        "location_name_groups": {"Everywhere": ["Location 1"]},
-        "checksum": "1234",
-    }
-
-    @override
-    def setUp(self) -> None:
-        self.cache = GamesPackageCache()
-
-    def test_get_static_is_same(self) -> None:
-        """Tests that get_static returns the same objects twice"""
-        reduced_games_package1, item_name_groups1, location_name_groups1 = self.cache.get_static(self.any_game)
-        reduced_games_package2, item_name_groups2, location_name_groups2 = self.cache.get_static(self.any_game)
-        self.assertIs(reduced_games_package1, reduced_games_package2)
-        self.assertIs(item_name_groups1, item_name_groups2)
-        self.assertIs(location_name_groups1, location_name_groups2)
-
-    def test_get_static_data_format(self) -> None:
-        """Tests that get_static returns data in the correct format"""
-        reduced_games_package, item_name_groups, location_name_groups = self.cache.get_static(self.any_game)
-        self.assertTrue(reduced_games_package["checksum"])
-        self.assertTrue(reduced_games_package["item_name_to_id"])
-        self.assertTrue(reduced_games_package["location_name_to_id"])
-        self.assertNotIn("item_name_groups", reduced_games_package)
-        self.assertNotIn("location_name_groups", reduced_games_package)
-        self.assertTrue(item_name_groups["Everything"])
-        self.assertTrue(location_name_groups["Everywhere"])
-
-    def test_get_static_is_serializable(self) -> None:
-        """Tests that get_static returns data that can be serialized"""
-        NetUtils.encode(self.cache.get_static(self.any_game))
-
-    def test_get_static_missing_raises(self) -> None:
-        """Tests that get_static raises KeyError if the world is missing"""
-        with self.assertRaises(KeyError):
-            _ = self.cache.get_static("Does not exist")
-
-    def test_eviction(self) -> None:
-        """Tests that unused items get evicted from cache"""
-        game_name = "Test"
-        before_add = len(self.cache._reduced_games_packages)
-        data = self.cache.get(game_name, self.example_games_package)
-        self.assertTrue(data)
-        self.assertEqual(before_add + 1, len(self.cache._reduced_games_packages))
-
-        del data
-        if len(self.cache._reduced_games_packages) != before_add:  # gc.collect() may not even be required
-            import gc
-
-            gc.collect()
-
-        self.assertEqual(before_add, len(self.cache._reduced_games_packages))
-
-    def test_get_required_field(self) -> None:
-        """Tests that missing required field raises a KeyError"""
-        for field in ("item_name_to_id", "location_name_to_id", "item_name_groups"):
-            with self.subTest(field=field):
-                games_package = deepcopy(self.example_games_package)
-                del games_package[field]  # type: ignore
-                with self.assertRaises(KeyError):
-                    _ = self.cache.get(self.any_game, games_package)
-
-    def test_get_optional_properties(self) -> None:
-        """Tests that missing optional field works"""
-        for field in ("checksum", "location_name_groups"):
-            with self.subTest(field=field):
-                games_package = deepcopy(self.example_games_package)
-                del games_package[field]  # type: ignore
-                _, item_name_groups, location_name_groups = self.cache.get(self.any_game, games_package)
-                self.assertTrue(item_name_groups)
-                self.assertEqual(field != "location_name_groups", bool(location_name_groups))
-
-    def test_item_name_deduplication(self) -> None:
-        n = 1
-        s1 = f"Item {n}"
-        s2 = f"Item {n}"
-        # check if the deduplication is actually gonna do anything
-        self.assertIsNot(s1, s2)
-        self.assertEqual(s1, s2)
-        # do the thing
-        game_name = "Test"
-        games_package: GamesPackage = {
-            "item_name_to_id": {s1: n},
-            "item_name_groups": {"Everything": [s2]},
-            "location_name_to_id": {},
-            "location_name_groups": {},
-            "checksum": "1234",
-        }
-        reduced_games_package, item_name_groups, location_name_groups = self.cache.get(game_name, games_package)
-        self.assertIs(
-            next(iter(reduced_games_package["item_name_to_id"].keys())),
-            item_name_groups["Everything"][0],
-        )
-
-    def test_location_name_deduplication(self) -> None:
-        n = 1
-        s1 = f"Location {n}"
-        s2 = f"Location {n}"
-        # check if the deduplication is actually gonna do anything
-        self.assertIsNot(s1, s2)
-        self.assertEqual(s1, s2)
-        # do the thing
-        game_name = "Test"
-        games_package: GamesPackage = {
-            "item_name_to_id": {},
-            "item_name_groups": {},
-            "location_name_to_id": {s1: n},
-            "location_name_groups": {"Everywhere": [s2]},
-            "checksum": "1234",
-        }
-        reduced_games_package, item_name_groups, location_name_groups = self.cache.get(game_name, games_package)
-        self.assertIs(
-            next(iter(reduced_games_package["location_name_to_id"].keys())),
-            location_name_groups["Everywhere"][0],
-        )

test/webhost/test_port_allocation.py

@@ -1,86 +0,0 @@
-import os
-import unittest
-
-from Utils import is_macos
-from WebHostLib.customserver import parse_game_ports, create_random_port_socket, get_used_ports
-
-ci = bool(os.environ.get("CI"))
-
-
-class TestPortAllocating(unittest.TestCase):
-    def test_parse_game_ports(self) -> None:
-        """Ensure that game ports with ranges are parsed correctly"""
-        val = parse_game_ports(("1000-2000", "2000-5000", "1000-2000", 20, 40, "20", "0"))
-
-        self.assertListEqual(val.parsed_ports,
-                             [range(1000, 2001), range(2000, 5001), range(1000, 2001), range(20, 21), range(40, 41),
-                              range(20, 21)], "The parsed game ports are not the expected values")
-        self.assertTrue(val.ephemeral_allowed, "The ephemeral allowed flag is not set even though it was passed")
-        self.assertListEqual(val.weights, [1001, 4002, 5003, 5004, 5005, 5006],
-                             "Cumulative weights are not the expected value")
-
-        val = parse_game_ports(())
-        self.assertListEqual(val.parsed_ports, [], "Empty list of game port returned something")
-        self.assertFalse(val.ephemeral_allowed, "Empty list returned that ephemeral is allowed")
-
-        val = parse_game_ports((0,))
-        self.assertListEqual(val.parsed_ports, [], "Empty list of ranges returned something")
-        self.assertTrue(val.ephemeral_allowed, "List with just 0 is not allowing ephemeral ports")
-
-        val = parse_game_ports((1,))
-        self.assertEqual(val.parsed_ports, [range(1, 2)], "Parsed ports doesn't contain the expected values")
-        self.assertFalse(val.ephemeral_allowed, "List with just single port returned that ephemeral is allowed")
-
-    def test_parse_game_port_errors(self) -> None:
-        """Ensure that game ports with incorrect values raise the expected error"""
-        with self.assertRaises(ValueError, msg="Negative numbers didn't get interpreted as an invalid range"):
-            parse_game_ports(tuple("-50215"))
-        with self.assertRaises(ValueError, msg="Text got interpreted as a valid number"):
-            parse_game_ports(tuple("dwafawg"))
-        with self.assertRaises(
-                ValueError,
-                msg="A range with an extra dash at the end didn't get interpreted as an invalid number because of it's end dash"
-        ):
-            parse_game_ports(tuple("20-21215-"))
-        with self.assertRaises(ValueError, msg="Text got interpreted as a valid number for the start of a range"):
-            parse_game_ports(tuple("f-21215"))
-
-    def test_random_port_socket_edge_cases(self) -> None:
-        """Verify if edge cases on creation of random port socket is working fine"""
-        # Try giving an empty tuple and fail over it
-        with self.assertRaises(OSError) as err:
-            create_random_port_socket(tuple(), "127.0.0.1")
-        self.assertEqual(err.exception.errno, 98, "Raised an unexpected error code")
-        self.assertEqual(err.exception.strerror, "No available ports", "Raised an unexpected error string")
-
-        # Try only having ephemeral ports enabled
-        try:
-            create_random_port_socket(("0",), "127.0.0.1").close()
-        except OSError as err:
-            self.assertEqual(err.errno, 98, "Raised an unexpected error code")
-            # If it returns our error string that means something is wrong with our code
-            self.assertNotEqual(err.strerror, "No available ports",
-                                "Raised an unexpected error string")
-
-    @unittest.skipUnless(ci, "can't guarantee free ports outside of CI")
-    def test_random_port_socket(self) -> None:
-        """Verify if returned sockets use the correct port ranges"""
-        sockets = []
-        for _ in range(6):
-            socket = create_random_port_socket(("8080-8085",), "127.0.0.1")
-            sockets.append(socket)
-            _, port = socket.getsockname()
-            self.assertIn(port, range(8080, 8086), "Port of socket was not inside the expected range")
-        for s in sockets:
-            s.close()
-
-        sockets.clear()
-        length = 5_000 if is_macos else (30_000 - len(get_used_ports()))
-        for _ in range(length):
-            socket = create_random_port_socket(("30000-65535",), "127.0.0.1")
-            sockets.append(socket)
-            _, port = socket.getsockname()
-            self.assertIn(port, range(30_000, 65536), "Port of socket was not inside the expected range")
-
-        for s in sockets:
-            s.close()

test/webhost_customserver/test_gamespackage_cache.py

@@ -1,147 +0,0 @@
-import typing as t
-from copy import deepcopy
-
-from typing_extensions import override
-
-from test.multiserver.test_gamespackage_cache import GamesPackageCacheTest
-
-import Utils
-from NetUtils import GamesPackage
-from apmw.webhost.customserver.gamespackagecache import DBGamesPackageCache
-
-
-class FakeGameDataPackage:
-    _rows: "t.ClassVar[dict[str, FakeGameDataPackage]]" = {}
-    data: bytes
-
-    @classmethod
-    def get(cls, checksum: str) -> "FakeGameDataPackage | None":
-        return cls._rows.get(checksum, None)
-
-    @classmethod
-    def add(cls, checksum: str, full_games_package: GamesPackage) -> None:
-        row = FakeGameDataPackage()
-        row.data = Utils.restricted_dumps(full_games_package)
-        cls._rows[checksum] = row
-
-
-class DBGamesPackageCacheTest(GamesPackageCacheTest):
-    cache: DBGamesPackageCache
-    any_game: t.ClassVar[str] = "My Game"
-    static_data: t.ClassVar[dict[str, GamesPackage]] = {  # noqa: pycharm doesn't understand this
-        "My Game": {
-            "item_name_to_id": {"Item 1": 1},
-            "location_name_to_id": {"Location 1": 1},
-            "item_name_groups": {"Everything": ["Item 1"]},
-            "location_name_groups": {"Everywhere": ["Location 1"]},
-            "checksum": "2345",
-        }
-    }
-    orig_db_type: t.ClassVar[type]
-
-    @override
-    @classmethod
-    def setUpClass(cls) -> None:
-        import WebHostLib.models
-
-        cls.orig_db_type = WebHostLib.models.GameDataPackage
-        WebHostLib.models.GameDataPackage = FakeGameDataPackage  # type: ignore
-
-    @override
-    def setUp(self) -> None:
-        self.cache = DBGamesPackageCache(self.static_data)
-
-    @override
-    @classmethod
-    def tearDownClass(cls) -> None:
-        import WebHostLib.models
-
-        WebHostLib.models.GameDataPackage = cls.orig_db_type  # type: ignore
-
-    def assert_conversion(
-        self,
-        full_games_package: GamesPackage,
-        reduced_games_package: dict[str, t.Any],
-        item_name_groups: dict[str, t.Any],
-        location_name_groups: dict[str, t.Any],
-    ) -> None:
-        for key in ("item_name_to_id", "location_name_to_id", "checksum"):
-            if key in full_games_package:
-                self.assertEqual(reduced_games_package[key], full_games_package[key])  # noqa: pycharm
-        self.assertEqual(item_name_groups, full_games_package["item_name_groups"])
-        self.assertEqual(location_name_groups, full_games_package["location_name_groups"])
-
-    def assert_static_conversion(
-        self,
-        full_games_package: GamesPackage,
-        reduced_games_package: dict[str, t.Any],
-        item_name_groups: dict[str, t.Any],
-        location_name_groups: dict[str, t.Any],
-    ) -> None:
-        self.assert_conversion(full_games_package, reduced_games_package, item_name_groups, location_name_groups)
-        for key in ("item_name_to_id", "location_name_to_id", "checksum"):
-            self.assertIs(reduced_games_package[key], full_games_package[key])  # noqa: pycharm
-
-    def test_get_static_contents(self) -> None:
-        """Tests that get_static returns the correct data"""
-        reduced_games_package, item_name_groups, location_name_groups = self.cache.get_static(self.any_game)
-        for key in ("item_name_to_id", "location_name_to_id", "checksum"):
-            self.assertIs(reduced_games_package[key], self.static_data[self.any_game][key])  # noqa: pycharm
-        self.assertEqual(item_name_groups, self.static_data[self.any_game]["item_name_groups"])
-        self.assertEqual(location_name_groups, self.static_data[self.any_game]["location_name_groups"])
-
-    def test_static_not_evicted(self) -> None:
-        """Tests that static data is not evicted from cache during gc"""
-        import gc
-
-        game_name = next(iter(self.static_data.keys()))
-        ids = [id(o) for o in self.cache.get_static(game_name)]
-        gc.collect()
-        self.assertEqual(ids, [id(o) for o in self.cache.get_static(game_name)])
-
-    def test_get_is_static(self) -> None:
-        """Tests that a get with correct checksum return the static items"""
-        # NOTE: this is only true for the DB cache, not the "regular" one, since we want to avoid loading worlds there
-        cks: GamesPackage = {"checksum": self.static_data[self.any_game]["checksum"]}  # noqa: pycharm doesn't like this
-        reduced_games_package1, item_name_groups1, location_name_groups1 = self.cache.get(self.any_game, cks)
-        reduced_games_package2, item_name_groups2, location_name_groups2 = self.cache.get_static(self.any_game)
-        self.assertIs(reduced_games_package1, reduced_games_package2)
-        self.assertEqual(location_name_groups1, location_name_groups2)
-        self.assertEqual(item_name_groups1, item_name_groups2)
-
-    def test_get_from_db(self) -> None:
-        """Tests that a get with only checksum will load the full data from db and is cached"""
-        game_name = "Another Game"
-        full_games_package = deepcopy(self.static_data[self.any_game])
-        full_games_package["checksum"] = "3456"
-        cks: GamesPackage = {"checksum": full_games_package["checksum"]}  # noqa: pycharm doesn't like this
-        FakeGameDataPackage.add(full_games_package["checksum"], full_games_package)
-        before_add = len(self.cache._reduced_games_packages)
-        data = self.cache.get(game_name, cks)
-        self.assert_conversion(full_games_package, *data)  # type: ignore
-        self.assertEqual(before_add + 1, len(self.cache._reduced_games_packages))
-
-    def test_get_missing_from_db_uses_full_games_package(self) -> None:
-        """Tests that a get with full data (missing from db) will use the full data and is cached"""
-        game_name = "Yet Another Game"
-        full_games_package = deepcopy(self.static_data[self.any_game])
-        full_games_package["checksum"] = "4567"
-        before_add = len(self.cache._reduced_games_packages)
-        data = self.cache.get(game_name, full_games_package)
-        self.assert_conversion(full_games_package, *data)  # type: ignore
-        self.assertEqual(before_add + 1, len(self.cache._reduced_games_packages))
-
-    def test_get_without_checksum_uses_full_games_package(self) -> None:
-        """Tests that a get with full data and no checksum will use the full data and is not cached"""
-        game_name = "Yet Another Game"
-        full_games_package = deepcopy(self.static_data[self.any_game])
-        del full_games_package["checksum"]
-        before_add = len(self.cache._reduced_games_packages)
-        data = self.cache.get(game_name, full_games_package)
-        self.assert_conversion(full_games_package, *data)  # type: ignore
-        self.assertEqual(before_add, len(self.cache._reduced_games_packages))
-
-    def test_get_missing_from_db_raises(self) -> None:
-        """Tests that a get that requires a row to exist raise an exception if it doesn't"""
-        with self.assertRaises(Exception):
-            _ = self.cache.get("Does not exist", {"checksum": "0000"})

worlds/generic/__init__.py

@@ -26,7 +26,10 @@ class GenericWeb(WebWorld):
                      'English', 'setup_en.md', 'setup/en', ['alwaysintreble'])
     triggers = Tutorial('Archipelago Triggers Guide', 'A guide to setting up and using triggers in your game settings.',
                         'English', 'triggers_en.md', 'triggers/en', ['alwaysintreble'])
-    tutorials = [setup, mac, commands, advanced_settings, triggers, plando]
+    other = Tutorial('Other Games and Tools',
+                     'A guide to additional games and tools that can be used with Archipelago.',
+                     'English', 'other_en.md', 'other/en', ['Berserker'])
+    tutorials = [setup, mac, commands, advanced_settings, triggers, plando, other]
 
 
 class GenericWorld(World):

worlds/generic/docs/other_en.md

@@ -0,0 +1,24 @@
+# Other Games and Tools
+
+This guide provides information on additional community resources, tools, and games that function with Archipelago.
+
+## Community Resources
+
+The Archipelago community is active across several platforms where you can find support, new games, and tools.
+
+### Discord Servers
+Archipelago has two primary Discord servers for community interaction, game support, and hosting public games:
+- **[Archipelago Official Discord](https://discord.gg/8Z65BR2)**: The main hub for the community, including general discussion, support, and public multiworld hosting.
+- **[Archipelago After Dark Discord](https://discord.gg/fqvNCCRsu4)**: An adults-only server for 18+ and unrated content.
+
+Both servers feature an **#apworld-index** channel. These channels are repositories for "APWorlds" — additional game implementations that can be easily added to your Archipelago installation to support more games.
+
+### Documentation
+- **[Archipelago Wiki](https://archipelago.miraheze.org/)**: A community-maintained wiki.
+
+## Community Tools
+
+These community-developed tools are frequently used alongside Archipelago to improve the player experience.
+
+### PopTracker
+**[PopTracker](https://github.com/black-sliver/PopTracker)** is a universal multi-platform tracking application designed for randomizers. It supports many Archipelago games through tracker packs, providing both manual and automatic autotracking capabilities by connecting directly to an Archipelago server.

worlds/kh2/Locations.py

@@ -1281,7 +1281,7 @@ exclusion_table = {
         LocationName.HadesCupTrophyParadoxCups,
         LocationName.MusicalOrichalcumPlus,
     ],
-    "HitlistCasual":          {
+    "HitlistCasual":          [
         LocationName.FuturePete,
         LocationName.BetwixtandBetweenBondofFlame,
         LocationName.GrimReaper2,
@@ -1299,7 +1299,7 @@ exclusion_table = {
         LocationName.MCP,
         LocationName.Lvl50,
         LocationName.Lvl99
-    },
+    ],
     "Cups":          {
         LocationName.ProtectBeltPainandPanicCup,
         LocationName.SerenityGemPainandPanicCup,