Merge remote-tracking branch 'imurx/custom-port-range' into active/rc-site

Co-authored-by: Doug Hoskisson <beauxq@users.noreply.github.com>

.gitignore

@@ -45,7 +45,6 @@ EnemizerCLI/
 /SNI/
 /sni-*/
 /appimagetool*
-/VC_redist.x64.exe
 /host.yaml
 /options.yaml
 /config.yaml

CommonClient.py

@@ -773,7 +773,7 @@ class CommonContext:
         if len(parts) == 1:
             parts = title.split(', ', 1)
         if len(parts) > 1:
-            text = f"{parts[1]}\n\n{text}" if text else parts[1]
+            text = parts[1] + '\n\n' + text
             title = parts[0]
         # display error
         self._messagebox = MessageBox(title, text, error=True)
@@ -896,8 +896,6 @@ async def server_loop(ctx: CommonContext, address: typing.Optional[str] = None)
                                    "May not be running Archipelago on that address or port.")
     except websockets.InvalidURI:
         ctx.handle_connection_loss("Failed to connect to the multiworld server (invalid URI)")
-    except asyncio.TimeoutError:
-        ctx.handle_connection_loss("Failed to connect to the multiworld server. Connection timed out.")
     except OSError:
         ctx.handle_connection_loss("Failed to connect to the multiworld server")
     except Exception:

MultiServer.py

@@ -44,8 +44,9 @@ import NetUtils
 import Utils
 from Utils import version_tuple, restricted_loads, Version, async_start, get_intended_text
 from NetUtils import Endpoint, ClientStatus, NetworkItem, decode, encode, NetworkPlayer, Permission, NetworkSlot, \
-    SlotType, LocationStore, MultiData, Hint, HintStatus
+    SlotType, LocationStore, MultiData, Hint, HintStatus, GamesPackage
 from BaseClasses import ItemClassification
+from apmw.multiserver.gamespackagecache import GamesPackageCache
 
 
 min_client_version = Version(0, 5, 0)
@@ -241,21 +242,38 @@ class Context:
     slot_info: typing.Dict[int, NetworkSlot]
     generator_version = Version(0, 0, 0)
     checksums: typing.Dict[str, str]
+    played_games: set[str]
     item_names: typing.Dict[str, typing.Dict[int, str]]
-    item_name_groups: typing.Dict[str, typing.Dict[str, typing.Set[str]]]
+    item_name_groups: typing.Dict[str, typing.Dict[str, list[str]]]
     location_names: typing.Dict[str, typing.Dict[int, str]]
-    location_name_groups: typing.Dict[str, typing.Dict[str, typing.Set[str]]]
+    location_name_groups: typing.Dict[str, typing.Dict[str, list[str]]]
     all_item_and_group_names: typing.Dict[str, typing.Set[str]]
     all_location_and_group_names: typing.Dict[str, typing.Set[str]]
     non_hintable_names: typing.Dict[str, typing.AbstractSet[str]]
     spheres: typing.List[typing.Dict[int, typing.Set[int]]]
     """ each sphere is { player: { location_id, ... } } """
+    games_package_cache: GamesPackageCache
     logger: logging.Logger
 
-    def __init__(self, host: str, port: int, server_password: str, password: str, location_check_points: int,
-                 hint_cost: int, item_cheat: bool, release_mode: str = "disabled", collect_mode="disabled",
-                 countdown_mode: str = "auto", remaining_mode: str = "disabled", auto_shutdown: typing.SupportsFloat = 0, 
-                 compatibility: int = 2, log_network: bool = False, logger: logging.Logger = logging.getLogger()):
+    def __init__(
+            self,
+            host: str,
+            port: int,
+            server_password: str,
+            password: str,
+            location_check_points: int,
+            hint_cost: int,
+            item_cheat: bool,
+            release_mode: str = "disabled",
+            collect_mode="disabled",
+            countdown_mode: str = "auto",
+            remaining_mode: str = "disabled",
+            auto_shutdown: typing.SupportsFloat = 0,
+            compatibility: int = 2,
+            log_network: bool = False,
+            games_package_cache: GamesPackageCache | None = None,
+            logger: logging.Logger = logging.getLogger(),
+    ) -> None:
         self.logger = logger
         super(Context, self).__init__()
         self.slot_info = {}
@@ -306,6 +324,7 @@ class Context:
         self.save_dirty = False
         self.tags = ['AP']
         self.games: typing.Dict[int, str] = {}
+        self.played_games = set()
         self.minimum_client_versions: typing.Dict[int, Version] = {}
         self.seed_name = ""
         self.groups = {}
@@ -315,9 +334,10 @@ class Context:
         self.stored_data_notification_clients = collections.defaultdict(weakref.WeakSet)
         self.read_data = {}
         self.spheres = []
+        self.games_package_cache = games_package_cache or GamesPackageCache()
 
         # init empty to satisfy linter, I suppose
-        self.gamespackage = {}
+        self.reduced_games_package = {}
         self.checksums = {}
         self.item_name_groups = {}
         self.location_name_groups = {}
@@ -329,50 +349,11 @@ class Context:
             lambda: Utils.KeyedDefaultDict(lambda code: f'Unknown location (ID:{code})'))
         self.non_hintable_names = collections.defaultdict(frozenset)
 
-        self._load_game_data()
-
-    # Data package retrieval
-    def _load_game_data(self):
-        import worlds
-        self.gamespackage = worlds.network_data_package["games"]
-
-        self.item_name_groups = {world_name: world.item_name_groups for world_name, world in
-                                 worlds.AutoWorldRegister.world_types.items()}
-        self.location_name_groups = {world_name: world.location_name_groups for world_name, world in
-                                     worlds.AutoWorldRegister.world_types.items()}
-        for world_name, world in worlds.AutoWorldRegister.world_types.items():
-            self.non_hintable_names[world_name] = world.hint_blacklist
-
-        for game_package in self.gamespackage.values():
-            # remove groups from data sent to clients
-            del game_package["item_name_groups"]
-            del game_package["location_name_groups"]
-
-    def _init_game_data(self):
-        for game_name, game_package in self.gamespackage.items():
-            if "checksum" in game_package:
-                self.checksums[game_name] = game_package["checksum"]
-            for item_name, item_id in game_package["item_name_to_id"].items():
-                self.item_names[game_name][item_id] = item_name
-            for location_name, location_id in game_package["location_name_to_id"].items():
-                self.location_names[game_name][location_id] = location_name
-            self.all_item_and_group_names[game_name] = \
-                set(game_package["item_name_to_id"]) | set(self.item_name_groups[game_name])
-            self.all_location_and_group_names[game_name] = \
-                set(game_package["location_name_to_id"]) | set(self.location_name_groups.get(game_name, []))
-
-        archipelago_item_names = self.item_names["Archipelago"]
-        archipelago_location_names = self.location_names["Archipelago"]
-        for game in [game_name for game_name in self.gamespackage if game_name != "Archipelago"]:
-            # Add Archipelago items and locations to each data package.
-            self.item_names[game].update(archipelago_item_names)
-            self.location_names[game].update(archipelago_location_names)
-
     def item_names_for_game(self, game: str) -> typing.Optional[typing.Dict[str, int]]:
-        return self.gamespackage[game]["item_name_to_id"] if game in self.gamespackage else None
+        return self.reduced_games_package[game]["item_name_to_id"] if game in self.reduced_games_package else None
 
     def location_names_for_game(self, game: str) -> typing.Optional[typing.Dict[str, int]]:
-        return self.gamespackage[game]["location_name_to_id"] if game in self.gamespackage else None
+        return self.reduced_games_package[game]["location_name_to_id"] if game in self.reduced_games_package else None
 
     # General networking
     async def send_msgs(self, endpoint: Endpoint, msgs: typing.Iterable[dict]) -> bool:
@@ -482,19 +463,17 @@ class Context:
             with open(multidatapath, 'rb') as f:
                 data = f.read()
 
-        self._load(self.decompress(data), {}, use_embedded_server_options)
+        self._load(self.decompress(data), use_embedded_server_options)
         self.data_filename = multidatapath
 
     @staticmethod
-    def decompress(data: bytes) -> dict:
+    def decompress(data: bytes) -> typing.Any:
         format_version = data[0]
         if format_version > 3:
             raise Utils.VersionException("Incompatible multidata.")
         return restricted_loads(zlib.decompress(data[1:]))
 
-    def _load(self, decoded_obj: MultiData, game_data_packages: typing.Dict[str, typing.Any],
-              use_embedded_server_options: bool):
-
+    def _load(self, decoded_obj: MultiData, use_embedded_server_options: bool) -> None:
         self.read_data = {}
         # there might be a better place to put this.
         race_mode = decoded_obj.get("race_mode", 0)
@@ -515,6 +494,7 @@ class Context:
 
         self.slot_info = decoded_obj["slot_info"]
         self.games = {slot: slot_info.game for slot, slot_info in self.slot_info.items()}
+        self.played_games = {"Archipelago"} | {self.games[x] for x in range(1, len(self.games) + 1)}
         self.groups = {slot: set(slot_info.group_members) for slot, slot_info in self.slot_info.items()
                        if slot_info.type == SlotType.group}
 
@@ -559,18 +539,11 @@ class Context:
             server_options = decoded_obj.get("server_options", {})
             self._set_options(server_options)
 
-        # embedded data package
-        for game_name, data in decoded_obj.get("datapackage", {}).items():
-            if game_name in game_data_packages:
-                data = game_data_packages[game_name]
-            self.logger.info(f"Loading embedded data package for game {game_name}")
-            self.gamespackage[game_name] = data
-            self.item_name_groups[game_name] = data["item_name_groups"]
-            if "location_name_groups" in data:
-                self.location_name_groups[game_name] = data["location_name_groups"]
-                del data["location_name_groups"]
-            del data["item_name_groups"]  # remove from data package, but keep in self.item_name_groups
+        # load and apply world data and (embedded) data package
+        self._load_world_data()
+        self._load_data_package(decoded_obj.get("datapackage", {}))
         self._init_game_data()
+
         for game_name, data in self.item_name_groups.items():
             self.read_data[f"item_name_groups_{game_name}"] = lambda lgame=game_name: self.item_name_groups[lgame]
         for game_name, data in self.location_name_groups.items():
@@ -579,6 +552,55 @@ class Context:
         # sorted access spheres
         self.spheres = decoded_obj.get("spheres", [])
 
+    def _load_world_data(self) -> None:
+        import worlds
+
+        for world_name, world in worlds.AutoWorldRegister.world_types.items():
+            # TODO: move hint_blacklist into GamesPackage?
+            self.non_hintable_names[world_name] = world.hint_blacklist
+
+    def _load_data_package(self, data_package: dict[str, GamesPackage]) -> None:
+        """Populates reduced_games_package, item_name_groups, location_name_groups from static data and data_package"""
+        # NOTE: for worlds loaded from db, only checksum is set in GamesPackage, but this is handled by cache
+        for game_name in sorted(self.played_games):
+            if game_name in data_package:
+                self.logger.info(f"Loading embedded data package for game {game_name}")
+                data = self.games_package_cache.get(game_name, data_package[game_name])
+            else:
+                # NOTE: we still allow uploading a game without datapackage. Once that is changed, we could drop this.
+                data = self.games_package_cache.get_static(game_name)
+            (
+                self.reduced_games_package[game_name],
+                self.item_name_groups[game_name],
+                self.location_name_groups[game_name],
+            ) = data
+
+        del self.games_package_cache  # Not used past this point. Free memory.
+
+    def _init_game_data(self) -> None:
+        """Update internal values from previously loaded data packages"""
+        for game_name, game_package in self.reduced_games_package.items():
+            if game_name not in self.played_games:
+                continue
+            if "checksum" in game_package:
+                self.checksums[game_name] = game_package["checksum"]
+            # NOTE: we could save more memory by moving the stuff below to data package cache as well
+            for item_name, item_id in game_package["item_name_to_id"].items():
+                self.item_names[game_name][item_id] = item_name
+            for location_name, location_id in game_package["location_name_to_id"].items():
+                self.location_names[game_name][location_id] = location_name
+            self.all_item_and_group_names[game_name] = \
+                set(game_package["item_name_to_id"]) | set(self.item_name_groups[game_name])
+            self.all_location_and_group_names[game_name] = \
+                set(game_package["location_name_to_id"]) | set(self.location_name_groups.get(game_name, []))
+
+        archipelago_item_names = self.item_names["Archipelago"]
+        archipelago_location_names = self.location_names["Archipelago"]
+        for game in [game_name for game_name in self.reduced_games_package if game_name != "Archipelago"]:
+            # Add Archipelago items and locations to each data package.
+            self.item_names[game].update(archipelago_item_names)
+            self.location_names[game].update(archipelago_location_names)
+
     # saving
 
     def save(self, now=False) -> bool:
@@ -919,12 +941,10 @@ async def server(websocket: "ServerConnection", path: str = "/", ctx: Context =
 
 
 async def on_client_connected(ctx: Context, client: Client):
-    games = {ctx.games[x] for x in range(1, len(ctx.games) + 1)}
-    games.add("Archipelago")
     await ctx.send_msgs(client, [{
         'cmd': 'RoomInfo',
         'password': bool(ctx.password),
-        'games': games,
+        'games': sorted(ctx.played_games),
         # tags are for additional features in the communication.
         # Name them by feature or fork, as you feel is appropriate.
         'tags': ctx.tags,
@@ -933,8 +953,7 @@ async def on_client_connected(ctx: Context, client: Client):
         'permissions': get_permissions(ctx),
         'hint_cost': ctx.hint_cost,
         'location_check_points': ctx.location_check_points,
-        'datapackage_checksums': {game: game_data["checksum"] for game, game_data
-                                  in ctx.gamespackage.items() if game in games and "checksum" in game_data},
+        'datapackage_checksums': ctx.checksums,
         'seed_name': ctx.seed_name,
         'time': time.time(),
     }])
@@ -1940,25 +1959,11 @@ async def process_client_cmd(ctx: Context, client: Client, args: dict):
             await ctx.send_msgs(client, reply)
 
     elif cmd == "GetDataPackage":
-        exclusions = args.get("exclusions", [])
-        if "games" in args:
-            games = {name: game_data for name, game_data in ctx.gamespackage.items()
-                     if name in set(args.get("games", []))}
-            await ctx.send_msgs(client, [{"cmd": "DataPackage",
-                                          "data": {"games": games}}])
-        # TODO: remove exclusions behaviour around 0.5.0
-        elif exclusions:
-            exclusions = set(exclusions)
-            games = {name: game_data for name, game_data in ctx.gamespackage.items()
-                     if name not in exclusions}
-
-            package = {"games": games}
-            await ctx.send_msgs(client, [{"cmd": "DataPackage",
-                                          "data": package}])
-
-        else:
-            await ctx.send_msgs(client, [{"cmd": "DataPackage",
-                                          "data": {"games": ctx.gamespackage}}])
+        games = {
+            name: game_data for name, game_data in ctx.reduced_games_package.items()
+            if name in set(args.get("games", []))
+        }
+        await ctx.send_msgs(client, [{"cmd": "DataPackage", "data": {"games": games}}])
 
     elif client.auth:
         if cmd == "ConnectUpdate":

WebHostLib/__init__.py

@@ -42,12 +42,11 @@ app.config["SELFLAUNCH"] = True  # application process is in charge of launching
 app.config["SELFLAUNCHCERT"] = None  # can point to a SSL Certificate to encrypt Room websocket connections
 app.config["SELFLAUNCHKEY"] = None  # can point to a SSL Certificate Key to encrypt Room websocket connections
 app.config["SELFGEN"] = True  # application process is in charge of scheduling Generations.
+app.config["GAME_PORTS"] = ["49152-65535", 0]
 # at what amount of worlds should scheduling be used, instead of rolling in the web-thread
 app.config["JOB_THRESHOLD"] = 1
 # after what time in seconds should generation be aborted, freeing the queue slot. Can be set to None to disable.
 app.config["JOB_TIME"] = 600
-# maximum time in seconds since last activity for a room to be hosted
-app.config["MAX_ROOM_TIMEOUT"] = 259200
 # memory limit for generator processes in bytes
 app.config["GENERATOR_MEMORY_LIMIT"] = 4294967296
 

WebHostLib/autolauncher.py

@@ -9,7 +9,7 @@ from threading import Event, Thread
 from typing import Any
 from uuid import UUID
 
-from pony.orm import db_session, select, commit, PrimaryKey, desc
+from pony.orm import db_session, select, commit, PrimaryKey
 
 from Utils import restricted_loads, utcnow
 from .locker import Locker, AlreadyRunningException
@@ -129,8 +129,7 @@ def autohost(config: dict):
                     with db_session:
                         rooms = select(
                             room for room in Room if
-                            room.last_activity >= utcnow() - timedelta(
-                                seconds=config["MAX_ROOM_TIMEOUT"])).order_by(desc(Room.last_port))
+                            room.last_activity >= utcnow() - timedelta(days=3))
                         for room in rooms:
                             # we have to filter twice, as the per-room timeout can't currently be PonyORM transpiled.
                             if room.last_activity >= utcnow() - timedelta(seconds=room.timeout + 5):
@@ -188,6 +187,7 @@ class MultiworldInstance():
         self.cert = config["SELFLAUNCHCERT"]
         self.key = config["SELFLAUNCHKEY"]
         self.host = config["HOST_ADDRESS"]
+        self.game_ports = config["GAME_PORTS"]
         self.rooms_to_start = multiprocessing.Queue()
         self.rooms_shutting_down = multiprocessing.Queue()
         self.name = f"MultiHoster{id}"
@@ -198,7 +198,7 @@ class MultiworldInstance():
 
         process = multiprocessing.Process(group=None, target=run_server_process,
                                           args=(self.name, self.ponyconfig, get_static_server_data(),
-                                                self.cert, self.key, self.host,
+                                                self.cert, self.key, self.host, self.game_ports,
                                                 self.rooms_to_start, self.rooms_shutting_down),
                                           name=self.name)
         process.start()

WebHostLib/customserver.py

@@ -4,6 +4,7 @@ import asyncio
 import collections
 import datetime
 import functools
+import itertools
 import logging
 import multiprocessing
 import pickle
@@ -13,7 +14,9 @@ import threading
 import time
 import typing
 import sys
+from asyncio import AbstractEventLoop
 
+import psutil
 import websockets
 from pony.orm import commit, db_session, select
 
@@ -24,8 +27,10 @@ from MultiServer import (
     server_per_message_deflate_factory,
 )
 from Utils import restricted_loads, cache_argsless
+from NetUtils import GamesPackage
+from apmw.webhost.customserver.gamespackagecache import DBGamesPackageCache
 from .locker import Locker
-from .models import Command, GameDataPackage, Room, db
+from .models import Command, Room, db
 
 
 class CustomClientMessageProcessor(ClientMessageProcessor):
@@ -62,18 +67,39 @@ class DBCommandProcessor(ServerCommandProcessor):
 
 class WebHostContext(Context):
     room_id: int
+    video: dict[tuple[int, int], tuple[str, str]]
+    main_loop: AbstractEventLoop
+    static_server_data: StaticServerData
 
-    def __init__(self, static_server_data: dict, logger: logging.Logger):
+    def __init__(
+            self,
+            static_server_data: StaticServerData,
+            games_package_cache: DBGamesPackageCache,
+            logger: logging.Logger,
+    ) -> None:
         # static server data is used during _load_game_data to load required data,
         # without needing to import worlds system, which takes quite a bit of memory
-        self.static_server_data = static_server_data
-        super(WebHostContext, self).__init__("", 0, "", "", 1,
-                                             40, True, "enabled", "enabled",
-                                             "enabled", 0, 2, logger=logger)
-        del self.static_server_data
-        self.main_loop = asyncio.get_running_loop()
-        self.video = {}
+        super(WebHostContext, self).__init__(
+            "",
+            0,
+            "",
+            "",
+            1,
+            40,
+            True,
+            "enabled",
+            "enabled",
+            "enabled",
+            0,
+            2,
+            games_package_cache=games_package_cache,
+            logger=logger,
+        )
         self.tags = ["AP", "WebHost"]
+        self.video = {}
+        self.main_loop = asyncio.get_running_loop()
+        self.static_server_data = static_server_data
+        self.games_package_cache = games_package_cache
 
     def __del__(self):
         try:
@@ -83,12 +109,6 @@ class WebHostContext(Context):
         except ImportError:
             self.logger.debug("Context destroyed")
 
-    def _load_game_data(self):
-        for key, value in self.static_server_data.items():
-            # NOTE: attributes are mutable and shared, so they will have to be copied before being modified
-            setattr(self, key, value)
-        self.non_hintable_names = collections.defaultdict(frozenset, self.non_hintable_names)
-
     async def listen_to_db_commands(self):
         cmdprocessor = DBCommandProcessor(self)
 
@@ -115,45 +135,17 @@ class WebHostContext(Context):
         if room.last_port:
             self.port = room.last_port
         else:
-            self.port = get_random_port()
+            self.port = 0
 
         multidata = self.decompress(room.seed.multidata)
-        game_data_packages = {}
+        return self._load(multidata, True)
 
-        static_gamespackage = self.gamespackage  # this is shared across all rooms
-        static_item_name_groups = self.item_name_groups
-        static_location_name_groups = self.location_name_groups
-        self.gamespackage = {"Archipelago": static_gamespackage.get("Archipelago", {})}  # this may be modified by _load
-        self.item_name_groups = {"Archipelago": static_item_name_groups.get("Archipelago", {})}
-        self.location_name_groups = {"Archipelago": static_location_name_groups.get("Archipelago", {})}
-        missing_checksum = False
-
-        for game in list(multidata.get("datapackage", {})):
-            game_data = multidata["datapackage"][game]
-            if "checksum" in game_data:
-                if static_gamespackage.get(game, {}).get("checksum") == game_data["checksum"]:
-                    # non-custom. remove from multidata and use static data
-                    # games package could be dropped from static data once all rooms embed data package
-                    del multidata["datapackage"][game]
-                else:
-                    row = GameDataPackage.get(checksum=game_data["checksum"])
-                    if row:  # None if rolled on >= 0.3.9 but uploaded to <= 0.3.8. multidata should be complete
-                        game_data_packages[game] = restricted_loads(row.data)
-                        continue
-                    else:
-                        self.logger.warning(f"Did not find game_data_package for {game}: {game_data['checksum']}")
-            else:
-                missing_checksum = True  # Game rolled on old AP and will load data package from multidata
-            self.gamespackage[game] = static_gamespackage.get(game, {})
-            self.item_name_groups[game] = static_item_name_groups.get(game, {})
-            self.location_name_groups[game] = static_location_name_groups.get(game, {})
-
-        if not game_data_packages and not missing_checksum:
-            # all static -> use the static dicts directly
-            self.gamespackage = static_gamespackage
-            self.item_name_groups = static_item_name_groups
-            self.location_name_groups = static_location_name_groups
-        return self._load(multidata, game_data_packages, True)
+    def _load_world_data(self):
+        # Use static_server_data, but skip static data package since that is in cache anyway.
+        # Also NOT importing worlds here!
+        # FIXME: does this copy the non_hintable_names (also for games not part of the room)?
+        self.non_hintable_names = collections.defaultdict(frozenset, self.static_server_data["non_hintable_names"])
+        del self.static_server_data  # Not used past this point. Free memory.
 
     def init_save(self, enabled: bool = True):
         self.saving = enabled
@@ -181,38 +173,117 @@ class WebHostContext(Context):
         return d
 
 
-def get_random_port():
-    return random.randint(49152, 65535)
+class GameRangePorts(typing.NamedTuple):
+    parsed_ports: list[range]
+    weights: list[int]
+    ephemeral_allowed: bool
+
+
+@functools.cache
+def parse_game_ports(game_ports: tuple[str | int, ...]) -> GameRangePorts:
+    parsed_ports: list[range] = []
+    weights: list[int] = []
+    ephemeral_allowed = False
+    total_length = 0
+
+    for item in game_ports:
+        if isinstance(item, str) and "-" in item:
+            start, end = map(int, item.split("-"))
+            x = range(start, end + 1)
+            total_length += len(x)
+            weights.append(total_length)
+            parsed_ports.append(x)
+        elif int(item) == 0:
+            ephemeral_allowed = True
+        else:
+            total_length += 1
+            weights.append(total_length)
+            num = int(item)
+            parsed_ports.append(range(num, num + 1))
+
+    return GameRangePorts(parsed_ports, weights, ephemeral_allowed)
+
+
+def weighted_random(ranges: list[range], cum_weights: list[int]) -> int:
+    [picked] = random.choices(ranges, cum_weights=cum_weights)
+    return random.randrange(picked.start, picked.stop, picked.step)
+
+
+def create_random_port_socket(game_ports: tuple[str | int, ...], host: str) -> socket.socket:
+    parsed_ports, weights, ephemeral_allowed = parse_game_ports(game_ports)
+    used_ports = get_used_ports()
+    i = 1024 if len(parsed_ports) > 0 else 0
+    while i > 0:
+        port_num = weighted_random(parsed_ports, weights)
+        if port_num in used_ports:
+            used_ports = get_used_ports()
+            continue
+
+        i -= 0
+
+        try:
+            return socket.create_server((host, port_num))
+        except OSError:
+            pass
+
+    if ephemeral_allowed:
+        return socket.create_server((host, 0))
+
+    raise OSError(98, "No available ports")
+
+
+def try_conns_per_process(p: psutil.Process) -> typing.Iterable[int]:
+    try:
+        return (c.laddr.port for c in p.net_connections("tcp4"))
+    except psutil.AccessDenied:
+        return ()
+
+
+def get_active_net_connections() -> typing.Iterable[int]:
+    # Don't even try to check if system using AIX
+    if psutil.AIX:
+        return ()
+
+    try:
+        return (c.laddr.port for c in psutil.net_connections("tcp4"))
+    # raises AccessDenied when done on macOS
+    except psutil.AccessDenied:
+        # flatten the list of iterables
+        return itertools.chain.from_iterable(map(
+            # get the net connections of the process and then map its ports
+            try_conns_per_process,
+            # this method has caching handled by psutil
+            psutil.process_iter(["net_connections"])
+        ))
+
+
+def get_used_ports():
+    last_used_ports: tuple[frozenset[int], float] | None = getattr(get_used_ports, "last", None)
+    t_hash = round(time.time() / 90)  # cache for 90 seconds
+    if last_used_ports is None or last_used_ports[1] != t_hash:
+        last_used_ports = (frozenset(get_active_net_connections()), t_hash)
+        setattr(get_used_ports, "last", last_used_ports)
+
+    return last_used_ports[0]
+
+
+class StaticServerData(typing.TypedDict, total=True):
+    non_hintable_names: dict[str, typing.AbstractSet[str]]
+    games_package: dict[str, GamesPackage]
 
 
 @cache_argsless
-def get_static_server_data() -> dict:
+def get_static_server_data() -> StaticServerData:
     import worlds
-    data = {
+
+    return {
         "non_hintable_names": {
             world_name: world.hint_blacklist
             for world_name, world in worlds.AutoWorldRegister.world_types.items()
         },
-        "gamespackage": {
-            world_name: {
-                key: value
-                for key, value in game_package.items()
-                if key not in ("item_name_groups", "location_name_groups")
-            }
-            for world_name, game_package in worlds.network_data_package["games"].items()
-        },
-        "item_name_groups": {
-            world_name: world.item_name_groups
-            for world_name, world in worlds.AutoWorldRegister.world_types.items()
-        },
-        "location_name_groups": {
-            world_name: world.location_name_groups
-            for world_name, world in worlds.AutoWorldRegister.world_types.items()
-        },
+        "games_package": worlds.network_data_package["games"]
     }
 
-    return data
-
 
 def set_up_logging(room_id) -> logging.Logger:
     import os
@@ -245,9 +316,19 @@ def tear_down_logging(room_id):
         del logging.Logger.manager.loggerDict[logger_name]
 
 
-def run_server_process(name: str, ponyconfig: dict, static_server_data: dict,
-                       cert_file: typing.Optional[str], cert_key_file: typing.Optional[str],
-                       host: str, rooms_to_run: multiprocessing.Queue, rooms_shutting_down: multiprocessing.Queue):
+def run_server_process(
+        name: str,
+        ponyconfig: dict[str, typing.Any],
+        static_server_data: StaticServerData,
+        cert_file: typing.Optional[str],
+        cert_key_file: typing.Optional[str],
+        host: str,
+        game_ports: typing.Iterable[str | int],
+        rooms_to_run: multiprocessing.Queue,
+        rooms_shutting_down: multiprocessing.Queue,
+) -> None:
+    import gc
+
     from setproctitle import setproctitle
 
     setproctitle(name)
@@ -263,6 +344,11 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict,
         resource.setrlimit(resource.RLIMIT_NOFILE, (file_limit, file_limit))
         del resource, file_limit
 
+    # prime the data package cache with static data
+    games_package_cache = DBGamesPackageCache(static_server_data["games_package"])
+    # convert to tuple because its hashable
+    game_ports = tuple(game_ports)
+
     # establish DB connection for multidata and multisave
     db.bind(**ponyconfig)
     db.generate_mapping(check_tables=False)
@@ -270,8 +356,6 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict,
     if "worlds" in sys.modules:
         raise Exception("Worlds system should not be loaded in the custom server.")
 
-    import gc
-
     if not cert_file:
         def get_ssl_context():
             return None
@@ -296,24 +380,30 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict,
         with Locker(f"RoomLocker {room_id}"):
             try:
                 logger = set_up_logging(room_id)
-                ctx = WebHostContext(static_server_data, logger)
+                ctx = WebHostContext(static_server_data, games_package_cache, logger)
                 ctx.load(room_id)
                 ctx.init_save()
                 assert ctx.server is None
-                try:
+                if ctx.port != 0:
+                    try:
+                        ctx.server = websockets.serve(
+                            functools.partial(server, ctx=ctx),
+                            ctx.host,
+                            ctx.port,
+                            ssl=get_ssl_context(),
+                            extensions=[server_per_message_deflate_factory],
+                        )
+                        await ctx.server
+                    except OSError:
+                        ctx.port = 0
+                if ctx.port == 0:
                     ctx.server = websockets.serve(
                         functools.partial(server, ctx=ctx),
-                        ctx.host,
-                        ctx.port,
+                        sock=create_random_port_socket(game_ports, ctx.host),
                         ssl=get_ssl_context(),
                         extensions=[server_per_message_deflate_factory],
                     )
                     await ctx.server
-                except OSError:  # likely port in use
-                    ctx.server = websockets.serve(
-                        functools.partial(server, ctx=ctx), ctx.host, 0, ssl=get_ssl_context())
-
-                    await ctx.server
                 port = 0
                 for wssocket in ctx.server.ws_server.sockets:
                     socketname = wssocket.getsockname()
@@ -388,7 +478,7 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict,
 
         def run(self):
             while 1:
-                next_room = rooms_to_run.get(block=True,  timeout=None)
+                next_room = rooms_to_run.get(block=True, timeout=None)
                 gc.collect()
                 task = asyncio.run_coroutine_threadsafe(start_room(next_room), loop)
                 self._tasks.append(task)

WebHostLib/static/styles/landing.css

@@ -33,6 +33,17 @@ html{
     z-index: 10;
 }
 
+#landing-header h5 {
+    color: #ffffff;
+    font-style: italic;
+    font-size: 28px;
+    margin-top: 15px;
+    margin-bottom: -43px;
+    text-shadow: 1px 1px 7px #000000;
+    font-kerning: none;
+    z-index: 10;
+}
+
 #landing-links{
     margin-left: auto;
     margin-right: auto;

WebHostLib/templates/landing.html

@@ -11,7 +11,7 @@
     <div id="landing-wrapper">
         <div id="landing-header">
             <img id="landing-logo" src="static/static/branding/landing-logo.png" alt="Archipelago Logo" />
-            <h4>multiworld multi-game randomizer</h4>
+            <h4>multiworld multi-game randomizer</h4><h5>beta</h5>
         </div>
         <div id="landing-links">
             <a href="/games" id="far-left-button">Supported<br />Games</a>
@@ -35,7 +35,8 @@
         </div>
         <div id="landing" class="grass-island">
             <div id="landing-body">
-                <p id="first-line">Welcome to Archipelago!</p>
+                <p id="first-line">Welcome to Archipelago Beta!</p>
+                <p>For the stable version, visit <a href="//archipelago.gg">Archipelago.gg</a>!</p>
                 <p>
                     This is a cross-game modification system which randomizes different games, then uses the result to
                     build a single unified multi-player game. Items from one game may be present in another, and

WebHostLib/templates/pageWrapper.html

@@ -21,6 +21,7 @@
                 </div>
             {% endif %}
         {% endwith %}
+        <div class="user-message">This is the beta site! For the stable version, visit <a href="https://archipelago.gg">Archipelago.gg</a>!</div>
 
         {% block body %}
         {% endblock %}

WebHostLib/templates/supportedGames.html

@@ -33,9 +33,7 @@
         <h1>Currently Supported Games</h1>
         <p>Below are the games that are currently included with the Archipelago software. To play a game that is not on
         this page, please refer to the <a href="/tutorial/Archipelago/setup/en#playing-with-custom-worlds">playing with
-                custom worlds</a> section of the setup guide and the
-            <a href="{{ url_for("tutorial", game="Archipelago", file="other_en") }}">other games and tools guide</a>
-            to find more.</p>
+        custom worlds</a> section of the setup guide.</p>
         <div class="js-only">
             <label for="game-search">Search for your game below!</label><br />
             <div class="page-controls">

apmw/multiserver/gamespackagecache.py

@@ -0,0 +1,96 @@
+import typing as t
+from weakref import WeakValueDictionary
+
+from NetUtils import GamesPackage
+
+GameAndChecksum = tuple[str, str | None]
+ItemNameGroups = dict[str, list[str]]
+LocationNameGroups = dict[str, list[str]]
+
+
+K = t.TypeVar("K")
+V = t.TypeVar("V")
+
+
+class DictLike(dict[K, V]):
+    __slots__ = ("__weakref__",)
+
+
+class GamesPackageCache:
+    # NOTE: this uses 3 separate collections because unpacking the get() result would end the container lifetime
+    _reduced_games_packages: WeakValueDictionary[GameAndChecksum, GamesPackage]
+    """Does not include item_name_groups nor location_name_groups"""
+    _item_name_groups: WeakValueDictionary[GameAndChecksum, dict[str, list[str]]]
+    _location_name_groups: WeakValueDictionary[GameAndChecksum, dict[str, list[str]]]
+
+    def __init__(self) -> None:
+        self._reduced_games_packages = WeakValueDictionary()
+        self._item_name_groups = WeakValueDictionary()
+        self._location_name_groups = WeakValueDictionary()
+
+    def _get(
+        self,
+        cache_key: GameAndChecksum,
+    ) -> tuple[GamesPackage | None, ItemNameGroups | None, LocationNameGroups | None]:
+        if cache_key[1] is None:
+            return None, None, None
+        return (
+            self._reduced_games_packages.get(cache_key, None),
+            self._item_name_groups.get(cache_key, None),
+            self._location_name_groups.get(cache_key, None),
+        )
+
+    def get(
+        self,
+        game: str,
+        full_games_package: GamesPackage,
+    ) -> tuple[GamesPackage, ItemNameGroups, LocationNameGroups]:
+        """Loads and caches embedded data package provided by multidata"""
+        cache_key = (game, full_games_package.get("checksum", None))
+        cached_reduced_games_package, cached_item_name_groups, cached_location_name_groups = self._get(cache_key)
+
+        if cached_reduced_games_package is None:
+            cached_reduced_games_package = t.cast(
+                t.Any,
+                DictLike(
+                    {
+                        "item_name_to_id": full_games_package["item_name_to_id"],
+                        "location_name_to_id": full_games_package["location_name_to_id"],
+                        "checksum": full_games_package.get("checksum", None),
+                    }
+                ),
+            )
+            if cache_key[1] is not None:  # only cache if checksum is available
+                self._reduced_games_packages[cache_key] = cached_reduced_games_package
+
+        if cached_item_name_groups is None:
+            # optimize strings to be references instead of copies
+            item_names = {name: name for name in cached_reduced_games_package["item_name_to_id"].keys()}
+            cached_item_name_groups = DictLike(
+                {
+                    group_name: [item_names.get(item_name, item_name) for item_name in group_items]
+                    for group_name, group_items in full_games_package["item_name_groups"].items()
+                }
+            )
+            if cache_key[1] is not None:  # only cache if checksum is available
+                self._item_name_groups[cache_key] = cached_item_name_groups
+
+        if cached_location_name_groups is None:
+            # optimize strings to be references instead of copies
+            location_names = {name: name for name in cached_reduced_games_package["location_name_to_id"].keys()}
+            cached_location_name_groups = DictLike(
+                {
+                    group_name: [location_names.get(location_name, location_name) for location_name in group_locations]
+                    for group_name, group_locations in full_games_package.get("location_name_groups", {}).items()
+                }
+            )
+            if cache_key[1] is not None:  # only cache if checksum is available
+                self._location_name_groups[cache_key] = cached_location_name_groups
+
+        return cached_reduced_games_package, cached_item_name_groups, cached_location_name_groups
+
+    def get_static(self, game: str) -> tuple[GamesPackage, ItemNameGroups, LocationNameGroups]:
+        """Loads legacy data package from installed worlds"""
+        import worlds
+
+        return self.get(game, worlds.network_data_package["games"][game])

apmw/webhost/customserver/gamespackagecache.py

@@ -0,0 +1,42 @@
+from typing_extensions import override
+
+from NetUtils import GamesPackage
+from Utils import restricted_loads
+from apmw.multiserver.gamespackagecache import GamesPackageCache, ItemNameGroups, LocationNameGroups
+
+
+class DBGamesPackageCache(GamesPackageCache):
+    _static: dict[str, tuple[GamesPackage, ItemNameGroups, LocationNameGroups]]
+
+    def __init__(self, static_games_package: dict[str, GamesPackage]) -> None:
+        super().__init__()
+        self._static = {
+            game: GamesPackageCache.get(self, game, games_package)
+            for game, games_package in static_games_package.items()
+        }
+
+    @override
+    def get(
+        self,
+        game: str,
+        full_games_package: GamesPackage,
+    ) -> tuple[GamesPackage, ItemNameGroups, LocationNameGroups]:
+        # for games started on webhost, full_games_package is likely unpopulated and only has the checksum field
+        cache_key = (game, full_games_package.get("checksum", None))
+        cached = self._get(cache_key)
+        if any(value is None for value in cached):
+            if "checksum" not in full_games_package:
+                return super().get(game, full_games_package)  # no checksum, assume fully populated
+
+            from WebHostLib.models import GameDataPackage
+
+            row: GameDataPackage | None = GameDataPackage.get(checksum=full_games_package["checksum"])
+            if row:  # None if rolled on >= 0.3.9 but uploaded to <= 0.3.8 ...
+                return super().get(game, restricted_loads(row.data))
+            return super().get(game, full_games_package)  # ... in which case full_games_package should be populated
+
+        return cached  # type: ignore # mypy doesn't understand any value is None
+
+    @override
+    def get_static(self, game: str) -> tuple[GamesPackage, ItemNameGroups, LocationNameGroups]:
+        return self._static[game]

docs/CODEOWNERS

@@ -19,6 +19,8 @@
 # NewSoupVi is acting maintainer, but world belongs to core with the exception of the music
 /worlds/apquest/ @NewSoupVi
 
+# Sudoku (APSudoku)
+/worlds/apsudoku/ @EmilyV99
 
 # Aquaria
 /worlds/aquaria/ @tioui

docs/webhost configuration sample.yaml

@@ -17,6 +17,12 @@
 # Web hosting port
 #PORT: 80
 
+# Ports used for game hosting. Values can be specific ports, port ranges or both. Default is: [49152-65535, 0]
+# Zero means it will use a random free port if there is no port in the next 1024 randomly chosen ports from the range
+# Examples of valid values: [40000-41000, 49152-65535]
+# If ports within the range(s) are already in use, the WebHost will fallback to the default [49152-65535, 0] range.
+#GAME_PORTS: [49152-65535, 0]
+
 # Place where uploads go.
 #UPLOAD_FOLDER: uploads
 

setup.py

@@ -71,6 +71,7 @@ non_apworlds: set[str] = {
     "Ocarina of Time",
     "Overcooked! 2",
     "Raft",
+    "Sudoku",
     "Super Mario 64",
     "VVVVVV",
     "Wargroove",

test/general/test_implemented.py

@@ -11,7 +11,7 @@ class TestImplemented(unittest.TestCase):
     def test_completion_condition(self):
         """Ensure a completion condition is set that has requirements."""
         for game_name, world_type in AutoWorldRegister.world_types.items():
-            if not world_type.hidden:
+            if not world_type.hidden and game_name not in {"Sudoku"}:
                 with self.subTest(game_name):
                     multiworld = setup_solo_multiworld(world_type)
                     self.assertFalse(multiworld.completion_condition[1](multiworld.state))
@@ -59,7 +59,7 @@ class TestImplemented(unittest.TestCase):
     def test_prefill_items(self):
         """Test that every world can reach every location from allstate before pre_fill."""
         for gamename, world_type in AutoWorldRegister.world_types.items():
-            if gamename not in ("Archipelago", "Final Fantasy", "Test Game"):
+            if gamename not in ("Archipelago", "Sudoku", "Final Fantasy", "Test Game"):
                 with self.subTest(gamename):
                     multiworld = setup_solo_multiworld(world_type, ("generate_early", "create_regions", "create_items",
                                                                     "set_rules", "connect_entrances", "generate_basic"))

test/general/test_options.py

@@ -109,7 +109,7 @@ class TestOptions(unittest.TestCase):
     def test_option_set_keys_random(self):
         """Tests that option sets do not contain 'random' and its variants as valid keys"""
         for game_name, world_type in AutoWorldRegister.world_types.items():
-            if game_name not in ("Archipelago", "Super Metroid"):
+            if game_name not in ("Archipelago", "Sudoku", "Super Metroid"):
                 for option_key, option in world_type.options_dataclass.type_hints.items():
                     if issubclass(option, OptionSet):
                         with self.subTest(game=game_name, option=option_key):

test/multiserver/test_gamespackage_cache.py

@@ -0,0 +1,132 @@
+import typing as t
+from copy import deepcopy
+from unittest import TestCase
+
+from typing_extensions import override
+
+import NetUtils
+from NetUtils import GamesPackage
+from apmw.multiserver.gamespackagecache import GamesPackageCache
+
+
+class GamesPackageCacheTest(TestCase):
+    cache: GamesPackageCache
+    any_game: t.ClassVar[str] = "APQuest"
+    example_games_package: GamesPackage = {
+        "item_name_to_id": {"Item 1": 1},
+        "item_name_groups": {"Everything": ["Item 1"]},
+        "location_name_to_id": {"Location 1": 1},
+        "location_name_groups": {"Everywhere": ["Location 1"]},
+        "checksum": "1234",
+    }
+
+    @override
+    def setUp(self) -> None:
+        self.cache = GamesPackageCache()
+
+    def test_get_static_is_same(self) -> None:
+        """Tests that get_static returns the same objects twice"""
+        reduced_games_package1, item_name_groups1, location_name_groups1 = self.cache.get_static(self.any_game)
+        reduced_games_package2, item_name_groups2, location_name_groups2 = self.cache.get_static(self.any_game)
+        self.assertIs(reduced_games_package1, reduced_games_package2)
+        self.assertIs(item_name_groups1, item_name_groups2)
+        self.assertIs(location_name_groups1, location_name_groups2)
+
+    def test_get_static_data_format(self) -> None:
+        """Tests that get_static returns data in the correct format"""
+        reduced_games_package, item_name_groups, location_name_groups = self.cache.get_static(self.any_game)
+        self.assertTrue(reduced_games_package["checksum"])
+        self.assertTrue(reduced_games_package["item_name_to_id"])
+        self.assertTrue(reduced_games_package["location_name_to_id"])
+        self.assertNotIn("item_name_groups", reduced_games_package)
+        self.assertNotIn("location_name_groups", reduced_games_package)
+        self.assertTrue(item_name_groups["Everything"])
+        self.assertTrue(location_name_groups["Everywhere"])
+
+    def test_get_static_is_serializable(self) -> None:
+        """Tests that get_static returns data that can be serialized"""
+        NetUtils.encode(self.cache.get_static(self.any_game))
+
+    def test_get_static_missing_raises(self) -> None:
+        """Tests that get_static raises KeyError if the world is missing"""
+        with self.assertRaises(KeyError):
+            _ = self.cache.get_static("Does not exist")
+
+    def test_eviction(self) -> None:
+        """Tests that unused items get evicted from cache"""
+        game_name = "Test"
+        before_add = len(self.cache._reduced_games_packages)
+        data = self.cache.get(game_name, self.example_games_package)
+        self.assertTrue(data)
+        self.assertEqual(before_add + 1, len(self.cache._reduced_games_packages))
+
+        del data
+        if len(self.cache._reduced_games_packages) != before_add:  # gc.collect() may not even be required
+            import gc
+
+            gc.collect()
+
+        self.assertEqual(before_add, len(self.cache._reduced_games_packages))
+
+    def test_get_required_field(self) -> None:
+        """Tests that missing required field raises a KeyError"""
+        for field in ("item_name_to_id", "location_name_to_id", "item_name_groups"):
+            with self.subTest(field=field):
+                games_package = deepcopy(self.example_games_package)
+                del games_package[field]  # type: ignore
+                with self.assertRaises(KeyError):
+                    _ = self.cache.get(self.any_game, games_package)
+
+    def test_get_optional_properties(self) -> None:
+        """Tests that missing optional field works"""
+        for field in ("checksum", "location_name_groups"):
+            with self.subTest(field=field):
+                games_package = deepcopy(self.example_games_package)
+                del games_package[field]  # type: ignore
+                _, item_name_groups, location_name_groups = self.cache.get(self.any_game, games_package)
+                self.assertTrue(item_name_groups)
+                self.assertEqual(field != "location_name_groups", bool(location_name_groups))
+
+    def test_item_name_deduplication(self) -> None:
+        n = 1
+        s1 = f"Item {n}"
+        s2 = f"Item {n}"
+        # check if the deduplication is actually gonna do anything
+        self.assertIsNot(s1, s2)
+        self.assertEqual(s1, s2)
+        # do the thing
+        game_name = "Test"
+        games_package: GamesPackage = {
+            "item_name_to_id": {s1: n},
+            "item_name_groups": {"Everything": [s2]},
+            "location_name_to_id": {},
+            "location_name_groups": {},
+            "checksum": "1234",
+        }
+        reduced_games_package, item_name_groups, location_name_groups = self.cache.get(game_name, games_package)
+        self.assertIs(
+            next(iter(reduced_games_package["item_name_to_id"].keys())),
+            item_name_groups["Everything"][0],
+        )
+
+    def test_location_name_deduplication(self) -> None:
+        n = 1
+        s1 = f"Location {n}"
+        s2 = f"Location {n}"
+        # check if the deduplication is actually gonna do anything
+        self.assertIsNot(s1, s2)
+        self.assertEqual(s1, s2)
+        # do the thing
+        game_name = "Test"
+        games_package: GamesPackage = {
+            "item_name_to_id": {},
+            "item_name_groups": {},
+            "location_name_to_id": {s1: n},
+            "location_name_groups": {"Everywhere": [s2]},
+            "checksum": "1234",
+        }
+        reduced_games_package, item_name_groups, location_name_groups = self.cache.get(game_name, games_package)
+        self.assertIs(
+            next(iter(reduced_games_package["location_name_to_id"].keys())),
+            location_name_groups["Everywhere"][0],
+        )

test/webhost/test_port_allocation.py

@@ -0,0 +1,86 @@
+import os
+import unittest
+
+from Utils import is_macos
+from WebHostLib.customserver import parse_game_ports, create_random_port_socket, get_used_ports
+
+ci = bool(os.environ.get("CI"))
+
+
+class TestPortAllocating(unittest.TestCase):
+    def test_parse_game_ports(self) -> None:
+        """Ensure that game ports with ranges are parsed correctly"""
+        val = parse_game_ports(("1000-2000", "2000-5000", "1000-2000", 20, 40, "20", "0"))
+
+        self.assertListEqual(val.parsed_ports,
+                             [range(1000, 2001), range(2000, 5001), range(1000, 2001), range(20, 21), range(40, 41),
+                              range(20, 21)], "The parsed game ports are not the expected values")
+        self.assertTrue(val.ephemeral_allowed, "The ephemeral allowed flag is not set even though it was passed")
+        self.assertListEqual(val.weights, [1001, 4002, 5003, 5004, 5005, 5006],
+                             "Cumulative weights are not the expected value")
+
+        val = parse_game_ports(())
+        self.assertListEqual(val.parsed_ports, [], "Empty list of game port returned something")
+        self.assertFalse(val.ephemeral_allowed, "Empty list returned that ephemeral is allowed")
+
+        val = parse_game_ports((0,))
+        self.assertListEqual(val.parsed_ports, [], "Empty list of ranges returned something")
+        self.assertTrue(val.ephemeral_allowed, "List with just 0 is not allowing ephemeral ports")
+
+        val = parse_game_ports((1,))
+        self.assertEqual(val.parsed_ports, [range(1, 2)], "Parsed ports doesn't contain the expected values")
+        self.assertFalse(val.ephemeral_allowed, "List with just single port returned that ephemeral is allowed")
+
+    def test_parse_game_port_errors(self) -> None:
+        """Ensure that game ports with incorrect values raise the expected error"""
+        with self.assertRaises(ValueError, msg="Negative numbers didn't get interpreted as an invalid range"):
+            parse_game_ports(tuple("-50215"))
+        with self.assertRaises(ValueError, msg="Text got interpreted as a valid number"):
+            parse_game_ports(tuple("dwafawg"))
+        with self.assertRaises(
+                ValueError,
+                msg="A range with an extra dash at the end didn't get interpreted as an invalid number because of it's end dash"
+        ):
+            parse_game_ports(tuple("20-21215-"))
+        with self.assertRaises(ValueError, msg="Text got interpreted as a valid number for the start of a range"):
+            parse_game_ports(tuple("f-21215"))
+
+    def test_random_port_socket_edge_cases(self) -> None:
+        """Verify if edge cases on creation of random port socket is working fine"""
+        # Try giving an empty tuple and fail over it
+        with self.assertRaises(OSError) as err:
+            create_random_port_socket(tuple(), "127.0.0.1")
+        self.assertEqual(err.exception.errno, 98, "Raised an unexpected error code")
+        self.assertEqual(err.exception.strerror, "No available ports", "Raised an unexpected error string")
+
+        # Try only having ephemeral ports enabled
+        try:
+            create_random_port_socket(("0",), "127.0.0.1").close()
+        except OSError as err:
+            self.assertEqual(err.errno, 98, "Raised an unexpected error code")
+            # If it returns our error string that means something is wrong with our code
+            self.assertNotEqual(err.strerror, "No available ports",
+                                "Raised an unexpected error string")
+
+    @unittest.skipUnless(ci, "can't guarantee free ports outside of CI")
+    def test_random_port_socket(self) -> None:
+        """Verify if returned sockets use the correct port ranges"""
+        sockets = []
+        for _ in range(6):
+            socket = create_random_port_socket(("8080-8085",), "127.0.0.1")
+            sockets.append(socket)
+            _, port = socket.getsockname()
+            self.assertIn(port, range(8080, 8086), "Port of socket was not inside the expected range")
+        for s in sockets:
+            s.close()
+
+        sockets.clear()
+        length = 5_000 if is_macos else (30_000 - len(get_used_ports()))
+        for _ in range(length):
+            socket = create_random_port_socket(("30000-65535",), "127.0.0.1")
+            sockets.append(socket)
+            _, port = socket.getsockname()
+            self.assertIn(port, range(30_000, 65536), "Port of socket was not inside the expected range")
+
+        for s in sockets:
+            s.close()

test/webhost_customserver/test_gamespackage_cache.py

@@ -0,0 +1,147 @@
+import typing as t
+from copy import deepcopy
+
+from typing_extensions import override
+
+from test.multiserver.test_gamespackage_cache import GamesPackageCacheTest
+
+import Utils
+from NetUtils import GamesPackage
+from apmw.webhost.customserver.gamespackagecache import DBGamesPackageCache
+
+
+class FakeGameDataPackage:
+    _rows: "t.ClassVar[dict[str, FakeGameDataPackage]]" = {}
+    data: bytes
+
+    @classmethod
+    def get(cls, checksum: str) -> "FakeGameDataPackage | None":
+        return cls._rows.get(checksum, None)
+
+    @classmethod
+    def add(cls, checksum: str, full_games_package: GamesPackage) -> None:
+        row = FakeGameDataPackage()
+        row.data = Utils.restricted_dumps(full_games_package)
+        cls._rows[checksum] = row
+
+
+class DBGamesPackageCacheTest(GamesPackageCacheTest):
+    cache: DBGamesPackageCache
+    any_game: t.ClassVar[str] = "My Game"
+    static_data: t.ClassVar[dict[str, GamesPackage]] = {  # noqa: pycharm doesn't understand this
+        "My Game": {
+            "item_name_to_id": {"Item 1": 1},
+            "location_name_to_id": {"Location 1": 1},
+            "item_name_groups": {"Everything": ["Item 1"]},
+            "location_name_groups": {"Everywhere": ["Location 1"]},
+            "checksum": "2345",
+        }
+    }
+    orig_db_type: t.ClassVar[type]
+
+    @override
+    @classmethod
+    def setUpClass(cls) -> None:
+        import WebHostLib.models
+
+        cls.orig_db_type = WebHostLib.models.GameDataPackage
+        WebHostLib.models.GameDataPackage = FakeGameDataPackage  # type: ignore
+
+    @override
+    def setUp(self) -> None:
+        self.cache = DBGamesPackageCache(self.static_data)
+
+    @override
+    @classmethod
+    def tearDownClass(cls) -> None:
+        import WebHostLib.models
+
+        WebHostLib.models.GameDataPackage = cls.orig_db_type  # type: ignore
+
+    def assert_conversion(
+        self,
+        full_games_package: GamesPackage,
+        reduced_games_package: dict[str, t.Any],
+        item_name_groups: dict[str, t.Any],
+        location_name_groups: dict[str, t.Any],
+    ) -> None:
+        for key in ("item_name_to_id", "location_name_to_id", "checksum"):
+            if key in full_games_package:
+                self.assertEqual(reduced_games_package[key], full_games_package[key])  # noqa: pycharm
+        self.assertEqual(item_name_groups, full_games_package["item_name_groups"])
+        self.assertEqual(location_name_groups, full_games_package["location_name_groups"])
+
+    def assert_static_conversion(
+        self,
+        full_games_package: GamesPackage,
+        reduced_games_package: dict[str, t.Any],
+        item_name_groups: dict[str, t.Any],
+        location_name_groups: dict[str, t.Any],
+    ) -> None:
+        self.assert_conversion(full_games_package, reduced_games_package, item_name_groups, location_name_groups)
+        for key in ("item_name_to_id", "location_name_to_id", "checksum"):
+            self.assertIs(reduced_games_package[key], full_games_package[key])  # noqa: pycharm
+
+    def test_get_static_contents(self) -> None:
+        """Tests that get_static returns the correct data"""
+        reduced_games_package, item_name_groups, location_name_groups = self.cache.get_static(self.any_game)
+        for key in ("item_name_to_id", "location_name_to_id", "checksum"):
+            self.assertIs(reduced_games_package[key], self.static_data[self.any_game][key])  # noqa: pycharm
+        self.assertEqual(item_name_groups, self.static_data[self.any_game]["item_name_groups"])
+        self.assertEqual(location_name_groups, self.static_data[self.any_game]["location_name_groups"])
+
+    def test_static_not_evicted(self) -> None:
+        """Tests that static data is not evicted from cache during gc"""
+        import gc
+
+        game_name = next(iter(self.static_data.keys()))
+        ids = [id(o) for o in self.cache.get_static(game_name)]
+        gc.collect()
+        self.assertEqual(ids, [id(o) for o in self.cache.get_static(game_name)])
+
+    def test_get_is_static(self) -> None:
+        """Tests that a get with correct checksum return the static items"""
+        # NOTE: this is only true for the DB cache, not the "regular" one, since we want to avoid loading worlds there
+        cks: GamesPackage = {"checksum": self.static_data[self.any_game]["checksum"]}  # noqa: pycharm doesn't like this
+        reduced_games_package1, item_name_groups1, location_name_groups1 = self.cache.get(self.any_game, cks)
+        reduced_games_package2, item_name_groups2, location_name_groups2 = self.cache.get_static(self.any_game)
+        self.assertIs(reduced_games_package1, reduced_games_package2)
+        self.assertEqual(location_name_groups1, location_name_groups2)
+        self.assertEqual(item_name_groups1, item_name_groups2)
+
+    def test_get_from_db(self) -> None:
+        """Tests that a get with only checksum will load the full data from db and is cached"""
+        game_name = "Another Game"
+        full_games_package = deepcopy(self.static_data[self.any_game])
+        full_games_package["checksum"] = "3456"
+        cks: GamesPackage = {"checksum": full_games_package["checksum"]}  # noqa: pycharm doesn't like this
+        FakeGameDataPackage.add(full_games_package["checksum"], full_games_package)
+        before_add = len(self.cache._reduced_games_packages)
+        data = self.cache.get(game_name, cks)
+        self.assert_conversion(full_games_package, *data)  # type: ignore
+        self.assertEqual(before_add + 1, len(self.cache._reduced_games_packages))
+
+    def test_get_missing_from_db_uses_full_games_package(self) -> None:
+        """Tests that a get with full data (missing from db) will use the full data and is cached"""
+        game_name = "Yet Another Game"
+        full_games_package = deepcopy(self.static_data[self.any_game])
+        full_games_package["checksum"] = "4567"
+        before_add = len(self.cache._reduced_games_packages)
+        data = self.cache.get(game_name, full_games_package)
+        self.assert_conversion(full_games_package, *data)  # type: ignore
+        self.assertEqual(before_add + 1, len(self.cache._reduced_games_packages))
+
+    def test_get_without_checksum_uses_full_games_package(self) -> None:
+        """Tests that a get with full data and no checksum will use the full data and is not cached"""
+        game_name = "Yet Another Game"
+        full_games_package = deepcopy(self.static_data[self.any_game])
+        del full_games_package["checksum"]
+        before_add = len(self.cache._reduced_games_packages)
+        data = self.cache.get(game_name, full_games_package)
+        self.assert_conversion(full_games_package, *data)  # type: ignore
+        self.assertEqual(before_add, len(self.cache._reduced_games_packages))
+
+    def test_get_missing_from_db_raises(self) -> None:
+        """Tests that a get that requires a row to exist raise an exception if it doesn't"""
+        with self.assertRaises(Exception):
+            _ = self.cache.get("Does not exist", {"checksum": "0000"})

worlds/apsudoku/__init__.py

@@ -0,0 +1,34 @@
+from typing import Dict
+
+from BaseClasses import Tutorial
+from ..AutoWorld import WebWorld, World
+
+class AP_SudokuWebWorld(WebWorld):
+    options_page = False
+    theme = 'partyTime'
+
+    setup_en = Tutorial(
+        tutorial_name='Setup Guide',
+        description='A guide to playing APSudoku',
+        language='English',
+        file_name='setup_en.md',
+        link='setup/en',
+        authors=['EmilyV']
+    )
+    
+    tutorials = [setup_en]
+
+class AP_SudokuWorld(World):
+    """
+    Play a little Sudoku while you're in BK mode to maybe get some useful hints
+    """
+    game = "Sudoku"
+    web = AP_SudokuWebWorld()
+
+    item_name_to_id: Dict[str, int] = {}
+    location_name_to_id: Dict[str, int] = {}
+
+    @classmethod
+    def stage_assert_generate(cls, multiworld):
+        raise Exception("APSudoku cannot be used for generating worlds, the client can instead connect to any slot from any world")
+

worlds/apsudoku/docs/en_Sudoku.md

@@ -0,0 +1,15 @@
+# APSudoku
+
+## Hint Games
+
+HintGames do not need to be added at the start of a seed, and do not create a 'slot'- instead, you connect the HintGame client to a different game's slot. By playing a HintGame, you can earn hints for the connected slot.
+
+## What is this game?
+
+Play Sudoku puzzles of varying difficulties, earning a hint for each puzzle correctly solved. Harder puzzles are more likely to grant a hint towards a Progression item, though otherwise what hint is granted is random.
+
+## Where is the options page?
+
+There is no options page; this game cannot be used in your .yamls. Instead, the client can connect to any slot in a multiworld.
+
+By using the connected room's Admin Password on the Admin Panel tab, you can configure some settings at any time to affect the entire room. This allows disabling hints entirely, as well as altering the hint odds for each difficulty.

worlds/apsudoku/docs/setup_en.md

@@ -0,0 +1,55 @@
+# APSudoku Setup Guide
+
+## Required Software
+- [APSudoku](https://github.com/APSudoku/APSudoku)
+
+## General Concept
+
+This is a HintGame client, which can connect to any multiworld slot, allowing you to play Sudoku to unlock random hints for that slot's locations.
+
+Does not need to be added at the start of a seed, as it does not create any slots of its own, nor does it have any YAML files.
+
+## Installation Procedures
+
+### Windows / Linux
+Go to the latest release from the [github APSudoku Releases page](https://github.com/APSudoku/APSudoku/releases/latest). Download and extract the appropriate file for your platform.
+
+### Web
+Go to the [github pages](apsudoku.github.io) or [itch.io](https://emilyv99.itch.io/apsudoku) site, and play in the browser.
+
+## Joining a MultiWorld Game
+
+1. Run the APSudoku executable.
+2. Under `Settings` → `Connection` at the top-right:
+	- Enter the server address and port number
+	- Enter the name of the slot you wish to connect to
+	- Enter the room password (optional)
+	- Select DeathLink related settings (optional)
+	- Press `Connect`
+4. Under the `Sudoku` tab
+	- Choose puzzle difficulty
+	- Click `Start` to generate a puzzle
+5. Try to solve the Sudoku. Click `Check` when done
+	- A correct solution rewards you with 1 hint for a location in the world you are connected to
+	- An incorrect solution has no penalty, unless DeathLink is enabled (see below)
+
+Info:
+- You can set various settings under `Settings` → `Sudoku`, and can change the colors used under `Settings` → `Theme`.
+- While connected, you can view the `Console` and `Hints` tabs for standard TextClient-like features
+- You can also use the `Tracking` tab to view either a basic tracker or a valid [GodotAP tracker pack](https://github.com/EmilyV99/GodotAP/blob/main/tracker_packs/GET_PACKS.md)
+- While connected, the number of "unhinted" locations for your slot is shown in the upper-left of the the `Sudoku` tab. (If this reads 0, no further hints can be earned for this slot, as every locations is already hinted)
+- Click the various `?` buttons for information on controls/how to play
+
+## Admin Settings
+
+By using the connected room's Admin Password on the Admin Panel tab, you can configure some settings at any time to affect the entire room.
+
+- You can disable APSudoku for the entire room, preventing any hints from being granted.
+- You can customize the reward weights for each difficulty, making progression hints more or less likely, and/or adding a chance to get "no hint" after a solve.
+
+## DeathLink Support
+
+If `DeathLink` is enabled when you click `Connect`:
+- Lose a life if you check an incorrect puzzle (not an _incomplete_ puzzle- if any cells are empty, you get off with a warning), or if you quit a puzzle without solving it (including disconnecting).
+- Your life count is customizable (default 0). Dying with 0 lives left kills linked players AND resets your puzzle.
+- On receiving a DeathLink from another player, your puzzle resets. 

worlds/generic/__init__.py

@@ -26,10 +26,7 @@ class GenericWeb(WebWorld):
                      'English', 'setup_en.md', 'setup/en', ['alwaysintreble'])
     triggers = Tutorial('Archipelago Triggers Guide', 'A guide to setting up and using triggers in your game settings.',
                         'English', 'triggers_en.md', 'triggers/en', ['alwaysintreble'])
-    other_games = Tutorial('Other Games and Tools',
-                           'A guide to additional games and tools that can be used with Archipelago.',
-                           'English', 'other_en.md', 'other/en', ['Berserker'])
-    tutorials = [setup, mac, commands, advanced_settings, triggers, plando, other_games]
+    tutorials = [setup, mac, commands, advanced_settings, triggers, plando]
 
 
 class GenericWorld(World):

worlds/generic/docs/other_en.md

@@ -1,33 +0,0 @@
-# Other Games And Tools
-
-This page provides information and links regarding various tools that may be of use with Archipelago, including additional playable games not supported by this website.
-
-You should only download and use files from sources you trust; sources listed here are not officially vetted for safety, so use your own judgement and caution.
-
-## Discord
-
-Currently, Discord is the primary hub for Archipelago; whether it be finding people to play with, developing new game implementations, or finding new playable games.
-
-The [Archipelago Official Discord](https://discord.gg/8Z65BR2) is the main hub, while the [Archipelago After Dark Discord](https://discord.gg/fqvNCCRsu4) houses additional games that may be unrated or 18+ in some territories.
-
-The `#apworld-index` channels in each of these servers contain lists of playable games which should be easily downloadable and playable with an Archipelago installation.
-
-## Wiki
-
-The community-maintained [Archipelago Wiki](https://archipelago.miraheze.org/) has information on many games as well, and acts as a great discord-free source of information.
-
-## Hint Games
-
-Hint Games are a special type of game which are not included as part of the multiworld generation process. Instead, they can log in to an ongoing multiworld, connecting to a slot designated for any game. Rather than earning items for other games in the multiworld, a Hint Game will allow you to earn hints for the slot you are connected to.
-
-Hint Games can be found from sources such as the Discord and the [Hint Game Category](https://archipelago.miraheze.org/wiki/Category:Hint_games) of the wiki, as detailed above.
-
-## Notable Tools
-
-### Options Creator
-
-The Options Creator is included in the Archipelago installation, and is accessible from the Archipelago Launcher. Using this simple GUI tool, you can easily create randomization options for any installed '.apworld'- perfect when using custom worlds you've installed that don't have options pages on the website.
-
-### PopTracker
-
-[PopTracker](https://github.com/black-sliver/PopTracker) is a popular tool in Randomizer communities, which many games support via custom PopTracker Packs. Many Archipelago packs include the ability to directly connect to your slot for auto-tracking capabilities. (Check each game's setup guide to see if it has PopTracker compatibility!)

worlds/kh2/Locations.py

@@ -1281,7 +1281,7 @@ exclusion_table = {
         LocationName.HadesCupTrophyParadoxCups,
         LocationName.MusicalOrichalcumPlus,
     ],
-    "HitlistCasual":          [
+    "HitlistCasual":          {
         LocationName.FuturePete,
         LocationName.BetwixtandBetweenBondofFlame,
         LocationName.GrimReaper2,
@@ -1299,7 +1299,7 @@ exclusion_table = {
         LocationName.MCP,
         LocationName.Lvl50,
         LocationName.Lvl99
-    ],
+    },
     "Cups":          {
         LocationName.ProtectBeltPainandPanicCup,
         LocationName.SerenityGemPainandPanicCup,