Bring change to Weighted Options as well

* CI: build: fail fast if setup.py fails on windows

* CI: build: fail for missing uploads, rework compression

Upload-artifact allows setting compression level now.
The change speeds up both upload and extraction.

* CI: match build gz in release

* CI: build: verify worlds all load

* CI: build: generate a game

* Generate: move worlds loaded exception to allow settings to init from worlds

* CI: build: build setup before running tests

.github/workflows/build.yml

@@ -40,6 +40,10 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           python setup.py build_exe --yes
+          if ( $? -eq $false ) {
+            Write-Error "setup.py failed!"
+            exit 1
+          }
           $NAME="$(ls build | Select-String -Pattern 'exe')".Split('.',2)[1]
           $ZIP_NAME="Archipelago_$NAME.7z"
           echo "$NAME -> $ZIP_NAME"
@@ -49,12 +53,6 @@ jobs:
           Rename-Item "exe.$NAME" Archipelago
           7z a -mx=9 -mhe=on -ms "../dist/$ZIP_NAME" Archipelago
           Rename-Item Archipelago "exe.$NAME"  # inno_setup.iss expects the original name
-      - name: Store 7z
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ env.ZIP_NAME }}
-          path: dist/${{ env.ZIP_NAME }}
-          retention-days: 7  # keep for 7 days, should be enough
       - name: Build Setup
         run: |
           & "${env:ProgramFiles(x86)}\Inno Setup 6\iscc.exe" inno_setup.iss /DNO_SIGNTOOL
@@ -65,11 +63,38 @@ jobs:
           $contents = Get-ChildItem -Path setups/*.exe -Force -Recurse
           $SETUP_NAME=$contents[0].Name
           echo "SETUP_NAME=$SETUP_NAME" >> $Env:GITHUB_ENV
+      - name: Check build loads expected worlds
+        shell: bash
+        run: |
+          cd build/exe*
+          mv Players/Templates/meta.yaml .
+          ls -1 Players/Templates | sort > setup-player-templates.txt
+          rm -R Players/Templates
+          timeout 30 ./ArchipelagoLauncher "Generate Template Options" || true
+          ls -1 Players/Templates | sort > generated-player-templates.txt
+          cmp setup-player-templates.txt generated-player-templates.txt \
+            || diff setup-player-templates.txt generated-player-templates.txt
+          mv meta.yaml Players/Templates/
+      - name: Test Generate
+        shell: bash
+        run: |
+          cd build/exe*
+          cp Players/Templates/Clique.yaml Players/
+          timeout 30 ./ArchipelagoGenerate
+      - name: Store 7z
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.ZIP_NAME }}
+          path: dist/${{ env.ZIP_NAME }}
+          compression-level: 0  # .7z is incompressible by zip
+          if-no-files-found: error
+          retention-days: 7  # keep for 7 days, should be enough
       - name: Store Setup
         uses: actions/upload-artifact@v4
         with:
           name: ${{ env.SETUP_NAME }}
           path: setups/${{ env.SETUP_NAME }}
+          if-no-files-found: error
           retention-days: 7  # keep for 7 days, should be enough
 
   build-ubuntu2004:
@@ -110,7 +135,7 @@ jobs:
           echo -e "setup.py dist output:\n `ls dist`"
           cd dist && export APPIMAGE_NAME="`ls *.AppImage`" && cd ..
           export TAR_NAME="${APPIMAGE_NAME%.AppImage}.tar.gz"
-          (cd build && DIR_NAME="`ls | grep exe`" && mv "$DIR_NAME" Archipelago && tar -czvf ../dist/$TAR_NAME Archipelago && mv Archipelago "$DIR_NAME")
+          (cd build && DIR_NAME="`ls | grep exe`" && mv "$DIR_NAME" Archipelago && tar -cv Archipelago | gzip -8 > ../dist/$TAR_NAME && mv Archipelago "$DIR_NAME")
           echo "APPIMAGE_NAME=$APPIMAGE_NAME" >> $GITHUB_ENV
           echo "TAR_NAME=$TAR_NAME" >> $GITHUB_ENV
       # - copy code above to release.yml -
@@ -118,15 +143,36 @@ jobs:
         run: |
           source venv/bin/activate
           python setup.py build_exe --yes
+      - name: Check build loads expected worlds
+        shell: bash
+        run: |
+          cd build/exe*
+          mv Players/Templates/meta.yaml .
+          ls -1 Players/Templates | sort > setup-player-templates.txt
+          rm -R Players/Templates
+          timeout 30 ./ArchipelagoLauncher "Generate Template Options" || true
+          ls -1 Players/Templates | sort > generated-player-templates.txt
+          cmp setup-player-templates.txt generated-player-templates.txt \
+            || diff setup-player-templates.txt generated-player-templates.txt
+          mv meta.yaml Players/Templates/
+      - name: Test Generate
+        shell: bash
+        run: |
+          cd build/exe*
+          cp Players/Templates/Clique.yaml Players/
+          timeout 30 ./ArchipelagoGenerate
       - name: Store AppImage
         uses: actions/upload-artifact@v4
         with:
           name: ${{ env.APPIMAGE_NAME }}
           path: dist/${{ env.APPIMAGE_NAME }}
+          if-no-files-found: error
           retention-days: 7
       - name: Store .tar.gz
         uses: actions/upload-artifact@v4
         with:
           name: ${{ env.TAR_NAME }}
           path: dist/${{ env.TAR_NAME }}
+          compression-level: 0  # .gz is incompressible by zip
+          if-no-files-found: error
           retention-days: 7

.github/workflows/ctest.yml

@@ -0,0 +1,54 @@
+# Run CMake / CTest C++ unit tests
+
+name: ctest
+
+on:
+  push:
+    paths:
+      - '**.cc?'
+      - '**.cpp'
+      - '**.cxx'
+      - '**.hh?'
+      - '**.hpp'
+      - '**.hxx'
+      - '**.CMakeLists'
+      - '.github/workflows/ctest.yml'
+  pull_request:
+    paths:
+      - '**.cc?'
+      - '**.cpp'
+      - '**.cxx'
+      - '**.hh?'
+      - '**.hpp'
+      - '**.hxx'
+      - '**.CMakeLists'
+      - '.github/workflows/ctest.yml'
+
+jobs:
+  ctest:
+    runs-on: ${{ matrix.os }}
+    name: Test C++ ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+
+    steps:
+    - uses: actions/checkout@v4
+    - uses: ilammy/msvc-dev-cmd@v1
+      if: startsWith(matrix.os,'windows')
+    - uses: Bacondish2023/setup-googletest@v1
+      with:
+        build-type: 'Release'
+    - name: Build tests
+      run: |
+        cd test/cpp
+        mkdir build
+        cmake -S . -B build/ -DCMAKE_BUILD_TYPE=Release
+        cmake --build build/ --config Release
+        ls
+    - name: Run tests
+      run: |
+        cd test/cpp
+        ctest --test-dir build/ -C Release --output-on-failure

.github/workflows/release.yml

@@ -69,7 +69,7 @@ jobs:
           echo -e "setup.py dist output:\n `ls dist`"
           cd dist && export APPIMAGE_NAME="`ls *.AppImage`" && cd ..
           export TAR_NAME="${APPIMAGE_NAME%.AppImage}.tar.gz"
-          (cd build && DIR_NAME="`ls | grep exe`" && mv "$DIR_NAME" Archipelago && tar -czvf ../dist/$TAR_NAME Archipelago && mv Archipelago "$DIR_NAME")
+          (cd build && DIR_NAME="`ls | grep exe`" && mv "$DIR_NAME" Archipelago && tar -cv Archipelago | gzip -8 > ../dist/$TAR_NAME && mv Archipelago "$DIR_NAME")
           echo "APPIMAGE_NAME=$APPIMAGE_NAME" >> $GITHUB_ENV
           echo "TAR_NAME=$TAR_NAME" >> $GITHUB_ENV
       # - code above copied from build.yml -

.github/workflows/unittests.yml

@@ -24,7 +24,7 @@ on:
       - '.github/workflows/unittests.yml'
 
 jobs:
-  build:
+  unit:
     runs-on: ${{ matrix.os }}
     name: Test Python ${{ matrix.python.version }} ${{ matrix.os }}
 
@@ -60,3 +60,32 @@ jobs:
     - name: Unittests
       run: |
         pytest -n auto
+
+  hosting:
+    runs-on: ${{ matrix.os }}
+    name: Test hosting with ${{ matrix.python.version }} on ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+        python:
+          - {version: '3.11'}  # current
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python.version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python.version }}
+    - name: Install dependencies
+      run: |
+        python -m venv venv
+        source venv/bin/activate
+        python -m pip install --upgrade pip
+        python ModuleUpdate.py --yes --force --append "WebHostLib/requirements.txt"
+    - name: Test hosting
+      run: |
+        source venv/bin/activate
+        export PYTHONPATH=$(pwd)
+        python test/hosting/__main__.py

.gitignore

@@ -62,6 +62,7 @@ Output Logs/
 /installdelete.iss
 /data/user.kv
 /datapackage
+/custom_worlds
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -177,6 +178,7 @@ dmypy.json
 cython_debug/
 
 # Cython intermediates
+_speedups.c
 _speedups.cpp
 _speedups.html
 

AHITClient.py

@@ -0,0 +1,8 @@
+from worlds.ahit.Client import launch
+import Utils
+import ModuleUpdate
+ModuleUpdate.update()
+
+if __name__ == "__main__":
+    Utils.init_logging("AHITClient", exception_logger="Client")
+    launch()

AdventureClient.py

@@ -80,7 +80,7 @@ class AdventureContext(CommonContext):
         self.local_item_locations = {}
         self.dragon_speed_info = {}
 
-        options = Utils.get_options()
+        options = Utils.get_settings()
         self.display_msgs = options["adventure_options"]["display_msgs"]
 
     async def server_auth(self, password_requested: bool = False):
@@ -102,7 +102,7 @@ class AdventureContext(CommonContext):
     def on_package(self, cmd: str, args: dict):
         if cmd == 'Connected':
             self.locations_array = None
-            if Utils.get_options()["adventure_options"].get("death_link", False):
+            if Utils.get_settings()["adventure_options"].get("death_link", False):
                 self.set_deathlink = True
             async_start(self.get_freeincarnates_used())
         elif cmd == "RoomInfo":
@@ -112,7 +112,7 @@ class AdventureContext(CommonContext):
             if ': !' not in msg:
                 self._set_message(msg, SYSTEM_MESSAGE_ID)
         elif cmd == "ReceivedItems":
-            msg = f"Received {', '.join([self.item_names[item.item] for item in args['items']])}"
+            msg = f"Received {', '.join([self.item_names.lookup_in_slot(item.item) for item in args['items']])}"
             self._set_message(msg, SYSTEM_MESSAGE_ID)
         elif cmd == "Retrieved":
             if f"adventure_{self.auth}_freeincarnates_used" in args["keys"]:
@@ -415,8 +415,8 @@ async def atari_sync_task(ctx: AdventureContext):
 
 
 async def run_game(romfile):
-    auto_start = Utils.get_options()["adventure_options"].get("rom_start", True)
-    rom_args = Utils.get_options()["adventure_options"].get("rom_args")
+    auto_start = Utils.get_settings()["adventure_options"].get("rom_start", True)
+    rom_args = Utils.get_settings()["adventure_options"].get("rom_args")
     if auto_start is True:
         import webbrowser
         webbrowser.open(romfile)

BaseClasses.py

@@ -728,7 +728,7 @@ class CollectionState():
                 return True
         return False
     
-    def has_from_list_exclusive(self, items: Iterable[str], player: int, count: int) -> bool:
+    def has_from_list_unique(self, items: Iterable[str], player: int, count: int) -> bool:
         """Returns True if the state contains at least `count` items matching any of the item names from a list.
         Ignores duplicates of the same item."""
         found: int = 0
@@ -743,7 +743,7 @@ class CollectionState():
         """Returns the cumulative count of items from a list present in state."""
         return sum(self.prog_items[player][item_name] for item_name in items)
     
-    def count_from_list_exclusive(self, items: Iterable[str], player: int) -> int:
+    def count_from_list_unique(self, items: Iterable[str], player: int) -> int:
         """Returns the cumulative count of items from a list present in state. Ignores duplicates of the same item."""
         return sum(self.prog_items[player][item_name] > 0 for item_name in items)
 
@@ -758,7 +758,7 @@ class CollectionState():
                 return True
         return False
 
-    def has_group_exclusive(self, item_name_group: str, player: int, count: int = 1) -> bool:
+    def has_group_unique(self, item_name_group: str, player: int, count: int = 1) -> bool:
         """Returns True if the state contains at least `count` items present in a specified item group.
         Ignores duplicates of the same item.
         """
@@ -778,7 +778,7 @@ class CollectionState():
             for item_name in self.multiworld.worlds[player].item_name_groups[item_name_group]
         )
 
-    def count_group_exclusive(self, item_name_group: str, player: int) -> int:
+    def count_group_unique(self, item_name_group: str, player: int) -> int:
         """Returns the cumulative count of items from an item group present in state.
         Ignores duplicates of the same item."""
         player_prog_items = self.prog_items[player]

CommonClient.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import collections
 import copy
 import logging
 import asyncio
@@ -8,6 +9,7 @@ import sys
 import typing
 import time
 import functools
+import warnings
 
 import ModuleUpdate
 ModuleUpdate.update()
@@ -173,10 +175,74 @@ class CommonContext:
     items_handling: typing.Optional[int] = None
     want_slot_data: bool = True  # should slot_data be retrieved via Connect
 
-    # data package
-    # Contents in flux until connection to server is made, to download correct data for this multiworld.
-    item_names: typing.Dict[int, str] = Utils.KeyedDefaultDict(lambda code: f'Unknown item (ID:{code})')
-    location_names: typing.Dict[int, str] = Utils.KeyedDefaultDict(lambda code: f'Unknown location (ID:{code})')
+    class NameLookupDict:
+        """A specialized dict, with helper methods, for id -> name item/location data package lookups by game."""
+        def __init__(self, ctx: CommonContext, lookup_type: typing.Literal["item", "location"]):
+            self.ctx: CommonContext = ctx
+            self.lookup_type: typing.Literal["item", "location"] = lookup_type
+            self._unknown_item: typing.Callable[[int], str] = lambda key: f"Unknown {lookup_type} (ID: {key})"
+            self._archipelago_lookup: typing.Dict[int, str] = {}
+            self._flat_store: typing.Dict[int, str] = Utils.KeyedDefaultDict(self._unknown_item)
+            self._game_store: typing.Dict[str, typing.ChainMap[int, str]] = collections.defaultdict(
+                lambda: collections.ChainMap(self._archipelago_lookup, Utils.KeyedDefaultDict(self._unknown_item)))
+            self.warned: bool = False
+
+        # noinspection PyTypeChecker
+        def __getitem__(self, key: str) -> typing.Mapping[int, str]:
+            # TODO: In a future version (0.6.0?) this should be simplified by removing implicit id lookups support.
+            if isinstance(key, int):
+                if not self.warned:
+                    # Use warnings instead of logger to avoid deprecation message from appearing on user side.
+                    self.warned = True
+                    warnings.warn(f"Implicit name lookup by id only is deprecated and only supported to maintain "
+                                  f"backwards compatibility for now. If multiple games share the same id for a "
+                                  f"{self.lookup_type}, name could be incorrect. Please use "
+                                  f"`{self.lookup_type}_names.lookup_in_game()` or "
+                                  f"`{self.lookup_type}_names.lookup_in_slot()` instead.")
+                return self._flat_store[key]  # type: ignore
+
+            return self._game_store[key]
+
+        def __len__(self) -> int:
+            return len(self._game_store)
+
+        def __iter__(self) -> typing.Iterator[str]:
+            return iter(self._game_store)
+
+        def __repr__(self) -> str:
+            return self._game_store.__repr__()
+
+        def lookup_in_game(self, code: int, game_name: typing.Optional[str] = None) -> str:
+            """Returns the name for an item/location id in the context of a specific game or own game if `game` is
+            omitted.
+            """
+            if game_name is None:
+                game_name = self.ctx.game
+                assert game_name is not None, f"Attempted to lookup {self.lookup_type} with no game name available."
+
+            return self._game_store[game_name][code]
+
+        def lookup_in_slot(self, code: int, slot: typing.Optional[int] = None) -> str:
+            """Returns the name for an item/location id in the context of a specific slot or own slot if `slot` is
+            omitted.
+            """
+            if slot is None:
+                slot = self.ctx.slot
+                assert slot is not None, f"Attempted to lookup {self.lookup_type} with no slot info available."
+
+            return self.lookup_in_game(code, self.ctx.slot_info[slot].game)
+
+        def update_game(self, game: str, name_to_id_lookup_table: typing.Dict[str, int]) -> None:
+            """Overrides existing lookup tables for a particular game."""
+            id_to_name_lookup_table = Utils.KeyedDefaultDict(self._unknown_item)
+            id_to_name_lookup_table.update({code: name for name, code in name_to_id_lookup_table.items()})
+            self._game_store[game] = collections.ChainMap(self._archipelago_lookup, id_to_name_lookup_table)
+            self._flat_store.update(id_to_name_lookup_table)  # Only needed for legacy lookup method.
+            if game == "Archipelago":
+                # Keep track of the Archipelago data package separately so if it gets updated in a custom datapackage,
+                # it updates in all chain maps automatically.
+                self._archipelago_lookup.clear()
+                self._archipelago_lookup.update(id_to_name_lookup_table)
 
     # defaults
     starting_reconnect_delay: int = 5
@@ -231,7 +297,7 @@ class CommonContext:
     # message box reporting a loss of connection
     _messagebox_connection_loss: typing.Optional["kvui.MessageBox"] = None
 
-    def __init__(self, server_address: typing.Optional[str], password: typing.Optional[str]) -> None:
+    def __init__(self, server_address: typing.Optional[str] = None, password: typing.Optional[str] = None) -> None:
         # server state
         self.server_address = server_address
         self.username = None
@@ -271,6 +337,9 @@ class CommonContext:
         self.exit_event = asyncio.Event()
         self.watcher_event = asyncio.Event()
 
+        self.item_names = self.NameLookupDict(self, "item")
+        self.location_names = self.NameLookupDict(self, "location")
+
         self.jsontotextparser = JSONtoTextParser(self)
         self.rawjsontotextparser = RawJSONtoTextParser(self)
         self.update_data_package(network_data_package)
@@ -424,6 +493,11 @@ class CommonContext:
         """Gets called before sending a Say to the server from the user.
         Returned text is sent, or sending is aborted if None is returned."""
         return text
+    
+    def on_ui_command(self, text: str) -> None:
+        """Gets called by kivy when the user executes a command starting with `/` or `!`.
+        The command processor is still called; this is just intended for command echoing."""
+        self.ui.print_json([{"text": text, "type": "color", "color": "orange"}])
 
     def update_permissions(self, permissions: typing.Dict[str, int]):
         for permission_name, permission_flag in permissions.items():
@@ -486,19 +560,17 @@ class CommonContext:
                         or remote_checksum != cache_checksum:
                     needed_updates.add(game)
                 else:
-                    self.update_game(cached_game)
+                    self.update_game(cached_game, game)
         if needed_updates:
             await self.send_msgs([{"cmd": "GetDataPackage", "games": [game_name]} for game_name in needed_updates])
 
-    def update_game(self, game_package: dict):
-        for item_name, item_id in game_package["item_name_to_id"].items():
-            self.item_names[item_id] = item_name
-        for location_name, location_id in game_package["location_name_to_id"].items():
-            self.location_names[location_id] = location_name
+    def update_game(self, game_package: dict, game: str):
+        self.item_names.update_game(game, game_package["item_name_to_id"])
+        self.location_names.update_game(game, game_package["location_name_to_id"])
 
     def update_data_package(self, data_package: dict):
         for game, game_data in data_package["games"].items():
-            self.update_game(game_data)
+            self.update_game(game_data, game)
 
     def consume_network_data_package(self, data_package: dict):
         self.update_data_package(data_package)

Fill.py

@@ -35,8 +35,8 @@ def fill_restrictive(multiworld: MultiWorld, base_state: CollectionState, locati
     """
     :param multiworld: Multiworld to be filled.
     :param base_state: State assumed before fill.
-    :param locations: Locations to be filled with item_pool
-    :param item_pool: Items to fill into the locations
+    :param locations: Locations to be filled with item_pool, gets mutated by removing locations that get filled.
+    :param item_pool: Items to fill into the locations, gets mutated by removing items that get placed.
     :param single_player_placement: if true, can speed up placement if everything belongs to a single player
     :param lock: locations are set to locked as they are filled
     :param swap: if true, swaps of already place items are done in the event of a dead end
@@ -220,7 +220,8 @@ def fill_restrictive(multiworld: MultiWorld, base_state: CollectionState, locati
 def remaining_fill(multiworld: MultiWorld,
                    locations: typing.List[Location],
                    itempool: typing.List[Item],
-                   name: str = "Remaining") -> None:
+                   name: str = "Remaining", 
+                   move_unplaceable_to_start_inventory: bool = False) -> None:
     unplaced_items: typing.List[Item] = []
     placements: typing.List[Location] = []
     swapped_items: typing.Counter[typing.Tuple[int, str]] = Counter()
@@ -284,13 +285,21 @@ def remaining_fill(multiworld: MultiWorld,
 
     if unplaced_items and locations:
         # There are leftover unplaceable items and locations that won't accept them
-        raise FillError(f"No more spots to place {len(unplaced_items)} items. Remaining locations are invalid.\n"
-                        f"Unplaced items:\n"
-                        f"{', '.join(str(item) for item in unplaced_items)}\n"
-                        f"Unfilled locations:\n"
-                        f"{', '.join(str(location) for location in locations)}\n"
-                        f"Already placed {len(placements)}:\n"
-                        f"{', '.join(str(place) for place in placements)}")
+        if move_unplaceable_to_start_inventory:
+            last_batch = []
+            for item in unplaced_items:
+                logging.debug(f"Moved {item} to start_inventory to prevent fill failure.")
+                multiworld.push_precollected(item)
+                last_batch.append(multiworld.worlds[item.player].create_filler())
+            remaining_fill(multiworld, locations, unplaced_items, name + " Start Inventory Retry")
+        else:
+            raise FillError(f"No more spots to place {len(unplaced_items)} items. Remaining locations are invalid.\n"
+                            f"Unplaced items:\n"
+                            f"{', '.join(str(item) for item in unplaced_items)}\n"
+                            f"Unfilled locations:\n"
+                            f"{', '.join(str(location) for location in locations)}\n"
+                            f"Already placed {len(placements)}:\n"
+                            f"{', '.join(str(place) for place in placements)}")
 
     itempool.extend(unplaced_items)
 
@@ -420,7 +429,8 @@ def distribute_early_items(multiworld: MultiWorld,
     return fill_locations, itempool
 
 
-def distribute_items_restrictive(multiworld: MultiWorld) -> None:
+def distribute_items_restrictive(multiworld: MultiWorld,
+                                 panic_method: typing.Literal["swap", "raise", "start_inventory"] = "swap") -> None:
     fill_locations = sorted(multiworld.get_unfilled_locations())
     multiworld.random.shuffle(fill_locations)
     # get items to distribute
@@ -470,8 +480,29 @@ def distribute_items_restrictive(multiworld: MultiWorld) -> None:
 
     if progitempool:
         # "advancement/progression fill"
-        fill_restrictive(multiworld, multiworld.state, defaultlocations, progitempool, single_player_placement=multiworld.players == 1,
-                         name="Progression")
+        if panic_method == "swap":
+            fill_restrictive(multiworld, multiworld.state, defaultlocations, progitempool,
+                             swap=True,
+                             name="Progression", single_player_placement=multiworld.players == 1)
+        elif panic_method == "raise":
+            fill_restrictive(multiworld, multiworld.state, defaultlocations, progitempool,
+                             swap=False,
+                             name="Progression", single_player_placement=multiworld.players == 1)
+        elif panic_method == "start_inventory":
+            fill_restrictive(multiworld, multiworld.state, defaultlocations, progitempool,
+                             swap=False, allow_partial=True,
+                             name="Progression", single_player_placement=multiworld.players == 1)
+            if progitempool:
+                for item in progitempool:
+                    logging.debug(f"Moved {item} to start_inventory to prevent fill failure.")
+                    multiworld.push_precollected(item)
+                    filleritempool.append(multiworld.worlds[item.player].create_filler())
+                logging.warning(f"{len(progitempool)} items moved to start inventory,"
+                                f" due to failure in Progression fill step.")
+                progitempool[:] = []
+
+        else:
+            raise ValueError(f"Generator Panic Method {panic_method} not recognized.")
         if progitempool:
             raise FillError(
                 f"Not enough locations for progression items. "
@@ -486,7 +517,9 @@ def distribute_items_restrictive(multiworld: MultiWorld) -> None:
 
     inaccessible_location_rules(multiworld, multiworld.state, defaultlocations)
 
-    remaining_fill(multiworld, excludedlocations, filleritempool, "Remaining Excluded")
+    remaining_fill(multiworld, excludedlocations, filleritempool, "Remaining Excluded",
+                   move_unplaceable_to_start_inventory=panic_method=="start_inventory")
+
     if excludedlocations:
         raise FillError(
             f"Not enough filler items for excluded locations. "
@@ -495,7 +528,8 @@ def distribute_items_restrictive(multiworld: MultiWorld) -> None:
 
     restitempool = filleritempool + usefulitempool
 
-    remaining_fill(multiworld, defaultlocations, restitempool)
+    remaining_fill(multiworld, defaultlocations, restitempool,
+                   move_unplaceable_to_start_inventory=panic_method=="start_inventory")
 
     unplaced = restitempool
     unfilled = defaultlocations

Generate.py

@@ -1,36 +1,32 @@
 from __future__ import annotations
 
 import argparse
+import copy
 import logging
 import os
 import random
 import string
+import sys
 import urllib.parse
 import urllib.request
 from collections import Counter
 from typing import Any, Dict, Tuple, Union
+from itertools import chain
 
 import ModuleUpdate
 
 ModuleUpdate.update()
 
-import copy
 import Utils
 import Options
 from BaseClasses import seeddigits, get_seed, PlandoOptions
-from Main import main as ERmain
-from settings import get_settings
 from Utils import parse_yamls, version_tuple, __version__, tuplize_version
-from worlds.alttp.EntranceRandomizer import parse_arguments
-from worlds.alttp.Text import TextTable
-from worlds.AutoWorld import AutoWorldRegister
-from worlds.generic import PlandoConnection
-from worlds import failed_world_loads
 
 
 def mystery_argparse():
-    options = get_settings()
-    defaults = options.generator
+    from settings import get_settings
+    settings = get_settings()
+    defaults = settings.generator
 
     parser = argparse.ArgumentParser(description="CMD Generation Interface, defaults come from host.yaml.")
     parser.add_argument('--weights_file_path', default=defaults.weights_file_path,
@@ -42,7 +38,7 @@ def mystery_argparse():
     parser.add_argument('--seed', help='Define seed number to generate.', type=int)
     parser.add_argument('--multi', default=defaults.players, type=lambda value: max(int(value), 1))
     parser.add_argument('--spoiler', type=int, default=defaults.spoiler)
-    parser.add_argument('--outputpath', default=options.general_options.output_path,
+    parser.add_argument('--outputpath', default=settings.general_options.output_path,
                         help="Path to output folder. Absolute or relative to cwd.")  # absolute or relative to cwd
     parser.add_argument('--race', action='store_true', default=defaults.race)
     parser.add_argument('--meta_file_path', default=defaults.meta_file_path)
@@ -62,20 +58,23 @@ def mystery_argparse():
     if not os.path.isabs(args.meta_file_path):
         args.meta_file_path = os.path.join(args.player_files_path, args.meta_file_path)
     args.plando: PlandoOptions = PlandoOptions.from_option_string(args.plando)
-    return args, options
+    return args
 
 
 def get_seed_name(random_source) -> str:
     return f"{random_source.randint(0, pow(10, seeddigits) - 1)}".zfill(seeddigits)
 
 
-def main(args=None, callback=ERmain):
+def main(args=None):
+    # __name__ == "__main__" check so unittests that already imported worlds don't trip this.
+    if __name__ == "__main__" and "worlds" in sys.modules:
+        raise Exception("Worlds system should not be loaded before logging init.")
+
     if not args:
-        args, options = mystery_argparse()
-    else:
-        options = get_settings()
+        args = mystery_argparse()
 
     seed = get_seed(args.seed)
+
     Utils.init_logging(f"Generate_{seed}", loglevel=args.log_level)
     random.seed(seed)
     seed_name = get_seed_name(random)
@@ -144,6 +143,9 @@ def main(args=None, callback=ERmain):
         raise Exception(f"No weights found. "
                         f"Provide a general weights file ({args.weights_file_path}) or individual player files. "
                         f"A mix is also permitted.")
+
+    from worlds.AutoWorld import AutoWorldRegister
+    from worlds.alttp.EntranceRandomizer import parse_arguments
     erargs = parse_arguments(['--multi', str(args.multi)])
     erargs.seed = seed
     erargs.plando_options = args.plando
@@ -235,7 +237,8 @@ def main(args=None, callback=ERmain):
         with open(os.path.join(args.outputpath if args.outputpath else ".", f"generate_{seed_name}.yaml"), "wt") as f:
             yaml.dump(important, f)
 
-    return callback(erargs, seed)
+    from Main import main as ERmain
+    return ERmain(erargs, seed)
 
 
 def read_weights_yamls(path) -> Tuple[Any, ...]:
@@ -319,18 +322,34 @@ def update_weights(weights: dict, new_weights: dict, update_type: str, name: str
     logging.debug(f'Applying {new_weights}')
     cleaned_weights = {}
     for option in new_weights:
-        option_name = option.lstrip("+")
+        option_name = option.lstrip("+-")
         if option.startswith("+") and option_name in weights:
             cleaned_value = weights[option_name]
             new_value = new_weights[option]
-            if isinstance(new_value, (set, dict)):
+            if isinstance(new_value, set):
                 cleaned_value.update(new_value)
             elif isinstance(new_value, list):
                 cleaned_value.extend(new_value)
+            elif isinstance(new_value, dict):
+                cleaned_value = dict(Counter(cleaned_value) + Counter(new_value))
             else:
                 raise Exception(f"Cannot apply merge to non-dict, set, or list type {option_name},"
                                 f" received {type(new_value).__name__}.")
             cleaned_weights[option_name] = cleaned_value
+        elif option.startswith("-") and option_name in weights:
+            cleaned_value = weights[option_name]
+            new_value = new_weights[option]
+            if isinstance(new_value, set):
+                cleaned_value.difference_update(new_value)
+            elif isinstance(new_value, list):
+                for element in new_value:
+                    cleaned_value.remove(element)
+            elif isinstance(new_value, dict):
+                cleaned_value = dict(Counter(cleaned_value) - Counter(new_value))
+            else:
+                raise Exception(f"Cannot apply remove to non-dict, set, or list type {option_name},"
+                                f" received {type(new_value).__name__}.")
+            cleaned_weights[option_name] = cleaned_value
         else:
             cleaned_weights[option_name] = new_weights[option]
     new_options = set(cleaned_weights) - set(weights)
@@ -344,6 +363,8 @@ def update_weights(weights: dict, new_weights: dict, update_type: str, name: str
 
 
 def roll_meta_option(option_key, game: str, category_dict: Dict) -> Any:
+    from worlds import AutoWorldRegister
+
     if not game:
         return get_choice(option_key, category_dict)
     if game in AutoWorldRegister.world_types:
@@ -415,23 +436,25 @@ def handle_option(ret: argparse.Namespace, game_weights: dict, option_key: str,
                 player_option = option.from_any(game_weights[option_key])
             else:
                 player_option = option.from_any(get_choice(option_key, game_weights))
-            del game_weights[option_key]
         else:
             player_option = option.from_any(option.default)  # call the from_any here to support default "random"
         setattr(ret, option_key, player_option)
     except Exception as e:
         raise Options.OptionError(f"Error generating option {option_key} in {ret.game}") from e
     else:
+        from worlds import AutoWorldRegister
         player_option.verify(AutoWorldRegister.world_types[ret.game], ret.name, plando_options)
 
 
 def roll_settings(weights: dict, plando_options: PlandoOptions = PlandoOptions.bosses):
+    from worlds import AutoWorldRegister
+
     if "linked_options" in weights:
         weights = roll_linked_options(weights)
 
-    valid_trigger_names = set()
+    valid_keys = set()
     if "triggers" in weights:
-        weights = roll_triggers(weights, weights["triggers"], valid_trigger_names)
+        weights = roll_triggers(weights, weights["triggers"], valid_keys)
 
     requirements = weights.get("requires", {})
     if requirements:
@@ -452,6 +475,7 @@ def roll_settings(weights: dict, plando_options: PlandoOptions = PlandoOptions.b
 
     ret.game = get_choice("game", weights)
     if ret.game not in AutoWorldRegister.world_types:
+        from worlds import failed_world_loads
         picks = Utils.get_fuzzy_results(ret.game, list(AutoWorldRegister.world_types) + failed_world_loads, limit=1)[0]
         if picks[0] in failed_world_loads:
             raise Exception(f"No functional world found to handle game {ret.game}. "
@@ -466,12 +490,14 @@ def roll_settings(weights: dict, plando_options: PlandoOptions = PlandoOptions.b
     world_type = AutoWorldRegister.world_types[ret.game]
     game_weights = weights[ret.game]
 
-    if any(weight.startswith("+") for weight in game_weights) or \
-       any(weight.startswith("+") for weight in weights):
-        raise Exception(f"Merge tag cannot be used outside of trigger contexts.")
+    for weight in chain(game_weights, weights):
+        if weight.startswith("+"):
+            raise Exception(f"Merge tag cannot be used outside of trigger contexts. Found {weight}")
+        if weight.startswith("-"):
+            raise Exception(f"Remove tag cannot be used outside of trigger contexts. Found {weight}")
 
     if "triggers" in game_weights:
-        weights = roll_triggers(weights, game_weights["triggers"], valid_trigger_names)
+        weights = roll_triggers(weights, game_weights["triggers"], valid_keys)
         game_weights = weights[ret.game]
 
     ret.name = get_choice('name', weights)
@@ -480,42 +506,20 @@ def roll_settings(weights: dict, plando_options: PlandoOptions = PlandoOptions.b
 
     for option_key, option in world_type.options_dataclass.type_hints.items():
         handle_option(ret, game_weights, option_key, option, plando_options)
+        valid_keys.add(option_key)
     for option_key in game_weights:
-        if option_key in {"triggers", *valid_trigger_names}:
+        if option_key in {"triggers", *valid_keys}:
             continue
         logging.warning(f"{option_key} is not a valid option name for {ret.game} and is not present in triggers.")
     if PlandoOptions.items in plando_options:
         ret.plando_items = game_weights.get("plando_items", [])
     if ret.game == "A Link to the Past":
-        roll_alttp_settings(ret, game_weights, plando_options)
-    if PlandoOptions.connections in plando_options:
-        ret.plando_connections = []
-        options = game_weights.get("plando_connections", [])
-        for placement in options:
-            if roll_percentage(get_choice("percentage", placement, 100)):
-                ret.plando_connections.append(PlandoConnection(
-                    get_choice("entrance", placement),
-                    get_choice("exit", placement),
-                    get_choice("direction", placement, "both")
-                ))
+        roll_alttp_settings(ret, game_weights)
 
     return ret
 
 
-def roll_alttp_settings(ret: argparse.Namespace, weights, plando_options):
-
-    ret.plando_texts = {}
-    if PlandoOptions.texts in plando_options:
-        tt = TextTable()
-        tt.removeUnwantedText()
-        options = weights.get("plando_texts", [])
-        for placement in options:
-            if roll_percentage(get_choice_legacy("percentage", placement, 100)):
-                at = str(get_choice_legacy("at", placement))
-                if at not in tt:
-                    raise Exception(f"No text target \"{at}\" found.")
-                ret.plando_texts[at] = str(get_choice_legacy("text", placement))
-
+def roll_alttp_settings(ret: argparse.Namespace, weights):
     ret.sprite_pool = weights.get('sprite_pool', [])
     ret.sprite = get_choice_legacy('sprite', weights, "Link")
     if 'random_sprite_on_event' in weights:

Launcher.py

@@ -19,7 +19,7 @@ import sys
 import webbrowser
 from os.path import isfile
 from shutil import which
-from typing import Sequence, Union, Optional
+from typing import Callable, Sequence, Union, Optional
 
 import Utils
 import settings
@@ -160,8 +160,12 @@ def launch(exe, in_terminal=False):
     subprocess.Popen(exe)
 
 
+refresh_components: Optional[Callable[[], None]] = None
+
+
 def run_gui():
     from kvui import App, ContainerLayout, GridLayout, Button, Label, ScrollBox, Widget
+    from kivy.core.window import Window
     from kivy.uix.image import AsyncImage
     from kivy.uix.relativelayout import RelativeLayout
 
@@ -169,11 +173,8 @@ def run_gui():
         base_title: str = "Archipelago Launcher"
         container: ContainerLayout
         grid: GridLayout
-
-        _tools = {c.display_name: c for c in components if c.type == Type.TOOL}
-        _clients = {c.display_name: c for c in components if c.type == Type.CLIENT}
-        _adjusters = {c.display_name: c for c in components if c.type == Type.ADJUSTER}
-        _miscs = {c.display_name: c for c in components if c.type == Type.MISC}
+        _tool_layout: Optional[ScrollBox] = None
+        _client_layout: Optional[ScrollBox] = None
 
         def __init__(self, ctx=None):
             self.title = self.base_title
@@ -181,18 +182,7 @@ def run_gui():
             self.icon = r"data/icon.png"
             super().__init__()
 
-        def build(self):
-            self.container = ContainerLayout()
-            self.grid = GridLayout(cols=2)
-            self.container.add_widget(self.grid)
-            self.grid.add_widget(Label(text="General", size_hint_y=None, height=40))
-            self.grid.add_widget(Label(text="Clients", size_hint_y=None, height=40))
-            tool_layout = ScrollBox()
-            tool_layout.layout.orientation = "vertical"
-            self.grid.add_widget(tool_layout)
-            client_layout = ScrollBox()
-            client_layout.layout.orientation = "vertical"
-            self.grid.add_widget(client_layout)
+        def _refresh_components(self) -> None:
 
             def build_button(component: Component) -> Widget:
                 """
@@ -217,14 +207,49 @@ def run_gui():
                     return box_layout
                 return button
 
+            # clear before repopulating
+            assert self._tool_layout and self._client_layout, "must call `build` first"
+            tool_children = reversed(self._tool_layout.layout.children)
+            for child in tool_children:
+                self._tool_layout.layout.remove_widget(child)
+            client_children = reversed(self._client_layout.layout.children)
+            for child in client_children:
+                self._client_layout.layout.remove_widget(child)
+
+            _tools = {c.display_name: c for c in components if c.type == Type.TOOL}
+            _clients = {c.display_name: c for c in components if c.type == Type.CLIENT}
+            _adjusters = {c.display_name: c for c in components if c.type == Type.ADJUSTER}
+            _miscs = {c.display_name: c for c in components if c.type == Type.MISC}
+
             for (tool, client) in itertools.zip_longest(itertools.chain(
-                    self._tools.items(), self._miscs.items(), self._adjusters.items()), self._clients.items()):
+                _tools.items(), _miscs.items(), _adjusters.items()
+            ), _clients.items()):
                 # column 1
                 if tool:
-                    tool_layout.layout.add_widget(build_button(tool[1]))
+                    self._tool_layout.layout.add_widget(build_button(tool[1]))
                 # column 2
                 if client:
-                    client_layout.layout.add_widget(build_button(client[1]))
+                    self._client_layout.layout.add_widget(build_button(client[1]))
+
+        def build(self):
+            self.container = ContainerLayout()
+            self.grid = GridLayout(cols=2)
+            self.container.add_widget(self.grid)
+            self.grid.add_widget(Label(text="General", size_hint_y=None, height=40))
+            self.grid.add_widget(Label(text="Clients", size_hint_y=None, height=40))
+            self._tool_layout = ScrollBox()
+            self._tool_layout.layout.orientation = "vertical"
+            self.grid.add_widget(self._tool_layout)
+            self._client_layout = ScrollBox()
+            self._client_layout.layout.orientation = "vertical"
+            self.grid.add_widget(self._client_layout)
+
+            self._refresh_components()
+
+            global refresh_components
+            refresh_components = self._refresh_components
+
+            Window.bind(on_drop_file=self._on_drop_file)
 
             return self.container
 
@@ -235,6 +260,14 @@ def run_gui():
             else:
                 launch(get_exe(button.component), button.component.cli)
 
+        def _on_drop_file(self, window: Window, filename: bytes, x: int, y: int) -> None:
+            """ When a patch file is dropped into the window, run the associated component. """
+            file, component = identify(filename.decode())
+            if file and component:
+                run_component(component, file)
+            else:
+                logging.warning(f"unable to identify component for {filename}")
+
         def _stop(self, *largs):
             # ran into what appears to be https://groups.google.com/g/kivy-users/c/saWDLoYCSZ4 with PyCharm.
             # Closing the window explicitly cleans it up.
@@ -243,10 +276,17 @@ def run_gui():
 
     Launcher().run()
 
+    # avoiding Launcher reference leak
+    # and don't try to do something with widgets after window closed
+    global refresh_components
+    refresh_components = None
+
 
 def run_component(component: Component, *args):
     if component.func:
         component.func(*args)
+        if refresh_components:
+            refresh_components()
     elif component.script_name:
         subprocess.run([*get_exe(component.script_name), *args])
     else:

Main.py

@@ -13,7 +13,7 @@ import worlds
 from BaseClasses import CollectionState, Item, Location, LocationProgressType, MultiWorld, Region
 from Fill import balance_multiworld_progression, distribute_items_restrictive, distribute_planned, flood_items
 from Options import StartInventoryPool
-from Utils import __version__, output_path, version_tuple
+from Utils import __version__, output_path, version_tuple, get_settings
 from settings import get_settings
 from worlds import AutoWorld
 from worlds.generic.Rules import exclusion_rules, locality_rules
@@ -272,7 +272,7 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
     if multiworld.algorithm == 'flood':
         flood_items(multiworld)  # different algo, biased towards early game progress items
     elif multiworld.algorithm == 'balanced':
-        distribute_items_restrictive(multiworld)
+        distribute_items_restrictive(multiworld, get_settings().generator.panic_method)
 
     AutoWorld.call_all(multiworld, 'post_fill')
 
@@ -372,6 +372,17 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
 
                 checks_in_area: Dict[int, Dict[str, Union[int, List[int]]]] = {}
 
+                # get spheres -> filter address==None -> skip empty
+                spheres: List[Dict[int, Set[int]]] = []
+                for sphere in multiworld.get_spheres():
+                    current_sphere: Dict[int, Set[int]] = collections.defaultdict(set)
+                    for sphere_location in sphere:
+                        if type(sphere_location.address) is int:
+                            current_sphere[sphere_location.player].add(sphere_location.address)
+
+                    if current_sphere:
+                        spheres.append(dict(current_sphere))
+
                 multidata = {
                     "slot_data": slot_data,
                     "slot_info": slot_info,
@@ -386,6 +397,7 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
                     "tags": ["AP"],
                     "minimum_versions": minimum_versions,
                     "seed_name": multiworld.seed_name,
+                    "spheres": spheres,
                     "datapackage": data_package,
                 }
                 AutoWorld.call_all(multiworld, "modify_multidata", multidata)

MultiServer.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import argparse
 import asyncio
 import collections
+import contextlib
 import copy
 import datetime
 import functools
@@ -37,7 +38,7 @@ except ImportError:
 
 import NetUtils
 import Utils
-from Utils import version_tuple, restricted_loads, Version, async_start
+from Utils import version_tuple, restricted_loads, Version, async_start, get_intended_text
 from NetUtils import Endpoint, ClientStatus, NetworkItem, decode, encode, NetworkPlayer, Permission, NetworkSlot, \
     SlotType, LocationStore
 
@@ -168,15 +169,20 @@ class Context:
     slot_info: typing.Dict[int, NetworkSlot]
     generator_version = Version(0, 0, 0)
     checksums: typing.Dict[str, str]
-    item_names: typing.Dict[int, str] = Utils.KeyedDefaultDict(lambda code: f'Unknown item (ID:{code})')
+    item_names: typing.Dict[str, typing.Dict[int, str]] = (
+        collections.defaultdict(lambda: Utils.KeyedDefaultDict(lambda code: f'Unknown item (ID:{code})')))
     item_name_groups: typing.Dict[str, typing.Dict[str, typing.Set[str]]]
-    location_names: typing.Dict[int, str] = Utils.KeyedDefaultDict(lambda code: f'Unknown location (ID:{code})')
+    location_names: typing.Dict[str, typing.Dict[int, str]] = (
+        collections.defaultdict(lambda: Utils.KeyedDefaultDict(lambda code: f'Unknown location (ID:{code})')))
     location_name_groups: typing.Dict[str, typing.Dict[str, typing.Set[str]]]
     all_item_and_group_names: typing.Dict[str, typing.Set[str]]
     all_location_and_group_names: typing.Dict[str, typing.Set[str]]
-    non_hintable_names: typing.Dict[str, typing.Set[str]]
+    non_hintable_names: typing.Dict[str, typing.AbstractSet[str]]
+    spheres: typing.List[typing.Dict[int, typing.Set[int]]]
+    """ each sphere is { player: { location_id, ... } } """
     logger: logging.Logger
 
+
     def __init__(self, host: str, port: int, server_password: str, password: str, location_check_points: int,
                  hint_cost: int, item_cheat: bool, release_mode: str = "disabled", collect_mode="disabled",
                  remaining_mode: str = "disabled", auto_shutdown: typing.SupportsFloat = 0, compatibility: int = 2,
@@ -226,7 +232,7 @@ class Context:
         self.embedded_blacklist = {"host", "port"}
         self.client_ids: typing.Dict[typing.Tuple[int, int], datetime.datetime] = {}
         self.auto_save_interval = 60  # in seconds
-        self.auto_saver_thread = None
+        self.auto_saver_thread: typing.Optional[threading.Thread] = None
         self.save_dirty = False
         self.tags = ['AP']
         self.games: typing.Dict[int, str] = {}
@@ -238,6 +244,7 @@ class Context:
         self.stored_data = {}
         self.stored_data_notification_clients = collections.defaultdict(weakref.WeakSet)
         self.read_data = {}
+        self.spheres = []
 
         # init empty to satisfy linter, I suppose
         self.gamespackage = {}
@@ -262,19 +269,31 @@ class Context:
         for world_name, world in worlds.AutoWorldRegister.world_types.items():
             self.non_hintable_names[world_name] = world.hint_blacklist
 
+        for game_package in self.gamespackage.values():
+            # remove groups from data sent to clients
+            del game_package["item_name_groups"]
+            del game_package["location_name_groups"]
+
     def _init_game_data(self):
         for game_name, game_package in self.gamespackage.items():
             if "checksum" in game_package:
                 self.checksums[game_name] = game_package["checksum"]
             for item_name, item_id in game_package["item_name_to_id"].items():
-                self.item_names[item_id] = item_name
+                self.item_names[game_name][item_id] = item_name
             for location_name, location_id in game_package["location_name_to_id"].items():
-                self.location_names[location_id] = location_name
+                self.location_names[game_name][location_id] = location_name
             self.all_item_and_group_names[game_name] = \
                 set(game_package["item_name_to_id"]) | set(self.item_name_groups[game_name])
             self.all_location_and_group_names[game_name] = \
                 set(game_package["location_name_to_id"]) | set(self.location_name_groups.get(game_name, []))
 
+        archipelago_item_names = self.item_names["Archipelago"]
+        archipelago_location_names = self.location_names["Archipelago"]
+        for game in [game_name for game_name in self.gamespackage if game_name != "Archipelago"]:
+            # Add Archipelago items and locations to each data package.
+            self.item_names[game].update(archipelago_item_names)
+            self.location_names[game].update(archipelago_location_names)
+
     def item_names_for_game(self, game: str) -> typing.Optional[typing.Dict[str, int]]:
         return self.gamespackage[game]["item_name_to_id"] if game in self.gamespackage else None
 
@@ -466,6 +485,9 @@ class Context:
         for game_name, data in self.location_name_groups.items():
             self.read_data[f"location_name_groups_{game_name}"] = lambda lgame=game_name: self.location_name_groups[lgame]
 
+        # sorted access spheres
+        self.spheres = decoded_obj.get("spheres", [])
+
     # saving
 
     def save(self, now=False) -> bool:
@@ -508,7 +530,7 @@ class Context:
                 self.logger.exception(e)
             self._start_async_saving()
 
-    def _start_async_saving(self):
+    def _start_async_saving(self, atexit_save: bool = True):
         if not self.auto_saver_thread:
             def save_regularly():
                 # time.time() is platform dependent, so using the expensive datetime method instead
@@ -532,8 +554,9 @@ class Context:
             self.auto_saver_thread = threading.Thread(target=save_regularly, daemon=True)
             self.auto_saver_thread.start()
 
-            import atexit
-            atexit.register(self._save, True)  # make sure we save on exit too
+            if atexit_save:
+                import atexit
+                atexit.register(self._save, True)  # make sure we save on exit too
 
     def get_save(self) -> dict:
         self.recheck_hints()
@@ -623,6 +646,16 @@ class Context:
         self.recheck_hints(team, slot)
         return self.hints[team, slot]
 
+    def get_sphere(self, player: int, location_id: int) -> int:
+        """Get sphere of a location, -1 if spheres are not available."""
+        if self.spheres:
+            for i, sphere in enumerate(self.spheres):
+                if location_id in sphere.get(player, set()):
+                    return i
+            raise KeyError(f"No Sphere found for location ID {location_id} belonging to player {player}. "
+                           f"Location or player may not exist.")
+        return -1
+
     def get_players_package(self):
         return [NetworkPlayer(t, p, self.get_aliased_name(t, p), n) for (t, p), n in self.player_names.items()]
 
@@ -765,10 +798,7 @@ async def on_client_connected(ctx: Context, client: Client):
         for slot, connected_clients in clients.items():
             if connected_clients:
                 name = ctx.player_names[team, slot]
-                players.append(
-                    NetworkPlayer(team, slot,
-                                  ctx.name_aliases.get((team, slot), name), name)
-                )
+                players.append(NetworkPlayer(team, slot, ctx.name_aliases.get((team, slot), name), name))
     games = {ctx.games[x] for x in range(1, len(ctx.games) + 1)}
     games.add("Archipelago")
     await ctx.send_msgs(client, [{
@@ -783,8 +813,6 @@ async def on_client_connected(ctx: Context, client: Client):
         'permissions': get_permissions(ctx),
         'hint_cost': ctx.hint_cost,
         'location_check_points': ctx.location_check_points,
-        'datapackage_versions': {game: game_data["version"] for game, game_data
-                                 in ctx.gamespackage.items() if game in games},
         'datapackage_checksums': {game: game_data["checksum"] for game, game_data
                                   in ctx.gamespackage.items() if game in games and "checksum" in game_data},
         'seed_name': ctx.seed_name,
@@ -988,8 +1016,8 @@ def register_location_checks(ctx: Context, team: int, slot: int, locations: typi
             send_items_to(ctx, team, target_player, new_item)
 
             ctx.logger.info('(Team #%d) %s sent %s to %s (%s)' % (
-                team + 1, ctx.player_names[(team, slot)], ctx.item_names[item_id],
-                ctx.player_names[(team, target_player)], ctx.location_names[location]))
+                team + 1, ctx.player_names[(team, slot)], ctx.item_names[ctx.slot_info[target_player].game][item_id],
+                ctx.player_names[(team, target_player)], ctx.location_names[ctx.slot_info[slot].game][location]))
             info_text = json_format_send_event(new_item, target_player)
             ctx.broadcast_team(team, [info_text])
 
@@ -1043,8 +1071,8 @@ def collect_hint_location_id(ctx: Context, team: int, slot: int, seeked_location
 
 def format_hint(ctx: Context, team: int, hint: NetUtils.Hint) -> str:
     text = f"[Hint]: {ctx.player_names[team, hint.receiving_player]}'s " \
-           f"{ctx.item_names[hint.item]} is " \
-           f"at {ctx.location_names[hint.location]} " \
+           f"{ctx.item_names[ctx.slot_info[hint.receiving_player].game][hint.item]} is " \
+           f"at {ctx.location_names[ctx.slot_info[hint.finding_player].game][hint.location]} " \
            f"in {ctx.player_names[team, hint.finding_player]}'s World"
 
     if hint.entrance:
@@ -1073,28 +1101,6 @@ def json_format_send_event(net_item: NetworkItem, receiving_player: int):
             "item": net_item}
 
 
-def get_intended_text(input_text: str, possible_answers) -> typing.Tuple[str, bool, str]:
-    picks = Utils.get_fuzzy_results(input_text, possible_answers, limit=2)
-    if len(picks) > 1:
-        dif = picks[0][1] - picks[1][1]
-        if picks[0][1] == 100:
-            return picks[0][0], True, "Perfect Match"
-        elif picks[0][1] < 75:
-            return picks[0][0], False, f"Didn't find something that closely matches '{input_text}', " \
-                                       f"did you mean '{picks[0][0]}'? ({picks[0][1]}% sure)"
-        elif dif > 5:
-            return picks[0][0], True, "Close Match"
-        else:
-            return picks[0][0], False, f"Too many close matches for '{input_text}', " \
-                                       f"did you mean '{picks[0][0]}'? ({picks[0][1]}% sure)"
-    else:
-        if picks[0][1] > 90:
-            return picks[0][0], True, "Only Option Match"
-        else:
-            return picks[0][0], False, f"Didn't find something that closely matches '{input_text}', " \
-                                       f"did you mean '{picks[0][0]}'? ({picks[0][1]}% sure)"
-
-
 class CommandMeta(type):
     def __new__(cls, name, bases, attrs):
         commands = attrs["commands"] = {}
@@ -1346,7 +1352,7 @@ class ClientMessageProcessor(CommonCommandProcessor):
         if self.ctx.remaining_mode == "enabled":
             remaining_item_ids = get_remaining(self.ctx, self.client.team, self.client.slot)
             if remaining_item_ids:
-                self.output("Remaining items: " + ", ".join(self.ctx.item_names[item_id]
+                self.output("Remaining items: " + ", ".join(self.ctx.item_names[self.client.slot.game][item_id]
                                                             for item_id in remaining_item_ids))
             else:
                 self.output("No remaining items found.")
@@ -1359,7 +1365,7 @@ class ClientMessageProcessor(CommonCommandProcessor):
             if self.ctx.client_game_state[self.client.team, self.client.slot] == ClientStatus.CLIENT_GOAL:
                 remaining_item_ids = get_remaining(self.ctx, self.client.team, self.client.slot)
                 if remaining_item_ids:
-                    self.output("Remaining items: " + ", ".join(self.ctx.item_names[item_id]
+                    self.output("Remaining items: " + ", ".join(self.ctx.item_names[self.client.slot.game][item_id]
                                                                 for item_id in remaining_item_ids))
                 else:
                     self.output("No remaining items found.")
@@ -1377,7 +1383,8 @@ class ClientMessageProcessor(CommonCommandProcessor):
         locations = get_missing_checks(self.ctx, self.client.team, self.client.slot)
 
         if locations:
-            names = [self.ctx.location_names[location] for location in locations]
+            game = self.ctx.slot_info[self.client.slot].game
+            names = [self.ctx.location_names[game][location] for location in locations]
             if filter_text:
                 location_groups = self.ctx.location_name_groups[self.ctx.games[self.client.slot]]
                 if filter_text in location_groups:  # location group name
@@ -1402,7 +1409,8 @@ class ClientMessageProcessor(CommonCommandProcessor):
         locations = get_checked_checks(self.ctx, self.client.team, self.client.slot)
 
         if locations:
-            names = [self.ctx.location_names[location] for location in locations]
+            game = self.ctx.slot_info[self.client.slot].game
+            names = [self.ctx.location_names[game][location] for location in locations]
             if filter_text:
                 location_groups = self.ctx.location_name_groups[self.ctx.games[self.client.slot]]
                 if filter_text in location_groups:  # location group name
@@ -1483,10 +1491,10 @@ class ClientMessageProcessor(CommonCommandProcessor):
         elif input_text.isnumeric():
             game = self.ctx.games[self.client.slot]
             hint_id = int(input_text)
-            hint_name = self.ctx.item_names[hint_id] \
-                if not for_location and hint_id in self.ctx.item_names \
-                else self.ctx.location_names[hint_id] \
-                if for_location and hint_id in self.ctx.location_names \
+            hint_name = self.ctx.item_names[game][hint_id] \
+                if not for_location and hint_id in self.ctx.item_names[game] \
+                else self.ctx.location_names[game][hint_id] \
+                if for_location and hint_id in self.ctx.location_names[game] \
                 else None
             if hint_name in self.ctx.non_hintable_names[game]:
                 self.output(f"Sorry, \"{hint_name}\" is marked as non-hintable.")
@@ -1548,6 +1556,9 @@ class ClientMessageProcessor(CommonCommandProcessor):
                 self.ctx.random.shuffle(not_found_hints)
                 # By popular vote, make hints prefer non-local placements
                 not_found_hints.sort(key=lambda hint: int(hint.receiving_player != hint.finding_player))
+                # By another popular vote, prefer early sphere
+                not_found_hints.sort(key=lambda hint: self.ctx.get_sphere(hint.finding_player, hint.location),
+                                     reverse=True)
 
                 hints = found_hints + old_hints
                 while can_pay > 0:
@@ -1557,10 +1568,10 @@ class ClientMessageProcessor(CommonCommandProcessor):
                     hints.append(hint)
                     can_pay -= 1
                     self.ctx.hints_used[self.client.team, self.client.slot] += 1
-                    points_available = get_client_points(self.ctx, self.client)
 
                 self.ctx.notify_hints(self.client.team, hints)
                 if not_found_hints:
+                    points_available = get_client_points(self.ctx, self.client)
                     if hints and cost and int((points_available // cost) == 0):
                         self.output(
                             f"There may be more hintables, however, you cannot afford to pay for any more. "
@@ -1921,8 +1932,6 @@ class ServerCommandProcessor(CommonCommandProcessor):
     def _cmd_exit(self) -> bool:
         """Shutdown the server"""
         self.ctx.server.ws_server.close()
-        if self.ctx.shutdown_task:
-            self.ctx.shutdown_task.cancel()
         self.ctx.exit_event.set()
         return True
 
@@ -2280,7 +2289,8 @@ def parse_args() -> argparse.Namespace:
 
 
 async def auto_shutdown(ctx, to_cancel=None):
-    await asyncio.sleep(ctx.auto_shutdown)
+    with contextlib.suppress(asyncio.TimeoutError):
+        await asyncio.wait_for(ctx.exit_event.wait(), ctx.auto_shutdown)
 
     def inactivity_shutdown():
         ctx.server.ws_server.close()
@@ -2300,7 +2310,8 @@ async def auto_shutdown(ctx, to_cancel=None):
             if seconds < 0:
                 inactivity_shutdown()
             else:
-                await asyncio.sleep(seconds)
+                with contextlib.suppress(asyncio.TimeoutError):
+                    await asyncio.wait_for(ctx.exit_event.wait(), seconds)
 
 
 def load_server_cert(path: str, cert_key: typing.Optional[str]) -> "ssl.SSLContext":

NetUtils.py

@@ -198,7 +198,8 @@ class JSONtoTextParser(metaclass=HandlerMeta):
         "slateblue": "6D8BE8",
         "plum": "AF99EF",
         "salmon": "FA8072",
-        "white": "FFFFFF"
+        "white": "FFFFFF",
+        "orange": "FF7700",
     }
 
     def __init__(self, ctx):
@@ -247,7 +248,7 @@ class JSONtoTextParser(metaclass=HandlerMeta):
 
     def _handle_item_id(self, node: JSONMessagePart):
         item_id = int(node["text"])
-        node["text"] = self.ctx.item_names[item_id]
+        node["text"] = self.ctx.item_names.lookup_in_slot(item_id, node["player"])
         return self._handle_item_name(node)
 
     def _handle_location_name(self, node: JSONMessagePart):
@@ -255,8 +256,8 @@ class JSONtoTextParser(metaclass=HandlerMeta):
         return self._handle_color(node)
 
     def _handle_location_id(self, node: JSONMessagePart):
-        item_id = int(node["text"])
-        node["text"] = self.ctx.location_names[item_id]
+        location_id = int(node["text"])
+        node["text"] = self.ctx.location_names.lookup_in_slot(location_id, node["player"])
         return self._handle_location_name(node)
 
     def _handle_entrance_name(self, node: JSONMessagePart):

Options.py

@@ -12,6 +12,7 @@ from copy import deepcopy
 from dataclasses import dataclass
 
 from schema import And, Optional, Or, Schema
+from typing_extensions import Self
 
 from Utils import get_fuzzy_results, is_iterable_except_str
 
@@ -734,6 +735,12 @@ class NamedRange(Range):
         elif value > self.range_end and value not in self.special_range_names.values():
             raise Exception(f"{value} is higher than maximum {self.range_end} for option {self.__class__.__name__} " +
                             f"and is also not one of the supported named special values: {self.special_range_names}")
+        
+        # See docstring
+        for key in self.special_range_names:
+            if key != key.lower():
+                raise Exception(f"{self.__class__.__name__} has an invalid special_range_names key: {key}. "
+                                f"NamedRange keys must use only lowercase letters, and ideally should be snake_case.")
         self.value = value
 
     @classmethod
@@ -746,6 +753,7 @@ class NamedRange(Range):
 
 class FreezeValidKeys(AssembleOptions):
     def __new__(mcs, name, bases, attrs):
+        assert not "_valid_keys" in attrs, "'_valid_keys' gets set by FreezeValidKeys, define 'valid_keys' instead."
         if "valid_keys" in attrs:
             attrs["_valid_keys"] = frozenset(attrs["valid_keys"])
         return super(FreezeValidKeys, mcs).__new__(mcs, name, bases, attrs)
@@ -895,6 +903,228 @@ class ItemSet(OptionSet):
     convert_name_groups = True
 
 
+class PlandoText(typing.NamedTuple):
+    at: str
+    text: typing.List[str]
+    percentage: int = 100
+
+
+PlandoTextsFromAnyType = typing.Union[
+    typing.Iterable[typing.Union[typing.Mapping[str, typing.Any], PlandoText, typing.Any]], typing.Any
+]
+
+
+class PlandoTexts(Option[typing.List[PlandoText]], VerifyKeys):
+    default = ()
+    supports_weighting = False
+    display_name = "Plando Texts"
+
+    def __init__(self, value: typing.Iterable[PlandoText]) -> None:
+        self.value = list(deepcopy(value))
+        super().__init__()
+
+    def verify(self, world: typing.Type[World], player_name: str, plando_options: "PlandoOptions") -> None:
+        from BaseClasses import PlandoOptions
+        if self.value and not (PlandoOptions.texts & plando_options):
+            # plando is disabled but plando options were given so overwrite the options
+            self.value = []
+            logging.warning(f"The plando texts module is turned off, "
+                            f"so text for {player_name} will be ignored.")
+
+    @classmethod
+    def from_any(cls, data: PlandoTextsFromAnyType) -> Self:
+        texts: typing.List[PlandoText] = []
+        if isinstance(data, typing.Iterable):
+            for text in data:
+                if isinstance(text, typing.Mapping):
+                    if random.random() < float(text.get("percentage", 100)/100):
+                        at = text.get("at", None)
+                        if at is not None:
+                            given_text = text.get("text", [])
+                            if isinstance(given_text, str):
+                                given_text = [given_text]
+                            texts.append(PlandoText(
+                                at,
+                                given_text,
+                                text.get("percentage", 100)
+                            ))
+                elif isinstance(text, PlandoText):
+                    if random.random() < float(text.percentage/100):
+                        texts.append(text)
+                else:
+                    raise Exception(f"Cannot create plando text from non-dictionary type, got {type(text)}")
+            cls.verify_keys([text.at for text in texts])
+            return cls(texts)
+        else:
+            raise NotImplementedError(f"Cannot Convert from non-list, got {type(data)}")
+
+    @classmethod
+    def get_option_name(cls, value: typing.List[PlandoText]) -> str:
+        return str({text.at: " ".join(text.text) for text in value})
+
+    def __iter__(self) -> typing.Iterator[PlandoText]:
+        yield from self.value
+
+    def __getitem__(self, index: typing.SupportsIndex) -> PlandoText:
+        return self.value.__getitem__(index)
+
+    def __len__(self) -> int:
+        return self.value.__len__()
+
+
+class ConnectionsMeta(AssembleOptions):
+    def __new__(mcs, name: str, bases: tuple[type, ...], attrs: dict[str, typing.Any]):
+        if name != "PlandoConnections":
+            assert "entrances" in attrs, f"Please define valid entrances for {name}"
+            attrs["entrances"] = frozenset((connection.lower() for connection in attrs["entrances"]))
+            assert "exits" in attrs, f"Please define valid exits for {name}"
+            attrs["exits"] = frozenset((connection.lower() for connection in attrs["exits"]))
+        if "__doc__" not in attrs:
+            attrs["__doc__"] = PlandoConnections.__doc__
+        cls = super().__new__(mcs, name, bases, attrs)
+        return cls
+
+
+class PlandoConnection(typing.NamedTuple):
+    class Direction:
+        entrance = "entrance"
+        exit = "exit"
+        both = "both"
+
+    entrance: str
+    exit: str
+    direction: typing.Literal["entrance", "exit", "both"]  # TODO: convert Direction to StrEnum once 3.8 is dropped
+    percentage: int = 100
+
+
+PlandoConFromAnyType = typing.Union[
+    typing.Iterable[typing.Union[typing.Mapping[str, typing.Any], PlandoConnection, typing.Any]], typing.Any
+]
+
+
+class PlandoConnections(Option[typing.List[PlandoConnection]], metaclass=ConnectionsMeta):
+    """Generic connections plando. Format is:
+    - entrance: "Entrance Name"
+      exit: "Exit Name"
+      direction: "Direction"
+      percentage: 100
+    Direction must be one of 'entrance', 'exit', or 'both', and defaults to 'both' if omitted.
+    Percentage is an integer from 1 to 100, and defaults to 100 when omitted."""
+
+    display_name = "Plando Connections"
+
+    default = ()
+    supports_weighting = False
+
+    entrances: typing.ClassVar[typing.AbstractSet[str]]
+    exits: typing.ClassVar[typing.AbstractSet[str]]
+
+    duplicate_exits: bool = False
+    """Whether or not exits should be allowed to be duplicate."""
+
+    def __init__(self, value: typing.Iterable[PlandoConnection]):
+        self.value = list(deepcopy(value))
+        super(PlandoConnections, self).__init__()
+
+    @classmethod
+    def validate_entrance_name(cls, entrance: str) -> bool:
+        return entrance.lower() in cls.entrances
+
+    @classmethod
+    def validate_exit_name(cls, exit: str) -> bool:
+        return exit.lower() in cls.exits
+
+    @classmethod
+    def can_connect(cls, entrance: str, exit: str) -> bool:
+        """Checks that a given entrance can connect to a given exit.
+        By default, this will always return true unless overridden."""
+        return True
+
+    @classmethod
+    def validate_plando_connections(cls, connections: typing.Iterable[PlandoConnection]) -> None:
+        used_entrances: typing.List[str] = []
+        used_exits: typing.List[str] = []
+        for connection in connections:
+            entrance = connection.entrance
+            exit = connection.exit
+            direction = connection.direction
+            if direction not in (PlandoConnection.Direction.entrance,
+                                 PlandoConnection.Direction.exit,
+                                 PlandoConnection.Direction.both):
+                raise ValueError(f"Unknown direction: {direction}")
+            if entrance in used_entrances:
+                raise ValueError(f"Duplicate Entrance {entrance} not allowed.")
+            if not cls.duplicate_exits and exit in used_exits:
+                raise ValueError(f"Duplicate Exit {exit} not allowed.")
+            used_entrances.append(entrance)
+            used_exits.append(exit)
+            if not cls.validate_entrance_name(entrance):
+                raise ValueError(f"{entrance.title()} is not a valid entrance.")
+            if not cls.validate_exit_name(exit):
+                raise ValueError(f"{exit.title()} is not a valid exit.")
+            if not cls.can_connect(entrance, exit):
+                raise ValueError(f"Connection between {entrance.title()} and {exit.title()} is invalid.")
+
+    @classmethod
+    def from_any(cls, data: PlandoConFromAnyType) -> Self:
+        if not isinstance(data, typing.Iterable):
+            raise Exception(f"Cannot create plando connections from non-List value, got {type(data)}.")
+
+        value: typing.List[PlandoConnection] = []
+        for connection in data:
+            if isinstance(connection, typing.Mapping):
+                percentage = connection.get("percentage", 100)
+                if random.random() < float(percentage / 100):
+                    entrance = connection.get("entrance", None)
+                    if is_iterable_except_str(entrance):
+                        entrance = random.choice(sorted(entrance))
+                    exit = connection.get("exit", None)
+                    if is_iterable_except_str(exit):
+                        exit = random.choice(sorted(exit))
+                    direction = connection.get("direction", "both")
+
+                    if not entrance or not exit:
+                        raise Exception("Plando connection must have an entrance and an exit.")
+                    value.append(PlandoConnection(
+                        entrance,
+                        exit,
+                        direction,
+                        percentage
+                    ))
+            elif isinstance(connection, PlandoConnection):
+                if random.random() < float(connection.percentage / 100):
+                    value.append(connection)
+            else:
+                raise Exception(f"Cannot create connection from non-Dict type, got {type(connection)}.")
+        cls.validate_plando_connections(value)
+        return cls(value)
+
+    def verify(self, world: typing.Type[World], player_name: str, plando_options: "PlandoOptions") -> None:
+        from BaseClasses import PlandoOptions
+        if self.value and not (PlandoOptions.connections & plando_options):
+            # plando is disabled but plando options were given so overwrite the options
+            self.value = []
+            logging.warning(f"The plando connections module is turned off, "
+                            f"so connections for {player_name} will be ignored.")
+
+    @classmethod
+    def get_option_name(cls, value: typing.List[PlandoConnection]) -> str:
+        return ", ".join(["%s %s %s" % (connection.entrance,
+                                        "<=>" if connection.direction == PlandoConnection.Direction.both else
+                                        "<=" if connection.direction == PlandoConnection.Direction.exit else
+                                        "=>",
+                                        connection.exit) for connection in value])
+
+    def __getitem__(self, index: typing.SupportsIndex) -> PlandoConnection:
+        return self.value.__getitem__(index)
+
+    def __iter__(self) -> typing.Iterator[PlandoConnection]:
+        yield from self.value
+
+    def __len__(self) -> int:
+        return len(self.value)
+
+
 class Accessibility(Choice):
     """Set rules for reachability of your items/locations.
     Locations: ensure everything can be reached and acquired.
@@ -909,8 +1139,10 @@ class Accessibility(Choice):
 
 
 class ProgressionBalancing(NamedRange):
-    """A system that can move progression earlier, to try and prevent the player from getting stuck and bored early.
-    A lower setting means more getting stuck. A higher setting means less getting stuck."""
+    """
+    A system that can move progression earlier, to try and prevent the player from getting stuck and bored early.
+    A lower setting means more getting stuck. A higher setting means less getting stuck.
+    """
     default = 50
     range_start = 0
     range_end = 99
@@ -983,7 +1215,7 @@ class LocalItems(ItemSet):
 
 class NonLocalItems(ItemSet):
     """Forces these items to be outside their native world."""
-    display_name = "Not Local Items"
+    display_name = "Non-local Items"
 
 
 class StartInventory(ItemDict):
@@ -1046,7 +1278,8 @@ class ItemLinks(OptionList):
     ])
 
     @staticmethod
-    def verify_items(items: typing.List[str], item_link: str, pool_name: str, world, allow_item_groups: bool = True) -> typing.Set:
+    def verify_items(items: typing.List[str], item_link: str, pool_name: str, world,
+                     allow_item_groups: bool = True) -> typing.Set:
         pool = set()
         for item_name in items:
             if item_name not in world.item_names and (not allow_item_groups or item_name not in world.item_name_groups):
@@ -1129,9 +1362,41 @@ class OptionGroup(typing.NamedTuple):
     """Name of the group to categorize these options in for display on the WebHost and in generated YAMLS."""
     options: typing.List[typing.Type[Option[typing.Any]]]
     """Options to be in the defined group."""
+    start_collapsed: bool = False
+    """Whether the group will start collapsed on the WebHost options pages."""
 
 
-def generate_yaml_templates(target_folder: typing.Union[str, "pathlib.Path"], generate_hidden: bool = True):
+item_and_loc_options = [LocalItems, NonLocalItems, StartInventory, StartInventoryPool, StartHints,
+                        StartLocationHints, ExcludeLocations, PriorityLocations, ItemLinks]
+"""
+Options that are always populated in "Item & Location Options" Option Group. Cannot be moved to another group.
+If desired, a custom "Item & Location Options" Option Group can be defined, but only for adding additional options to
+it.
+"""
+
+
+def get_option_groups(world: typing.Type[World], visibility_level: Visibility = Visibility.template) -> typing.Dict[
+        str, typing.Dict[str, typing.Type[Option[typing.Any]]]]:
+    """Generates and returns a dictionary for the option groups of a specified world."""
+    option_groups = {option: option_group.name
+                     for option_group in world.web.option_groups
+                     for option in option_group.options}
+    # add a default option group for uncategorized options to get thrown into
+    ordered_groups = ["Game Options"]
+    [ordered_groups.append(group) for group in option_groups.values() if group not in ordered_groups]
+    grouped_options = {group: {} for group in ordered_groups}
+    for option_name, option in world.options_dataclass.type_hints.items():
+        if visibility_level & option.visibility:
+            grouped_options[option_groups.get(option, "Game Options")][option_name] = option
+
+    # if the world doesn't have any ungrouped options, this group will be empty so just remove it
+    if not grouped_options["Game Options"]:
+        del grouped_options["Game Options"]
+
+    return grouped_options
+
+
+def generate_yaml_templates(target_folder: typing.Union[str, "pathlib.Path"], generate_hidden: bool = True) -> None:
     import os
 
     import yaml
@@ -1169,17 +1434,7 @@ def generate_yaml_templates(target_folder: typing.Union[str, "pathlib.Path"], ge
 
     for game_name, world in AutoWorldRegister.world_types.items():
         if not world.hidden or generate_hidden:
-
-            option_groups = {option: option_group.name
-                             for option_group in world.web.option_groups
-                             for option in option_group.options}
-            ordered_groups = ["Game Options"]
-            [ordered_groups.append(group) for group in option_groups.values() if group not in ordered_groups]
-            grouped_options = {group: {} for group in ordered_groups}
-            for option_name, option in world.options_dataclass.type_hints.items():
-                if option.visibility >= Visibility.template:
-                    grouped_options[option_groups.get(option, "Game Options")][option_name] = option
-
+            grouped_options = get_option_groups(world)
             with open(local_path("data", "options.yaml")) as f:
                 file_data = f.read()
             res = Template(file_data).render(

README.md

@@ -1,8 +1,10 @@
 # [Archipelago](https://archipelago.gg) ![Discord Shield](https://discordapp.com/api/guilds/731205301247803413/widget.png?style=shield) | [Install](https://github.com/ArchipelagoMW/Archipelago/releases)
 
-Archipelago provides a generic framework for developing multiworld capability for game randomizers. In all cases, presently, Archipelago is also the randomizer itself.
+Archipelago provides a generic framework for developing multiworld capability for game randomizers. In all cases,
+presently, Archipelago is also the randomizer itself.
 
 Currently, the following games are supported:
+
 * The Legend of Zelda: A Link to the Past
 * Factorio
 * Minecraft
@@ -67,7 +69,9 @@ Currently, the following games are supported:
 * Yoshi's Island
 * Mario & Luigi: Superstar Saga
 * Bomb Rush Cyberfunk
+* Aquaria
 * Yu-Gi-Oh! Ultimate Masters: World Championship Tournament 2006
+* A Hat in Time
 
 For setup and instructions check out our [tutorials page](https://archipelago.gg/tutorial/).
 Downloads can be found at [Releases](https://github.com/ArchipelagoMW/Archipelago/releases), including compiled
@@ -75,36 +79,57 @@ windows binaries.
 
 ## History
 
-Archipelago is built upon a strong legacy of brilliant hobbyists. We want to honor that legacy by showing it here. The repositories which Archipelago is built upon, inspired by, or otherwise owes its gratitude to are:
+Archipelago is built upon a strong legacy of brilliant hobbyists. We want to honor that legacy by showing it here.
+The repositories which Archipelago is built upon, inspired by, or otherwise owes its gratitude to are:
 
 * [bonta0's MultiWorld](https://github.com/Bonta0/ALttPEntranceRandomizer/tree/multiworld_31)
 * [AmazingAmpharos' Entrance Randomizer](https://github.com/AmazingAmpharos/ALttPEntranceRandomizer)
 * [VT Web Randomizer](https://github.com/sporchia/alttp_vt_randomizer)
 * [Dessyreqt's alttprandomizer](https://github.com/Dessyreqt/alttprandomizer)
-* [Zarby89's](https://github.com/Ijwu/Enemizer/commits?author=Zarby89) and [sosuke3's](https://github.com/Ijwu/Enemizer/commits?author=sosuke3) contributions to Enemizer, which make the vast majority of Enemizer contributions.
+* [Zarby89's](https://github.com/Ijwu/Enemizer/commits?author=Zarby89)
+  and [sosuke3's](https://github.com/Ijwu/Enemizer/commits?author=sosuke3) contributions to Enemizer, which make up the
+  vast majority of Enemizer contributions.
 
-We recognize that there is a strong community of incredibly smart people that have come before us and helped pave the path. Just because one person's name may be in a repository title does not mean that only one person made that project happen. We can't hope to perfectly cover every single contribution that lead up to Archipelago but we hope to honor them fairly.
+We recognize that there is a strong community of incredibly smart people that have come before us and helped pave the
+path. Just because one person's name may be in a repository title does not mean that only one person made that project
+happen. We can't hope to perfectly cover every single contribution that lead up to Archipelago, but we hope to honor
+them fairly.
 
 ### Path to the Archipelago
-Archipelago was directly forked from bonta0's `multiworld_31` branch of ALttPEntranceRandomizer (this project has a long legacy of its own, please check it out linked above) on January 12, 2020. The repository was then named to _MultiWorld-Utilities_ to better encompass its intended function. As Archipelago matured, then known as "Berserker's MultiWorld" by some, we found it necessary to transform our repository into a root level repository (as opposed to a 'forked repo') and change the name (which came later) to better reflect our project.
+
+Archipelago was directly forked from bonta0's `multiworld_31` branch of ALttPEntranceRandomizer (this project has a
+long legacy of its own, please check it out linked above) on January 12, 2020. The repository was then named to
+_MultiWorld-Utilities_ to better encompass its intended function. As Archipelago matured, then known as
+"Berserker's MultiWorld" by some, we found it necessary to transform our repository into a root level repository
+(as opposed to a 'forked repo') and change the name (which came later) to better reflect our project.
 
 ## Running Archipelago
-For most people, all you need to do is head over to the [releases](https://github.com/ArchipelagoMW/Archipelago/releases) page then download and run the appropriate installer, or AppImage for Linux-based systems.
 
-If you are a developer or are running on a platform with no compiled releases available, please see our doc on [running Archipelago from source](docs/running%20from%20source.md).
+For most people, all you need to do is head over to
+the [releases page](https://github.com/ArchipelagoMW/Archipelago/releases), then download and run the appropriate
+installer, or AppImage for Linux-based systems.
+
+If you are a developer or are running on a platform with no compiled releases available, please see our doc on
+[running Archipelago from source](docs/running%20from%20source.md).
 
 ## Related Repositories
-This project makes use of multiple other projects. We wouldn't be here without these other repositories and the contributions of their developers, past and present.
+
+This project makes use of multiple other projects. We wouldn't be here without these other repositories and the
+contributions of their developers, past and present.
 
 * [z3randomizer](https://github.com/ArchipelagoMW/z3randomizer)
 * [Enemizer](https://github.com/Ijwu/Enemizer)
 * [Ocarina of Time Randomizer](https://github.com/TestRunnerSRL/OoT-Randomizer)
 
 ## Contributing
-For contribution guidelines, please see our [Contributing doc.](/docs/contributing.md)
+
+To contribute to Archipelago, including the WebHost, core program, or by adding a new game, see our
+[Contributing guidelines](/docs/contributing.md).
 
 ## FAQ
-For Frequently asked questions, please see the website's [FAQ Page.](https://archipelago.gg/faq/en/)
+
+For Frequently asked questions, please see the website's [FAQ Page](https://archipelago.gg/faq/en/).
 
 ## Code of Conduct
-Please refer to our [code of conduct.](/docs/code_of_conduct.md)
+
+Please refer to our [code of conduct](/docs/code_of_conduct.md).

UndertaleClient.py

@@ -247,8 +247,8 @@ async def process_undertale_cmd(ctx: UndertaleContext, cmd: str, args: dict):
             with open(os.path.join(ctx.save_game_folder, filename), "w") as f:
                 toDraw = ""
                 for i in range(20):
-                    if i < len(str(ctx.item_names[l.item])):
-                        toDraw += str(ctx.item_names[l.item])[i]
+                    if i < len(str(ctx.item_names.lookup_in_slot(l.item))):
+                        toDraw += str(ctx.item_names.lookup_in_slot(l.item))[i]
                     else:
                         break
                 f.write(toDraw)

Utils.py

@@ -46,7 +46,7 @@ class Version(typing.NamedTuple):
         return ".".join(str(item) for item in self)
 
 
-__version__ = "0.4.6"
+__version__ = "0.5.0"
 version_tuple = tuplize_version(__version__)
 
 is_linux = sys.platform.startswith("linux")
@@ -101,8 +101,7 @@ def cache_self1(function: typing.Callable[[S, T], RetType]) -> typing.Callable[[
 
     @functools.wraps(function)
     def wrap(self: S, arg: T) -> RetType:
-        cache: Optional[Dict[T, RetType]] = typing.cast(Optional[Dict[T, RetType]],
-                                                        getattr(self, cache_name, None))
+        cache: Optional[Dict[T, RetType]] = getattr(self, cache_name, None)
         if cache is None:
             res = function(self, arg)
             setattr(self, cache_name, {arg: res})
@@ -209,10 +208,11 @@ def output_path(*path: str) -> str:
 
 def open_file(filename: typing.Union[str, "pathlib.Path"]) -> None:
     if is_windows:
-        os.startfile(filename)
+        os.startfile(filename)  # type: ignore
     else:
         from shutil import which
         open_command = which("open") if is_macos else (which("xdg-open") or which("gnome-open") or which("kde-open"))
+        assert open_command, "Didn't find program for open_file! Please report this together with system details."
         subprocess.call([open_command, filename])
 
 
@@ -300,21 +300,21 @@ def get_options() -> Settings:
     return get_settings()
 
 
-def persistent_store(category: str, key: typing.Any, value: typing.Any):
+def persistent_store(category: str, key: str, value: typing.Any):
     path = user_path("_persistent_storage.yaml")
-    storage: dict = persistent_load()
-    category = storage.setdefault(category, {})
-    category[key] = value
+    storage = persistent_load()
+    category_dict = storage.setdefault(category, {})
+    category_dict[key] = value
     with open(path, "wt") as f:
         f.write(dump(storage, Dumper=Dumper))
 
 
-def persistent_load() -> typing.Dict[str, dict]:
-    storage = getattr(persistent_load, "storage", None)
+def persistent_load() -> Dict[str, Dict[str, Any]]:
+    storage: Union[Dict[str, Dict[str, Any]], None] = getattr(persistent_load, "storage", None)
     if storage:
         return storage
     path = user_path("_persistent_storage.yaml")
-    storage: dict = {}
+    storage = {}
     if os.path.exists(path):
         try:
             with open(path, "r") as f:
@@ -323,7 +323,7 @@ def persistent_load() -> typing.Dict[str, dict]:
             logging.debug(f"Could not read store: {e}")
     if storage is None:
         storage = {}
-    persistent_load.storage = storage
+    setattr(persistent_load, "storage", storage)
     return storage
 
 
@@ -365,6 +365,7 @@ def store_data_package_for_checksum(game: str, data: typing.Dict[str, Any]) -> N
         except Exception as e:
             logging.debug(f"Could not store data package: {e}")
 
+
 def get_default_adjuster_settings(game_name: str) -> Namespace:
     import LttPAdjuster
     adjuster_settings = Namespace()
@@ -383,7 +384,9 @@ def get_adjuster_settings(game_name: str) -> Namespace:
     default_settings = get_default_adjuster_settings(game_name)
 
     # Fill in any arguments from the argparser that we haven't seen before
-    return Namespace(**vars(adjuster_settings), **{k:v for k,v in vars(default_settings).items() if k not in vars(adjuster_settings)})
+    return Namespace(**vars(adjuster_settings), **{
+        k: v for k, v in vars(default_settings).items() if k not in vars(adjuster_settings)
+    })
 
 
 @cache_argsless
@@ -407,13 +410,13 @@ safe_builtins = frozenset((
 class RestrictedUnpickler(pickle.Unpickler):
     generic_properties_module: Optional[object]
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
         super(RestrictedUnpickler, self).__init__(*args, **kwargs)
         self.options_module = importlib.import_module("Options")
         self.net_utils_module = importlib.import_module("NetUtils")
         self.generic_properties_module = None
 
-    def find_class(self, module, name):
+    def find_class(self, module: str, name: str) -> type:
         if module == "builtins" and name in safe_builtins:
             return getattr(builtins, name)
         # used by MultiServer -> savegame/multidata
@@ -437,7 +440,7 @@ class RestrictedUnpickler(pickle.Unpickler):
         raise pickle.UnpicklingError(f"global '{module}.{name}' is forbidden")
 
 
-def restricted_loads(s):
+def restricted_loads(s: bytes) -> Any:
     """Helper function analogous to pickle.loads()."""
     return RestrictedUnpickler(io.BytesIO(s)).load()
 
@@ -455,6 +458,15 @@ class KeyedDefaultDict(collections.defaultdict):
     """defaultdict variant that uses the missing key as argument to default_factory"""
     default_factory: typing.Callable[[typing.Any], typing.Any]
 
+    def __init__(self,
+                 default_factory: typing.Callable[[Any], Any] = None,
+                 seq: typing.Union[typing.Mapping, typing.Iterable, None] = None,
+                 **kwargs):
+        if seq is not None:
+            super().__init__(default_factory, seq, **kwargs)
+        else:
+            super().__init__(default_factory, **kwargs)
+
     def __missing__(self, key):
         self[key] = value = self.default_factory(key)
         return value
@@ -493,7 +505,7 @@ def init_logging(name: str, loglevel: typing.Union[str, int] = logging.INFO, wri
     file_handler.setFormatter(logging.Formatter(log_format))
 
     class Filter(logging.Filter):
-        def __init__(self, filter_name, condition):
+        def __init__(self, filter_name: str, condition: typing.Callable[[logging.LogRecord], bool]) -> None:
             super().__init__(filter_name)
             self.condition = condition
 
@@ -541,10 +553,11 @@ def init_logging(name: str, loglevel: typing.Union[str, int] = logging.INFO, wri
         f"Archipelago ({__version__}) logging initialized"
         f" on {platform.platform()}"
         f" running Python {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
+        f"{' (frozen)' if is_frozen() else ''}"
     )
 
 
-def stream_input(stream, queue):
+def stream_input(stream: typing.TextIO, queue: "asyncio.Queue[str]"):
     def queuer():
         while 1:
             try:
@@ -572,7 +585,7 @@ class VersionException(Exception):
     pass
 
 
-def chaining_prefix(index: int, labels: typing.Tuple[str]) -> str:
+def chaining_prefix(index: int, labels: typing.Sequence[str]) -> str:
     text = ""
     max_label = len(labels) - 1
     while index > max_label:
@@ -595,7 +608,7 @@ def format_SI_prefix(value, power=1000, power_labels=("", "k", "M", "G", "T", "P
     return f"{value.quantize(decimal.Decimal('1.00'))} {chaining_prefix(n, power_labels)}"
 
 
-def get_fuzzy_results(input_word: str, wordlist: typing.Sequence[str], limit: typing.Optional[int] = None) \
+def get_fuzzy_results(input_word: str, word_list: typing.Collection[str], limit: typing.Optional[int] = None) \
         -> typing.List[typing.Tuple[str, int]]:
     import jellyfish
 
@@ -603,21 +616,55 @@ def get_fuzzy_results(input_word: str, wordlist: typing.Sequence[str], limit: ty
         return (1 - jellyfish.damerau_levenshtein_distance(word1.lower(), word2.lower())
                 / max(len(word1), len(word2)))
 
-    limit: int = limit if limit else len(wordlist)
+    limit = limit if limit else len(word_list)
     return list(
         map(
             lambda container: (container[0], int(container[1]*100)),  # convert up to limit to int %
             sorted(
-                map(lambda candidate:
-                    (candidate,  get_fuzzy_ratio(input_word, candidate)),
-                    wordlist),
+                map(lambda candidate: (candidate, get_fuzzy_ratio(input_word, candidate)), word_list),
                 key=lambda element: element[1],
-                reverse=True)[0:limit]
+                reverse=True
+            )[0:limit]
         )
     )
 
 
-def open_filename(title: str, filetypes: typing.Sequence[typing.Tuple[str, typing.Sequence[str]]], suggest: str = "") \
+def get_intended_text(input_text: str, possible_answers) -> typing.Tuple[str, bool, str]:
+    picks = get_fuzzy_results(input_text, possible_answers, limit=2)
+    if len(picks) > 1:
+        dif = picks[0][1] - picks[1][1]
+        if picks[0][1] == 100:
+            return picks[0][0], True, "Perfect Match"
+        elif picks[0][1] < 75:
+            return picks[0][0], False, f"Didn't find something that closely matches '{input_text}', " \
+                                       f"did you mean '{picks[0][0]}'? ({picks[0][1]}% sure)"
+        elif dif > 5:
+            return picks[0][0], True, "Close Match"
+        else:
+            return picks[0][0], False, f"Too many close matches for '{input_text}', " \
+                                       f"did you mean '{picks[0][0]}'? ({picks[0][1]}% sure)"
+    else:
+        if picks[0][1] > 90:
+            return picks[0][0], True, "Only Option Match"
+        else:
+            return picks[0][0], False, f"Didn't find something that closely matches '{input_text}', " \
+                                       f"did you mean '{picks[0][0]}'? ({picks[0][1]}% sure)"
+
+
+def get_input_text_from_response(text: str, command: str) -> typing.Optional[str]:
+    if "did you mean " in text:
+        for question in ("Didn't find something that closely matches",
+                         "Too many close matches"):
+            if text.startswith(question):
+                name = get_text_between(text, "did you mean '",
+                                        "'? (")
+                return f"!{command} {name}"
+    elif text.startswith("Missing: "):
+        return text.replace("Missing: ", "!hint_location ")
+    return None
+
+
+def open_filename(title: str, filetypes: typing.Iterable[typing.Tuple[str, typing.Iterable[str]]], suggest: str = "") \
         -> typing.Optional[str]:
     logging.info(f"Opening file input dialog for {title}.")
 
@@ -734,7 +781,7 @@ def messagebox(title: str, text: str, error: bool = False) -> None:
         root.update()
 
 
-def title_sorted(data: typing.Sequence, key=None, ignore: typing.Set = frozenset(("a", "the"))):
+def title_sorted(data: typing.Iterable, key=None, ignore: typing.AbstractSet[str] = frozenset(("a", "the"))):
     """Sorts a sequence of text ignoring typical articles like "a" or "the" in the beginning."""
     def sorter(element: Union[str, Dict[str, Any]]) -> str:
         if (not isinstance(element, str)):
@@ -788,7 +835,7 @@ class DeprecateDict(dict):
     log_message: str
     should_error: bool
 
-    def __init__(self, message, error: bool = False) -> None:
+    def __init__(self, message: str, error: bool = False) -> None:
         self.log_message = message
         self.should_error = error
         super().__init__()

WargrooveClient.py

@@ -176,7 +176,7 @@ class WargrooveContext(CommonContext):
                 if not os.path.isfile(path):
                     open(path, 'w').close()
                     # Announcing commander unlocks
-                    item_name = self.item_names[network_item.item]
+                    item_name = self.item_names.lookup_in_slot(network_item.item)
                     if item_name in faction_table.keys():
                         for commander in faction_table[item_name]:
                             logger.info(f"{commander.name} has been unlocked!")
@@ -197,7 +197,7 @@ class WargrooveContext(CommonContext):
                     open(print_path, 'w').close()
                     with open(print_path, 'w') as f:
                         f.write("Received " +
-                                self.item_names[network_item.item] +
+                                self.item_names.lookup_in_slot(network_item.item) +
                                 " from " +
                                 self.player_names[network_item.player])
                         f.close()
@@ -342,7 +342,7 @@ class WargrooveContext(CommonContext):
             faction_items = 0
             faction_item_names = [faction + ' Commanders' for faction in faction_table.keys()]
             for network_item in self.items_received:
-                if self.item_names[network_item.item] in faction_item_names:
+                if self.item_names.lookup_in_slot(network_item.item) in faction_item_names:
                     faction_items += 1
             starting_groove = (faction_items - 1) * self.starting_groove_multiplier
             # Must be an integer larger than 0

WebHost.py

@@ -12,6 +12,9 @@ ModuleUpdate.update()
 import Utils
 import settings
 
+if typing.TYPE_CHECKING:
+    from flask import Flask
+
 Utils.local_path.cached_path = os.path.dirname(__file__) or "."  # py3.8 is not abs. remove "." when dropping 3.8
 settings.no_gui = True
 configpath = os.path.abspath("config.yaml")
@@ -19,7 +22,7 @@ if not os.path.exists(configpath):  # fall back to config.yaml in home
     configpath = os.path.abspath(Utils.user_path('config.yaml'))
 
 
-def get_app():
+def get_app() -> "Flask":
     from WebHostLib import register, cache, app as raw_app
     from WebHostLib.models import db
 
@@ -55,6 +58,7 @@ def create_ordered_tutorials_file() -> typing.List[typing.Dict[str, typing.Any]]
             worlds[game] = world
 
     base_target_path = Utils.local_path("WebHostLib", "static", "generated", "docs")
+    shutil.rmtree(base_target_path, ignore_errors=True)
     for game, world in worlds.items():
         # copy files from world's docs folder to the generated folder
         target_path = os.path.join(base_target_path, game)
@@ -117,7 +121,7 @@ if __name__ == "__main__":
     logging.basicConfig(format='[%(asctime)s] %(message)s', level=logging.INFO)
 
     from WebHostLib.lttpsprites import update_sprites_lttp
-    from WebHostLib.autolauncher import autohost, autogen
+    from WebHostLib.autolauncher import autohost, autogen, stop
     from WebHostLib.options import create as create_options_files
 
     try:
@@ -138,3 +142,11 @@ if __name__ == "__main__":
         else:
             from waitress import serve
             serve(app, port=app.config["PORT"], threads=app.config["WAITRESS_THREADS"])
+    else:
+        from time import sleep
+        try:
+            while True:
+                sleep(1)  # wait for process to be killed
+        except (SystemExit, KeyboardInterrupt):
+            pass
+    stop()  # stop worker threads

WebHostLib/api/__init__.py

@@ -5,7 +5,6 @@ from uuid import UUID
 from flask import Blueprint, abort, url_for
 
 import worlds.Files
-from .. import cache
 from ..models import Room, Seed
 
 api_endpoints = Blueprint('api', __name__, url_prefix="/api")
@@ -49,30 +48,4 @@ def room_info(room: UUID):
     }
 
 
-@api_endpoints.route('/datapackage')
-@cache.cached()
-def get_datapackage():
-    from worlds import network_data_package
-    return network_data_package
-
-
-@api_endpoints.route('/datapackage_version')
-@cache.cached()
-def get_datapackage_versions():
-    from worlds import AutoWorldRegister
-
-    version_package = {game: world.data_version for game, world in AutoWorldRegister.world_types.items()}
-    return version_package
-
-
-@api_endpoints.route('/datapackage_checksum')
-@cache.cached()
-def get_datapackage_checksums():
-    from worlds import network_data_package
-    version_package = {
-        game: game_data["checksum"] for game, game_data in network_data_package["games"].items()
-    }
-    return version_package
-
-
-from . import generate, user  # trigger registration
+from . import generate, user, datapackage  # trigger registration

WebHostLib/api/datapackage.py

@@ -0,0 +1,32 @@
+from flask import abort
+
+from Utils import restricted_loads
+from WebHostLib import cache
+from WebHostLib.models import GameDataPackage
+from . import api_endpoints
+
+
+@api_endpoints.route('/datapackage')
+@cache.cached()
+def get_datapackage():
+    from worlds import network_data_package
+    return network_data_package
+
+
+@api_endpoints.route('/datapackage/<string:checksum>')
+@cache.memoize(timeout=3600)
+def get_datapackage_by_checksum(checksum: str):
+    package = GameDataPackage.get(checksum=checksum)
+    if package:
+        return restricted_loads(package.data)
+    return abort(404)
+
+
+@api_endpoints.route('/datapackage_checksum')
+@cache.cached()
+def get_datapackage_checksums():
+    from worlds import network_data_package
+    version_package = {
+        game: game_data["checksum"] for game, game_data in network_data_package["games"].items()
+    }
+    return version_package

WebHostLib/autolauncher.py

@@ -3,16 +3,26 @@ from __future__ import annotations
 import json
 import logging
 import multiprocessing
-import time
 import typing
-from uuid import UUID
 from datetime import timedelta, datetime
+from threading import Event, Thread
+from uuid import UUID
 
 from pony.orm import db_session, select, commit
 
 from Utils import restricted_loads
 from .locker import Locker, AlreadyRunningException
 
+_stop_event = Event()
+
+
+def stop():
+    """Stops previously launched threads"""
+    global _stop_event
+    stop_event = _stop_event
+    _stop_event = Event()  # new event for new threads
+    stop_event.set()
+
 
 def handle_generation_success(seed_id):
     logging.info(f"Generation finished for seed {seed_id}")
@@ -63,6 +73,7 @@ def cleanup():
 
 def autohost(config: dict):
     def keep_running():
+        stop_event = _stop_event
         try:
             with Locker("autohost"):
                 cleanup()
@@ -72,26 +83,25 @@ def autohost(config: dict):
                     hosters.append(hoster)
                     hoster.start()
 
-                while 1:
-                    time.sleep(0.1)
+                while not stop_event.wait(0.1):
                     with db_session:
                         rooms = select(
                             room for room in Room if
                             room.last_activity >= datetime.utcnow() - timedelta(days=3))
                         for room in rooms:
                             # we have to filter twice, as the per-room timeout can't currently be PonyORM transpiled.
-                            if room.last_activity >= datetime.utcnow() - timedelta(seconds=room.timeout):
+                            if room.last_activity >= datetime.utcnow() - timedelta(seconds=room.timeout + 5):
                                 hosters[room.id.int % len(hosters)].start_room(room.id)
 
         except AlreadyRunningException:
             logging.info("Autohost reports as already running, not starting another.")
 
-    import threading
-    threading.Thread(target=keep_running, name="AP_Autohost").start()
+    Thread(target=keep_running, name="AP_Autohost").start()
 
 
 def autogen(config: dict):
     def keep_running():
+        stop_event = _stop_event
         try:
             with Locker("autogen"):
 
@@ -112,8 +122,7 @@ def autogen(config: dict):
                             commit()
                         select(generation for generation in Generation if generation.state == STATE_ERROR).delete()
 
-                    while 1:
-                        time.sleep(0.1)
+                    while not stop_event.wait(0.1):
                         with db_session:
                             # for update locks the database row(s) during transaction, preventing writes from elsewhere
                             to_start = select(
@@ -124,8 +133,7 @@ def autogen(config: dict):
         except AlreadyRunningException:
             logging.info("Autogen reports as already running, not starting another.")
 
-    import threading
-    threading.Thread(target=keep_running, name="AP_Autogen").start()
+    Thread(target=keep_running, name="AP_Autogen").start()
 
 
 multiworlds: typing.Dict[type(Room.id), MultiworldInstance] = {}

WebHostLib/customserver.py

@@ -74,6 +74,7 @@ class WebHostContext(Context):
 
     def _load_game_data(self):
         for key, value in self.static_server_data.items():
+            # NOTE: attributes are mutable and shared, so they will have to be copied before being modified
             setattr(self, key, value)
         self.non_hintable_names = collections.defaultdict(frozenset, self.non_hintable_names)
 
@@ -101,18 +102,37 @@ class WebHostContext(Context):
 
         multidata = self.decompress(room.seed.multidata)
         game_data_packages = {}
+
+        static_gamespackage = self.gamespackage  # this is shared across all rooms
+        static_item_name_groups = self.item_name_groups
+        static_location_name_groups = self.location_name_groups
+        self.gamespackage = {"Archipelago": static_gamespackage.get("Archipelago", {})}  # this may be modified by _load
+        self.item_name_groups = {"Archipelago": static_item_name_groups.get("Archipelago", {})}
+        self.location_name_groups = {"Archipelago": static_location_name_groups.get("Archipelago", {})}
+
         for game in list(multidata.get("datapackage", {})):
             game_data = multidata["datapackage"][game]
             if "checksum" in game_data:
-                if self.gamespackage.get(game, {}).get("checksum") == game_data["checksum"]:
-                    # non-custom. remove from multidata
+                if static_gamespackage.get(game, {}).get("checksum") == game_data["checksum"]:
+                    # non-custom. remove from multidata and use static data
                     # games package could be dropped from static data once all rooms embed data package
                     del multidata["datapackage"][game]
                 else:
                     row = GameDataPackage.get(checksum=game_data["checksum"])
                     if row:  # None if rolled on >= 0.3.9 but uploaded to <= 0.3.8. multidata should be complete
                         game_data_packages[game] = Utils.restricted_loads(row.data)
+                        continue
+                    else:
+                        self.logger.warning(f"Did not find game_data_package for {game}: {game_data['checksum']}")
+            self.gamespackage[game] = static_gamespackage.get(game, {})
+            self.item_name_groups[game] = static_item_name_groups.get(game, {})
+            self.location_name_groups[game] = static_location_name_groups.get(game, {})
 
+        if not game_data_packages:
+            # all static -> use the static dicts directly
+            self.gamespackage = static_gamespackage
+            self.item_name_groups = static_item_name_groups
+            self.location_name_groups = static_location_name_groups
         return self._load(multidata, game_data_packages, True)
 
     @db_session
@@ -122,7 +142,7 @@ class WebHostContext(Context):
             savegame_data = Room.get(id=self.room_id).multisave
             if savegame_data:
                 self.set_save(restricted_loads(Room.get(id=self.room_id).multisave))
-            self._start_async_saving()
+            self._start_async_saving(atexit_save=False)
         threading.Thread(target=self.listen_to_db_commands, daemon=True).start()
 
     @db_session
@@ -148,17 +168,28 @@ def get_random_port():
 def get_static_server_data() -> dict:
     import worlds
     data = {
-        "non_hintable_names": {},
-        "gamespackage": worlds.network_data_package["games"],
-        "item_name_groups": {world_name: world.item_name_groups for world_name, world in
-                             worlds.AutoWorldRegister.world_types.items()},
-        "location_name_groups": {world_name: world.location_name_groups for world_name, world in
-                                 worlds.AutoWorldRegister.world_types.items()},
+        "non_hintable_names": {
+            world_name: world.hint_blacklist
+            for world_name, world in worlds.AutoWorldRegister.world_types.items()
+        },
+        "gamespackage": {
+            world_name: {
+                key: value
+                for key, value in game_package.items()
+                if key not in ("item_name_groups", "location_name_groups")
+            }
+            for world_name, game_package in worlds.network_data_package["games"].items()
+        },
+        "item_name_groups": {
+            world_name: world.item_name_groups
+            for world_name, world in worlds.AutoWorldRegister.world_types.items()
+        },
+        "location_name_groups": {
+            world_name: world.location_name_groups
+            for world_name, world in worlds.AutoWorldRegister.world_types.items()
+        },
     }
 
-    for world_name, world in worlds.AutoWorldRegister.world_types.items():
-        data["non_hintable_names"][world_name] = world.hint_blacklist
-
     return data
 
 
@@ -212,68 +243,101 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict,
     loop = asyncio.get_event_loop()
 
     async def start_room(room_id):
-        try:
-            logger = set_up_logging(room_id)
-            ctx = WebHostContext(static_server_data, logger)
-            ctx.load(room_id)
-            ctx.init_save()
+        with Locker(f"RoomLocker {room_id}"):
             try:
-                ctx.server = websockets.serve(functools.partial(server, ctx=ctx), ctx.host, ctx.port, ssl=ssl_context)
+                logger = set_up_logging(room_id)
+                ctx = WebHostContext(static_server_data, logger)
+                ctx.load(room_id)
+                ctx.init_save()
+                try:
+                    ctx.server = websockets.serve(
+                        functools.partial(server, ctx=ctx), ctx.host, ctx.port, ssl=ssl_context)
 
-                await ctx.server
-            except OSError:  # likely port in use
-                ctx.server = websockets.serve(functools.partial(server, ctx=ctx), ctx.host, 0, ssl=ssl_context)
+                    await ctx.server
+                except OSError:  # likely port in use
+                    ctx.server = websockets.serve(
+                        functools.partial(server, ctx=ctx), ctx.host, 0, ssl=ssl_context)
 
-                await ctx.server
-            port = 0
-            for wssocket in ctx.server.ws_server.sockets:
-                socketname = wssocket.getsockname()
-                if wssocket.family == socket.AF_INET6:
-                    # Prefer IPv4, as most users seem to not have working ipv6 support
-                    if not port:
+                    await ctx.server
+                port = 0
+                for wssocket in ctx.server.ws_server.sockets:
+                    socketname = wssocket.getsockname()
+                    if wssocket.family == socket.AF_INET6:
+                        # Prefer IPv4, as most users seem to not have working ipv6 support
+                        if not port:
+                            port = socketname[1]
+                    elif wssocket.family == socket.AF_INET:
                         port = socketname[1]
-                elif wssocket.family == socket.AF_INET:
-                    port = socketname[1]
-            if port:
-                ctx.logger.info(f'Hosting game at {host}:{port}')
+                if port:
+                    ctx.logger.info(f'Hosting game at {host}:{port}')
+                    with db_session:
+                        room = Room.get(id=ctx.room_id)
+                        room.last_port = port
+                else:
+                    ctx.logger.exception("Could not determine port. Likely hosting failure.")
                 with db_session:
-                    room = Room.get(id=ctx.room_id)
-                    room.last_port = port
+                    ctx.auto_shutdown = Room.get(id=room_id).timeout
+                if ctx.saving:
+                    setattr(asyncio.current_task(), "save", lambda: ctx._save(True))
+                ctx.shutdown_task = asyncio.create_task(auto_shutdown(ctx, []))
+                await ctx.shutdown_task
+
+            except (KeyboardInterrupt, SystemExit):
+                if ctx.saving:
+                    ctx._save()
+                    setattr(asyncio.current_task(), "save", None)
+            except Exception as e:
+                with db_session:
+                    room = Room.get(id=room_id)
+                    room.last_port = -1
+                logger.exception(e)
+                raise
             else:
-                ctx.logger.exception("Could not determine port. Likely hosting failure.")
-            with db_session:
-                ctx.auto_shutdown = Room.get(id=room_id).timeout
-            ctx.shutdown_task = asyncio.create_task(auto_shutdown(ctx, []))
-            await ctx.shutdown_task
-
-            # ensure auto launch is on the same page in regard to room activity.
-            with db_session:
-                room: Room = Room.get(id=ctx.room_id)
-                room.last_activity = datetime.datetime.utcnow() - datetime.timedelta(seconds=room.timeout + 60)
-
-        except (KeyboardInterrupt, SystemExit):
-            with db_session:
-                room = Room.get(id=room_id)
-                # ensure the Room does not spin up again on its own, minute of safety buffer
-                room.last_activity = datetime.datetime.utcnow() - datetime.timedelta(minutes=1, seconds=room.timeout)
-        except Exception:
-            with db_session:
-                room = Room.get(id=room_id)
-                room.last_port = -1
-                # ensure the Room does not spin up again on its own, minute of safety buffer
-                room.last_activity = datetime.datetime.utcnow() - datetime.timedelta(minutes=1, seconds=room.timeout)
-            raise
-        finally:
-            rooms_shutting_down.put(room_id)
+                if ctx.saving:
+                    ctx._save()
+                    setattr(asyncio.current_task(), "save", None)
+            finally:
+                try:
+                    ctx.save_dirty = False  # make sure the saving thread does not write to DB after final wakeup
+                    ctx.exit_event.set()  # make sure the saving thread stops at some point
+                    # NOTE: async saving should probably be an async task and could be merged with shutdown_task
+                    with (db_session):
+                        # ensure the Room does not spin up again on its own, minute of safety buffer
+                        room = Room.get(id=room_id)
+                        room.last_activity = datetime.datetime.utcnow() - \
+                                             datetime.timedelta(minutes=1, seconds=room.timeout)
+                    logging.info(f"Shutting down room {room_id} on {name}.")
+                finally:
+                    await asyncio.sleep(5)
+                    rooms_shutting_down.put(room_id)
 
     class Starter(threading.Thread):
+        _tasks: typing.List[asyncio.Future]
+
+        def __init__(self):
+            super().__init__()
+            self._tasks = []
+
+        def _done(self, task: asyncio.Future):
+            self._tasks.remove(task)
+            task.result()
+
         def run(self):
             while 1:
                 next_room = rooms_to_run.get(block=True,  timeout=None)
-                asyncio.run_coroutine_threadsafe(start_room(next_room), loop)
+                task = asyncio.run_coroutine_threadsafe(start_room(next_room), loop)
+                self._tasks.append(task)
+                task.add_done_callback(self._done)
                 logging.info(f"Starting room {next_room} on {name}.")
 
     starter = Starter()
     starter.daemon = True
     starter.start()
-    loop.run_forever()
+    try:
+        loop.run_forever()
+    finally:
+        # save all tasks that want to be saved during shutdown
+        for task in asyncio.all_tasks(loop):
+            save: typing.Optional[typing.Callable[[], typing.Any]] = getattr(task, "save", None)
+            if save:
+                save()

WebHostLib/generate.py

@@ -6,7 +6,7 @@ import random
 import tempfile
 import zipfile
 from collections import Counter
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union, Set
 
 from flask import flash, redirect, render_template, request, session, url_for
 from pony.orm import commit, db_session
@@ -16,6 +16,7 @@ from Generate import PlandoOptions, handle_name
 from Main import main as ERmain
 from Utils import __version__
 from WebHostLib import app
+from settings import ServerOptions, GeneratorOptions
 from worlds.alttp.EntranceRandomizer import parse_arguments
 from .check import get_yaml_data, roll_options
 from .models import Generation, STATE_ERROR, STATE_QUEUED, Seed, UUID
@@ -23,25 +24,22 @@ from .upload import upload_zip_to_db
 
 
 def get_meta(options_source: dict, race: bool = False) -> Dict[str, Union[List[str], Dict[str, Any]]]:
-    plando_options = {
-        options_source.get("plando_bosses", ""),
-        options_source.get("plando_items", ""),
-        options_source.get("plando_connections", ""),
-        options_source.get("plando_texts", "")
-    }
-    plando_options -= {""}
+    plando_options: Set[str] = set()
+    for substr in ("bosses", "items", "connections", "texts"):
+        if options_source.get(f"plando_{substr}", substr in GeneratorOptions.plando_options):
+            plando_options.add(substr)
 
     server_options = {
-        "hint_cost": int(options_source.get("hint_cost", 10)),
-        "release_mode": options_source.get("release_mode", "goal"),
-        "remaining_mode": options_source.get("remaining_mode", "disabled"),
-        "collect_mode": options_source.get("collect_mode", "disabled"),
-        "item_cheat": bool(int(options_source.get("item_cheat", 1))),
+        "hint_cost": int(options_source.get("hint_cost", ServerOptions.hint_cost)),
+        "release_mode": options_source.get("release_mode", ServerOptions.release_mode),
+        "remaining_mode": options_source.get("remaining_mode", ServerOptions.remaining_mode),
+        "collect_mode": options_source.get("collect_mode", ServerOptions.collect_mode),
+        "item_cheat": bool(int(options_source.get("item_cheat", not ServerOptions.disable_item_cheat))),
         "server_password": options_source.get("server_password", None),
     }
     generator_options = {
-        "spoiler": int(options_source.get("spoiler", 0)),
-        "race": race
+        "spoiler": int(options_source.get("spoiler", GeneratorOptions.spoiler)),
+        "race": race,
     }
 
     if race:
@@ -70,37 +68,41 @@ def generate(race=False):
                 flash(options)
             else:
                 meta = get_meta(request.form, race)
-                results, gen_options = roll_options(options, set(meta["plando_options"]))
-
-                if any(type(result) == str for result in results.values()):
-                    return render_template("checkResult.html", results=results)
-                elif len(gen_options) > app.config["MAX_ROLL"]:
-                    flash(f"Sorry, generating of multiworlds is limited to {app.config['MAX_ROLL']} players. "
-                          f"If you have a larger group, please generate it yourself and upload it.")
-                elif len(gen_options) >= app.config["JOB_THRESHOLD"]:
-                    gen = Generation(
-                        options=pickle.dumps({name: vars(options) for name, options in gen_options.items()}),
-                        # convert to json compatible
-                        meta=json.dumps(meta),
-                        state=STATE_QUEUED,
-                        owner=session["_id"])
-                    commit()
-
-                    return redirect(url_for("wait_seed", seed=gen.id))
-                else:
-                    try:
-                        seed_id = gen_game({name: vars(options) for name, options in gen_options.items()},
-                                           meta=meta, owner=session["_id"].int)
-                    except BaseException as e:
-                        from .autolauncher import handle_generation_failure
-                        handle_generation_failure(e)
-                        return render_template("seedError.html", seed_error=(e.__class__.__name__ + ": " + str(e)))
-
-                    return redirect(url_for("view_seed", seed=seed_id))
+                return start_generation(options, meta)
 
     return render_template("generate.html", race=race, version=__version__)
 
 
+def start_generation(options: Dict[str, Union[dict, str]], meta: Dict[str, Any]):
+    results, gen_options = roll_options(options, set(meta["plando_options"]))
+
+    if any(type(result) == str for result in results.values()):
+        return render_template("checkResult.html", results=results)
+    elif len(gen_options) > app.config["MAX_ROLL"]:
+        flash(f"Sorry, generating of multiworlds is limited to {app.config['MAX_ROLL']} players. "
+              f"If you have a larger group, please generate it yourself and upload it.")
+    elif len(gen_options) >= app.config["JOB_THRESHOLD"]:
+        gen = Generation(
+            options=pickle.dumps({name: vars(options) for name, options in gen_options.items()}),
+            # convert to json compatible
+            meta=json.dumps(meta),
+            state=STATE_QUEUED,
+            owner=session["_id"])
+        commit()
+
+        return redirect(url_for("wait_seed", seed=gen.id))
+    else:
+        try:
+            seed_id = gen_game({name: vars(options) for name, options in gen_options.items()},
+                               meta=meta, owner=session["_id"].int)
+        except BaseException as e:
+            from .autolauncher import handle_generation_failure
+            handle_generation_failure(e)
+            return render_template("seedError.html", seed_error=(e.__class__.__name__ + ": " + str(e)))
+
+        return redirect(url_for("view_seed", seed=seed_id))
+
+
 def gen_game(gen_options: dict, meta: Optional[Dict[str, Any]] = None, owner=None, sid=None):
     if not meta:
         meta: Dict[str, Any] = {}

WebHostLib/options.py

@@ -1,83 +1,60 @@
 import collections.abc
-import os
-import yaml
-import requests
 import json
-import flask
+import os
+from textwrap import dedent
+from typing import Dict, Union
+
+import yaml
+from flask import redirect, render_template, request, Response
 
 import Options
-from Options import Visibility
-from flask import redirect, render_template, request, Response
-from worlds.AutoWorld import AutoWorldRegister
 from Utils import local_path
-from textwrap import dedent
+from worlds.AutoWorld import AutoWorldRegister
 from . import app, cache
+from .generate import get_meta
 
 
-def create():
+def create() -> None:
     target_folder = local_path("WebHostLib", "static", "generated")
     yaml_folder = os.path.join(target_folder, "configs")
 
     Options.generate_yaml_templates(yaml_folder)
 
 
-def get_world_theme(game_name: str):
+def get_world_theme(game_name: str) -> str:
     if game_name in AutoWorldRegister.world_types:
         return AutoWorldRegister.world_types[game_name].web.theme
     return 'grass'
 
 
-def render_options_page(template: str, world_name: str, is_complex: bool = False):
+def render_options_page(template: str, world_name: str, is_complex: bool = False) -> Union[Response, str]:
     world = AutoWorldRegister.world_types[world_name]
     if world.hidden or world.web.options_page is False:
         return redirect("games")
+    visibility_flag = Options.Visibility.complex_ui if is_complex else Options.Visibility.simple_ui
 
-    option_groups = {option: option_group.name
-                     for option_group in world.web.option_groups
-                     for option in option_group.options}
-    ordered_groups = ["Game Options", *[group.name for group in world.web.option_groups]]
-    grouped_options = {group: {} for group in ordered_groups}
-    for option_name, option in world.options_dataclass.type_hints.items():
-        # Exclude settings from options pages if their visibility is disabled
-        if not is_complex and option.visibility < Visibility.simple_ui:
-            continue
-
-        if is_complex and option.visibility < Visibility.complex_ui:
-            continue
-
-        grouped_options[option_groups.get(option, "Game Options")][option_name] = option
+    start_collapsed = {"Game Options": False}
+    for group in world.web.option_groups:
+        start_collapsed[group.name] = group.start_collapsed
 
     return render_template(
         template,
         world_name=world_name,
         world=world,
-        option_groups=grouped_options,
+        option_groups=Options.get_option_groups(world, visibility_level=visibility_flag),
+        start_collapsed=start_collapsed,
         issubclass=issubclass,
         Options=Options,
         theme=get_world_theme(world_name),
     )
 
 
-def generate_game(player_name: str, formatted_options: dict):
-    payload = {
-        "race": 0,
-        "hint_cost": 10,
-        "forfeit_mode": "auto",
-        "remaining_mode": "disabled",
-        "collect_mode": "goal",
-        "weights": {
-            player_name: formatted_options,
-        },
-    }
-    r = requests.post("https://archipelago.gg/api/generate", json=payload)
-    if 200 <= r.status_code <= 299:
-        response_data = r.json()
-        return redirect(response_data["url"])
-    else:
-        return r.text
+def generate_game(options: Dict[str, Union[dict, str]]) -> Union[Response, str]:
+    from .generate import start_generation
+    return start_generation(options, get_meta({}))
 
 
-def send_yaml(player_name: str, formatted_options: dict):
+def send_yaml(player_name: str, formatted_options: dict) -> Response:
     response = Response(yaml.dump(formatted_options, sort_keys=False))
     response.headers["Content-Type"] = "text/yaml"
     response.headers["Content-Disposition"] = f"attachment; filename={player_name}.yaml"
@@ -85,7 +62,7 @@ def send_yaml(player_name: str, formatted_options: dict):
 
 
 @app.template_filter("dedent")
-def filter_dedent(text: str):
+def filter_dedent(text: str) -> str:
     return dedent(text).strip("\n ")
 
 
@@ -98,10 +75,34 @@ def test_ordered(obj):
 @cache.cached()
 def option_presets(game: str) -> Response:
     world = AutoWorldRegister.world_types[game]
-    presets = {}
 
-    if world.web.options_presets:
-        presets = presets | world.web.options_presets
+    presets = {}
+    for preset_name, preset in world.web.options_presets.items():
+        presets[preset_name] = {}
+        for preset_option_name, preset_option in preset.items():
+            if preset_option == "random":
+                presets[preset_name][preset_option_name] = preset_option
+                continue
+
+            option = world.options_dataclass.type_hints[preset_option_name].from_any(preset_option)
+            if isinstance(option, Options.NamedRange) and isinstance(preset_option, str):
+                assert preset_option in option.special_range_names, \
+                    f"Invalid preset value '{preset_option}' for '{preset_option_name}' in '{preset_name}'. " \
+                    f"Expected {option.special_range_names.keys()} or {option.range_start}-{option.range_end}."
+
+                presets[preset_name][preset_option_name] = option.value
+            elif isinstance(option, (Options.Range, Options.OptionSet, Options.OptionList, Options.ItemDict)):
+                presets[preset_name][preset_option_name] = option.value
+            elif isinstance(preset_option, str):
+                # Ensure the option value is valid for Choice and Toggle options
+                assert option.name_lookup[option.value] == preset_option, \
+                    f"Invalid option value '{preset_option}' for '{preset_option_name}' in preset '{preset_name}'. " \
+                    f"Values must not be resolved to a different option via option.from_text (or an alias)."
+                # Use the name of the option
+                presets[preset_name][preset_option_name] = option.current_key
+            else:
+                # Use the name of the option
+                presets[preset_name][preset_option_name] = option.current_key
 
     class SetEncoder(json.JSONEncoder):
         def default(self, obj):
@@ -111,7 +112,7 @@ def option_presets(game: str) -> Response:
             return json.JSONEncoder.default(self, obj)
 
     json_data = json.dumps(presets, cls=SetEncoder)
-    response = flask.Response(json_data)
+    response = Response(json_data)
     response.headers["Content-Type"] = "application/json"
     return response
 
@@ -169,7 +170,7 @@ def generate_weighted_yaml(game: str):
         }
 
         if intent_generate:
-            return generate_game(player_name, formatted_options)
+            return generate_game({player_name: formatted_options})
 
         else:
             return send_yaml(player_name, formatted_options)
@@ -196,9 +197,9 @@ def generate_yaml(game: str):
             else:
                 options[key] = val
 
-        # Detect and build ItemDict options from their name pattern
         for key, val in options.copy().items():
             key_parts = key.rsplit("||", 2)
+            # Detect and build ItemDict options from their name pattern
             if key_parts[-1] == "qty":
                 if key_parts[0] not in options:
                     options[key_parts[0]] = {}
@@ -206,6 +207,13 @@ def generate_yaml(game: str):
                     options[key_parts[0]][key_parts[1]] = int(val)
                 del options[key]
 
+            # Detect keys which end with -custom, indicating a TextChoice with a possible custom value
+            elif key_parts[-1].endswith("-custom"):
+                if val:
+                    options[key_parts[-1][:-7]] = val
+
+                del options[key]
+
         # Detect random-* keys and set their options accordingly
         for key, val in options.copy().items():
             if key.startswith("random-"):
@@ -243,7 +251,7 @@ def generate_yaml(game: str):
         }
 
         if intent_generate:
-            return generate_game(player_name, formatted_options)
+            return generate_game({player_name: formatted_options})
 
         else:
             return send_yaml(player_name, formatted_options)

WebHostLib/requirements.txt

@@ -1,9 +1,10 @@
-flask>=3.0.0
+flask>=3.0.3
+werkzeug>=3.0.3
 pony>=0.7.17
-waitress>=2.1.2
-Flask-Caching>=2.1.0
-Flask-Compress>=1.14
-Flask-Limiter>=3.5.0
+waitress>=3.0.0
+Flask-Caching>=2.3.0
+Flask-Compress>=1.15
+Flask-Limiter>=3.7.0
 bokeh>=3.1.1; python_version <= '3.8'
-bokeh>=3.3.2; python_version >= '3.9'
-markupsafe>=2.1.3
+bokeh>=3.4.1; python_version >= '3.9'
+markupsafe>=2.1.5

WebHostLib/static/assets/trackerCommon.js

@@ -27,7 +27,7 @@ const adjustTableHeight = () => {
  * @returns {string}
  */
 const secondsToHours = (seconds) => {
-    let hours   = Math.floor(seconds / 3600);
+    let hours = Math.floor(seconds / 3600);
     let minutes = Math.floor((seconds - (hours * 3600)) / 60).toString().padStart(2, '0');
     return `${hours}:${minutes}`;
 };
@@ -38,18 +38,18 @@ window.addEventListener('load', () => {
         info: false,
         dom: "t",
         stateSave: true,
-        stateSaveCallback: function(settings, data) {
+        stateSaveCallback: function (settings, data) {
             delete data.search;
             localStorage.setItem(`DataTables_${settings.sInstance}_/tracker`, JSON.stringify(data));
         },
-        stateLoadCallback: function(settings) {
+        stateLoadCallback: function (settings) {
             return JSON.parse(localStorage.getItem(`DataTables_${settings.sInstance}_/tracker`));
         },
-        footerCallback: function(tfoot, data, start, end, display) {
+        footerCallback: function (tfoot, data, start, end, display) {
             if (tfoot) {
                 const activityData = this.api().column('lastActivity:name').data().toArray().filter(x => !isNaN(x));
                 Array.from(tfoot?.children).find(td => td.classList.contains('last-activity')).innerText =
-                  (activityData.length) ? secondsToHours(Math.min(...activityData)) : 'None';
+                    (activityData.length) ? secondsToHours(Math.min(...activityData)) : 'None';
             }
         },
         columnDefs: [
@@ -123,49 +123,64 @@ window.addEventListener('load', () => {
             event.preventDefault();
         }
     });
-    const tracker = document.getElementById('tracker-wrapper').getAttribute('data-tracker');
-    const target_second = document.getElementById('tracker-wrapper').getAttribute('data-second') + 3;
+    const target_second = parseInt(document.getElementById('tracker-wrapper').getAttribute('data-second')) + 3;
+    console.log("Target second of refresh: " + target_second);
 
-    function getSleepTimeSeconds(){
+    function getSleepTimeSeconds() {
         // -40 % 60 is -40, which is absolutely wrong and should burn
         var sleepSeconds = (((target_second - new Date().getSeconds()) % 60) + 60) % 60;
         return sleepSeconds || 60;
     }
 
+    let update_on_view = false;
     const update = () => {
-        const target = $("<div></div>");
-        console.log("Updating Tracker...");
-        target.load(location.href, function (response, status) {
-            if (status === "success") {
-                target.find(".table").each(function (i, new_table) {
-                    const new_trs = $(new_table).find("tbody>tr");
-                    const footer_tr = $(new_table).find("tfoot>tr");
-                    const old_table = tables.eq(i);
-                    const topscroll = $(old_table.settings()[0].nScrollBody).scrollTop();
-                    const leftscroll = $(old_table.settings()[0].nScrollBody).scrollLeft();
-                    old_table.clear();
-                    if (footer_tr.length) {
-                        $(old_table.table).find("tfoot").html(footer_tr);
-                    }
-                    old_table.rows.add(new_trs);
-                    old_table.draw();
-                    $(old_table.settings()[0].nScrollBody).scrollTop(topscroll);
-                    $(old_table.settings()[0].nScrollBody).scrollLeft(leftscroll);
-                });
-                $("#multi-stream-link").replaceWith(target.find("#multi-stream-link"));
-            } else {
-                console.log("Failed to connect to Server, in order to update Table Data.");
-                console.log(response);
-            }
-        })
-        setTimeout(update, getSleepTimeSeconds()*1000);
+        if (document.hidden) {
+            console.log("Document reporting as not visible, not updating Tracker...");
+            update_on_view = true;
+        } else {
+            update_on_view = false;
+            const target = $("<div></div>");
+            console.log("Updating Tracker...");
+            target.load(location.href, function (response, status) {
+                if (status === "success") {
+                    target.find(".table").each(function (i, new_table) {
+                        const new_trs = $(new_table).find("tbody>tr");
+                        const footer_tr = $(new_table).find("tfoot>tr");
+                        const old_table = tables.eq(i);
+                        const topscroll = $(old_table.settings()[0].nScrollBody).scrollTop();
+                        const leftscroll = $(old_table.settings()[0].nScrollBody).scrollLeft();
+                        old_table.clear();
+                        if (footer_tr.length) {
+                            $(old_table.table).find("tfoot").html(footer_tr);
+                        }
+                        old_table.rows.add(new_trs);
+                        old_table.draw();
+                        $(old_table.settings()[0].nScrollBody).scrollTop(topscroll);
+                        $(old_table.settings()[0].nScrollBody).scrollLeft(leftscroll);
+                    });
+                    $("#multi-stream-link").replaceWith(target.find("#multi-stream-link"));
+                } else {
+                    console.log("Failed to connect to Server, in order to update Table Data.");
+                    console.log(response);
+                }
+            })
+        }
+        updater = setTimeout(update, getSleepTimeSeconds() * 1000);
     }
-    setTimeout(update, getSleepTimeSeconds()*1000);
+    let updater = setTimeout(update, getSleepTimeSeconds() * 1000);
 
     window.addEventListener('resize', () => {
         adjustTableHeight();
         tables.draw();
     });
 
+    window.addEventListener('visibilitychange', () => {
+        if (!document.hidden && update_on_view) {
+            console.log("Page became visible, tracker should be refreshed.");
+            clearTimeout(updater);
+            update();
+        }
+    });
+
     adjustTableHeight();
 });

WebHostLib/static/styles/playerOptions/playerOptions.css

@@ -15,7 +15,7 @@ html {
   border-radius: 8px;
   padding: 1rem;
   color: #eeffeb;
-  word-break: break-all;
+  word-break: break-word;
 }
 #player-options #player-options-header h1 {
   margin-bottom: 0;

WebHostLib/static/styles/playerOptions/playerOptions.scss

@@ -16,7 +16,7 @@ html{
     border-radius: 8px;
     padding: 1rem;
     color: #eeffeb;
-    word-break: break-all;
+    word-break: break-word;
 
     #player-options-header{
         h1{

WebHostLib/templates/hostRoom.html

@@ -24,7 +24,8 @@
             <br />
         {% endif %}
         {% if room.tracker %}
-            This room has a <a href="{{ url_for("get_multiworld_tracker", tracker=room.tracker) }}">Multiworld Tracker</a> enabled.
+            This room has a <a href="{{ url_for("get_multiworld_tracker", tracker=room.tracker) }}">Multiworld Tracker</a>
+            and a <a href="{{ url_for("get_multiworld_sphere_tracker", tracker=room.tracker) }}">Sphere Tracker</a> enabled.
             <br />
         {% endif %}
         The server for this room will be paused after {{ room.timeout//60//60 }} hours of inactivity.

WebHostLib/templates/multispheretracker.html

@@ -0,0 +1,72 @@
+{% extends "tablepage.html" %}
+{% block head %}
+    {{ super() }}
+    <title>Multiworld Sphere Tracker</title>
+    <link rel="stylesheet" type="text/css" href="{{ url_for("static", filename="styles/tracker.css") }}" />
+    <script type="application/ecmascript" src="{{ url_for("static", filename="assets/trackerCommon.js") }}"></script>
+{% endblock %}
+
+{% block body %}
+    {% include "header/dirtHeader.html" %}
+
+    <div id="tracker-wrapper" data-tracker="{{ room.tracker | suuid }}">
+        <div id="tracker-header-bar">
+            <input placeholder="Search" id="search" />
+
+            <div class="info">
+                {% if tracker_data.get_spheres() %}
+                This tracker lists already found locations by their logical access sphere.
+                It ignores items that cannot be sent
+                and will therefore differ from the sphere numbers in the spoiler playthrough.
+                This tracker will automatically update itself periodically.
+                {% else %}
+                This Multiworld has no Sphere data, likely due to being too old, cannot display data.
+                {% endif %}
+            </div>
+        </div>
+
+        <div id="tables-container">
+        {%- for team, players in tracker_data.get_all_players().items() %}
+            <div class="table-wrapper">
+                <table id="checks-table" class="table non-unique-item-table">
+                    <thead>
+                        <tr>
+                            <th>Sphere</th>
+                            {#- Mimicking hint table header for familiarity. #}
+                            <th>Finder</th>
+                            <th>Receiver</th>
+                            <th>Item</th>
+                            <th>Location</th>
+                            <th>Game</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                    {%- for sphere in tracker_data.get_spheres() %}
+                    {%- set current_sphere = loop.index %}
+                    {%- for player, sphere_location_ids in sphere.items() %}
+                        {%- set checked_locations = tracker_data.get_player_checked_locations(team, player) %}
+                        {%- set finder_game = tracker_data.get_player_game(team, player) %}
+                        {%- set player_location_data = tracker_data.get_player_locations(team, player) %}
+                        {%- for location_id in sphere_location_ids.intersection(checked_locations) %}
+                        <tr>
+                            {%- set item_id, receiver, item_flags = player_location_data[location_id] %}
+                            {%- set receiver_game = tracker_data.get_player_game(team, receiver) %}
+                            <td>{{ current_sphere }}</td>
+                            <td>{{ tracker_data.get_player_name(team, player) }}</td>
+                            <td>{{ tracker_data.get_player_name(team, receiver) }}</td>
+                            <td>{{ tracker_data.item_id_to_name[receiver_game][item_id] }}</td>
+                            <td>{{ tracker_data.location_id_to_name[finder_game][location_id] }}</td>
+                            <td>{{ finder_game }}</td>
+                        </tr>
+                        {%- endfor %}
+
+                    {%- endfor %}
+                    {%- endfor %}
+                    </tbody>
+                </table>
+            </div>
+
+        {%- endfor -%}
+        </div>
+    </div>
+{% endblock %}

WebHostLib/templates/multitracker.html

@@ -10,7 +10,7 @@
     {% include "header/dirtHeader.html" %}
     {% include "multitrackerNavigation.html" %}
 
-    <div id="tracker-wrapper" data-tracker="{{ room.tracker | suuid }}">
+    <div id="tracker-wrapper" data-tracker="{{ room.tracker | suuid }}" data-second="{{ saving_second }}">
         <div id="tracker-header-bar">
             <input placeholder="Search" id="search" />
 

WebHostLib/templates/ootTracker.html

@@ -1,180 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <title>{{ player_name }}&apos;s Tracker</title>
-    <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='styles/ootTracker.css') }}"/>
-    <script type="application/ecmascript" src="{{ url_for('static', filename='assets/ootTracker.js') }}"></script>
-</head>
-
-<body>
-    <div id="player-tracker-wrapper" data-tracker="{{ room.tracker|suuid }}">
-        <table id="inventory-table">
-            <tr>
-                <td><img src="{{ ocarina_url }}" class="{{ 'acquired' if 'Ocarina' in acquired_items }}" title="Ocarina" /></td>
-                <td><img src="{{ icons['Bombs'] }}" class="{{ 'acquired' if 'Bomb Bag' in acquired_items }}" title="Bombs" /></td>
-                <td><img src="{{ icons['Bow'] }}" class="{{ 'acquired' if 'Bow' in acquired_items }}" title="Fairy Bow" /></td>
-                <td><img src="{{ icons['Fire Arrows'] }}" class="{{ 'acquired' if 'Fire Arrows' in acquired_items }}" title="Fire Arrows" /></td>
-                <td><img src="{{ icons['Kokiri Sword'] }}" class="{{ 'acquired' if 'Kokiri Sword' in acquired_items }}" title="Kokiri Sword" /></td>
-                <td><img src="{{ icons['Biggoron Sword'] }}" class="{{ 'acquired' if 'Biggoron Sword' in acquired_items }}" title="Biggoron's Sword" /></td>
-                <td><img src="{{ icons['Mirror Shield'] }}" class="{{ 'acquired' if 'Mirror Shield' in acquired_items }}" title="Mirror Shield" /></td>
-            </tr>
-            <tr>
-                <td><img src="{{ icons['Slingshot'] }}" class="{{ 'acquired' if 'Slingshot' in acquired_items }}" title="Slingshot" /></td>
-                <td><img src="{{ icons['Bombchus'] }}" class="{{ 'acquired' if has_bombchus }}" title="Bombchus" /></td>
-                <td>
-                    <div class="counted-item">
-                        <img src="{{ hookshot_url }}" class="{{ 'acquired' if 'Progressive Hookshot' in acquired_items }}" title="Progressive Hookshot" />
-                        <div class="item-count">{{ hookshot_length }}</div>
-                    </div>
-                </td>
-                <td><img src="{{ icons['Ice Arrows'] }}" class="{{ 'acquired' if 'Ice Arrows' in acquired_items }}" title="Ice Arrows" /></td>
-                <td><img src="{{ strength_upgrade_url }}" class="{{ 'acquired' if 'Progressive Strength Upgrade' in acquired_items }}" title="Progressive Strength Upgrade" /></td>
-                <td><img src="{{ icons['Goron Tunic'] }}" class="{{ 'acquired' if 'Goron Tunic' in acquired_items }}" title="Goron Tunic" /></td>
-                <td><img src="{{ icons['Zora Tunic'] }}" class="{{ 'acquired' if 'Zora Tunic' in acquired_items }}" title="Zora Tunic" /></td>
-            </tr>
-            <tr>
-                <td><img src="{{ icons['Boomerang'] }}" class="{{ 'acquired' if 'Boomerang' in acquired_items }}" title="Boomerang" /></td>
-                <td><img src="{{ icons['Lens of Truth'] }}" class="{{ 'acquired' if 'Lens of Truth' in acquired_items }}" title="Lens of Truth" /></td>
-                <td><img src="{{ icons['Megaton Hammer'] }}" class="{{ 'acquired' if 'Megaton Hammer' in acquired_items }}" title="Megaton Hammer" /></td>
-                <td><img src="{{ icons['Light Arrows'] }}" class="{{ 'acquired' if 'Light Arrows' in acquired_items }}" title="Light Arrows" /></td>
-                <td><img src="{{ scale_url }}" class="{{ 'acquired' if 'Progressive Scale' in acquired_items }}" title="Progressive Scale" /></td>
-                <td><img src="{{ icons['Iron Boots'] }}" class="{{ 'acquired' if 'Iron Boots' in acquired_items }}" title="Iron Boots" /></td>
-                <td><img src="{{ icons['Hover Boots'] }}" class="{{ 'acquired' if 'Hover Boots' in acquired_items }}" title="Hover Boots" /></td>
-            </tr>
-            <tr>
-                <td>
-                    <div class="counted-item">
-                        <img src="{{ bottle_url }}" class="{{ 'acquired' if bottle_count > 0 }}" title="Bottles" />
-                        <div class="item-count">{{ bottle_count if bottle_count > 0 else '' }}</div>
-                    </div>
-                </td>
-                <td><img src="{{ icons['Dins Fire'] }}" class="{{ 'acquired' if 'Dins Fire' in acquired_items }}" title="Din's Fire" /></td>
-                <td><img src="{{ icons['Farores Wind'] }}" class="{{ 'acquired' if 'Farores Wind' in acquired_items }}" title="Farore's Wind" /></td>
-                <td><img src="{{ icons['Nayrus Love'] }}" class="{{ 'acquired' if 'Nayrus Love' in acquired_items }}" title="Nayru's Love" /></td>
-                <td>
-                    <div class="counted-item">
-                        <img src="{{ wallet_url }}" class="{{ 'acquired' if 'Progressive Wallet' in acquired_items }}" title="Progressive Wallet" />
-                        <div class="item-count">{{ wallet_size }}</div>
-                    </div>
-                </td>
-                <td><img src="{{ magic_meter_url }}" class="{{ 'acquired' if 'Magic Meter' in acquired_items }}" title="Magic Meter" /></td>
-                <td><img src="{{ icons['Gerudo Membership Card'] }}" class="{{ 'acquired' if 'Gerudo Membership Card' in acquired_items }}" title="Gerudo Membership Card" /></td>
-            </tr>
-            <tr>
-                <td>
-                    <div class="counted-item">
-                        <img src="{{ icons['Zeldas Lullaby'] }}" class="{{ 'acquired' if 'Zeldas Lullaby' in acquired_items }}" title="Zelda's Lullaby" id="lullaby"/>
-                        <div class="item-count">Zelda</div>
-                    </div>
-                </td>
-                <td>
-                    <div class="counted-item">
-                        <img src="{{ icons['Eponas Song'] }}" class="{{ 'acquired' if 'Eponas Song' in acquired_items }}" title="Epona's Song" id="epona" />
-                        <div class="item-count">Epona</div>
-                    </div>
-                </td>
-                <td>
-                    <div class="counted-item">
-                        <img src="{{ icons['Sarias Song'] }}" class="{{ 'acquired' if 'Sarias Song' in acquired_items }}" title="Saria's Song" id="saria"/>
-                        <div class="item-count">Saria</div>
-                    </div>
-                </td>
-                <td>
-                    <div class="counted-item">
-                        <img src="{{ icons['Suns Song'] }}" class="{{ 'acquired' if 'Suns Song' in acquired_items }}" title="Sun's Song" id="sun"/>
-                        <div class="item-count">Sun</div>
-                    </div>
-                </td>
-                <td>
-                    <div class="counted-item">
-                        <img src="{{ icons['Song of Time'] }}" class="{{ 'acquired' if 'Song of Time' in acquired_items }}" title="Song of Time" id="time"/>
-                        <div class="item-count">Time</div>
-                    </div>
-                </td>
-                <td>
-                    <div class="counted-item">
-                        <img src="{{ icons['Song of Storms'] }}" class="{{ 'acquired' if 'Song of Storms' in acquired_items }}" title="Song of Storms" />
-                        <div class="item-count">Storms</div>
-                    </div>
-                </td>
-                <td>
-                    <div class="counted-item">
-                        <img src="{{ icons['Gold Skulltula Token'] }}" class="{{ 'acquired' if token_count > 0 }}" title="Gold Skulltula Tokens" />
-                        <div class="item-count">{{ token_count }}</div>
-                    </div>
-                </td>
-            </tr>
-            <tr>
-                <td>
-                    <div class="counted-item">
-                        <img src="{{ icons['Minuet of Forest'] }}" class="{{ 'acquired' if 'Minuet of Forest' in acquired_items }}" title="Minuet of Forest" />
-                        <div class="item-count">Min</div>
-                    </div>
-                </td>
-                <td>
-                    <div class="counted-item">
-                        <img src="{{ icons['Bolero of Fire'] }}" class="{{ 'acquired' if 'Bolero of Fire' in acquired_items }}" title="Bolero of Fire" />
-                        <div class="item-count">Bol</div>
-                    </div>
-                </td>
-                <td>
-                    <div class="counted-item">
-                        <img src="{{ icons['Serenade of Water'] }}" class="{{ 'acquired' if 'Serenade of Water' in acquired_items }}" title="Serenade of Water" />
-                        <div class="item-count">Ser</div>
-                    </div>
-                </td>
-                <td>
-                    <div class="counted-item">
-                        <img src="{{ icons['Requiem of Spirit'] }}" class="{{ 'acquired' if 'Requiem of Spirit' in acquired_items }}" title="Requiem of Spirit" />
-                        <div class="item-count">Req</div>
-                    </div>
-                </td>
-                <td>
-                    <div class="counted-item">
-                        <img src="{{ icons['Nocturne of Shadow'] }}" class="{{ 'acquired' if 'Nocturne of Shadow' in acquired_items }}" title="Nocturne of Shadow" />
-                        <div class="item-count">Noc</div>
-                    </div>
-                </td>
-                <td>
-                    <div class="counted-item">
-                        <img src="{{ icons['Prelude of Light'] }}" class="{{ 'acquired' if 'Prelude of Light' in acquired_items }}" title="Prelude of Light" />
-                        <div class="item-count">Pre</div>
-                    </div>
-                </td>
-                <td>
-                    <div class="counted-item">
-                        <img src="{{ icons['Triforce'] if game_finished else icons['Triforce Piece'] }}" class="{{ 'acquired' if game_finished or piece_count > 0 }}" title="{{ 'Triforce' if game_finished else 'Triforce Pieces' }}" id=triforce />
-                        <div class="item-count">{{ piece_count if piece_count > 0 else '' }}</div>
-                    </div>
-                </td>
-            </tr>
-        </table>
-        <table id="location-table">
-            <tr>
-                <td></td>
-                <td><img src="{{ icons['Small Key'] }}" title="Small Keys" /></td>
-                <td><img src="{{ icons['Boss Key'] }}" title="Boss Key" /></td>
-                <td class="right-align">Items</td>
-            </tr>
-            {% for area in checks_done %}
-            <tr class="location-category" id="{{area}}-header">
-                <td>{{ area }} {{'▼' if area != 'Total'}}</td>
-                <td class="smallkeys">{{ small_key_counts.get(area, '-') }}</td>
-                <td class="bosskeys">{{ boss_key_counts.get(area, '-') }}</td>
-                <td class="counter">{{ checks_done[area] }} / {{ checks_in_area[area] }}</td>
-            </tr>
-            <tbody class="locations hide" id="{{area}}">
-                {% for location in location_info[area] %}
-                <tr>
-                    <td class="location-name">{{ location }}</td>
-                    <td></td>
-                    <td></td>
-                    <td class="counter">{{ '✔' if location_info[area][location] else '' }}</td>
-                </tr>
-                {% endfor %}
-            </tbody>
-            {% endfor %}
-        </table>
-    </div>
-</body>
-</html>

WebHostLib/templates/playerOptions/macros.html

@@ -57,9 +57,9 @@
         <select id="{{ option_name }}-select" data-option-name="{{ option_name }}" {{ "disabled" if option.default == "random" }}>
             {% for key, val in option.special_range_names.items() %}
                 {% if option.default == val %}
-                    <option value="{{ val }}" selected>{{ key }} ({{ val }})</option>
+                    <option value="{{ val }}" selected>{{ key|replace("_", " ")|title }} ({{ val }})</option>
                 {% else %}
-                    <option value="{{ val }}">{{ key }} ({{ val }})</option>
+                    <option value="{{ val }}">{{ key|replace("_", " ")|title }} ({{ val }})</option>
                 {% endif %}
             {% endfor %}
             <option value="custom" hidden>Custom</option>
@@ -114,7 +114,7 @@
 {% macro ItemDict(option_name, option, world) %}
     {{ OptionTitle(option_name, option) }}
     <div class="option-container">
-        {% for item_name in world.item_names|sort %}
+        {% for item_name in (option.valid_keys|sort if (option.valid_keys|length > 0) else world.item_names|sort) %}
             <div class="option-entry">
                 <label for="{{ option_name }}-{{ item_name }}-qty">{{ item_name }}</label>
                 <input type="number" id="{{ option_name }}-{{ item_name }}-qty" name="{{ option_name }}||{{ item_name }}||qty" value="{{ option.default[item_name]|default("0") }}" data-option-name="{{ option_name }}" data-item-name="{{ item_name }}" />
@@ -141,7 +141,7 @@
         {% for group_name in world.location_name_groups.keys()|sort %}
             {% if group_name != "Everywhere" %}
                 <div class="option-entry">
-                    <input type="checkbox" id="{{ option_name }}-{{ group_name }}" name="{{ option_name }}" value="{{ group_name }}" {{ "checked" if grop_name in option.default }} />
+                    <input type="checkbox" id="{{ option_name }}-{{ group_name }}" name="{{ option_name }}" value="{{ group_name }}" {{ "checked" if group_name in option.default }} />
                     <label for="{{ option_name }}-{{ group_name }}">{{ group_name }}</label>
                 </div>
             {% endif %}
@@ -149,7 +149,7 @@
         {% if world.location_name_groups.keys()|length > 1 %}
             <div class="option-divider">&nbsp;</div>
         {% endif %}
-        {% for location_name in world.location_names|sort %}
+        {% for location_name in (option.valid_keys|sort if (option.valid_keys|length > 0) else world.location_names|sort) %}
             <div class="option-entry">
                 <input type="checkbox" id="{{ option_name }}-{{ location_name }}" name="{{ option_name }}" value="{{ location_name }}" {{ "checked" if location_name in option.default }} />
                 <label for="{{ option_name }}-{{ location_name }}">{{ location_name }}</label>
@@ -172,7 +172,7 @@
         {% if world.item_name_groups.keys()|length > 1 %}
             <div class="option-divider">&nbsp;</div>
         {% endif %}
-        {% for item_name in world.item_names|sort %}
+        {% for item_name in (option.valid_keys|sort if (option.valid_keys|length > 0) else world.item_names|sort) %}
             <div class="option-entry">
                 <input type="checkbox" id="{{ option_name }}-{{ item_name }}" name="{{ option_name }}" value="{{ item_name }}" {{ "checked" if item_name in option.default }} />
                 <label for="{{ option_name }}-{{ item_name }}">{{ item_name }}</label>

WebHostLib/templates/playerOptions/playerOptions.html

@@ -69,7 +69,7 @@
 
             <div id="option-groups">
                 {% for group_name, group_options in option_groups.items() %}
-                    <details class="group-container" {% if loop.index == 1 %}open{% endif %}>
+                    <details class="group-container" {% if not start_collapsed[group_name] %}open{% endif %}>
                         <summary class="h2">{{ group_name }}</summary>
                         <div class="game-options">
                             <div class="left">

WebHostLib/templates/weightedOptions/macros.html

@@ -1,9 +1,9 @@
 {% macro Toggle(option_name, option) %}
     <table>
         <tbody>
-            {{ RangeRow(option_name, option, "No", "false") }}
-            {{ RangeRow(option_name, option, "Yes", "true") }}
-            {{ RandomRows(option_name, option) }}
+            {{ RangeRow(option_name, option, "No", "false", False, "true" if option.default else "false") }}
+            {{ RangeRow(option_name, option, "Yes", "true", False, "true" if option.default else "false") }}
+            {{ RandomRow(option_name, option) }}
         </tbody>
     </table>
 {% endmacro %}
@@ -18,10 +18,14 @@
         <tbody>
             {% for id, name in option.name_lookup.items() %}
                 {% if name != 'random' %}
-                    {{ RangeRow(option_name, option, option.get_option_name(id), name) }}
+                    {% if option.default != 'random' %}
+                        {{ RangeRow(option_name, option, option.get_option_name(id), name, False, name if option.get_option_name(option.default)|lower == name else None) }}
+                    {% else %}
+                        {{ RangeRow(option_name, option, option.get_option_name(id), name) }}
+                    {% endif %}
                 {% endif %}
             {% endfor %}
-            {{ RandomRows(option_name, option) }}
+            {{ RandomRow(option_name, option) }}
         </tbody>
     </table>
 {% endmacro %}
@@ -34,10 +38,10 @@
         Normal range: {{ option.range_start }} - {{ option.range_end }}
         {% if option.special_range_names %}
             <br /><br />
-            The following values has special meaning, and may fall outside the normal range.
+            The following values have special meanings, and may fall outside the normal range.
             <ul>
                 {% for name, value in option.special_range_names.items() %}
-                    <li>{{ value }}: {{ name }}</li>
+                    <li>{{ value }}: {{ name|replace("_", " ")|title }}</li>
                 {% endfor %}
             </ul>
         {% endif %}
@@ -72,7 +76,9 @@
         </div>
         <table>
             <tbody>
-                <!-- This table to be filled by JS -->
+                {% if option.default %}
+                    {{ RangeRow(option_name, option, option.default, option.default) }}
+                {% endif %}
             </tbody>
         </table>
     </div>
@@ -90,10 +96,14 @@
         <tbody>
             {% for id, name in option.name_lookup.items() %}
                 {% if name != 'random' %}
-                    {{ RangeRow(option_name, option, option.get_option_name(id), name) }}
+                    {% if option.default != 'random' %}
+                        {{ RangeRow(option_name, option, option.get_option_name(id), name, False, name if option.get_option_name(option.default)|lower == name else None) }}
+                    {% else %}
+                        {{ RangeRow(option_name, option, option.get_option_name(id), name) }}
+                    {% endif %}
                 {% endif %}
             {% endfor %}
-            {{ RandomRows(option_name, option) }}
+            {{ RandomRow(option_name, option) }}
         </tbody>
     </table>
 {% endmacro %}
@@ -105,14 +115,14 @@
 
 {% macro ItemDict(option_name, option, world) %}
     <div class="dict-container">
-        {% for item_name in world.item_names|sort %}
+        {% for item_name in (option.valid_keys|sort if (option.valid_keys|length > 0) else world.item_names|sort) %}
             <div class="dict-entry">
                 <label for="{{ option_name }}-{{ item_name }}-qty">{{ item_name }}</label>
                 <input
                     type="number"
                     id="{{ option_name }}-{{ item_name }}-qty"
                     name="{{ option_name }}||{{ item_name }}"
-                    value="0"
+                    value="{{ option.default[item_name] if item_name in option.default else "0" }}"
                 />
             </div>
         {% endfor %}
@@ -121,13 +131,14 @@
 
 {% macro OptionList(option_name, option) %}
     <div class="list-container">
-        {% for key in option.valid_keys|sort %}
+        {% for key in (option.valid_keys if option.valid_keys is ordered else option.valid_keys|sort) %}
             <div class="list-entry">
                 <input
                     type="checkbox"
                     id="{{ option_name }}-{{ key }}"
                     name="{{ option_name }}||{{ key }}"
                     value="1"
+                    checked="{{ "checked" if key in option.default else "" }}"
                 />
                 <label for="{{ option_name }}-{{ key }}">
                     {{ key }}
@@ -142,7 +153,7 @@
         {% for group_name in world.location_name_groups.keys()|sort %}
             {% if group_name != "Everywhere" %}
                 <div class="set-entry">
-                    <input type="checkbox" id="{{ option_name }}-{{ group_name }}" name="{{ option_name }}||{{ group_name }}" value="1" {{ "checked" if grop_name in option.default }} />
+                    <input type="checkbox" id="{{ option_name }}-{{ group_name }}" name="{{ option_name }}||{{ group_name }}" value="1" {{ "checked" if group_name in option.default }} />
                     <label for="{{ option_name }}-{{ group_name }}">{{ group_name }}</label>
                 </div>
             {% endif %}
@@ -150,7 +161,7 @@
         {% if world.location_name_groups.keys()|length > 1 %}
             <div class="divider">&nbsp;</div>
         {% endif %}
-        {% for location_name in world.location_names|sort %}
+        {% for location_name in (option.valid_keys|sort if (option.valid_keys|length > 0) else world.location_names|sort) %}
             <div class="set-entry">
                 <input type="checkbox" id="{{ option_name }}-{{ location_name }}" name="{{ option_name }}||{{ location_name }}" value="1" {{ "checked" if location_name in option.default }} />
                 <label for="{{ option_name }}-{{ location_name }}">{{ location_name }}</label>
@@ -172,7 +183,7 @@
         {% if world.item_name_groups.keys()|length > 1 %}
             <div class="set-divider">&nbsp;</div>
         {% endif %}
-        {% for item_name in world.item_names|sort %}
+        {% for item_name in (option.valid_keys|sort if (option.valid_keys|length > 0) else world.item_names|sort) %}
             <div class="set-entry">
                 <input type="checkbox" id="{{ option_name }}-{{ item_name }}" name="{{ option_name }}||{{ item_name }}" value="1" {{ "checked" if item_name in option.default }} />
                 <label for="{{ option_name }}-{{ item_name }}">{{ item_name }}</label>
@@ -183,7 +194,7 @@
 
 {% macro OptionSet(option_name, option) %}
     <div class="set-container">
-        {% for key in option.valid_keys|sort %}
+        {% for key in (option.valid_keys if option.valid_keys is ordered else option.valid_keys|sort) %}
             <div class="set-entry">
                 <input type="checkbox" id="{{ option_name }}-{{ key }}" name="{{ option_name }}||{{ key }}" value="1" {{ "checked" if key in option.default }} />
                 <label for="{{ option_name }}-{{ key }}">{{ key }}</label>
@@ -200,13 +211,17 @@
     </td>
 {% endmacro %}
 
+{% macro RandomRow(option_name, option, extra_column=False) %}
+    {{ RangeRow(option_name, option, "Random", "random") }}
+{% endmacro %}
+
 {% macro RandomRows(option_name, option, extra_column=False) %}
     {% for key, value in {"Random": "random", "Random (Low)": "random-low", "Random (Middle)": "random-middle", "Random (High)": "random-high"}.items() %}
         {{ RangeRow(option_name, option, key, value) }}
     {% endfor %}
 {% endmacro %}
 
-{% macro RangeRow(option_name, option, display_value, value, can_delete=False) %}
+{% macro RangeRow(option_name, option, display_value, value, can_delete=False, default_override=None) %}
     <tr data-row="{{ option_name }}-{{ value }}-row" data-option-name="{{ option_name }}" data-value="{{ value }}">
         <td class="td-left">
             <label for="{{ option_name }}||{{ value }}">
@@ -220,7 +235,7 @@
                 name="{{ option_name }}||{{ value }}"
                 min="0"
                 max="50"
-                {% if option.default == value %}
+                {% if option.default == value or default_override == value %}
                     value="25"
                 {% else %}
                     value="0"
@@ -229,7 +244,7 @@
         </td>
         <td class="td-right">
             <span id="{{ option_name }}||{{ value }}-value">
-                {% if option.default == value %}
+                {% if option.default == value or default_override == value %}
                     25
                 {% else %}
                     0

WebHostLib/templates/weightedOptions/weightedOptions.html

@@ -51,7 +51,7 @@
 
             <div id="{{ world_name }}-container">
                 {% for group_name, group_options in option_groups.items() %}
-                    <details {% if loop.index == 1 %}open{% endif %}>
+                    <details {% if not start_collapsed[group_name] %}open{% endif %}>
                         <summary class="h2">{{ group_name }}</summary>
                         {% for option_name, option in group_options.items() %}
                             <div class="option-wrapper">

WebHostLib/tracker.py

@@ -3,8 +3,9 @@ import collections
 from dataclasses import dataclass
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, NamedTuple, Counter
 from uuid import UUID
+from email.utils import parsedate_to_datetime
 
-from flask import render_template
+from flask import render_template, make_response, Response, request
 from werkzeug.exceptions import abort
 
 from MultiServer import Context, get_saving_second
@@ -291,47 +292,47 @@ class TrackerData:
 
         return video_feeds
 
+    @_cache_results
+    def get_spheres(self) -> List[List[int]]:
+        """ each sphere is { player: { location_id, ... } } """
+        return self._multidata.get("spheres", [])
+
+
+def _process_if_request_valid(incoming_request, room: Optional[Room]) -> Optional[Response]:
+    if not room:
+        abort(404)
+
+    if_modified = incoming_request.headers.get("If-Modified-Since", None)
+    if if_modified:
+        if_modified = parsedate_to_datetime(if_modified)
+        # if_modified has less precision than last_activity, so we bring them to same precision
+        if if_modified >= room.last_activity.replace(microsecond=0):
+            return make_response("",  304)
+
 
 @app.route("/tracker/<suuid:tracker>/<int:tracked_team>/<int:tracked_player>")
-def get_player_tracker(tracker: UUID, tracked_team: int, tracked_player: int, generic: bool = False) -> str:
+def get_player_tracker(tracker: UUID, tracked_team: int, tracked_player: int, generic: bool = False) -> Response:
     key = f"{tracker}_{tracked_team}_{tracked_player}_{generic}"
-    tracker_page = cache.get(key)
-    if tracker_page:
-        return tracker_page
+    response: Optional[Response] = cache.get(key)
+    if response:
+        return response
 
-    timeout, tracker_page = get_timeout_and_tracker(tracker, tracked_team, tracked_player, generic)
-    cache.set(key, tracker_page, timeout)
-    return tracker_page
-
-
-@app.route("/generic_tracker/<suuid:tracker>/<int:tracked_team>/<int:tracked_player>")
-def get_generic_game_tracker(tracker: UUID, tracked_team: int, tracked_player: int) -> str:
-    return get_player_tracker(tracker, tracked_team, tracked_player, True)
-
-
-@app.route("/tracker/<suuid:tracker>", defaults={"game": "Generic"})
-@app.route("/tracker/<suuid:tracker>/<game>")
-@cache.memoize(timeout=TRACKER_CACHE_TIMEOUT_IN_SECONDS)
-def get_multiworld_tracker(tracker: UUID, game: str):
     # Room must exist.
     room = Room.get(tracker=tracker)
-    if not room:
-        abort(404)
 
-    tracker_data = TrackerData(room)
-    enabled_trackers = list(get_enabled_multiworld_trackers(room).keys())
-    if game not in _multiworld_trackers:
-        return render_generic_multiworld_tracker(tracker_data, enabled_trackers)
+    response = _process_if_request_valid(request, room)
+    if response:
+        return response
 
-    return _multiworld_trackers[game](tracker_data, enabled_trackers)
+    timeout, last_modified, tracker_page = get_timeout_and_player_tracker(room, tracked_team, tracked_player, generic)
+    response = make_response(tracker_page)
+    response.last_modified = last_modified
+    cache.set(key, response, timeout)
+    return response
 
 
-def get_timeout_and_tracker(tracker: UUID, tracked_team: int, tracked_player: int, generic: bool) -> Tuple[int, str]:
-    # Room must exist.
-    room = Room.get(tracker=tracker)
-    if not room:
-        abort(404)
-
+def get_timeout_and_player_tracker(room: Room, tracked_team: int, tracked_player: int, generic: bool)\
+        -> Tuple[int, datetime.datetime, str]:
     tracker_data = TrackerData(room)
 
     # Load and render the game-specific player tracker, or fallback to generic tracker if none exists.
@@ -341,7 +342,48 @@ def get_timeout_and_tracker(tracker: UUID, tracked_team: int, tracked_player: in
     else:
         tracker = render_generic_tracker(tracker_data, tracked_team, tracked_player)
 
-    return (tracker_data.get_room_saving_second() - datetime.datetime.now().second) % 60 or 60, tracker
+    return ((tracker_data.get_room_saving_second() - datetime.datetime.now().second)
+            % TRACKER_CACHE_TIMEOUT_IN_SECONDS or TRACKER_CACHE_TIMEOUT_IN_SECONDS, room.last_activity, tracker)
+
+
+@app.route("/generic_tracker/<suuid:tracker>/<int:tracked_team>/<int:tracked_player>")
+def get_generic_game_tracker(tracker: UUID, tracked_team: int, tracked_player: int) -> Response:
+    return get_player_tracker(tracker, tracked_team, tracked_player, True)
+
+
+@app.route("/tracker/<suuid:tracker>", defaults={"game": "Generic"})
+@app.route("/tracker/<suuid:tracker>/<game>")
+def get_multiworld_tracker(tracker: UUID, game: str) -> Response:
+    key = f"{tracker}_{game}"
+    response: Optional[Response] = cache.get(key)
+    if response:
+        return response
+
+    # Room must exist.
+    room = Room.get(tracker=tracker)
+
+    response = _process_if_request_valid(request, room)
+    if response:
+        return response
+
+    timeout, last_modified, tracker_page = get_timeout_and_multiworld_tracker(room, game)
+    response = make_response(tracker_page)
+    response.last_modified = last_modified
+    cache.set(key, response, timeout)
+    return response
+
+
+def get_timeout_and_multiworld_tracker(room: Room, game: str)\
+        -> Tuple[int, datetime.datetime, str]:
+    tracker_data = TrackerData(room)
+    enabled_trackers = list(get_enabled_multiworld_trackers(room).keys())
+    if game in _multiworld_trackers:
+        tracker = _multiworld_trackers[game](tracker_data, enabled_trackers)
+    else:
+        tracker = render_generic_multiworld_tracker(tracker_data, enabled_trackers)
+
+    return ((tracker_data.get_room_saving_second() - datetime.datetime.now().second)
+            % TRACKER_CACHE_TIMEOUT_IN_SECONDS or TRACKER_CACHE_TIMEOUT_IN_SECONDS, room.last_activity, tracker)
 
 
 def get_enabled_multiworld_trackers(room: Room) -> Dict[str, Callable]:
@@ -411,9 +453,30 @@ def render_generic_multiworld_tracker(tracker_data: TrackerData, enabled_tracker
         videos=tracker_data.get_room_videos(),
         item_id_to_name=tracker_data.item_id_to_name,
         location_id_to_name=tracker_data.location_id_to_name,
+        saving_second=tracker_data.get_room_saving_second(),
     )
 
 
+def render_generic_multiworld_sphere_tracker(tracker_data: TrackerData) -> str:
+    return render_template(
+        "multispheretracker.html",
+        room=tracker_data.room,
+        tracker_data=tracker_data,
+    )
+
+
+@app.route("/sphere_tracker/<suuid:tracker>")
+@cache.memoize(timeout=TRACKER_CACHE_TIMEOUT_IN_SECONDS)
+def get_multiworld_sphere_tracker(tracker: UUID):
+    # Room must exist.
+    room = Room.get(tracker=tracker)
+    if not room:
+        abort(404)
+
+    tracker_data = TrackerData(room)
+    return render_generic_multiworld_sphere_tracker(tracker_data)
+
+
 # TODO: This is a temporary solution until a proper Tracker API can be implemented for tracker templates and data to
 #       live in their respective world folders.
 

Zelda1Client.py

@@ -152,7 +152,7 @@ def get_payload(ctx: ZeldaContext):
 
 
 def reconcile_shops(ctx: ZeldaContext):
-    checked_location_names = [ctx.location_names[location] for location in ctx.checked_locations]
+    checked_location_names = [ctx.location_names.lookup_in_slot(location) for location in ctx.checked_locations]
     shops = [location for location in checked_location_names if "Shop" in location]
     left_slots = [shop for shop in shops if "Left" in shop]
     middle_slots = [shop for shop in shops if "Middle" in shop]
@@ -190,7 +190,7 @@ async def parse_locations(locations_array, ctx: ZeldaContext, force: bool, zone=
         locations_checked = []
         location = None
         for location in ctx.missing_locations:
-            location_name = ctx.location_names[location]
+            location_name = ctx.location_names.lookup_in_slot(location)
 
             if location_name in Locations.overworld_locations and zone == "overworld":
                 status = locations_array[Locations.major_location_offsets[location_name]]

_speedups.pyx

@@ -1,5 +1,6 @@
 #cython: language_level=3
-#distutils: language = c++
+#distutils: language = c
+#distutils: depends = intset.h
 
 """
 Provides faster implementation of some core parts.
@@ -13,7 +14,6 @@ from cpython cimport PyObject
 from typing import Any, Dict, Iterable, Iterator, Generator, Sequence, Tuple, TypeVar, Union, Set, List, TYPE_CHECKING
 from cymem.cymem cimport Pool
 from libc.stdint cimport int64_t, uint32_t
-from libcpp.set cimport set as std_set
 from collections import defaultdict
 
 cdef extern from *:
@@ -31,6 +31,27 @@ ctypedef int64_t ap_id_t
 cdef ap_player_t MAX_PLAYER_ID = 1000000  # limit the size of indexing array
 cdef size_t INVALID_SIZE = <size_t>(-1)  # this is all 0xff... adding 1 results in 0, but it's not negative
 
+# configure INTSET for player
+cdef extern from *:
+    """
+    #define INTSET_NAME ap_player_set
+    #define INTSET_TYPE uint32_t  // has to match ap_player_t
+    """
+
+# create INTSET for player
+cdef extern from "intset.h":
+    """
+    #undef INTSET_NAME
+    #undef INTSET_TYPE
+    """
+    ctypedef struct ap_player_set:
+        pass
+
+    ap_player_set* ap_player_set_new(size_t bucket_count) nogil
+    void ap_player_set_free(ap_player_set* set) nogil
+    bint ap_player_set_add(ap_player_set* set, ap_player_t val) nogil
+    bint ap_player_set_contains(ap_player_set* set, ap_player_t val) nogil
+
 
 cdef struct LocationEntry:
     # layout is so that
@@ -185,7 +206,7 @@ cdef class LocationStore:
     def find_item(self, slots: Set[int], seeked_item_id: int) -> Generator[Tuple[int, int, int, int, int], None, None]:
         cdef ap_id_t item = seeked_item_id
         cdef ap_player_t receiver
-        cdef std_set[ap_player_t] receivers
+        cdef ap_player_set* receivers
         cdef size_t slot_count = len(slots)
         if slot_count == 1:
             # specialized implementation for single slot
@@ -197,13 +218,20 @@ cdef class LocationStore:
                             yield entry.sender, entry.location, entry.item, entry.receiver, entry.flags
         elif slot_count:
             # generic implementation with lookup in set
-            for receiver in slots:
-                receivers.insert(receiver)
-            with nogil:
-                for entry in self.entries[:self.entry_count]:
-                    if entry.item == item and receivers.count(entry.receiver):
-                        with gil:
-                            yield entry.sender, entry.location, entry.item, entry.receiver, entry.flags
+            receivers = ap_player_set_new(min(1023, slot_count))  # limit top level struct to 16KB
+            if not receivers:
+                raise MemoryError()
+            try:
+                for receiver in slots:
+                    if not ap_player_set_add(receivers, receiver):
+                        raise MemoryError()
+                with nogil:
+                    for entry in self.entries[:self.entry_count]:
+                        if entry.item == item and ap_player_set_contains(receivers, entry.receiver):
+                            with gil:
+                                yield entry.sender, entry.location, entry.item, entry.receiver, entry.flags
+            finally:
+                ap_player_set_free(receivers)
 
     def get_for_player(self, slot: int) -> Dict[int, Set[int]]:
         cdef ap_player_t receiver = slot

_speedups.pyxbld

@@ -1,8 +1,10 @@
-# This file is required to get pyximport to work with C++.
-# Switching from std::set to a pure C implementation is still on the table to simplify everything.
+# This file is used when doing pyximport
+import os
 
 def make_ext(modname, pyxfilename):
     from distutils.extension import Extension
     return Extension(name=modname,
                      sources=[pyxfilename],
-                     language='c++')
+                     depends=["intset.h"],
+                     include_dirs=[os.getcwd()],
+                     language="c")

data/client.kv

@@ -13,6 +13,7 @@
     plum: "AF99EF"  # typically progression item
     salmon: "FA8072"  # typically trap item
     white: "FFFFFF"  # not used, if you want to change the generic text color change color in Label
+    orange: "FF7700"  # Used for command echo
 <Label>:
     color: "FFFFFF"
 <TabbedPanel>:

docs/CODEOWNERS

@@ -6,25 +6,24 @@
 #
 # All usernames must be GitHub usernames (and are case sensitive).
 
-###################
-## Active Worlds ##
-###################
-
 # Adventure
 /worlds/adventure/ @JusticePS
 
+# A Hat in Time
+/worlds/ahit/ @CookieCat45
+
 # A Link to the Past
 /worlds/alttp/ @Berserker66
 
+# Sudoku (APSudoku)
+/worlds/apsudoku/ @EmilyV99
+
 # Aquaria
 /worlds/aquaria/ @tioui
 
 # ArchipIDLE
 /worlds/archipidle/ @LegendaryLinux
 
-# Sudoku (BK Sudoku)
-/worlds/bk_sudoku/ @Jarno458
-
 # Blasphemous
 /worlds/blasphemous/ @TRPG0
 
@@ -64,9 +63,6 @@
 # Factorio
 /worlds/factorio/ @Berserker66
 
-# Final Fantasy
-/worlds/ff1/ @jtoyoda
-
 # Final Fantasy Mystic Quest
 /worlds/ffmq/ @Alchav @wildham0
 
@@ -74,7 +70,7 @@
 /worlds/heretic/ @Daivuk
 
 # Hollow Knight
-/worlds/hk/ @BadMagic100 @ThePhar
+/worlds/hk/ @BadMagic100 @qwint
 
 # Hylics 2
 /worlds/hylics2/ @TRPG0
@@ -204,7 +200,7 @@
 /worlds/yoshisisland/ @PinkSwitch
 
 #Yu-Gi-Oh! Ultimate Masters: World Championship Tournament 2006
-/worlds/yugioh06/ @rensen
+/worlds/yugioh06/ @Rensen3
 
 # Zillion
 /worlds/zillion/ @beauxq
@@ -212,9 +208,22 @@
 # Zork Grand Inquisitor
 /worlds/zork_grand_inquisitor/ @nbrochu
 
-##################################
-## Disabled Unmaintained Worlds ##
-##################################
+
+## Active Unmaintained Worlds
+
+# The following worlds in this repo are currently unmaintained, but currently still work in core. If any update breaks
+# compatibility, these worlds may be moved to `worlds_disabled`. If you are interested in stepping up as maintainer for
+# any of these worlds, please review `/docs/world maintainer.md` documentation.
+
+# Final Fantasy (1)
+# /worlds/ff1/
+
+
+## Disabled Unmaintained Worlds
+
+# The following worlds in this repo are currently unmaintained and disabled as they do not work in core. If you are
+# interested in stepping up as maintainer for any of these worlds, please review `/docs/world maintainer.md`
+# documentation.
 
 # Ori and the Blind Forest
-# /worlds_disabled/oribf/ <Unmaintained>
+# /worlds_disabled/oribf/

docs/contributing.md

@@ -1,43 +1,49 @@
 # Contributing
-Contributions are welcome. We have a few requests for new contributors:
+
+All contributions are welcome, though we have a few requests of contributors, whether they be for core, webhost, or new 
+game contributions:
 
 * **Follow styling guidelines.**
   Please take a look at the [code style documentation](/docs/style.md)
   to ensure ease of communication and uniformity.
 
-* **Ensure that critical changes are covered by tests.** 
-It is strongly recommended that unit tests are used to avoid regression and to ensure everything is still working.
-If you wish to contribute by adding a new game, please take a look at the [logic unit test documentation](/docs/tests.md).
-If you wish to contribute to the website, please take a look at [these tests](/test/webhost).
+* **Ensure that critical changes are covered by tests.**
+  It is strongly recommended that unit tests are used to avoid regression and to ensure everything is still working.
+  If you wish to contribute by adding a new game, please take a look at
+  the [logic unit test documentation](/docs/tests.md).
+  If you wish to contribute to the website, please take a look at [these tests](/test/webhost).
 
 * **Do not introduce unit test failures/regressions.**
-Archipelago supports multiple versions of Python. You may need to download older Python versions to fully test
-your changes. Currently, the oldest supported version is [Python 3.8](https://www.python.org/downloads/release/python-380/). 
-It is recommended that automated github actions are turned on in your fork to have github run all of the unit tests after pushing.
-You can turn them on here:  
-![Github actions example](./img/github-actions-example.png)
+  Archipelago supports multiple versions of Python. You may need to download older Python versions to fully test
+  your changes. Currently, the oldest supported version
+  is [Python 3.8](https://www.python.org/downloads/release/python-380/).
+  It is recommended that automated github actions are turned on in your fork to have github run unit tests after
+  pushing.
+  You can turn them on here:  
+  ![Github actions example](./img/github-actions-example.png)
 
 * **When reviewing PRs, please leave a message about what was done.**
-We don't have full test coverage, so manual testing can help.
-For code changes that could affect multiple worlds or that could have changes in unexpected code paths, manual testing
-or checking if all code paths are covered by automated tests is desired. The original author may not have been able
-to test all possibly affected worlds, or didn't know it would affect another world. In such cases, it is helpful to
-state which games or settings were rolled, if any.
-Please also tell us if you looked at code, just did functional testing, did both, or did neither.
-If testing the PR depends on other PRs, please state what you merged into what for testing.
-We cannot determine what "LGTM" means without additional context, so that should not be the norm.
+  We don't have full test coverage, so manual testing can help.
+  For code changes that could affect multiple worlds or that could have changes in unexpected code paths, manual testing
+  or checking if all code paths are covered by automated tests is desired. The original author may not have been able
+  to test all possibly affected worlds, or didn't know it would affect another world. In such cases, it is helpful to
+  state which games or settings were rolled, if any.
+  Please also tell us if you looked at code, just did functional testing, did both, or did neither.
+  If testing the PR depends on other PRs, please state what you merged into what for testing.
+  We cannot determine what "LGTM" means without additional context, so that should not be the norm.
 
-Other than these requests, we tend to judge code on a case-by-case basis. 
+Other than these requests, we tend to judge code on a case-by-case basis.
 
 For contribution to the website, please refer to the [WebHost README](/WebHostLib/README.md).
 
 If you want to contribute to the core, you will be subject to stricter review on your pull requests. It is recommended
 that you get in touch with other core maintainers via the [Discord](https://archipelago.gg/discord).
 
-If you want to add Archipelago support for a new game, please take a look at the [adding games documentation](/docs/adding%20games.md), which details what is required 
-to implement support for a game, as well as tips for how to get started.
-If you want to merge a new game into the main Archipelago repo, please make sure to read the responsibilities as a 
-[world maintainer](/docs/world%20maintainer.md).  
+If you want to add Archipelago support for a new game, please take a look at
+the [adding games documentation](/docs/adding%20games.md)
+which details what is required to implement support for a game, and has tips on to get started.
+If you want to merge a new game into the main Archipelago repo, please make sure to read the responsibilities as a
+[world maintainer](/docs/world%20maintainer.md).
 
-For other questions, feel free to explore the [main documentation folder](/docs/) and ask us questions in the #archipelago-dev channel 
-of the [Discord](https://archipelago.gg/discord).
+For other questions, feel free to explore the [main documentation folder](/docs), and ask us questions in the
+#ap-world-dev channel of the [Discord](https://archipelago.gg/discord).

docs/network protocol.md

@@ -53,7 +53,7 @@ Example:
 ```
 
 ## (Server -> Client)
-These packets are are sent from the multiworld server to the client. They are not messages which the server accepts.
+These packets are sent from the multiworld server to the client. They are not messages which the server accepts.
 * [RoomInfo](#RoomInfo)
 * [ConnectionRefused](#ConnectionRefused)
 * [Connected](#Connected)
@@ -80,7 +80,6 @@ Sent to clients when they connect to an Archipelago server.
 | hint_cost             | int                                           | The percentage of total locations that need to be checked to receive a hint from the server.                                                                                                                                          |
 | location_check_points | int                                           | The amount of hint points you receive per item/location check completed.                                                                                                                                                              |
 | games                 | list\[str\]                                   | List of games present in this multiworld.                                                                                                                                                                                             |
-| datapackage_versions  | dict\[str, int\]                              | Data versions of the individual games' data packages the server will send. Used to decide which games' caches are outdated. See [Data Package Contents](#Data-Package-Contents). **Deprecated. Use `datapackage_checksums` instead.** |
 | datapackage_checksums | dict[str, str]                                | Checksum hash of the individual games' data packages the server will send. Used by newer clients to decide which games' caches are outdated. See [Data Package Contents](#Data-Package-Contents) for more information.                | 
 | seed_name             | str                                           | Uniquely identifying name of this generation                                                                                                                                                                                          |
 | time                  | float                                         | Unix time stamp of "now". Send for time synchronization if wanted for things like the DeathLink Bounce.                                                                                                                               |
@@ -500,9 +499,9 @@ In JSON this may look like:
     {"item": 3, "location": 3, "player": 3, "flags": 0}
 ]
 ```
-`item` is the item id of the item. Item ids are in the range of ± 2<sup>53</sup>-1.
+`item` is the item id of the item. Item ids are only supported in the range of [-2<sup>53</sup>, 2<sup>53</sup> - 1], with anything ≤ 0 reserved for Archipelago use.
 
-`location` is the location id of the item inside the world. Location ids are in the range of ± 2<sup>53</sup>-1.
+`location` is the location id of the item inside the world. Location ids are only supported in the range of [-2<sup>53</sup>, 2<sup>53</sup> - 1], with anything ≤ 0 reserved for Archipelago use.
 
 `player` is the player slot of the world the item is located in, except when inside an [LocationInfo](#LocationInfo) Packet then it will be the slot of the player to receive the item
 
@@ -646,15 +645,47 @@ class Hint(typing.NamedTuple):
 ```
 
 ### Data Package Contents
-A data package is a JSON object which may contain arbitrary metadata to enable a client to interact with the Archipelago server most easily. Currently, this package is used to send ID to name mappings so that clients need not maintain their own mappings.
+A data package is a JSON object which may contain arbitrary metadata to enable a client to interact with the Archipelago
+server most easily and not maintain their own mappings. Some contents include:
 
-We encourage clients to cache the data package they receive on disk, or otherwise not tied to a session. You will know when your cache is outdated if the [RoomInfo](#RoomInfo) packet or the datapackage itself denote a different version. A special case is datapackage version 0, where it is expected the package is custom and should not be cached.
+   - Name to ID mappings for items and locations.
+   - A checksum of each game's data package for clients to tell if a cached package is invalid.
 
-Note: 
- * Any ID is unique to its type across AP: Item 56 only exists once and Location 56 only exists once.
- * Any Name is unique to its type across its own Game only: Single Arrow can exist in two games.
- * The IDs from the game "Archipelago" may be used in any other game. 
-   Especially Location ID -1: Cheat Console and -2: Server (typically Remote Start Inventory)
+We encourage clients to cache the data package they receive on disk, or otherwise not tied to a session. You will know 
+when your cache is outdated if the [RoomInfo](#RoomInfo) packet or the datapackage itself denote a different checksum
+than any locally cached ones.
+
+**Important Notes about IDs and Names**: 
+
+* IDs ≤ 0 are reserved for "Archipelago" and should not be used by other world implementations.
+* The IDs from the game "Archipelago" (in `worlds/generic`) may be used in any world.
+  * Especially Location ID `-1`: `Cheat Console` and `-2`: `Server` (typically Remote Start Inventory)
+* Any names and IDs are only unique in its own world data package, but different games may reuse these names or IDs.
+  * At runtime, you will need to look up the game of the player to know which item or location ID/Name to lookup in the
+    data package. This can be easily achieved by reviewing the `slot_info` for a particular player ID prior to lookup.
+  * For example, a data package like this is valid (Some properties such as `checksum` were omitted):
+    ```json
+    {
+      "games": {
+        "Game A": {
+          "location_name_to_id": {
+            "Boss Chest": 40
+          },
+          "item_name_to_id": {
+            "Item X": 12
+          }
+        },
+        "Game B": {
+          "location_name_to_id": {
+            "Minigame Prize": 40
+          },
+          "item_name_to_id": {
+            "Item X": 40
+          }
+        }
+      }
+    }
+    ```
 
 #### Contents
 | Name | Type | Notes |
@@ -668,7 +699,6 @@ GameData is a **dict** but contains these keys and values. It's broken out into
 |---------------------|----------------|-------------------------------------------------------------------------------------------------------------------------------|
 | item_name_to_id     | dict[str, int] | Mapping of all item names to their respective ID.                                                                             |
 | location_name_to_id | dict[str, int] | Mapping of all location names to their respective ID.                                                                         |
-| version             | int            | Version number of this game's data. Deprecated. Used by older clients to request an updated datapackage if cache is outdated. |
 | checksum            | str            | A checksum hash of this game's data.                                                                                          |
 
 ### Tags

docs/options api.md

@@ -86,17 +86,29 @@ class ExampleWorld(World):
 ```
 
 ### Option Groups
-Options may be categorized into groups for display on the WebHost. Option groups are displayed alphabetically on the
-player-options and weighted-options pages. Options without a group name are categorized into a generic "Game Options"
-group.
+Options may be categorized into groups for display on the WebHost. Option groups are displayed in the order specified
+by your world on the player-options and weighted-options pages. In the generated template files, there will be a comment
+with the group name at the beginning of each group of options. The `start_collapsed` Boolean only affects how the groups
+appear on the WebHost, with the grouping being collapsed when this is `True`.
+
+Options without a group name are categorized into a generic "Game Options" group, which is always the first group. If
+every option for your world is in a group, this group will be removed. There is also an "Items & Location Options"
+group, which is automatically created using certain specified `item_and_loc_options`. These specified options cannot be
+removed from this group.
+
+Both the "Game Options" and "Item & Location Options" groups can be overridden by creating your own groups with
+those names, letting you add options to them and change whether they start collapsed. The "Item &
+Location Options" group can also be moved to a different position in the group ordering, but "Game Options" will always
+be first, regardless of where it is in your list.
 
 ```python
 from worlds.AutoWorld import WebWorld
 from Options import OptionGroup
+from . import Options
 
 class MyWorldWeb(WebWorld):
     option_groups = [
-        OptionGroup('Color Options', [
+        OptionGroup("Color Options", [
             Options.ColorblindMode,
             Options.FlashReduction,
             Options.UIColors,
@@ -120,7 +132,8 @@ or if I need a boolean object, such as in my slot_data I can access it as:
 start_with_sword = bool(self.options.starting_sword.value)
 ```
 All numeric options (i.e. Toggle, Choice, Range) can be compared to integers, strings that match their attributes,
-strings that match the option attributes after "option_" is stripped, and the attributes themselves.
+strings that match the option attributes after "option_" is stripped, and the attributes themselves. The option can
+also be checked to see if it exists within a collection, but this will fail for a set of strings due to hashing.
 ```python
 # options.py
 class Logic(Choice):
@@ -132,6 +145,12 @@ class Logic(Choice):
     alias_extra_hard = 2
     crazy = 4  # won't be listed as an option and only exists as an attribute on the class
 
+class Weapon(Choice):
+    option_none = 0
+    option_sword = 1
+    option_bow = 2
+    option_hammer = 3
+
 # __init__.py
 from .options import Logic
 
@@ -145,6 +164,16 @@ elif self.options.logic == Logic.option_extreme:
     do_extreme_things()
 elif self.options.logic == "crazy":
     do_insane_things()
+
+# check if the current option is in a collection of integers using the class attributes
+if self.options.weapon in {Weapon.option_bow, Weapon.option_sword}:
+    do_stuff()
+# in order to make a set of strings work, we have to compare against current_key
+elif self.options.weapon.current_key in {"none", "hammer"}:
+    do_something_else()
+# though it's usually better to just use a tuple instead
+elif self.options.weapon in ("none", "hammer"):
+    do_something_else()
 ```
 ## Generic Option Classes
 These options are generically available to every game automatically, but can be overridden for slightly different

docs/world api.md

@@ -121,6 +121,53 @@ class RLWeb(WebWorld):
     # ...
 ```
 
+* `location_descriptions` (optional) WebWorlds can provide a map that contains human-friendly descriptions of locations 
+or location groups.
+
+  ```python
+  # locations.py
+  location_descriptions = {
+      "Red Potion #6": "In a secret destructible block under the second stairway",
+      "L2 Spaceship": """
+        The group of all items in the spaceship in Level 2.
+  
+        This doesn't include the item on the spaceship door, since it can be
+        accessed without the Spaceship Key.
+      """
+  }
+  
+  # __init__.py
+  from worlds.AutoWorld import WebWorld
+  from .locations import location_descriptions
+  
+  
+  class MyGameWeb(WebWorld):
+      location_descriptions = location_descriptions
+  ```
+
+* `item_descriptions` (optional) WebWorlds can provide a map that contains human-friendly descriptions of items or item 
+groups.
+
+  ```python
+  # items.py
+  item_descriptions = {
+      "Red Potion": "A standard health potion",
+      "Spaceship Key": """
+        The key to the spaceship in Level 2.
+  
+        This is necessary to get to the Star Realm.
+      """,
+  }
+  
+  # __init__.py
+  from worlds.AutoWorld import WebWorld
+  from .items import item_descriptions
+  
+  
+  class MyGameWeb(WebWorld):
+      item_descriptions = item_descriptions
+  ```
+
 ### MultiWorld Object
 
 The `MultiWorld` object references the whole multiworld (all items and locations for all players) and is accessible
@@ -178,36 +225,6 @@ Classification is one of `LocationProgressType.DEFAULT`, `PRIORITY` or `EXCLUDED
 The Fill algorithm will force progression items to be placed at priority locations, giving a higher chance of them being
 required, and will prevent progression and useful items from being placed at excluded locations.
 
-#### Documenting Locations
-
-Worlds can optionally provide a `location_descriptions` map which contains human-friendly descriptions of locations and
-location groups. These descriptions will show up in location-selection options on the options pages.
-
-```python
-# locations.py
-
-location_descriptions = {
-    "Red Potion #6": "In a secret destructible block under the second stairway",
-    "L2 Spaceship":
-        """
-        The group of all items in the spaceship in Level 2.
-
-        This doesn't include the item on the spaceship door, since it can be accessed without the Spaceship Key.
-        """
-}
-```
-
-```python
-# __init__.py
-
-from worlds.AutoWorld import World
-from .locations import location_descriptions
-
-
-class MyGameWorld(World):
-    location_descriptions = location_descriptions
-```
-
 ### Items
 
 Items are all things that can "drop" for your game. This may be RPG items like weapons, or technologies you normally
@@ -232,36 +249,6 @@ Other classifications include:
 * `progression_skip_balancing`: the combination of `progression` and `skip_balancing`, i.e., a progression item that
   will not be moved around by progression balancing; used, e.g., for currency or tokens, to not flood early spheres
 
-#### Documenting Items
-
-Worlds can optionally provide an `item_descriptions` map which contains human-friendly descriptions of items and item
-groups. These descriptions will show up in item-selection options on the options pages.
-
-```python
-# items.py
-
-item_descriptions = {
-    "Red Potion": "A standard health potion",
-    "Spaceship Key":
-        """
-        The key to the spaceship in Level 2.
-
-        This is necessary to get to the Star Realm.
-        """
-}
-```
-
-```python
-# __init__.py
-
-from worlds.AutoWorld import World
-from .items import item_descriptions
-
-
-class MyGameWorld(World):
-    item_descriptions = item_descriptions
-```
-
 ### Events
 
 An Event is a special combination of a Location and an Item, with both having an `id` of `None`. These can be used to

inno_setup.iss

@@ -75,7 +75,7 @@ Name: "{commondesktop}\{#MyAppName} Launcher"; Filename: "{app}\ArchipelagoLaunc
 [Run]
 
 Filename: "{tmp}\vc_redist.x64.exe"; Parameters: "/passive /norestart"; Check: IsVCRedist64BitNeeded; StatusMsg: "Installing VC++ redistributable..."
-Filename: "{app}\ArchipelagoLttPAdjuster"; Parameters: "--update_sprites"; StatusMsg: "Updating Sprite Library..."; Flags: nowait; Components: lttp_sprites
+Filename: "{app}\ArchipelagoLttPAdjuster"; Parameters: "--update_sprites"; StatusMsg: "Updating Sprite Library..."; Components: lttp_sprites
 Filename: "{app}\ArchipelagoLauncher"; Parameters: "--update_settings"; StatusMsg: "Updating host.yaml..."; Flags: runasoriginaluser runhidden
 Filename: "{app}\ArchipelagoLauncher"; Description: "{cm:LaunchProgram,{#StringChange('Launcher', '&', '&&')}}"; Flags: nowait postinstall skipifsilent
 
@@ -87,7 +87,14 @@ Type: files; Name: "{app}\lib\worlds\_bizhawk.apworld"
 Type: files; Name: "{app}\ArchipelagoLttPClient.exe"
 Type: files; Name: "{app}\ArchipelagoPokemonClient.exe"
 Type: files; Name: "{app}\data\lua\connector_pkmn_rb.lua"
-Type: filesandordirs; Name: "{app}\lib\worlds\rogue-legacy*"
+Type: filesandordirs; Name: "{app}\lib\worlds\rogue-legacy"
+Type: dirifempty; Name: "{app}\lib\worlds\rogue-legacy"
+Type: files; Name: "{app}\lib\worlds\sc2wol.apworld"
+Type: filesandordirs; Name: "{app}\lib\worlds\sc2wol"
+Type: dirifempty; Name: "{app}\lib\worlds\sc2wol"
+Type: filesandordirs; Name: "{app}\lib\worlds\bk_sudoku"
+Type: dirifempty; Name: "{app}\lib\worlds\bk_sudoku"
+Type: files; Name: "{app}\ArchipelagoLauncher(DEBUG).exe"
 Type: filesandordirs; Name: "{app}\SNI\lua*"
 Type: filesandordirs; Name: "{app}\EnemizerCLI*"
 #include "installdelete.iss"
@@ -209,6 +216,11 @@ Root: HKCR; Subkey: "{#MyAppName}multidata";                     ValueData: "Arc
 Root: HKCR; Subkey: "{#MyAppName}multidata\DefaultIcon";         ValueData: "{app}\ArchipelagoServer.exe,0";                         ValueType: string;  ValueName: "";
 Root: HKCR; Subkey: "{#MyAppName}multidata\shell\open\command";  ValueData: """{app}\ArchipelagoServer.exe"" ""%1""";                ValueType: string;  ValueName: "";
 
+Root: HKCR; Subkey: ".apworld";                                 ValueData: "{#MyAppName}worlddata";  Flags: uninsdeletevalue; ValueType: string;  ValueName: "";
+Root: HKCR; Subkey: "{#MyAppName}worlddata";                    ValueData: "Archipelago World Data"; Flags: uninsdeletekey;   ValueType: string;  ValueName: "";
+Root: HKCR; Subkey: "{#MyAppName}worlddata\DefaultIcon";        ValueData: "{app}\ArchipelagoLauncher.exe,0";                 ValueType: string;  ValueName: "";
+Root: HKCR; Subkey: "{#MyAppName}worlddata\shell\open\command"; ValueData: """{app}\ArchipelagoLauncher.exe"" ""%1""";
+
 Root: HKCR; Subkey: "archipelago"; ValueType: "string"; ValueData: "Archipegalo Protocol"; Flags: uninsdeletekey;
 Root: HKCR; Subkey: "archipelago"; ValueType: "string"; ValueName: "URL Protocol"; ValueData: "";
 Root: HKCR; Subkey: "archipelago\DefaultIcon"; ValueType: "string"; ValueData: "{app}\ArchipelagoTextClient.exe,0";

intset.h

@@ -0,0 +1,135 @@
+/* A specialized unordered_set implementation for literals, where bucket_count
+ * is defined at initialization rather than increased automatically.
+ */
+#include <stddef.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifndef INTSET_NAME
+#error "Please #define INTSET_NAME ... before including intset.h"
+#endif
+
+#ifndef INTSET_TYPE
+#error "Please #define INTSET_TYPE ... before including intset.h"
+#endif
+
+/* macros to generate unique names from INTSET_NAME */
+#ifndef INTSET_CONCAT
+#define INTSET_CONCAT_(a, b) a ## b
+#define INTSET_CONCAT(a, b) INTSET_CONCAT_(a, b)
+#define INTSET_FUNC_(a, b) INTSET_CONCAT(a, _ ## b)
+#endif
+
+#define INTSET_FUNC(name) INTSET_FUNC_(INTSET_NAME, name)
+#define INTSET_BUCKET INTSET_CONCAT(INTSET_NAME, Bucket)
+#define INTSET_UNION INTSET_CONCAT(INTSET_NAME, Union)
+
+#if defined(_MSC_VER)
+#pragma warning(push)
+#pragma warning(disable : 4200)
+#endif
+
+
+typedef struct {
+    size_t count;
+    union INTSET_UNION {
+        INTSET_TYPE val;
+        INTSET_TYPE *data;
+    } v;
+} INTSET_BUCKET;
+
+typedef struct {
+    size_t bucket_count;
+    INTSET_BUCKET buckets[];
+} INTSET_NAME;
+
+static INTSET_NAME *INTSET_FUNC(new)(size_t buckets)
+{
+    size_t i, size;
+    INTSET_NAME *set;
+
+    if (buckets < 1)
+        buckets = 1;
+    if ((SIZE_MAX - sizeof(INTSET_NAME)) / sizeof(INTSET_BUCKET) < buckets)
+        return NULL;
+    size = sizeof(INTSET_NAME) + buckets * sizeof(INTSET_BUCKET);
+    set = (INTSET_NAME*)malloc(size);
+    if (!set)
+        return NULL;
+    memset(set, 0, size); /* gcc -fanalyzer does not understand this sets all buckets' count to 0 */
+    for (i = 0; i < buckets; i++) {
+        set->buckets[i].count = 0;
+    }
+    set->bucket_count = buckets;
+    return set;
+}
+
+static void INTSET_FUNC(free)(INTSET_NAME *set)
+{
+    size_t i;
+    if (!set)
+        return;
+    for (i = 0; i < set->bucket_count; i++) {
+        if (set->buckets[i].count > 1)
+            free(set->buckets[i].v.data);
+    }
+    free(set);
+}
+
+static bool INTSET_FUNC(contains)(INTSET_NAME *set, INTSET_TYPE val)
+{
+    size_t i;
+    INTSET_BUCKET* bucket = &set->buckets[(size_t)val % set->bucket_count];
+    if (bucket->count == 1)
+        return bucket->v.val == val;
+    for (i = 0; i < bucket->count; ++i) {
+        if (bucket->v.data[i] == val)
+            return true;
+    }
+    return false;
+}
+
+static bool INTSET_FUNC(add)(INTSET_NAME *set, INTSET_TYPE val)
+{
+    INTSET_BUCKET* bucket;
+
+    if (INTSET_FUNC(contains)(set, val))
+        return true; /* ok */
+
+    bucket = &set->buckets[(size_t)val % set->bucket_count];
+    if (bucket->count == 0) {
+        bucket->v.val = val;
+        bucket->count = 1;
+    } else if (bucket->count == 1) {
+        INTSET_TYPE old = bucket->v.val;
+        bucket->v.data = (INTSET_TYPE*)malloc(2 * sizeof(INTSET_TYPE));
+        if (!bucket->v.data) {
+            bucket->v.val = old;
+            return false; /* error */
+        }
+        bucket->v.data[0] = old;
+        bucket->v.data[1] = val;
+        bucket->count = 2;
+    } else {
+        size_t new_bucket_size;
+        INTSET_TYPE* new_bucket_data;
+
+        new_bucket_size = (bucket->count + 1) * sizeof(INTSET_TYPE);
+        new_bucket_data = (INTSET_TYPE*)realloc(bucket->v.data, new_bucket_size);
+        if (!new_bucket_data)
+            return false; /* error */
+        bucket->v.data = new_bucket_data;
+        bucket->v.data[bucket->count++] = val;
+    }
+    return true; /* success */
+}
+
+
+#if defined(_MSC_VER)
+#pragma warning(pop)
+#endif
+
+#undef INTSET_FUNC
+#undef INTSET_BUCKET
+#undef INTSET_UNION

kvui.py

@@ -3,6 +3,7 @@ import logging
 import sys
 import typing
 import re
+from collections import deque
 
 if sys.platform == "win32":
     import ctypes
@@ -64,7 +65,7 @@ from kivy.uix.popup import Popup
 fade_in_animation = Animation(opacity=0, duration=0) + Animation(opacity=1, duration=0.25)
 
 from NetUtils import JSONtoTextParser, JSONMessagePart, SlotType
-from Utils import async_start
+from Utils import async_start, get_input_text_from_response
 
 if typing.TYPE_CHECKING:
     import CommonClient
@@ -285,16 +286,10 @@ class SelectableLabel(RecycleDataViewBehavior, TooltipLabel):
                 temp = MarkupLabel(text=self.text).markup
                 text = "".join(part for part in temp if not part.startswith(("[color", "[/color]", "[ref=", "[/ref]")))
                 cmdinput = App.get_running_app().textinput
-                if not cmdinput.text and " did you mean " in text:
-                    for question in ("Didn't find something that closely matches, did you mean ",
-                                     "Too many close matches, did you mean "):
-                        if text.startswith(question):
-                            name = Utils.get_text_between(text, question,
-                                                          "? (")
-                            cmdinput.text = f"!{App.get_running_app().last_autofillable_command} {name}"
-                            break
-                elif not cmdinput.text and text.startswith("Missing: "):
-                    cmdinput.text = text.replace("Missing: ", "!hint_location ")
+                if not cmdinput.text:
+                    input_text = get_input_text_from_response(text, App.get_running_app().last_autofillable_command)
+                    if input_text is not None:
+                        cmdinput.text = input_text
 
                 Clipboard.copy(text.replace("&", "&").replace("&bl;", "[").replace("&br;", "]"))
                 return self.parent.select_with_touch(self.index, touch)
@@ -386,6 +381,57 @@ class ConnectBarTextInput(TextInput):
         return super(ConnectBarTextInput, self).insert_text(s, from_undo=from_undo)
 
 
+def is_command_input(string: str) -> bool:
+    return len(string) > 0 and string[0] in "/!"
+
+
+class CommandPromptTextInput(TextInput):
+    MAXIMUM_HISTORY_MESSAGES = 50
+
+    def __init__(self, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self._command_history_index = -1
+        self._command_history: typing.Deque[str] = deque(maxlen=CommandPromptTextInput.MAXIMUM_HISTORY_MESSAGES)
+    
+    def update_history(self, new_entry: str) -> None:
+        self._command_history_index = -1
+        if is_command_input(new_entry):
+            self._command_history.appendleft(new_entry)
+
+    def keyboard_on_key_down(
+        self,
+        window,
+        keycode: typing.Tuple[int, str],
+        text: typing.Optional[str],
+        modifiers: typing.List[str]
+    ) -> bool:
+        """
+        :param window: The kivy window object
+        :param keycode: A tuple of (keycode, keyname). Keynames are always lowercase
+        :param text: The text printed by this key, not accounting for modifiers, or `None` if no text.
+                     Seems to pretty naively interpret the keycode as unicode, so numlock can return odd characters.
+        :param modifiers: A list of string modifiers, like `ctrl` or `numlock`
+        """
+        if keycode[1] == 'up':
+            self._change_to_history_text_if_available(self._command_history_index + 1)
+            return True
+        if keycode[1] == 'down':
+            self._change_to_history_text_if_available(self._command_history_index - 1)
+            return True
+        return super().keyboard_on_key_down(window, keycode, text, modifiers)
+    
+    def _change_to_history_text_if_available(self, new_index: int) -> None:
+        if new_index < -1:
+            return
+        if new_index >= len(self._command_history):
+            return
+        self._command_history_index = new_index
+        if new_index == -1:
+            self.text = ""
+            return
+        self.text = self._command_history[self._command_history_index]
+
+
 class MessageBox(Popup):
     class MessageBoxLabel(Label):
         def __init__(self, **kwargs):
@@ -421,7 +467,7 @@ class GameManager(App):
         self.commandprocessor = ctx.command_processor(ctx)
         self.icon = r"data/icon.png"
         self.json_to_kivy_parser = KivyJSONtoTextParser(ctx)
-        self.log_panels = {}
+        self.log_panels: typing.Dict[str, Widget] = {}
 
         # keep track of last used command to autofill on click
         self.last_autofillable_command = "hint"
@@ -505,7 +551,7 @@ class GameManager(App):
         info_button = Button(size=(dp(100), dp(30)), text="Command:", size_hint_x=None)
         info_button.bind(on_release=self.command_button_action)
         bottom_layout.add_widget(info_button)
-        self.textinput = TextInput(size_hint_y=None, height=dp(30), multiline=False, write_tab=False)
+        self.textinput = CommandPromptTextInput(size_hint_y=None, height=dp(30), multiline=False, write_tab=False)
         self.textinput.bind(on_text_validate=self.on_message)
         self.textinput.text_validate_unfocus = False
         bottom_layout.add_widget(self.textinput)
@@ -563,14 +609,18 @@ class GameManager(App):
 
         self.ctx.exit_event.set()
 
-    def on_message(self, textinput: TextInput):
+    def on_message(self, textinput: CommandPromptTextInput):
         try:
             input_text = textinput.text.strip()
             textinput.text = ""
+            textinput.update_history(input_text)
 
             if self.ctx.input_requests > 0:
                 self.ctx.input_requests -= 1
                 self.ctx.input_queue.put_nowait(input_text)
+            elif is_command_input(input_text):
+                self.ctx.on_ui_command(input_text)
+                self.commandprocessor(input_text)
             elif input_text:
                 self.commandprocessor(input_text)
 
@@ -683,10 +733,18 @@ class HintLog(RecycleView):
         for hint in hints:
             data.append({
                 "receiving": {"text": self.parser.handle_node({"type": "player_id", "text": hint["receiving_player"]})},
-                "item": {"text": self.parser.handle_node(
-                    {"type": "item_id", "text": hint["item"], "flags": hint["item_flags"]})},
+                "item": {"text": self.parser.handle_node({
+                    "type": "item_id",
+                    "text": hint["item"],
+                    "flags": hint["item_flags"],
+                    "player": hint["receiving_player"],
+                })},
                 "finding": {"text": self.parser.handle_node({"type": "player_id", "text": hint["finding_player"]})},
-                "location": {"text": self.parser.handle_node({"type": "location_id", "text": hint["location"]})},
+                "location": {"text": self.parser.handle_node({
+                    "type": "location_id",
+                    "text": hint["location"],
+                    "player": hint["finding_player"],
+                })},
                 "entrance": {"text": self.parser.handle_node({"type": "color" if hint["entrance"] else "text",
                                                               "color": "blue", "text": hint["entrance"]
                                                               if hint["entrance"] else "Vanilla"})},

requirements.txt

@@ -2,13 +2,13 @@ colorama>=0.4.6
 websockets>=12.0
 PyYAML>=6.0.1
 jellyfish>=1.0.3
-jinja2>=3.1.3
-schema>=0.7.5
+jinja2>=3.1.4
+schema>=0.7.7
 kivy>=2.3.0
 bsdiff4>=1.2.4
-platformdirs>=4.1.0
-certifi>=2023.11.17
-cython>=3.0.8
+platformdirs>=4.2.2
+certifi>=2024.6.2
+cython>=3.0.10
 cymem>=2.0.8
-orjson>=3.9.10
-typing_extensions>=4.7.0
+orjson>=3.10.3
+typing_extensions>=4.12.1

settings.py

@@ -643,17 +643,6 @@ class GeneratorOptions(Group):
         PLAYTHROUGH = 2
         FULL = 3
 
-    class GlitchTriforceRoom(IntEnum):
-        """
-        Glitch to Triforce room from Ganon
-        When disabled, you have to have a weapon that can hurt ganon (master sword or swordless/easy item functionality
-        + hammer) and have completed the goal required for killing ganon to be able to access the triforce room.
-        1 -> Enabled.
-        0 -> Disabled (except in no-logic)
-        """
-        OFF = 0
-        ON = 1
-
     class PlandoOptions(str):
         """
         List of options that can be plando'd. Can be combined, for example "bosses, items"
@@ -665,6 +654,14 @@ class GeneratorOptions(Group):
         OFF = 0
         ON = 1
 
+    class PanicMethod(str):
+        """
+        What to do if the current item placements appear unsolvable.
+        raise -> Raise an exception and abort.
+        swap -> Attempt to fix it by swapping prior placements around. (Default)
+        start_inventory -> Move remaining items to start_inventory, generate additional filler items to fill locations.
+        """
+
     enemizer_path: EnemizerPath = EnemizerPath("EnemizerCLI/EnemizerCLI.Core")  # + ".exe" is implied on Windows
     player_files_path: PlayerFilesPath = PlayerFilesPath("Players")
     players: Players = Players(0)
@@ -673,6 +670,7 @@ class GeneratorOptions(Group):
     spoiler: Spoiler = Spoiler(3)
     race: Race = Race(0)
     plando_options: PlandoOptions = PlandoOptions("bosses, connections, texts")
+    panic_method: PanicMethod = PanicMethod("swap")
 
 
 class SNIOptions(Group):

setup.py

@@ -21,7 +21,7 @@ from pathlib import Path
 
 # This is a bit jank. We need cx-Freeze to be able to run anything from this script, so install it
 try:
-    requirement = 'cx-Freeze>=7.0.0'
+    requirement = 'cx-Freeze==7.0.0'
     import pkg_resources
     try:
         pkg_resources.require(requirement)
@@ -190,7 +190,7 @@ if is_windows:
     c = next(component for component in components if component.script_name == "Launcher")
     exes.append(cx_Freeze.Executable(
         script=f"{c.script_name}.py",
-        target_name=f"{c.frozen_name}(DEBUG).exe",
+        target_name=f"{c.frozen_name}Debug.exe",
         icon=resolve_icon(c.icon),
     ))
 

test/cpp/CMakeLists.txt

@@ -0,0 +1,49 @@
+cmake_minimum_required(VERSION 3.5)
+project(ap-cpp-tests)
+
+enable_testing()
+
+find_package(GTest REQUIRED)
+
+if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
+    add_definitions("/source-charset:utf-8")
+    set(CMAKE_CXX_FLAGS_DEBUG "/MTd")
+    set(CMAKE_CXX_FLAGS_RELEASE "/MT")
+elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    # enable static analysis for gcc
+    add_compile_options(-fanalyzer -Werror)
+    # disable stuff that gets triggered by googletest
+    add_compile_options(-Wno-analyzer-malloc-leak)
+    # enable asan for gcc
+    add_compile_options(-fsanitize=address)
+    add_link_options(-fsanitize=address)
+endif ()
+
+add_executable(test_default)
+
+target_include_directories(test_default
+        PRIVATE
+        ${GTEST_INCLUDE_DIRS}
+)
+
+target_link_libraries(test_default
+        ${GTEST_BOTH_LIBRARIES}
+)
+
+add_test(
+        NAME test_default
+        COMMAND  test_default
+)
+
+set_property(
+        TEST test_default
+        PROPERTY ENVIRONMENT "ASAN_OPTIONS=allocator_may_return_null=1"
+)
+
+file(GLOB ITEMS *)
+foreach(item ${ITEMS})
+    if(IS_DIRECTORY ${item} AND EXISTS ${item}/CMakeLists.txt)
+        message(${item})
+        add_subdirectory(${item})
+    endif()
+endforeach()

test/cpp/README.md

@@ -0,0 +1,32 @@
+# C++ tests
+
+Test framework for C and C++ code in AP.
+
+## Adding a Test
+
+### GoogleTest
+
+Adding GoogleTests is as simple as creating a directory with
+* one or more `test_*.cpp` files that define tests using
+  [GoogleTest API](https://google.github.io/googletest/)
+* a `CMakeLists.txt` that adds the .cpp files to `test_default` target using
+  [target_sources](https://cmake.org/cmake/help/latest/command/target_sources.html)
+
+### CTest
+
+If either GoogleTest is not suitable for the test or the build flags / sources / libraries are incompatible,
+you can add another CTest to the project using add_target and add_test, similar to how it's done for `test_default`.
+
+## Running Tests
+
+* Install [CMake](https://cmake.org/).
+* Build and/or install GoogleTest and make sure
+  [CMake can find it](https://cmake.org/cmake/help/latest/module/FindGTest.html), or
+  [create a parent `CMakeLists.txt` that fetches GoogleTest](https://google.github.io/googletest/quickstart-cmake.html).
+* Enter the directory with the top-most `CMakeLists.txt` and run
+  ```sh
+  mkdir build
+  cmake -S . -B build/ -DCMAKE_BUILD_TYPE=Release
+  cmake --build build/ --config Release && \
+  ctest --test-dir build/ -C Release --output-on-failure
+  ```

test/cpp/intset/CMakeLists.txt

@@ -0,0 +1,4 @@
+target_sources(test_default
+        PRIVATE
+        ${CMAKE_CURRENT_SOURCE_DIR}/test_intset.cpp
+)

test/cpp/intset/test_intset.cpp

@@ -0,0 +1,105 @@
+#include <limits>
+#include <cstdint>
+#include <gtest/gtest.h>
+
+// uint32Set
+#define INTSET_NAME uint32Set
+#define INTSET_TYPE uint32_t
+#include "../../../intset.h"
+#undef INTSET_NAME
+#undef INTSET_TYPE
+
+// int64Set
+#define INTSET_NAME int64Set
+#define INTSET_TYPE int64_t
+#include "../../../intset.h"
+
+
+TEST(IntsetTest, ZeroBuckets)
+{
+    // trying to allocate with zero buckets has to either fail or be functioning
+    uint32Set *set = uint32Set_new(0);
+    if (!set)
+        return; // failed -> OK
+
+    EXPECT_FALSE(uint32Set_contains(set, 1));
+    EXPECT_TRUE(uint32Set_add(set, 1));
+    EXPECT_TRUE(uint32Set_contains(set, 1));
+    uint32Set_free(set);
+}
+
+TEST(IntsetTest, Duplicate)
+{
+    // adding the same number again can't fail
+    uint32Set *set = uint32Set_new(2);
+    ASSERT_TRUE(set);
+    EXPECT_TRUE(uint32Set_add(set, 0));
+    EXPECT_TRUE(uint32Set_add(set, 0));
+    EXPECT_TRUE(uint32Set_contains(set, 0));
+    uint32Set_free(set);
+}
+
+TEST(IntsetTest, SetAllocFailure)
+{
+    // try to allocate 100TB of RAM, should fail and return NULL
+    if (sizeof(size_t) < 8)
+        GTEST_SKIP() << "Alloc error not testable on 32bit";
+    int64Set *set = int64Set_new(6250000000000ULL);
+    EXPECT_FALSE(set);
+    int64Set_free(set);
+}
+
+TEST(IntsetTest, SetAllocOverflow)
+{
+    // try to overflow argument passed to malloc
+    int64Set *set = int64Set_new(std::numeric_limits<size_t>::max());
+    EXPECT_FALSE(set);
+    int64Set_free(set);
+}
+
+TEST(IntsetTest, NullFree)
+{
+    // free(NULL) should not try to free buckets
+    uint32Set_free(NULL);
+    int64Set_free(NULL);
+}
+
+TEST(IntsetTest, BucketRealloc)
+{
+    // add a couple of values to the same bucket to test growing the bucket
+    uint32Set* set = uint32Set_new(1);
+    ASSERT_TRUE(set);
+    EXPECT_FALSE(uint32Set_contains(set, 0));
+    EXPECT_TRUE(uint32Set_add(set, 0));
+    EXPECT_TRUE(uint32Set_contains(set, 0));
+    for (uint32_t i = 1; i < 32; ++i) {
+        EXPECT_TRUE(uint32Set_add(set, i));
+        EXPECT_TRUE(uint32Set_contains(set, i - 1));
+        EXPECT_TRUE(uint32Set_contains(set, i));
+        EXPECT_FALSE(uint32Set_contains(set, i + 1));
+    }
+    uint32Set_free(set);
+}
+
+TEST(IntSet, Max)
+{
+    constexpr auto n = std::numeric_limits<uint32_t>::max();
+    uint32Set *set = uint32Set_new(1);
+    ASSERT_TRUE(set);
+    EXPECT_FALSE(uint32Set_contains(set, n));
+    EXPECT_TRUE(uint32Set_add(set, n));
+    EXPECT_TRUE(uint32Set_contains(set, n));
+    uint32Set_free(set);
+}
+
+TEST(InsetTest, Negative)
+{
+    constexpr auto n = std::numeric_limits<int64_t>::min();
+    static_assert(n < 0, "n not negative");
+    int64Set *set = int64Set_new(3);
+    ASSERT_TRUE(set);
+    EXPECT_FALSE(int64Set_contains(set, n));
+    EXPECT_TRUE(int64Set_add(set, n));
+    EXPECT_TRUE(int64Set_contains(set, n));
+    int64Set_free(set);
+}

test/general/__init__.py

@@ -2,6 +2,7 @@ from argparse import Namespace
 from typing import List, Optional, Tuple, Type, Union
 
 from BaseClasses import CollectionState, Item, ItemClassification, Location, MultiWorld, Region
+from worlds import network_data_package
 from worlds.AutoWorld import World, call_all
 
 gen_steps = ("generate_early", "create_regions", "create_items", "set_rules", "generate_basic", "pre_fill")
@@ -60,6 +61,10 @@ class TestWorld(World):
     hidden = True
 
 
+# add our test world to the data package, so we can test it later
+network_data_package["games"][TestWorld.game] = TestWorld.get_data_package_data()
+
+
 def generate_test_multiworld(players: int = 1) -> MultiWorld:
     """
     Generates a multiworld using a special Test Case World class, and seed of 0.

test/general/test_client_server_interaction.py

@@ -0,0 +1,23 @@
+import unittest
+
+from Utils import get_intended_text, get_input_text_from_response
+
+
+class TestClient(unittest.TestCase):
+    def test_autofill_hint_from_fuzzy_hint(self) -> None:
+        tests = (
+            ("item", ["item1", "item2"]),          # Multiple close matches
+            ("itm", ["item1", "item21"]),          # No close match, multiple option
+            ("item", ["item1"]),                   # No close match, single option
+            ("item", ["\"item\" 'item' (item)"]),  # Testing different special characters
+        )
+
+        for input_text, possible_answers in tests:
+            item_name, usable, response = get_intended_text(input_text, possible_answers)
+            self.assertFalse(usable, "This test must be updated, it seems get_fuzzy_results behavior changed")
+
+            hint_command = get_input_text_from_response(response, "hint")
+            self.assertIsNotNone(hint_command,
+                                 "The response to fuzzy hints is no longer recognized by the hint autofill")
+            self.assertEqual(hint_command, f"!hint {item_name}",
+                             "The hint command autofilled by the response is not correct")

test/general/test_ids.py

@@ -1,27 +1,12 @@
 import unittest
 
 from Fill import distribute_items_restrictive
+from worlds import network_data_package
 from worlds.AutoWorld import AutoWorldRegister, call_all
 from . import setup_solo_multiworld
 
 
 class TestIDs(unittest.TestCase):
-    def test_unique_items(self):
-        """Tests that every game has a unique ID per item in the datapackage"""
-        known_item_ids = set()
-        for gamename, world_type in AutoWorldRegister.world_types.items():
-            current = len(known_item_ids)
-            known_item_ids |= set(world_type.item_id_to_name)
-            self.assertEqual(len(known_item_ids) - len(world_type.item_id_to_name), current)
-
-    def test_unique_locations(self):
-        """Tests that every game has a unique ID per location in the datapackage"""
-        known_location_ids = set()
-        for gamename, world_type in AutoWorldRegister.world_types.items():
-            current = len(known_location_ids)
-            known_location_ids |= set(world_type.location_id_to_name)
-            self.assertEqual(len(known_location_ids) - len(world_type.location_id_to_name), current)
-
     def test_range_items(self):
         """There are Javascript clients, which are limited to Number.MAX_SAFE_INTEGER due to 64bit float precision."""
         for gamename, world_type in AutoWorldRegister.world_types.items():
@@ -100,3 +85,4 @@ class TestIDs(unittest.TestCase):
                                           f"{loc_name} is not a valid item name for location_name_to_id")
                     self.assertIsInstance(loc_id, int,
                                           f"{loc_id} for {loc_name} should be an int")
+                self.assertEqual(datapackage["checksum"], network_data_package["games"][gamename]["checksum"])

test/general/test_items.py

@@ -64,15 +64,6 @@ class TestBase(unittest.TestCase):
                 for item in multiworld.itempool:
                     self.assertIn(item.name, world_type.item_name_to_id)
 
-    def test_item_descriptions_have_valid_names(self):
-        """Ensure all item descriptions match an item name or item group name"""
-        for game_name, world_type in AutoWorldRegister.world_types.items():
-            valid_names = world_type.item_names.union(world_type.item_name_groups)
-            for name in world_type.item_descriptions:
-                with self.subTest("Name should be valid", game=game_name, item=name):
-                    self.assertIn(name, valid_names,
-                                  "All item descriptions must match defined item names")
-
     def test_itempool_not_modified(self):
         """Test that worlds don't modify the itempool after `create_items`"""
         gen_steps = ("generate_early", "create_regions", "create_items")

test/general/test_locations.py

@@ -66,12 +66,3 @@ class TestBase(unittest.TestCase):
                         for location in locations:
                             self.assertIn(location, world_type.location_name_to_id)
                         self.assertNotIn(group_name, world_type.location_name_to_id)
-
-    def test_location_descriptions_have_valid_names(self):
-        """Ensure all location descriptions match a location name or location group name"""
-        for game_name, world_type in AutoWorldRegister.world_types.items():
-            valid_names = world_type.location_names.union(world_type.location_name_groups)
-            for name in world_type.location_descriptions:
-                with self.subTest("Name should be valid", game=game_name, location=name):
-                    self.assertIn(name, valid_names,
-                                  "All location descriptions must match defined location names")

test/general/test_player_options.py

@@ -31,7 +31,7 @@ class TestPlayerOptions(unittest.TestCase):
         self.assertEqual(new_weights["list_2"], ["string_3"])
         self.assertEqual(new_weights["list_1"], ["string", "string_2"])
         self.assertEqual(new_weights["dict_1"]["option_a"], 50)
-        self.assertEqual(new_weights["dict_1"]["option_b"], 0)
+        self.assertEqual(new_weights["dict_1"]["option_b"], 50)
         self.assertEqual(new_weights["dict_1"]["option_c"], 50)
         self.assertNotIn("option_f", new_weights["dict_2"])
         self.assertEqual(new_weights["dict_2"]["option_g"], 50)

test/hosting/__main__.py

@@ -0,0 +1,191 @@
+# A bunch of tests to verify MultiServer and custom webhost server work as expected.
+# This spawns processes and may modify your local AP, so this is not run as part of unit testing.
+# Run with `python test/hosting` instead,
+import logging
+import traceback
+from tempfile import TemporaryDirectory
+from time import sleep
+from typing import Any
+
+from test.hosting.client import Client
+from test.hosting.generate import generate_local
+from test.hosting.serve import ServeGame, LocalServeGame, WebHostServeGame
+from test.hosting.webhost import (create_room, get_app, get_multidata_for_room, set_multidata_for_room, start_room,
+                                  stop_autohost, upload_multidata)
+from test.hosting.world import copy as copy_world, delete as delete_world
+
+failure = False
+fail_fast = True
+
+
+def assert_true(condition: Any, msg: str = "") -> None:
+    global failure
+    if not condition:
+        failure = True
+        msg = f": {msg}" if msg else ""
+        raise AssertionError(f"Assertion failed{msg}")
+
+
+def assert_equal(first: Any, second: Any, msg: str = "") -> None:
+    global failure
+    if first != second:
+        failure = True
+        msg = f": {msg}" if msg else ""
+        raise AssertionError(f"Assertion failed: {first} == {second}{msg}")
+
+
+if fail_fast:
+    expect_true = assert_true
+    expect_equal = assert_equal
+else:
+    def expect_true(condition: Any, msg: str = "") -> None:
+        global failure
+        if not condition:
+            failure = True
+            tb = "".join(traceback.format_stack()[:-1])
+            msg = f": {msg}" if msg else ""
+            logging.error(f"Expectation failed{msg}\n{tb}")
+
+    def expect_equal(first: Any, second: Any, msg: str = "") -> None:
+        global failure
+        if first != second:
+            failure = True
+            tb = "".join(traceback.format_stack()[:-1])
+            msg = f": {msg}" if msg else ""
+            logging.error(f"Expectation failed {first} == {second}{msg}\n{tb}")
+
+
+if __name__ == "__main__":
+    import warnings
+    warnings.simplefilter("ignore", ResourceWarning)
+    warnings.simplefilter("ignore", UserWarning)
+
+    spacer = '=' * 80
+
+    with TemporaryDirectory() as tempdir:
+        multis = [["Clique"], ["Temp World"], ["Clique", "Temp World"]]
+        p1_games = []
+        data_paths = []
+        rooms = []
+
+        copy_world("Clique", "Temp World")
+        try:
+            for n, games in enumerate(multis, 1):
+                print(f"Generating [{n}] {', '.join(games)}")
+                multidata = generate_local(games, tempdir)
+                print(f"Generated [{n}] {', '.join(games)} as {multidata}\n")
+                p1_games.append(games[0])
+                data_paths.append(multidata)
+        finally:
+            delete_world("Temp World")
+
+        webapp = get_app(tempdir)
+        webhost_client = webapp.test_client()
+        for n, multidata in enumerate(data_paths, 1):
+            seed = upload_multidata(webhost_client, multidata)
+            room = create_room(webhost_client, seed)
+            print(f"Uploaded [{n}] {multidata} as {room}\n")
+            rooms.append(room)
+
+        print("Starting autohost")
+        from WebHostLib.autolauncher import autohost
+        try:
+            autohost(webapp.config)
+
+            host: ServeGame
+            for n, (multidata, room, game, multi_games) in enumerate(zip(data_paths, rooms, p1_games, multis), 1):
+                involved_games = {"Archipelago"} | set(multi_games)
+                for collected_items in range(3):
+                    print(f"\nTesting [{n}] {game} in {multidata} on MultiServer with {collected_items} items collected")
+                    with LocalServeGame(multidata) as host:
+                        with Client(host.address, game, "Player1") as client:
+                            local_data_packages = client.games_packages
+                            local_collected_items = len(client.checked_locations)
+                            if collected_items < 2:  # Clique only has 2 Locations
+                                client.collect_any()
+                            # TODO: Ctrl+C test here as well
+
+                    for game_name in sorted(involved_games):
+                        expect_true(game_name in local_data_packages,
+                                    f"{game_name} missing from MultiServer datap ackage")
+                        expect_true("item_name_groups" not in local_data_packages.get(game_name, {}),
+                                    f"item_name_groups are not supposed to be in MultiServer data for {game_name}")
+                        expect_true("location_name_groups" not in local_data_packages.get(game_name, {}),
+                                    f"location_name_groups are not supposed to be in MultiServer data for {game_name}")
+                    for game_name in local_data_packages:
+                        expect_true(game_name in involved_games,
+                                    f"Received unexpected extra data package for {game_name} from MultiServer")
+                    assert_equal(local_collected_items, collected_items,
+                                 "MultiServer did not load or save correctly")
+
+                    print(f"\nTesting [{n}] {game} in {multidata} on customserver with {collected_items} items collected")
+                    prev_host_adr: str
+                    with WebHostServeGame(webhost_client, room) as host:
+                        prev_host_adr = host.address
+                        with Client(host.address, game, "Player1") as client:
+                            web_data_packages = client.games_packages
+                            web_collected_items = len(client.checked_locations)
+                            if collected_items < 2:  # Clique only has 2 Locations
+                                client.collect_any()
+                            if collected_items == 1:
+                                sleep(1)  # wait for the server to collect the item
+                                stop_autohost(True)  # simulate Ctrl+C
+                                sleep(3)
+                                autohost(webapp.config)  # this will spin the room right up again
+                                sleep(1)  # make log less annoying
+                                # if saving failed, the next iteration will fail below
+
+                    # verify server shut down
+                    try:
+                        with Client(prev_host_adr, game, "Player1") as client:
+                            assert_true(False, "Server did not shut down")
+                    except ConnectionError:
+                        pass
+
+                    for game_name in sorted(involved_games):
+                        expect_true(game_name in web_data_packages,
+                                    f"{game_name} missing from customserver data package")
+                        expect_true("item_name_groups" not in web_data_packages.get(game_name, {}),
+                                    f"item_name_groups are not supposed to be in customserver data for {game_name}")
+                        expect_true("location_name_groups" not in web_data_packages.get(game_name, {}),
+                                    f"location_name_groups are not supposed to be in customserver data for {game_name}")
+                    for game_name in web_data_packages:
+                        expect_true(game_name in involved_games,
+                                    f"Received unexpected extra data package for {game_name} from customserver")
+                    assert_equal(web_collected_items, collected_items,
+                                 "customserver did not load or save correctly during/after "
+                                 + ("Ctrl+C" if collected_items == 2 else "/exit"))
+
+                    # compare customserver to MultiServer
+                    expect_equal(local_data_packages, web_data_packages,
+                                 "customserver datapackage differs from MultiServer")
+
+            sleep(5.5)  # make sure all tasks actually stopped
+
+            # raise an exception in customserver and verify the save doesn't get destroyed
+            # local variables room is the last room's id here
+            old_data = get_multidata_for_room(webhost_client, room)
+            print(f"Destroying multidata for {room}")
+            set_multidata_for_room(webhost_client, room, bytes([0]))
+            try:
+                start_room(webhost_client, room, timeout=7)
+            except TimeoutError:
+                pass
+            else:
+                assert_true(False, "Room started with destroyed multidata")
+            print(f"Restoring multidata for {room}")
+            set_multidata_for_room(webhost_client, room, old_data)
+            with WebHostServeGame(webhost_client, room) as host:
+                with Client(host.address, game, "Player1") as client:
+                    assert_equal(len(client.checked_locations), 2,
+                                 "Save was destroyed during exception in customserver")
+                    print("Save file is not busted 🥳")
+
+        finally:
+            print("Stopping autohost")
+            stop_autohost(False)
+
+    if failure:
+        print("Some tests failed")
+        exit(1)
+    exit(0)

test/hosting/client.py

@@ -0,0 +1,110 @@
+import json
+import sys
+from typing import Any, Collection, Dict, Iterable, Optional
+from websockets import ConnectionClosed
+from websockets.sync.client import connect, ClientConnection
+from threading import Thread
+
+
+__all__ = [
+    "Client"
+]
+
+
+class Client:
+    """Incomplete, minimalistic sync test client for AP network protocol"""
+
+    recv_timeout = 1.0
+
+    host: str
+    game: str
+    slot: str
+    password: Optional[str]
+
+    _ws: Optional[ClientConnection]
+
+    games: Iterable[str]
+    data_package_checksums: Dict[str, Any]
+    games_packages: Dict[str, Any]
+    missing_locations: Collection[int]
+    checked_locations: Collection[int]
+
+    def __init__(self, host: str, game: str, slot: str, password: Optional[str] = None) -> None:
+        self.host = host
+        self.game = game
+        self.slot = slot
+        self.password = password
+        self._ws = None
+        self.games = []
+        self.data_package_checksums = {}
+        self.games_packages = {}
+        self.missing_locations = []
+        self.checked_locations = []
+
+    def __enter__(self) -> "Client":
+        try:
+            self.connect()
+        except BaseException:
+            self.__exit__(*sys.exc_info())
+            raise
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:  # type: ignore
+        self.close()
+
+    def _poll(self) -> None:
+        assert self._ws
+        try:
+            while True:
+                self._ws.recv()
+        except (TimeoutError, ConnectionClosed, KeyboardInterrupt, SystemExit):
+            pass
+
+    def connect(self) -> None:
+        self._ws = connect(f"ws://{self.host}")
+        room_info = json.loads(self._ws.recv(self.recv_timeout))[0]
+        self.games = sorted(room_info["games"])
+        self.data_package_checksums = room_info["datapackage_checksums"]
+        self._ws.send(json.dumps([{
+            "cmd": "GetDataPackage",
+            "games": list(self.games),
+        }]))
+        data_package_msg = json.loads(self._ws.recv(self.recv_timeout))[0]
+        self.games_packages = data_package_msg["data"]["games"]
+        self._ws.send(json.dumps([{
+            "cmd": "Connect",
+            "game": self.game,
+            "name": self.slot,
+            "password": self.password,
+            "uuid": "",
+            "version": {
+                "class": "Version",
+                "major": 0,
+                "minor": 4,
+                "build": 6,
+            },
+            "items_handling": 0,
+            "tags": [],
+            "slot_data": False,
+        }]))
+        connect_result_msg = json.loads(self._ws.recv(self.recv_timeout))[0]
+        if connect_result_msg["cmd"] != "Connected":
+            raise ConnectionError(", ".join(connect_result_msg.get("errors", [connect_result_msg["cmd"]])))
+        self.missing_locations = connect_result_msg["missing_locations"]
+        self.checked_locations = connect_result_msg["checked_locations"]
+
+    def close(self) -> None:
+        if self._ws:
+            Thread(target=self._poll).start()
+            self._ws.close()
+
+    def collect(self, locations: Iterable[int]) -> None:
+        if not self._ws:
+            raise ValueError("Not connected")
+        self._ws.send(json.dumps([{
+            "cmd": "LocationChecks",
+            "locations": locations,
+        }]))
+
+    def collect_any(self) -> None:
+        self.collect([next(iter(self.missing_locations))])

test/hosting/generate.py

@@ -0,0 +1,75 @@
+import json
+import sys
+import warnings
+from pathlib import Path
+from typing import Iterable, Union, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from multiprocessing.managers import ListProxy  # noqa
+
+__all__ = [
+    "generate_local",
+]
+
+
+def _generate_local_inner(games: Iterable[str],
+                          dest: Union[Path, str],
+                          results: "ListProxy[Union[Path, BaseException]]") -> None:
+    original_argv = sys.argv
+    warnings.simplefilter("ignore")
+    try:
+        from tempfile import TemporaryDirectory
+
+        if not isinstance(dest, Path):
+            dest = Path(dest)
+
+        with TemporaryDirectory() as players_dir:
+            with TemporaryDirectory() as output_dir:
+                import Generate
+
+                for n, game in enumerate(games, 1):
+                    player_path = Path(players_dir) / f"{n}.yaml"
+                    with open(player_path, "w", encoding="utf-8") as f:
+                        f.write(json.dumps({
+                            "name": f"Player{n}",
+                            "game": game,
+                            game: {"hard_mode": "true"},
+                            "description": f"generate_local slot {n} ('Player{n}'): {game}",
+                        }))
+
+                # this is basically copied from test/programs/test_generate.py
+                # uses a reproducible seed that is different for each set of games
+                sys.argv = [sys.argv[0], "--seed", str(hash(tuple(games))),
+                            "--player_files_path", players_dir,
+                            "--outputpath", output_dir]
+                Generate.main()
+                output_files = list(Path(output_dir).glob('*.zip'))
+                assert len(output_files) == 1
+                final_file = dest / output_files[0].name
+                output_files[0].rename(final_file)
+                results.append(final_file)
+    except BaseException as e:
+        results.append(e)
+        raise e
+    finally:
+        sys.argv = original_argv
+
+
+def generate_local(games: Iterable[str], dest: Union[Path, str]) -> Path:
+    from multiprocessing import Manager, Process, set_start_method
+
+    try:
+        set_start_method("spawn")
+    except RuntimeError:
+        pass
+
+    manager = Manager()
+    results: "ListProxy[Union[Path, Exception]]" = manager.list()
+
+    p = Process(target=_generate_local_inner, args=(games, dest, results))
+    p.start()
+    p.join()
+    result = results[0]
+    if isinstance(result, BaseException):
+        raise Exception("Could not generate multiworld") from result
+    return result

test/hosting/serve.py

@@ -0,0 +1,115 @@
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from threading import Event
+    from werkzeug.test import Client as FlaskClient
+
+__all__ = [
+    "ServeGame",
+    "LocalServeGame",
+    "WebHostServeGame",
+]
+
+
+class ServeGame:
+    address: str
+
+
+def _launch_multiserver(multidata: Path, ready: "Event", stop: "Event") -> None:
+    import os
+    import warnings
+
+    original_argv = sys.argv
+    original_stdin = sys.stdin
+    warnings.simplefilter("ignore")
+    try:
+        import asyncio
+        from MultiServer import main, parse_args
+
+        sys.argv = [sys.argv[0], str(multidata), "--host", "127.0.0.1"]
+        r, w = os.pipe()
+        sys.stdin = os.fdopen(r, "r")
+
+        async def set_ready() -> None:
+            await asyncio.sleep(.01)  # switch back to other task once more
+            ready.set()  # server should be up, set ready state
+
+        async def wait_stop() -> None:
+            await asyncio.get_event_loop().run_in_executor(None, stop.wait)
+            os.fdopen(w, "w").write("/exit")
+
+        async def run() -> None:
+            # this will run main() until first await, then switch to set_ready()
+            await asyncio.gather(
+                main(parse_args()),
+                set_ready(),
+                wait_stop(),
+            )
+
+        asyncio.run(run())
+    finally:
+        sys.argv = original_argv
+        sys.stdin = original_stdin
+
+
+class LocalServeGame(ServeGame):
+    from multiprocessing import Process
+
+    _multidata: Path
+    _proc: Process
+    _stop: "Event"
+
+    def __init__(self, multidata: Path) -> None:
+        self.address = ""
+        self._multidata = multidata
+
+    def __enter__(self) -> "LocalServeGame":
+        from multiprocessing import Manager, Process, set_start_method
+
+        try:
+            set_start_method("spawn")
+        except RuntimeError:
+            pass
+
+        manager = Manager()
+        ready: "Event" = manager.Event()
+        self._stop = manager.Event()
+
+        self._proc = Process(target=_launch_multiserver, args=(self._multidata, ready, self._stop))
+        try:
+            self._proc.start()
+            ready.wait(30)
+            self.address = "localhost:38281"
+            return self
+        except BaseException:
+            self.__exit__(*sys.exc_info())
+            raise
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:  # type: ignore
+        try:
+            self._stop.set()
+            self._proc.join(30)
+        except TimeoutError:
+            self._proc.terminate()
+            self._proc.join()
+
+
+class WebHostServeGame(ServeGame):
+    _client: "FlaskClient"
+    _room: str
+
+    def __init__(self, app_client: "FlaskClient", room: str) -> None:
+        self.address = ""
+        self._client = app_client
+        self._room = room
+
+    def __enter__(self) -> "WebHostServeGame":
+        from .webhost import start_room
+        self.address = start_room(self._client, self._room)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:  # type: ignore
+        from .webhost import stop_room
+        stop_room(self._client, self._room, timeout=30)

test/hosting/webhost.py

@@ -0,0 +1,208 @@
+import re
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional, cast
+
+if TYPE_CHECKING:
+    from flask import Flask
+    from werkzeug.test import Client as FlaskClient
+
+__all__ = [
+    "get_app",
+    "upload_multidata",
+    "create_room",
+    "start_room",
+    "stop_room",
+    "set_room_timeout",
+    "get_multidata_for_room",
+    "set_multidata_for_room",
+    "stop_autohost",
+]
+
+
+def get_app(tempdir: str) -> "Flask":
+    from WebHostLib import app as raw_app
+    from WebHost import get_app
+    raw_app.config["PONY"] = {
+        "provider": "sqlite",
+        "filename": str(Path(tempdir) / "host.db"),
+        "create_db": True,
+    }
+    raw_app.config.update({
+        "TESTING": True,
+        "HOST_ADDRESS": "localhost",
+        "HOSTERS": 1,
+    })
+    return get_app()
+
+
+def upload_multidata(app_client: "FlaskClient", multidata: Path) -> str:
+    response = app_client.post("/uploads", data={
+        "file": multidata.open("rb"),
+    })
+    assert response.status_code < 400, f"Upload of {multidata} failed: status {response.status_code}"
+    assert "Location" in response.headers, f"Upload of {multidata} failed: no redirect"
+    location = response.headers["Location"]
+    assert isinstance(location, str)
+    assert location.startswith("/seed/"), f"Upload of {multidata} failed: unexpected redirect"
+    return location[6:]
+
+
+def create_room(app_client: "FlaskClient", seed: str, auto_start: bool = False) -> str:
+    response = app_client.get(f"/new_room/{seed}")
+    assert response.status_code < 400, f"Creating room for {seed} failed: status {response.status_code}"
+    assert "Location" in response.headers, f"Creating room for {seed} failed: no redirect"
+    location = response.headers["Location"]
+    assert isinstance(location, str)
+    assert location.startswith("/room/"), f"Creating room for {seed} failed: unexpected redirect"
+    room_id = location[6:]
+
+    if not auto_start:
+        # by default, creating a room will auto-start it, so we update last activity here
+        stop_room(app_client, room_id, simulate_idle=False)
+
+    return room_id
+
+
+def start_room(app_client: "FlaskClient", room_id: str, timeout: float = 30) -> str:
+    from time import sleep
+
+    import pony.orm
+
+    poll_interval = .2
+
+    print(f"Starting room {room_id}")
+    no_timeout = timeout <= 0
+    while no_timeout or timeout > 0:
+        try:
+            response = app_client.get(f"/room/{room_id}")
+        except pony.orm.core.OptimisticCheckError:
+            # hoster wrote to room during our transaction
+            continue
+
+        assert response.status_code == 200, f"Starting room for {room_id} failed: status {response.status_code}"
+        match = re.search(r"/connect ([\w:.\-]+)", response.text)
+        if match:
+            return match[1]
+        timeout -= poll_interval
+        sleep(poll_interval)
+    raise TimeoutError("Room did not start")
+
+
+def stop_room(app_client: "FlaskClient",
+              room_id: str,
+              timeout: Optional[float] = None,
+              simulate_idle: bool = True) -> None:
+    from datetime import datetime, timedelta
+    from time import sleep
+
+    from pony.orm import db_session
+
+    from WebHostLib.models import Command, Room
+    from WebHostLib import app
+
+    poll_interval = 2
+
+    print(f"Stopping room {room_id}")
+    room_uuid = app.url_map.converters["suuid"].to_python(None, room_id)  # type: ignore[arg-type]
+
+    if timeout is not None:
+        sleep(.1)  # should not be required, but other things might use threading
+
+    with db_session:
+        room: Room = Room.get(id=room_uuid)
+        if simulate_idle:
+            new_last_activity = datetime.utcnow() - timedelta(seconds=room.timeout + 5)
+        else:
+            new_last_activity = datetime.utcnow() - timedelta(days=3)
+        room.last_activity = new_last_activity
+        address = f"localhost:{room.last_port}" if room.last_port > 0 else None
+        if address:
+            original_timeout = room.timeout
+            room.timeout = 1  # avoid spinning it up again
+            Command(room=room, commandtext="/exit")
+
+    try:
+        if address and timeout is not None:
+            print("waiting for shutdown")
+            import socket
+            host_str, port_str = tuple(address.split(":"))
+            address_tuple = host_str, int(port_str)
+
+            no_timeout = timeout <= 0
+            while no_timeout or timeout > 0:
+                s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+                try:
+                    s.connect(address_tuple)
+                    s.close()
+                except ConnectionRefusedError:
+                    return
+                sleep(poll_interval)
+                timeout -= poll_interval
+
+            raise TimeoutError("Room did not stop")
+    finally:
+        with db_session:
+            room = Room.get(id=room_uuid)
+            room.last_port = 0  # easier to detect when the host is up this way
+            if address:
+                room.timeout = original_timeout
+                room.last_activity = new_last_activity
+                print("timeout restored")
+
+
+def set_room_timeout(room_id: str, timeout: float) -> None:
+    from pony.orm import db_session
+
+    from WebHostLib.models import Room
+    from WebHostLib import app
+
+    room_uuid = app.url_map.converters["suuid"].to_python(None, room_id)  # type: ignore[arg-type]
+    with db_session:
+        room: Room = Room.get(id=room_uuid)
+        room.timeout = timeout
+
+
+def get_multidata_for_room(webhost_client: "FlaskClient", room_id: str) -> bytes:
+    from pony.orm import db_session
+
+    from WebHostLib.models import Room
+    from WebHostLib import app
+
+    room_uuid = app.url_map.converters["suuid"].to_python(None, room_id)  # type: ignore[arg-type]
+    with db_session:
+        room: Room = Room.get(id=room_uuid)
+        return cast(bytes, room.seed.multidata)
+
+
+def set_multidata_for_room(webhost_client: "FlaskClient", room_id: str, data: bytes) -> None:
+    from pony.orm import db_session
+
+    from WebHostLib.models import Room
+    from WebHostLib import app
+
+    room_uuid = app.url_map.converters["suuid"].to_python(None, room_id)  # type: ignore[arg-type]
+    with db_session:
+        room: Room = Room.get(id=room_uuid)
+        room.seed.multidata = data
+
+
+def stop_autohost(graceful: bool = True) -> None:
+    import os
+    import signal
+
+    import multiprocessing
+
+    from WebHostLib.autolauncher import stop
+
+    stop()
+    proc: multiprocessing.process.BaseProcess
+    for proc in filter(lambda child: child.name.startswith("MultiHoster"), multiprocessing.active_children()):
+        if graceful and proc.pid:
+            os.kill(proc.pid, getattr(signal, "CTRL_C_EVENT", signal.SIGINT))
+        else:
+            proc.kill()
+        try:
+            proc.join(30)
+        except TimeoutError:
+            proc.kill()
+            proc.join()

test/hosting/world.py

@@ -0,0 +1,42 @@
+import re
+import shutil
+from pathlib import Path
+from typing import Dict
+
+
+__all__ = ["copy", "delete"]
+
+
+_new_worlds: Dict[str, str] = {}
+
+
+def copy(src: str, dst: str) -> None:
+    from Utils import get_file_safe_name
+    from worlds import AutoWorldRegister
+
+    assert dst not in _new_worlds, "World already created"
+    if '"' in dst or "\\" in dst:  # easier to reject than to escape
+        raise ValueError(f"Unsupported symbols in {dst}")
+    dst_folder_name = get_file_safe_name(dst.lower())
+    src_cls = AutoWorldRegister.world_types[src]
+    src_folder = Path(src_cls.__file__).parent
+    worlds_folder = src_folder.parent
+    if (not src_cls.__file__.endswith("__init__.py") or not src_folder.is_dir()
+            or not (worlds_folder / "generic").is_dir()):
+        raise ValueError(f"Unsupported layout for copy_world from {src}")
+    dst_folder = worlds_folder / dst_folder_name
+    if dst_folder.is_dir():
+        raise ValueError(f"Destination {dst_folder} already exists")
+    shutil.copytree(src_folder, dst_folder)
+    _new_worlds[dst] = str(dst_folder)
+    with open(dst_folder / "__init__.py", "r", encoding="utf-8-sig") as f:
+        contents = f.read()
+    contents = re.sub(r'game\s*=\s*[\'"]' + re.escape(src) + r'[\'"]', f'game = "{dst}"', contents)
+    with open(dst_folder / "__init__.py", "w", encoding="utf-8") as f:
+        f.write(contents)
+
+
+def delete(name: str) -> None:
+    assert name in _new_worlds, "World not created by this script"
+    shutil.rmtree(_new_worlds[name])
+    del _new_worlds[name]

test/netutils/test_location_store.py

@@ -1,4 +1,5 @@
 # Tests for _speedups.LocationStore and NetUtils._LocationStore
+import os
 import typing
 import unittest
 import warnings
@@ -7,6 +8,8 @@ from NetUtils import LocationStore, _LocationStore
 State = typing.Dict[typing.Tuple[int, int], typing.Set[int]]
 RawLocations = typing.Dict[int, typing.Dict[int, typing.Tuple[int, int, int]]]
 
+ci = bool(os.environ.get("CI"))  # always set in GitHub actions
+
 sample_data: RawLocations = {
     1: {
         11: (21, 2, 7),
@@ -24,6 +27,9 @@ sample_data: RawLocations = {
     3: {
         9: (99, 4, 0),
     },
+    5: {
+        9: (99, 5, 0),
+    }
 }
 
 empty_state: State = {
@@ -45,14 +51,14 @@ class Base:
         store: typing.Union[LocationStore, _LocationStore]
 
         def test_len(self) -> None:
-            self.assertEqual(len(self.store), 4)
+            self.assertEqual(len(self.store), 5)
             self.assertEqual(len(self.store[1]), 3)
 
         def test_key_error(self) -> None:
             with self.assertRaises(KeyError):
                 _ = self.store[0]
             with self.assertRaises(KeyError):
-                _ = self.store[5]
+                _ = self.store[6]
             locations = self.store[1]  # no Exception
             with self.assertRaises(KeyError):
                 _ = locations[7]
@@ -71,7 +77,7 @@ class Base:
             self.assertEqual(self.store[1].get(10, (None, None, None)), (None, None, None))
 
         def test_iter(self) -> None:
-            self.assertEqual(sorted(self.store), [1, 2, 3, 4])
+            self.assertEqual(sorted(self.store), [1, 2, 3, 4, 5])
             self.assertEqual(len(self.store), len(sample_data))
             self.assertEqual(list(self.store[1]), [11, 12, 13])
             self.assertEqual(len(self.store[1]), len(sample_data[1]))
@@ -85,13 +91,26 @@ class Base:
             self.assertEqual(sorted(self.store[1].items())[0][1], self.store[1][11])
 
         def test_find_item(self) -> None:
+            # empty player set
             self.assertEqual(sorted(self.store.find_item(set(), 99)), [])
+            # no such player, single
+            self.assertEqual(sorted(self.store.find_item({6}, 99)), [])
+            # no such player, set
+            self.assertEqual(sorted(self.store.find_item({7, 8, 9}, 99)), [])
+            # no such item
             self.assertEqual(sorted(self.store.find_item({3}, 1)), [])
-            self.assertEqual(sorted(self.store.find_item({5}, 99)), [])
+            # valid matches
             self.assertEqual(sorted(self.store.find_item({3}, 99)),
                              [(4, 9, 99, 3, 0)])
             self.assertEqual(sorted(self.store.find_item({3, 4}, 99)),
                              [(3, 9, 99, 4, 0), (4, 9, 99, 3, 0)])
+            self.assertEqual(sorted(self.store.find_item({2, 3, 4}, 99)),
+                             [(3, 9, 99, 4, 0), (4, 9, 99, 3, 0)])
+            # test hash collision in set
+            self.assertEqual(sorted(self.store.find_item({3, 5}, 99)),
+                             [(4, 9, 99, 3, 0), (5, 9, 99, 5, 0)])
+            self.assertEqual(sorted(self.store.find_item(set(range(2048)), 13)),
+                             [(1, 13, 13, 1, 0)])
 
         def test_get_for_player(self) -> None:
             self.assertEqual(self.store.get_for_player(3), {4: {9}})
@@ -196,18 +215,20 @@ class TestPurePythonLocationStoreConstructor(Base.TestLocationStoreConstructor):
         super().setUp()
 
 
-@unittest.skipIf(LocationStore is _LocationStore, "_speedups not available")
+@unittest.skipIf(LocationStore is _LocationStore and not ci, "_speedups not available")
 class TestSpeedupsLocationStore(Base.TestLocationStore):
     """Run base method tests for cython implementation."""
     def setUp(self) -> None:
+        self.assertFalse(LocationStore is _LocationStore, "Failed to load _speedups")
         self.store = LocationStore(sample_data)
         super().setUp()
 
 
-@unittest.skipIf(LocationStore is _LocationStore, "_speedups not available")
+@unittest.skipIf(LocationStore is _LocationStore and not ci, "_speedups not available")
 class TestSpeedupsLocationStoreConstructor(Base.TestLocationStoreConstructor):
     """Run base constructor tests and tests the additional constraints for cython implementation."""
     def setUp(self) -> None:
+        self.assertFalse(LocationStore is _LocationStore, "Failed to load _speedups")
         self.type = LocationStore
         super().setUp()
 

test/options/test_option_classes.py

@@ -0,0 +1,67 @@
+import unittest
+
+from Options import Choice, DefaultOnToggle, Toggle
+
+
+class TestNumericOptions(unittest.TestCase):
+    def test_numeric_option(self) -> None:
+        """Tests the initialization and equivalency comparisons of the base Numeric Option class."""
+        class TestChoice(Choice):
+            option_zero = 0
+            option_one = 1
+            option_two = 2
+            alias_three = 1
+            non_option_attr = 2
+
+        class TestToggle(Toggle):
+            pass
+
+        class TestDefaultOnToggle(DefaultOnToggle):
+            pass
+
+        with self.subTest("choice"):
+            choice_option_default = TestChoice.from_any(TestChoice.default)
+            choice_option_string = TestChoice.from_any("one")
+            choice_option_int = TestChoice.from_any(2)
+            choice_option_alias = TestChoice.from_any("three")
+            choice_option_attr = TestChoice.from_any(TestChoice.option_two)
+            
+            self.assertEqual(choice_option_default, TestChoice.option_zero,
+                "assigning default didn't match default value")
+            self.assertEqual(choice_option_string, "one")
+            self.assertEqual(choice_option_int, 2)
+            self.assertEqual(choice_option_alias, TestChoice.alias_three)
+            self.assertEqual(choice_option_attr, TestChoice.non_option_attr)
+
+            self.assertRaises(KeyError, TestChoice.from_any, "four")
+            
+            self.assertIn(choice_option_int, [1, 2, 3])
+            self.assertIn(choice_option_int, {2})
+            self.assertIn(choice_option_int, (2,))
+
+            self.assertIn(choice_option_string, ["one", "two", "three"])
+            # this fails since the hash is derived from the value
+            self.assertNotIn(choice_option_string, {"one"})
+            self.assertIn(choice_option_string, ("one",))
+
+        with self.subTest("toggle"):
+            toggle_default = TestToggle.from_any(TestToggle.default)
+            toggle_string = TestToggle.from_any("false")
+            toggle_int = TestToggle.from_any(0)
+            toggle_alias = TestToggle.from_any("off")
+
+            self.assertFalse(toggle_default)
+            self.assertFalse(toggle_string)
+            self.assertFalse(toggle_int)
+            self.assertFalse(toggle_alias)
+
+        with self.subTest("on toggle"):
+            toggle_default = TestDefaultOnToggle.from_any(TestDefaultOnToggle.default)
+            toggle_string = TestDefaultOnToggle.from_any("true")
+            toggle_int = TestDefaultOnToggle.from_any(1)
+            toggle_alias = TestDefaultOnToggle.from_any("on")
+
+            self.assertTrue(toggle_default)
+            self.assertTrue(toggle_string)
+            self.assertTrue(toggle_int)
+            self.assertTrue(toggle_alias)

test/programs/test_common_client.py

@@ -0,0 +1,106 @@
+import unittest
+
+import NetUtils
+from CommonClient import CommonContext
+
+
+class TestCommonContext(unittest.IsolatedAsyncioTestCase):
+    async def asyncSetUp(self):
+        self.ctx = CommonContext()
+        self.ctx.slot = 1  # Pretend we're player 1 for this.
+        self.ctx.slot_info.update({
+            1: NetUtils.NetworkSlot("Player 1", "__TestGame1", NetUtils.SlotType.player),
+            2: NetUtils.NetworkSlot("Player 2", "__TestGame1", NetUtils.SlotType.player),
+            3: NetUtils.NetworkSlot("Player 3", "__TestGame2", NetUtils.SlotType.player),
+        })
+        self.ctx.consume_players_package([
+            NetUtils.NetworkPlayer(1, 1, "Player 1", "Player 1"),
+            NetUtils.NetworkPlayer(1, 2, "Player 2", "Player 2"),
+            NetUtils.NetworkPlayer(1, 3, "Player 3", "Player 3"),
+        ])
+        # Using IDs outside the "safe range" for testing purposes only. If this fails unit tests, it's because
+        # another world is not following the spec for allowed ID ranges.
+        self.ctx.update_data_package({
+            "games": {
+                "__TestGame1": {
+                    "location_name_to_id": {
+                        "Test Location 1 - Safe": 2**54 + 1,
+                        "Test Location 2 - Duplicate": 2**54 + 2,
+                    },
+                    "item_name_to_id": {
+                        "Test Item 1 - Safe": 2**54 + 1,
+                        "Test Item 2 - Duplicate": 2**54 + 2,
+                    },
+                },
+                "__TestGame2": {
+                    "location_name_to_id": {
+                        "Test Location 3 - Duplicate": 2**54 + 2,
+                    },
+                    "item_name_to_id": {
+                        "Test Item 3 - Duplicate": 2**54 + 2,
+                    },
+                },
+            },
+        })
+
+    async def test_archipelago_datapackage_lookups_exist(self):
+        assert "Archipelago" in self.ctx.item_names, "Archipelago item names entry does not exist"
+        assert "Archipelago" in self.ctx.location_names, "Archipelago location names entry does not exist"
+
+    async def test_implicit_name_lookups(self):
+        # Items
+        assert self.ctx.item_names[2**54 + 1] == "Test Item 1 - Safe"
+        assert self.ctx.item_names[2**54 + 3] == f"Unknown item (ID: {2**54+3})"
+        assert self.ctx.item_names[-1] == "Nothing"
+
+        # Locations
+        assert self.ctx.location_names[2**54 + 1] == "Test Location 1 - Safe"
+        assert self.ctx.location_names[2**54 + 3] == f"Unknown location (ID: {2**54+3})"
+        assert self.ctx.location_names[-1] == "Cheat Console"
+
+    async def test_explicit_name_lookups(self):
+        # Items
+        assert self.ctx.item_names["__TestGame1"][2**54+1] == "Test Item 1 - Safe"
+        assert self.ctx.item_names["__TestGame1"][2**54+2] == "Test Item 2 - Duplicate"
+        assert self.ctx.item_names["__TestGame1"][2**54+3] == f"Unknown item (ID: {2**54+3})"
+        assert self.ctx.item_names["__TestGame1"][-1] == "Nothing"
+        assert self.ctx.item_names["__TestGame2"][2**54+1] == f"Unknown item (ID: {2**54+1})"
+        assert self.ctx.item_names["__TestGame2"][2**54+2] == "Test Item 3 - Duplicate"
+        assert self.ctx.item_names["__TestGame2"][2**54+3] == f"Unknown item (ID: {2**54+3})"
+        assert self.ctx.item_names["__TestGame2"][-1] == "Nothing"
+
+        # Locations
+        assert self.ctx.location_names["__TestGame1"][2**54+1] == "Test Location 1 - Safe"
+        assert self.ctx.location_names["__TestGame1"][2**54+2] == "Test Location 2 - Duplicate"
+        assert self.ctx.location_names["__TestGame1"][2**54+3] == f"Unknown location (ID: {2**54+3})"
+        assert self.ctx.location_names["__TestGame1"][-1] == "Cheat Console"
+        assert self.ctx.location_names["__TestGame2"][2**54+1] == f"Unknown location (ID: {2**54+1})"
+        assert self.ctx.location_names["__TestGame2"][2**54+2] == "Test Location 3 - Duplicate"
+        assert self.ctx.location_names["__TestGame2"][2**54+3] == f"Unknown location (ID: {2**54+3})"
+        assert self.ctx.location_names["__TestGame2"][-1] == "Cheat Console"
+
+    async def test_lookup_helper_functions(self):
+        # Checking own slot.
+        assert self.ctx.item_names.lookup_in_slot(2 ** 54 + 1) == "Test Item 1 - Safe"
+        assert self.ctx.item_names.lookup_in_slot(2 ** 54 + 2) == "Test Item 2 - Duplicate"
+        assert self.ctx.item_names.lookup_in_slot(2 ** 54 + 3) == f"Unknown item (ID: {2 ** 54 + 3})"
+        assert self.ctx.item_names.lookup_in_slot(-1) == f"Nothing"
+
+        # Checking others' slots.
+        assert self.ctx.item_names.lookup_in_slot(2 ** 54 + 1, 2) == "Test Item 1 - Safe"
+        assert self.ctx.item_names.lookup_in_slot(2 ** 54 + 2, 2) == "Test Item 2 - Duplicate"
+        assert self.ctx.item_names.lookup_in_slot(2 ** 54 + 1, 3) == f"Unknown item (ID: {2 ** 54 + 1})"
+        assert self.ctx.item_names.lookup_in_slot(2 ** 54 + 2, 3) == "Test Item 3 - Duplicate"
+
+        # Checking by game.
+        assert self.ctx.item_names.lookup_in_game(2 ** 54 + 1, "__TestGame1") == "Test Item 1 - Safe"
+        assert self.ctx.item_names.lookup_in_game(2 ** 54 + 2, "__TestGame1") == "Test Item 2 - Duplicate"
+        assert self.ctx.item_names.lookup_in_game(2 ** 54 + 3, "__TestGame1") == f"Unknown item (ID: {2 ** 54 + 3})"
+        assert self.ctx.item_names.lookup_in_game(2 ** 54 + 1, "__TestGame2") == f"Unknown item (ID: {2 ** 54 + 1})"
+        assert self.ctx.item_names.lookup_in_game(2 ** 54 + 2, "__TestGame2") == "Test Item 3 - Duplicate"
+
+        # Checking with Archipelago ids are valid in any game package.
+        assert self.ctx.item_names.lookup_in_slot(-1, 2) == "Nothing"
+        assert self.ctx.item_names.lookup_in_slot(-1, 3) == "Nothing"
+        assert self.ctx.item_names.lookup_in_game(-1, "__TestGame1") == "Nothing"
+        assert self.ctx.item_names.lookup_in_game(-1, "__TestGame2") == "Nothing"

test/webhost/test_descriptions.py

@@ -0,0 +1,23 @@
+import unittest
+
+from worlds.AutoWorld import AutoWorldRegister
+
+
+class TestWebDescriptions(unittest.TestCase):
+    def test_item_descriptions_have_valid_names(self) -> None:
+        """Ensure all item descriptions match an item name or item group name"""
+        for game_name, world_type in AutoWorldRegister.world_types.items():
+            valid_names = world_type.item_names.union(world_type.item_name_groups)
+            for name in world_type.web.item_descriptions:
+                with self.subTest("Name should be valid", game=game_name, item=name):
+                    self.assertIn(name, valid_names,
+                                  "All item descriptions must match defined item names")
+
+    def test_location_descriptions_have_valid_names(self) -> None:
+        """Ensure all location descriptions match a location name or location group name"""
+        for game_name, world_type in AutoWorldRegister.world_types.items():
+            valid_names = world_type.location_names.union(world_type.location_name_groups)
+            for name in world_type.web.location_descriptions:
+                with self.subTest("Name should be valid", game=game_name, location=name):
+                    self.assertIn(name, valid_names,
+                                  "All location descriptions must match defined location names")

test/webhost/test_option_presets.py

@@ -1,7 +1,7 @@
 import unittest
 
 from worlds import AutoWorldRegister
-from Options import Choice, NamedRange, Toggle, Range
+from Options import ItemDict, NamedRange, NumericOption, OptionList, OptionSet
 
 
 class TestOptionPresets(unittest.TestCase):
@@ -14,7 +14,7 @@ class TestOptionPresets(unittest.TestCase):
                     with self.subTest(game=game_name, preset=preset_name, option=option_name):
                         try:
                             option = world_type.options_dataclass.type_hints[option_name].from_any(option_value)
-                            supported_types = [Choice, Toggle, Range, NamedRange]
+                            supported_types = [NumericOption, OptionSet, OptionList, ItemDict]
                             if not any([issubclass(option.__class__, t) for t in supported_types]):
                                 self.fail(f"'{option_name}' in preset '{preset_name}' for game '{game_name}' "
                                           f"is not a supported type for webhost. "

typings/kivy/core/window.pyi

@@ -0,0 +1,15 @@
+from typing import Callable, ClassVar
+
+from kivy.event import EventDispatcher
+
+
+class WindowBase(EventDispatcher):
+    width: ClassVar[int]  # readonly AliasProperty
+    height: ClassVar[int]  # readonly AliasProperty
+
+    @staticmethod
+    def bind(**kwargs: Callable[..., None]) -> None: ...
+
+
+class Window(WindowBase):
+    ...

typings/kivy/event.pyi

@@ -0,0 +1,2 @@
+class EventDispatcher:
+    ...

typings/kivy/uix/boxlayout.pyi

@@ -0,0 +1,6 @@
+from typing import Literal
+from .layout import Layout
+
+
+class BoxLayout(Layout):
+    orientation: Literal['horizontal', 'vertical']

typings/kivy/uix/layout.pyi

@@ -1,8 +1,14 @@
-from typing import Any
+from typing import Any, Sequence
+
 from .widget import Widget
 
 
 class Layout(Widget):
+    @property
+    def children(self) -> Sequence[Widget]: ...
+
     def add_widget(self, widget: Widget) -> None: ...
 
+    def remove_widget(self, widget: Widget) -> None: ...
+
     def do_layout(self, *largs: Any, **kwargs: Any) -> None: ...

typings/schema/__init__.pyi

@@ -0,0 +1,17 @@
+from typing import Any, Callable
+
+
+class And:
+    def __init__(self, __type: type, __func: Callable[[Any], bool]) -> None: ...
+
+
+class Or:
+    def __init__(self, *args: object) -> None: ...
+
+
+class Schema:
+    def __init__(self, __x: object) -> None: ...
+
+
+class Optional(Schema):
+    ...

worlds/AutoWorld.py

@@ -3,18 +3,14 @@ from __future__ import annotations
 import hashlib
 import logging
 import pathlib
-from random import Random
-import re
 import sys
 import time
+from random import Random
 from dataclasses import make_dataclass
-from typing import (Any, Callable, ClassVar, Dict, FrozenSet, List, Mapping,
-                    Optional, Set, TextIO, Tuple, TYPE_CHECKING, Type, Union)
+from typing import (Any, Callable, ClassVar, Dict, FrozenSet, List, Mapping, Optional, Set, TextIO, Tuple,
+                    TYPE_CHECKING, Type, Union)
 
-from Options import (
-    ExcludeLocations, ItemLinks, LocalItems, NonLocalItems, OptionGroup, PerGameCommonOptions,
-    PriorityLocations, StartHints, StartInventory, StartInventoryPool, StartLocationHints
-)
+from Options import item_and_loc_options, OptionGroup, PerGameCommonOptions
 from BaseClasses import CollectionState
 
 if TYPE_CHECKING:
@@ -55,17 +51,12 @@ class AutoWorldRegister(type):
         dct["item_name_groups"] = {group_name: frozenset(group_set) for group_name, group_set
                                    in dct.get("item_name_groups", {}).items()}
         dct["item_name_groups"]["Everything"] = dct["item_names"]
-        dct["item_descriptions"] = {name: _normalize_description(description) for name, description
-                                    in dct.get("item_descriptions", {}).items()}
-        dct["item_descriptions"]["Everything"] = "All items in the entire game."
+
         dct["location_names"] = frozenset(dct["location_name_to_id"])
         dct["location_name_groups"] = {group_name: frozenset(group_set) for group_name, group_set
                                        in dct.get("location_name_groups", {}).items()}
         dct["location_name_groups"]["Everywhere"] = dct["location_names"]
         dct["all_item_and_group_names"] = frozenset(dct["item_names"] | set(dct.get("item_name_groups", {})))
-        dct["location_descriptions"] = {name: _normalize_description(description) for name, description
-                                    in dct.get("location_descriptions", {}).items()}
-        dct["location_descriptions"]["Everywhere"] = "All locations in the entire game."
 
         # move away from get_required_client_version function
         if "game" in dct:
@@ -125,13 +116,19 @@ class WebWorldRegister(type):
         # don't allow an option to appear in multiple groups, allow "Item & Location Options" to appear anywhere by the
         # dev, putting it at the end if they don't define options in it
         option_groups: List[OptionGroup] = dct.get("option_groups", [])
-        item_and_loc_options = [LocalItems, NonLocalItems, StartInventory, StartInventoryPool, StartHints,
-                                StartLocationHints, ExcludeLocations, PriorityLocations, ItemLinks]
+        prebuilt_options = ["Game Options", "Item & Location Options"]
         seen_options = []
         item_group_in_list = False
         for group in option_groups:
-            assert group.name != "Game Options", "Game Options is a pre-determined group and can not be defined."
+            assert group.options, "A custom defined Option Group must contain at least one Option."
+            # catch incorrectly titled versions of the prebuilt groups so they don't create extra groups
+            title_name = group.name.title()
+            assert title_name not in prebuilt_options or title_name == group.name, \
+                f"Prebuilt group name \"{group.name}\" must be \"{title_name}\""
+
             if group.name == "Item & Location Options":
+                assert not any(option in item_and_loc_options for option in group.options), \
+                    f"Item and Location Options cannot be specified multiple times"
                 group.options.extend(item_and_loc_options)
                 item_group_in_list = True
             else:
@@ -143,7 +140,7 @@ class WebWorldRegister(type):
                 assert option not in seen_options, f"{option} found in two option groups"
                 seen_options.append(option)
         if not item_group_in_list:
-            option_groups.append(OptionGroup("Item & Location Options", item_and_loc_options))
+            option_groups.append(OptionGroup("Item & Location Options", item_and_loc_options, True))
         return super().__new__(mcs, name, bases, dct)
 
 
@@ -226,6 +223,12 @@ class WebWorld(metaclass=WebWorldRegister):
     option_groups: ClassVar[List[OptionGroup]] = []
     """Ordered list of option groupings. Any options not set in a group will be placed in a pre-built "Game Options"."""
 
+    location_descriptions: Dict[str, str] = {}
+    """An optional map from location names (or location group names) to brief descriptions for users."""
+
+    item_descriptions: Dict[str, str] = {}
+    """An optional map from item names (or item group names) to brief descriptions for users."""
+
 
 class World(metaclass=AutoWorldRegister):
     """A World object encompasses a game's Items, Locations, Rules and additional data or functionality required.
@@ -252,35 +255,9 @@ class World(metaclass=AutoWorldRegister):
     item_name_groups: ClassVar[Dict[str, Set[str]]] = {}
     """maps item group names to sets of items. Example: {"Weapons": {"Sword", "Bow"}}"""
 
-    item_descriptions: ClassVar[Dict[str, str]] = {}
-    """An optional map from item names (or item group names) to brief descriptions for users.
-
-    Individual newlines and indentation will be collapsed into spaces before these descriptions are
-    displayed. This may cover only a subset of items.
-    """
-
     location_name_groups: ClassVar[Dict[str, Set[str]]] = {}
     """maps location group names to sets of locations. Example: {"Sewer": {"Sewer Key Drop 1", "Sewer Key Drop 2"}}"""
 
-    location_descriptions: ClassVar[Dict[str, str]] = {}
-    """An optional map from location names (or location group names) to brief descriptions for users.
-
-    Individual newlines and indentation will be collapsed into spaces before these descriptions are
-    displayed. This may cover only a subset of locations.
-    """
-
-    data_version: ClassVar[int] = 0
-    """
-    Increment this every time something in your world's names/id mappings changes.
-
-    When this is set to 0, that world's DataPackage is considered in "testing mode", which signals to servers/clients
-    that it should not be cached, and clients should request that world's DataPackage every connection. Not
-    recommended for production-ready worlds.
-
-    Deprecated. Clients should utilize `checksum` to determine if DataPackage has changed since last connection and
-    request a new DataPackage, if necessary.
-    """
-
     required_client_version: Tuple[int, int, int] = (0, 1, 6)
     """
     override this if changes to a world break forward-compatibility of the client
@@ -554,7 +531,6 @@ class World(metaclass=AutoWorldRegister):
             "item_name_to_id": cls.item_name_to_id,
             "location_name_groups": sorted_location_name_groups,
             "location_name_to_id": cls.location_name_to_id,
-            "version": cls.data_version,
         }
         res["checksum"] = data_package_checksum(res)
         return res
@@ -572,18 +548,3 @@ def data_package_checksum(data: "GamesPackage") -> str:
     assert sorted(data) == list(data), "Data not ordered"
     from NetUtils import encode
     return hashlib.sha1(encode(data).encode()).hexdigest()
-
-
-def _normalize_description(description):
-    """
-    Normalizes a description in item_descriptions or location_descriptions.
-
-    This allows authors to write descritions with nice indentation and line lengths in their world
-    definitions without having it affect the rendered format.
-    """
-    # First, collapse the whitespace around newlines and the ends of the description.
-    description = re.sub(r' *\n *', '\n', description.strip())
-    # Next, condense individual newlines into spaces.
-    description = re.sub(r'(?<!\n)\n(?!\n)', ' ', description)
-    return description
-

worlds/LauncherComponents.py

@@ -1,8 +1,11 @@
+import bisect
+import logging
+import pathlib
 import weakref
 from enum import Enum, auto
-from typing import Optional, Callable, List, Iterable
+from typing import Optional, Callable, List, Iterable, Tuple
 
-from Utils import local_path
+from Utils import local_path, open_filename
 
 
 class Type(Enum):
@@ -49,8 +52,10 @@ class Component:
     def __repr__(self):
         return f"{self.__class__.__name__}({self.display_name})"
 
+
 processes = weakref.WeakSet()
 
+
 def launch_subprocess(func: Callable, name: str = None):
     global processes
     import multiprocessing
@@ -58,6 +63,7 @@ def launch_subprocess(func: Callable, name: str = None):
     process.start()
     processes.add(process)
 
+
 class SuffixIdentifier:
     suffixes: Iterable[str]
 
@@ -77,6 +83,80 @@ def launch_textclient():
     launch_subprocess(CommonClient.run_as_textclient, name="TextClient")
 
 
+def _install_apworld(apworld_src: str = "") -> Optional[Tuple[pathlib.Path, pathlib.Path]]:
+    if not apworld_src:
+        apworld_src = open_filename('Select APWorld file to install', (('APWorld', ('.apworld',)),))
+        if not apworld_src:
+            # user closed menu
+            return
+
+    if not apworld_src.endswith(".apworld"):
+        raise Exception(f"Wrong file format, looking for .apworld. File identified: {apworld_src}")
+
+    apworld_path = pathlib.Path(apworld_src)
+
+    module_name = pathlib.Path(apworld_path.name).stem
+    try:
+        import zipfile
+        zipfile.ZipFile(apworld_path).open(module_name + "/__init__.py")
+    except ValueError as e:
+        raise Exception("Archive appears invalid or damaged.") from e
+    except KeyError as e:
+        raise Exception("Archive appears to not be an apworld. (missing __init__.py)") from e
+
+    import worlds
+    if worlds.user_folder is None:
+        raise Exception("Custom Worlds directory appears to not be writable.")
+    for world_source in worlds.world_sources:
+        if apworld_path.samefile(world_source.resolved_path):
+            # Note that this doesn't check if the same world is already installed.
+            # It only checks if the user is trying to install the apworld file
+            # that comes from the installation location (worlds or custom_worlds)
+            raise Exception(f"APWorld is already installed at {world_source.resolved_path}.")
+
+    # TODO: run generic test suite over the apworld.
+    # TODO: have some kind of version system to tell from metadata if the apworld should be compatible.
+
+    target = pathlib.Path(worlds.user_folder) / apworld_path.name
+    import shutil
+    shutil.copyfile(apworld_path, target)
+
+    # If a module with this name is already loaded, then we can't load it now.
+    # TODO: We need to be able to unload a world module,
+    # so the user can update a world without restarting the application.
+    found_already_loaded = False
+    for loaded_world in worlds.world_sources:
+        loaded_name = pathlib.Path(loaded_world.path).stem
+        if module_name == loaded_name:
+            found_already_loaded = True
+            break
+    if found_already_loaded:
+        raise Exception(f"Installed APWorld successfully, but '{module_name}' is already loaded,\n"
+                        "so a Launcher restart is required to use the new installation.")
+    world_source = worlds.WorldSource(str(target), is_zip=True)
+    bisect.insort(worlds.world_sources, world_source)
+    world_source.load()
+
+    return apworld_path, target
+
+
+def install_apworld(apworld_path: str = "") -> None:
+    try:
+        res = _install_apworld(apworld_path)
+        if res is None:
+            logging.info("Aborting APWorld installation.")
+            return
+        source, target = res
+    except Exception as e:
+        import Utils
+        Utils.messagebox(e.__class__.__name__, str(e), error=True)
+        logging.exception(e)
+    else:
+        import Utils
+        logging.info(f"Installed APWorld successfully, copied {source} to {target}.")
+        Utils.messagebox("Install complete.", f"Installed APWorld from {source}.")
+
+
 components: List[Component] = [
     # Launcher
     Component('Launcher', 'Launcher', component_type=Type.HIDDEN),
@@ -84,6 +164,7 @@ components: List[Component] = [
     Component('Host', 'MultiServer', 'ArchipelagoServer', cli=True,
               file_identifier=SuffixIdentifier('.archipelago', '.zip')),
     Component('Generate', 'Generate', cli=True),
+    Component("Install APWorld", func=install_apworld, file_identifier=SuffixIdentifier(".apworld")),
     Component('Text Client', 'CommonClient', 'ArchipelagoTextClient', func=launch_textclient),
     Component('Links Awakening DX Client', 'LinksAwakeningClient',
               file_identifier=SuffixIdentifier('.apladx')),

worlds/__init__.py

@@ -1,16 +1,22 @@
 import importlib
+import importlib.util
+import logging
 import os
 import sys
 import warnings
 import zipimport
 import time
 import dataclasses
-from typing import Dict, List, TypedDict, Optional
+from typing import Dict, List, TypedDict
 
 from Utils import local_path, user_path
 
 local_folder = os.path.dirname(__file__)
-user_folder = user_path("worlds") if user_path() != local_path() else None
+user_folder = user_path("worlds") if user_path() != local_path() else user_path("custom_worlds")
+try:
+    os.makedirs(user_folder, exist_ok=True)
+except OSError:  # can't access/write?
+    user_folder = None
 
 __all__ = {
     "network_data_package",
@@ -33,7 +39,6 @@ class GamesPackage(TypedDict, total=False):
     location_name_groups: Dict[str, List[str]]
     location_name_to_id: Dict[str, int]
     checksum: str
-    version: int  # TODO: Remove support after per game data packages API change.
 
 
 class DataPackage(TypedDict):
@@ -45,7 +50,7 @@ class WorldSource:
     path: str  # typically relative path from this module
     is_zip: bool = False
     relative: bool = True  # relative to regular world import folder
-    time_taken: Optional[float] = None
+    time_taken: float = -1.0
 
     def __repr__(self) -> str:
         return f"{self.__class__.__name__}({self.path}, is_zip={self.is_zip}, relative={self.relative})"
@@ -89,7 +94,6 @@ class WorldSource:
             print(f"Could not load world {self}:", file=file_like)
             traceback.print_exc(file=file_like)
             file_like.seek(0)
-            import logging
             logging.exception(file_like.read())
             failed_world_loads.append(os.path.basename(self.path).rsplit(".", 1)[0])
             return False
@@ -104,7 +108,12 @@ for folder in (folder for folder in (user_folder, local_folder) if folder):
         if not entry.name.startswith(("_", ".")):
             file_name = entry.name if relative else os.path.join(folder, entry.name)
             if entry.is_dir():
-                world_sources.append(WorldSource(file_name, relative=relative))
+                if os.path.isfile(os.path.join(entry.path, '__init__.py')):
+                    world_sources.append(WorldSource(file_name, relative=relative))
+                elif os.path.isfile(os.path.join(entry.path, '__init__.pyc')):
+                    world_sources.append(WorldSource(file_name, relative=relative))
+                else:
+                    logging.warning(f"excluding {entry.name} from world sources because it has no __init__.py")
             elif entry.is_file() and entry.name.endswith(".apworld"):
                 world_sources.append(WorldSource(file_name, is_zip=True, relative=relative))
 

worlds/_bizhawk/__init__.py

@@ -103,7 +103,7 @@ async def connect(ctx: BizHawkContext) -> bool:
             return True
         except (TimeoutError, ConnectionRefusedError):
             continue
-    
+
     # No ports worked
     ctx.streams = None
     ctx.connection_status = ConnectionStatus.NOT_CONNECTED

worlds/_bizhawk/client.py

@@ -2,7 +2,6 @@
 A module containing the BizHawkClient base class and metaclass
 """
 
-
 from __future__ import annotations
 
 import abc
@@ -12,14 +11,13 @@ from worlds.LauncherComponents import Component, SuffixIdentifier, Type, compone
 
 if TYPE_CHECKING:
     from .context import BizHawkClientContext
-else:
-    BizHawkClientContext = object
 
 
 def launch_client(*args) -> None:
     from .context import launch
     launch_subprocess(launch, name="BizHawkClient")
 
+
 component = Component("BizHawk Client", "BizHawkClient", component_type=Type.CLIENT, func=launch_client,
                       file_identifier=SuffixIdentifier())
 components.append(component)
@@ -56,7 +54,7 @@ class AutoBizHawkClientRegister(abc.ABCMeta):
         return new_class
 
     @staticmethod
-    async def get_handler(ctx: BizHawkClientContext, system: str) -> Optional[BizHawkClient]:
+    async def get_handler(ctx: "BizHawkClientContext", system: str) -> Optional[BizHawkClient]:
         for systems, handlers in AutoBizHawkClientRegister.game_handlers.items():
             if system in systems:
                 for handler in handlers.values():
@@ -77,7 +75,7 @@ class BizHawkClient(abc.ABC, metaclass=AutoBizHawkClientRegister):
     """The file extension(s) this client is meant to open and patch (e.g. ".apz3")"""
 
     @abc.abstractmethod
-    async def validate_rom(self, ctx: BizHawkClientContext) -> bool:
+    async def validate_rom(self, ctx: "BizHawkClientContext") -> bool:
         """Should return whether the currently loaded ROM should be handled by this client. You might read the game name
         from the ROM header, for example. This function will only be asked to validate ROMs from the system set by the
         client class, so you do not need to check the system yourself.
@@ -86,18 +84,18 @@ class BizHawkClient(abc.ABC, metaclass=AutoBizHawkClientRegister):
         as necessary (such as setting `ctx.game = self.game`, modifying `ctx.items_handling`, etc...)."""
         ...
 
-    async def set_auth(self, ctx: BizHawkClientContext) -> None:
+    async def set_auth(self, ctx: "BizHawkClientContext") -> None:
         """Should set ctx.auth in anticipation of sending a `Connected` packet. You may override this if you store slot
         name in your patched ROM. If ctx.auth is not set after calling, the player will be prompted to enter their
         username."""
         pass
 
     @abc.abstractmethod
-    async def game_watcher(self, ctx: BizHawkClientContext) -> None:
+    async def game_watcher(self, ctx: "BizHawkClientContext") -> None:
         """Runs on a loop with the approximate interval `ctx.watcher_timeout`. The currently loaded ROM is guaranteed
         to have passed your validator when this function is called, and the emulator is very likely to be connected."""
         ...
 
-    def on_package(self, ctx: BizHawkClientContext, cmd: str, args: dict) -> None:
+    def on_package(self, ctx: "BizHawkClientContext", cmd: str, args: dict) -> None:
         """For handling packages from the server. Called from `BizHawkClientContext.on_package`."""
         pass

worlds/_bizhawk/context.py

@@ -3,7 +3,6 @@ A module containing context and functions relevant to running the client. This m
 checking or launching the client, otherwise it will probably cause circular import issues.
 """
 
-
 import asyncio
 import enum
 import subprocess
@@ -77,7 +76,7 @@ class BizHawkClientContext(CommonContext):
         if self.client_handler is not None:
             self.client_handler.on_package(self, cmd, args)
 
-    async def server_auth(self, password_requested: bool = False):
+    async def server_auth(self, password_requested: bool=False):
         self.password_requested = password_requested
 
         if self.bizhawk_ctx.connection_status != ConnectionStatus.CONNECTED:
@@ -103,7 +102,7 @@ class BizHawkClientContext(CommonContext):
         await self.send_connect()
         self.auth_status = AuthStatus.PENDING
 
-    async def disconnect(self, allow_autoreconnect: bool = False):
+    async def disconnect(self, allow_autoreconnect: bool=False):
         self.auth_status = AuthStatus.NOT_AUTHENTICATED
         await super().disconnect(allow_autoreconnect)
 
@@ -148,7 +147,8 @@ async def _game_watcher(ctx: BizHawkClientContext):
                 script_version = await get_script_version(ctx.bizhawk_ctx)
 
                 if script_version != EXPECTED_SCRIPT_VERSION:
-                    logger.info(f"Connector script is incompatible. Expected version {EXPECTED_SCRIPT_VERSION} but got {script_version}. Disconnecting.")
+                    logger.info(f"Connector script is incompatible. Expected version {EXPECTED_SCRIPT_VERSION} but "
+                                f"got {script_version}. Disconnecting.")
                     disconnect(ctx.bizhawk_ctx)
                     continue
 
@@ -168,6 +168,7 @@ async def _game_watcher(ctx: BizHawkClientContext):
                 ctx.auth = None
                 ctx.username = None
                 ctx.client_handler = None
+                ctx.finished_game = False
                 await ctx.disconnect(False)
             ctx.rom_hash = rom_hash
 
@@ -177,7 +178,8 @@ async def _game_watcher(ctx: BizHawkClientContext):
 
                 if ctx.client_handler is None:
                     if not showed_no_handler_message:
-                        logger.info("No handler was found for this game")
+                        logger.info("No handler was found for this game. Double-check that the apworld is installed "
+                                    "correctly and that you loaded the right ROM file.")
                         showed_no_handler_message = True
                     continue
                 else:

worlds/_sc2common/bot/sc2process.py

@@ -28,6 +28,11 @@ class kill_switch:
         logger.debug("kill_switch: Add switch")
         cls._to_kill.append(value)
 
+    @classmethod
+    def kill(cls, value):
+        logger.info(f"kill_switch: Process cleanup for 1 process")
+        value._clean(verbose=False)
+
     @classmethod
     def kill_all(cls):
         logger.info(f"kill_switch: Process cleanup for {len(cls._to_kill)} processes")
@@ -116,7 +121,7 @@ class SC2Process:
     async def __aexit__(self, *args):
         logger.exception("async exit")
         await self._close_connection()
-        kill_switch.kill_all()
+        kill_switch.kill(self)
         signal.signal(signal.SIGINT, signal.SIG_DFL)
 
     @property

worlds/adventure/Options.py

@@ -2,7 +2,8 @@ from __future__ import annotations
 
 from typing import Dict
 
-from Options import Choice, Option, DefaultOnToggle, DeathLink, Range, Toggle
+from dataclasses import dataclass
+from Options import Choice, Option, DefaultOnToggle, DeathLink, Range, Toggle, PerGameCommonOptions
 
 
 class FreeincarnateMax(Range):
@@ -223,22 +224,22 @@ class StartCastle(Choice):
     option_white = 2
     default = option_yellow
 
+@dataclass
+class AdventureOptions(PerGameCommonOptions):
+    dragon_slay_check: DragonSlayCheck
+    death_link: DeathLink
+    bat_logic: BatLogic
+    freeincarnate_max: FreeincarnateMax
+    dragon_rando_type: DragonRandoType
+    connector_multi_slot: ConnectorMultiSlot
+    yorgle_speed: YorgleStartingSpeed
+    yorgle_min_speed: YorgleMinimumSpeed
+    grundle_speed: GrundleStartingSpeed
+    grundle_min_speed: GrundleMinimumSpeed
+    rhindle_speed: RhindleStartingSpeed
+    rhindle_min_speed: RhindleMinimumSpeed
+    difficulty_switch_a: DifficultySwitchA
+    difficulty_switch_b: DifficultySwitchB
+    start_castle: StartCastle
 
-adventure_option_definitions: Dict[str, type(Option)] = {
-    "dragon_slay_check": DragonSlayCheck,
-    "death_link": DeathLink,
-    "bat_logic": BatLogic,
-    "freeincarnate_max": FreeincarnateMax,
-    "dragon_rando_type": DragonRandoType,
-    "connector_multi_slot": ConnectorMultiSlot,
-    "yorgle_speed": YorgleStartingSpeed,
-    "yorgle_min_speed": YorgleMinimumSpeed,
-    "grundle_speed": GrundleStartingSpeed,
-    "grundle_min_speed": GrundleMinimumSpeed,
-    "rhindle_speed": RhindleStartingSpeed,
-    "rhindle_min_speed": RhindleMinimumSpeed,
-    "difficulty_switch_a": DifficultySwitchA,
-    "difficulty_switch_b": DifficultySwitchB,
-    "start_castle": StartCastle,
 
-}

worlds/adventure/Regions.py

@@ -1,4 +1,5 @@
 from BaseClasses import MultiWorld, Region, Entrance, LocationProgressType
+from Options import PerGameCommonOptions
 from .Locations import location_table, LocationData, AdventureLocation, dragon_room_to_region
 
 
@@ -24,7 +25,7 @@ def connect(world: MultiWorld, player: int, source: str, target: str, rule: call
         connect(world, player, target, source, rule, True)
 
 
-def create_regions(multiworld: MultiWorld, player: int, dragon_rooms: []) -> None:
+def create_regions(options: PerGameCommonOptions, multiworld: MultiWorld, player: int, dragon_rooms: []) -> None:
 
     menu = Region("Menu", player, multiworld)
 
@@ -74,7 +75,7 @@ def create_regions(multiworld: MultiWorld, player: int, dragon_rooms: []) -> Non
     credits_room_far_side.exits.append(Entrance(player, "CreditsFromFarSide", credits_room_far_side))
     multiworld.regions.append(credits_room_far_side)
 
-    dragon_slay_check = multiworld.dragon_slay_check[player].value
+    dragon_slay_check = options.dragon_slay_check.value
     priority_locations = determine_priority_locations(multiworld, dragon_slay_check)
 
     for name, location_data in location_table.items():

worlds/adventure/Rules.py

@@ -6,7 +6,7 @@ from BaseClasses import LocationProgressType
 
 def set_rules(self) -> None:
     world = self.multiworld
-    use_bat_logic = world.bat_logic[self.player].value == BatLogic.option_use_logic
+    use_bat_logic = self.options.bat_logic.value == BatLogic.option_use_logic
 
     set_rule(world.get_entrance("YellowCastlePort", self.player),
              lambda state: state.has("Yellow Key", self.player))
@@ -28,7 +28,7 @@ def set_rules(self) -> None:
                  lambda state: state.has("Bridge", self.player) or
                                state.has("Magnet", self.player))
 
-    dragon_slay_check = world.dragon_slay_check[self.player].value
+    dragon_slay_check = self.options.dragon_slay_check.value
     if dragon_slay_check:
         if self.difficulty_switch_b == DifficultySwitchB.option_hard_with_unlock_item:
             set_rule(world.get_location("Slay Yorgle", self.player),

worlds/adventure/__init__.py

@@ -15,7 +15,8 @@ from Options import AssembleOptions
 from worlds.AutoWorld import WebWorld, World
 from Fill import fill_restrictive
 from worlds.generic.Rules import add_rule, set_rule
-from .Options import adventure_option_definitions, DragonRandoType, DifficultySwitchA, DifficultySwitchB
+from .Options import DragonRandoType, DifficultySwitchA, DifficultySwitchB, \
+    AdventureOptions
 from .Rom import get_base_rom_bytes, get_base_rom_path, AdventureDeltaPatch, apply_basepatch, \
     AdventureAutoCollectLocation
 from .Items import item_table, ItemData, nothing_item_id, event_table, AdventureItem, standard_item_max
@@ -109,11 +110,10 @@ class AdventureWorld(World):
     game: ClassVar[str] = "Adventure"
     web: ClassVar[WebWorld] = AdventureWeb()
 
-    option_definitions: ClassVar[Dict[str, AssembleOptions]] = adventure_option_definitions
+    options_dataclass = AdventureOptions
     settings: ClassVar[AdventureSettings]
     item_name_to_id: ClassVar[Dict[str, int]] = {name: data.id for name, data in item_table.items()}
     location_name_to_id: ClassVar[Dict[str, int]] = {name: data.location_id for name, data in location_table.items()}
-    data_version: ClassVar[int] = 1
     required_client_version: Tuple[int, int, int] = (0, 3, 9)
 
     def __init__(self, world: MultiWorld, player: int):
@@ -150,18 +150,18 @@ class AdventureWorld(World):
             bytearray(f"ADVENTURE{__version__.replace('.', '')[:3]}_{self.player}_{self.multiworld.seed}", "utf8")[:21]
         self.rom_name.extend([0] * (21 - len(self.rom_name)))
 
-        self.dragon_rando_type = self.multiworld.dragon_rando_type[self.player].value
-        self.dragon_slay_check = self.multiworld.dragon_slay_check[self.player].value
-        self.connector_multi_slot = self.multiworld.connector_multi_slot[self.player].value
-        self.yorgle_speed = self.multiworld.yorgle_speed[self.player].value
-        self.yorgle_min_speed = self.multiworld.yorgle_min_speed[self.player].value
-        self.grundle_speed = self.multiworld.grundle_speed[self.player].value
-        self.grundle_min_speed = self.multiworld.grundle_min_speed[self.player].value
-        self.rhindle_speed = self.multiworld.rhindle_speed[self.player].value
-        self.rhindle_min_speed = self.multiworld.rhindle_min_speed[self.player].value
-        self.difficulty_switch_a = self.multiworld.difficulty_switch_a[self.player].value
-        self.difficulty_switch_b = self.multiworld.difficulty_switch_b[self.player].value
-        self.start_castle = self.multiworld.start_castle[self.player].value
+        self.dragon_rando_type = self.options.dragon_rando_type.value
+        self.dragon_slay_check = self.options.dragon_slay_check.value
+        self.connector_multi_slot = self.options.connector_multi_slot.value
+        self.yorgle_speed = self.options.yorgle_speed.value
+        self.yorgle_min_speed = self.options.yorgle_min_speed.value
+        self.grundle_speed = self.options.grundle_speed.value
+        self.grundle_min_speed = self.options.grundle_min_speed.value
+        self.rhindle_speed = self.options.rhindle_speed.value
+        self.rhindle_min_speed = self.options.rhindle_min_speed.value
+        self.difficulty_switch_a = self.options.difficulty_switch_a.value
+        self.difficulty_switch_b = self.options.difficulty_switch_b.value
+        self.start_castle = self.options.start_castle.value
         self.created_items = 0
 
         if self.dragon_slay_check == 0:
@@ -228,7 +228,7 @@ class AdventureWorld(World):
         extra_filler_count = num_locations - self.created_items
 
         # traps would probably go here, if enabled
-        freeincarnate_max = self.multiworld.freeincarnate_max[self.player].value
+        freeincarnate_max = self.options.freeincarnate_max.value
         actual_freeincarnates = min(extra_filler_count, freeincarnate_max)
         self.multiworld.itempool += [self.create_item("Freeincarnate") for _ in range(actual_freeincarnates)]
         self.created_items += actual_freeincarnates
@@ -248,7 +248,7 @@ class AdventureWorld(World):
                 self.created_items += 1
 
     def create_regions(self) -> None:
-        create_regions(self.multiworld, self.player, self.dragon_rooms)
+        create_regions(self.options, self.multiworld, self.player, self.dragon_rooms)
 
     set_rules = set_rules
 
@@ -355,7 +355,7 @@ class AdventureWorld(World):
         auto_collect_locations: [AdventureAutoCollectLocation] = []
         local_item_to_location: {int, int} = {}
         bat_no_touch_locs: [LocationData] = []
-        bat_logic: int = self.multiworld.bat_logic[self.player].value
+        bat_logic: int = self.options.bat_logic.value
         try:
             rom_deltas: { int, int } = {}
             self.place_dragons(rom_deltas)
@@ -422,7 +422,7 @@ class AdventureWorld(World):
                 item_position_data_start = get_item_position_data_start(unplaced_item.table_index)
                 rom_deltas[item_position_data_start] = 0xff
 
-            if self.multiworld.connector_multi_slot[self.player].value:
+            if self.options.connector_multi_slot.value:
                 rom_deltas[connector_port_offset] = (self.player & 0xff)
             else:
                 rom_deltas[connector_port_offset] = 0

worlds/ahit/Client.py

@@ -0,0 +1,232 @@
+import asyncio
+import Utils
+import websockets
+import functools
+from copy import deepcopy
+from typing import List, Any, Iterable
+from NetUtils import decode, encode, JSONtoTextParser, JSONMessagePart, NetworkItem
+from MultiServer import Endpoint
+from CommonClient import CommonContext, gui_enabled, ClientCommandProcessor, logger, get_base_parser
+
+DEBUG = False
+
+
+class AHITJSONToTextParser(JSONtoTextParser):
+    def _handle_color(self, node: JSONMessagePart):
+        return self._handle_text(node)  # No colors for the in-game text
+
+
+class AHITCommandProcessor(ClientCommandProcessor):
+    def _cmd_ahit(self):
+        """Check AHIT Connection State"""
+        if isinstance(self.ctx, AHITContext):
+            logger.info(f"AHIT Status: {self.ctx.get_ahit_status()}")
+
+
+class AHITContext(CommonContext):
+    command_processor = AHITCommandProcessor
+    game = "A Hat in Time"
+
+    def __init__(self, server_address, password):
+        super().__init__(server_address, password)
+        self.proxy = None
+        self.proxy_task = None
+        self.gamejsontotext = AHITJSONToTextParser(self)
+        self.autoreconnect_task = None
+        self.endpoint = None
+        self.items_handling = 0b111
+        self.room_info = None
+        self.connected_msg = None
+        self.game_connected = False
+        self.awaiting_info = False
+        self.full_inventory: List[Any] = []
+        self.server_msgs: List[Any] = []
+
+    async def server_auth(self, password_requested: bool = False):
+        if password_requested and not self.password:
+            await super(AHITContext, self).server_auth(password_requested)
+
+        await self.get_username()
+        await self.send_connect()
+
+    def get_ahit_status(self) -> str:
+        if not self.is_proxy_connected():
+            return "Not connected to A Hat in Time"
+
+        return "Connected to A Hat in Time"
+
+    async def send_msgs_proxy(self, msgs: Iterable[dict]) -> bool:
+        """ `msgs` JSON serializable """
+        if not self.endpoint or not self.endpoint.socket.open or self.endpoint.socket.closed:
+            return False
+
+        if DEBUG:
+            logger.info(f"Outgoing message: {msgs}")
+
+        await self.endpoint.socket.send(msgs)
+        return True
+
+    async def disconnect(self, allow_autoreconnect: bool = False):
+        await super().disconnect(allow_autoreconnect)
+
+    async def disconnect_proxy(self):
+        if self.endpoint and not self.endpoint.socket.closed:
+            await self.endpoint.socket.close()
+        if self.proxy_task is not None:
+            await self.proxy_task
+
+    def is_connected(self) -> bool:
+        return self.server and self.server.socket.open
+
+    def is_proxy_connected(self) -> bool:
+        return self.endpoint and self.endpoint.socket.open
+
+    def on_print_json(self, args: dict):
+        text = self.gamejsontotext(deepcopy(args["data"]))
+        msg = {"cmd": "PrintJSON", "data": [{"text": text}], "type": "Chat"}
+        self.server_msgs.append(encode([msg]))
+
+        if self.ui:
+            self.ui.print_json(args["data"])
+        else:
+            text = self.jsontotextparser(args["data"])
+            logger.info(text)
+
+    def update_items(self):
+        # just to be safe - we might still have an inventory from a different room
+        if not self.is_connected():
+            return
+
+        self.server_msgs.append(encode([{"cmd": "ReceivedItems", "index": 0, "items": self.full_inventory}]))
+
+    def on_package(self, cmd: str, args: dict):
+        if cmd == "Connected":
+            self.connected_msg = encode([args])
+            if self.awaiting_info:
+                self.server_msgs.append(self.room_info)
+                self.update_items()
+                self.awaiting_info = False
+
+        elif cmd == "ReceivedItems":
+            if args["index"] == 0:
+                self.full_inventory.clear()
+
+            for item in args["items"]:
+                self.full_inventory.append(NetworkItem(*item))
+
+            self.server_msgs.append(encode([args]))
+
+        elif cmd == "RoomInfo":
+            self.seed_name = args["seed_name"]
+            self.room_info = encode([args])
+
+        else:
+            if cmd != "PrintJSON":
+                self.server_msgs.append(encode([args]))
+
+    def run_gui(self):
+        from kvui import GameManager
+
+        class AHITManager(GameManager):
+            logging_pairs = [
+                ("Client", "Archipelago")
+            ]
+            base_title = "Archipelago A Hat in Time Client"
+
+        self.ui = AHITManager(self)
+        self.ui_task = asyncio.create_task(self.ui.async_run(), name="UI")
+
+
+async def proxy(websocket, path: str = "/", ctx: AHITContext = None):
+    ctx.endpoint = Endpoint(websocket)
+    try:
+        await on_client_connected(ctx)
+
+        if ctx.is_proxy_connected():
+            async for data in websocket:
+                if DEBUG:
+                    logger.info(f"Incoming message: {data}")
+
+                for msg in decode(data):
+                    if msg["cmd"] == "Connect":
+                        # Proxy is connecting, make sure it is valid
+                        if msg["game"] != "A Hat in Time":
+                            logger.info("Aborting proxy connection: game is not A Hat in Time")
+                            await ctx.disconnect_proxy()
+                            break
+
+                        if ctx.seed_name:
+                            seed_name = msg.get("seed_name", "")
+                            if seed_name != "" and seed_name != ctx.seed_name:
+                                logger.info("Aborting proxy connection: seed mismatch from save file")
+                                logger.info(f"Expected: {ctx.seed_name}, got: {seed_name}")
+                                text = encode([{"cmd": "PrintJSON",
+                                                "data": [{"text": "Connection aborted - save file to seed mismatch"}]}])
+                                await ctx.send_msgs_proxy(text)
+                                await ctx.disconnect_proxy()
+                                break
+
+                        if ctx.connected_msg and ctx.is_connected():
+                            await ctx.send_msgs_proxy(ctx.connected_msg)
+                            ctx.update_items()
+                        continue
+
+                    if not ctx.is_proxy_connected():
+                        break
+
+                    await ctx.send_msgs([msg])
+
+    except Exception as e:
+        if not isinstance(e, websockets.WebSocketException):
+            logger.exception(e)
+    finally:
+        await ctx.disconnect_proxy()
+
+
+async def on_client_connected(ctx: AHITContext):
+    if ctx.room_info and ctx.is_connected():
+        await ctx.send_msgs_proxy(ctx.room_info)
+    else:
+        ctx.awaiting_info = True
+
+
+async def proxy_loop(ctx: AHITContext):
+    try:
+        while not ctx.exit_event.is_set():
+            if len(ctx.server_msgs) > 0:
+                for msg in ctx.server_msgs:
+                    await ctx.send_msgs_proxy(msg)
+
+                ctx.server_msgs.clear()
+            await asyncio.sleep(0.1)
+    except Exception as e:
+        logger.exception(e)
+        logger.info("Aborting AHIT Proxy Client due to errors")
+
+
+def launch():
+    async def main():
+        parser = get_base_parser()
+        args = parser.parse_args()
+
+        ctx = AHITContext(args.connect, args.password)
+        logger.info("Starting A Hat in Time proxy server")
+        ctx.proxy = websockets.serve(functools.partial(proxy, ctx=ctx),
+                                     host="localhost", port=11311, ping_timeout=999999, ping_interval=999999)
+        ctx.proxy_task = asyncio.create_task(proxy_loop(ctx), name="ProxyLoop")
+
+        if gui_enabled:
+            ctx.run_gui()
+        ctx.run_cli()
+
+        await ctx.proxy
+        await ctx.proxy_task
+        await ctx.exit_event.wait()
+
+    Utils.init_logging("AHITClient")
+    # options = Utils.get_options()
+
+    import colorama
+    colorama.init()
+    asyncio.run(main())
+    colorama.deinit()

worlds/ahit/DeathWishLocations.py

@@ -0,0 +1,243 @@
+from .Types import HatInTimeLocation, HatInTimeItem
+from .Regions import create_region
+from BaseClasses import Region, LocationProgressType, ItemClassification
+from worlds.generic.Rules import add_rule
+from typing import List, TYPE_CHECKING
+from .Locations import death_wishes
+from .Options import EndGoal
+
+if TYPE_CHECKING:
+    from . import HatInTimeWorld
+
+
+dw_prereqs = {
+    "So You're Back From Outer Space":  ["Beat the Heat"],
+    "Snatcher's Hit List":              ["Beat the Heat"],
+    "Snatcher Coins in Mafia Town":     ["So You're Back From Outer Space"],
+    "Rift Collapse: Mafia of Cooks":    ["So You're Back From Outer Space"],
+    "Collect-a-thon":                   ["So You're Back From Outer Space"],
+    "She Speedran from Outer Space":    ["Rift Collapse: Mafia of Cooks"],
+    "Mafia's Jumps":                    ["She Speedran from Outer Space"],
+    "Vault Codes in the Wind":          ["Collect-a-thon", "She Speedran from Outer Space"],
+    "Encore! Encore!":                  ["Collect-a-thon"],
+
+    "Security Breach":                          ["Beat the Heat"],
+    "Rift Collapse: Dead Bird Studio":          ["Security Breach"],
+    "The Great Big Hootenanny":                 ["Security Breach"],
+    "10 Seconds until Self-Destruct":           ["The Great Big Hootenanny"],
+    "Killing Two Birds":                        ["Rift Collapse: Dead Bird Studio", "10 Seconds until Self-Destruct"],
+    "Community Rift: Rhythm Jump Studio":       ["10 Seconds until Self-Destruct"],
+    "Snatcher Coins in Battle of the Birds":    ["The Great Big Hootenanny"],
+    "Zero Jumps":                               ["Rift Collapse: Dead Bird Studio"],
+    "Snatcher Coins in Nyakuza Metro":          ["Killing Two Birds"],
+
+    "Speedrun Well":                    ["Beat the Heat"],
+    "Rift Collapse: Sleepy Subcon":     ["Speedrun Well"],
+    "Boss Rush":                        ["Speedrun Well"],
+    "Quality Time with Snatcher":       ["Rift Collapse: Sleepy Subcon"],
+    "Breaching the Contract":           ["Boss Rush", "Quality Time with Snatcher"],
+    "Community Rift: Twilight Travels": ["Quality Time with Snatcher"],
+    "Snatcher Coins in Subcon Forest":  ["Rift Collapse: Sleepy Subcon"],
+
+    "Bird Sanctuary":                       ["Beat the Heat"],
+    "Snatcher Coins in Alpine Skyline":     ["Bird Sanctuary"],
+    "Wound-Up Windmill":                    ["Bird Sanctuary"],
+    "Rift Collapse: Alpine Skyline":        ["Bird Sanctuary"],
+    "Camera Tourist":                       ["Rift Collapse: Alpine Skyline"],
+    "Community Rift: The Mountain Rift":    ["Rift Collapse: Alpine Skyline"],
+    "The Illness has Speedrun":             ["Rift Collapse: Alpine Skyline", "Wound-Up Windmill"],
+
+    "The Mustache Gauntlet":            ["Wound-Up Windmill"],
+    "No More Bad Guys":                 ["The Mustache Gauntlet"],
+    "Seal the Deal":                    ["Encore! Encore!", "Killing Two Birds",
+                                         "Breaching the Contract", "No More Bad Guys"],
+
+    "Rift Collapse: Deep Sea":          ["Rift Collapse: Mafia of Cooks", "Rift Collapse: Dead Bird Studio",
+                                         "Rift Collapse: Sleepy Subcon", "Rift Collapse: Alpine Skyline"],
+
+    "Cruisin' for a Bruisin'":          ["Rift Collapse: Deep Sea"],
+}
+
+dw_candles = [
+    "Snatcher's Hit List",
+    "Zero Jumps",
+    "Camera Tourist",
+    "Snatcher Coins in Mafia Town",
+    "Snatcher Coins in Battle of the Birds",
+    "Snatcher Coins in Subcon Forest",
+    "Snatcher Coins in Alpine Skyline",
+    "Snatcher Coins in Nyakuza Metro",
+]
+
+annoying_dws = [
+    "Vault Codes in the Wind",
+    "Boss Rush",
+    "Camera Tourist",
+    "The Mustache Gauntlet",
+    "Rift Collapse: Deep Sea",
+    "Cruisin' for a Bruisin'",
+    "Seal the Deal",  # Non-excluded if goal
+]
+
+# includes the above as well
+annoying_bonuses = [
+    "So You're Back From Outer Space",
+    "Encore! Encore!",
+    "Snatcher's Hit List",
+    "Vault Codes in the Wind",
+    "10 Seconds until Self-Destruct",
+    "Killing Two Birds",
+    "Zero Jumps",
+    "Boss Rush",
+    "Bird Sanctuary",
+    "The Mustache Gauntlet",
+    "Wound-Up Windmill",
+    "Camera Tourist",
+    "Rift Collapse: Deep Sea",
+    "Cruisin' for a Bruisin'",
+    "Seal the Deal",
+]
+
+dw_classes = {
+    "Beat the Heat":                    "Hat_SnatcherContract_DeathWish_HeatingUpHarder",
+    "So You're Back From Outer Space":  "Hat_SnatcherContract_DeathWish_BackFromSpace",
+    "Snatcher's Hit List":              "Hat_SnatcherContract_DeathWish_KillEverybody",
+    "Collect-a-thon":                   "Hat_SnatcherContract_DeathWish_PonFrenzy",
+    "Rift Collapse: Mafia of Cooks":    "Hat_SnatcherContract_DeathWish_RiftCollapse_MafiaTown",
+    "Encore! Encore!":                  "Hat_SnatcherContract_DeathWish_MafiaBossEX",
+    "She Speedran from Outer Space":    "Hat_SnatcherContract_DeathWish_Speedrun_MafiaAlien",
+    "Mafia's Jumps":                    "Hat_SnatcherContract_DeathWish_NoAPresses_MafiaAlien",
+    "Vault Codes in the Wind":          "Hat_SnatcherContract_DeathWish_MovingVault",
+    "Snatcher Coins in Mafia Town":     "Hat_SnatcherContract_DeathWish_Tokens_MafiaTown",
+
+    "Security Breach":                          "Hat_SnatcherContract_DeathWish_DeadBirdStudioMoreGuards",
+    "The Great Big Hootenanny":                 "Hat_SnatcherContract_DeathWish_DifficultParade",
+    "Rift Collapse: Dead Bird Studio":          "Hat_SnatcherContract_DeathWish_RiftCollapse_Birds",
+    "10 Seconds until Self-Destruct":           "Hat_SnatcherContract_DeathWish_TrainRushShortTime",
+    "Killing Two Birds":                        "Hat_SnatcherContract_DeathWish_BirdBossEX",
+    "Snatcher Coins in Battle of the Birds":    "Hat_SnatcherContract_DeathWish_Tokens_Birds",
+    "Zero Jumps":                               "Hat_SnatcherContract_DeathWish_NoAPresses",
+
+    "Speedrun Well":                    "Hat_SnatcherContract_DeathWish_Speedrun_SubWell",
+    "Rift Collapse: Sleepy Subcon":     "Hat_SnatcherContract_DeathWish_RiftCollapse_Subcon",
+    "Boss Rush":                        "Hat_SnatcherContract_DeathWish_BossRush",
+    "Quality Time with Snatcher":       "Hat_SnatcherContract_DeathWish_SurvivalOfTheFittest",
+    "Breaching the Contract":           "Hat_SnatcherContract_DeathWish_SnatcherEX",
+    "Snatcher Coins in Subcon Forest":  "Hat_SnatcherContract_DeathWish_Tokens_Subcon",
+
+    "Bird Sanctuary":                   "Hat_SnatcherContract_DeathWish_NiceBirdhouse",
+    "Rift Collapse: Alpine Skyline":    "Hat_SnatcherContract_DeathWish_RiftCollapse_Alps",
+    "Wound-Up Windmill":                "Hat_SnatcherContract_DeathWish_FastWindmill",
+    "The Illness has Speedrun":         "Hat_SnatcherContract_DeathWish_Speedrun_Illness",
+    "Snatcher Coins in Alpine Skyline": "Hat_SnatcherContract_DeathWish_Tokens_Alps",
+    "Camera Tourist":                   "Hat_SnatcherContract_DeathWish_CameraTourist_1",
+
+    "The Mustache Gauntlet":            "Hat_SnatcherContract_DeathWish_HardCastle",
+    "No More Bad Guys":                 "Hat_SnatcherContract_DeathWish_MuGirlEX",
+
+    "Seal the Deal":                    "Hat_SnatcherContract_DeathWish_BossRushEX",
+    "Rift Collapse: Deep Sea":          "Hat_SnatcherContract_DeathWish_RiftCollapse_Cruise",
+    "Cruisin' for a Bruisin'":          "Hat_SnatcherContract_DeathWish_EndlessTasks",
+
+    "Community Rift: Rhythm Jump Studio":   "Hat_SnatcherContract_DeathWish_CommunityRift_RhythmJump",
+    "Community Rift: Twilight Travels":     "Hat_SnatcherContract_DeathWish_CommunityRift_TwilightTravels",
+    "Community Rift: The Mountain Rift":    "Hat_SnatcherContract_DeathWish_CommunityRift_MountainRift",
+
+    "Snatcher Coins in Nyakuza Metro":      "Hat_SnatcherContract_DeathWish_Tokens_Metro",
+}
+
+
+def create_dw_regions(world: "HatInTimeWorld"):
+    if world.options.DWExcludeAnnoyingContracts:
+        for name in annoying_dws:
+            world.excluded_dws.append(name)
+
+    if not world.options.DWEnableBonus or world.options.DWAutoCompleteBonuses:
+        for name in death_wishes:
+            world.excluded_bonuses.append(name)
+    elif world.options.DWExcludeAnnoyingBonuses:
+        for name in annoying_bonuses:
+            world.excluded_bonuses.append(name)
+
+    if world.options.DWExcludeCandles:
+        for name in dw_candles:
+            if name not in world.excluded_dws:
+                world.excluded_dws.append(name)
+
+    spaceship = world.multiworld.get_region("Spaceship", world.player)
+    dw_map: Region = create_region(world, "Death Wish Map")
+    entrance = spaceship.connect(dw_map, "-> Death Wish Map")
+    add_rule(entrance, lambda state: state.has("Time Piece", world.player, world.options.DWTimePieceRequirement))
+
+    if world.options.DWShuffle:
+        # Connect Death Wishes randomly to one another in a linear sequence
+        dw_list: List[str] = []
+        for name in death_wishes.keys():
+            # Don't shuffle excluded or invalid Death Wishes
+            if not world.is_dlc2() and name == "Snatcher Coins in Nyakuza Metro" or world.is_dw_excluded(name):
+                continue
+
+            dw_list.append(name)
+
+        world.random.shuffle(dw_list)
+        count = world.random.randint(world.options.DWShuffleCountMin.value, world.options.DWShuffleCountMax.value)
+        dw_shuffle: List[str] = []
+        total = min(len(dw_list), count)
+        for i in range(total):
+            dw_shuffle.append(dw_list[i])
+
+        # Seal the Deal is always last if it's the goal
+        if world.options.EndGoal == EndGoal.option_seal_the_deal:
+            if "Seal the Deal" in dw_shuffle:
+                dw_shuffle.remove("Seal the Deal")
+
+            dw_shuffle.append("Seal the Deal")
+
+        world.dw_shuffle = dw_shuffle
+        prev_dw = dw_map
+        for death_wish_name in dw_shuffle:
+            dw = create_region(world, death_wish_name)
+            prev_dw.connect(dw)
+            create_dw_locations(world, dw)
+            prev_dw = dw
+    else:
+        # DWShuffle is disabled, use vanilla connections
+        for key in death_wishes.keys():
+            if key == "Snatcher Coins in Nyakuza Metro" and not world.is_dlc2():
+                world.excluded_dws.append(key)
+                continue
+
+            dw = create_region(world, key)
+            if key == "Beat the Heat":
+                dw_map.connect(dw, f"{dw_map.name} -> Beat the Heat")
+            elif key in dw_prereqs.keys():
+                for name in dw_prereqs[key]:
+                    parent = world.multiworld.get_region(name, world.player)
+                    parent.connect(dw, f"{parent.name} -> {key}")
+
+            create_dw_locations(world, dw)
+
+
+def create_dw_locations(world: "HatInTimeWorld", dw: Region):
+    loc_id = death_wishes[dw.name]
+    main_objective = HatInTimeLocation(world.player, f"{dw.name} - Main Objective", loc_id, dw)
+    full_clear = HatInTimeLocation(world.player, f"{dw.name} - All Clear", loc_id + 1, dw)
+    main_stamp = HatInTimeLocation(world.player, f"Main Stamp - {dw.name}", None, dw)
+    bonus_stamps = HatInTimeLocation(world.player, f"Bonus Stamps - {dw.name}", None, dw)
+    main_stamp.show_in_spoiler = False
+    bonus_stamps.show_in_spoiler = False
+    dw.locations.append(main_stamp)
+    dw.locations.append(bonus_stamps)
+    main_stamp.place_locked_item(HatInTimeItem(f"1 Stamp - {dw.name}",
+                                               ItemClassification.progression, None, world.player))
+    bonus_stamps.place_locked_item(HatInTimeItem(f"2 Stamp - {dw.name}",
+                                                 ItemClassification.progression, None, world.player))
+
+    if dw.name in world.excluded_dws:
+        main_objective.progress_type = LocationProgressType.EXCLUDED
+        full_clear.progress_type = LocationProgressType.EXCLUDED
+    elif world.is_bonus_excluded(dw.name):
+        full_clear.progress_type = LocationProgressType.EXCLUDED
+
+    dw.locations.append(main_objective)
+    dw.locations.append(full_clear)

worlds/ahit/DeathWishRules.py

@@ -0,0 +1,462 @@
+from worlds.AutoWorld import CollectionState
+from .Rules import can_use_hat, can_use_hookshot, can_hit, zipline_logic, get_difficulty, has_paintings
+from .Types import HatType, Difficulty, HatInTimeLocation, HatInTimeItem, LocData, HitType
+from .DeathWishLocations import dw_prereqs, dw_candles
+from BaseClasses import Entrance, Location, ItemClassification
+from worlds.generic.Rules import add_rule, set_rule
+from typing import List, Callable, TYPE_CHECKING
+from .Locations import death_wishes
+from .Options import EndGoal
+
+if TYPE_CHECKING:
+    from . import HatInTimeWorld
+
+
+# Any speedruns expect the player to have Sprint Hat
+dw_requirements = {
+    "Beat the Heat": LocData(hit_type=HitType.umbrella),
+    "So You're Back From Outer Space": LocData(hookshot=True),
+    "Mafia's Jumps": LocData(required_hats=[HatType.ICE]),
+    "Vault Codes in the Wind": LocData(required_hats=[HatType.SPRINT]),
+
+    "Security Breach": LocData(hit_type=HitType.umbrella_or_brewing),
+    "10 Seconds until Self-Destruct": LocData(hookshot=True),
+    "Community Rift: Rhythm Jump Studio": LocData(required_hats=[HatType.ICE]),
+
+    "Speedrun Well": LocData(hookshot=True, hit_type=HitType.umbrella_or_brewing),
+    "Boss Rush": LocData(hit_type=HitType.umbrella, hookshot=True),
+    "Community Rift: Twilight Travels": LocData(hookshot=True, required_hats=[HatType.DWELLER]),
+
+    "Bird Sanctuary": LocData(hookshot=True),
+    "Wound-Up Windmill": LocData(hookshot=True),
+    "The Illness has Speedrun": LocData(hookshot=True),
+    "Community Rift: The Mountain Rift": LocData(hookshot=True, required_hats=[HatType.DWELLER]),
+    "Camera Tourist": LocData(misc_required=["Camera Badge"]),
+
+    "The Mustache Gauntlet": LocData(hookshot=True, required_hats=[HatType.DWELLER]),
+
+    "Rift Collapse: Deep Sea": LocData(hookshot=True),
+}
+
+# Includes main objective requirements
+dw_bonus_requirements = {
+    # Some One-Hit Hero requirements need badge pins as well because of Hookshot
+    "So You're Back From Outer Space": LocData(required_hats=[HatType.SPRINT]),
+    "Encore! Encore!": LocData(misc_required=["One-Hit Hero Badge"]),
+
+    "10 Seconds until Self-Destruct": LocData(misc_required=["One-Hit Hero Badge", "Badge Pin"]),
+
+    "Boss Rush": LocData(misc_required=["One-Hit Hero Badge", "Badge Pin"]),
+    "Community Rift: Twilight Travels": LocData(required_hats=[HatType.BREWING]),
+
+    "Bird Sanctuary": LocData(misc_required=["One-Hit Hero Badge", "Badge Pin"], required_hats=[HatType.DWELLER]),
+    "Wound-Up Windmill": LocData(misc_required=["One-Hit Hero Badge", "Badge Pin"]),
+    "The Illness has Speedrun": LocData(required_hats=[HatType.SPRINT]),
+
+    "The Mustache Gauntlet": LocData(required_hats=[HatType.ICE]),
+
+    "Rift Collapse: Deep Sea": LocData(required_hats=[HatType.DWELLER]),
+}
+
+dw_stamp_costs = {
+    "So You're Back From Outer Space":  2,
+    "Collect-a-thon":                   5,
+    "She Speedran from Outer Space":    8,
+    "Encore! Encore!":                  10,
+
+    "Security Breach":                  4,
+    "The Great Big Hootenanny":         7,
+    "10 Seconds until Self-Destruct":   15,
+    "Killing Two Birds":                25,
+    "Snatcher Coins in Nyakuza Metro":  30,
+
+    "Speedrun Well":                10,
+    "Boss Rush":                    15,
+    "Quality Time with Snatcher":   20,
+    "Breaching the Contract":       40,
+
+    "Bird Sanctuary":           15,
+    "Wound-Up Windmill":        30,
+    "The Illness has Speedrun": 35,
+
+    "The Mustache Gauntlet":    35,
+    "No More Bad Guys":         50,
+    "Seal the Deal":            70,
+}
+
+required_snatcher_coins = {
+    "Snatcher Coins in Mafia Town": ["Snatcher Coin - Top of HQ", "Snatcher Coin - Top of Tower",
+                                     "Snatcher Coin - Under Ruined Tower"],
+
+    "Snatcher Coins in Battle of the Birds": ["Snatcher Coin - Top of Red House", "Snatcher Coin - Train Rush",
+                                              "Snatcher Coin - Picture Perfect"],
+
+    "Snatcher Coins in Subcon Forest": ["Snatcher Coin - Swamp Tree", "Snatcher Coin - Manor Roof",
+                                        "Snatcher Coin - Giant Time Piece"],
+
+    "Snatcher Coins in Alpine Skyline": ["Snatcher Coin - Goat Village Top", "Snatcher Coin - Lava Cake",
+                                         "Snatcher Coin - Windmill"],
+
+    "Snatcher Coins in Nyakuza Metro": ["Snatcher Coin - Green Clean Tower", "Snatcher Coin - Bluefin Cat Train",
+                                        "Snatcher Coin - Pink Paw Fence"],
+}
+
+
+def set_dw_rules(world: "HatInTimeWorld"):
+    if "Snatcher's Hit List" not in world.excluded_dws or "Camera Tourist" not in world.excluded_dws:
+        set_enemy_rules(world)
+
+    dw_list: List[str] = []
+    if world.options.DWShuffle:
+        dw_list = world.dw_shuffle
+    else:
+        for name in death_wishes.keys():
+            dw_list.append(name)
+
+    for name in dw_list:
+        if name == "Snatcher Coins in Nyakuza Metro" and not world.is_dlc2():
+            continue
+
+        dw = world.multiworld.get_region(name, world.player)
+        if not world.options.DWShuffle and name in dw_stamp_costs.keys():
+            for entrance in dw.entrances:
+                add_rule(entrance, lambda state, n=name: state.has("Stamps", world.player, dw_stamp_costs[n]))
+
+        main_objective = world.multiworld.get_location(f"{name} - Main Objective", world.player)
+        all_clear = world.multiworld.get_location(f"{name} - All Clear", world.player)
+        main_stamp = world.multiworld.get_location(f"Main Stamp - {name}", world.player)
+        bonus_stamps = world.multiworld.get_location(f"Bonus Stamps - {name}", world.player)
+        if not world.options.DWEnableBonus:
+            # place nothing, but let the locations exist still, so we can use them for bonus stamp rules
+            all_clear.address = None
+            all_clear.place_locked_item(HatInTimeItem("Nothing", ItemClassification.filler, None, world.player))
+            all_clear.show_in_spoiler = False
+
+        # No need for rules if excluded - stamps will be auto-granted
+        if world.is_dw_excluded(name):
+            continue
+
+        modify_dw_rules(world, name)
+        add_dw_rules(world, main_objective)
+        add_dw_rules(world, all_clear)
+        add_rule(main_stamp, main_objective.access_rule)
+        add_rule(all_clear, main_objective.access_rule)
+        # Only set bonus stamp rules if we don't auto complete bonuses
+        if not world.options.DWAutoCompleteBonuses and not world.is_bonus_excluded(all_clear.name):
+            add_rule(bonus_stamps, all_clear.access_rule)
+
+    if world.options.DWShuffle:
+        for i in range(len(world.dw_shuffle)-1):
+            name = world.dw_shuffle[i+1]
+            prev_dw = world.multiworld.get_region(world.dw_shuffle[i], world.player)
+            entrance = world.multiworld.get_entrance(f"{prev_dw.name} -> {name}", world.player)
+            add_rule(entrance, lambda state, n=prev_dw.name: state.has(f"1 Stamp - {n}", world.player))
+    else:
+        for key, reqs in dw_prereqs.items():
+            if key == "Snatcher Coins in Nyakuza Metro" and not world.is_dlc2():
+                continue
+
+            access_rules: List[Callable[[CollectionState], bool]] = []
+            entrances: List[Entrance] = []
+
+            for parent in reqs:
+                entrance = world.multiworld.get_entrance(f"{parent} -> {key}", world.player)
+                entrances.append(entrance)
+
+                if not world.is_dw_excluded(parent):
+                    access_rules.append(lambda state, n=parent: state.has(f"1 Stamp - {n}", world.player))
+
+            for entrance in entrances:
+                for rule in access_rules:
+                    add_rule(entrance, rule)
+
+    if world.options.EndGoal == EndGoal.option_seal_the_deal:
+        world.multiworld.completion_condition[world.player] = lambda state: \
+            state.has("1 Stamp - Seal the Deal", world.player)
+
+
+def add_dw_rules(world: "HatInTimeWorld", loc: Location):
+    bonus: bool = "All Clear" in loc.name
+    if not bonus:
+        data = dw_requirements.get(loc.parent_region.name)
+    else:
+        data = dw_bonus_requirements.get(loc.parent_region.name)
+
+    if data is None:
+        return
+
+    if data.hookshot:
+        add_rule(loc, lambda state: can_use_hookshot(state, world))
+
+    for hat in data.required_hats:
+        add_rule(loc, lambda state, h=hat: can_use_hat(state, world, h))
+
+    for misc in data.misc_required:
+        add_rule(loc, lambda state, item=misc: state.has(item, world.player))
+
+    if data.paintings > 0 and world.options.ShuffleSubconPaintings:
+        add_rule(loc, lambda state, paintings=data.paintings: has_paintings(state, world, paintings))
+
+    if data.hit_type is not HitType.none and world.options.UmbrellaLogic:
+        if data.hit_type == HitType.umbrella:
+            add_rule(loc, lambda state: state.has("Umbrella", world.player))
+
+        elif data.hit_type == HitType.umbrella_or_brewing:
+            add_rule(loc, lambda state: state.has("Umbrella", world.player)
+                     or can_use_hat(state, world, HatType.BREWING))
+
+        elif data.hit_type == HitType.dweller_bell:
+            add_rule(loc, lambda state: state.has("Umbrella", world.player)
+                     or can_use_hat(state, world, HatType.BREWING)
+                     or can_use_hat(state, world, HatType.DWELLER))
+
+
+def modify_dw_rules(world: "HatInTimeWorld", name: str):
+    difficulty: Difficulty = get_difficulty(world)
+    main_objective = world.multiworld.get_location(f"{name} - Main Objective", world.player)
+    full_clear = world.multiworld.get_location(f"{name} - All Clear", world.player)
+
+    if name == "The Illness has Speedrun":
+        # All stamps with hookshot only in Expert
+        if difficulty >= Difficulty.EXPERT:
+            set_rule(full_clear, lambda state: True)
+        else:
+            add_rule(main_objective, lambda state: state.has("Umbrella", world.player))
+
+    elif name == "The Mustache Gauntlet":
+        add_rule(main_objective, lambda state: state.has("Umbrella", world.player)
+                 or can_use_hat(state, world, HatType.ICE) or can_use_hat(state, world, HatType.BREWING))
+
+    elif name == "Vault Codes in the Wind":
+        # Sprint is normally expected here
+        if difficulty >= Difficulty.HARD:
+            set_rule(main_objective, lambda state: True)
+
+    elif name == "Speedrun Well":
+        # All stamps with nothing :)
+        if difficulty >= Difficulty.EXPERT:
+            set_rule(main_objective, lambda state: True)
+
+    elif name == "Mafia's Jumps":
+        if difficulty >= Difficulty.HARD:
+            set_rule(main_objective, lambda state: True)
+            set_rule(full_clear, lambda state: True)
+
+    elif name == "So You're Back from Outer Space":
+        # Without Hookshot
+        if difficulty >= Difficulty.HARD:
+            set_rule(main_objective, lambda state: True)
+
+    elif name == "Wound-Up Windmill":
+        # No badge pin required. Player can switch to One Hit Hero after the checkpoint and do level without it.
+        if difficulty >= Difficulty.MODERATE:
+            set_rule(full_clear, lambda state: can_use_hookshot(state, world)
+                     and state.has("One-Hit Hero Badge", world.player))
+
+    if name in dw_candles:
+        set_candle_dw_rules(name, world)
+
+
+def set_candle_dw_rules(name: str, world: "HatInTimeWorld"):
+    main_objective = world.multiworld.get_location(f"{name} - Main Objective", world.player)
+    full_clear = world.multiworld.get_location(f"{name} - All Clear", world.player)
+
+    if name == "Zero Jumps":
+        add_rule(main_objective, lambda state: state.has("Zero Jumps", world.player))
+        add_rule(full_clear, lambda state: state.has("Zero Jumps", world.player, 4)
+                 and state.has("Train Rush (Zero Jumps)", world.player) and can_use_hat(state, world, HatType.ICE))
+
+        # No Ice Hat/painting required in Expert for Toilet Zero Jumps
+        # This painting wall can only be skipped via cherry hover.
+        if get_difficulty(world) < Difficulty.EXPERT or world.options.NoPaintingSkips:
+            set_rule(world.multiworld.get_location("Toilet of Doom (Zero Jumps)", world.player),
+                     lambda state: can_use_hookshot(state, world) and can_hit(state, world)
+                     and has_paintings(state, world, 1, False))
+        else:
+            set_rule(world.multiworld.get_location("Toilet of Doom (Zero Jumps)", world.player),
+                     lambda state: can_use_hookshot(state, world) and can_hit(state, world))
+
+        set_rule(world.multiworld.get_location("Contractual Obligations (Zero Jumps)", world.player),
+                 lambda state: has_paintings(state, world, 1, False))
+
+    elif name == "Snatcher's Hit List":
+        add_rule(main_objective, lambda state: state.has("Mafia Goon", world.player))
+        add_rule(full_clear, lambda state: state.has("Enemy", world.player, 12))
+
+    elif name == "Camera Tourist":
+        add_rule(main_objective, lambda state: state.has("Enemy", world.player, 8))
+        add_rule(full_clear, lambda state: state.has("Boss", world.player, 6)
+                 and state.has("Triple Enemy Photo", world.player))
+
+    elif "Snatcher Coins" in name:
+        coins: List[str] = []
+        for coin in required_snatcher_coins[name]:
+            coins.append(coin)
+            add_rule(full_clear, lambda state, c=coin: state.has(c, world.player))
+
+        # any coin works for the main objective
+        add_rule(main_objective, lambda state: state.has(coins[0], world.player)
+                 or state.has(coins[1], world.player)
+                 or state.has(coins[2], world.player))
+
+
+def create_enemy_events(world: "HatInTimeWorld"):
+    no_tourist = "Camera Tourist" in world.excluded_dws
+    for enemy, regions in hit_list.items():
+        if no_tourist and enemy in bosses:
+            continue
+
+        for area in regions:
+            if (area == "Bon Voyage!" or area == "Time Rift - Deep Sea") and not world.is_dlc1():
+                continue
+
+            if area == "Time Rift - Tour" and (not world.is_dlc1() or world.options.ExcludeTour):
+                continue
+
+            if area == "Bluefin Tunnel" and not world.is_dlc2():
+                continue
+
+            if world.options.DWShuffle and area in death_wishes.keys() and area not in world.dw_shuffle:
+                continue
+
+            region = world.multiworld.get_region(area, world.player)
+            event = HatInTimeLocation(world.player, f"{enemy} - {area}", None, region)
+            event.place_locked_item(HatInTimeItem(enemy, ItemClassification.progression, None, world.player))
+            region.locations.append(event)
+            event.show_in_spoiler = False
+
+    for name in triple_enemy_locations:
+        if name == "Time Rift - Tour" and (not world.is_dlc1() or world.options.ExcludeTour):
+            continue
+
+        if world.options.DWShuffle and name in death_wishes.keys() and name not in world.dw_shuffle:
+            continue
+
+        region = world.multiworld.get_region(name, world.player)
+        event = HatInTimeLocation(world.player, f"Triple Enemy Photo - {name}", None, region)
+        event.place_locked_item(HatInTimeItem("Triple Enemy Photo", ItemClassification.progression, None, world.player))
+        region.locations.append(event)
+        event.show_in_spoiler = False
+        if name == "The Mustache Gauntlet":
+            add_rule(event, lambda state: can_use_hookshot(state, world) and can_use_hat(state, world, HatType.DWELLER))
+
+
+def set_enemy_rules(world: "HatInTimeWorld"):
+    no_tourist = "Camera Tourist" in world.excluded_dws or "Camera Tourist" in world.excluded_bonuses
+
+    for enemy, regions in hit_list.items():
+        if no_tourist and enemy in bosses:
+            continue
+
+        for area in regions:
+            if (area == "Bon Voyage!" or area == "Time Rift - Deep Sea") and not world.is_dlc1():
+                continue
+
+            if area == "Time Rift - Tour" and (not world.is_dlc1() or world.options.ExcludeTour):
+                continue
+
+            if area == "Bluefin Tunnel" and not world.is_dlc2():
+                continue
+
+            if world.options.DWShuffle and area in death_wishes and area not in world.dw_shuffle:
+                continue
+
+            event = world.multiworld.get_location(f"{enemy} - {area}", world.player)
+
+            if enemy == "Toxic Flower":
+                add_rule(event, lambda state: can_use_hookshot(state, world))
+
+                if area == "The Illness has Spread":
+                    add_rule(event, lambda state: not zipline_logic(world) or
+                             state.has("Zipline Unlock - The Birdhouse Path", world.player)
+                             or state.has("Zipline Unlock - The Lava Cake Path", world.player)
+                             or state.has("Zipline Unlock - The Windmill Path", world.player))
+
+            elif enemy == "Director":
+                if area == "Dead Bird Studio Basement":
+                    add_rule(event, lambda state: can_use_hookshot(state, world))
+
+            elif enemy == "Snatcher" or enemy == "Mustache Girl":
+                if area == "Boss Rush":
+                    # need to be able to kill toilet and snatcher
+                    add_rule(event, lambda state: can_hit(state, world) and can_use_hookshot(state, world))
+                    if enemy == "Mustache Girl":
+                        add_rule(event, lambda state: can_hit(state, world, True) and can_use_hookshot(state, world))
+
+                elif area == "The Finale" and enemy == "Mustache Girl":
+                    add_rule(event, lambda state: can_use_hookshot(state, world)
+                             and can_use_hat(state, world, HatType.DWELLER))
+
+            elif enemy == "Shock Squid" or enemy == "Ninja Cat":
+                if area == "Time Rift - Deep Sea":
+                    add_rule(event, lambda state: can_use_hookshot(state, world))
+
+
+# Enemies for Snatcher's Hit List/Camera Tourist, and where to find them
+hit_list = {
+    "Mafia Goon":       ["Mafia Town Area", "Time Rift - Mafia of Cooks", "Time Rift - Tour",
+                         "Bon Voyage!", "The Mustache Gauntlet", "Rift Collapse: Mafia of Cooks",
+                         "So You're Back From Outer Space"],
+
+    "Sleepy Raccoon":   ["She Came from Outer Space", "Down with the Mafia!", "The Twilight Bell",
+                         "She Speedran from Outer Space", "Mafia's Jumps", "The Mustache Gauntlet",
+                         "Time Rift - Sleepy Subcon", "Rift Collapse: Sleepy Subcon"],
+
+    "UFO":              ["Picture Perfect", "So You're Back From Outer Space", "Community Rift: Rhythm Jump Studio"],
+
+    "Rat":              ["Down with the Mafia!", "Bluefin Tunnel"],
+
+    "Shock Squid":      ["Bon Voyage!", "Time Rift - Sleepy Subcon", "Time Rift - Deep Sea",
+                         "Rift Collapse: Sleepy Subcon"],
+
+    "Shromb Egg":       ["The Birdhouse", "Bird Sanctuary"],
+
+    "Spider":           ["Subcon Forest Area", "The Mustache Gauntlet", "Speedrun Well",
+                         "The Lava Cake", "The Windmill"],
+
+    "Crow":             ["Mafia Town Area", "The Birdhouse", "Time Rift - Tour", "Bird Sanctuary",
+                         "Time Rift - Alpine Skyline", "Rift Collapse: Alpine Skyline"],
+
+    "Pompous Crow":     ["The Birdhouse", "Time Rift - The Lab", "Bird Sanctuary", "The Mustache Gauntlet"],
+
+    "Fiery Crow":       ["The Finale", "The Lava Cake", "The Mustache Gauntlet"],
+
+    "Express Owl":      ["The Finale", "Time Rift - The Owl Express", "Time Rift - Deep Sea"],
+
+    "Ninja Cat":        ["The Birdhouse", "The Windmill", "Bluefin Tunnel", "The Mustache Gauntlet",
+                         "Time Rift - Curly Tail Trail", "Time Rift - Alpine Skyline", "Time Rift - Deep Sea",
+                         "Rift Collapse: Alpine Skyline"],
+
+    # Bosses
+    "Mafia Boss":       ["Down with the Mafia!", "Encore! Encore!", "Boss Rush"],
+
+    "Conductor":        ["Dead Bird Studio Basement", "Killing Two Birds", "Boss Rush"],
+    "Toilet":           ["Toilet of Doom", "Boss Rush"],
+
+    "Snatcher":         ["Your Contract has Expired", "Breaching the Contract", "Boss Rush",
+                         "Quality Time with Snatcher"],
+
+    "Toxic Flower":     ["The Illness has Spread", "The Illness has Speedrun"],
+
+    "Mustache Girl":    ["The Finale", "Boss Rush", "No More Bad Guys"],
+}
+
+# Camera Tourist has a bonus that requires getting three different types of enemies in one photo.
+triple_enemy_locations = [
+    "She Came from Outer Space",
+    "She Speedran from Outer Space",
+    "Mafia's Jumps",
+    "The Mustache Gauntlet",
+    "The Birdhouse",
+    "Bird Sanctuary",
+    "Time Rift - Tour",
+]
+
+bosses = [
+    "Mafia Boss",
+    "Conductor",
+    "Toilet",
+    "Snatcher",
+    "Toxic Flower",
+    "Mustache Girl",
+]

worlds/ahit/Items.py

@@ -0,0 +1,302 @@
+from BaseClasses import Item, ItemClassification
+from .Types import HatDLC, HatType, hat_type_to_item, Difficulty, ItemData, HatInTimeItem
+from .Locations import get_total_locations
+from .Rules import get_difficulty
+from .Options import get_total_time_pieces, CTRLogic
+from typing import List, Dict, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from . import HatInTimeWorld
+
+
+def create_itempool(world: "HatInTimeWorld") -> List[Item]:
+    itempool: List[Item] = []
+    if world.has_yarn():
+        yarn_pool: List[Item] = create_multiple_items(world, "Yarn",
+                                                      world.options.YarnAvailable.value,
+                                                      ItemClassification.progression_skip_balancing)
+
+        for i in range(int(len(yarn_pool) * (0.01 * world.options.YarnBalancePercent))):
+            yarn_pool[i].classification = ItemClassification.progression
+
+        itempool += yarn_pool
+
+    for name in item_table.keys():
+        if name == "Yarn":
+            continue
+
+        if not item_dlc_enabled(world, name):
+            continue
+
+        if not world.options.HatItems and name in hat_type_to_item.values():
+            continue
+
+        item_type: ItemClassification = item_table.get(name).classification
+
+        if world.is_dw_only():
+            if item_type is ItemClassification.progression \
+               or item_type is ItemClassification.progression_skip_balancing:
+                continue
+        else:
+            if name == "Scooter Badge":
+                if world.options.CTRLogic is CTRLogic.option_scooter or get_difficulty(world) >= Difficulty.MODERATE:
+                    item_type = ItemClassification.progression
+            elif name == "No Bonk Badge" and world.is_dw():
+                item_type = ItemClassification.progression
+
+        # some death wish bonuses require one hit hero + hookshot
+        if world.is_dw() and name == "Badge Pin" and not world.is_dw_only():
+            item_type = ItemClassification.progression
+
+        if item_type is ItemClassification.filler or item_type is ItemClassification.trap:
+            continue
+
+        if name in act_contracts.keys() and not world.options.ShuffleActContracts:
+            continue
+
+        if name in alps_hooks.keys() and not world.options.ShuffleAlpineZiplines:
+            continue
+
+        if name == "Progressive Painting Unlock" and not world.options.ShuffleSubconPaintings:
+            continue
+
+        if world.options.StartWithCompassBadge and name == "Compass Badge":
+            continue
+
+        if name == "Time Piece":
+            tp_list: List[Item] = create_multiple_items(world, name, get_total_time_pieces(world), item_type)
+            for i in range(int(len(tp_list) * (0.01 * world.options.TimePieceBalancePercent))):
+                tp_list[i].classification = ItemClassification.progression
+
+            itempool += tp_list
+            continue
+
+        itempool += create_multiple_items(world, name, item_frequencies.get(name, 1), item_type)
+
+    itempool += create_junk_items(world, get_total_locations(world) - len(itempool))
+    return itempool
+
+
+def calculate_yarn_costs(world: "HatInTimeWorld"):
+    min_yarn_cost = int(min(world.options.YarnCostMin.value, world.options.YarnCostMax.value))
+    max_yarn_cost = int(max(world.options.YarnCostMin.value, world.options.YarnCostMax.value))
+
+    max_cost = 0
+    for i in range(5):
+        hat: HatType = HatType(i)
+        if not world.is_hat_precollected(hat):
+            cost: int = world.random.randint(min_yarn_cost, max_yarn_cost)
+            world.hat_yarn_costs[hat] = cost
+            max_cost += cost
+        else:
+            world.hat_yarn_costs[hat] = 0
+
+    available_yarn: int = world.options.YarnAvailable.value
+    if max_cost > available_yarn:
+        world.options.YarnAvailable.value = max_cost
+        available_yarn = max_cost
+
+    extra_yarn = max_cost + world.options.MinExtraYarn - available_yarn
+    if extra_yarn > 0:
+        world.options.YarnAvailable.value += extra_yarn
+
+
+def item_dlc_enabled(world: "HatInTimeWorld", name: str) -> bool:
+    data = item_table[name]
+
+    if data.dlc_flags == HatDLC.none:
+        return True
+    elif data.dlc_flags == HatDLC.dlc1 and world.is_dlc1():
+        return True
+    elif data.dlc_flags == HatDLC.dlc2 and world.is_dlc2():
+        return True
+    elif data.dlc_flags == HatDLC.death_wish and world.is_dw():
+        return True
+
+    return False
+
+
+def create_item(world: "HatInTimeWorld", name: str) -> Item:
+    data = item_table[name]
+    return HatInTimeItem(name, data.classification, data.code, world.player)
+
+
+def create_multiple_items(world: "HatInTimeWorld", name: str, count: int = 1,
+                          item_type: ItemClassification = ItemClassification.progression) -> List[Item]:
+
+    data = item_table[name]
+    itemlist: List[Item] = []
+
+    for i in range(count):
+        itemlist += [HatInTimeItem(name, item_type, data.code, world.player)]
+
+    return itemlist
+
+
+def create_junk_items(world: "HatInTimeWorld", count: int) -> List[Item]:
+    trap_chance = world.options.TrapChance.value
+    junk_pool: List[Item] = []
+    junk_list: Dict[str, int] = {}
+    trap_list: Dict[str, int] = {}
+    ic: ItemClassification
+
+    for name in item_table.keys():
+        ic = item_table[name].classification
+        if ic == ItemClassification.filler:
+            if world.is_dw_only() and "Pons" in name:
+                continue
+
+            junk_list[name] = junk_weights.get(name)
+
+        elif trap_chance > 0 and ic == ItemClassification.trap:
+            if name == "Baby Trap":
+                trap_list[name] = world.options.BabyTrapWeight.value
+            elif name == "Laser Trap":
+                trap_list[name] = world.options.LaserTrapWeight.value
+            elif name == "Parade Trap":
+                trap_list[name] = world.options.ParadeTrapWeight.value
+
+    for i in range(count):
+        if trap_chance > 0 and world.random.randint(1, 100) <= trap_chance:
+            junk_pool.append(world.create_item(
+                world.random.choices(list(trap_list.keys()), weights=list(trap_list.values()), k=1)[0]))
+        else:
+            junk_pool.append(world.create_item(
+                world.random.choices(list(junk_list.keys()), weights=list(junk_list.values()), k=1)[0]))
+
+    return junk_pool
+
+
+def get_shop_trap_name(world: "HatInTimeWorld") -> str:
+    rand = world.random.randint(1, 9)
+    name = ""
+    if rand == 1:
+        name = "Time Plece"
+    elif rand == 2:
+        name = "Time Piece (Trust me bro)"
+    elif rand == 3:
+        name = "TimePiece"
+    elif rand == 4:
+        name = "Time Piece?"
+    elif rand == 5:
+        name = "Time Pizza"
+    elif rand == 6:
+        name = "Time piece"
+    elif rand == 7:
+        name = "TIme Piece"
+    elif rand == 8:
+        name = "Time Piece (maybe)"
+    elif rand == 9:
+        name = "Time Piece ;)"
+
+    return name
+
+
+ahit_items = {
+    "Yarn": ItemData(2000300001, ItemClassification.progression_skip_balancing),
+    "Time Piece": ItemData(2000300002, ItemClassification.progression_skip_balancing),
+
+    # for HatItems option
+    "Sprint Hat": ItemData(2000300049, ItemClassification.progression),
+    "Brewing Hat": ItemData(2000300050, ItemClassification.progression),
+    "Ice Hat": ItemData(2000300051, ItemClassification.progression),
+    "Dweller Mask": ItemData(2000300052, ItemClassification.progression),
+    "Time Stop Hat": ItemData(2000300053, ItemClassification.progression),
+
+    # Badges
+    "Projectile Badge": ItemData(2000300024, ItemClassification.useful),
+    "Fast Hatter Badge": ItemData(2000300025, ItemClassification.useful),
+    "Hover Badge": ItemData(2000300026, ItemClassification.useful),
+    "Hookshot Badge": ItemData(2000300027, ItemClassification.progression),
+    "Item Magnet Badge": ItemData(2000300028, ItemClassification.useful),
+    "No Bonk Badge": ItemData(2000300029, ItemClassification.useful),
+    "Compass Badge": ItemData(2000300030, ItemClassification.useful),
+    "Scooter Badge": ItemData(2000300031, ItemClassification.useful),
+    "One-Hit Hero Badge": ItemData(2000300038, ItemClassification.progression, HatDLC.death_wish),
+    "Camera Badge": ItemData(2000300042, ItemClassification.progression, HatDLC.death_wish),
+
+    # Relics
+    "Relic (Burger Patty)": ItemData(2000300006, ItemClassification.progression),
+    "Relic (Burger Cushion)": ItemData(2000300007, ItemClassification.progression),
+    "Relic (Mountain Set)": ItemData(2000300008, ItemClassification.progression),
+    "Relic (Train)": ItemData(2000300009, ItemClassification.progression),
+    "Relic (UFO)": ItemData(2000300010, ItemClassification.progression),
+    "Relic (Cow)": ItemData(2000300011, ItemClassification.progression),
+    "Relic (Cool Cow)": ItemData(2000300012, ItemClassification.progression),
+    "Relic (Tin-foil Hat Cow)": ItemData(2000300013, ItemClassification.progression),
+    "Relic (Crayon Box)": ItemData(2000300014, ItemClassification.progression),
+    "Relic (Red Crayon)": ItemData(2000300015, ItemClassification.progression),
+    "Relic (Blue Crayon)": ItemData(2000300016, ItemClassification.progression),
+    "Relic (Green Crayon)": ItemData(2000300017, ItemClassification.progression),
+    # DLC
+    "Relic (Cake Stand)": ItemData(2000300018, ItemClassification.progression, HatDLC.dlc1),
+    "Relic (Shortcake)": ItemData(2000300019, ItemClassification.progression, HatDLC.dlc1),
+    "Relic (Chocolate Cake Slice)": ItemData(2000300020, ItemClassification.progression, HatDLC.dlc1),
+    "Relic (Chocolate Cake)": ItemData(2000300021, ItemClassification.progression, HatDLC.dlc1),
+    "Relic (Necklace Bust)": ItemData(2000300022, ItemClassification.progression, HatDLC.dlc2),
+    "Relic (Necklace)": ItemData(2000300023, ItemClassification.progression, HatDLC.dlc2),
+
+    # Garbage items
+    "25 Pons": ItemData(2000300034, ItemClassification.filler),
+    "50 Pons": ItemData(2000300035, ItemClassification.filler),
+    "100 Pons": ItemData(2000300036, ItemClassification.filler),
+    "Health Pon": ItemData(2000300037, ItemClassification.filler),
+    "Random Cosmetic": ItemData(2000300044, ItemClassification.filler),
+
+    # Traps
+    "Baby Trap": ItemData(2000300039, ItemClassification.trap),
+    "Laser Trap": ItemData(2000300040, ItemClassification.trap),
+    "Parade Trap": ItemData(2000300041, ItemClassification.trap),
+
+    # Other
+    "Badge Pin": ItemData(2000300043, ItemClassification.useful),
+    "Umbrella": ItemData(2000300033, ItemClassification.progression),
+    "Progressive Painting Unlock": ItemData(2000300003, ItemClassification.progression),
+    # DLC
+    "Metro Ticket - Yellow": ItemData(2000300045, ItemClassification.progression, HatDLC.dlc2),
+    "Metro Ticket - Green": ItemData(2000300046, ItemClassification.progression, HatDLC.dlc2),
+    "Metro Ticket - Blue": ItemData(2000300047, ItemClassification.progression, HatDLC.dlc2),
+    "Metro Ticket - Pink": ItemData(2000300048, ItemClassification.progression, HatDLC.dlc2),
+}
+
+act_contracts = {
+    "Snatcher's Contract - The Subcon Well": ItemData(2000300200, ItemClassification.progression),
+    "Snatcher's Contract - Toilet of Doom": ItemData(2000300201, ItemClassification.progression),
+    "Snatcher's Contract - Queen Vanessa's Manor": ItemData(2000300202, ItemClassification.progression),
+    "Snatcher's Contract - Mail Delivery Service": ItemData(2000300203, ItemClassification.progression),
+}
+
+alps_hooks = {
+    "Zipline Unlock - The Birdhouse Path": ItemData(2000300204, ItemClassification.progression),
+    "Zipline Unlock - The Lava Cake Path": ItemData(2000300205, ItemClassification.progression),
+    "Zipline Unlock - The Windmill Path": ItemData(2000300206, ItemClassification.progression),
+    "Zipline Unlock - The Twilight Bell Path": ItemData(2000300207, ItemClassification.progression),
+}
+
+relic_groups = {
+    "Burger": {"Relic (Burger Patty)", "Relic (Burger Cushion)"},
+    "Train": {"Relic (Mountain Set)", "Relic (Train)"},
+    "UFO": {"Relic (UFO)", "Relic (Cow)", "Relic (Cool Cow)", "Relic (Tin-foil Hat Cow)"},
+    "Crayon": {"Relic (Crayon Box)", "Relic (Red Crayon)", "Relic (Blue Crayon)", "Relic (Green Crayon)"},
+    "Cake": {"Relic (Cake Stand)", "Relic (Chocolate Cake)", "Relic (Chocolate Cake Slice)", "Relic (Shortcake)"},
+    "Necklace": {"Relic (Necklace Bust)", "Relic (Necklace)"},
+}
+
+item_frequencies = {
+    "Badge Pin": 2,
+    "Progressive Painting Unlock": 3,
+}
+
+junk_weights = {
+    "25 Pons": 50,
+    "50 Pons": 25,
+    "100 Pons": 10,
+    "Health Pon": 35,
+    "Random Cosmetic": 35,
+}
+
+item_table = {
+    **ahit_items,
+    **act_contracts,
+    **alps_hooks,
+}