Core/LttP: remove can_reach_private

* - Added missing special order requirements, mostly for the regions where to place the collected items, or the NPC to talk to when done

* - Added missing requirement on being able to go see the wizard cutscene in order to interact with bundles

.github/workflows/analyze-modified-files.yml

@@ -0,0 +1,80 @@
+name: Analyze modified files
+
+on:
+  pull_request:
+    paths:
+      - "**.py"
+  push:
+    paths:
+      - "**.py"
+
+env:
+  BASE: ${{ github.event.pull_request.base.sha }}
+  HEAD: ${{ github.event.pull_request.head.sha }}
+  BEFORE: ${{ github.event.before }}
+  AFTER: ${{ github.event.after }}
+
+jobs:
+  flake8-or-mypy:
+    strategy:
+      fail-fast: false
+      matrix:
+        task: [flake8, mypy]
+
+    name: ${{ matrix.task }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: "Determine modified files (pull_request)"
+        if: github.event_name == 'pull_request'
+        run: |
+          git fetch origin $BASE $HEAD
+          DIFF=$(git diff --diff-filter=d --name-only $BASE...$HEAD -- "*.py")
+          echo "modified files:"
+          echo "$DIFF"
+          echo "diff=${DIFF//$'\n'/$' '}" >> $GITHUB_ENV
+
+      - name: "Determine modified files (push)"
+        if: github.event_name == 'push' && github.event.before != '0000000000000000000000000000000000000000'
+        run: |
+          git fetch origin $BEFORE $AFTER
+          DIFF=$(git diff --diff-filter=d --name-only $BEFORE..$AFTER -- "*.py")
+          echo "modified files:"
+          echo "$DIFF"
+          echo "diff=${DIFF//$'\n'/$' '}" >> $GITHUB_ENV
+
+      - name: "Treat all files as modified (new branch)"
+        if: github.event_name == 'push' && github.event.before == '0000000000000000000000000000000000000000'
+        run: |
+          echo "diff=." >> $GITHUB_ENV
+
+      - uses: actions/setup-python@v4
+        if: env.diff != ''
+        with:
+          python-version: 3.8
+
+      - name: "Install dependencies"
+        if: env.diff != ''
+        run: |
+          python -m pip install --upgrade pip ${{ matrix.task }}
+          python ModuleUpdate.py --append "WebHostLib/requirements.txt" --force --yes
+
+      - name: "flake8: Stop the build if there are Python syntax errors or undefined names"
+        continue-on-error: false
+        if: env.diff != '' && matrix.task == 'flake8'
+        run: |
+          flake8 --count --select=E9,F63,F7,F82 --show-source --statistics ${{ env.diff }}
+
+      - name: "flake8: Lint modified files"
+        continue-on-error: true
+        if: env.diff != '' && matrix.task == 'flake8'
+        run: |
+          flake8 --count --max-complexity=10 --max-doc-length=120 --max-line-length=120 --statistics ${{ env.diff }}
+
+      - name: "mypy: Type check modified files"
+        continue-on-error: true
+        if: env.diff != '' && matrix.task == 'mypy'
+        run: |
+          mypy --follow-imports=silent --install-types --non-interactive --strict ${{ env.diff }}

.github/workflows/build.yml

@@ -38,12 +38,13 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           python setup.py build_exe --yes
-          $NAME="$(ls build)".Split('.',2)[1]
+          $NAME="$(ls build | Select-String -Pattern 'exe')".Split('.',2)[1]
           $ZIP_NAME="Archipelago_$NAME.7z"
+          echo "$NAME -> $ZIP_NAME"
           echo "ZIP_NAME=$ZIP_NAME" >> $Env:GITHUB_ENV
           New-Item -Path dist -ItemType Directory -Force
           cd build
-          Rename-Item exe.$NAME Archipelago
+          Rename-Item "exe.$NAME" Archipelago
           7z a -mx=9 -mhe=on -ms "../dist/$ZIP_NAME" Archipelago
       - name: Store 7z
         uses: actions/upload-artifact@v3
@@ -65,10 +66,10 @@ jobs:
       - name: Get a recent python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.11'
       - name: Install build-time dependencies
         run: |
-          echo "PYTHON=python3.9" >> $GITHUB_ENV
+          echo "PYTHON=python3.11" >> $GITHUB_ENV
           wget -nv https://github.com/AppImage/AppImageKit/releases/download/$APPIMAGETOOL_VERSION/appimagetool-x86_64.AppImage
           chmod a+rx appimagetool-x86_64.AppImage
           ./appimagetool-x86_64.AppImage --appimage-extract

.github/workflows/lint.yml

@@ -1,35 +0,0 @@
-# This workflow will install Python dependencies, run tests and lint with a single version of Python
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-
-name: lint
-
-on:
-  push:
-    paths:
-      - '**.py'
-  pull_request:
-    paths:
-      - '**.py'
-
-jobs:
-  flake8:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.9
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip wheel
-        pip install flake8
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Lint with flake8
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics

.github/workflows/release.yml

@@ -44,10 +44,10 @@ jobs:
       - name: Get a recent python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.9'
+          python-version: '3.11'
       - name: Install build-time dependencies
         run: |
-          echo "PYTHON=python3.9" >> $GITHUB_ENV
+          echo "PYTHON=python3.11" >> $GITHUB_ENV
           wget -nv https://github.com/AppImage/AppImageKit/releases/download/$APPIMAGETOOL_VERSION/appimagetool-x86_64.AppImage
           chmod a+rx appimagetool-x86_64.AppImage
           ./appimagetool-x86_64.AppImage --appimage-extract

.github/workflows/unittests.yml

@@ -36,12 +36,13 @@ jobs:
           - {version: '3.8'}
           - {version: '3.9'}
           - {version: '3.10'}
+          - {version: '3.11'}
         include:
           - python: {version: '3.8'}  # win7 compat
             os: windows-latest
-          - python: {version: '3.10'}  # current
+          - python: {version: '3.11'}  # current
             os: windows-latest
-          - python: {version: '3.10'}  # current
+          - python: {version: '3.11'}  # current
             os: macos-latest
 
     steps:
@@ -55,6 +56,7 @@ jobs:
         python -m pip install --upgrade pip
         pip install pytest pytest-subtests
         python ModuleUpdate.py --yes --force --append "WebHostLib/requirements.txt"
+        python Launcher.py --update_settings  # make sure host.yaml exists for tests
     - name: Unittests
       run: |
         pytest

.gitignore

@@ -28,6 +28,7 @@
 *.apsave
 *.BIN
 
+setups
 build
 bundle/components.wxs
 dist
@@ -36,6 +37,7 @@ README.html
 EnemizerCLI/
 /Players/
 /SNI/
+/host.yaml
 /options.yaml
 /config.yaml
 /logs/
@@ -167,6 +169,10 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
+# Cython intermediates
+_speedups.cpp
+_speedups.html
+
 # minecraft server stuff
 jdk*/
 minecraft*/
@@ -176,6 +182,9 @@ minecraft_versions.json
 # pyenv
 .python-version
 
+#undertale stuff
+/Undertale/
+
 # OS General Files
 .DS_Store
 .AppleDouble

AdventureClient.py

@@ -396,7 +396,7 @@ async def atari_sync_task(ctx: AdventureContext):
                     ctx.atari_streams = await asyncio.wait_for(
                         asyncio.open_connection("localhost",
                                                 port),
-                                                timeout=10)
+                        timeout=10)
                     ctx.atari_status = CONNECTION_TENTATIVE_STATUS
                 except TimeoutError:
                     logger.debug("Connection Timed Out, Trying Again")

BaseClasses.py

@@ -7,9 +7,10 @@ import random
 import secrets
 import typing  # this can go away when Python 3.8 support is dropped
 from argparse import Namespace
-from collections import ChainMap, Counter, OrderedDict, deque
+from collections import ChainMap, Counter, deque
 from enum import IntEnum, IntFlag
-from typing import Any, Callable, Dict, Iterable, List, NamedTuple, Optional, Set, Tuple, TypedDict, Union
+from typing import Any, Callable, Dict, Iterable, Iterator, List, NamedTuple, Optional, Set, Tuple, TypedDict, Union, \
+    Type, ClassVar
 
 import NetUtils
 import Options
@@ -28,15 +29,15 @@ class Group(TypedDict, total=False):
     link_replacement: bool
 
 
-class ThreadBarrierProxy():
+class ThreadBarrierProxy:
     """Passes through getattr while passthrough is True"""
-    def __init__(self, obj: Any):
+    def __init__(self, obj: object) -> None:
         self.passthrough = True
         self.obj = obj
 
-    def __getattr__(self, item):
+    def __getattr__(self, name: str) -> Any:
         if self.passthrough:
-            return getattr(self.obj, item)
+            return getattr(self.obj, name)
         else:
             raise RuntimeError("You are in a threaded context and global random state was removed for your safety. "
                                "Please use multiworld.per_slot_randoms[player] or randomize ahead of output.")
@@ -81,6 +82,7 @@ class MultiWorld():
 
     random: random.Random
     per_slot_randoms: Dict[int, random.Random]
+    """Deprecated. Please use `self.random` instead."""
 
     class AttributeProxy():
         def __init__(self, rule):
@@ -96,7 +98,6 @@ class MultiWorld():
         self.player_types = {player: NetUtils.SlotType.player for player in self.player_ids}
         self.glitch_triforce = False
         self.algorithm = 'balanced'
-        self.dungeons: Dict[Tuple[str, int], Dungeon] = {}
         self.groups = {}
         self.regions = []
         self.shops = []
@@ -243,6 +244,7 @@ class MultiWorld():
                 setattr(self, option_key, getattr(args, option_key, {}))
 
             self.worlds[player] = world_type(self, player)
+            self.worlds[player].random = self.per_slot_randoms[player]
 
     def set_item_links(self):
         item_links = {}
@@ -386,12 +388,6 @@ class MultiWorld():
             self._recache()
             return self._location_cache[location, player]
 
-    def get_dungeon(self, dungeonname: str, player: int) -> Dungeon:
-        try:
-            return self.dungeons[dungeonname, player]
-        except KeyError as e:
-            raise KeyError('No such dungeon %s for player %d' % (dungeonname, player)) from e
-
     def get_all_state(self, use_cache: bool) -> CollectionState:
         cached = getattr(self, "_all_state", None)
         if use_cache and cached:
@@ -491,8 +487,10 @@ class MultiWorld():
     def get_unfilled_locations_for_players(self, location_names: List[str], players: Iterable[int]):
         for player in players:
             if not location_names:
-                location_names = [location.name for location in self.get_unfilled_locations(player)]
-            for location_name in location_names:
+                valid_locations = [location.name for location in self.get_unfilled_locations(player)]
+            else:
+                valid_locations = location_names
+            for location_name in valid_locations:
                 location = self._location_cache.get((location_name, player), None)
                 if location is not None and location.item is None:
                     yield location
@@ -793,79 +791,6 @@ class CollectionState():
             self.stale[item.player] = True
 
 
-class Region:
-    name: str
-    _hint_text: str
-    player: int
-    multiworld: Optional[MultiWorld]
-    entrances: List[Entrance]
-    exits: List[Entrance]
-    locations: List[Location]
-    dungeon: Optional[Dungeon] = None
-
-    def __init__(self, name: str, player: int, multiworld: MultiWorld, hint: Optional[str] = None):
-        self.name = name
-        self.entrances = []
-        self.exits = []
-        self.locations = []
-        self.multiworld = multiworld
-        self._hint_text = hint
-        self.player = player
-
-    def can_reach(self, state: CollectionState) -> bool:
-        if state.stale[self.player]:
-            state.update_reachable_regions(self.player)
-        return self in state.reachable_regions[self.player]
-
-    def can_reach_private(self, state: CollectionState) -> bool:
-        for entrance in self.entrances:
-            if entrance.can_reach(state):
-                if not self in state.path:
-                    state.path[self] = (self.name, state.path.get(entrance, None))
-                return True
-        return False
-
-    @property
-    def hint_text(self) -> str:
-        return self._hint_text if self._hint_text else self.name
-
-    def get_connecting_entrance(self, is_main_entrance: typing.Callable[[Entrance], bool]) -> Entrance:
-        for entrance in self.entrances:
-            if is_main_entrance(entrance):
-                return entrance
-        for entrance in self.entrances:  # BFS might be better here, trying DFS for now.
-            return entrance.parent_region.get_connecting_entrance(is_main_entrance)
-
-    def add_locations(self, locations: Dict[str, Optional[int]], location_type: Optional[typing.Type[Location]] = None) -> None:
-        """Adds locations to the Region object, where location_type is your Location class and locations is a dict of
-        location names to address."""
-        if location_type is None:
-            location_type = Location
-        for location, address in locations.items():
-            self.locations.append(location_type(self.player, location, address, self))
-
-    def add_exits(self, exits: Dict[str, Optional[str]], rules: Dict[str, Callable[[CollectionState], bool]] = None) -> None:
-        """
-        Connects current region to regions in exit dictionary. Passed region names must exist first.
-
-        :param exits: exits from the region. format is {"connecting_region", "exit_name"}
-        :param rules: rules for the exits from this region. format is {"connecting_region", rule}
-        """
-        for exiting_region, name in exits.items():
-            ret = Entrance(self.player, name, self) if name \
-                else Entrance(self.player, f"{self.name} -> {exiting_region}", self)
-            if rules and exiting_region in rules:
-                ret.access_rule = rules[exiting_region]
-            self.exits.append(ret)
-            ret.connect(self.multiworld.get_region(exiting_region, self.player))
-
-    def __repr__(self):
-        return self.__str__()
-
-    def __str__(self):
-        return self.multiworld.get_name_string_for_object(self) if self.multiworld else f'{self.name} (Player {self.player})'
-
-
 class Entrance:
     access_rule: Callable[[CollectionState], bool] = staticmethod(lambda state: True)
     hide_path: bool = False
@@ -904,41 +829,92 @@ class Entrance:
         return world.get_name_string_for_object(self) if world else f'{self.name} (Player {self.player})'
 
 
-class Dungeon(object):
-    def __init__(self, name: str, regions: List[Region], big_key: Item, small_keys: List[Item],
-                 dungeon_items: List[Item], player: int):
+class Region:
+    name: str
+    _hint_text: str
+    player: int
+    multiworld: Optional[MultiWorld]
+    entrances: List[Entrance]
+    exits: List[Entrance]
+    locations: List[Location]
+    entrance_type: ClassVar[Type[Entrance]] = Entrance
+
+    def __init__(self, name: str, player: int, multiworld: MultiWorld, hint: Optional[str] = None):
         self.name = name
-        self.regions = regions
-        self.big_key = big_key
-        self.small_keys = small_keys
-        self.dungeon_items = dungeon_items
-        self.bosses = dict()
+        self.entrances = []
+        self.exits = []
+        self.locations = []
+        self.multiworld = multiworld
+        self._hint_text = hint
         self.player = player
-        self.multiworld = None
+
+    def can_reach(self, state: CollectionState) -> bool:
+        if state.stale[self.player]:
+            state.update_reachable_regions(self.player)
+        return self in state.reachable_regions[self.player]
 
     @property
-    def boss(self) -> Optional[Boss]:
-        return self.bosses.get(None, None)
+    def hint_text(self) -> str:
+        return self._hint_text if self._hint_text else self.name
 
-    @boss.setter
-    def boss(self, value: Optional[Boss]):
-        self.bosses[None] = value
+    def get_connecting_entrance(self, is_main_entrance: Callable[[Entrance], bool]) -> Entrance:
+        for entrance in self.entrances:
+            if is_main_entrance(entrance):
+                return entrance
+        for entrance in self.entrances:  # BFS might be better here, trying DFS for now.
+            return entrance.parent_region.get_connecting_entrance(is_main_entrance)
 
-    @property
-    def keys(self) -> List[Item]:
-        return self.small_keys + ([self.big_key] if self.big_key else [])
+    def add_locations(self, locations: Dict[str, Optional[int]],
+                      location_type: Optional[Type[Location]] = None) -> None:
+        """
+        Adds locations to the Region object, where location_type is your Location class and locations is a dict of
+        location names to address.
+        
+        :param locations: dictionary of locations to be created and added to this Region `{name: ID}`
+        :param location_type: Location class to be used to create the locations with"""
+        if location_type is None:
+            location_type = Location
+        for location, address in locations.items():
+            self.locations.append(location_type(self.player, location, address, self))
+    
+    def connect(self, connecting_region: Region, name: Optional[str] = None,
+                rule: Optional[Callable[[CollectionState], bool]] = None) -> None:
+        """
+        Connects this Region to another Region, placing the provided rule on the connection.
+        
+        :param connecting_region: Region object to connect to path is `self -> exiting_region`
+        :param name: name of the connection being created
+        :param rule: callable to determine access of this connection to go from self to the exiting_region"""
+        exit_ = self.create_exit(name if name else f"{self.name} -> {connecting_region.name}")
+        if rule:
+            exit_.access_rule = rule
+        exit_.connect(connecting_region)
+    
+    def create_exit(self, name: str) -> Entrance:
+        """
+        Creates and returns an Entrance object as an exit of this region.
+        
+        :param name: name of the Entrance being created
+        """
+        exit_ = self.entrance_type(self.player, name, self)
+        self.exits.append(exit_)
+        return exit_
 
-    @property
-    def all_items(self) -> List[Item]:
-        return self.dungeon_items + self.keys
+    def add_exits(self, exits: Union[Iterable[str], Dict[str, Optional[str]]],
+                  rules: Dict[str, Callable[[CollectionState], bool]] = None) -> None:
+        """
+        Connects current region to regions in exit dictionary. Passed region names must exist first.
 
-    def is_dungeon_item(self, item: Item) -> bool:
-        return item.player == self.player and item.name in (dungeon_item.name for dungeon_item in self.all_items)
-
-    def __eq__(self, other: Dungeon) -> bool:
-        if not other:
-            return False
-        return self.name == other.name and self.player == other.player
+        :param exits: exits from the region. format is {"connecting_region": "exit_name"}. if a non dict is provided,
+        created entrances will be named "self.name -> connecting_region"
+        :param rules: rules for the exits from this region. format is {"connecting_region", rule}
+        """
+        if not isinstance(exits, Dict):
+            exits = dict.fromkeys(exits)
+        for connecting_region, name in exits.items():
+            self.connect(self.multiworld.get_region(connecting_region, self.player),
+                         name,
+                         rules[connecting_region] if rules and connecting_region in rules else None)
 
     def __repr__(self):
         return self.__str__()
@@ -947,20 +923,6 @@ class Dungeon(object):
         return self.multiworld.get_name_string_for_object(self) if self.multiworld else f'{self.name} (Player {self.player})'
 
 
-class Boss():
-    def __init__(self, name: str, enemizer_name: str, defeat_rule: Callable, player: int):
-        self.name = name
-        self.enemizer_name = enemizer_name
-        self.defeat_rule = defeat_rule
-        self.player = player
-
-    def can_defeat(self, state) -> bool:
-        return self.defeat_rule(state, self.player)
-
-    def __repr__(self):
-        return f"Boss({self.name})"
-
-
 class LocationProgressType(IntEnum):
     DEFAULT = 1
     PRIORITY = 2
@@ -1093,15 +1055,19 @@ class Item:
     def flags(self) -> int:
         return self.classification.as_flag()
 
-    def __eq__(self, other):
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Item):
+            return NotImplemented
         return self.name == other.name and self.player == other.player
 
-    def __lt__(self, other: Item) -> bool:
+    def __lt__(self, other: object) -> bool:
+        if not isinstance(other, Item):
+            return NotImplemented
         if other.player != self.player:
             return other.player < self.player
         return self.name < other.name
 
-    def __hash__(self):
+    def __hash__(self) -> int:
         return hash((self.name, self.player))
 
     def __repr__(self) -> str:
@@ -1113,33 +1079,44 @@ class Item:
         return f"{self.name} (Player {self.player})"
 
 
-class Spoiler():
-    multiworld: MultiWorld
-    unreachables: Set[Location]
+class EntranceInfo(TypedDict, total=False):
+    player: int
+    entrance: str
+    exit: str
+    direction: str
 
-    def __init__(self, world):
-        self.multiworld = world
+
+class Spoiler:
+    multiworld: MultiWorld
+    hashes: Dict[int, str]
+    entrances: Dict[Tuple[str, str, int], EntranceInfo]
+    playthrough: Dict[str, Union[List[str], Dict[str, str]]]  # sphere "0" is list, others are dict
+    unreachables: Set[Location]
+    paths: Dict[str, List[Union[Tuple[str, str], Tuple[str, None]]]]  # last step takes no further exits
+
+    def __init__(self, multiworld: MultiWorld) -> None:
+        self.multiworld = multiworld
         self.hashes = {}
-        self.entrances = OrderedDict()
+        self.entrances = {}
         self.playthrough = {}
         self.unreachables = set()
         self.paths = {}
 
-    def set_entrance(self, entrance: str, exit_: str, direction: str, player: int):
+    def set_entrance(self, entrance: str, exit_: str, direction: str, player: int) -> None:
         if self.multiworld.players == 1:
-            self.entrances[(entrance, direction, player)] = OrderedDict(
-                [('entrance', entrance), ('exit', exit_), ('direction', direction)])
+            self.entrances[(entrance, direction, player)] = \
+                {"entrance": entrance, "exit": exit_, "direction": direction}
         else:
-            self.entrances[(entrance, direction, player)] = OrderedDict(
-                [('player', player), ('entrance', entrance), ('exit', exit_), ('direction', direction)])
+            self.entrances[(entrance, direction, player)] = \
+                {"player": player, "entrance": entrance, "exit": exit_, "direction": direction}
 
-    def create_playthrough(self, create_paths: bool = True):
+    def create_playthrough(self, create_paths: bool = True) -> None:
         """Destructive to the world while it is run, damage gets repaired afterwards."""
         from itertools import chain
         # get locations containing progress items
         multiworld = self.multiworld
         prog_locations = {location for location in multiworld.get_filled_locations() if location.item.advancement}
-        state_cache = [None]
+        state_cache: List[Optional[CollectionState]] = [None]
         collection_spheres: List[Set[Location]] = []
         state = CollectionState(multiworld)
         sphere_candidates = set(prog_locations)
@@ -1248,17 +1225,17 @@ class Spoiler():
         for item in removed_precollected:
             multiworld.push_precollected(item)
 
-    def create_paths(self, state: CollectionState, collection_spheres: List[Set[Location]]):
+    def create_paths(self, state: CollectionState, collection_spheres: List[Set[Location]]) -> None:
         from itertools import zip_longest
         multiworld = self.multiworld
 
-        def flist_to_iter(node):
-            while node:
-                value, node = node
-                yield value
+        def flist_to_iter(path_value: Optional[PathValue]) -> Iterator[str]:
+            while path_value:
+                region_or_entrance, path_value = path_value
+                yield region_or_entrance
 
-        def get_path(state, region):
-            reversed_path_as_flist = state.path.get(region, (region, None))
+        def get_path(state: CollectionState, region: Region) -> List[Union[Tuple[str, str], Tuple[str, None]]]:
+            reversed_path_as_flist: PathValue = state.path.get(region, (str(region), None))
             string_path_flat = reversed(list(map(str, flist_to_iter(reversed_path_as_flist))))
             # Now we combine the flat string list into (region, exit) pairs
             pathsiter = iter(string_path_flat)
@@ -1284,14 +1261,11 @@ class Spoiler():
                         self.paths[str(multiworld.get_region('Inverted Big Bomb Shop', player))] = \
                             get_path(state, multiworld.get_region('Inverted Big Bomb Shop', player))
 
-    def to_file(self, filename: str):
-        def write_option(option_key: str, option_obj: type(Options.Option)):
+    def to_file(self, filename: str) -> None:
+        def write_option(option_key: str, option_obj: Options.AssembleOptions) -> None:
             res = getattr(self.multiworld, option_key)[player]
             display_name = getattr(option_obj, "display_name", option_key)
-            try:
-                outfile.write(f'{display_name + ":":33}{res.current_option_name}\n')
-            except:
-                raise Exception
+            outfile.write(f"{display_name + ':':33}{res.current_option_name}\n")
 
         with open(filename, 'w', encoding="utf-8-sig") as outfile:
             outfile.write(
@@ -1324,15 +1298,15 @@ class Spoiler():
             AutoWorld.call_all(self.multiworld, "write_spoiler", outfile)
 
             locations = [(str(location), str(location.item) if location.item is not None else "Nothing")
-                              for location in self.multiworld.get_locations() if location.show_in_spoiler]
+                         for location in self.multiworld.get_locations() if location.show_in_spoiler]
             outfile.write('\n\nLocations:\n\n')
             outfile.write('\n'.join(
                 ['%s: %s' % (location, item) for location, item in locations]))
 
             outfile.write('\n\nPlaythrough:\n\n')
             outfile.write('\n'.join(['%s: {\n%s\n}' % (sphere_nr, '\n'.join(
-                ['  %s: %s' % (location, item) for (location, item) in sphere.items()] if sphere_nr != '0' else [
-                    f'  {item}' for item in sphere])) for (sphere_nr, sphere) in self.playthrough.items()]))
+                [f"  {location}: {item}" for (location, item) in sphere.items()] if isinstance(sphere, dict) else
+                [f"  {item}" for item in sphere])) for (sphere_nr, sphere) in self.playthrough.items()]))
             if self.unreachables:
                 outfile.write('\n\nUnreachable Items:\n\n')
                 outfile.write(
@@ -1393,23 +1367,21 @@ class PlandoOptions(IntFlag):
     @classmethod
     def _handle_part(cls, part: str, base: PlandoOptions) -> PlandoOptions:
         try:
-            part = cls[part]
+            return base | cls[part]
         except Exception as e:
             raise KeyError(f"{part} is not a recognized name for a plando module. "
-                           f"Known options: {', '.join(flag.name for flag in cls)}") from e
-        else:
-            return base | part
+                           f"Known options: {', '.join(str(flag.name) for flag in cls)}") from e
 
     def __str__(self) -> str:
         if self.value:
-            return ", ".join(flag.name for flag in PlandoOptions if self.value & flag.value)
+            return ", ".join(str(flag.name) for flag in PlandoOptions if self.value & flag.value)
         return "None"
 
 
 seeddigits = 20
 
 
-def get_seed(seed=None) -> int:
+def get_seed(seed: Optional[int] = None) -> int:
     if seed is None:
         random.seed(None)
         return random.randint(0, pow(10, seeddigits) - 1)

CommonClient.py

@@ -23,6 +23,7 @@ from NetUtils import Endpoint, decode, NetworkItem, encode, JSONtoTextParser, \
 from Utils import Version, stream_input, async_start
 from worlds import network_data_package, AutoWorldRegister
 import os
+import ssl
 
 if typing.TYPE_CHECKING:
     import kvui
@@ -33,6 +34,12 @@ logger = logging.getLogger("Client")
 gui_enabled = not sys.stdout or "--nogui" not in sys.argv
 
 
+@Utils.cache_argsless
+def get_ssl_context():
+    import certifi
+    return ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=certifi.where())
+
+
 class ClientCommandProcessor(CommandProcessor):
     def __init__(self, ctx: CommonContext):
         self.ctx = ctx
@@ -184,6 +191,10 @@ class CommonContext:
     server_locations: typing.Set[int]  # all locations the server knows of, missing_location | checked_locations
     locations_info: typing.Dict[int, NetworkItem]
 
+    # data storage
+    stored_data: typing.Dict[str, typing.Any]
+    stored_data_notification_keys: typing.Set[str]
+
     # internals
     # current message box through kvui
     _messagebox: typing.Optional["kvui.MessageBox"] = None
@@ -219,6 +230,9 @@ class CommonContext:
         self.server_locations = set()  # all locations the server knows of, missing_location | checked_locations
         self.locations_info = {}
 
+        self.stored_data = {}
+        self.stored_data_notification_keys = set()
+
         self.input_queue = asyncio.Queue()
         self.input_requests = 0
 
@@ -460,6 +474,21 @@ class CommonContext:
         for game, game_data in data_package["games"].items():
             Utils.store_data_package_for_checksum(game, game_data)
 
+    # data storage
+
+    def set_notify(self, *keys: str) -> None:
+        """Subscribe to be notified of changes to selected data storage keys.
+
+        The values can be accessed via the "stored_data" attribute of this context, which is a dictionary mapping the
+        names of the data storage keys to the latest values received from the server.
+        """
+        if new_keys := (set(keys) - self.stored_data_notification_keys):
+            self.stored_data_notification_keys.update(new_keys)
+            async_start(self.send_msgs([{"cmd": "Get",
+                                         "keys": list(new_keys)},
+                                        {"cmd": "SetNotify",
+                                         "keys": list(new_keys)}]))
+
     # DeathLink hooks
 
     def on_deathlink(self, data: typing.Dict[str, typing.Any]) -> None:
@@ -589,7 +618,8 @@ async def server_loop(ctx: CommonContext, address: typing.Optional[str] = None)
 
     logger.info(f'Connecting to Archipelago server at {address}')
     try:
-        socket = await websockets.connect(address, port=port, ping_timeout=None, ping_interval=None)
+        socket = await websockets.connect(address, port=port, ping_timeout=None, ping_interval=None,
+                                          ssl=get_ssl_context() if address.startswith("wss://") else None)
         if ctx.ui is not None:
             ctx.ui.update_address_bar(server_url.netloc)
         ctx.server = Endpoint(socket)
@@ -604,6 +634,7 @@ async def server_loop(ctx: CommonContext, address: typing.Optional[str] = None)
     except websockets.InvalidMessage:
         # probably encrypted
         if address.startswith("ws://"):
+            # try wss
             await server_loop(ctx, "ws" + address[1:])
         else:
             ctx.handle_connection_loss(f"Lost connection to the multiworld server due to InvalidMessage"
@@ -728,6 +759,11 @@ async def process_server_cmd(ctx: CommonContext, args: dict):
         if ctx.locations_scouted:
             msgs.append({"cmd": "LocationScouts",
                          "locations": list(ctx.locations_scouted)})
+        if ctx.stored_data_notification_keys:
+            msgs.append({"cmd": "Get",
+                         "keys": list(ctx.stored_data_notification_keys)})
+            msgs.append({"cmd": "SetNotify",
+                         "keys": list(ctx.stored_data_notification_keys)})
         if msgs:
             await ctx.send_msgs(msgs)
         if ctx.finished_game:
@@ -791,8 +827,13 @@ async def process_server_cmd(ctx: CommonContext, args: dict):
         # we can skip checking "DeathLink" in ctx.tags, as otherwise we wouldn't have been send this
         if "DeathLink" in tags and ctx.last_death_link != args["data"]["time"]:
             ctx.on_deathlink(args["data"])
+
+    elif cmd == "Retrieved":
+        ctx.stored_data.update(args["keys"])
+
     elif cmd == "SetReply":
-        if args["key"] == "EnergyLink":
+        ctx.stored_data[args["key"]] = args["value"]
+        if args["key"].startswith("EnergyLink"):
             ctx.current_energy_link_value = args["value"]
             if ctx.ui:
                 ctx.ui.set_new_energy_link_value()
@@ -832,10 +873,9 @@ def get_base_parser(description: typing.Optional[str] = None):
     return parser
 
 
-if __name__ == '__main__':
-    # Text Mode to use !hint and such with games that have no text entry
-
+def run_as_textclient():
     class TextContext(CommonContext):
+        # Text Mode to use !hint and such with games that have no text entry
         tags = {"AP", "TextOnly"}
         game = ""  # empty matches any game since 0.3.2
         items_handling = 0b111  # receive all items for /received
@@ -850,12 +890,11 @@ if __name__ == '__main__':
         def on_package(self, cmd: str, args: dict):
             if cmd == "Connected":
                 self.game = self.slot_info[self.slot].game
-        
+
         async def disconnect(self, allow_autoreconnect: bool = False):
             self.game = ""
             await super().disconnect(allow_autoreconnect)
 
-
     async def main(args):
         ctx = TextContext(args.connect, args.password)
         ctx.auth = args.name
@@ -868,7 +907,6 @@ if __name__ == '__main__':
         await ctx.exit_event.wait()
         await ctx.shutdown()
 
-
     import colorama
 
     parser = get_base_parser(description="Gameless Archipelago Client, for text interfacing.")
@@ -888,3 +926,7 @@ if __name__ == '__main__':
 
     asyncio.run(main(args))
     colorama.deinit()
+
+
+if __name__ == '__main__':
+    run_as_textclient()

FF1Client.py

@@ -33,7 +33,7 @@ class FF1CommandProcessor(ClientCommandProcessor):
             logger.info(f"NES Status: {self.ctx.nes_status}")
 
     def _cmd_toggle_msgs(self):
-        """Toggle displaying messages in bizhawk"""
+        """Toggle displaying messages in EmuHawk"""
         global DISPLAY_MSGS
         DISPLAY_MSGS = not DISPLAY_MSGS
         logger.info(f"Messages are now {'enabled' if DISPLAY_MSGS  else 'disabled'}")

FactorioClient.py

@@ -1,553 +1,12 @@
 from __future__ import annotations
-import os
-import logging
-import json
-import string
-import copy
-import re
-import subprocess
-import sys
-import time
-import random
-import typing
 
 import ModuleUpdate
 ModuleUpdate.update()
 
-import factorio_rcon
-import colorama
-import asyncio
-from queue import Queue
+from worlds.factorio.Client import check_stdin, launch
 import Utils
 
-def check_stdin() -> None:
-    if Utils.is_windows and sys.stdin:
-        print("WARNING: Console input is not routed reliably on Windows, use the GUI instead.")
-
 if __name__ == "__main__":
     Utils.init_logging("FactorioClient", exception_logger="Client")
     check_stdin()
-
-from CommonClient import CommonContext, server_loop, ClientCommandProcessor, logger, gui_enabled, get_base_parser
-from MultiServer import mark_raw
-from NetUtils import NetworkItem, ClientStatus, JSONtoTextParser, JSONMessagePart
-from Utils import async_start
-
-from worlds.factorio import Factorio
-
-
-class FactorioCommandProcessor(ClientCommandProcessor):
-    ctx: FactorioContext
-
-    def _cmd_energy_link(self):
-        """Print the status of the energy link."""
-        self.output(f"Energy Link: {self.ctx.energy_link_status}")
-
-    @mark_raw
-    def _cmd_factorio(self, text: str) -> bool:
-        """Send the following command to the bound Factorio Server."""
-        if self.ctx.rcon_client:
-            # TODO: Print the command non-silently only for race seeds, or otherwise block anything but /factorio /save in race seeds.
-            self.ctx.print_to_game(f"/factorio {text}")
-            result = self.ctx.rcon_client.send_command(text)
-            if result:
-                self.output(result)
-            return True
-        return False
-
-    def _cmd_resync(self):
-        """Manually trigger a resync."""
-        self.ctx.awaiting_bridge = True
-
-    def _cmd_toggle_send_filter(self):
-        """Toggle filtering of item sends that get displayed in-game to only those that involve you."""
-        self.ctx.toggle_filter_item_sends()
-
-    def _cmd_toggle_chat(self):
-        """Toggle sending of chat messages from players on the Factorio server to Archipelago."""
-        self.ctx.toggle_bridge_chat_out()
-
-class FactorioContext(CommonContext):
-    command_processor = FactorioCommandProcessor
-    game = "Factorio"
-    items_handling = 0b111  # full remote
-
-    # updated by spinup server
-    mod_version: Utils.Version = Utils.Version(0, 0, 0)
-
-    def __init__(self, server_address, password):
-        super(FactorioContext, self).__init__(server_address, password)
-        self.send_index: int = 0
-        self.rcon_client = None
-        self.awaiting_bridge = False
-        self.write_data_path = None
-        self.death_link_tick: int = 0  # last send death link on Factorio layer
-        self.factorio_json_text_parser = FactorioJSONtoTextParser(self)
-        self.energy_link_increment = 0
-        self.last_deplete = 0
-        self.filter_item_sends: bool = False
-        self.multiplayer: bool = False  # whether multiple different players have connected
-        self.bridge_chat_out: bool = True
-
-    async def server_auth(self, password_requested: bool = False):
-        if password_requested and not self.password:
-            await super(FactorioContext, self).server_auth(password_requested)
-
-        if self.rcon_client:
-            await get_info(self, self.rcon_client)  # retrieve current auth code
-        else:
-            raise Exception("Cannot connect to a server with unknown own identity, "
-                            "bridge to Factorio first.")
-
-        await self.send_connect()
-
-    def on_print(self, args: dict):
-        super(FactorioContext, self).on_print(args)
-        if self.rcon_client:
-            if not args['text'].startswith(self.player_names[self.slot] + ":"):
-                self.print_to_game(args['text'])
-
-    def on_print_json(self, args: dict):
-        if self.rcon_client:
-            if (not self.filter_item_sends or not self.is_uninteresting_item_send(args)) \
-                    and not self.is_echoed_chat(args):
-                text = self.factorio_json_text_parser(copy.deepcopy(args["data"]))
-                if not text.startswith(self.player_names[self.slot] + ":"): # TODO: Remove string heuristic in the future.
-                    self.print_to_game(text)
-        super(FactorioContext, self).on_print_json(args)
-
-    @property
-    def savegame_name(self) -> str:
-        return f"AP_{self.seed_name}_{self.auth}_Save.zip"
-
-    def print_to_game(self, text):
-        self.rcon_client.send_command(f"/ap-print [font=default-large-bold]Archipelago:[/font] "
-                                      f"{text}")
-
-    @property
-    def energy_link_status(self) -> str:
-        if not self.energy_link_increment:
-            return "Disabled"
-        elif self.current_energy_link_value is None:
-            return "Standby"
-        else:
-            return f"{Utils.format_SI_prefix(self.current_energy_link_value)}J"
-
-    def on_deathlink(self, data: dict):
-        if self.rcon_client:
-            self.rcon_client.send_command(f"/ap-deathlink {data['source']}")
-        super(FactorioContext, self).on_deathlink(data)
-
-    def on_package(self, cmd: str, args: dict):
-        if cmd in {"Connected", "RoomUpdate"}:
-            # catch up sync anything that is already cleared.
-            if "checked_locations" in args and args["checked_locations"]:
-                self.rcon_client.send_commands({item_name: f'/ap-get-technology ap-{item_name}-\t-1' for
-                                                item_name in args["checked_locations"]})
-            if cmd == "Connected" and self.energy_link_increment:
-                async_start(self.send_msgs([{
-                    "cmd": "SetNotify", "keys": ["EnergyLink"]
-                }]))
-        elif cmd == "SetReply":
-            if args["key"] == "EnergyLink":
-                if self.energy_link_increment and args.get("last_deplete", -1) == self.last_deplete:
-                    # it's our deplete request
-                    gained = int(args["original_value"] - args["value"])
-                    gained_text = Utils.format_SI_prefix(gained) + "J"
-                    if gained:
-                        logger.debug(f"EnergyLink: Received {gained_text}. "
-                                     f"{Utils.format_SI_prefix(args['value'])}J remaining.")
-                        self.rcon_client.send_command(f"/ap-energylink {gained}")
-
-    def on_user_say(self, text: str) -> typing.Optional[str]:
-        # Mirror chat sent from the UI to the Factorio server.
-        self.print_to_game(f"{self.player_names[self.slot]}: {text}")
-        return text
-
-    async def chat_from_factorio(self, user: str, message: str) -> None:
-        if not self.bridge_chat_out:
-            return
-
-        # Pass through commands
-        if message.startswith("!"):
-            await self.send_msgs([{"cmd": "Say", "text": message}])
-            return
-
-        # Omit messages that contain local coordinates
-        if "[gps=" in message:
-            return
-
-        prefix = f"({user}) " if self.multiplayer else ""
-        await self.send_msgs([{"cmd": "Say", "text": f"{prefix}{message}"}])
-
-    def toggle_filter_item_sends(self) -> None:
-        self.filter_item_sends = not self.filter_item_sends
-        if self.filter_item_sends:
-            announcement = "Item sends are now filtered."
-        else:
-            announcement = "Item sends are no longer filtered."
-        logger.info(announcement)
-        self.print_to_game(announcement)
-
-    def toggle_bridge_chat_out(self) -> None:
-        self.bridge_chat_out = not self.bridge_chat_out
-        if self.bridge_chat_out:
-            announcement = "Chat is now bridged to Archipelago."
-        else:
-            announcement = "Chat is no longer bridged to Archipelago."
-        logger.info(announcement)
-        self.print_to_game(announcement)
-
-    def run_gui(self):
-        from kvui import GameManager
-
-        class FactorioManager(GameManager):
-            logging_pairs = [
-                ("Client", "Archipelago"),
-                ("FactorioServer", "Factorio Server Log"),
-                ("FactorioWatcher", "Bridge Data Log"),
-            ]
-            base_title = "Archipelago Factorio Client"
-
-        self.ui = FactorioManager(self)
-        self.ui_task = asyncio.create_task(self.ui.async_run(), name="UI")
-
-
-async def game_watcher(ctx: FactorioContext):
-    bridge_logger = logging.getLogger("FactorioWatcher")
-    next_bridge = time.perf_counter() + 1
-    try:
-        while not ctx.exit_event.is_set():
-            # TODO: restore on-demand refresh
-            if ctx.rcon_client and time.perf_counter() > next_bridge:
-                next_bridge = time.perf_counter() + 1
-                ctx.awaiting_bridge = False
-                data = json.loads(ctx.rcon_client.send_command("/ap-sync"))
-                if not ctx.auth:
-                    pass  # auth failed, wait for new attempt
-                elif data["slot_name"] != ctx.auth:
-                    bridge_logger.warning(f"Connected World is not the expected one {data['slot_name']} != {ctx.auth}")
-                elif data["seed_name"] != ctx.seed_name:
-                    bridge_logger.warning(
-                        f"Connected Multiworld is not the expected one {data['seed_name']} != {ctx.seed_name}")
-                else:
-                    data = data["info"]
-                    research_data = data["research_done"]
-                    research_data = {int(tech_name.split("-")[1]) for tech_name in research_data}
-                    victory = data["victory"]
-                    await ctx.update_death_link(data["death_link"])
-                    ctx.multiplayer = data.get("multiplayer", False)
-
-                    if not ctx.finished_game and victory:
-                        await ctx.send_msgs([{"cmd": "StatusUpdate", "status": ClientStatus.CLIENT_GOAL}])
-                        ctx.finished_game = True
-
-                    if ctx.locations_checked != research_data:
-                        bridge_logger.debug(
-                            f"New researches done: "
-                            f"{[ctx.location_names[rid] for rid in research_data - ctx.locations_checked]}")
-                        ctx.locations_checked = research_data
-                        await ctx.send_msgs([{"cmd": 'LocationChecks', "locations": tuple(research_data)}])
-                    death_link_tick = data.get("death_link_tick", 0)
-                    if death_link_tick != ctx.death_link_tick:
-                        ctx.death_link_tick = death_link_tick
-                        if "DeathLink" in ctx.tags:
-                            async_start(ctx.send_death())
-                    if ctx.energy_link_increment:
-                        in_world_bridges = data["energy_bridges"]
-                        if in_world_bridges:
-                            in_world_energy = data["energy"]
-                            if in_world_energy < (ctx.energy_link_increment * in_world_bridges):
-                                # attempt to refill
-                                ctx.last_deplete = time.time()
-                                async_start(ctx.send_msgs([{
-                                    "cmd": "Set", "key": "EnergyLink", "operations":
-                                        [{"operation": "add", "value": -ctx.energy_link_increment * in_world_bridges},
-                                         {"operation": "max", "value": 0}],
-                                    "last_deplete": ctx.last_deplete
-                                }]))
-                            # Above Capacity - (len(Bridges) * ENERGY_INCREMENT)
-                            elif in_world_energy > (in_world_bridges * ctx.energy_link_increment * 5) - \
-                                ctx.energy_link_increment*in_world_bridges:
-                                value = ctx.energy_link_increment * in_world_bridges
-                                async_start(ctx.send_msgs([{
-                                    "cmd": "Set", "key": "EnergyLink", "operations":
-                                        [{"operation": "add", "value": value}]
-                                }]))
-                                ctx.rcon_client.send_command(
-                                    f"/ap-energylink -{value}")
-                                logger.debug(f"EnergyLink: Sent {Utils.format_SI_prefix(value)}J")
-
-            await asyncio.sleep(0.1)
-
-    except Exception as e:
-        logging.exception(e)
-        logging.error("Aborted Factorio Server Bridge")
-
-
-def stream_factorio_output(pipe, queue, process):
-    pipe.reconfigure(errors="replace")
-
-    def queuer():
-        while process.poll() is None:
-            text = pipe.readline().strip()
-            if text:
-                queue.put_nowait(text)
-
-    from threading import Thread
-
-    thread = Thread(target=queuer, name="Factorio Output Queue", daemon=True)
-    thread.start()
-    return thread
-
-
-async def factorio_server_watcher(ctx: FactorioContext):
-    savegame_name = os.path.abspath(ctx.savegame_name)
-    if not os.path.exists(savegame_name):
-        logger.info(f"Creating savegame {savegame_name}")
-        subprocess.run((
-            executable, "--create", savegame_name, "--preset", "archipelago"
-        ))
-    factorio_process = subprocess.Popen((executable, "--start-server", ctx.savegame_name,
-                                         *(str(elem) for elem in server_args)),
-                                        stderr=subprocess.PIPE,
-                                        stdout=subprocess.PIPE,
-                                        stdin=subprocess.DEVNULL,
-                                        encoding="utf-8")
-    factorio_server_logger.info("Started Factorio Server")
-    factorio_queue = Queue()
-    stream_factorio_output(factorio_process.stdout, factorio_queue, factorio_process)
-    stream_factorio_output(factorio_process.stderr, factorio_queue, factorio_process)
-    try:
-        while not ctx.exit_event.is_set():
-            if factorio_process.poll() is not None:
-                factorio_server_logger.info("Factorio server has exited.")
-                ctx.exit_event.set()
-
-            while not factorio_queue.empty():
-                msg = factorio_queue.get()
-                factorio_queue.task_done()
-
-                if not ctx.rcon_client and "Starting RCON interface at IP ADDR:" in msg:
-                    ctx.rcon_client = factorio_rcon.RCONClient("localhost", rcon_port, rcon_password)
-                    if not ctx.server:
-                        logger.info("Established bridge to Factorio Server. "
-                                    "Ready to connect to Archipelago via /connect")
-                        check_stdin()
-
-                if not ctx.awaiting_bridge and "Archipelago Bridge Data available for game tick " in msg:
-                    ctx.awaiting_bridge = True
-                    factorio_server_logger.debug(msg)
-                elif re.match(r"^[0-9.]+ Script @[^ ]+\.lua:\d+: Player command energy-link$", msg):
-                    factorio_server_logger.debug(msg)
-                    ctx.print_to_game(f"Energy Link: {ctx.energy_link_status}")
-                elif re.match(r"^[0-9.]+ Script @[^ ]+\.lua:\d+: Player command toggle-ap-send-filter$", msg):
-                    factorio_server_logger.debug(msg)
-                    ctx.toggle_filter_item_sends()
-                elif re.match(r"^[0-9.]+ Script @[^ ]+\.lua:\d+: Player command toggle-ap-chat$", msg):
-                    factorio_server_logger.debug(msg)
-                    ctx.toggle_bridge_chat_out()
-                else:
-                    factorio_server_logger.info(msg)
-                    match = re.match(r"^\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d \[CHAT\] ([^:]+): (.*)$", msg)
-                    if match:
-                        await ctx.chat_from_factorio(match.group(1), match.group(2))
-            if ctx.rcon_client:
-                commands = {}
-                while ctx.send_index < len(ctx.items_received):
-                    transfer_item: NetworkItem = ctx.items_received[ctx.send_index]
-                    item_id = transfer_item.item
-                    player_name = ctx.player_names[transfer_item.player]
-                    if item_id not in Factorio.item_id_to_name:
-                        factorio_server_logger.error(f"Cannot send unknown item ID: {item_id}")
-                    else:
-                        item_name = Factorio.item_id_to_name[item_id]
-                        factorio_server_logger.info(f"Sending {item_name} to Nauvis from {player_name}.")
-                        commands[ctx.send_index] = f'/ap-get-technology {item_name}\t{ctx.send_index}\t{player_name}'
-                    ctx.send_index += 1
-                if commands:
-                    ctx.rcon_client.send_commands(commands)
-            await asyncio.sleep(0.1)
-
-    except Exception as e:
-        logging.exception(e)
-        logging.error("Aborted Factorio Server Bridge")
-        ctx.exit_event.set()
-
-    finally:
-        if factorio_process.poll() is not None:
-            if ctx.rcon_client:
-                ctx.rcon_client.close()
-                ctx.rcon_client = None
-            return
-
-        sent_quit = False
-        if ctx.rcon_client:
-            # Attempt clean quit through RCON.
-            try:
-                ctx.rcon_client.send_command("/quit")
-            except factorio_rcon.RCONNetworkError:
-                pass
-            else:
-                sent_quit = True
-            ctx.rcon_client.close()
-            ctx.rcon_client = None
-        if not sent_quit:
-            # Attempt clean quit using SIGTERM. (Note that on Windows this kills the process instead.)
-            factorio_process.terminate()
-
-        try:
-            factorio_process.wait(10)
-        except subprocess.TimeoutExpired:
-            factorio_process.kill()
-
-
-async def get_info(ctx: FactorioContext, rcon_client: factorio_rcon.RCONClient):
-    info = json.loads(rcon_client.send_command("/ap-rcon-info"))
-    ctx.auth = info["slot_name"]
-    ctx.seed_name = info["seed_name"]
-    # 0.2.0 addition, not present earlier
-    death_link = bool(info.get("death_link", False))
-    ctx.energy_link_increment = info.get("energy_link", 0)
-    logger.debug(f"Energy Link Increment: {ctx.energy_link_increment}")
-    if ctx.energy_link_increment and ctx.ui:
-        ctx.ui.enable_energy_link()
-    await ctx.update_death_link(death_link)
-
-
-async def factorio_spinup_server(ctx: FactorioContext) -> bool:
-    savegame_name = os.path.abspath("Archipelago.zip")
-    if not os.path.exists(savegame_name):
-        logger.info(f"Creating savegame {savegame_name}")
-        subprocess.run((
-            executable, "--create", savegame_name
-        ))
-    factorio_process = subprocess.Popen(
-        (executable, "--start-server", savegame_name, *(str(elem) for elem in server_args)),
-        stderr=subprocess.PIPE,
-        stdout=subprocess.PIPE,
-        stdin=subprocess.DEVNULL,
-        encoding="utf-8")
-    factorio_server_logger.info("Started Information Exchange Factorio Server")
-    factorio_queue = Queue()
-    stream_factorio_output(factorio_process.stdout, factorio_queue, factorio_process)
-    stream_factorio_output(factorio_process.stderr, factorio_queue, factorio_process)
-    rcon_client = None
-    try:
-        while not ctx.auth:
-            while not factorio_queue.empty():
-                msg = factorio_queue.get()
-                factorio_server_logger.info(msg)
-                if "Loading mod AP-" in msg and msg.endswith("(data.lua)"):
-                    parts = msg.split()
-                    ctx.mod_version = Utils.Version(*(int(number) for number in parts[-2].split(".")))
-                elif "Write data path: " in msg:
-                    ctx.write_data_path = Utils.get_text_between(msg, "Write data path: ", " [")
-                    if "AppData" in ctx.write_data_path:
-                        logger.warning("It appears your mods are loaded from Appdata, "
-                                       "this can lead to problems with multiple Factorio instances. "
-                                       "If this is the case, you will get a file locked error running Factorio.")
-                if not rcon_client and "Starting RCON interface at IP ADDR:" in msg:
-                    rcon_client = factorio_rcon.RCONClient("localhost", rcon_port, rcon_password)
-                    if ctx.mod_version == ctx.__class__.mod_version:
-                        raise Exception("No Archipelago mod was loaded. Aborting.")
-                    await get_info(ctx, rcon_client)
-            await asyncio.sleep(0.01)
-
-    except Exception as e:
-        logger.exception(e, extra={"compact_gui": True})
-        msg = "Aborted Factorio Server Bridge"
-        logger.error(msg)
-        ctx.gui_error(msg, e)
-        ctx.exit_event.set()
-
-    else:
-        logger.info(
-            f"Got World Information from AP Mod {tuple(ctx.mod_version)} for seed {ctx.seed_name} in slot {ctx.auth}")
-        return True
-    finally:
-        factorio_process.terminate()
-        factorio_process.wait(5)
-    return False
-
-
-async def main(args):
-    ctx = FactorioContext(args.connect, args.password)
-    ctx.filter_item_sends = initial_filter_item_sends
-    ctx.bridge_chat_out = initial_bridge_chat_out
-    ctx.server_task = asyncio.create_task(server_loop(ctx), name="ServerLoop")
-
-    if gui_enabled:
-        ctx.run_gui()
-    ctx.run_cli()
-
-    factorio_server_task = asyncio.create_task(factorio_spinup_server(ctx), name="FactorioSpinupServer")
-    successful_launch = await factorio_server_task
-    if successful_launch:
-        factorio_server_task = asyncio.create_task(factorio_server_watcher(ctx), name="FactorioServer")
-        progression_watcher = asyncio.create_task(
-            game_watcher(ctx), name="FactorioProgressionWatcher")
-
-        await ctx.exit_event.wait()
-        ctx.server_address = None
-
-        await progression_watcher
-        await factorio_server_task
-
-    await ctx.shutdown()
-
-
-class FactorioJSONtoTextParser(JSONtoTextParser):
-    def _handle_color(self, node: JSONMessagePart):
-        colors = node["color"].split(";")
-        for color in colors:
-            if color in self.color_codes:
-                node["text"] = f"[color=#{self.color_codes[color]}]{node['text']}[/color]"
-            return self._handle_text(node)
-        return self._handle_text(node)
-
-
-if __name__ == '__main__':
-    parser = get_base_parser(description="Optional arguments to FactorioClient follow. "
-                                         "Remaining arguments get passed into bound Factorio instance."
-                                         "Refer to Factorio --help for those.")
-    parser.add_argument('--rcon-port', default='24242', type=int, help='Port to use to communicate with Factorio')
-    parser.add_argument('--rcon-password', help='Password to authenticate with RCON.')
-    parser.add_argument('--server-settings', help='Factorio server settings configuration file.')
-
-    args, rest = parser.parse_known_args()
-    colorama.init()
-    rcon_port = args.rcon_port
-    rcon_password = args.rcon_password if args.rcon_password else ''.join(
-        random.choice(string.ascii_letters) for x in range(32))
-
-    factorio_server_logger = logging.getLogger("FactorioServer")
-    options = Utils.get_options()
-    executable = options["factorio_options"]["executable"]
-    server_settings = args.server_settings if args.server_settings else options["factorio_options"].get("server_settings", None)
-    if server_settings:
-        server_settings = os.path.abspath(server_settings)
-    if not isinstance(options["factorio_options"]["filter_item_sends"], bool):
-        logging.warning(f"Warning: Option filter_item_sends should be a bool.")
-    initial_filter_item_sends = bool(options["factorio_options"]["filter_item_sends"])
-    if not isinstance(options["factorio_options"]["bridge_chat_out"], bool):
-        logging.warning(f"Warning: Option bridge_chat_out should be a bool.")
-    initial_bridge_chat_out = bool(options["factorio_options"]["bridge_chat_out"])
-
-    if not os.path.exists(os.path.dirname(executable)):
-        raise FileNotFoundError(f"Path {os.path.dirname(executable)} does not exist or could not be accessed.")
-    if os.path.isdir(executable):  # user entered a path to a directory, let's find the executable therein
-        executable = os.path.join(executable, "factorio")
-    if not os.path.isfile(executable):
-        if os.path.isfile(executable + ".exe"):
-            executable = executable + ".exe"
-        else:
-            raise FileNotFoundError(f"Path {executable} is not an executable file.")
-
-    if server_settings and os.path.isfile(server_settings):
-        server_args = ("--rcon-port", rcon_port, "--rcon-password", rcon_password, "--server-settings", server_settings, *rest)
-    else:
-        server_args = ("--rcon-port", rcon_port, "--rcon-password", rcon_password, *rest)
-
-    asyncio.run(main(args))
-    colorama.deinit()
+    launch()

Fill.py

@@ -39,8 +39,9 @@ def fill_restrictive(world: MultiWorld, base_state: CollectionState, locations:
     """
     unplaced_items: typing.List[Item] = []
     placements: typing.List[Location] = []
+    cleanup_required = False
 
-    swapped_items: typing.Counter[typing.Tuple[int, str]] = Counter()
+    swapped_items: typing.Counter[typing.Tuple[int, str, bool]] = Counter()
     reachable_items: typing.Dict[int, typing.Deque[Item]] = {}
     for item in item_pool:
         reachable_items.setdefault(item.player, deque()).append(item)
@@ -50,7 +51,10 @@ def fill_restrictive(world: MultiWorld, base_state: CollectionState, locations:
         items_to_place = [items.pop()
                           for items in reachable_items.values() if items]
         for item in items_to_place:
-            item_pool.remove(item)
+            for p, pool_item in enumerate(item_pool):
+                if pool_item is item:
+                    item_pool.pop(p)
+                    break
         maximum_exploration_state = sweep_from_pool(
             base_state, item_pool + unplaced_items)
 
@@ -84,25 +88,28 @@ def fill_restrictive(world: MultiWorld, base_state: CollectionState, locations:
             else:
                 # we filled all reachable spots.
                 if swap:
-                    # try swapping this item with previously placed items
-                    for (i, location) in enumerate(placements):
+                    # try swapping this item with previously placed items in a safe way then in an unsafe way
+                    swap_attempts = ((i, location, unsafe)
+                                     for unsafe in (False, True)
+                                     for i, location in enumerate(placements))
+                    for (i, location, unsafe) in swap_attempts:
                         placed_item = location.item
                         # Unplaceable items can sometimes be swapped infinitely. Limit the
                         # number of times we will swap an individual item to prevent this
-                        swap_count = swapped_items[placed_item.player,
-                                                   placed_item.name]
+                        swap_count = swapped_items[placed_item.player, placed_item.name, unsafe]
                         if swap_count > 1:
                             continue
 
                         location.item = None
                         placed_item.location = None
-                        swap_state = sweep_from_pool(base_state, [placed_item])
-                        # swap_state assumes we can collect placed item before item_to_place
+                        swap_state = sweep_from_pool(base_state, [placed_item] if unsafe else [])
+                        # unsafe means swap_state assumes we can somehow collect placed_item before item_to_place
+                        # by continuing to swap, which is not guaranteed. This is unsafe because there is no mechanic
+                        # to clean that up later, so there is a chance generation fails.
                         if (not single_player_placement or location.player == item_to_place.player) \
                                 and location.can_fill(swap_state, item_to_place, perform_access_check):
 
-                            # Verify that placing this item won't reduce available locations, which could happen with rules
-                            # that want to not have both items. Left in until removal is proven useful.
+                            # Verify placing this item won't reduce available locations, which would be a useless swap.
                             prev_state = swap_state.copy()
                             prev_loc_count = len(
                                 world.get_reachable_locations(prev_state))
@@ -117,13 +124,15 @@ def fill_restrictive(world: MultiWorld, base_state: CollectionState, locations:
                                 spot_to_fill = placements.pop(i)
 
                                 swap_count += 1
-                                swapped_items[placed_item.player,
-                                              placed_item.name] = swap_count
+                                swapped_items[placed_item.player, placed_item.name, unsafe] = swap_count
 
                                 reachable_items[placed_item.player].appendleft(
                                     placed_item)
                                 item_pool.append(placed_item)
 
+                                # cleanup at the end to hopefully get better errors
+                                cleanup_required = True
+
                                 break
 
                         # Item can't be placed here, restore original item
@@ -144,6 +153,16 @@ def fill_restrictive(world: MultiWorld, base_state: CollectionState, locations:
             if on_place:
                 on_place(spot_to_fill)
 
+    if cleanup_required:
+        # validate all placements and remove invalid ones
+        state = sweep_from_pool(base_state, [])
+        for placement in placements:
+            if world.accessibility[placement.item.player] != "minimal" and not placement.can_reach(state):
+                placement.item.location = None
+                unplaced_items.append(placement.item)
+                placement.item = None
+                locations.append(placement)
+
     if allow_excluded:
         # check if partial fill is the result of excluded locations, in which case retry
         excluded_locations = [

Generate.py

@@ -7,55 +7,52 @@ import random
 import string
 import urllib.parse
 import urllib.request
-from collections import Counter, ChainMap
-from typing import Dict, Tuple, Callable, Any, Union
+from collections import ChainMap, Counter
+from typing import Any, Callable, Dict, Tuple, Union
 
 import ModuleUpdate
 
 ModuleUpdate.update()
 
+import copy
 import Utils
-from worlds.alttp import Options as LttPOptions
-from worlds.generic import PlandoConnection
-from Utils import parse_yamls, version_tuple, __version__, tuplize_version, get_options, user_path
-from worlds.alttp.EntranceRandomizer import parse_arguments
-from Main import main as ERmain
-from BaseClasses import seeddigits, get_seed, PlandoOptions
 import Options
+from BaseClasses import seeddigits, get_seed, PlandoOptions
+from Main import main as ERmain
+from settings import get_settings
+from Utils import parse_yamls, version_tuple, __version__, tuplize_version, user_path
+from worlds.alttp import Options as LttPOptions
+from worlds.alttp.EntranceRandomizer import parse_arguments
 from worlds.alttp.Text import TextTable
 from worlds.AutoWorld import AutoWorldRegister
-import copy
-
-
-
+from worlds.generic import PlandoConnection
 
 
 def mystery_argparse():
-    options = get_options()
-    defaults = options["generator"]
-
-    def resolve_path(path: str, resolver: Callable[[str], str]) -> str:
-        return path if os.path.isabs(path) else resolver(path)
+    options = get_settings()
+    defaults = options.generator
 
     parser = argparse.ArgumentParser(description="CMD Generation Interface, defaults come from host.yaml.")
-    parser.add_argument('--weights_file_path', default=defaults["weights_file_path"],
+    parser.add_argument('--weights_file_path', default=defaults.weights_file_path,
                         help='Path to the weights file to use for rolling game settings, urls are also valid')
     parser.add_argument('--samesettings', help='Rolls settings per weights file rather than per player',
                         action='store_true')
-    parser.add_argument('--player_files_path', default=resolve_path(defaults["player_files_path"], user_path),
+    parser.add_argument('--player_files_path', default=defaults.player_files_path,
                         help="Input directory for player files.")
     parser.add_argument('--seed', help='Define seed number to generate.', type=int)
-    parser.add_argument('--multi', default=defaults["players"], type=lambda value: max(int(value), 1))
-    parser.add_argument('--spoiler', type=int, default=defaults["spoiler"])
-    parser.add_argument('--outputpath', default=resolve_path(options["general_options"]["output_path"], user_path),
+    parser.add_argument('--multi', default=defaults.players, type=lambda value: max(int(value), 1))
+    parser.add_argument('--spoiler', type=int, default=defaults.spoiler)
+    parser.add_argument('--outputpath', default=options.general_options.output_path,
                         help="Path to output folder. Absolute or relative to cwd.")  # absolute or relative to cwd
-    parser.add_argument('--race', action='store_true', default=defaults["race"])
-    parser.add_argument('--meta_file_path', default=defaults["meta_file_path"])
+    parser.add_argument('--race', action='store_true', default=defaults.race)
+    parser.add_argument('--meta_file_path', default=defaults.meta_file_path)
     parser.add_argument('--log_level', default='info', help='Sets log level')
     parser.add_argument('--yaml_output', default=0, type=lambda value: max(int(value), 0),
                         help='Output rolled mystery results to yaml up to specified number (made for async multiworld)')
-    parser.add_argument('--plando', default=defaults["plando_options"],
+    parser.add_argument('--plando', default=defaults.plando_options,
                         help='List of options that can be set manually. Can be combined, for example "bosses, items"')
+    parser.add_argument("--skip_prog_balancing", action="store_true",
+                        help="Skip progression balancing step during generation.")
     args = parser.parse_args()
     if not os.path.isabs(args.weights_file_path):
         args.weights_file_path = os.path.join(args.player_files_path, args.weights_file_path)
@@ -72,12 +69,16 @@ def get_seed_name(random_source) -> str:
 def main(args=None, callback=ERmain):
     if not args:
         args, options = mystery_argparse()
+    else:
+        options = get_settings()
 
     seed = get_seed(args.seed)
+    Utils.init_logging(f"Generate_{seed}", loglevel=args.log_level)
     random.seed(seed)
     seed_name = get_seed_name(random)
 
     if args.race:
+        logging.info("Race mode enabled. Using non-deterministic random source.")
         random.seed()  # reset to time-based random source
 
     weights_cache: Dict[str, Tuple[Any, ...]] = {}
@@ -85,16 +86,16 @@ def main(args=None, callback=ERmain):
         try:
             weights_cache[args.weights_file_path] = read_weights_yamls(args.weights_file_path)
         except Exception as e:
-            raise ValueError(f"File {args.weights_file_path} is destroyed. Please fix your yaml.") from e
-        print(f"Weights: {args.weights_file_path} >> "
-              f"{get_choice('description', weights_cache[args.weights_file_path][-1], 'No description specified')}")
+            raise ValueError(f"File {args.weights_file_path} is invalid. Please fix your yaml.") from e
+        logging.info(f"Weights: {args.weights_file_path} >> "
+                     f"{get_choice('description', weights_cache[args.weights_file_path][-1], 'No description specified')}")
 
     if args.meta_file_path and os.path.exists(args.meta_file_path):
         try:
             meta_weights = read_weights_yamls(args.meta_file_path)[-1]
         except Exception as e:
-            raise ValueError(f"File {args.meta_file_path} is destroyed. Please fix your yaml.") from e
-        print(f"Meta: {args.meta_file_path} >> {get_choice('meta_description', meta_weights)}")
+            raise ValueError(f"File {args.meta_file_path} is invalid. Please fix your yaml.") from e
+        logging.info(f"Meta: {args.meta_file_path} >> {get_choice('meta_description', meta_weights)}")
         try:  # meta description allows us to verify that the file named meta.yaml is intentionally a meta file
             del(meta_weights["meta_description"])
         except Exception as e:
@@ -113,35 +114,35 @@ def main(args=None, callback=ERmain):
             try:
                 weights_cache[fname] = read_weights_yamls(path)
             except Exception as e:
-                raise ValueError(f"File {fname} is destroyed. Please fix your yaml.") from e
+                raise ValueError(f"File {fname} is invalid. Please fix your yaml.") from e
 
     # sort dict for consistent results across platforms:
     weights_cache = {key: value for key, value in sorted(weights_cache.items())}
     for filename, yaml_data in weights_cache.items():
         if filename not in {args.meta_file_path, args.weights_file_path}:
             for yaml in yaml_data:
-                print(f"P{player_id} Weights: {filename} >> "
-                      f"{get_choice('description', yaml, 'No description specified')}")
+                logging.info(f"P{player_id} Weights: {filename} >> "
+                             f"{get_choice('description', yaml, 'No description specified')}")
                 player_files[player_id] = filename
                 player_id += 1
 
     args.multi = max(player_id - 1, args.multi)
-    print(f"Generating for {args.multi} player{'s' if args.multi > 1 else ''}, {seed_name} Seed {seed} with plando: "
-          f"{args.plando}")
+    logging.info(f"Generating for {args.multi} player{'s' if args.multi > 1 else ''}, "
+                 f"{seed_name} Seed {seed} with plando: {args.plando}")
 
     if not weights_cache:
-        raise Exception(f"No weights found. Provide a general weights file ({args.weights_file_path}) or individual player files. "
+        raise Exception(f"No weights found. "
+                        f"Provide a general weights file ({args.weights_file_path}) or individual player files. "
                         f"A mix is also permitted.")
     erargs = parse_arguments(['--multi', str(args.multi)])
     erargs.seed = seed
     erargs.plando_options = args.plando
-    erargs.glitch_triforce = options["generator"]["glitch_triforce_room"]
+    erargs.glitch_triforce = options.generator.glitch_triforce_room
     erargs.spoiler = args.spoiler
     erargs.race = args.race
     erargs.outputname = seed_name
     erargs.outputpath = args.outputpath
-
-    Utils.init_logging(f"Generate_{seed}", loglevel=args.log_level)
+    erargs.skip_prog_balancing = args.skip_prog_balancing
 
     settings_cache: Dict[str, Tuple[argparse.Namespace, ...]] = \
         {fname: (tuple(roll_settings(yaml, args.plando) for yaml in yamls) if args.samesettings else None)
@@ -194,7 +195,7 @@ def main(args=None, callback=ERmain):
 
                     player += 1
             except Exception as e:
-                raise ValueError(f"File {path} is destroyed. Please fix your yaml.") from e
+                raise ValueError(f"File {path} is invalid. Please fix your yaml.") from e
         else:
             raise RuntimeError(f'No weights specified for player {player}')
 
@@ -373,7 +374,7 @@ def roll_linked_options(weights: dict) -> dict:
             else:
                 logging.debug(f"linked option {option_set['name']} skipped.")
         except Exception as e:
-            raise ValueError(f"Linked option {option_set['name']} is destroyed. "
+            raise ValueError(f"Linked option {option_set['name']} is invalid. "
                              f"Please fix your linked option.") from e
     return weights
 
@@ -403,7 +404,7 @@ def roll_triggers(weights: dict, triggers: list) -> dict:
                     update_weights(currently_targeted_weights, category_options, "Triggered", option_set["option_name"])
 
         except Exception as e:
-            raise ValueError(f"Your trigger number {i + 1} is destroyed. "
+            raise ValueError(f"Your trigger number {i + 1} is invalid. "
                              f"Please fix your triggers.") from e
     return weights
 
@@ -449,6 +450,11 @@ def roll_settings(weights: dict, plando_options: PlandoOptions = PlandoOptions.b
             raise Exception(f"Option {option_key} has to be in a game's section, not on its own.")
 
     ret.game = get_choice("game", weights)
+    if ret.game not in AutoWorldRegister.world_types:
+        picks = Utils.get_fuzzy_results(ret.game, AutoWorldRegister.world_types, limit=1)[0]
+        raise Exception(f"No world found to handle game {ret.game}. Did you mean '{picks[0]}' ({picks[1]}% sure)? "
+                        f"Check your spelling or installation of that world.")
+
     if ret.game not in weights:
         raise Exception(f"No game options for selected game \"{ret.game}\" found.")
 
@@ -463,32 +469,29 @@ def roll_settings(weights: dict, plando_options: PlandoOptions = PlandoOptions.b
     for option_key, option in Options.common_options.items():
         setattr(ret, option_key, option.from_any(get_choice(option_key, weights, option.default)))
 
-    if ret.game in AutoWorldRegister.world_types:
-        for option_key, option in world_type.option_definitions.items():
+    for option_key, option in world_type.option_definitions.items():
+        handle_option(ret, game_weights, option_key, option, plando_options)
+    for option_key, option in Options.per_game_common_options.items():
+        # skip setting this option if already set from common_options, defaulting to root option
+        if option_key not in world_type.option_definitions and \
+                (option_key not in Options.common_options or option_key in game_weights):
             handle_option(ret, game_weights, option_key, option, plando_options)
-        for option_key, option in Options.per_game_common_options.items():
-            # skip setting this option if already set from common_options, defaulting to root option
-            if option_key not in world_type.option_definitions and \
-                    (option_key not in Options.common_options or option_key in game_weights):
-                handle_option(ret, game_weights, option_key, option, plando_options)
-        if PlandoOptions.items in plando_options:
-            ret.plando_items = game_weights.get("plando_items", [])
-        if ret.game == "Minecraft" or ret.game == "Ocarina of Time":
-            # bad hardcoded behavior to make this work for now
-            ret.plando_connections = []
-            if PlandoOptions.connections in plando_options:
-                options = game_weights.get("plando_connections", [])
-                for placement in options:
-                    if roll_percentage(get_choice("percentage", placement, 100)):
-                        ret.plando_connections.append(PlandoConnection(
-                            get_choice("entrance", placement),
-                            get_choice("exit", placement),
-                            get_choice("direction", placement)
-                        ))
-        elif ret.game == "A Link to the Past":
-            roll_alttp_settings(ret, game_weights, plando_options)
-    else:
-        raise Exception(f"Unsupported game {ret.game}")
+    if PlandoOptions.items in plando_options:
+        ret.plando_items = game_weights.get("plando_items", [])
+    if ret.game == "Minecraft" or ret.game == "Ocarina of Time":
+        # bad hardcoded behavior to make this work for now
+        ret.plando_connections = []
+        if PlandoOptions.connections in plando_options:
+            options = game_weights.get("plando_connections", [])
+            for placement in options:
+                if roll_percentage(get_choice("percentage", placement, 100)):
+                    ret.plando_connections.append(PlandoConnection(
+                        get_choice("entrance", placement),
+                        get_choice("exit", placement),
+                        get_choice("direction", placement)
+                    ))
+    elif ret.game == "A Link to the Past":
+        roll_alttp_settings(ret, game_weights, plando_options)
 
     return ret
 

Launcher.py

@@ -11,6 +11,7 @@ Scroll down to components= to add components to the launcher as well as setup.py
 
 import argparse
 import itertools
+import logging
 import multiprocessing
 import shlex
 import subprocess
@@ -21,6 +22,7 @@ from shutil import which
 from typing import Sequence, Union, Optional
 
 import Utils
+import settings
 from worlds.LauncherComponents import Component, components, Type, SuffixIdentifier, icon_paths
 
 if __name__ == "__main__":
@@ -32,7 +34,8 @@ from Utils import is_frozen, user_path, local_path, init_logging, open_filename,
 
 
 def open_host_yaml():
-    file = user_path('host.yaml')
+    file = settings.get_settings().filename
+    assert file, "host.yaml missing"
     if is_linux:
         exe = which('sensible-editor') or which('gedit') or \
               which('xdg-open') or which('gnome-open') or which('kde-open')
@@ -55,7 +58,7 @@ def open_patch():
     except Exception as e:
         messagebox('Error', str(e), error=True)
     else:
-        file, _, component = identify(filename)
+        file, component = identify(filename)
         if file and component:
             launch([*get_exe(component), file], component.cli)
 
@@ -83,6 +86,11 @@ def open_folder(folder_path):
         webbrowser.open(folder_path)
 
 
+def update_settings():
+    from settings import get_settings
+    get_settings().save()
+
+
 components.extend([
     # Functions
     Component("Open host.yaml", func=open_host_yaml),
@@ -96,11 +104,13 @@ components.extend([
 
 def identify(path: Union[None, str]):
     if path is None:
-        return None, None, None
+        return None, None
     for component in components:
         if component.handles_file(path):
-            return path, component.script_name, component
-    return (None, None, None) if '/' in path or '\\' in path else (None, path, None)
+            return path,  component
+        elif path == component.display_name or path == component.script_name:
+            return None, component
+    return None, None
 
 
 def get_exe(component: Union[str, Component]) -> Optional[Sequence[str]]:
@@ -155,10 +165,10 @@ def run_gui():
         container: ContainerLayout
         grid: GridLayout
 
-        _tools = {c.display_name: c for c in components if c.type == Type.TOOL and isfile(get_exe(c)[-1])}
-        _clients = {c.display_name: c for c in components if c.type == Type.CLIENT and isfile(get_exe(c)[-1])}
-        _adjusters = {c.display_name: c for c in components if c.type == Type.ADJUSTER and isfile(get_exe(c)[-1])}
-        _funcs = {c.display_name: c for c in components if c.type == Type.FUNC}
+        _tools = {c.display_name: c for c in components if c.type == Type.TOOL}
+        _clients = {c.display_name: c for c in components if c.type == Type.CLIENT}
+        _adjusters = {c.display_name: c for c in components if c.type == Type.ADJUSTER}
+        _miscs = {c.display_name: c for c in components if c.type == Type.MISC}
 
         def __init__(self, ctx=None):
             self.title = self.base_title
@@ -199,7 +209,7 @@ def run_gui():
                     button_layout.add_widget(button)
 
             for (tool, client) in itertools.zip_longest(itertools.chain(
-                    self._tools.items(), self._funcs.items(), self._adjusters.items()), self._clients.items()):
+                    self._tools.items(), self._miscs.items(), self._adjusters.items()), self._clients.items()):
                 # column 1
                 if tool:
                     build_button(tool[1])
@@ -215,14 +225,29 @@ def run_gui():
 
         @staticmethod
         def component_action(button):
-            if button.component.type == Type.FUNC:
+            if button.component.func:
                 button.component.func()
             else:
                 launch(get_exe(button.component), button.component.cli)
 
+        def _stop(self, *largs):
+            # ran into what appears to be https://groups.google.com/g/kivy-users/c/saWDLoYCSZ4 with PyCharm.
+            # Closing the window explicitly cleans it up.
+            self.root_window.close()
+            super()._stop(*largs)
+
     Launcher().run()
 
 
+def run_component(component: Component, *args):
+    if component.func:
+        component.func(*args)
+    elif component.script_name:
+        subprocess.run([*get_exe(component.script_name), *args])
+    else:
+        logging.warning(f"Component {component} does not appear to be executable.")
+
+
 def main(args: Optional[Union[argparse.Namespace, dict]] = None):
     if isinstance(args, argparse.Namespace):
         args = {k: v for k, v in args._get_kwargs()}
@@ -230,25 +255,40 @@ def main(args: Optional[Union[argparse.Namespace, dict]] = None):
         args = {}
 
     if "Patch|Game|Component" in args:
-        file, component, _ = identify(args["Patch|Game|Component"])
+        file, component = identify(args["Patch|Game|Component"])
         if file:
             args['file'] = file
         if component:
             args['component'] = component
+        if not component:
+            logging.warning(f"Could not identify Component responsible for {args['Patch|Game|Component']}")
 
+    if args["update_settings"]:
+        update_settings()
     if 'file' in args:
-        subprocess.run([*get_exe(args['component']), args['file'], *args['args']])
+        run_component(args["component"], args["file"], *args["args"])
     elif 'component' in args:
-        subprocess.run([*get_exe(args['component']), *args['args']])
-    else:
+        run_component(args["component"], *args["args"])
+    elif not args["update_settings"]:
         run_gui()
 
 
 if __name__ == '__main__':
     init_logging('Launcher')
-    multiprocessing.freeze_support()
+    Utils.freeze_support()
+    multiprocessing.set_start_method("spawn")  # if launched process uses kivy, fork won't work
     parser = argparse.ArgumentParser(description='Archipelago Launcher')
-    parser.add_argument('Patch|Game|Component', type=str, nargs='?',
-                        help="Pass either a patch file, a generated game or the name of a component to run.")
-    parser.add_argument('args', nargs="*", help="Arguments to pass to component.")
+    run_group = parser.add_argument_group("Run")
+    run_group.add_argument("--update_settings", action="store_true",
+                           help="Update host.yaml and exit.")
+    run_group.add_argument("Patch|Game|Component", type=str, nargs="?",
+                           help="Pass either a patch file, a generated game or the name of a component to run.")
+    run_group.add_argument("args", nargs="*",
+                           help="Arguments to pass to component.")
     main(parser.parse_args())
+
+    from worlds.LauncherComponents import processes
+    for process in processes:
+        # we await all child processes to close before we tear down the process host
+        # this makes it feel like each one is its own program, as the Launcher is closed now
+        process.join()

LinksAwakeningClient.py

@@ -9,15 +9,18 @@ if __name__ == "__main__":
 import asyncio
 import base64
 import binascii
+import colorama
 import io
-import logging
+import os
+import re
 import select
+import shlex
 import socket
+import struct
+import sys
+import subprocess
 import time
 import typing
-import urllib
-
-import colorama
 
 
 from CommonClient import (CommonContext, get_base_parser, gui_enabled, logger,
@@ -30,6 +33,7 @@ from worlds.ladx.LADXR.checkMetadata import checkMetadataTable
 from worlds.ladx.Locations import get_locations_to_id, meta_to_name
 from worlds.ladx.Tracker import LocationTracker, MagpieBridge
 
+
 class GameboyException(Exception):
     pass
 
@@ -91,7 +95,7 @@ class LAClientConstants:
     # wLinkSendShopTarget = 0xDDFF
 
 
-    wRecvIndex = 0xDDFE  # 0xDB58
+    wRecvIndex = 0xDDFD # Two bytes
     wCheckAddress = 0xC0FF - 0x4
     WRamCheckSize = 0x4
     WRamSafetyValue = bytearray([0]*WRamCheckSize)
@@ -115,17 +119,17 @@ class RAGameboy():
         assert (self.socket)
         self.socket.setblocking(False)
 
-    def get_retroarch_version(self):
-        self.send(b'VERSION\n')
-        select.select([self.socket], [], [])
-        response_str, addr = self.socket.recvfrom(16)
+    async def send_command(self, command, timeout=1.0):
+        self.send(f'{command}\n')
+        response_str = await self.async_recv()
+        self.check_command_response(command, response_str)
         return response_str.rstrip()
 
-    def get_retroarch_status(self, timeout):
-        self.send(b'GET_STATUS\n')
-        select.select([self.socket], [], [], timeout)
-        response_str, addr = self.socket.recvfrom(1000, )
-        return response_str.rstrip()
+    async def get_retroarch_version(self):
+        return await self.send_command("VERSION")
+
+    async def get_retroarch_status(self):
+        return await self.send_command("GET_STATUS")
 
     def set_cache_limits(self, cache_start, cache_size):
         self.cache_start = cache_start
@@ -141,8 +145,8 @@ class RAGameboy():
         response, _ = self.socket.recvfrom(4096)
         return response
 
-    async def async_recv(self):
-        response = await asyncio.get_event_loop().sock_recv(self.socket, 4096)
+    async def async_recv(self, timeout=1.0):
+        response = await asyncio.wait_for(asyncio.get_event_loop().sock_recv(self.socket, 4096), timeout)
         return response
 
     async def check_safe_gameplay(self, throw=True):
@@ -169,6 +173,8 @@ class RAGameboy():
                 raise InvalidEmulatorStateError()
             return False
         if not await check_wram():
+            if throw:
+                raise InvalidEmulatorStateError()
             return False
         return True
 
@@ -227,20 +233,30 @@ class RAGameboy():
 
         return r
 
+    def check_command_response(self, command: str, response: bytes):
+        if command == "VERSION":
+            ok = re.match("\d+\.\d+\.\d+", response.decode('ascii')) is not None
+        else:
+            ok = response.startswith(command.encode())
+        if not ok:
+            logger.warning(f"Bad response to command {command} - {response}")
+            raise BadRetroArchResponse()
+
     def read_memory(self, address, size=1):
         command = "READ_CORE_MEMORY"
 
         self.send(f'{command} {hex(address)} {size}\n')
         response = self.recv()
 
+        self.check_command_response(command, response)
+
         splits = response.decode().split(" ", 2)
-
-        assert (splits[0] == command)
         # Ignore the address for now
-
-        # TODO: transform to bytes
-        if splits[2][:2] == "-1" or splits[0] != "READ_CORE_MEMORY":
+        if splits[2][:2] == "-1":
             raise BadRetroArchResponse()
+
+        # TODO: check response address, check hex behavior between RA and BH
+
         return bytearray.fromhex(splits[2])
 
     async def async_read_memory(self, address, size=1):
@@ -248,14 +264,21 @@ class RAGameboy():
 
         self.send(f'{command} {hex(address)} {size}\n')
         response = await self.async_recv()
+        self.check_command_response(command, response)
         response = response[:-1]
         splits = response.decode().split(" ", 2)
+        try:
+            response_addr = int(splits[1], 16)
+        except ValueError:
+            raise BadRetroArchResponse()
 
-        assert (splits[0] == command)
-        # Ignore the address for now
+        if response_addr != address:
+            raise BadRetroArchResponse()
 
-        # TODO: transform to bytes
-        return bytearray.fromhex(splits[2])
+        ret = bytearray.fromhex(splits[2])
+        if len(ret) > size:
+            raise BadRetroArchResponse()
+        return ret
 
     def write_memory(self, address, bytes):
         command = "WRITE_CORE_MEMORY"
@@ -263,7 +286,7 @@ class RAGameboy():
         self.send(f'{command} {hex(address)} {" ".join(hex(b) for b in bytes)}')
         select.select([self.socket], [], [])
         response, _ = self.socket.recvfrom(4096)
-
+        self.check_command_response(command, response)
         splits = response.decode().split(" ", 3)
 
         assert (splits[0] == command)
@@ -281,6 +304,9 @@ class LinksAwakeningClient():
     pending_deathlink = False
     deathlink_debounce = True
     recvd_checks = {}
+    retroarch_address = None
+    retroarch_port = None
+    gameboy = None
 
     def msg(self, m):
         logger.info(m)
@@ -288,50 +314,48 @@ class LinksAwakeningClient():
         self.gameboy.send(s)
 
     def __init__(self, retroarch_address="127.0.0.1", retroarch_port=55355):
-        self.gameboy = RAGameboy(retroarch_address, retroarch_port)
+        self.retroarch_address = retroarch_address
+        self.retroarch_port = retroarch_port
+        pass
 
+    stop_bizhawk_spam = False
     async def wait_for_retroarch_connection(self):
-        logger.info("Waiting on connection to Retroarch...")
+        if not self.stop_bizhawk_spam:
+            logger.info("Waiting on connection to Retroarch...")
+            self.stop_bizhawk_spam = True
+        self.gameboy = RAGameboy(self.retroarch_address, self.retroarch_port)
+
         while True:
             try:
-                version = self.gameboy.get_retroarch_version()
+                version = await self.gameboy.get_retroarch_version()
                 NO_CONTENT = b"GET_STATUS CONTENTLESS"
                 status = NO_CONTENT
                 core_type = None
                 GAME_BOY = b"game_boy"
                 while status == NO_CONTENT or core_type != GAME_BOY:
-                    try:
-                        status = self.gameboy.get_retroarch_status(0.1)
-                        if status.count(b" ") < 2:
-                            await asyncio.sleep(1.0)
-                            continue
-                        
-                        GET_STATUS, PLAYING, info = status.split(b" ", 2)
-                        if status.count(b",") < 2:
-                            await asyncio.sleep(1.0)
-                            continue
-                        core_type, rom_name, self.game_crc = info.split(b",", 2)
-                        if core_type != GAME_BOY:
-                            logger.info(
-                                f"Core type should be '{GAME_BOY}', found {core_type} instead - wrong type of ROM?")
-                            await asyncio.sleep(1.0)
-                            continue
-                    except (BlockingIOError, TimeoutError) as e:
-                        await asyncio.sleep(0.1)
-                        pass
-                logger.info(f"Connected to Retroarch {version} {info}")
-                self.gameboy.read_memory(0x1000)
+                    status = await self.gameboy.get_retroarch_status()
+                    if status.count(b" ") < 2:
+                        await asyncio.sleep(1.0)
+                        continue
+                    GET_STATUS, PLAYING, info = status.split(b" ", 2)
+                    if status.count(b",") < 2:
+                        await asyncio.sleep(1.0)
+                        continue
+                    core_type, rom_name, self.game_crc = info.split(b",", 2)
+                    if core_type != GAME_BOY:
+                        logger.info(
+                            f"Core type should be '{GAME_BOY}', found {core_type} instead - wrong type of ROM?")
+                        await asyncio.sleep(1.0)
+                        continue
+                self.stop_bizhawk_spam = False
+                logger.info(f"Connected to Retroarch {version.decode('ascii')} running {rom_name.decode('ascii')}")
                 return
-            except ConnectionResetError:
+            except (BlockingIOError, TimeoutError, ConnectionResetError):
                 await asyncio.sleep(1.0)
                 pass
 
-    def reset_auth(self):
-        auth = binascii.hexlify(self.gameboy.read_memory(0x0134, 12)).decode()
-
-        if self.auth:
-            assert (auth == self.auth)
-
+    async def reset_auth(self):
+        auth = binascii.hexlify(await self.gameboy.async_read_memory(0x0134, 12)).decode()
         self.auth = auth
 
     async def wait_and_init_tracker(self):
@@ -365,14 +389,16 @@ class LinksAwakeningClient():
                                   item_id, from_player])
         status |= 1
         status = self.gameboy.write_memory(LAClientConstants.wLinkStatusBits, [status])
-        self.gameboy.write_memory(LAClientConstants.wRecvIndex, [next_index])
+        self.gameboy.write_memory(LAClientConstants.wRecvIndex, struct.pack(">H", next_index))
 
+    should_reset_auth = False
     async def wait_for_game_ready(self):
         logger.info("Waiting on game to be in valid state...")
         while not await self.gameboy.check_safe_gameplay(throw=False):
-            pass
-        logger.info("Ready!")
-    last_index = 0
+            if self.should_reset_auth:
+                self.should_reset_auth = False
+                raise GameboyException("Resetting due to wrong archipelago server")
+        logger.info("Game connection ready!")
 
     async def is_victory(self):
         return (await self.gameboy.read_memory_cache([LAClientConstants.wGameplayType]))[LAClientConstants.wGameplayType] == 1
@@ -382,11 +408,6 @@ class LinksAwakeningClient():
         await self.item_tracker.readItems()
         await self.gps_tracker.read_location()
 
-        next_index = self.gameboy.read_memory(LAClientConstants.wRecvIndex)[0]
-        if next_index != self.last_index:
-            self.last_index = next_index
-            # logger.info(f"Got new index {next_index}")
-
         current_health = (await self.gameboy.read_memory_cache([LAClientConstants.wLinkHealth]))[LAClientConstants.wLinkHealth]
         if self.deathlink_debounce and current_health != 0:
             self.deathlink_debounce = False
@@ -404,7 +425,7 @@ class LinksAwakeningClient():
         if await self.is_victory():
             await win_cb()
 
-        recv_index = (await self.gameboy.async_read_memory_safe(LAClientConstants.wRecvIndex))[0]
+        recv_index = struct.unpack(">H", await self.gameboy.async_read_memory(LAClientConstants.wRecvIndex, 2))[0]
 
         # Play back one at a time
         if recv_index in self.recvd_checks:
@@ -438,12 +459,16 @@ class LinksAwakeningContext(CommonContext):
     found_checks = []
     last_resend = time.time()
 
-    magpie = MagpieBridge()
+    magpie_enabled = False
+    magpie = None
     magpie_task = None
     won = False
 
-    def __init__(self, server_address: typing.Optional[str], password: typing.Optional[str]) -> None:
+    def __init__(self, server_address: typing.Optional[str], password: typing.Optional[str], magpie: typing.Optional[bool]) -> None:
         self.client = LinksAwakeningClient()
+        if magpie:
+            self.magpie_enabled = True
+            self.magpie = MagpieBridge()
         super().__init__(server_address, password)
 
     def run_gui(self) -> None:
@@ -462,16 +487,17 @@ class LinksAwakeningContext(CommonContext):
             def build(self):
                 b = super().build()
 
-                button = Button(text="", size=(30, 30), size_hint_x=None,
-                                on_press=lambda _: webbrowser.open('https://magpietracker.us/?enable_autotracker=1'))
-                image = Image(size=(16, 16), texture=magpie_logo())
-                button.add_widget(image)
+                if self.ctx.magpie_enabled:
+                    button = Button(text="", size=(30, 30), size_hint_x=None,
+                                    on_press=lambda _: webbrowser.open('https://magpietracker.us/?enable_autotracker=1'))
+                    image = Image(size=(16, 16), texture=magpie_logo())
+                    button.add_widget(image)
 
-                def set_center(_, center):
-                    image.center = center
-                button.bind(center=set_center)
+                    def set_center(_, center):
+                        image.center = center
+                    button.bind(center=set_center)
 
-                self.connect_layout.add_widget(button)
+                    self.connect_layout.add_widget(button)
                 return b
 
         self.ui = LADXManager(self)
@@ -481,6 +507,15 @@ class LinksAwakeningContext(CommonContext):
         message = [{"cmd": 'LocationChecks', "locations": self.found_checks}]
         await self.send_msgs(message)
 
+    had_invalid_slot_data = None
+    def event_invalid_slot(self):
+        # The next time we try to connect, reset the game loop for new auth
+        self.had_invalid_slot_data = True
+        self.auth = None
+        # Don't try to autoreconnect, it will just fail
+        self.disconnected_intentionally = True
+        CommonContext.event_invalid_slot(self)
+
     ENABLE_DEATHLINK = False
     async def send_deathlink(self):
         if self.ENABLE_DEATHLINK:
@@ -506,13 +541,23 @@ class LinksAwakeningContext(CommonContext):
     def new_checks(self, item_ids, ladxr_ids):
         self.found_checks += item_ids
         create_task_log_exception(self.send_checks())
-        create_task_log_exception(self.magpie.send_new_checks(ladxr_ids))
+        if self.magpie_enabled:
+            create_task_log_exception(self.magpie.send_new_checks(ladxr_ids))
 
     async def server_auth(self, password_requested: bool = False):
         if password_requested and not self.password:
             await super(LinksAwakeningContext, self).server_auth(password_requested)
+
+        if self.had_invalid_slot_data:
+            # We are connecting when previously we had the wrong ROM or server - just in case
+            # re-read the ROM so that if the user had the correct address but wrong ROM, we
+            # allow a successful reconnect
+            self.client.should_reset_auth = True
+            self.had_invalid_slot_data = False
+
+        while self.client.auth == None:
+            await asyncio.sleep(0.1)
         self.auth = self.client.auth
-        await self.get_username()
         await self.send_connect()
 
     def on_package(self, cmd: str, args: dict):
@@ -520,9 +565,13 @@ class LinksAwakeningContext(CommonContext):
             self.game = self.slot_info[self.slot].game
         # TODO - use watcher_event
         if cmd == "ReceivedItems":
-            for index, item in enumerate(args["items"], args["index"]):
+            for index, item in enumerate(args["items"], start=args["index"]):
                 self.client.recvd_checks[index] = item
 
+    async def sync(self):
+        sync_msg = [{'cmd': 'Sync'}]
+        await self.send_msgs(sync_msg)
+
     item_id_lookup = get_locations_to_id()
 
     async def run_game_loop(self):
@@ -537,18 +586,33 @@ class LinksAwakeningContext(CommonContext):
         async def deathlink():
             await self.send_deathlink()
 
-        self.magpie_task = asyncio.create_task(self.magpie.serve())
-        
+        if self.magpie_enabled:
+            self.magpie_task = asyncio.create_task(self.magpie.serve())
+
         # yield to allow UI to start
         await asyncio.sleep(0)
 
         while True:
             try:
                 # TODO: cancel all client tasks
-                logger.info("(Re)Starting game loop")
+                if not self.client.stop_bizhawk_spam:
+                    logger.info("(Re)Starting game loop")
                 self.found_checks.clear()
+                # On restart of game loop, clear all checks, just in case we swapped ROMs
+                # this isn't totally neccessary, but is extra safety against cross-ROM contamination
+                self.client.recvd_checks.clear()
                 await self.client.wait_for_retroarch_connection()
-                self.client.reset_auth()
+                await self.client.reset_auth()
+                # If we find ourselves with new auth after the reset, reconnect
+                if self.auth and self.client.auth != self.auth:
+                    # It would be neat to reconnect here, but connection needs this loop to be running
+                    logger.info("Detected new ROM, disconnecting...")
+                    await self.disconnect()
+                    continue
+
+                if not self.client.recvd_checks:
+                    await self.sync()
+
                 await self.client.wait_and_init_tracker()
 
                 while True:
@@ -558,39 +622,62 @@ class LinksAwakeningContext(CommonContext):
                     if self.last_resend + 5.0 < now:
                         self.last_resend = now
                         await self.send_checks()
-                    self.magpie.set_checks(self.client.tracker.all_checks)
-                    await self.magpie.set_item_tracker(self.client.item_tracker)
-                    await self.magpie.send_gps(self.client.gps_tracker)
+                    if self.magpie_enabled:
+                        try:
+                            self.magpie.set_checks(self.client.tracker.all_checks)
+                            await self.magpie.set_item_tracker(self.client.item_tracker)
+                            await self.magpie.send_gps(self.client.gps_tracker)
+                        except Exception:
+                            # Don't let magpie errors take out the client
+                            pass
+                    if self.client.should_reset_auth:
+                        self.client.should_reset_auth = False
+                        raise GameboyException("Resetting due to wrong archipelago server")
+            except (GameboyException, asyncio.TimeoutError, TimeoutError, ConnectionResetError):
+                await asyncio.sleep(1.0)
 
-            except GameboyException:
-                time.sleep(1.0)
-                pass
+def run_game(romfile: str) -> None:
+    auto_start = typing.cast(typing.Union[bool, str],
+                            Utils.get_options()["ladx_options"].get("rom_start", True))
+    if auto_start is True:
+        import webbrowser
+        webbrowser.open(romfile)
+    elif isinstance(auto_start, str):
+        args = shlex.split(auto_start)
+        # Specify full path to ROM as we are going to cd in popen
+        full_rom_path = os.path.realpath(romfile)
+        args.append(full_rom_path)
+        try:
+            # set cwd so that paths to lua scripts are always relative to our client
+            if getattr(sys, 'frozen', False):
+                # The application is frozen
+                script_dir = os.path.dirname(sys.executable)
+            else:
+                script_dir = os.path.dirname(os.path.realpath(__file__))
 
+            subprocess.Popen(args, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=script_dir)
+        except FileNotFoundError:
+            logger.error(f"Couldn't launch ROM, {args[0]} is missing")
 
 async def main():
     parser = get_base_parser(description="Link's Awakening Client.")
     parser.add_argument("--url", help="Archipelago connection url")
-
+    parser.add_argument("--no-magpie", dest='magpie', default=True, action='store_false', help="Disable magpie bridge")
     parser.add_argument('diff_file', default="", type=str, nargs="?",
                         help='Path to a .apladx Archipelago Binary Patch file')
+
     args = parser.parse_args()
-    logger.info(args)
 
     if args.diff_file:
         import Patch
         logger.info("patch file was supplied - creating rom...")
         meta, rom_file = Patch.create_rom_file(args.diff_file)
-        if "server" in meta:
-            args.url = meta["server"]
+        if "server" in meta and not args.connect:
+            args.connect = meta["server"]
         logger.info(f"wrote rom file to {rom_file}")
 
-    if args.url:
-        url = urllib.parse.urlparse(args.url)
-        args.connect = url.netloc
-        if url.password:
-            args.password = urllib.parse.unquote(url.password)
 
-    ctx = LinksAwakeningContext(args.connect, args.password)
+    ctx = LinksAwakeningContext(args.connect, args.password, args.magpie)
 
     ctx.server_task = asyncio.create_task(server_loop(ctx), name="server loop")
 
@@ -600,6 +687,10 @@ async def main():
         ctx.run_gui()
     ctx.run_cli()
 
+    # Down below run_gui so that we get errors out of the process
+    if args.diff_file:
+        run_game(rom_file)
+
     await ctx.exit_event.wait()
     await ctx.shutdown()
 

LttPAdjuster.py

@@ -25,7 +25,7 @@ ModuleUpdate.update()
 
 from worlds.alttp.Rom import Sprite, LocalRom, apply_rom_settings, get_base_rom_bytes
 from Utils import output_path, local_path, user_path, open_file, get_cert_none_ssl_context, persistent_store, \
-    get_adjuster_settings, tkinter_center_window, init_logging
+    get_adjuster_settings, get_adjuster_settings_no_defaults, tkinter_center_window, init_logging
 
 
 GAME_ALTTP = "A Link to the Past"
@@ -43,8 +43,49 @@ class ArgumentDefaultsHelpFormatter(argparse.RawTextHelpFormatter):
     def _get_help_string(self, action):
         return textwrap.dedent(action.help)
 
+# See argparse.BooleanOptionalAction
+class BooleanOptionalActionWithDisable(argparse.Action):
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=None,
+                 type=None,
+                 choices=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
 
-def main():
+        _option_strings = []
+        for option_string in option_strings:
+            _option_strings.append(option_string)
+
+            if option_string.startswith('--'):
+                option_string = '--disable' + option_string[2:]
+                _option_strings.append(option_string)
+
+        if help is not None and default is not None:
+            help += " (default: %(default)s)"
+
+        super().__init__(
+            option_strings=_option_strings,
+            dest=dest,
+            nargs=0,
+            default=default,
+            type=type,
+            choices=choices,
+            required=required,
+            help=help,
+            metavar=metavar)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        if option_string in self.option_strings:
+            setattr(namespace, self.dest, not option_string.startswith('--disable'))
+
+    def format_usage(self):
+        return ' | '.join(self.option_strings)
+
+
+def get_argparser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 
     parser.add_argument('rom', nargs="?", default='AP_LttP.sfc', help='Path to an ALttP rom to adjust.')
@@ -52,6 +93,8 @@ def main():
                         help='Path to an ALttP Japan(1.0) rom to use as a base.')
     parser.add_argument('--loglevel', default='info', const='info', nargs='?',
                         choices=['error', 'info', 'warning', 'debug'], help='Select level of logging for output.')
+    parser.add_argument('--auto_apply', default='ask',
+                        choices=['ask', 'always', 'never'], help='Whether or not to apply settings automatically in the future.')
     parser.add_argument('--menuspeed', default='normal', const='normal', nargs='?',
                         choices=['normal', 'instant', 'double', 'triple', 'quadruple', 'half'],
                         help='''\
@@ -61,7 +104,7 @@ def main():
     parser.add_argument('--quickswap', help='Enable quick item swapping with L and R.', action='store_true')
     parser.add_argument('--deathlink', help='Enable DeathLink system.', action='store_true')
     parser.add_argument('--allowcollect', help='Allow collection of other player items', action='store_true')
-    parser.add_argument('--disablemusic', help='Disables game music.', action='store_true')
+    parser.add_argument('--music', default=True, help='Enables/Disables game music.', action=BooleanOptionalActionWithDisable)
     parser.add_argument('--triforcehud', default='hide_goal', const='hide_goal', nargs='?',
                         choices=['normal', 'hide_goal', 'hide_required', 'hide_both'],
                         help='''\
@@ -85,9 +128,6 @@ def main():
     parser.add_argument('--ow_palettes', default='default',
                         choices=['default', 'random', 'blackout', 'puke', 'classic', 'grayscale', 'negative', 'dizzy',
                                  'sick'])
-    # parser.add_argument('--link_palettes', default='default',
-    #                     choices=['default', 'random', 'blackout', 'puke', 'classic', 'grayscale', 'negative', 'dizzy',
-    #                              'sick'])
     parser.add_argument('--shield_palettes', default='default',
                         choices=['default', 'random', 'blackout', 'puke', 'classic', 'grayscale', 'negative', 'dizzy',
                                  'sick'])
@@ -107,16 +147,23 @@ def main():
                              Alternatively, can be a ALttP Rom patched with a Link
                              sprite that will be extracted.
                              ''')
+    parser.add_argument('--sprite_pool', nargs='+', default=[], help='''
+                             A list of sprites to pull from.
+                        ''')
     parser.add_argument('--oof', help='''\
                              Path to a sound effect to replace Link's "oof" sound.
                              Needs to be in a .brr format and have a length of no
                              more than 2673 bytes, created from a 16-bit signed PCM
                              .wav at 12khz. https://github.com/boldowa/snesbrr
                              ''')
-    parser.add_argument('--names', default='', type=str)
     parser.add_argument('--update_sprites', action='store_true', help='Update Sprite Database, then exit.')
-    args = parser.parse_args()
-    args.music = not args.disablemusic
+    return parser
+
+
+def main():
+    parser = get_argparser()
+    args = parser.parse_args(namespace=get_adjuster_settings_no_defaults(GAME_ALTTP))
+    
     # set up logger
     loglevel = {'error': logging.ERROR, 'info': logging.INFO, 'warning': logging.WARNING, 'debug': logging.DEBUG}[
         args.loglevel]
@@ -193,7 +240,7 @@ def adjustGUI():
     from tkinter import Tk, LEFT, BOTTOM, TOP, \
         StringVar, Frame, Label, X, Entry, Button, filedialog, messagebox, ttk
     from argparse import Namespace
-    from Main import __version__ as MWVersion
+    from Utils import __version__ as MWVersion
     adjustWindow = Tk()
     adjustWindow.wm_title("Archipelago %s LttP Adjuster" % MWVersion)
     set_icon(adjustWindow)
@@ -528,9 +575,6 @@ class AttachTooltip(object):
 
 def get_rom_frame(parent=None):
     adjuster_settings = get_adjuster_settings(GAME_ALTTP)
-    if not adjuster_settings:
-        adjuster_settings = Namespace()
-        adjuster_settings.baserom = "Zelda no Densetsu - Kamigami no Triforce (Japan).sfc"
 
     romFrame = Frame(parent)
     baseRomLabel = Label(romFrame, text='LttP Base Rom: ')
@@ -558,33 +602,8 @@ def get_rom_frame(parent=None):
 
     return romFrame, romVar
 
-
 def get_rom_options_frame(parent=None):
     adjuster_settings = get_adjuster_settings(GAME_ALTTP)
-    defaults = {
-        "auto_apply": 'ask',
-        "music": True,
-        "reduceflashing": True,
-        "deathlink": False,
-        "sprite": None,
-        "oof": None,
-        "quickswap": True,
-        "menuspeed": 'normal',
-        "heartcolor": 'red',
-        "heartbeep": 'normal',
-        "ow_palettes": 'default',
-        "uw_palettes": 'default',
-        "hud_palettes": 'default',
-        "sword_palettes": 'default',
-        "shield_palettes": 'default',
-        "sprite_pool": [],
-        "allowcollect": False,
-    }
-    if not adjuster_settings:
-        adjuster_settings = Namespace()
-    for key, defaultvalue in defaults.items():
-        if not hasattr(adjuster_settings, key):
-            setattr(adjuster_settings, key, defaultvalue)
 
     romOptionsFrame = LabelFrame(parent, text="Rom options")
     romOptionsFrame.columnconfigure(0, weight=1)

MMBN3Client.py

@@ -0,0 +1,376 @@
+import asyncio
+import hashlib
+import json
+import os
+import multiprocessing
+import subprocess
+import zipfile
+
+from asyncio import StreamReader, StreamWriter
+
+import bsdiff4
+
+from CommonClient import CommonContext, server_loop, gui_enabled, \
+    ClientCommandProcessor, logger, get_base_parser
+import Utils
+from NetUtils import ClientStatus
+from worlds.mmbn3.Items import items_by_id
+from worlds.mmbn3.Rom import get_base_rom_path
+from worlds.mmbn3.Locations import all_locations, scoutable_locations
+
+SYSTEM_MESSAGE_ID = 0
+
+CONNECTION_TIMING_OUT_STATUS = "Connection timing out. Please restart your emulator, then restart connector_mmbn3.lua"
+CONNECTION_REFUSED_STATUS = \
+    "Connection refused. Please start your emulator and make sure connector_mmbn3.lua is running"
+CONNECTION_RESET_STATUS = "Connection was reset. Please restart your emulator, then restart connector_mmbn3.lua"
+CONNECTION_TENTATIVE_STATUS = "Initial Connection Made"
+CONNECTION_CONNECTED_STATUS = "Connected"
+CONNECTION_INITIAL_STATUS = "Connection has not been initiated"
+CONNECTION_INCORRECT_ROM = "Supplied Base Rom does not match US GBA Blue Version. Please provide the correct ROM version"
+
+script_version: int = 2
+
+debugEnabled = False
+locations_checked = []
+items_sent = []
+itemIndex = 1
+
+CHECKSUM_BLUE = "6fe31df0144759b34ad666badaacc442"
+
+
+class MMBN3CommandProcessor(ClientCommandProcessor):
+    def __init__(self, ctx):
+        super().__init__(ctx)
+
+    def _cmd_gba(self):
+        """Check GBA Connection State"""
+        if isinstance(self.ctx, MMBN3Context):
+            logger.info(f"GBA Status: {self.ctx.gba_status}")
+
+    def _cmd_debug(self):
+        """Toggle the Debug Text overlay in ROM"""
+        global debugEnabled
+        debugEnabled = not debugEnabled
+        logger.info("Debug Overlay Enabled" if debugEnabled else "Debug Overlay Disabled")
+
+
+class MMBN3Context(CommonContext):
+    command_processor = MMBN3CommandProcessor
+    game = "MegaMan Battle Network 3"
+    items_handling = 0b001  # full local
+
+    def __init__(self, server_address, password):
+        super().__init__(server_address, password)
+        self.gba_streams: (StreamReader, StreamWriter) = None
+        self.gba_sync_task = None
+        self.gba_status = CONNECTION_INITIAL_STATUS
+        self.awaiting_rom = False
+        self.location_table = {}
+        self.version_warning = False
+        self.auth_name = None
+        self.slot_data = dict()
+        self.patching_error = False
+        self.sent_hints = []
+
+    async def server_auth(self, password_requested: bool = False):
+        if password_requested and not self.password:
+            await super(MMBN3Context, self).server_auth(password_requested)
+
+        if self.auth_name is None:
+            self.awaiting_rom = True
+            logger.info("No ROM detected, awaiting conection to Bizhawk to authenticate to the multiworld server")
+            return
+
+        logger.info("Attempting to decode from ROM... ")
+        self.awaiting_rom = False
+        self.auth = self.auth_name.decode("utf8").replace('\x00', '')
+        logger.info("Connecting as "+self.auth)
+        await self.send_connect(name=self.auth)
+
+    def run_gui(self):
+        from kvui import GameManager
+
+        class MMBN3Manager(GameManager):
+            logging_pairs = [
+                ("Client", "Archipelago")
+            ]
+            base_title = "Archipelago MegaMan Battle Network 3 Client"
+
+        self.ui = MMBN3Manager(self)
+        self.ui_task = asyncio.create_task(self.ui.async_run(), name="UI")
+
+    def on_package(self, cmd: str, args: dict):
+        if cmd == 'Connected':
+            self.slot_data = args.get("slot_data", {})
+            print(self.slot_data)
+
+class ItemInfo:
+    id = 0x00
+    sender = ""
+    type = ""
+    count = 1
+    itemName = "Unknown"
+    itemID = 0x00  # Item ID, Chip ID, etc.
+    subItemID = 0x00  # Code for chips, color for programs
+    itemIndex = 1
+
+    def __init__(self, id, sender, type):
+        self.id = id
+        self.sender = sender
+        self.type = type
+
+    def get_json(self):
+        json_data = {
+            "id": self.id,
+            "sender": self.sender,
+            "type": self.type,
+            "itemName": self.itemName,
+            "itemID": self.itemID,
+            "subItemID": self.subItemID,
+            "count": self.count,
+            "itemIndex": self.itemIndex
+        }
+        return json_data
+
+
+def get_payload(ctx: MMBN3Context):
+    global debugEnabled
+
+    items_sent = []
+    for i, item in enumerate(ctx.items_received):
+        item_data = items_by_id[item.item]
+        new_item = ItemInfo(i, ctx.player_names[item.player], item_data.type)
+        new_item.itemIndex = i+1
+        new_item.itemName = item_data.itemName
+        new_item.type = item_data.type
+        new_item.itemID = item_data.itemID
+        new_item.subItemID = item_data.subItemID
+        new_item.count = item_data.count
+        items_sent.append(new_item)
+
+    return json.dumps({
+        "items": [item.get_json() for item in items_sent],
+        "debug": debugEnabled
+        })
+
+
+async def parse_payload(payload: dict, ctx: MMBN3Context, force: bool):
+    # Game completion handling
+    if payload["gameComplete"] and not ctx.finished_game:
+        await ctx.send_msgs([{
+            "cmd": "StatusUpdate",
+            "status": ClientStatus.CLIENT_GOAL
+        }])
+        ctx.finished_game = True
+
+    # Locations handling
+    if ctx.location_table != payload["locations"]:
+        ctx.location_table = payload["locations"]
+        locs = [loc.id for loc in all_locations
+                if check_location_packet(loc, ctx.location_table)]
+        await ctx.send_msgs([{
+            "cmd": "LocationChecks",
+            "locations": locs
+        }])
+
+    # If trade hinting is enabled, send scout checks
+    if ctx.slot_data.get("trade_quest_hinting", 0) == 2:
+        trade_bits = [loc.id for loc in scoutable_locations
+                        if check_location_scouted(loc, payload["locations"])]
+        scouted_locs = [loc for loc in trade_bits if loc not in ctx.sent_hints]
+        if len(scouted_locs) > 0:
+            ctx.sent_hints.extend(scouted_locs)
+            await ctx.send_msgs([{
+                "cmd": "LocationScouts",
+                "locations": scouted_locs,
+                "create_as_hint": 2
+            }])
+
+
+def check_location_packet(location, memory):
+    if len(memory) == 0:
+        return False
+    # Our keys have to be strings to come through the JSON lua plugin so we have to turn our memory address into a string as well
+    location_key = hex(location.flag_byte)[2:]
+    byte = memory.get(location_key)
+    if byte is not None:
+        return byte & location.flag_mask
+
+
+def check_location_scouted(location, memory):
+    if len(memory) == 0:
+        return False
+    location_key = hex(location.hint_flag)[2:]
+    byte = memory.get(location_key)
+    if byte is not None:
+        return byte & location.hint_flag_mask
+
+
+async def gba_sync_task(ctx: MMBN3Context):
+    logger.info("Starting GBA connector. Use /gba for status information.")
+    if ctx.patching_error:
+        logger.error('Unable to Patch ROM. No ROM provided or ROM does not match US GBA Blue Version.')
+    while not ctx.exit_event.is_set():
+        error_status = None
+        if ctx.gba_streams:
+            (reader, writer) = ctx.gba_streams
+            msg = get_payload(ctx).encode()
+            writer.write(msg)
+            writer.write(b'\n')
+            try:
+                await asyncio.wait_for(writer.drain(), timeout=1.5)
+                try:
+                    # Data will return a dict with up to four fields
+                    # 1. str: player name (always)
+                    # 2. int: script version (always)
+                    # 3. dict[str, byte]: value of location's memory byte
+                    # 4. bool: whether the game currently registers as complete
+                    data = await asyncio.wait_for(reader.readline(), timeout=10)
+                    data_decoded = json.loads(data.decode())
+                    reported_version = data_decoded.get("scriptVersion", 0)
+                    if reported_version >= script_version:
+                        if ctx.game is not None and "locations" in data_decoded:
+                            # Not just a keep alive ping, parse
+                            asyncio.create_task((parse_payload(data_decoded, ctx, False)))
+                        if not ctx.auth:
+                            ctx.auth_name = bytes(data_decoded["playerName"])
+
+                            if ctx.awaiting_rom:
+                                logger.info("Awaiting data from ROM...")
+                                await ctx.server_auth(False)
+                    else:
+                        if not ctx.version_warning:
+                            logger.warning(f"Your Lua script is version {reported_version}, expected {script_version}."
+                                           "Please update to the latest version."
+                                           "Your connection to the Archipelago server will not be accepted.")
+                            ctx.version_warning = True
+                except asyncio.TimeoutError:
+                    logger.debug("Read Timed Out, Reconnecting")
+                    error_status = CONNECTION_TIMING_OUT_STATUS
+                    writer.close()
+                    ctx.gba_streams = None
+                except ConnectionResetError:
+                    logger.debug("Read failed due to Connection Lost, Reconnecting")
+                    error_status = CONNECTION_RESET_STATUS
+                    writer.close()
+                    ctx.gba_streams = None
+            except TimeoutError:
+                logger.debug("Connection Timed Out, Reconnecting")
+                error_status = CONNECTION_TIMING_OUT_STATUS
+                writer.close()
+                ctx.gba_streams = None
+            except ConnectionResetError:
+                logger.debug("Connection Lost, Reconnecting")
+                error_status = CONNECTION_RESET_STATUS
+                writer.close()
+                ctx.gba_streams = None
+            if ctx.gba_status == CONNECTION_TENTATIVE_STATUS:
+                if not error_status:
+                    logger.info("Successfully Connected to GBA")
+                    ctx.gba_status = CONNECTION_CONNECTED_STATUS
+                else:
+                    ctx.gba_status = f"Was tentatively connected but error occurred: {error_status}"
+            elif error_status:
+                ctx.gba_status = error_status
+                logger.info("Lost connection to GBA and attempting to reconnect. Use /gba for status updates")
+        else:
+            try:
+                logger.debug("Attempting to connect to GBA")
+                ctx.gba_streams = await asyncio.wait_for(asyncio.open_connection("localhost", 28922), timeout=10)
+                ctx.gba_status = CONNECTION_TENTATIVE_STATUS
+            except TimeoutError:
+                logger.debug("Connection Timed Out, Trying Again")
+                ctx.gba_status = CONNECTION_TIMING_OUT_STATUS
+                continue
+            except ConnectionRefusedError:
+                logger.debug("Connection Refused, Trying Again")
+                ctx.gba_status = CONNECTION_REFUSED_STATUS
+                continue
+
+
+async def run_game(romfile):
+    options = Utils.get_options().get("mmbn3_options", None)
+    if options is None:
+        auto_start = True
+    else:
+        auto_start = options.get("rom_start", True)
+    if auto_start:
+        import webbrowser
+        webbrowser.open(romfile)
+    elif os.path.isfile(auto_start):
+        subprocess.Popen([auto_start, romfile],
+                         stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+
+async def patch_and_run_game(apmmbn3_file):
+    base_name = os.path.splitext(apmmbn3_file)[0]
+
+    with zipfile.ZipFile(apmmbn3_file, 'r') as patch_archive:
+        try:
+            with patch_archive.open("delta.bsdiff4", 'r') as stream:
+                patch_data = stream.read()
+        except KeyError:
+            raise FileNotFoundError("Patch file missing from archive.")
+    rom_file = get_base_rom_path()
+
+    with open(rom_file, 'rb') as rom:
+        rom_bytes = rom.read()
+
+    patched_bytes = bsdiff4.patch(rom_bytes, patch_data)
+    patched_rom_file = base_name+".gba"
+    with open(patched_rom_file, 'wb') as patched_rom:
+        patched_rom.write(patched_bytes)
+
+    asyncio.create_task(run_game(patched_rom_file))
+
+
+def confirm_checksum():
+    rom_file = get_base_rom_path()
+    if not os.path.exists(rom_file):
+        return False
+
+    with open(rom_file, 'rb') as rom:
+        rom_bytes = rom.read()
+
+    basemd5 = hashlib.md5()
+    basemd5.update(rom_bytes)
+    return CHECKSUM_BLUE == basemd5.hexdigest()
+
+
+if __name__ == "__main__":
+    Utils.init_logging("MMBN3Client")
+
+    async def main():
+        multiprocessing.freeze_support()
+        parser = get_base_parser()
+        parser.add_argument("patch_file", default="", type=str, nargs="?",
+                            help="Path to an APMMBN3 file")
+        args = parser.parse_args()
+        checksum_matches = confirm_checksum()
+        if checksum_matches:
+            if args.patch_file:
+                asyncio.create_task(patch_and_run_game(args.patch_file))
+
+        ctx = MMBN3Context(args.connect, args.password)
+        if not checksum_matches:
+            ctx.patching_error = True
+        ctx.server_task = asyncio.create_task(server_loop(ctx), name="Server Loop")
+        if gui_enabled:
+            ctx.run_gui()
+        ctx.run_cli()
+
+        ctx.gba_sync_task = asyncio.create_task(gba_sync_task(ctx), name="GBA Sync")
+        await ctx.exit_event.wait()
+        ctx.server_address = None
+        await ctx.shutdown()
+
+        if ctx.gba_sync_task:
+            await ctx.gba_sync_task
+
+    import colorama
+
+    colorama.init()
+
+    asyncio.run(main())
+    colorama.deinit()

Main.py

@@ -7,29 +7,24 @@ import tempfile
 import time
 import zipfile
 import zlib
-from typing import Dict, List, Optional, Set, Tuple
+from typing import Dict, List, Optional, Set, Tuple, Union
 
 import worlds
 from BaseClasses import CollectionState, Item, Location, LocationProgressType, MultiWorld, Region
 from Fill import balance_multiworld_progression, distribute_items_restrictive, distribute_planned, flood_items
 from Options import StartInventoryPool
-from Utils import __version__, get_options, output_path, version_tuple
+from settings import get_settings
+from Utils import __version__, output_path, version_tuple
 from worlds import AutoWorld
-from worlds.alttp.Regions import is_main_entrance
-from worlds.alttp.Shops import FillDisabledShopSlots
-from worlds.alttp.SubClasses import LTTPRegionType
 from worlds.generic.Rules import exclusion_rules, locality_rules
 
-ordered_areas = (
-    'Light World', 'Dark World', 'Hyrule Castle', 'Agahnims Tower', 'Eastern Palace', 'Desert Palace',
-    'Tower of Hera', 'Palace of Darkness', 'Swamp Palace', 'Skull Woods', 'Thieves Town', 'Ice Palace',
-    'Misery Mire', 'Turtle Rock', 'Ganons Tower', "Total"
-)
+__all__ = ["main"]
 
 
 def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = None):
     if not baked_server_options:
-        baked_server_options = get_options()["server_options"]
+        baked_server_options = get_settings().server_options.as_dict()
+    assert isinstance(baked_server_options, dict)
     if args.outputpath:
         os.makedirs(args.outputpath, exist_ok=True)
         output_path.cached_path = args.outputpath
@@ -138,12 +133,6 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
         world.non_local_items[player].value -= world.local_items[player].value
         world.non_local_items[player].value -= set(world.local_early_items[player])
 
-    if world.players > 1:
-        locality_rules(world)
-    else:
-        world.non_local_items[1].value = set()
-        world.local_items[1].value = set()
-
     AutoWorld.call_all(world, "set_rules")
 
     for player in world.player_ids:
@@ -152,6 +141,13 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
         for location_name in world.priority_locations[player].value:
             world.get_location(location_name, player).progress_type = LocationProgressType.PRIORITY
 
+    # Set local and non-local item rules.
+    if world.players > 1:
+        locality_rules(world)
+    else:
+        world.non_local_items[1].value = set()
+        world.local_items[1].value = set()
+    
     AutoWorld.call_all(world, "generate_basic")
 
     # remove starting inventory from pool items.
@@ -283,8 +279,10 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
 
     AutoWorld.call_all(world, 'post_fill')
 
-    if world.players > 1:
+    if world.players > 1 and not args.skip_prog_balancing:
         balance_multiworld_progression(world)
+    else:
+        logger.info("Progression balancing skipped.")
 
     logger.info(f'Beginning output...')
 
@@ -309,35 +307,6 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
             er_hint_data: Dict[int, Dict[int, str]] = {}
             AutoWorld.call_all(world, 'extend_hint_information', er_hint_data)
 
-            checks_in_area = {player: {area: list() for area in ordered_areas}
-                              for player in range(1, world.players + 1)}
-
-            for player in range(1, world.players + 1):
-                checks_in_area[player]["Total"] = 0
-
-            for location in world.get_filled_locations():
-                if type(location.address) is int:
-                    if location.game != "A Link to the Past":
-                        checks_in_area[location.player]["Light World"].append(location.address)
-                    else:
-                        main_entrance = location.parent_region.get_connecting_entrance(is_main_entrance)
-                        if location.parent_region.dungeon:
-                            dungeonname = {'Inverted Agahnims Tower': 'Agahnims Tower',
-                                           'Inverted Ganons Tower': 'Ganons Tower'} \
-                                .get(location.parent_region.dungeon.name, location.parent_region.dungeon.name)
-                            checks_in_area[location.player][dungeonname].append(location.address)
-                        elif location.parent_region.type == LTTPRegionType.LightWorld:
-                            checks_in_area[location.player]["Light World"].append(location.address)
-                        elif location.parent_region.type == LTTPRegionType.DarkWorld:
-                            checks_in_area[location.player]["Dark World"].append(location.address)
-                        elif main_entrance.parent_region.type == LTTPRegionType.LightWorld:
-                            checks_in_area[location.player]["Light World"].append(location.address)
-                        elif main_entrance.parent_region.type == LTTPRegionType.DarkWorld:
-                            checks_in_area[location.player]["Dark World"].append(location.address)
-                    checks_in_area[location.player]["Total"] += 1
-
-            FillDisabledShopSlots(world)
-
             def write_multidata():
                 import NetUtils
                 slot_data = {}
@@ -397,10 +366,11 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
                     for game_world in world.worlds.values()
                 }
 
+                checks_in_area: Dict[int, Dict[str, Union[int, List[int]]]] = {}
+
                 multidata = {
                     "slot_data": slot_data,
                     "slot_info": slot_info,
-                    "names": names,  # TODO: remove after 0.3.9
                     "connect_names": {name: (0, player) for player, name in world.player_name.items()},
                     "locations": locations_data,
                     "checks_in_area": checks_in_area,

MinecraftClient.py

@@ -299,7 +299,7 @@ if __name__ == '__main__':
 
     versions = get_minecraft_versions(data_version, channel)
 
-    forge_dir = Utils.user_path(options["minecraft_options"]["forge_directory"])
+    forge_dir = options["minecraft_options"]["forge_directory"]
     max_heap = options["minecraft_options"]["max_heap_size"]
     forge_version = args.forge or versions["forge"]
     java_version = args.java or versions["java"]

MultiServer.py

@@ -38,7 +38,7 @@ import NetUtils
 import Utils
 from Utils import version_tuple, restricted_loads, Version, async_start
 from NetUtils import Endpoint, ClientStatus, NetworkItem, decode, encode, NetworkPlayer, Permission, NetworkSlot, \
-    SlotType
+    SlotType, LocationStore
 
 min_client_version = Version(0, 1, 6)
 colorama.init()
@@ -152,7 +152,9 @@ class Context:
                       "compatibility": int}
     # team -> slot id -> list of clients authenticated to slot.
     clients: typing.Dict[int, typing.Dict[int, typing.List[Client]]]
-    locations: typing.Dict[int, typing.Dict[int, typing.Tuple[int, int, int]]]
+    locations: LocationStore  # typing.Dict[int, typing.Dict[int, typing.Tuple[int, int, int]]]
+    location_checks: typing.Dict[typing.Tuple[int, int], typing.Set[int]]
+    hints_used: typing.Dict[typing.Tuple[int, int], int]
     groups: typing.Dict[int, typing.Set[int]]
     save_version = 2
     stored_data: typing.Dict[str, object]
@@ -187,8 +189,6 @@ class Context:
         self.player_name_lookup: typing.Dict[str, team_slot] = {}
         self.connect_names = {}  # names of slots clients can connect to
         self.allow_releases = {}
-        #                          player          location_id     item_id  target_player_id
-        self.locations = {}
         self.host = host
         self.port = port
         self.server_password = server_password
@@ -284,6 +284,7 @@ class Context:
         except websockets.ConnectionClosed:
             logging.exception(f"Exception during send_msgs, could not send {msg}")
             await self.disconnect(endpoint)
+            return False
         else:
             if self.log_network:
                 logging.info(f"Outgoing message: {msg}")
@@ -297,6 +298,7 @@ class Context:
         except websockets.ConnectionClosed:
             logging.exception("Exception during send_encoded_msgs")
             await self.disconnect(endpoint)
+            return False
         else:
             if self.log_network:
                 logging.info(f"Outgoing message: {msg}")
@@ -311,6 +313,7 @@ class Context:
             websockets.broadcast(sockets, msg)
         except RuntimeError:
             logging.exception("Exception during broadcast_send_encoded_msgs")
+            return False
         else:
             if self.log_network:
                 logging.info(f"Outgoing broadcast: {msg}")
@@ -413,7 +416,7 @@ class Context:
         self.seed_name = decoded_obj["seed_name"]
         self.random.seed(self.seed_name)
         self.connect_names = decoded_obj['connect_names']
-        self.locations = decoded_obj['locations']
+        self.locations = LocationStore(decoded_obj.pop("locations"))  # pre-emptively free memory
         self.slot_data = decoded_obj['slot_data']
         for slot, data in self.slot_data.items():
             self.read_data[f"slot_data_{slot}"] = lambda data=data: data
@@ -792,7 +795,7 @@ async def on_client_joined(ctx: Context, client: Client):
     ctx.broadcast_text_all(
         f"{ctx.get_aliased_name(client.team, client.slot)} (Team #{client.team + 1}) "
         f"{verb} {ctx.games[client.slot]} has joined. "
-        f"Client({version_str}), {client.tags}).",
+        f"Client({version_str}), {client.tags}.",
         {"type": "Join", "team": client.team, "slot": client.slot, "tags": client.tags})
     ctx.notify_client(client, "Now that you are connected, "
                               "you can use !help to list commands to run via the server. "
@@ -902,11 +905,7 @@ def release_player(ctx: Context, team: int, slot: int):
 
 def collect_player(ctx: Context, team: int, slot: int, is_group: bool = False):
     """register any locations that are in the multidata, pointing towards this player"""
-    all_locations = collections.defaultdict(set)
-    for source_slot, location_data in ctx.locations.items():
-        for location_id, values in location_data.items():
-            if values[1] == slot:
-                all_locations[source_slot].add(location_id)
+    all_locations = ctx.locations.get_for_player(slot)
 
     ctx.broadcast_text_all("%s (Team #%d) has collected their items from other worlds."
                            % (ctx.player_names[(team, slot)], team + 1),
@@ -925,11 +924,7 @@ def collect_player(ctx: Context, team: int, slot: int, is_group: bool = False):
 
 
 def get_remaining(ctx: Context, team: int, slot: int) -> typing.List[int]:
-    items = []
-    for location_id in ctx.locations[slot]:
-        if location_id not in ctx.location_checks[team, slot]:
-            items.append(ctx.locations[slot][location_id][0])  # item ID
-    return sorted(items)
+    return ctx.locations.get_remaining(ctx.location_checks, team, slot)
 
 
 def send_items_to(ctx: Context, team: int, target_slot: int, *items: NetworkItem):
@@ -977,13 +972,12 @@ def collect_hints(ctx: Context, team: int, slot: int, item: typing.Union[int, st
             slots.add(group_id)
 
     seeked_item_id = item if isinstance(item, int) else ctx.item_names_for_game(ctx.games[slot])[item]
-    for finding_player, check_data in ctx.locations.items():
-        for location_id, (item_id, receiving_player, item_flags) in check_data.items():
-            if receiving_player in slots and item_id == seeked_item_id:
-                found = location_id in ctx.location_checks[team, finding_player]
-                entrance = ctx.er_hint_data.get(finding_player, {}).get(location_id, "")
-                hints.append(NetUtils.Hint(receiving_player, finding_player, location_id, item_id, found, entrance,
-                                           item_flags))
+    for finding_player, location_id, item_id, receiving_player, item_flags \
+            in ctx.locations.find_item(slots, seeked_item_id):
+        found = location_id in ctx.location_checks[team, finding_player]
+        entrance = ctx.er_hint_data.get(finding_player, {}).get(location_id, "")
+        hints.append(NetUtils.Hint(receiving_player, finding_player, location_id, item_id, found, entrance,
+                                   item_flags))
 
     return hints
 
@@ -1555,15 +1549,11 @@ class ClientMessageProcessor(CommonCommandProcessor):
 
 
 def get_checked_checks(ctx: Context, team: int, slot: int) -> typing.List[int]:
-    return [location_id for
-            location_id in ctx.locations[slot] if
-            location_id in ctx.location_checks[team, slot]]
+    return ctx.locations.get_checked(ctx.location_checks, team, slot)
 
 
 def get_missing_checks(ctx: Context, team: int, slot: int) -> typing.List[int]:
-    return [location_id for
-            location_id in ctx.locations[slot] if
-            location_id not in ctx.location_checks[team, slot]]
+    return ctx.locations.get_missing(ctx.location_checks, team, slot)
 
 
 def get_client_points(ctx: Context, client: Client) -> int:
@@ -2128,13 +2118,15 @@ class ServerCommandProcessor(CommonCommandProcessor):
 async def console(ctx: Context):
     import sys
     queue = asyncio.Queue()
-    Utils.stream_input(sys.stdin, queue)
+    worker = Utils.stream_input(sys.stdin, queue)
     while not ctx.exit_event.is_set():
         try:
             # I don't get why this while loop is needed. Works fine without it on clients,
             # but the queue.get() for server never fulfills if the queue is empty when entering the await.
             while queue.qsize() == 0:
                 await asyncio.sleep(0.05)
+                if not worker.is_alive():
+                    return
             input_text = await queue.get()
             queue.task_done()
             ctx.commandprocessor(input_text)
@@ -2145,7 +2137,7 @@ async def console(ctx: Context):
 
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser()
-    defaults = Utils.get_options()["server_options"]
+    defaults = Utils.get_options()["server_options"].as_dict()
     parser.add_argument('multidata', nargs="?", default=defaults["multidata"])
     parser.add_argument('--host', default=defaults["host"])
     parser.add_argument('--port', default=defaults["port"], type=int)
@@ -2254,12 +2246,15 @@ async def main(args: argparse.Namespace):
                 if not isinstance(e, ImportError):
                     logging.error(f"Failed to load tkinter ({e})")
                 logging.info("Pass a multidata filename on command line to run headless.")
-                exit(1)
+                # when cx_Freeze'd the built-in exit is not available, so we import sys.exit instead
+                import sys
+                sys.exit(1)
             raise
 
         if not data_filename:
             logging.info("No file selected. Exiting.")
-            exit(1)
+            import sys
+            sys.exit(1)
 
     try:
         ctx.load(data_filename, args.use_embedded_options)

NetUtils.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import typing
 import enum
+import warnings
 from json import JSONEncoder, JSONDecoder
 
 import websockets
@@ -343,3 +344,77 @@ class Hint(typing.NamedTuple):
     @property
     def local(self):
         return self.receiving_player == self.finding_player
+
+
+class _LocationStore(dict, typing.MutableMapping[int, typing.Dict[int, typing.Tuple[int, int, int]]]):
+    def __init__(self, values: typing.MutableMapping[int, typing.Dict[int, typing.Tuple[int, int, int]]]):
+        super().__init__(values)
+
+        if not self:
+            raise ValueError(f"Rejecting game with 0 players")
+
+        if len(self) != max(self):
+            raise ValueError("Player IDs not continuous")
+
+        if len(self.get(0, {})):
+            raise ValueError("Invalid player id 0 for location")
+
+    def find_item(self, slots: typing.Set[int], seeked_item_id: int
+                  ) -> typing.Generator[typing.Tuple[int, int, int, int, int], None, None]:
+        for finding_player, check_data in self.items():
+            for location_id, (item_id, receiving_player, item_flags) in check_data.items():
+                if receiving_player in slots and item_id == seeked_item_id:
+                    yield finding_player, location_id, item_id, receiving_player, item_flags
+
+    def get_for_player(self, slot: int) -> typing.Dict[int, typing.Set[int]]:
+        import collections
+        all_locations: typing.Dict[int, typing.Set[int]] = collections.defaultdict(set)
+        for source_slot, location_data in self.items():
+            for location_id, values in location_data.items():
+                if values[1] == slot:
+                    all_locations[source_slot].add(location_id)
+        return all_locations
+
+    def get_checked(self, state: typing.Dict[typing.Tuple[int, int], typing.Set[int]], team: int, slot: int
+                    ) -> typing.List[int]:
+        checked = state[team, slot]
+        if not checked:
+            # This optimizes the case where everyone connects to a fresh game at the same time.
+            return []
+        return [location_id for
+                location_id in self[slot] if
+                location_id in checked]
+
+    def get_missing(self, state: typing.Dict[typing.Tuple[int, int], typing.Set[int]], team: int, slot: int
+                    ) -> typing.List[int]:
+        checked = state[team, slot]
+        if not checked:
+            # This optimizes the case where everyone connects to a fresh game at the same time.
+            return list(self[slot])
+        return [location_id for
+                location_id in self[slot] if
+                location_id not in checked]
+
+    def get_remaining(self, state: typing.Dict[typing.Tuple[int, int], typing.Set[int]], team: int, slot: int
+                      ) -> typing.List[int]:
+        checked = state[team, slot]
+        player_locations = self[slot]
+        return sorted([player_locations[location_id][0] for
+                       location_id in player_locations if
+                       location_id not in checked])
+
+
+if typing.TYPE_CHECKING:  # type-check with pure python implementation until we have a typing stub
+    LocationStore = _LocationStore
+else:
+    try:
+        import pyximport
+        pyximport.install()
+    except ImportError:
+        pyximport = None
+    try:
+        from _speedups import LocationStore
+    except ImportError:
+        warnings.warn("_speedups not available. Falling back to pure python LocationStore. "
+                      "Install a matching C++ compiler for your platform to compile _speedups.")
+        LocationStore = _LocationStore

OoTAdjuster.py

@@ -44,7 +44,7 @@ def adjustGUI():
         StringVar, IntVar, Checkbutton, Frame, Label, X, Entry, Button, \
         OptionMenu, filedialog, messagebox, ttk
     from argparse import Namespace
-    from Main import __version__ as MWVersion
+    from Utils import __version__ as MWVersion
 
     window = tk.Tk()
     window.wm_title(f"Archipelago {MWVersion} OoT Adjuster")

OoTClient.py

@@ -100,7 +100,7 @@ class OoTContext(CommonContext):
             await super(OoTContext, self).server_auth(password_requested)
         if not self.auth:
             self.awaiting_rom = True
-            logger.info('Awaiting connection to Bizhawk to get player information')
+            logger.info('Awaiting connection to EmuHawk to get player information')
             return
 
         await self.send_connect()
@@ -296,8 +296,6 @@ async def patch_and_run_game(apz5_file):
     comp_path = base_name + '.z64'
     # Load vanilla ROM, patch file, compress ROM
     rom_file_name = Utils.get_options()["oot_options"]["rom_file"]
-    if not os.path.exists(rom_file_name):
-        rom_file_name = Utils.user_path(rom_file_name)
     rom = Rom(rom_file_name)
 
     sub_file = None

Options.py

@@ -1,13 +1,15 @@
 from __future__ import annotations
+
 import abc
 import logging
-from copy import deepcopy
 import math
 import numbers
-import typing
 import random
+import typing
+from copy import deepcopy
+
+from schema import And, Optional, Or, Schema
 
-from schema import Schema, And, Or, Optional
 from Utils import get_fuzzy_results
 
 if typing.TYPE_CHECKING:
@@ -769,7 +771,7 @@ class VerifyKeys(metaclass=FreezeValidKeys):
                                     f"Did you mean '{picks[0][0]}' ({picks[0][1]}% sure)")
 
 
-class OptionDict(Option[typing.Dict[str, typing.Any]], VerifyKeys):
+class OptionDict(Option[typing.Dict[str, typing.Any]], VerifyKeys, typing.Mapping[str, typing.Any]):
     default: typing.Dict[str, typing.Any] = {}
     supports_weighting = False
 
@@ -787,8 +789,14 @@ class OptionDict(Option[typing.Dict[str, typing.Any]], VerifyKeys):
     def get_option_name(self, value):
         return ", ".join(f"{key}: {v}" for key, v in value.items())
 
-    def __contains__(self, item):
-        return item in self.value
+    def __getitem__(self, item: str) -> typing.Any:
+        return self.value.__getitem__(item)
+
+    def __iter__(self) -> typing.Iterator[str]:
+        return self.value.__iter__()
+
+    def __len__(self) -> int:
+        return self.value.__len__()
 
 
 class ItemDict(OptionDict):
@@ -949,6 +957,7 @@ class DeathLink(Toggle):
 
 class ItemLinks(OptionList):
     """Share part of your item pool with other players."""
+    display_name = "Item Links"
     default = []
     schema = Schema([
         {

PokemonClient.py

@@ -29,6 +29,9 @@ for location in location_data:
             location_map[type(location.ram_address).__name__][location.ram_address.flag] = location.address
             location_bytes_bits[location.address] = {'byte': location.ram_address.byte, 'bit': location.ram_address.bit}
 
+location_name_to_id = {location.name: location.address for location in location_data if location.type == "Item"
+                           and location.address is not None}
+
 SYSTEM_MESSAGE_ID = 0
 
 CONNECTION_TIMING_OUT_STATUS = "Connection timing out. Please restart your emulator, then restart pkmn_rb.lua"
@@ -72,13 +75,14 @@ class GBContext(CommonContext):
         self.items_handling = 0b001
         self.sent_release = False
         self.sent_collect = False
+        self.auto_hints = set()
 
     async def server_auth(self, password_requested: bool = False):
         if password_requested and not self.password:
             await super(GBContext, self).server_auth(password_requested)
         if not self.auth:
             self.awaiting_rom = True
-            logger.info('Awaiting connection to Bizhawk to get Player information')
+            logger.info('Awaiting connection to EmuHawk to get Player information')
             return
 
         await self.send_connect()
@@ -153,6 +157,33 @@ async def parse_locations(data: List, ctx: GBContext):
                     locations.append(loc_id)
             elif flags[flag_type][location_bytes_bits[loc_id]['byte']] & 1 << location_bytes_bits[loc_id]['bit']:
                 locations.append(loc_id)
+
+    hints = []
+    if flags["EventFlag"][280] & 16:
+        hints.append("Cerulean Bicycle Shop")
+    if flags["EventFlag"][280] & 32:
+        hints.append("Route 2 Gate - Oak's Aide")
+    if flags["EventFlag"][280] & 64:
+        hints.append("Route 11 Gate 2F - Oak's Aide")
+    if flags["EventFlag"][280] & 128:
+        hints.append("Route 15 Gate 2F - Oak's Aide")
+    if flags["EventFlag"][281] & 1:
+        hints += ["Celadon Prize Corner - Item Prize 1", "Celadon Prize Corner - Item Prize 2",
+                  "Celadon Prize Corner - Item Prize 3"]
+    if (location_name_to_id["Fossil - Choice A"] in ctx.checked_locations and location_name_to_id["Fossil - Choice B"]
+            not in ctx.checked_locations):
+        hints.append("Fossil - Choice B")
+    elif (location_name_to_id["Fossil - Choice B"] in ctx.checked_locations and location_name_to_id["Fossil - Choice A"]
+            not in ctx.checked_locations):
+        hints.append("Fossil - Choice A")
+    hints = [
+        location_name_to_id[loc] for loc in hints if location_name_to_id[loc] not in ctx.auto_hints and
+        location_name_to_id[loc] in ctx.missing_locations and location_name_to_id[loc] not in ctx.locations_checked
+    ]
+    if hints:
+        await ctx.send_msgs([{"cmd": "LocationScouts", "locations": hints, "create_as_hint": 2}])
+    ctx.auto_hints.update(hints)
+
     if flags["EventFlag"][280] & 1 and not ctx.finished_game:
         await ctx.send_msgs([
                     {"cmd": "StatusUpdate",

README.md

@@ -45,6 +45,12 @@ Currently, the following games are supported:
 * Adventure
 * DLC Quest
 * Noita
+* Undertale
+* Bumper Stickers
+* Mega Man Battle Network 3: Blue Version
+* Muse Dash
+* DOOM 1993
+* Terraria
 
 For setup and instructions check out our [tutorials page](https://archipelago.gg/tutorial/).
 Downloads can be found at [Releases](https://github.com/ArchipelagoMW/Archipelago/releases), including compiled

SNIClient.py

@@ -315,7 +315,7 @@ def launch_sni() -> None:
             f"please start it yourself if it is not running")
 
 
-async def _snes_connect(ctx: SNIContext, address: str) -> WebSocketClientProtocol:
+async def _snes_connect(ctx: SNIContext, address: str, retry: bool = True) -> WebSocketClientProtocol:
     address = f"ws://{address}" if "://" not in address else address
     snes_logger.info("Connecting to SNI at %s ..." % address)
     seen_problems: typing.Set[str] = set()
@@ -336,6 +336,8 @@ async def _snes_connect(ctx: SNIContext, address: str) -> WebSocketClientProtoco
             await asyncio.sleep(1)
         else:
             return snes_socket
+        if not retry:
+            break
 
 
 class SNESRequest(typing.TypedDict):
@@ -563,14 +565,16 @@ async def snes_write(ctx: SNIContext, write_list: typing.List[typing.Tuple[int,
         PutAddress_Request: SNESRequest = {"Opcode": "PutAddress", "Operands": [], 'Space': 'SNES'}
         try:
             for address, data in write_list:
-                PutAddress_Request['Operands'] = [hex(address)[2:], hex(len(data))[2:]]
-                # REVIEW: above: `if snes_socket is None: return False`
-                # Does it need to be checked again?
-                if ctx.snes_socket is not None:
-                    await ctx.snes_socket.send(dumps(PutAddress_Request))
-                    await ctx.snes_socket.send(data)
-                else:
-                    snes_logger.warning(f"Could not send data to SNES: {data}")
+                while data:
+                    # Divide the write into packets of 256 bytes.
+                    PutAddress_Request['Operands'] = [hex(address)[2:], hex(min(len(data), 256))[2:]]
+                    if ctx.snes_socket is not None:
+                        await ctx.snes_socket.send(dumps(PutAddress_Request))
+                        await ctx.snes_socket.send(data[:256])
+                        address += 256
+                        data = data[256:]
+                    else:
+                        snes_logger.warning(f"Could not send data to SNES: {data}")
         except ConnectionClosed:
             return False
 
@@ -684,6 +688,8 @@ async def main() -> None:
         logging.info(f"Wrote rom file to {romfile}")
         if args.diff_file.endswith(".apsoe"):
             import webbrowser
+            async_start(run_game(romfile))
+            await _snes_connect(SNIContext(args.snes, args.connect, args.password), args.snes, False)
             webbrowser.open(f"http://www.evermizer.com/apclient/#server={meta['server']}")
             logging.info("Starting Evermizer Client in your Browser...")
             import time

Starcraft2Client.py

@@ -25,11 +25,10 @@ logger = logging.getLogger("Client")
 sc2_logger = logging.getLogger("Starcraft2")
 
 import nest_asyncio
-import sc2
-from sc2.bot_ai import BotAI
-from sc2.data import Race
-from sc2.main import run_game
-from sc2.player import Bot
+from worlds._sc2common import bot
+from worlds._sc2common.bot.data import Race
+from worlds._sc2common.bot.main import run_game
+from worlds._sc2common.bot.player import Bot
 from worlds.sc2wol import SC2WoLWorld
 from worlds.sc2wol.Items import lookup_id_to_name, item_table, ItemData, type_flaggroups
 from worlds.sc2wol.Locations import SC2WOL_LOC_ID_OFFSET
@@ -240,8 +239,6 @@ class SC2Context(CommonContext):
         from kivy.uix.floatlayout import FloatLayout
         from kivy.properties import StringProperty
 
-        import Utils
-
         class HoverableButton(HoverBehavior, Button):
             pass
 
@@ -544,11 +541,11 @@ async def starcraft_launch(ctx: SC2Context, mission_id: int):
     sc2_logger.info(f"Launching {lookup_id_to_mission[mission_id]}. If game does not launch check log file for errors.")
 
     with DllDirectory(None):
-        run_game(sc2.maps.get(maps_table[mission_id - 1]), [Bot(Race.Terran, ArchipelagoBot(ctx, mission_id),
+        run_game(bot.maps.get(maps_table[mission_id - 1]), [Bot(Race.Terran, ArchipelagoBot(ctx, mission_id),
                                                                 name="Archipelago", fullscreen=True)], realtime=True)
 
 
-class ArchipelagoBot(sc2.bot_ai.BotAI):
+class ArchipelagoBot(bot.bot_ai.BotAI):
     game_running: bool = False
     mission_completed: bool = False
     boni: typing.List[bool]
@@ -867,7 +864,7 @@ def check_game_install_path() -> bool:
         documentspath = buf.value
         einfo = str(documentspath / Path("StarCraft II\\ExecuteInfo.txt"))
     else:
-        einfo = str(sc2.paths.get_home() / Path(sc2.paths.USERPATH[sc2.paths.PF]))
+        einfo = str(bot.paths.get_home() / Path(bot.paths.USERPATH[bot.paths.PF]))
 
     # Check if the file exists.
     if os.path.isfile(einfo):
@@ -883,7 +880,7 @@ def check_game_install_path() -> bool:
                                    f"try again.")
                 return False
             if os.path.exists(base):
-                executable = sc2.paths.latest_executeble(Path(base).expanduser() / "Versions")
+                executable = bot.paths.latest_executeble(Path(base).expanduser() / "Versions")
 
                 # Finally, check the path for an actual executable.
                 # If we find one, great. Set up the SC2PATH.

UndertaleClient.py

@@ -0,0 +1,512 @@
+from __future__ import annotations
+import os
+import sys
+import asyncio
+import typing
+import bsdiff4
+import shutil
+
+import Utils
+
+from NetUtils import NetworkItem, ClientStatus
+from worlds import undertale
+from MultiServer import mark_raw
+from CommonClient import CommonContext, server_loop, \
+    gui_enabled, ClientCommandProcessor, logger, get_base_parser
+from Utils import async_start
+
+
+class UndertaleCommandProcessor(ClientCommandProcessor):
+    def __init__(self, ctx):
+        super().__init__(ctx)
+
+    def _cmd_resync(self):
+        """Manually trigger a resync."""
+        if isinstance(self.ctx, UndertaleContext):
+            self.output(f"Syncing items.")
+            self.ctx.syncing = True
+
+    def _cmd_patch(self):
+        """Patch the game."""
+        if isinstance(self.ctx, UndertaleContext):
+            os.makedirs(name=os.getcwd() + "\\Undertale", exist_ok=True)
+            self.ctx.patch_game()
+            self.output("Patched.")
+
+    def _cmd_savepath(self, directory: str):
+        """Redirect to proper save data folder. (Use before connecting!)"""
+        if isinstance(self.ctx, UndertaleContext):
+            UndertaleContext.save_game_folder = directory
+            self.output("Changed to the following directory: " + directory)
+
+    @mark_raw
+    def _cmd_auto_patch(self, steaminstall: typing.Optional[str] = None):
+        """Patch the game automatically."""
+        if isinstance(self.ctx, UndertaleContext):
+            os.makedirs(name=os.getcwd() + "\\Undertale", exist_ok=True)
+            tempInstall = steaminstall
+            if not os.path.isfile(os.path.join(tempInstall, "data.win")):
+                tempInstall = None
+            if tempInstall is None:
+                tempInstall = "C:\\Program Files (x86)\\Steam\\steamapps\\common\\Undertale"
+                if not os.path.exists("C:\\Program Files (x86)\\Steam\\steamapps\\common\\Undertale"):
+                    tempInstall = "C:\\Program Files\\Steam\\steamapps\\common\\Undertale"
+            elif not os.path.exists(tempInstall):
+                tempInstall = "C:\\Program Files (x86)\\Steam\\steamapps\\common\\Undertale"
+                if not os.path.exists("C:\\Program Files (x86)\\Steam\\steamapps\\common\\Undertale"):
+                    tempInstall = "C:\\Program Files\\Steam\\steamapps\\common\\Undertale"
+            if not os.path.exists(tempInstall) or not os.path.exists(tempInstall) or not os.path.isfile(os.path.join(tempInstall, "data.win")):
+                self.output("ERROR: Cannot find Undertale. Please rerun the command with the correct folder."
+                            " command. \"/auto_patch (Steam directory)\".")
+            else:
+                for file_name in os.listdir(tempInstall):
+                    if file_name != "steam_api.dll":
+                        shutil.copy(tempInstall+"\\"+file_name,
+                               os.getcwd() + "\\Undertale\\" + file_name)
+                self.ctx.patch_game()
+                self.output("Patching successful!")
+
+    def _cmd_online(self):
+        """Makes you no longer able to see other Undertale players."""
+        if isinstance(self.ctx, UndertaleContext):
+            self.ctx.update_online_mode(not ("Online" in self.ctx.tags))
+            if "Online" in self.ctx.tags:
+                self.output(f"Now online.")
+            else:
+                self.output(f"Now offline.")
+
+    def _cmd_deathlink(self):
+        """Toggles deathlink"""
+        if isinstance(self.ctx, UndertaleContext):
+            self.ctx.deathlink_status = not self.ctx.deathlink_status
+            if self.ctx.deathlink_status:
+                self.output(f"Deathlink enabled.")
+            else:
+                self.output(f"Deathlink disabled.")
+
+
+class UndertaleContext(CommonContext):
+    tags = {"AP", "Online"}
+    game = "Undertale"
+    command_processor = UndertaleCommandProcessor
+    items_handling = 0b111
+    route = None
+    pieces_needed = None
+    completed_routes = None
+    completed_count = 0
+    save_game_folder = os.path.expandvars(r"%localappdata%/UNDERTALE")
+
+    def __init__(self, server_address, password):
+        super().__init__(server_address, password)
+        self.pieces_needed = 0
+        self.finished_game = False
+        self.game = "Undertale"
+        self.got_deathlink = False
+        self.syncing = False
+        self.deathlink_status = False
+        self.tem_armor = False
+        self.completed_count = 0
+        self.completed_routes = {"pacifist": 0, "genocide": 0, "neutral": 0}
+        # self.save_game_folder: files go in this path to pass data between us and the actual game
+        self.save_game_folder = os.path.expandvars(r"%localappdata%/UNDERTALE")
+
+    def patch_game(self):
+        with open(os.getcwd() + "/Undertale/data.win", "rb") as f:
+            patchedFile = bsdiff4.patch(f.read(), undertale.data_path("patch.bsdiff"))
+        with open(os.getcwd() + "/Undertale/data.win", "wb") as f:
+            f.write(patchedFile)
+        os.makedirs(name=os.getcwd() + "\\Undertale\\" + "Custom Sprites", exist_ok=True)
+        with open(os.path.expandvars(os.getcwd() + "\\Undertale\\" + "Custom Sprites\\" +
+                                     "Which Character.txt"), "w") as f:
+            f.writelines(["// Put the folder name of the sprites you want to play as, make sure it is the only "
+                          "line other than this one.\n", "frisk"])
+            f.close()
+
+    async def server_auth(self, password_requested: bool = False):
+        if password_requested and not self.password:
+            await super().server_auth(password_requested)
+        await self.get_username()
+        await self.send_connect()
+
+    def clear_undertale_files(self):
+        path = self.save_game_folder
+        self.finished_game = False
+        for root, dirs, files in os.walk(path):
+            for file in files:
+                if "check.spot" == file or "scout" == file:
+                    os.remove(os.path.join(root, file))
+                elif file.endswith((".item", ".victory", ".route", ".playerspot", ".mad", 
+                                            ".youDied", ".LV", ".mine", ".flag", ".hint")):
+                    os.remove(os.path.join(root, file))
+
+    async def connect(self, address: typing.Optional[str] = None):
+        self.clear_undertale_files()
+        await super().connect(address)
+
+    async def disconnect(self, allow_autoreconnect: bool = False):
+        self.clear_undertale_files()
+        await super().disconnect(allow_autoreconnect)
+
+    async def connection_closed(self):
+        self.clear_undertale_files()
+        await super().connection_closed()
+
+    async def shutdown(self):
+        self.clear_undertale_files()
+        await super().shutdown()
+
+    def update_online_mode(self, online):
+        old_tags = self.tags.copy()
+        if online:
+            self.tags.add("Online")
+        else:
+            self.tags -= {"Online"}
+        if old_tags != self.tags and self.server and not self.server.socket.closed:
+            async_start(self.send_msgs([{"cmd": "ConnectUpdate", "tags": self.tags}]))
+
+    def on_package(self, cmd: str, args: dict):
+        if cmd == "Connected":
+            self.game = self.slot_info[self.slot].game
+        async_start(process_undertale_cmd(self, cmd, args))
+
+    def run_gui(self):
+        from kvui import GameManager
+
+        class UTManager(GameManager):
+            logging_pairs = [
+                ("Client", "Archipelago")
+            ]
+            base_title = "Archipelago Undertale Client"
+
+        self.ui = UTManager(self)
+        self.ui_task = asyncio.create_task(self.ui.async_run(), name="UI")
+
+    def on_deathlink(self, data: typing.Dict[str, typing.Any]):
+        self.got_deathlink = True
+        super().on_deathlink(data)
+
+
+def to_room_name(place_name: str):
+    if place_name == "Old Home Exit":
+        return "room_ruinsexit"
+    elif place_name == "Snowdin Forest":
+        return "room_tundra1"
+    elif place_name == "Snowdin Town Exit":
+        return "room_fogroom"
+    elif place_name == "Waterfall":
+        return "room_water1"
+    elif place_name == "Waterfall Exit":
+        return "room_fire2"
+    elif place_name == "Hotland":
+        return "room_fire_prelab"
+    elif place_name == "Hotland Exit":
+        return "room_fire_precore"
+    elif place_name == "Core":
+        return "room_fire_core1"
+
+
+async def process_undertale_cmd(ctx: UndertaleContext, cmd: str, args: dict):
+    if cmd == "Connected":
+        if not os.path.exists(ctx.save_game_folder):
+            os.mkdir(ctx.save_game_folder)
+        ctx.route = args["slot_data"]["route"]
+        ctx.pieces_needed = args["slot_data"]["key_pieces"]
+        ctx.tem_armor = args["slot_data"]["temy_armor_include"]
+
+        await ctx.send_msgs([{"cmd": "Get", "keys": [str(ctx.slot)+" RoutesDone neutral",
+                                                     str(ctx.slot)+" RoutesDone pacifist",
+                                                     str(ctx.slot)+" RoutesDone genocide"]}])
+        await ctx.send_msgs([{"cmd": "SetNotify", "keys": [str(ctx.slot)+" RoutesDone neutral",
+                                                           str(ctx.slot)+" RoutesDone pacifist",
+                                                           str(ctx.slot)+" RoutesDone genocide"]}])
+        if args["slot_data"]["only_flakes"]:
+            with open(os.path.join(ctx.save_game_folder, "GenoNoChest.flag"), "w") as f:
+                f.close()
+        if not args["slot_data"]["key_hunt"]:
+            ctx.pieces_needed = 0
+        if args["slot_data"]["rando_love"]:
+            filename = f"LOVErando.LV"
+            with open(os.path.join(ctx.save_game_folder, filename), "w") as f:
+                f.close()
+        if args["slot_data"]["rando_stats"]:
+            filename = f"STATrando.LV"
+            with open(os.path.join(ctx.save_game_folder, filename), "w") as f:
+                f.close()
+        filename = f"{ctx.route}.route"
+        with open(os.path.join(ctx.save_game_folder, filename), "w") as f:
+            f.close()
+        filename = f"check.spot"
+        with open(os.path.join(ctx.save_game_folder, filename), "a") as f:
+            for ss in set(args["checked_locations"]):
+                f.write(str(ss-12000)+"\n")
+            f.close()
+    elif cmd == "LocationInfo":
+        for l in args["locations"]:
+            locationid = l.location
+            filename = f"{str(locationid-12000)}.hint"
+            with open(os.path.join(ctx.save_game_folder, filename), "w") as f:
+                toDraw = ""
+                for i in range(20):
+                    if i < len(str(ctx.item_names[l.item])):
+                        toDraw += str(ctx.item_names[l.item])[i]
+                    else:
+                        break
+                f.write(toDraw)
+                f.close()
+    elif cmd == "Retrieved":
+        if str(ctx.slot)+" RoutesDone neutral" in args["keys"]:
+            if args["keys"][str(ctx.slot)+" RoutesDone neutral"] is not None:
+                ctx.completed_routes["neutral"] = args["keys"][str(ctx.slot)+" RoutesDone neutral"]
+        if str(ctx.slot)+" RoutesDone genocide" in args["keys"]:
+            if args["keys"][str(ctx.slot)+" RoutesDone genocide"] is not None:
+                ctx.completed_routes["genocide"] = args["keys"][str(ctx.slot)+" RoutesDone genocide"]
+        if str(ctx.slot)+" RoutesDone pacifist" in args["keys"]:
+            if args["keys"][str(ctx.slot) + " RoutesDone pacifist"] is not None:
+                ctx.completed_routes["pacifist"] = args["keys"][str(ctx.slot)+" RoutesDone pacifist"]
+    elif cmd == "SetReply":
+        if args["value"] is not None:
+            if str(ctx.slot)+" RoutesDone pacifist" == args["key"]:
+                ctx.completed_routes["pacifist"] = args["value"]
+            elif str(ctx.slot)+" RoutesDone genocide" == args["key"]:
+                ctx.completed_routes["genocide"] = args["value"]
+            elif str(ctx.slot)+" RoutesDone neutral" == args["key"]:
+                ctx.completed_routes["neutral"] = args["value"]
+    elif cmd == "ReceivedItems":
+        start_index = args["index"]
+
+        if start_index == 0:
+            ctx.items_received = []
+        elif start_index != len(ctx.items_received):
+            sync_msg = [{"cmd": "Sync"}]
+            if ctx.locations_checked:
+                sync_msg.append({"cmd": "LocationChecks",
+                                 "locations": list(ctx.locations_checked)})
+            await ctx.send_msgs(sync_msg)
+        if start_index == len(ctx.items_received):
+            counter = -1
+            placedWeapon = 0
+            placedArmor = 0
+            for item in args["items"]:
+                id = NetworkItem(*item).location
+                while NetworkItem(*item).location < 0 and \
+                        counter <= id:
+                    id -= 1
+                if NetworkItem(*item).location < 0:
+                    counter -= 1
+                filename = f"{str(id)}PLR{str(NetworkItem(*item).player)}.item"
+                with open(os.path.join(ctx.save_game_folder, filename), "w") as f:
+                    if NetworkItem(*item).item == 77701:
+                        if placedWeapon == 0:
+                            f.write(str(77013-11000))
+                        elif placedWeapon == 1:
+                            f.write(str(77014-11000))
+                        elif placedWeapon == 2:
+                            f.write(str(77025-11000))
+                        elif placedWeapon == 3:
+                            f.write(str(77045-11000))
+                        elif placedWeapon == 4:
+                            f.write(str(77049-11000))
+                        elif placedWeapon == 5:
+                            f.write(str(77047-11000))
+                        elif placedWeapon == 6:
+                            if str(ctx.route) == "genocide" or str(ctx.route) == "all_routes":
+                                f.write(str(77052-11000))
+                            else:
+                                f.write(str(77051-11000))
+                        else:
+                            f.write(str(77003-11000))
+                        placedWeapon += 1
+                    elif NetworkItem(*item).item == 77702:
+                        if placedArmor == 0:
+                            f.write(str(77012-11000))
+                        elif placedArmor == 1:
+                            f.write(str(77015-11000))
+                        elif placedArmor == 2:
+                            f.write(str(77024-11000))
+                        elif placedArmor == 3:
+                            f.write(str(77044-11000))
+                        elif placedArmor == 4:
+                            f.write(str(77048-11000))
+                        elif placedArmor == 5:
+                            if str(ctx.route) == "genocide":
+                                f.write(str(77053-11000))
+                            else:
+                                f.write(str(77046-11000))
+                        elif placedArmor == 6 and ((not str(ctx.route) == "genocide") or ctx.tem_armor):
+                            if str(ctx.route) == "all_routes":
+                                f.write(str(77053-11000))
+                            elif str(ctx.route) == "genocide":
+                                f.write(str(77064-11000))
+                            else:
+                                f.write(str(77050-11000))
+                        elif placedArmor == 7 and ctx.tem_armor and not str(ctx.route) == "genocide":
+                            f.write(str(77064-11000))
+                        else:
+                            f.write(str(77004-11000))
+                        placedArmor += 1
+                    else:
+                        f.write(str(NetworkItem(*item).item-11000))
+                    f.close()
+                ctx.items_received.append(NetworkItem(*item))
+                if [item.item for item in ctx.items_received].count(77000) >= ctx.pieces_needed > 0:
+                    filename = f"{str(-99999)}PLR{str(0)}.item"
+                    with open(os.path.join(ctx.save_game_folder, filename), "w") as f:
+                        f.write(str(77787 - 11000))
+                        f.close()
+                    filename = f"{str(-99998)}PLR{str(0)}.item"
+                    with open(os.path.join(ctx.save_game_folder, filename), "w") as f:
+                        f.write(str(77789 - 11000))
+                        f.close()
+        ctx.watcher_event.set()
+
+    elif cmd == "RoomUpdate":
+        if "checked_locations" in args:
+            filename = f"check.spot"
+            with open(os.path.join(ctx.save_game_folder, filename), "a") as f:
+                for ss in set(args["checked_locations"]):
+                    f.write(str(ss-12000)+"\n")
+                f.close()
+
+    elif cmd == "Bounced":
+        tags = args.get("tags", [])
+        if "Online" in tags:
+            data = args.get("data", {})
+            if data["player"] != ctx.slot and data["player"] is not None:
+                filename = f"FRISK" + str(data["player"]) + ".playerspot"
+                with open(os.path.join(ctx.save_game_folder, filename), "w") as f:
+                    f.write(str(data["x"]) + str(data["y"]) + str(data["room"]) + str(
+                        data["spr"]) + str(data["frm"]))
+                    f.close()
+
+
+async def multi_watcher(ctx: UndertaleContext):
+    while not ctx.exit_event.is_set():
+        path = ctx.save_game_folder
+        for root, dirs, files in os.walk(path):
+            for file in files:
+                if "spots.mine" in file and "Online" in ctx.tags:
+                    with open(root + "/" + file, "r") as mine:
+                        this_x = mine.readline()
+                        this_y = mine.readline()
+                        this_room = mine.readline()
+                        this_sprite = mine.readline()
+                        this_frame = mine.readline()
+                        mine.close()
+                    message = [{"cmd": "Bounce", "tags": ["Online"],
+                                "data": {"player": ctx.slot, "x": this_x, "y": this_y, "room": this_room,
+                                         "spr": this_sprite, "frm": this_frame}}]
+                    await ctx.send_msgs(message)
+
+        await asyncio.sleep(0.1)
+
+
+async def game_watcher(ctx: UndertaleContext):
+    while not ctx.exit_event.is_set():
+        await ctx.update_death_link(ctx.deathlink_status)
+        path = ctx.save_game_folder
+        if ctx.syncing:
+            for root, dirs, files in os.walk(path):
+                for file in files:
+                    if ".item" in file:
+                        os.remove(root+"/"+file)
+            sync_msg = [{"cmd": "Sync"}]
+            if ctx.locations_checked:
+                sync_msg.append({"cmd": "LocationChecks", "locations": list(ctx.locations_checked)})
+            await ctx.send_msgs(sync_msg)
+            ctx.syncing = False
+        if ctx.got_deathlink:
+            ctx.got_deathlink = False
+            with open(os.path.join(ctx.save_game_folder, "WelcomeToTheDead.youDied"), "w") as f:
+                f.close()
+        sending = []
+        victory = False
+        found_routes = 0
+        for root, dirs, files in os.walk(path):
+            for file in files:
+                if "DontBeMad.mad" in file:
+                    os.remove(root+"/"+file)
+                    if "DeathLink" in ctx.tags:
+                        await ctx.send_death()
+                if "scout" == file:
+                    sending = []
+                    try:
+                        with open(root+"/"+file, "r") as f:
+                            lines = f.readlines()
+                        for l in lines:
+                            if ctx.server_locations.__contains__(int(l)+12000):
+                                sending = sending + [int(l.rstrip('\n'))+12000]
+                    finally:
+                        await ctx.send_msgs([{"cmd": "LocationScouts", "locations": sending,
+                                                          "create_as_hint": int(2)}])
+                        os.remove(root+"/"+file)
+                if "check.spot" in file:
+                    sending = []
+                    try:
+                        with open(root+"/"+file, "r") as f:
+                            lines = f.readlines()
+                        for l in lines:
+                            sending = sending+[(int(l.rstrip('\n')))+12000]
+                    finally:
+                        await ctx.send_msgs([{"cmd": "LocationChecks", "locations": sending}])
+                if "victory" in file and str(ctx.route) in file:
+                    victory = True
+                if ".playerspot" in file and "Online" not in ctx.tags:
+                    os.remove(root+"/"+file)
+                if "victory" in file:
+                    if str(ctx.route) == "all_routes":
+                        if "neutral" in file and ctx.completed_routes["neutral"] != 1:
+                            await ctx.send_msgs([{"cmd": "Set", "key": str(ctx.slot)+" RoutesDone neutral",
+                                                  "default": 0, "want_reply": True, "operations": [{"operation": "max",
+                                                                                                    "value": 1}]}])
+                        elif "pacifist" in file and ctx.completed_routes["pacifist"] != 1:
+                            await ctx.send_msgs([{"cmd": "Set", "key": str(ctx.slot)+" RoutesDone pacifist",
+                                                  "default": 0, "want_reply": True, "operations": [{"operation": "max",
+                                                                                                    "value": 1}]}])
+                        elif "genocide" in file and ctx.completed_routes["genocide"] != 1:
+                            await ctx.send_msgs([{"cmd": "Set", "key": str(ctx.slot)+" RoutesDone genocide",
+                                                  "default": 0, "want_reply": True, "operations": [{"operation": "max",
+                                                                                                    "value": 1}]}])
+        if str(ctx.route) == "all_routes":
+            found_routes += ctx.completed_routes["neutral"]
+            found_routes += ctx.completed_routes["pacifist"]
+            found_routes += ctx.completed_routes["genocide"]
+        if str(ctx.route) == "all_routes" and found_routes >= 3:
+            victory = True
+        ctx.locations_checked = sending
+        if (not ctx.finished_game) and victory:
+            await ctx.send_msgs([{"cmd": "StatusUpdate", "status": ClientStatus.CLIENT_GOAL}])
+            ctx.finished_game = True
+        await asyncio.sleep(0.1)
+
+
+def main():
+    Utils.init_logging("UndertaleClient", exception_logger="Client")
+
+    async def _main():
+        ctx = UndertaleContext(None, None)
+        ctx.server_task = asyncio.create_task(server_loop(ctx), name="server loop")
+        asyncio.create_task(
+            game_watcher(ctx), name="UndertaleProgressionWatcher")
+
+        asyncio.create_task(
+            multi_watcher(ctx), name="UndertaleMultiplayerWatcher")
+
+        if gui_enabled:
+            ctx.run_gui()
+        ctx.run_cli()
+
+        await ctx.exit_event.wait()
+        await ctx.shutdown()
+
+    import colorama
+
+    colorama.init()
+
+    asyncio.run(_main())
+    colorama.deinit()
+
+
+if __name__ == "__main__":
+    parser = get_base_parser(description="Undertale Client, for text interfacing.")
+    args = parser.parse_args()
+    main()

Utils.py

@@ -13,8 +13,10 @@ import io
 import collections
 import importlib
 import logging
-from typing import BinaryIO, Coroutine, Optional, Set, Dict, Any, Union
 
+from argparse import Namespace
+from settings import Settings, get_settings
+from typing import BinaryIO, Coroutine, Optional, Set, Dict, Any, Union
 from yaml import load, load_all, dump, SafeLoader
 
 try:
@@ -42,7 +44,7 @@ class Version(typing.NamedTuple):
         return ".".join(str(item) for item in self)
 
 
-__version__ = "0.4.1"
+__version__ = "0.4.2"
 version_tuple = tuplize_version(__version__)
 
 is_linux = sys.platform.startswith("linux")
@@ -138,13 +140,16 @@ def user_path(*path: str) -> str:
         user_path.cached_path = local_path()
     else:
         user_path.cached_path = home_path()
-        # populate home from local - TODO: upgrade feature
-        if user_path.cached_path != local_path() and not os.path.exists(user_path("host.yaml")):
-            import shutil
-            for dn in ("Players", "data/sprites"):
-                shutil.copytree(local_path(dn), user_path(dn), dirs_exist_ok=True)
-            for fn in ("manifest.json", "host.yaml"):
-                shutil.copy2(local_path(fn), user_path(fn))
+        # populate home from local
+        if user_path.cached_path != local_path():
+            import filecmp
+            if not os.path.exists(user_path("manifest.json")) or \
+                    not filecmp.cmp(local_path("manifest.json"), user_path("manifest.json"), shallow=True):
+                import shutil
+                for dn in ("Players", "data/sprites"):
+                    shutil.copytree(local_path(dn), user_path(dn), dirs_exist_ok=True)
+                for fn in ("manifest.json",):
+                    shutil.copy2(local_path(fn), user_path(fn))
 
     return os.path.join(user_path.cached_path, *path)
 
@@ -238,151 +243,15 @@ def get_public_ipv6() -> str:
     return ip
 
 
-OptionsType = typing.Dict[str, typing.Dict[str, typing.Any]]
+OptionsType = Settings  # TODO: remove ~2 versions after 0.4.1
 
 
 @cache_argsless
-def get_default_options() -> OptionsType:
-    # Refer to host.yaml for comments as to what all these options mean.
-    options = {
-        "general_options": {
-            "output_path": "output",
-        },
-        "factorio_options": {
-            "executable": os.path.join("factorio", "bin", "x64", "factorio"),
-            "filter_item_sends": False,
-            "bridge_chat_out": True,
-        },
-        "sni_options": {
-            "sni_path": "SNI",
-            "snes_rom_start": True,
-        },
-        "sm_options": {
-            "rom_file": "Super Metroid (JU).sfc",
-        },
-        "soe_options": {
-            "rom_file": "Secret of Evermore (USA).sfc",
-        },
-        "lttp_options": {
-            "rom_file": "Zelda no Densetsu - Kamigami no Triforce (Japan).sfc",
-        },
-        "ladx_options": {
-            "rom_file": "Legend of Zelda, The - Link's Awakening DX (USA, Europe) (SGB Enhanced).gbc",
-        },
-        "server_options": {
-            "host": None,
-            "port": 38281,
-            "password": None,
-            "multidata": None,
-            "savefile": None,
-            "disable_save": False,
-            "loglevel": "info",
-            "server_password": None,
-            "disable_item_cheat": False,
-            "location_check_points": 1,
-            "hint_cost": 10,
-            "release_mode": "goal",
-            "collect_mode": "disabled",
-            "remaining_mode": "goal",
-            "auto_shutdown": 0,
-            "compatibility": 2,
-            "log_network": 0
-        },
-        "generator": {
-            "enemizer_path": os.path.join("EnemizerCLI", "EnemizerCLI.Core"),
-            "player_files_path": "Players",
-            "players": 0,
-            "weights_file_path": "weights.yaml",
-            "meta_file_path": "meta.yaml",
-            "spoiler": 3,
-            "glitch_triforce_room": 1,
-            "race": 0,
-            "plando_options": "bosses",
-        },
-        "minecraft_options": {
-            "forge_directory": "Minecraft Forge server",
-            "max_heap_size": "2G",
-            "release_channel": "release"
-        },
-        "oot_options": {
-            "rom_file": "The Legend of Zelda - Ocarina of Time.z64",
-            "rom_start": True
-        },
-        "dkc3_options": {
-            "rom_file": "Donkey Kong Country 3 - Dixie Kong's Double Trouble! (USA) (En,Fr).sfc",
-        },
-        "smw_options": {
-            "rom_file": "Super Mario World (USA).sfc",
-        },
-        "zillion_options": {
-            "rom_file": "Zillion (UE) [!].sms",
-            # RetroArch doesn't make it easy to launch a game from the command line.
-            # You have to know the path to the emulator core library on the user's computer.
-            "rom_start": "retroarch",
-        },
-        "pokemon_rb_options": {
-            "red_rom_file": "Pokemon Red (UE) [S][!].gb",
-            "blue_rom_file": "Pokemon Blue (UE) [S][!].gb",
-            "rom_start": True
-        },
-        "ffr_options": {
-            "display_msgs": True,
-        },
-        "lufia2ac_options": {
-            "rom_file": "Lufia II - Rise of the Sinistrals (USA).sfc",
-        },
-        "tloz_options": {
-            "rom_file": "Legend of Zelda, The (U) (PRG0) [!].nes",
-            "rom_start": True,
-            "display_msgs": True,
-        },
-        "wargroove_options": {
-            "root_directory": "C:/Program Files (x86)/Steam/steamapps/common/Wargroove"
-        },
-        "adventure_options": {
-            "rom_file": "ADVNTURE.BIN",
-            "display_msgs": True,
-            "rom_start": True,
-            "rom_args": ""
-        },
-    }
-    return options
+def get_default_options() -> Settings:  # TODO: remove ~2 versions after 0.4.1
+    return Settings(None)
 
 
-def update_options(src: dict, dest: dict, filename: str, keys: list) -> OptionsType:
-    for key, value in src.items():
-        new_keys = keys.copy()
-        new_keys.append(key)
-        option_name = '.'.join(new_keys)
-        if key not in dest:
-            dest[key] = value
-            if filename.endswith("options.yaml"):
-                logging.info(f"Warning: {filename} is missing {option_name}")
-        elif isinstance(value, dict):
-            if not isinstance(dest.get(key, None), dict):
-                if filename.endswith("options.yaml"):
-                    logging.info(f"Warning: {filename} has {option_name}, but it is not a dictionary. overwriting.")
-                dest[key] = value
-            else:
-                dest[key] = update_options(value, dest[key], filename, new_keys)
-    return dest
-
-
-@cache_argsless
-def get_options() -> OptionsType:
-    filenames = ("options.yaml", "host.yaml")
-    locations: typing.List[str] = []
-    if os.path.join(os.getcwd()) != local_path():
-        locations += filenames  # use files from cwd only if it's not the local_path
-    locations += [user_path(filename) for filename in filenames]
-
-    for location in locations:
-        if os.path.exists(location):
-            with open(location) as f:
-                options = parse_yaml(f.read())
-            return update_options(get_default_options(), options, location, list())
-
-    raise FileNotFoundError(f"Could not find {filenames[1]} to load options.")
+get_options = get_settings  # TODO: add a warning ~2 versions after 0.4.1 and remove once all games are ported
 
 
 def persistent_store(category: str, key: typing.Any, value: typing.Any):
@@ -450,12 +319,27 @@ def store_data_package_for_checksum(game: str, data: typing.Dict[str, Any]) -> N
         except Exception as e:
             logging.debug(f"Could not store data package: {e}")
 
+def get_default_adjuster_settings(game_name: str) -> Namespace:
+    import LttPAdjuster
+    adjuster_settings = Namespace()
+    if game_name == LttPAdjuster.GAME_ALTTP:
+        return LttPAdjuster.get_argparser().parse_known_args(args=[])[0]
 
-def get_adjuster_settings(game_name: str) -> typing.Dict[str, typing.Any]:
-    adjuster_settings = persistent_load().get("adjuster", {}).get(game_name, {})
     return adjuster_settings
 
 
+def get_adjuster_settings_no_defaults(game_name: str) -> Namespace:
+    return persistent_load().get("adjuster", {}).get(game_name, Namespace())
+
+
+def get_adjuster_settings(game_name: str) -> Namespace:
+    adjuster_settings = get_adjuster_settings_no_defaults(game_name)
+    default_settings = get_default_adjuster_settings(game_name)
+
+    # Fill in any arguments from the argparser that we haven't seen before
+    return Namespace(**vars(adjuster_settings), **{k:v for k,v in vars(default_settings).items() if k not in vars(adjuster_settings)})
+
+
 @cache_argsless
 def get_unique_identifier():
     uuid = persistent_load().get("client", {}).get("uuid", None)
@@ -549,6 +433,7 @@ def init_logging(name: str, loglevel: typing.Union[str, int] = logging.INFO, wri
         root_logger.removeHandler(handler)
         handler.close()
     root_logger.setLevel(loglevel)
+    logging.getLogger("websockets").setLevel(loglevel)  # make sure level is applied for websockets
     if "a" not in write_mode:
         name += f"_{datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S')}"
     file_handler = logging.FileHandler(
@@ -672,7 +557,7 @@ def get_fuzzy_results(input_word: str, wordlist: typing.Sequence[str], limit: ty
     )
 
 
-def open_filename(title: str, filetypes: typing.Sequence[typing.Tuple[str, typing.Sequence[str]]]) \
+def open_filename(title: str, filetypes: typing.Sequence[typing.Tuple[str, typing.Sequence[str]]], suggest: str = "") \
         -> typing.Optional[str]:
     def run(*args: str):
         return subprocess.run(args, capture_output=True, text=True).stdout.split("\n", 1)[0] or None
@@ -683,11 +568,12 @@ def open_filename(title: str, filetypes: typing.Sequence[typing.Tuple[str, typin
         kdialog = which("kdialog")
         if kdialog:
             k_filters = '|'.join((f'{text} (*{" *".join(ext)})' for (text, ext) in filetypes))
-            return run(kdialog, f"--title={title}", "--getopenfilename", ".", k_filters)
+            return run(kdialog, f"--title={title}", "--getopenfilename", suggest or ".", k_filters)
         zenity = which("zenity")
         if zenity:
             z_filters = (f'--file-filter={text} ({", ".join(ext)}) | *{" *".join(ext)}' for (text, ext) in filetypes)
-            return run(zenity, f"--title={title}", "--file-selection", *z_filters)
+            selection = (f'--filename="{suggest}',) if suggest else ()
+            return run(zenity, f"--title={title}", "--file-selection", *z_filters, *selection)
 
     # fall back to tk
     try:
@@ -700,7 +586,38 @@ def open_filename(title: str, filetypes: typing.Sequence[typing.Tuple[str, typin
     else:
         root = tkinter.Tk()
         root.withdraw()
-        return tkinter.filedialog.askopenfilename(title=title, filetypes=((t[0], ' '.join(t[1])) for t in filetypes))
+        return tkinter.filedialog.askopenfilename(title=title, filetypes=((t[0], ' '.join(t[1])) for t in filetypes),
+                                                  initialfile=suggest or None)
+
+
+def open_directory(title: str, suggest: str = "") -> typing.Optional[str]:
+    def run(*args: str):
+        return subprocess.run(args, capture_output=True, text=True).stdout.split("\n", 1)[0] or None
+
+    if is_linux:
+        # prefer native dialog
+        from shutil import which
+        kdialog = None#which("kdialog")
+        if kdialog:
+            return run(kdialog, f"--title={title}", "--getexistingdirectory", suggest or ".")
+        zenity = None#which("zenity")
+        if zenity:
+            z_filters = ("--directory",)
+            selection = (f'--filename="{suggest}',) if suggest else ()
+            return run(zenity, f"--title={title}", "--file-selection", *z_filters, *selection)
+
+    # fall back to tk
+    try:
+        import tkinter
+        import tkinter.filedialog
+    except Exception as e:
+        logging.error('Could not load tkinter, which is likely not installed. '
+                      f'This attempt was made because open_filename was used for "{title}".')
+        raise e
+    else:
+        root = tkinter.Tk()
+        root.withdraw()
+        return tkinter.filedialog.askdirectory(title=title, mustexist=True, initialdir=suggest or None)
 
 
 def messagebox(title: str, text: str, error: bool = False) -> None:
@@ -765,10 +682,10 @@ def read_snes_rom(stream: BinaryIO, strip_header: bool = True) -> bytearray:
     return buffer
 
 
-_faf_tasks: "Set[asyncio.Task[None]]" = set()
+_faf_tasks: "Set[asyncio.Task[typing.Any]]" = set()
 
 
-def async_start(co: Coroutine[typing.Any, typing.Any, bool], name: Optional[str] = None) -> None:
+def async_start(co: Coroutine[None, None, typing.Any], name: Optional[str] = None) -> None:
     """
     Use this to start a task when you don't keep a reference to it or immediately await it,
     to prevent early garbage collection. "fire-and-forget"
@@ -781,6 +698,60 @@ def async_start(co: Coroutine[typing.Any, typing.Any, bool], name: Optional[str]
     # ```
     # This implementation follows the pattern given in that documentation.
 
-    task = asyncio.create_task(co, name=name)
+    task: asyncio.Task[typing.Any] = asyncio.create_task(co, name=name)
     _faf_tasks.add(task)
     task.add_done_callback(_faf_tasks.discard)
+
+
+def deprecate(message: str):
+    if __debug__:
+        raise Exception(message)
+    import warnings
+    warnings.warn(message)
+
+def _extend_freeze_support() -> None:
+    """Extend multiprocessing.freeze_support() to also work on Non-Windows for spawn."""
+    # upstream issue: https://github.com/python/cpython/issues/76327
+    # code based on https://github.com/pyinstaller/pyinstaller/blob/develop/PyInstaller/hooks/rthooks/pyi_rth_multiprocessing.py#L26
+    import multiprocessing
+    import multiprocessing.spawn
+
+    def _freeze_support() -> None:
+        """Minimal freeze_support. Only apply this if frozen."""
+        from subprocess import _args_from_interpreter_flags
+
+        # Prevent `spawn` from trying to read `__main__` in from the main script
+        multiprocessing.process.ORIGINAL_DIR = None
+
+        # Handle the first process that MP will create
+        if (
+            len(sys.argv) >= 2 and sys.argv[-2] == '-c' and sys.argv[-1].startswith((
+                'from multiprocessing.semaphore_tracker import main',  # Py<3.8
+                'from multiprocessing.resource_tracker import main',  # Py>=3.8
+                'from multiprocessing.forkserver import main'
+            )) and set(sys.argv[1:-2]) == set(_args_from_interpreter_flags())
+        ):
+            exec(sys.argv[-1])
+            sys.exit()
+
+        # Handle the second process that MP will create
+        if multiprocessing.spawn.is_forking(sys.argv):
+            kwargs = {}
+            for arg in sys.argv[2:]:
+                name, value = arg.split('=')
+                if value == 'None':
+                    kwargs[name] = None
+                else:
+                    kwargs[name] = int(value)
+            multiprocessing.spawn.spawn_main(**kwargs)
+            sys.exit()
+
+    if not is_windows and is_frozen():
+        multiprocessing.freeze_support = multiprocessing.spawn.freeze_support = _freeze_support
+
+
+def freeze_support() -> None:
+    """This behaves like multiprocessing.freeze_support but also works on Non-Windows."""
+    import multiprocessing
+    _extend_freeze_support()
+    multiprocessing.freeze_support()

WebHost.py

@@ -10,6 +10,7 @@ ModuleUpdate.update()
 
 # in case app gets imported by something like gunicorn
 import Utils
+import settings
 
 Utils.local_path.cached_path = os.path.dirname(__file__) or "."  # py3.8 is not abs. remove "." when dropping 3.8
 
@@ -21,6 +22,7 @@ from WebHostLib.autolauncher import autohost, autogen
 from WebHostLib.lttpsprites import update_sprites_lttp
 from WebHostLib.options import create as create_options_files
 
+settings.no_gui = True
 configpath = os.path.abspath("config.yaml")
 if not os.path.exists(configpath):  # fall back to config.yaml in home
     configpath = os.path.abspath(Utils.user_path('config.yaml'))
@@ -72,6 +74,7 @@ def create_ordered_tutorials_file() -> typing.List[typing.Dict[str, typing.Any]]
             with zipfile.ZipFile(zipfile_path) as zf:
                 for zfile in zf.infolist():
                     if not zfile.is_dir() and "/docs/" in zfile.filename:
+                        zfile.filename = os.path.basename(zfile.filename)
                         zf.extract(zfile, target_path)
         else:
             source_path = Utils.local_path(os.path.dirname(world.__file__), "docs")

WebHostLib/api/generate.py

@@ -2,7 +2,8 @@ import json
 import pickle
 from uuid import UUID
 
-from flask import request, session, url_for, Markup
+from flask import request, session, url_for
+from markupsafe import Markup
 from pony.orm import commit
 
 from WebHostLib import app

WebHostLib/check.py

@@ -1,7 +1,8 @@
 import zipfile
 from typing import *
 
-from flask import request, flash, redirect, url_for, render_template, Markup
+from flask import request, flash, redirect, url_for, render_template
+from markupsafe import Markup
 
 from WebHostLib import app
 
@@ -91,7 +92,7 @@ def roll_options(options: Dict[str, Union[dict, str]],
                         rolled_results[f"{filename}/{i + 1}"] = roll_settings(yaml_data,
                                                                               plando_options=plando_options)
             except Exception as e:
-                results[filename] = f"Failed to generate mystery in {filename}: {e}"
+                results[filename] = f"Failed to generate options in {filename}: {e}"
             else:
                 results[filename] = True
     return results, rolled_results

WebHostLib/customserver.py

@@ -18,7 +18,7 @@ from pony.orm import commit, db_session, select
 import Utils
 
 from MultiServer import Context, server, auto_shutdown, ServerCommandProcessor, ClientMessageProcessor, load_server_cert
-from Utils import get_public_ipv4, get_public_ipv6, restricted_loads, cache_argsless
+from Utils import restricted_loads, cache_argsless
 from .models import Command, GameDataPackage, Room, db
 
 
@@ -169,13 +169,11 @@ def run_server_process(room_id, ponyconfig: dict, static_server_data: dict,
         ctx.init_save()
         ssl_context = load_server_cert(cert_file, cert_key_file) if cert_file else None
         try:
-            ctx.server = websockets.serve(functools.partial(server, ctx=ctx), ctx.host, ctx.port, ping_timeout=None,
-                                          ping_interval=None, ssl=ssl_context)
+            ctx.server = websockets.serve(functools.partial(server, ctx=ctx), ctx.host, ctx.port, ssl=ssl_context)
 
             await ctx.server
         except Exception:  # likely port in use - in windows this is OSError, but I didn't check the others
-            ctx.server = websockets.serve(functools.partial(server, ctx=ctx), ctx.host, 0, ping_timeout=None,
-                                          ping_interval=None, ssl=ssl_context)
+            ctx.server = websockets.serve(functools.partial(server, ctx=ctx), ctx.host, 0, ssl=ssl_context)
 
             await ctx.server
         port = 0

WebHostLib/generate.py

@@ -106,7 +106,7 @@ def gen_game(gen_options: dict, meta: Optional[Dict[str, Any]] = None, owner=Non
         meta: Dict[str, Any] = {}
 
     meta.setdefault("server_options", {}).setdefault("hint_cost", 10)
-    race = meta["generator_options"].setdefault("race", False)
+    race = meta.setdefault("generator_options", {}).setdefault("race", False)
 
     def task():
         target = tempfile.TemporaryDirectory()
@@ -123,13 +123,14 @@ def gen_game(gen_options: dict, meta: Optional[Dict[str, Any]] = None, owner=Non
         erargs = parse_arguments(['--multi', str(playercount)])
         erargs.seed = seed
         erargs.name = {x: "" for x in range(1, playercount + 1)}  # only so it can be overwritten in mystery
-        erargs.spoiler = meta["generator_options"]["spoiler"]
+        erargs.spoiler = meta["generator_options"].get("spoiler", 0)
         erargs.race = race
         erargs.outputname = seedname
         erargs.outputpath = target.name
         erargs.teams = 1
         erargs.plando_options = PlandoOptions.from_set(meta.setdefault("plando_options",
                                                                        {"bosses", "items", "connections", "texts"}))
+        erargs.skip_prog_balancing = False
 
         name_counter = Counter()
         for player, (playerfile, settings) in enumerate(gen_options.items(), 1):

WebHostLib/models.py

@@ -21,7 +21,7 @@ class Slot(db.Entity):
 class Room(db.Entity):
     id = PrimaryKey(UUID, default=uuid4)
     last_activity = Required(datetime, default=lambda: datetime.utcnow(), index=True)
-    creation_time = Required(datetime, default=lambda: datetime.utcnow())
+    creation_time = Required(datetime, default=lambda: datetime.utcnow(), index=True)  # index used by landing page
     owner = Required(UUID, index=True)
     commands = Set('Command')
     seed = Required('Seed', index=True)
@@ -38,7 +38,7 @@ class Seed(db.Entity):
     rooms = Set(Room)
     multidata = Required(bytes, lazy=True)
     owner = Required(UUID, index=True)
-    creation_time = Required(datetime, default=lambda: datetime.utcnow())
+    creation_time = Required(datetime, default=lambda: datetime.utcnow(), index=True)  # index used by landing page
     slots = Set(Slot)
     spoiler = Optional(LongStr, lazy=True)
     meta = Required(LongStr, default=lambda: "{\"race\": false}")  # additional meta information/tags

WebHostLib/options.py

@@ -44,7 +44,7 @@ def create():
         # Generate JSON files for player-settings pages
         player_settings = {
             "baseOptions": {
-                "description": "Generated by https://archipelago.gg/",
+                "description": f"Generated by https://archipelago.gg/ for {game_name}",
                 "game": game_name,
                 "name": "Player",
             },

WebHostLib/requirements.txt

@@ -1,7 +1,9 @@
 flask>=2.2.3
-pony>=0.7.16
+pony>=0.7.16; python_version <= '3.10'
+pony @ https://github.com/Berserker66/pony/releases/download/v0.7.16/pony-0.7.16-py3-none-any.whl#0.7.16 ; python_version >= '3.11'
 waitress>=2.1.2
 Flask-Caching>=2.0.2
 Flask-Compress>=1.13
 Flask-Limiter>=3.3.0
-bokeh>=3.1.0
+bokeh>=3.1.1
+markupsafe>=2.1.3

WebHostLib/static/assets/player-settings.js

@@ -148,7 +148,7 @@ const buildOptionsTable = (settings, romOpts = false) => {
         randomButton.classList.add('randomize-button');
         randomButton.setAttribute('data-key', setting);
         randomButton.setAttribute('data-tooltip', 'Toggle randomization for this option!');
-        randomButton.addEventListener('click', (event) => toggleRandomize(event, [select]));
+        randomButton.addEventListener('click', (event) => toggleRandomize(event, select));
         if (currentSettings[gameName][setting] === 'random') {
           randomButton.classList.add('active');
           select.disabled = true;
@@ -185,7 +185,7 @@ const buildOptionsTable = (settings, romOpts = false) => {
         randomButton.classList.add('randomize-button');
         randomButton.setAttribute('data-key', setting);
         randomButton.setAttribute('data-tooltip', 'Toggle randomization for this option!');
-        randomButton.addEventListener('click', (event) => toggleRandomize(event, [range]));
+        randomButton.addEventListener('click', (event) => toggleRandomize(event, range));
         if (currentSettings[gameName][setting] === 'random') {
           randomButton.classList.add('active');
           range.disabled = true;
@@ -269,7 +269,7 @@ const buildOptionsTable = (settings, romOpts = false) => {
         randomButton.setAttribute('data-key', setting);
         randomButton.setAttribute('data-tooltip', 'Toggle randomization for this option!');
         randomButton.addEventListener('click', (event) => toggleRandomize(
-            event, [specialRange, specialRangeSelect])
+            event, specialRange, specialRangeSelect)
         );
         if (currentSettings[gameName][setting] === 'random') {
           randomButton.classList.add('active');
@@ -294,23 +294,25 @@ const buildOptionsTable = (settings, romOpts = false) => {
   return table;
 };
 
-const toggleRandomize = (event, inputElements) => {
+const toggleRandomize = (event, inputElement, optionalSelectElement = null) => {
   const active = event.target.classList.contains('active');
   const randomButton = event.target;
 
   if (active) {
     randomButton.classList.remove('active');
-    for (const element of inputElements) {
-      element.disabled = undefined;
-      updateGameSetting(element);
+    inputElement.disabled = undefined;
+    if (optionalSelectElement) {
+      optionalSelectElement.disabled = undefined;
     }
   } else {
     randomButton.classList.add('active');
-    for (const element of inputElements) {
-      element.disabled = true;
-      updateGameSetting(randomButton);
+    inputElement.disabled = true;
+    if (optionalSelectElement) {
+      optionalSelectElement.disabled = true;
     }
   }
+
+  updateGameSetting(randomButton);
 };
 
 const updateBaseSetting = (event) => {
@@ -364,6 +366,7 @@ const generateGame = (raceMode = false) => {
     weights: { player: settings },
     presetData: { player: settings },
     playerCount: 1,
+    spoiler: 3,
     race: raceMode ? '1' : '0',
   }).then((response) => {
     window.location.href = response.data.url;

WebHostLib/static/assets/trackerCommon.js

@@ -14,6 +14,17 @@ const adjustTableHeight = () => {
     }
 };
 
+/**
+ * Convert an integer number of seconds into a human readable HH:MM format
+ * @param {Number} seconds
+ * @returns {string}
+ */
+const secondsToHours = (seconds) => {
+    let hours   = Math.floor(seconds / 3600);
+    let minutes = Math.floor((seconds - (hours * 3600)) / 60).toString().padStart(2, '0');
+    return `${hours}:${minutes}`;
+};
+
 window.addEventListener('load', () => {
     const tables = $(".table").DataTable({
         paging: false,
@@ -27,7 +38,18 @@ window.addEventListener('load', () => {
         stateLoadCallback: function(settings) {
             return JSON.parse(localStorage.getItem(`DataTables_${settings.sInstance}_/tracker`));
         },
+        footerCallback: function(tfoot, data, start, end, display) {
+            if (tfoot) {
+                const activityData = this.api().column('lastActivity:name').data().toArray().filter(x => !isNaN(x));
+                Array.from(tfoot?.children).find(td => td.classList.contains('last-activity')).innerText =
+                  (activityData.length) ? secondsToHours(Math.min(...activityData)) : 'None';
+            }
+        },
         columnDefs: [
+            {
+                targets: 'last-activity',
+                name: 'lastActivity'
+            },
             {
                 targets: 'hours',
                 render: function (data, type, row) {
@@ -40,11 +62,7 @@ window.addEventListener('load', () => {
                     if (data === "None")
                         return data;
 
-                    let hours   = Math.floor(data / 3600);
-                    let minutes = Math.floor((data - (hours * 3600)) / 60);
-
-                    if (minutes < 10) {minutes = "0"+minutes;}
-                    return hours+':'+minutes;
+                    return secondsToHours(data);
                 }
             },
             {
@@ -114,11 +132,16 @@ window.addEventListener('load', () => {
             if (status === "success") {
                 target.find(".table").each(function (i, new_table) {
                     const new_trs = $(new_table).find("tbody>tr");
+                    const footer_tr = $(new_table).find("tfoot>tr");
                     const old_table = tables.eq(i);
                     const topscroll = $(old_table.settings()[0].nScrollBody).scrollTop();
                     const leftscroll = $(old_table.settings()[0].nScrollBody).scrollLeft();
                     old_table.clear();
-                    old_table.rows.add(new_trs).draw();
+                    if (footer_tr.length) {
+                        $(old_table.table).find("tfoot").html(footer_tr);
+                    }
+                    old_table.rows.add(new_trs);
+                    old_table.draw();
                     $(old_table.settings()[0].nScrollBody).scrollTop(topscroll);
                     $(old_table.settings()[0].nScrollBody).scrollLeft(leftscroll);
                 });

WebHostLib/static/assets/weighted-settings.js

@@ -1199,6 +1199,7 @@ const generateGame = (raceMode = false) => {
     weights: { player: JSON.stringify(settings) },
     presetData: { player: JSON.stringify(settings) },
     playerCount: 1,
+    spoiler: 3,
     race: raceMode ? '1' : '0',
   }).then((response) => {
     window.location.href = response.data.url;

WebHostLib/static/styles/player-settings.css

@@ -5,7 +5,8 @@ html{
 }
 
 #player-settings{
-    max-width: 1000px;
+    box-sizing: border-box;
+    max-width: 1024px;
     margin-left: auto;
     margin-right: auto;
     background-color: rgba(0, 0, 0, 0.15);
@@ -163,6 +164,11 @@ html{
     background-color: #ffef00; /* Same as .interactive in globalStyles.css */
 }
 
+#player-settings table .randomize-button[data-tooltip]::after {
+    left: unset;
+    right: 0;
+}
+
 #player-settings table label{
     display: block;
     min-width: 200px;
@@ -177,18 +183,31 @@ html{
     vertical-align: top;
 }
 
-@media all and (max-width: 1000px), all and (orientation: portrait){
+@media all and (max-width: 1024px) {
+    #player-settings {
+        border-radius: 0;
+    }
+
     #player-settings #game-options{
         justify-content: flex-start;
         flex-wrap: wrap;
     }
 
-    #player-settings .left, #player-settings .right{
-        flex-grow: unset;
+    #player-settings .left,
+    #player-settings .right {
+        margin: 0;
+    }
+
+    #game-options table {
+        margin-bottom: 0;
     }
 
     #game-options table label{
         display: block;
         min-width: 200px;
     }
+
+    #game-options table tr td {
+        width: 50%;
+    }
 }

WebHostLib/static/styles/tracker.css

@@ -55,16 +55,16 @@ table.dataTable thead{
     font-family: LexendDeca-Regular, sans-serif;
 }
 
-table.dataTable tbody{
+table.dataTable tbody, table.dataTable tfoot{
     background-color: #dce2bd;
     font-family: LexendDeca-Light, sans-serif;
 }
 
-table.dataTable tbody tr:hover{
+table.dataTable tbody tr:hover, table.dataTable tfoot tr:hover{
     background-color: #e2eabb;
 }
 
-table.dataTable tbody td{
+table.dataTable tbody td, table.dataTable tfoot td{
     padding: 4px 6px;
 }
 
@@ -97,10 +97,14 @@ table.dataTable thead th.lower-row{
     top: 46px;
 }
 
-table.dataTable tbody td{
+table.dataTable tbody td, table.dataTable tfoot td{
     border: 1px solid #bba967;
 }
 
+table.dataTable tfoot td{
+    font-weight: bold;
+}
+
 div.dataTables_scrollBody{
     background-color: inherit !important;
 }

WebHostLib/templates/hintTable.html

@@ -0,0 +1,28 @@
+{% for team, hints in hints.items() %}
+    <div class="table-wrapper">
+        <table id="hints-table" class="table non-unique-item-table" data-order='[[5, "asc"], [0, "asc"]]'>
+            <thead>
+            <tr>
+                <th>Finder</th>
+                <th>Receiver</th>
+                <th>Item</th>
+                <th>Location</th>
+                <th>Entrance</th>
+                <th>Found</th>
+            </tr>
+            </thead>
+            <tbody>
+            {%- for hint in hints -%}
+                <tr>
+                    <td>{{ long_player_names[team, hint.finding_player] }}</td>
+                    <td>{{ long_player_names[team, hint.receiving_player] }}</td>
+                    <td>{{ hint.item|item_name }}</td>
+                    <td>{{ hint.location|location_name }}</td>
+                    <td>{% if hint.entrance %}{{ hint.entrance }}{% else %}Vanilla{% endif %}</td>
+                    <td>{% if hint.found %}✔{% endif %}</td>
+                </tr>
+            {%- endfor -%}
+            </tbody>
+        </table>
+    </div>
+{% endfor %}

WebHostLib/templates/islandFooter.html

@@ -1,6 +1,6 @@
 {% block footer %}
     <footer id="island-footer">
-        <div id="copyright-notice">Copyright 2022 Archipelago</div>
+        <div id="copyright-notice">Copyright 2023 Archipelago</div>
         <div id="links">
             <a href="/sitemap">Site Map</a>
             -

WebHostLib/templates/lttpMultiTracker.html

@@ -128,20 +128,30 @@
                                     tracked_team=team, tracked_player=player)}}">{{ loop.index }}</a></td>
                                     <td>{{ player_names[(team, loop.index)]|e }}</td>
                                     {%- for area in ordered_areas -%}
-                                        {%- set checks_done = checks[area] -%}
-                                        {%- set checks_total = checks_in_area[player][area] -%}
-                                        {%- if checks_done == checks_total -%}
-                                            <td class="item-acquired center-column">
-                                                {{ checks_done }}/{{ checks_total }}</td>
-                                        {%- else -%}
-                                            <td class="center-column">{{ checks_done }}/{{ checks_total }}</td>
-                                        {%- endif -%}
-                                        {%- if area in key_locations -%}
-                                            <td class="center-column">{{ inventory[team][player][small_key_ids[area]] }}</td>
-                                        {%- endif -%}
-                                        {%- if area in big_key_locations -%}
-                                            <td class="center-column">{% if inventory[team][player][big_key_ids[area]] %}✔️{% endif %}</td>
-                                        {%- endif -%}
+                                        {% if player in checks_in_area and area in checks_in_area[player] %}
+                                            {%- set checks_done = checks[area] -%}
+                                            {%- set checks_total = checks_in_area[player][area] -%}
+                                            {%- if checks_done == checks_total -%}
+                                                <td class="item-acquired center-column">
+                                                    {{ checks_done }}/{{ checks_total }}</td>
+                                            {%- else -%}
+                                                <td class="center-column">{{ checks_done }}/{{ checks_total }}</td>
+                                            {%- endif -%}
+                                            {%- if area in key_locations -%}
+                                                <td class="center-column">{{ inventory[team][player][small_key_ids[area]] }}</td>
+                                            {%- endif -%}
+                                            {%- if area in big_key_locations -%}
+                                                <td class="center-column">{% if inventory[team][player][big_key_ids[area]] %}✔️{% endif %}</td>
+                                            {%- endif -%}
+                                        {% else %}
+                                            <td class="center-column"></td>
+                                            {%- if area in key_locations -%}
+                                                <td class="center-column"></td>
+                                            {%- endif -%}
+                                            {%- if area in big_key_locations -%}
+                                                <td class="center-column"></td>
+                                            {%- endif -%}
+                                        {% endif %}
                                     {%- endfor -%}
                                     <td class="center-column">{{ percent_total_checks_done[team][player] }}</td>
                                     {%- if activity_timers[(team, player)] -%}
@@ -155,34 +165,7 @@
                     </table>
                 </div>
             {% endfor %}
-            {% for team, hints in hints.items() %}
-                <div class="table-wrapper">
-                    <table id="hints-table" class="table non-unique-item-table" data-order='[[5, "asc"], [0, "asc"]]'>
-                        <thead>
-                        <tr>
-                            <th>Finder</th>
-                            <th>Receiver</th>
-                            <th>Item</th>
-                            <th>Location</th>
-                            <th>Entrance</th>
-                            <th>Found</th>
-                        </tr>
-                        </thead>
-                        <tbody>
-                        {%- for hint in hints -%}
-                            <tr>
-                                <td>{{ long_player_names[team, hint.finding_player] }}</td>
-                                <td>{{ long_player_names[team, hint.receiving_player] }}</td>
-                                <td>{{ hint.item|item_name }}</td>
-                                <td>{{ hint.location|location_name }}</td>
-                                <td>{% if hint.entrance %}{{ hint.entrance }}{% else %}Vanilla{% endif %}</td>
-                                <td>{% if hint.found %}✔{% endif %}</td>
-                            </tr>
-                        {%- endfor -%}
-                        </tbody>
-                    </table>
-                </div>
-            {% endfor %}
+            {% include "hintTable.html" with context %}
         </div>
     </div>
 {% endblock %}

WebHostLib/templates/multiFactorioTracker.html

@@ -27,12 +27,14 @@
 {% endblock %}
 {% block custom_table_row scoped  %}
 {% if games[player] == "Factorio" %}
-<td class="center-column">{% if inventory[team][player][131161] or inventory[team][player][131281] %}✔{% endif %}</td>
-<td class="center-column">{% if inventory[team][player][131172] or inventory[team][player][131281] > 1%}✔{% endif %}</td>
-<td class="center-column">{% if inventory[team][player][131195] or inventory[team][player][131281] > 2%}✔{% endif %}</td>
-<td class="center-column">{% if inventory[team][player][131240] or inventory[team][player][131281] > 3%}✔{% endif %}</td>
-<td class="center-column">{% if inventory[team][player][131240] or inventory[team][player][131281] > 4%}✔{% endif %}</td>
-<td class="center-column">{% if inventory[team][player][131220] or inventory[team][player][131281] > 5%}✔{% endif %}</td>
+{% set player_inventory = inventory[team][player] %}
+{% set prog_science = player_inventory[custom_items["progressive-science-pack"]] %}
+<td class="center-column">{% if player_inventory[custom_items["logistic-science-pack"]] or prog_science %}✔{% endif %}</td>
+<td class="center-column">{% if player_inventory[custom_items["military-science-pack"]] or prog_science > 1%}✔{% endif %}</td>
+<td class="center-column">{% if player_inventory[custom_items["chemical-science-pack"]] or prog_science > 2%}✔{% endif %}</td>
+<td class="center-column">{% if player_inventory[custom_items["production-science-pack"]] or prog_science > 3%}✔{% endif %}</td>
+<td class="center-column">{% if player_inventory[custom_items["utility-science-pack"]] or prog_science > 4%}✔{% endif %}</td>
+<td class="center-column">{% if player_inventory[custom_items["space-science-pack"]] or prog_science > 5%}✔{% endif %}</td>
 {% else %}
 <td class="center-column">❌</td>
 <td class="center-column">❌</td>

WebHostLib/templates/multiTracker.html

@@ -31,13 +31,13 @@
                                 <th>#</th>
                                 <th>Name</th>
                                 <th>Game</th>
+                                <th>Status</th>
                                 {% block custom_table_headers %}
                                 {# implement this block in game-specific multi trackers #}
                                 {% endblock %}
                                 <th class="center-column">Checks</th>
                                 <th class="center-column">&percnt;</th>
-                                <th class="center-column">Status</th>
-                                <th class="center-column hours">Last<br>Activity</th>
+                                <th class="center-column hours last-activity">Last<br>Activity</th>
                             </tr>
                         </thead>
                         <tbody>
@@ -47,13 +47,15 @@
                                     tracked_team=team, tracked_player=player)}}">{{ loop.index }}</a></td>
                                     <td>{{ player_names[(team, loop.index)]|e }}</td>
                                     <td>{{ games[player] }}</td>
+                                    <td>{{ {0: "Disconnected", 5: "Connected", 10: "Ready", 20: "Playing",
+                                            30: "Goal Completed"}.get(states[team, player], "Unknown State") }}</td>
                                     {% block custom_table_row scoped %}
                                     {# implement this block in game-specific multi trackers #}
                                     {% endblock %}
-                                    <td class="center-column">{{ checks["Total"] }}/{{ checks_in_area[player]["Total"] }}</td>
+                                    <td class="center-column" data-sort="{{ checks["Total"] }}">
+                                      {{ checks["Total"] }}/{{ locations[player] | length }}
+                                    </td>
                                     <td class="center-column">{{ percent_total_checks_done[team][player] }}</td>
-                                    <td>{{ {0: "Disconnected", 5: "Connected", 10: "Ready", 20: "Playing",
-                                            30: "Goal Completed"}.get(states[team, player], "Unknown State") }}</td>
                                     {%- if activity_timers[team, player] -%}
                                         <td class="center-column">{{ activity_timers[team, player].total_seconds() }}</td>
                                     {%- else -%}
@@ -62,37 +64,23 @@
                                 </tr>
                             {%- endfor -%}
                         </tbody>
+                        {% if not self.custom_table_headers() | trim %}
+                            <tfoot>
+                                <tr>
+                                    <td></td>
+                                    <td>Total</td>
+                                    <td>All Games</td>
+                                    <td>{{ completed_worlds }}/{{ players|length }} Complete</td>
+                                    <td class="center-column">{{ players.values()|sum(attribute='Total') }}/{{ total_locations[team] }}</td>
+                                    <td class="center-column">{{ (players.values()|sum(attribute='Total') / total_locations[team] * 100) | int }}</td>
+                                    <td class="center-column last-activity"></td>
+                                </tr>
+                            </tfoot>
+                        {% endif %}
                     </table>
                 </div>
             {% endfor %}
-            {% for team, hints in hints.items() %}
-                <div class="table-wrapper">
-                    <table id="hints-table" class="table non-unique-item-table" data-order='[[5, "asc"], [0, "asc"]]'>
-                        <thead>
-                        <tr>
-                            <th>Finder</th>
-                            <th>Receiver</th>
-                            <th>Item</th>
-                            <th>Location</th>
-                            <th>Entrance</th>
-                            <th>Found</th>
-                        </tr>
-                        </thead>
-                        <tbody>
-                        {%- for hint in hints -%}
-                            <tr>
-                                <td>{{ long_player_names[team, hint.finding_player] }}</td>
-                                <td>{{ long_player_names[team, hint.receiving_player] }}</td>
-                                <td>{{ hint.item|item_name }}</td>
-                                <td>{{ hint.location|location_name }}</td>
-                                <td>{% if hint.entrance %}{{ hint.entrance }}{% else %}Vanilla{% endif %}</td>
-                                <td>{% if hint.found %}✔{% endif %}</td>
-                            </tr>
-                        {%- endfor -%}
-                        </tbody>
-                    </table>
-                </div>
-            {% endfor %}
+            {% include "hintTable.html" with context %}
         </div>
     </div>
 {% endblock %}

WebHostLib/templates/siteMap.html

@@ -32,7 +32,6 @@
         <h2>Tutorials</h2>
         <ul>
             <li><a href="/tutorial/Archipelago/setup/en">Multiworld Setup Tutorial</a></li>
-            <li><a href="/tutorial/Archipelago/using_website/en">Website User Guide</a></li>
             <li><a href="/tutorial/Archipelago/mac/en">Setup Guide for Mac</a></li>
             <li><a href="/tutorial/Archipelago/commands/en">Server and Client Commands</a></li>
             <li><a href="/tutorial/Archipelago/advanced_settings/en">Advanced YAML Guide</a></li>

WebHostLib/tracker.py

@@ -1,7 +1,7 @@
 import collections
 import datetime
 import typing
-from typing import Counter, Optional, Dict, Any, Tuple
+from typing import Counter, Optional, Dict, Any, Tuple, List
 from uuid import UUID
 
 from flask import render_template
@@ -9,7 +9,7 @@ from jinja2 import pass_context, runtime
 from werkzeug.exceptions import abort
 
 from MultiServer import Context, get_saving_second
-from NetUtils import SlotType
+from NetUtils import SlotType, NetworkSlot
 from Utils import restricted_loads
 from worlds import lookup_any_item_id_to_name, lookup_any_location_id_to_name, network_data_package
 from worlds.alttp import Items
@@ -264,16 +264,17 @@ def get_static_room_data(room: Room):
     multidata = Context.decompress(room.seed.multidata)
     # in > 100 players this can take a bit of time and is the main reason for the cache
     locations: Dict[int, Dict[int, Tuple[int, int, int]]] = multidata['locations']
-    names: Dict[int, Dict[int, str]] = multidata["names"]
-    games = {}
+    names: List[List[str]] = multidata.get("names", [])
+    games = multidata.get("games", {})
     groups = {}
     custom_locations = {}
     custom_items = {}
     if "slot_info" in multidata:
-        games = {slot: slot_info.game for slot, slot_info in multidata["slot_info"].items()}
-        groups = {slot: slot_info.group_members for slot, slot_info in multidata["slot_info"].items()
+        slot_info_dict: Dict[int, NetworkSlot] = multidata["slot_info"]
+        games = {slot: slot_info.game for slot, slot_info in slot_info_dict.items()}
+        groups = {slot: slot_info.group_members for slot, slot_info in slot_info_dict.items()
                   if slot_info.type == SlotType.group}
-
+        names = [[slot_info.name for slot, slot_info in sorted(slot_info_dict.items())]]
         for game in games.values():
             if game not in multidata["datapackage"]:
                 continue
@@ -290,8 +291,7 @@ def get_static_room_data(room: Room):
                 {id_: name for name, id_ in game_data["location_name_to_id"].items()})
             custom_items.update(
                 {id_: name for name, id_ in game_data["item_name_to_id"].items()})
-    elif "games" in multidata:
-        games = multidata["games"]
+
     seed_checks_in_area = checks_in_area.copy()
 
     use_door_tracker = False
@@ -302,14 +302,17 @@ def get_static_room_data(room: Room):
             seed_checks_in_area[area] += len(checks)
         seed_checks_in_area["Total"] = 249
 
-    player_checks_in_area = {playernumber: {areaname: len(multidata["checks_in_area"][playernumber][areaname])
-    if areaname != "Total" else multidata["checks_in_area"][playernumber]["Total"]
-                                            for areaname in ordered_areas}
-                             for playernumber in range(1, len(names[0]) + 1)
-                             if playernumber not in groups}
+    player_checks_in_area = {
+        playernumber: {
+            areaname: len(multidata["checks_in_area"][playernumber][areaname]) if areaname != "Total" else
+            multidata["checks_in_area"][playernumber]["Total"]
+            for areaname in ordered_areas
+        }
+        for playernumber in multidata["checks_in_area"]
+    }
+
     player_location_to_area = {playernumber: get_location_table(multidata["checks_in_area"][playernumber])
-                               for playernumber in range(1, len(names[0]) + 1)
-                               if playernumber not in groups}
+                               for playernumber in multidata["checks_in_area"]}
     saving_second = get_saving_second(multidata["seed_name"])
     result = locations, names, use_door_tracker, player_checks_in_area, player_location_to_area, \
              multidata["precollected_items"], games, multidata["slot_data"], groups, saving_second, \
@@ -343,7 +346,7 @@ def _get_player_tracker(tracker: UUID, tracked_team: int, tracked_player: int, w
         precollected_items, games, slot_data, groups, saving_second, custom_locations, custom_items = \
         get_static_room_data(room)
     player_name = names[tracked_team][tracked_player - 1]
-    location_to_area = player_location_to_area[tracked_player]
+    location_to_area = player_location_to_area.get(tracked_player, {})
     inventory = collections.Counter()
     checks_done = {loc_name: 0 for loc_name in default_locations}
 
@@ -375,15 +378,18 @@ def _get_player_tracker(tracker: UUID, tracked_team: int, tracked_player: int, w
                     if recipient in slots_aimed_at_player:  # a check done for the tracked player
                         attribute_item_solo(inventory, item)
                     if ms_player == tracked_player:  # a check done by the tracked player
-                        checks_done[location_to_area[location]] += 1
+                        area_name = location_to_area.get(location, None)
+                        if area_name:
+                            checks_done[area_name] += 1
                         checks_done["Total"] += 1
     specific_tracker = game_specific_trackers.get(games[tracked_player], None)
     if specific_tracker and not want_generic:
         tracker =  specific_tracker(multisave, room, locations, inventory, tracked_team, tracked_player, player_name,
-                                seed_checks_in_area, checks_done, slot_data[tracked_player], saving_second)
+                                    seed_checks_in_area, checks_done, slot_data[tracked_player], saving_second)
     else:
-        tracker =  __renderGenericTracker(multisave, room, locations, inventory, tracked_team, tracked_player, player_name,
-                                      seed_checks_in_area, checks_done, saving_second, custom_locations, custom_items)
+        tracker =  __renderGenericTracker(multisave, room, locations, inventory, tracked_team, tracked_player,
+                                          player_name, seed_checks_in_area, checks_done, saving_second,
+                                          custom_locations, custom_items)
 
     return (saving_second - datetime.datetime.now().second) % 60 or 60, tracker
 
@@ -1360,6 +1366,10 @@ def _get_multiworld_tracker_data(tracker: UUID) -> typing.Optional[typing.Dict[s
                                 for playernumber in range(1, len(team) + 1) if playernumber not in groups}
                     for teamnumber, team in enumerate(names)}
 
+    total_locations = {teamnumber: sum(len(locations[playernumber])
+                                for playernumber in range(1, len(team) + 1) if playernumber not in groups)
+                    for teamnumber, team in enumerate(names)}
+
     hints = {team: set() for team in range(len(names))}
     if room.multisave:
         multisave = restricted_loads(room.multisave)
@@ -1373,10 +1383,10 @@ def _get_multiworld_tracker_data(tracker: UUID) -> typing.Optional[typing.Dict[s
         if player in groups:
             continue
         player_locations = locations[player]
-        checks_done[team][player]["Total"] = sum(1 for loc in locations_checked if loc in player_locations)
+        checks_done[team][player]["Total"] = len(locations_checked)
         percent_total_checks_done[team][player] = int(checks_done[team][player]["Total"] /
-                                                      checks_in_area[player]["Total"] * 100) \
-            if checks_in_area[player]["Total"] else 100
+                                                      len(player_locations) * 100) \
+            if player_locations else 100
 
     activity_timers = {}
     now = datetime.datetime.utcnow()
@@ -1384,11 +1394,14 @@ def _get_multiworld_tracker_data(tracker: UUID) -> typing.Optional[typing.Dict[s
         activity_timers[team, player] = now - datetime.datetime.utcfromtimestamp(timestamp)
 
     player_names = {}
+    completed_worlds = 0
     states: typing.Dict[typing.Tuple[int, int], int] = {}
     for team, names in enumerate(names):
         for player, name in enumerate(names, 1):
             player_names[team, player] = name
             states[team, player] = multisave.get("client_game_state", {}).get((team, player), 0)
+            if states[team, player] == 30:  # Goal Completed
+                completed_worlds += 1
     long_player_names = player_names.copy()
     for (team, player), alias in multisave.get("name_aliases", {}).items():
         player_names[team, player] = alias
@@ -1398,12 +1411,16 @@ def _get_multiworld_tracker_data(tracker: UUID) -> typing.Optional[typing.Dict[s
     for (team, player), data in multisave.get("video", []):
         video[team, player] = data
 
-    return dict(player_names=player_names, room=room, checks_done=checks_done,
-                percent_total_checks_done=percent_total_checks_done, checks_in_area=checks_in_area,
-                activity_timers=activity_timers, video=video, hints=hints,
-                long_player_names=long_player_names,
-                multisave=multisave, precollected_items=precollected_items, groups=groups,
-                locations=locations, games=games, states=states)
+    return dict(
+        player_names=player_names, room=room, checks_done=checks_done,
+        percent_total_checks_done=percent_total_checks_done, checks_in_area=checks_in_area,
+        activity_timers=activity_timers, video=video, hints=hints,
+        long_player_names=long_player_names,
+        multisave=multisave, precollected_items=precollected_items, groups=groups,
+        locations=locations, total_locations=total_locations, games=games, states=states,
+        completed_worlds=completed_worlds,
+        custom_locations=custom_locations, custom_items=custom_items,
+    )
 
 
 def _get_inventory_data(data: typing.Dict[str, typing.Any]) -> typing.Dict[int, typing.Dict[int, int]]:
@@ -1509,8 +1526,8 @@ def get_LttP_multiworld_tracker(tracker: UUID):
                 checks_done[team][player][player_location_to_area[player][location]] += 1
                 checks_done[team][player]["Total"] += 1
         percent_total_checks_done[team][player] = int(
-            checks_done[team][player]["Total"] / seed_checks_in_area[player]["Total"] * 100) if \
-        seed_checks_in_area[player]["Total"] else 100
+            checks_done[team][player]["Total"] / len(player_locations) * 100) if \
+        player_locations else 100
 
     for (team, player), game_state in multisave.get("client_game_state", {}).items():
         if player in groups:

WebHostLib/upload.py

@@ -7,12 +7,13 @@ import zipfile
 import zlib
 
 from io import BytesIO
-from flask import request, flash, redirect, url_for, session, render_template, Markup
+from flask import request, flash, redirect, url_for, session, render_template
+from markupsafe import Markup
 from pony.orm import commit, flush, select, rollback
 from pony.orm.core import TransactionIntegrityError
 
 import MultiServer
-from NetUtils import NetworkSlot, SlotType
+from NetUtils import SlotType
 from Utils import VersionException, __version__
 from worlds.Files import AutoPatchRegister
 from . import app

Zelda1Client.py

@@ -46,7 +46,7 @@ class ZeldaCommandProcessor(ClientCommandProcessor):
             logger.info(f"NES Status: {self.ctx.nes_status}")
 
     def _cmd_toggle_msgs(self):
-        """Toggle displaying messages in bizhawk"""
+        """Toggle displaying messages in EmuHawk"""
         global DISPLAY_MSGS
         DISPLAY_MSGS = not DISPLAY_MSGS
         logger.info(f"Messages are now {'enabled' if DISPLAY_MSGS else 'disabled'}")

ZillionClient.py

@@ -423,9 +423,9 @@ async def zillion_sync_task(ctx: ZillionContext) -> None:
                                         async_start(ctx.send_connect())
                                         log_no_spam("logging in to server...")
                                         await asyncio.wait((
-                                            ctx.got_slot_data.wait(),
-                                            ctx.exit_event.wait(),
-                                            asyncio.sleep(6)
+                                            asyncio.create_task(ctx.got_slot_data.wait()),
+                                            asyncio.create_task(ctx.exit_event.wait()),
+                                            asyncio.create_task(asyncio.sleep(6))
                                         ), return_when=asyncio.FIRST_COMPLETED)  # to not spam connect packets
                             else:  # not correct seed name
                                 log_no_spam("incorrect seed - did you mix up roms?")
@@ -447,9 +447,9 @@ async def zillion_sync_task(ctx: ZillionContext) -> None:
                     ctx.known_name = name
                     async_start(ctx.connect())
                     await asyncio.wait((
-                        ctx.got_room_info.wait(),
-                        ctx.exit_event.wait(),
-                        asyncio.sleep(6)
+                        asyncio.create_task(ctx.got_room_info.wait()),
+                        asyncio.create_task(ctx.exit_event.wait()),
+                        asyncio.create_task(asyncio.sleep(6))
                     ), return_when=asyncio.FIRST_COMPLETED)
             else:  # no name found in game
                 if not help_message_shown:

_speedups.pyx

@@ -0,0 +1,347 @@
+#cython: language_level=3
+#distutils: language = c++
+
+"""
+Provides faster implementation of some core parts.
+This is deliberately .pyx because using a non-compiled "pure python" may be slower.
+"""
+
+# pip install cython cymem
+import cython
+import warnings
+from cpython cimport PyObject
+from typing import Any, Dict, Iterable, Iterator, Generator, Sequence, Tuple, TypeVar, Union, Set, List, TYPE_CHECKING
+from cymem.cymem cimport Pool
+from libc.stdint cimport int64_t, uint32_t
+from libcpp.set cimport set as std_set
+from collections import defaultdict
+
+cdef extern from *:
+    """
+    // avoid warning from cython-generated code with MSVC + pyximport
+    #ifdef _MSC_VER
+    #pragma warning( disable: 4551 )
+    #endif
+    """
+
+ctypedef uint32_t ap_player_t  # on AMD64 this is faster (and smaller) than 64bit ints
+ctypedef uint32_t ap_flags_t
+ctypedef int64_t ap_id_t
+
+cdef ap_player_t MAX_PLAYER_ID = 1000000  # limit the size of indexing array
+cdef size_t INVALID_SIZE = <size_t>(-1)  # this is all 0xff... adding 1 results in 0, but it's not negative
+
+
+cdef struct LocationEntry:
+    # layout is so that
+    # 64bit player: location+sender and item+receiver 128bit comparisons, if supported
+    # 32bit player: aligned to 32/64bit with no unused space
+    ap_id_t location
+    ap_player_t sender
+    ap_player_t receiver
+    ap_id_t item
+    ap_flags_t flags
+
+
+cdef struct IndexEntry:
+    size_t start
+    size_t count
+
+
+cdef class LocationStore:
+    """Compact store for locations and their items in a MultiServer"""
+    # The original implementation uses Dict[int, Dict[int, Tuple(int, int, int]]
+    # with sender, location, (item, receiver, flags).
+    # This implementation is a flat list of (sender, location, item, receiver, flags) using native integers
+    # as well as some mapping arrays used to speed up stuff, saving a lot of memory while speeding up hints.
+    # Using std::map might be worth investigating, but memory overhead would be ~100% compared to arrays.
+
+    cdef Pool _mem
+    cdef object _len
+    cdef LocationEntry* entries  # 3.2MB/100k items
+    cdef size_t entry_count
+    cdef IndexEntry* sender_index  # 16KB/1000 players
+    cdef size_t sender_index_size
+    cdef list _keys  # ~36KB/1000 players, speed up iter (28 per int + 8 per list entry)
+    cdef list _items  # ~64KB/1000 players, speed up items (56 per tuple + 8 per list entry)
+    cdef list _proxies  # ~92KB/1000 players, speed up self[player] (56 per struct + 28 per len + 8 per list entry)
+    cdef PyObject** _raw_proxies  # 8K/1000 players, faster access to _proxies, but does not keep a ref
+
+    def get_size(self):
+        from sys import getsizeof
+        size = getsizeof(self) + getsizeof(self._mem) + getsizeof(self._len) \
+                + sizeof(LocationEntry) * self.entry_count + sizeof(IndexEntry) * self.sender_index_size
+        size += getsizeof(self._keys) + getsizeof(self._items) + getsizeof(self._proxies)
+        size += sum(sizeof(key) for key in self._keys)
+        size += sum(sizeof(item) for item in self._items)
+        size += sum(sizeof(proxy) for proxy in self._proxies)
+        size += sizeof(self._raw_proxies[0]) * self.sender_index_size
+        return size
+
+    def __cinit__(self, locations_dict: Dict[int, Dict[int, Sequence[int]]]) -> None:
+        self._mem = None
+        self._keys = None
+        self._items = None
+        self._proxies = None
+        self._len = 0
+        self.entries = NULL
+        self.entry_count = 0
+        self.sender_index = NULL
+        self.sender_index_size = 0
+        self._raw_proxies = NULL
+
+    def __init__(self, locations_dict: Dict[int, Dict[int, Sequence[int]]]) -> None:
+        self._mem = Pool()
+        cdef object key
+        self._keys = []
+        self._items = []
+        self._proxies = []
+
+        # iterate over everything to get all maxima and validate everything
+        cdef size_t max_sender = INVALID_SIZE  # keep track of highest used player id for indexing
+        cdef size_t sender_count = 0
+        cdef size_t count = 0
+        for sender, locations in locations_dict.items():
+            # we don't require the dict to be sorted here
+            if not isinstance(sender, int) or sender < 1 or sender > MAX_PLAYER_ID:
+                raise ValueError(f"Invalid player id {sender} for location")
+            if max_sender == INVALID_SIZE:
+                max_sender = sender
+            else:
+                max_sender = max(max_sender, sender)
+            for location, data in locations.items():
+                receiver = data[1]
+                if receiver < 1 or receiver > MAX_PLAYER_ID:
+                    raise ValueError(f"Invalid player id {receiver} for item")
+                count += 1
+            sender_count += 1
+
+        if not sender_count:
+            raise ValueError(f"Rejecting game with 0 players")
+
+        if sender_count != max_sender:
+            # we assume player 0 will never have locations
+            raise ValueError("Player IDs not continuous")
+
+        if not count:
+            warnings.warn("Game has no locations")
+
+        # allocate the arrays and invalidate index (0xff...)
+        self.entries = <LocationEntry*>self._mem.alloc(count, sizeof(LocationEntry))
+        self.sender_index = <IndexEntry*>self._mem.alloc(max_sender + 1, sizeof(IndexEntry))
+        self._raw_proxies = <PyObject**>self._mem.alloc(max_sender + 1, sizeof(PyObject*))
+
+        # build entries and index
+        cdef size_t i = 0
+        for sender, locations in sorted(locations_dict.items()):
+            self.sender_index[sender].start = i
+            self.sender_index[sender].count = 0
+            # Sorting locations here makes it possible to write a faster lookup without an additional index.
+            for location, data in sorted(locations.items()):
+                self.entries[i].sender = sender
+                self.entries[i].location = location
+                self.entries[i].item = data[0]
+                self.entries[i].receiver = data[1]
+                if len(data) > 2:
+                    self.entries[i].flags = data[2]  # initialized to 0 during alloc
+                # Ignoring extra data. warn?
+                self.sender_index[sender].count += 1
+                i += 1
+
+        # build pyobject caches
+        self._proxies.append(None)  # player 0
+        assert self.sender_index[0].count == 0
+        for i in range(1, max_sender + 1):
+            assert self.sender_index[i].count == 0 or (
+                    self.sender_index[i].start < count and
+                    self.sender_index[i].start + self.sender_index[i].count <= count)
+            key = i  # allocate python integer
+            proxy = PlayerLocationProxy(self, i)
+            self._keys.append(key)
+            self._items.append((key, proxy))
+            self._proxies.append(proxy)
+            self._raw_proxies[i] = <PyObject*>proxy
+
+        self.sender_index_size = max_sender + 1
+        self.entry_count = count
+        self._len = sender_count
+
+    # fake dict access
+    def __len__(self) -> int:
+        return self._len
+
+    def __iter__(self) -> Iterator[int]:
+        return self._keys.__iter__()
+
+    def __getitem__(self, key: int) -> Any:
+        # figure out if player actually exists in the multidata and return a proxy
+        cdef size_t i = key  # NOTE: this may raise TypeError
+        if i < 1 or i >= self.sender_index_size:
+            raise KeyError(key)
+        return <object>self._raw_proxies[key]
+
+    T = TypeVar('T')
+
+    def get(self, key: int, default: T) -> Union[PlayerLocationProxy, T]:
+        # calling into self.__getitem__ here is slow, but this is not used in MultiServer
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+    def items(self) -> Iterable[Tuple[int, PlayerLocationProxy]]:
+        return self._items
+
+    # specialized accessors
+    def find_item(self, slots: Set[int], seeked_item_id: int) -> Generator[Tuple[int, int, int, int, int], None, None]:
+        cdef ap_id_t item = seeked_item_id
+        cdef ap_player_t receiver
+        cdef std_set[ap_player_t] receivers
+        cdef size_t slot_count = len(slots)
+        if slot_count == 1:
+            # specialized implementation for single slot
+            receiver = list(slots)[0]
+            with nogil:
+                for entry in self.entries[:self.entry_count]:
+                    if entry.item == item and entry.receiver == receiver:
+                        with gil:
+                            yield entry.sender, entry.location, entry.item, entry.receiver, entry.flags
+        elif slot_count:
+            # generic implementation with lookup in set
+            for receiver in slots:
+                receivers.insert(receiver)
+            with nogil:
+                for entry in self.entries[:self.entry_count]:
+                    if entry.item == item and receivers.count(entry.receiver):
+                        with gil:
+                            yield entry.sender, entry.location, entry.item, entry.receiver, entry.flags
+
+    def get_for_player(self, slot: int) -> Dict[int, Set[int]]:
+        cdef ap_player_t receiver = slot
+        all_locations: Dict[int, Set[int]] = {}
+        with nogil:
+            for entry in self.entries[:self.entry_count]:
+                if entry.receiver == receiver:
+                    with gil:
+                        sender: int = entry.sender
+                        if sender not in all_locations:
+                            all_locations[sender] = set()
+                        all_locations[sender].add(entry.location)
+        return all_locations
+
+    if TYPE_CHECKING:
+        State = Dict[Tuple[int, int], Set[int]]
+    else:
+        State = Union[Tuple[int, int], Set[int], defaultdict]
+
+    def get_checked(self, state: State, team: int, slot: int) -> List[int]:
+        # This used to validate checks actually exist. A remnant from the past.
+        # If the order of locations becomes relevant at some point, we could not do sorted(set), so leaving it.
+        cdef set checked = state[team, slot]
+
+        if not len(checked):
+            # Skips loop if none have been checked.
+            # This optimizes the case where everyone connects to a fresh game at the same time.
+            return []
+
+        # Unless the set is close to empty, it's cheaper to use the python set directly, so we do that.
+        cdef LocationEntry* entry
+        cdef ap_player_t sender = slot
+        cdef size_t start = self.sender_index[sender].start
+        cdef size_t count = self.sender_index[sender].count
+        return [entry.location for
+                entry in self.entries[start:start+count] if
+                entry.location in checked]
+
+    def get_missing(self, state: State, team: int, slot: int) -> List[int]:
+        cdef LocationEntry* entry
+        cdef ap_player_t sender = slot
+        cdef size_t start = self.sender_index[sender].start
+        cdef size_t count = self.sender_index[sender].count
+        cdef set checked = state[team, slot]
+        if not len(checked):
+            # Skip `in` if none have been checked.
+            # This optimizes the case where everyone connects to a fresh game at the same time.
+            return [entry.location for
+                    entry in self.entries[start:start + count]]
+        else:
+            # Unless the set is close to empty, it's cheaper to use the python set directly, so we do that.
+            return [entry.location for
+                    entry in self.entries[start:start + count] if
+                    entry.location not in checked]
+
+    def get_remaining(self, state: State, team: int, slot: int) -> List[int]:
+        cdef LocationEntry* entry
+        cdef ap_player_t sender = slot
+        cdef size_t start = self.sender_index[sender].start
+        cdef size_t count = self.sender_index[sender].count
+        cdef set checked = state[team, slot]
+        return sorted([entry.item for
+                       entry in self.entries[start:start+count] if
+                       entry.location not in checked])
+
+
+@cython.internal  # unsafe. disable direct import
+cdef class PlayerLocationProxy:
+    cdef LocationStore _store
+    cdef size_t _player
+    cdef object _len
+
+    def __init__(self, store: LocationStore, player: int) -> None:
+        self._store = store
+        self._player = player
+        self._len = self._store.sender_index[self._player].count
+
+    def __len__(self) -> int:
+        return self._store.sender_index[self._player].count
+
+    def __iter__(self) -> Generator[int, None, None]:
+        cdef LocationEntry* entry
+        cdef size_t i
+        cdef size_t off = self._store.sender_index[self._player].start
+        for i in range(self._store.sender_index[self._player].count):
+            entry = self._store.entries + off + i
+            yield entry.location
+
+    cdef LocationEntry* _get(self, ap_id_t loc):
+        # This requires locations to be sorted.
+        # This is always going to be slower than a pure python dict, because constructing the result tuple takes as long
+        # as the search in a python dict, which stores a pointer to an existing tuple.
+        cdef LocationEntry* entry = NULL
+        # binary search
+        cdef size_t l = self._store.sender_index[self._player].start
+        cdef size_t r = l + self._store.sender_index[self._player].count
+        cdef size_t m
+        while l < r:
+            m = (l + r) // 2
+            entry = self._store.entries + m
+            if entry.location < loc:
+                l = m + 1
+            else:
+                r = m
+        if entry:  # count != 0
+            entry = self._store.entries + l
+            if entry.location == loc:
+                return entry
+        return NULL
+
+    def __getitem__(self, key: int) -> Tuple[int, int, int]:
+        cdef LocationEntry* entry = self._get(key)
+        if entry:
+            return entry.item, entry.receiver, entry.flags
+        raise KeyError(f"No location {key} for player {self._player}")
+
+    T = TypeVar('T')
+
+    def get(self, key: int, default: T) -> Union[Tuple[int, int, int], T]:
+        cdef LocationEntry* entry = self._get(key)
+        if entry:
+            return entry.item, entry.receiver, entry.flags
+        return default
+
+    def items(self) -> Generator[Tuple[int, Tuple[int, int, int]], None, None]:
+        cdef LocationEntry* entry
+        start = self._store.sender_index[self._player].start
+        count = self._store.sender_index[self._player].count
+        for entry in self._store.entries[start:start+count]:
+            yield entry.location, (entry.item, entry.receiver, entry.flags)

_speedups.pyxbld

@@ -0,0 +1,8 @@
+# This file is required to get pyximport to work with C++.
+# Switching from std::set to a pure C implementation is still on the table to simplify everything.
+
+def make_ext(modname, pyxfilename):
+    from distutils.extension import Extension
+    return Extension(name=modname,
+                     sources=[pyxfilename],
+                     language='c++')

data/lua/common.lua

@@ -27,8 +27,8 @@ end
 
 local is23Or24Or25 = (bizhawk_version=="2.3.1") or (bizhawk_major == 2 and bizhawk_minor >= 3 and bizhawk_minor <= 5)
 local isGreaterOrEqualTo26 = bizhawk_major > 2 or (bizhawk_major == 2 and bizhawk_minor >= 6)
-local isUntestedBizhawk = bizhawk_major > 2 or (bizhawk_major == 2 and bizhawk_minor > 9)
-local untestedBizhawkMessage = "Warning: this version of bizhawk is newer than we know about. If it doesn't work, consider downgrading to 2.9"
+local isUntestedBizHawk = bizhawk_major > 2 or (bizhawk_major == 2 and bizhawk_minor > 9)
+local untestedBizHawkMessage = "Warning: this version of BizHawk is newer than we know about. If it doesn't work, consider downgrading to 2.9"
 
 u8 = memory.read_u8
 wU8 = memory.write_u8
@@ -94,12 +94,12 @@ function drawMessages()
   end
 end
 
-function checkBizhawkVersion()
+function checkBizHawkVersion()
   if not is23Or24Or25 and not isGreaterOrEqualTo26 then
-    print("Must use a version of bizhawk 2.3.1 or higher")
+    print("Must use a version of BizHawk 2.3.1 or higher")
     return false
-  elseif isUntestedBizhawk then
-    print(untestedBizhawkMessage)
+  elseif isUntestedBizHawk then
+    print(untestedBizHawkMessage)
   end
   return true
 end

data/lua/connector_adventure.lua

@@ -457,7 +457,7 @@ end
 
 function main()
     memory.usememorydomain("System Bus")
-    if not checkBizhawkVersion() then
+    if not checkBizHawkVersion() then
         return
     end
     local playerSlot = memory.read_u8(PlayerSlotAddress)

data/lua/connector_ff1.lua

@@ -414,7 +414,7 @@ function receive()
 end
 
 function main()
-    if not checkBizhawkVersion() then
+    if not checkBizHawkVersion() then
         return
     end
     server, error = socket.bind('localhost', 52980)

data/lua/connector_ladx_bizhawk.lua

@@ -3,8 +3,8 @@
 -- SPDX-License-Identifier: MIT
 
 -- This script attempts to implement the basic functionality needed in order for
--- the LADXR Archipelago client to be able to talk to BizHawk instead of RetroArch
--- by reproducing the RetroArch API with BizHawk's Lua interface.
+-- the LADXR Archipelago client to be able to talk to EmuHawk instead of RetroArch
+-- by reproducing the RetroArch API with EmuHawk's Lua interface.
 --
 -- RetroArch UDP API: https://github.com/libretro/RetroArch/blob/master/command.c
 --
@@ -16,19 +16,19 @@
 -- commands are supported right now.
 --
 -- USAGE:
---  Load this script in BizHawk ("Tools" -> "Lua Console" -> "Script" -> "Open Script")
+--  Load this script in EmuHawk ("Tools" -> "Lua Console" -> "Script" -> "Open Script", or drag+drop)
 --
 -- All inconsistencies (like missing newlines for some commands) of the RetroArch
 -- UDP API (network_cmd_enable) are reproduced as-is in order for clients written to work with
 -- RetroArch's current API to "just work"(tm).
 --
 -- This script has only been tested on GB(C). If you have made sure it works for N64 or other
--- cores supported by BizHawk, please let me know. Note that GET_STATUS, at the very least, will
+-- cores supported by EmuHawk, please let me know. Note that GET_STATUS, at the very least, will
 -- have to be adjusted.
 --
 --
 -- NOTE:
---  BizHawk's Lua API is very trigger-happy on throwing exceptions.
+--  EmuHawk's Lua API is very trigger-happy on throwing exceptions.
 --  Emulation will continue fine, but the RetroArch API layer will stop working. This
 --  is indicated only by an exception visible in the Lua console, which most players
 --  will probably not have in the foreground.
@@ -43,13 +43,13 @@
 
 
 local socket = require("socket")
-local udp = socket.socket.udp()
+udp = socket.socket.udp()
 require('common')
 
 udp:setsockname('127.0.0.1', 55355)
 udp:settimeout(0)
 
-while true do
+function on_vblank()
     -- Attempt to lessen the CPU load by only polling the UDP socket every x frames.
     -- x = 10 is entirely arbitrary, very little thought went into it.
     -- We could try to make use of client.get_approx_framerate() here, but the values returned
@@ -82,7 +82,7 @@ while true do
                 -- "GET_STATUS PLAYING game_boy,AP_62468482466172374046_P1_Lonk,crc32=3ecb7b6f"
                 -- CRC32 isn't readily available through the Lua API. We could calculate
                 -- it ourselves, but since LADXR doesn't make use of this field it is
-                -- simply replaced by the hash that BizHawk _does_ make available.
+                -- simply replaced by the hash that EmuHawk _does_ make available.
 
                 udp:sendto(
                     "GET_STATUS " .. status .. " game_boy," ..
@@ -112,6 +112,7 @@ while true do
             for _, v in ipairs(mem) do
                 hex_string = hex_string .. string.format("%02X ", v)
             end
+
             hex_string = hex_string:sub(1, -2) -- Hang head in shame, remove last " "
             local reply = string.format("%s %02x %s\n", command, address, hex_string)
             udp:sendto(reply, msg_or_ip, port_or_nil)
@@ -135,6 +136,10 @@ while true do
             udp:sendto(reply, msg_or_ip, port_or_nil)
         end
     end
-
-    emu.frameadvance()
+end
+
+event.onmemoryexecute(on_vblank, 0x40, "ap_connector_vblank")
+
+while true do
+    emu.yield()
 end

data/lua/connector_mmbn3.lua

@@ -0,0 +1,723 @@
+local socket = require("socket")
+local json = require('json')
+local math = require('math')
+require('common')
+
+local last_modified_date = '2023-31-05' -- Should be the last modified date
+local script_version = 4
+
+local bizhawk_version = client.getversion()
+local bizhawk_major, bizhawk_minor, bizhawk_patch = bizhawk_version:match("(%d+)%.(%d+)%.?(%d*)")
+bizhawk_major = tonumber(bizhawk_major)
+bizhawk_minor = tonumber(bizhawk_minor)
+if bizhawk_patch == "" then
+  bizhawk_patch = 0
+else
+  bizhawk_patch = tonumber(bizhawk_patch)
+end
+
+local STATE_OK = "Ok"
+local STATE_TENTATIVELY_CONNECTED = "Tentatively Connected"
+local STATE_INITIAL_CONNECTION_MADE = "Initial Connection Made"
+local STATE_UNINITIALIZED = "Uninitialized"
+
+local prevstate = ""
+local curstate =  STATE_UNINITIALIZED
+local mmbn3Socket = nil
+local frame = 0
+
+-- States
+local ITEMSTATE_NONINITIALIZED = "Game Not Yet Started" -- Game has not yet started
+local ITEMSTATE_NONITEM = "Non-Itemable State" -- Do not send item now. RAM is not capable of holding
+local ITEMSTATE_IDLE = "Item State Ready" -- Ready for the next item if there are any
+local ITEMSTATE_SENT = "Item Sent Not Claimed" -- The ItemBit is set, but the dialog has not been closed yet
+local itemState = ITEMSTATE_NONINITIALIZED
+
+local itemQueued = nil
+local itemQueueCounter = 120
+
+local debugEnabled = false
+local game_complete = false
+
+local backup_bytes = nil
+
+local itemsReceived  = {}
+local previousMessageBit = 0x00
+
+local key_item_start_address = 0x20019C0
+
+-- The Canary Byte is a flag byte that is intentionally left unused. If this byte is FF, then we know the flag
+-- data cannot be trusted, so we don't send checks.
+local canary_byte = 0x20001A9
+
+local charDict = {
+    [' ']=0x00,['0']=0x01,['1']=0x02,['2']=0x03,['3']=0x04,['4']=0x05,['5']=0x06,['6']=0x07,['7']=0x08,['8']=0x09,['9']=0x0A,
+    ['A']=0x0B,['B']=0x0C,['C']=0x0D,['D']=0x0E,['E']=0x0F,['F']=0x10,['G']=0x11,['H']=0x12,['I']=0x13,['J']=0x14,['K']=0x15,
+    ['L']=0x16,['M']=0x17,['N']=0x18,['O']=0x19,['P']=0x1A,['Q']=0x1B,['R']=0x1C,['S']=0x1D,['T']=0x1E,['U']=0x1F,['V']=0x20,
+    ['W']=0x21,['X']=0x22,['Y']=0x23,['Z']=0x24,['a']=0x25,['b']=0x26,['c']=0x27,['d']=0x28,['e']=0x29,['f']=0x2A,['g']=0x2B,
+    ['h']=0x2C,['i']=0x2D,['j']=0x2E,['k']=0x2F,['l']=0x30,['m']=0x31,['n']=0x32,['o']=0x33,['p']=0x34,['q']=0x35,['r']=0x36,
+    ['s']=0x37,['t']=0x38,['u']=0x39,['v']=0x3A,['w']=0x3B,['x']=0x3C,['y']=0x3D,['z']=0x3E,['-']=0x3F,['×']=0x40,[']=']=0x41,
+    [':']=0x42,['+']=0x43,['÷']=0x44,['※']=0x45,['*']=0x46,['!']=0x47,['?']=0x48,['%']=0x49,['&']=0x4A,[',']=0x4B,['⋯']=0x4C,
+    ['.']=0x4D,['・']=0x4E,[';']=0x4F,['\'']=0x50,['\"']=0x51,['~']=0x52,['/']=0x53,['(']=0x54,[')']=0x55,['「']=0x56,['」']=0x57,
+    ["[V2]"]=0x58,["[V3]"]=0x59,["[V4]"]=0x5A,["[V5]"]=0x5B,['@']=0x5C,['♥']=0x5D,['♪']=0x5E,["[MB]"]=0x5F,['■']=0x60,['_']=0x61,
+    ["[circle1]"]=0x62,["[circle2]"]=0x63,["[cross1]"]=0x64,["[cross2]"]=0x65,["[bracket1]"]=0x66,["[bracket2]"]=0x67,["[ModTools1]"]=0x68,
+    ["[ModTools2]"]=0x69,["[ModTools3]"]=0x6A,['Σ']=0x6B,['Ω']=0x6C,['α']=0x6D,['β']=0x6E,['#']=0x6F,['…']=0x70,['>']=0x71,
+    ['<']=0x72,['エ']=0x73,["[BowneGlobal1]"]=0x74,["[BowneGlobal2]"]=0x75,["[BowneGlobal3]"]=0x76,["[BowneGlobal4]"]=0x77,
+    ["[BowneGlobal5]"]=0x78,["[BowneGlobal6]"]=0x79,["[BowneGlobal7]"]=0x7A,["[BowneGlobal8]"]=0x7B,["[BowneGlobal9]"]=0x7C,
+    ["[BowneGlobal10]"]=0x7D,["[BowneGlobal11]"]=0x7E,['\n']=0xE8
+}
+
+local TableConcat = function(t1,t2)
+   for i=1,#t2 do
+      t1[#t1+1] = t2[i]
+   end
+   return t1
+end
+local int32ToByteList_le = function(x)
+    bytes = {}
+    hexString = string.format("%08x", x)
+    for i=#hexString, 1, -2 do
+      hbyte = hexString:sub(i-1, i)
+      table.insert(bytes,tonumber(hbyte,16))
+    end
+    return bytes
+end
+local int16ToByteList_le = function(x)
+    bytes = {}
+    hexString = string.format("%04x", x)
+    for i=#hexString, 1, -2 do
+      hbyte = hexString:sub(i-1, i)
+      table.insert(bytes,tonumber(hbyte,16))
+    end
+    return bytes
+end
+
+local IsInMenu = function()
+    return bit.band(memory.read_u8(0x0200027A),0x10) ~= 0
+end
+local IsInTransition = function()
+    return bit.band(memory.read_u8(0x02001880), 0x10) ~= 0
+end
+local IsInDialog = function()
+    return bit.band(memory.read_u8(0x02009480),0x01) ~= 0
+end
+local IsInBattle = function()
+    return memory.read_u8(0x020097F8) == 0x08
+end
+local IsItemQueued = function()
+    return memory.read_u8(0x2000224) == 0x01
+end
+
+-- This function actually determines when you're on ANY full-screen menu (navi cust, link battle, etc.) but we
+-- don't want to check any locations there either so it's fine.
+local IsOnTitle = function()
+    return bit.band(memory.read_u8(0x020097F8),0x04) == 0
+end
+local IsItemable = function()
+    return not IsInMenu() and not IsInTransition() and not IsInDialog() and not IsInBattle() and not IsOnTitle() and not IsItemQueued()
+end
+
+local is_game_complete = function()
+    if IsOnTitle() or itemState == ITEMSTATE_NONINITIALIZED then return game_complete end
+
+    -- If the game is already marked complete, do not read memory
+    if game_complete then return true end
+    local is_alpha_defeated = bit.band(memory.read_u8(0x2000433), 0x01) ~= 0
+
+    if (is_alpha_defeated) then
+        game_complete = true
+        return true
+    end
+
+    -- Game is still ongoing
+    return false
+end
+
+local saveItemIndexToRAM = function(newIndex)
+    memory.write_s16_le(0x20000AE,newIndex)
+end
+
+local loadItemIndexFromRAM = function()
+    last_index = memory.read_s16_le(0x20000AE)
+    if (last_index < 0) then
+        last_index = 0
+        saveItemIndexToRAM(0)
+    end
+    return last_index
+end
+
+local loadPlayerNameFromROM = function()
+    return memory.read_bytes_as_array(0x7FFFC0,63,"ROM")
+end
+
+local check_all_locations = function()
+    local location_checks = {}
+    -- Title Screen should not check items
+    if itemState == ITEMSTATE_NONINITIALIZED or IsInTransition() then
+        return location_checks
+    end
+    if memory.read_u8(canary_byte) == 0xFF then
+        return location_checks
+    end
+    for k,v in pairs(memory.read_bytes_as_dict(0x02000000, 0x434)) do
+        str_k = string.format("%x", k)
+        location_checks[str_k] = v
+    end
+    return location_checks
+end
+
+local Check_Progressive_Undernet_ID = function()
+    ordered_offsets = { 0x020019DB,0x020019DC,0x020019DD,0x020019DE,0x020019DF,0x020019E0,0x020019FA,0x020019E2 }
+    for i=1,#ordered_offsets do
+        offset=ordered_offsets[i]
+
+        if memory.read_u8(offset) == 0 then
+            return i
+        end
+    end
+    return 9
+end
+local GenerateTextBytes = function(message)
+    bytes = {}
+    for i = 1, #message do
+        local c = message:sub(i,i)
+        table.insert(bytes, charDict[c])
+    end
+    return bytes
+end
+
+-- Item Message Generation functions
+local Next_Progressive_Undernet_ID = function(index)
+    ordered_IDs = { 27,28,29,30,31,32,58,34}
+    if index > #ordered_IDs then
+        --It shouldn't reach this point, but if it does, just give another GigFreez I guess
+        return 34
+    end
+    item_index=ordered_IDs[index]
+    return item_index
+end
+local Extra_Progressive_Undernet = function()
+    fragBytes = int32ToByteList_le(20)
+    bytes = {
+        0xF6, 0x50, fragBytes[1], fragBytes[2], fragBytes[3], fragBytes[4], 0xFF, 0xFF, 0xFF
+    }
+    bytes = TableConcat(bytes, GenerateTextBytes("The extra data\ndecompiles into:\n\"20 BugFrags\"!!"))
+    return bytes
+end
+
+local GenerateChipGet = function(chip, code, amt)
+    chipBytes = int16ToByteList_le(chip)
+    bytes = {
+        0xF6, 0x10, chipBytes[1], chipBytes[2], code, amt,
+        charDict['G'], charDict['o'], charDict['t'], charDict[' '], charDict['a'], charDict[' '], charDict['c'], charDict['h'], charDict['i'], charDict['p'], charDict[' '], charDict['f'], charDict['o'], charDict['r'], charDict['\n'],
+
+    }
+    if chip < 256 then
+        bytes = TableConcat(bytes, {
+            charDict['\"'], 0xF9,0x00,chipBytes[1],0x01,0x00,0xF9,0x00,code,0x03, charDict['\"'],charDict['!'],charDict['!']
+        })
+    else
+        bytes = TableConcat(bytes, {
+            charDict['\"'], 0xF9,0x00,chipBytes[1],0x02,0x00,0xF9,0x00,code,0x03, charDict['\"'],charDict['!'],charDict['!']
+        })
+    end
+    return bytes
+end
+local GenerateKeyItemGet = function(item, amt)
+    bytes = {
+        0xF6, 0x00, item, amt,
+        charDict['G'], charDict['o'], charDict['t'], charDict[' '], charDict['a'], charDict['\n'],
+        charDict['\"'], 0xF9, 0x00, item, 0x00, charDict['\"'],charDict['!'],charDict['!']
+    }
+    return bytes
+end
+local GenerateSubChipGet = function(subchip, amt)
+    -- SubChips have an extra bit of trouble. If you have too many, they're supposed to skip to another text bank that doesn't give you the item
+    -- Instead, I'm going to just let it get eaten
+    bytes = {
+        0xF6, 0x20, subchip, amt, 0xFF, 0xFF, 0xFF,
+        charDict['G'], charDict['o'], charDict['t'], charDict[' '], charDict['a'], charDict['\n'],
+        charDict['S'], charDict['u'], charDict['b'], charDict['C'], charDict['h'], charDict['i'], charDict['p'], charDict[' '], charDict['f'], charDict['o'], charDict['r'], charDict['\n'],
+        charDict['\"'], 0xF9, 0x00, subchip, 0x00, charDict['\"'],charDict['!'],charDict['!']
+    }
+    return bytes
+end
+local GenerateZennyGet = function(amt)
+    zennyBytes = int32ToByteList_le(amt)
+    bytes = {
+        0xF6, 0x30, zennyBytes[1], zennyBytes[2], zennyBytes[3], zennyBytes[4], 0xFF, 0xFF, 0xFF,
+        charDict['G'], charDict['o'], charDict['t'], charDict[' '], charDict['a'], charDict['\n'], charDict['\"']
+    }
+    -- The text needs to be added one char at a time, so we need to convert the number to a string then iterate through it
+    zennyStr = tostring(amt)
+    for i = 1, #zennyStr do
+        local c = zennyStr:sub(i,i)
+        table.insert(bytes, charDict[c])
+    end
+    bytes = TableConcat(bytes, {
+        charDict[' '], charDict['Z'], charDict['e'], charDict['n'], charDict['n'], charDict['y'], charDict['s'], charDict['\"'],charDict['!'],charDict['!']
+    })
+    return bytes
+end
+local GenerateProgramGet = function(program, color, amt)
+    bytes = {
+        0xF6, 0x40, (program * 4), amt, color,
+        charDict['G'], charDict['o'], charDict['t'], charDict[' '], charDict['a'], charDict[' '], charDict['N'], charDict['a'], charDict['v'], charDict['i'], charDict['\n'],
+        charDict['C'], charDict['u'], charDict['s'], charDict['t'], charDict['o'], charDict['m'], charDict['i'], charDict['z'], charDict['e'], charDict['r'], charDict[' '], charDict['P'], charDict['r'], charDict['o'], charDict['g'], charDict['r'], charDict['a'], charDict['m'], charDict[':'], charDict['\n'],
+        charDict['\"'], 0xF9, 0x00, program, 0x05, charDict['\"'],charDict['!'],charDict['!']
+    }
+
+    return bytes
+end
+local GenerateBugfragGet = function(amt)
+    fragBytes = int32ToByteList_le(amt)
+    bytes = {
+        0xF6, 0x50, fragBytes[1], fragBytes[2], fragBytes[3], fragBytes[4], 0xFF, 0xFF, 0xFF,
+        charDict['G'], charDict['o'], charDict['t'], charDict[':'], charDict['\n'], charDict['\"']
+    }
+    -- The text needs to be added one char at a time, so we need to convert the number to a string then iterate through it
+    bugFragStr = tostring(amt)
+    for i = 1, #bugFragStr do
+        local c = bugFragStr:sub(i,i)
+        table.insert(bytes, charDict[c])
+    end
+    bytes = TableConcat(bytes, {
+        charDict[' '], charDict['B'], charDict['u'], charDict['g'], charDict['F'], charDict['r'], charDict['a'], charDict['g'], charDict['s'], charDict['\"'],charDict['!'],charDict['!']
+    })
+    return bytes
+end
+local GenerateGetMessageFromItem = function(item)
+    --Special case for progressive undernet
+    if item["type"] == "undernet" then
+        undernet_id = Check_Progressive_Undernet_ID()
+        if undernet_id > 8 then
+            return Extra_Progressive_Undernet()
+        end
+        return GenerateKeyItemGet(Next_Progressive_Undernet_ID(undernet_id),1)
+    elseif item["type"] == "chip" then
+        return GenerateChipGet(item["itemID"], item["subItemID"], item["count"])
+    elseif item["type"] == "key" then
+        return GenerateKeyItemGet(item["itemID"], item["count"])
+    elseif item["type"] == "subchip" then
+        return GenerateSubChipGet(item["itemID"], item["count"])
+    elseif item["type"] == "zenny" then
+        return GenerateZennyGet(item["count"])
+    elseif item["type"] == "program" then
+        return GenerateProgramGet(item["itemID"], item["subItemID"], item["count"])
+    elseif item["type"] == "bugfrag" then
+        return GenerateBugfragGet(item["count"])
+    end
+
+    return GenerateTextBytes("Empty Message")
+end
+
+local GetMessage = function(item)
+    startBytes = {0x02, 0x00}
+    playerLockBytes = {0xF8,0x00, 0xF8, 0x10}
+    msgOpenBytes = {0xF1, 0x02}
+    textBytes = GenerateTextBytes("Receiving\ndata from\n"..item["sender"]..".")
+    dotdotWaitBytes = {0xEA,0x00,0x0A,0x00,0x4D,0xEA,0x00,0x0A,0x00,0x4D}
+    continueBytes = {0xEB, 0xE9}
+    -- continueBytes = {0xE9}
+    playReceiveAnimationBytes = {0xF8,0x04,0x18}
+    chipGiveBytes = GenerateGetMessageFromItem(item)
+    playerFinishBytes = {0xF8, 0x0C}
+    playerUnlockBytes={0xEB, 0xF8, 0x08}
+    -- playerUnlockBytes={0xF8, 0x08}
+    endMessageBytes = {0xF8, 0x10, 0xE7}
+
+    bytes = {}
+    bytes = TableConcat(bytes,startBytes)
+    bytes = TableConcat(bytes,playerLockBytes)
+    bytes = TableConcat(bytes,msgOpenBytes)
+    bytes = TableConcat(bytes,textBytes)
+    bytes = TableConcat(bytes,dotdotWaitBytes)
+    bytes = TableConcat(bytes,continueBytes)
+    bytes = TableConcat(bytes,playReceiveAnimationBytes)
+    bytes = TableConcat(bytes,chipGiveBytes)
+    bytes = TableConcat(bytes,playerFinishBytes)
+    bytes = TableConcat(bytes,playerUnlockBytes)
+    bytes = TableConcat(bytes,endMessageBytes)
+    return bytes
+end
+
+local getChipCodeIndex = function(chip_id, chip_code)
+    chipCodeArrayStartAddress = 0x8011510 + (0x20 * chip_id)
+    for i=1,6 do
+        currentCode = memory.read_u8(chipCodeArrayStartAddress + (i-1))
+        if currentCode == chip_code then
+            return i-1
+        end
+    end
+    return 0
+end
+
+local getProgramColorIndex = function(program_id, program_color)
+    -- The general case, most programs use white pink or yellow. This is the values the enums already have
+    if program_id >= 20 and program_id <= 47 then
+        return program_color-1
+    end
+    --The final three programs only have a color index 0, so just return those
+    if program_id > 47 then
+        return 0
+    end
+    --BrakChrg as an AP item only comes in orange, index 0
+    if program_id == 3 then
+        return 0
+    end
+    -- every other AP obtainable program returns only color index 3
+    return 3
+end
+
+local addChip = function(chip_id, chip_code, amount)
+    chipStartAddress = 0x02001F60
+    chipOffset = 0x12 * chip_id
+    chip_code_index = getChipCodeIndex(chip_id, chip_code)
+    currentChipAddress = chipStartAddress + chipOffset + chip_code_index
+    currentChipCount = memory.read_u8(currentChipAddress)
+    memory.write_u8(currentChipAddress,currentChipCount+amount)
+end
+
+local addProgram = function(program_id, program_color, amount)
+    programStartAddress = 0x02001A80
+    programOffset = 0x04 * program_id
+    program_code_index = getProgramColorIndex(program_id, program_color)
+    currentProgramAddress = programStartAddress + programOffset + program_code_index
+    currentProgramCount = memory.read_u8(currentProgramAddress)
+    memory.write_u8(currentProgramAddress, currentProgramCount+amount)
+end
+
+local addSubChip = function(subchip_id, amount)
+    subChipStartAddress = 0x02001A30
+    --SubChip indices start after the key items, so subtract 112 from the index to get the actual subchip index
+    currentSubChipAddress = subChipStartAddress + (subchip_id - 112)
+    currentSubChipCount = memory.read_u8(currentSubChipAddress)
+    --TODO check submem, reject if number too big
+    memory.write_u8(currentSubChipAddress, currentSubChipCount+amount)
+end
+
+local changeZenny = function(val)
+	if val == nil then
+		return 0
+	end
+	if memory.read_u32_le(0x20018F4) <= math.abs(tonumber(val)) and tonumber(val) < 0 then
+		memory.write_u32_le(0x20018f4, 0)
+		val = 0
+		return "empty"
+	end
+	memory.write_u32_le(0x20018f4, memory.read_u32_le(0x20018F4) + tonumber(val))
+	if memory.read_u32_le(0x20018F4) > 999999 then
+		memory.write_u32_le(0x20018F4, 999999)
+	end
+	return val
+end
+
+local changeFrags = function(val)
+	if val == nil then
+		return 0
+	end
+	if memory.read_u16_le(0x20018F8) <= math.abs(tonumber(val)) and tonumber(val) < 0 then
+		memory.write_u16_le(0x20018f8, 0)
+		val = 0
+		return "empty"
+	end
+	memory.write_u16_le(0x20018f8, memory.read_u16_le(0x20018F8) + tonumber(val))
+	if memory.read_u16_le(0x20018F8) > 9999 then
+		memory.write_u16_le(0x20018F8, 9999)
+	end
+	return val
+end
+
+-- Fix Health Pools
+local fix_hp = function()
+	-- Current Health fix
+	if IsInBattle() and not (memory.read_u16_le(0x20018A0) == memory.read_u16_le(0x2037294)) then
+		memory.write_u16_le(0x20018A0, memory.read_u16_le(0x2037294))
+	end
+
+	-- Max Health Fix
+	if IsInBattle() and not (memory.read_u16_le(0x20018A2) == memory.read_u16_le(0x2037296)) then
+		memory.write_u16_le(0x20018A2, memory.read_u16_le(0x2037296))
+	end
+end
+
+local changeRegMemory = function(amt)
+    regMemoryAddress = 0x02001897
+    currentRegMem = memory.read_u8(regMemoryAddress)
+    memory.write_u8(regMemoryAddress, currentRegMem + amt)
+end
+
+local changeMaxHealth = function(val)
+	fix_hp()
+	if val == nil then
+		fix_hp()
+		return 0
+	end
+	if math.abs(tonumber(val)) >= memory.read_u16_le(0x20018A2) and tonumber(val) < 0 then
+		memory.write_u16_le(0x20018A2, 0)
+		if IsInBattle() then
+			memory.write_u16_le(0x2037296, memory.read_u16_le(0x20018A2))
+			if memory.read_u16_le(0x2037296) >= memory.read_u16_le(0x20018A2) then
+				memory.write_u16_le(0x2037296, memory.read_u16_le(0x20018A2))
+			end
+		end
+		fix_hp()
+		return "lethal"
+	end
+	memory.write_u16_le(0x20018A2, memory.read_u16_le(0x20018A2) + tonumber(val))
+	if memory.read_u16_le(0x20018A2) > 9999 then
+		memory.write_u16_le(0x20018A2, 9999)
+	end
+	if IsInBattle() then
+		memory.write_u16_le(0x2037296, memory.read_u16_le(0x20018A2))
+	end
+	fix_hp()
+	return val
+end
+
+local SendItem = function(item)
+    if item["type"] == "undernet" then
+        undernet_id = Check_Progressive_Undernet_ID()
+        if undernet_id > 8 then
+            -- Generate Extra BugFrags
+            changeFrags(20)
+            gui.addmessage("Receiving extra Undernet Rank from "..item["sender"]..", +20 BugFrags")
+            -- print("Receiving extra Undernet Rank from "..item["sender"]..", +20 BugFrags")
+        else
+            itemAddress = key_item_start_address + Next_Progressive_Undernet_ID(undernet_id)
+
+            itemCount = memory.read_u8(itemAddress)
+            itemCount = itemCount + item["count"]
+            memory.write_u8(itemAddress, itemCount)
+            gui.addmessage("Received Undernet Rank from player "..item["sender"])
+            -- print("Received Undernet Rank from player "..item["sender"])
+        end
+    elseif item["type"] == "chip" then
+        addChip(item["itemID"], item["subItemID"], item["count"])
+        gui.addmessage("Received Chip "..item["itemName"].." from player "..item["sender"])
+        -- print("Received Chip "..item["itemName"].." from player "..item["sender"])
+    elseif item["type"] == "key" then
+        itemAddress = key_item_start_address + item["itemID"]
+        itemCount = memory.read_u8(itemAddress)
+        itemCount = itemCount + item["count"]
+        memory.write_u8(itemAddress, itemCount)
+        -- HPMemory will increase the internal counter but not actually increase the HP. If the item is one of those, do that
+        if item["itemID"] == 96 then
+            changeMaxHealth(20)
+        end
+        -- Same for the RegUps, but there's three of those
+        if item["itemID"] == 98 then
+            changeRegMemory(1)
+        end
+        if item["itemID"] == 99 then
+            changeRegMemory(2)
+        end
+        if item["itemID"] == 100 then
+            changeRegMemory(3)
+        end
+        gui.addmessage("Received Key Item "..item["itemName"].." from player "..item["sender"])
+        -- print("Received Key Item "..item["itemName"].." from player "..item["sender"])
+    elseif item["type"] == "subchip" then
+        addSubChip(item["itemID"], item["count"])
+        gui.addmessage("Received SubChip "..item["itemName"].." from player "..item["sender"])
+        -- print("Received SubChip "..item["itemName"].." from player "..item["sender"])
+    elseif item["type"] == "zenny" then
+        changeZenny(item["count"])
+        gui.addmessage("Received "..item["count"].."z from "..item["sender"])
+        -- print("Received "..item["count"].."z from "..item["sender"])
+    elseif item["type"] == "program" then
+        addProgram(item["itemID"], item["subItemID"], item["count"])
+        gui.addmessage("Received Program "..item["itemName"].." from player "..item["sender"])
+        -- print("Received Program "..item["itemName"].." from player "..item["sender"])
+    elseif item["type"] == "bugfrag" then
+        changeFrags(item["count"])
+        gui.addmessage("Received "..item["count"].." BugFrag(s) from "..item["sender"])
+        -- print("Received "..item["count"].." BugFrag(s) from "..item["sender"])
+    end
+end
+
+-- Set the flags for opening the shortcuts as soon as the Cybermetro passes are received to save having to check email
+local OpenShortcuts = function()
+    if (memory.read_u8(key_item_start_address + 92) > 0) then
+        memory.write_u8(0x2000032, bit.bor(memory.read_u8(0x2000032),0x10))
+    end
+    -- if CSciPass
+    if (memory.read_u8(key_item_start_address + 93) > 0) then
+        memory.write_u8(0x2000032, bit.bor(memory.read_u8(0x2000032),0x08))
+    end
+    if (memory.read_u8(key_item_start_address + 94) > 0) then
+        memory.write_u8(0x2000032, bit.bor(memory.read_u8(0x2000032),0x20))
+    end
+    if (memory.read_u8(key_item_start_address + 95) > 0) then
+       memory.write_u8(0x2000032, bit.bor(memory.read_u8(0x2000032),0x40))
+    end
+end
+
+local RestoreItemRam = function()
+    if backup_bytes ~= nil then
+        memory.write_bytes_as_array(0x203fe10, backup_bytes)
+    end
+    backup_bytes = nil
+end
+
+local process_block = function(block)
+    -- Sometimes the block is nothing, if this is the case then quietly stop processing
+    if block == nil then
+        return
+    end
+    debugEnabled = block['debug']
+    -- Queue item for receiving, if one exists
+    if (itemsReceived ~= block['items']) then
+        itemsReceived = block['items']
+    end
+    return
+end
+
+local itemStateMachineProcess = function()
+    if itemState == ITEMSTATE_NONINITIALIZED then
+        itemQueueCounter = 120
+        -- Only exit this state the first time a dialog window pops up. This way we know for sure that we're ready to receive
+        if not IsInMenu() and (IsInDialog() or IsInTransition()) then
+            itemState = ITEMSTATE_NONITEM
+        end
+    elseif itemState == ITEMSTATE_NONITEM then
+        itemQueueCounter = 120
+        -- Always attempt to restore the previously stored memory in this state
+        -- Exit this state whenever the game is in an itemable status
+        if IsItemable() then
+            itemState = ITEMSTATE_IDLE
+        end
+    elseif itemState == ITEMSTATE_IDLE then
+        -- Remain Idle until an item is sent or we enter a non itemable status
+        if not IsItemable() then
+            itemState = ITEMSTATE_NONITEM
+        end
+        if itemQueueCounter == 0 then
+            if #itemsReceived > loadItemIndexFromRAM() and not IsItemQueued() then
+                itemQueued = itemsReceived[loadItemIndexFromRAM()+1]
+                SendItem(itemQueued)
+                itemState = ITEMSTATE_SENT
+            end
+        else
+            itemQueueCounter = itemQueueCounter - 1
+        end
+    elseif itemState == ITEMSTATE_SENT then
+        -- Once the item is sent, wait for the dialog to close. Then clear the item bit and be ready for the next item.
+        if IsInTransition() or IsInMenu() or IsOnTitle() then
+            itemState = ITEMSTATE_NONITEM
+            itemQueued = nil
+            RestoreItemRam()
+        elseif not IsInDialog() then
+            itemState = ITEMSTATE_IDLE
+            saveItemIndexToRAM(itemQueued["itemIndex"])
+            itemQueued = nil
+            RestoreItemRam()
+        end
+    end
+end
+local receive = function()
+    l, e = mmbn3Socket:receive()
+
+    -- Handle incoming message
+    if e == 'closed' then
+        if curstate == STATE_OK then
+            print("Connection closed")
+        end
+        curstate = STATE_UNINITIALIZED
+        return
+    elseif e == 'timeout' then
+        print("timeout")
+        return
+    elseif e ~= nil then
+        print(e)
+        curstate = STATE_UNINITIALIZED
+        return
+    end
+    process_block(json.decode(l))
+end
+
+local send = function()
+    -- Determine message to send back
+    local retTable = {}
+    retTable["playerName"] = loadPlayerNameFromROM()
+    retTable["scriptVersion"] = script_version
+    retTable["locations"] = check_all_locations()
+    retTable["gameComplete"] = is_game_complete()
+
+    -- Send the message
+    msg = json.encode(retTable).."\n"
+    local ret, error = mmbn3Socket:send(msg)
+
+    if ret == nil then
+        print(error)
+    elseif curstate == STATE_INITIAL_CONNECTION_MADE then
+        curstate = STATE_TENTATIVELY_CONNECTED
+    elseif curstate == STATE_TENTATIVELY_CONNECTED then
+        print("Connected!")
+        curstate = STATE_OK
+    end
+end
+
+function main()
+    if (bizhawk_major > 2 or (bizhawk_major == 2 and bizhawk_minor >= 7)==false) then
+        print("Must use a version of bizhawk 2.7.0 or higher")
+        return
+    end
+    server, error = socket.bind('localhost', 28922)
+
+    while true do
+        frame = frame + 1
+
+        if not (curstate == prevstate) then
+            prevstate = curstate
+        end
+
+        itemStateMachineProcess()
+
+        if (curstate == STATE_OK) or (curstate == STATE_INITIAL_CONNECTION_MADE) or (curstate == STATE_TENTATIVELY_CONNECTED) then
+            -- If we're connected and everything's fine, receive and send data from the network
+            if (frame % 60 == 0) then
+                receive()
+                send()
+                -- Perform utility functions which read and write data but aren't directly related to checks
+                OpenShortcuts()
+            end
+        elseif (curstate == STATE_UNINITIALIZED) then
+            -- If we're uninitialized, attempt to make the connection.
+            if (frame % 120 == 0) then
+                server:settimeout(2)
+                local client, timeout = server:accept()
+                if timeout == nil then
+                    print('Initial Connection Made')
+                    curstate = STATE_INITIAL_CONNECTION_MADE
+                    mmbn3Socket = client
+                    mmbn3Socket:settimeout(0)
+                else
+                    print('Connection failed, ensure MMBN3Client is running and rerun connector_mmbn3.lua')
+                    return
+                end
+            end
+        end
+
+        -- Handle the debug data display
+        gui.cleartext()
+        if debugEnabled then
+            -- gui.text(0,0,"Item Queued: "..tostring(IsItemQueued()))
+            -- gui.text(0,16,"In Battle: "..tostring(IsInBattle()))
+            -- gui.text(0,32,"In Dialog: "..tostring(IsInDialog()))
+            -- gui.text(0,48,"In Menu: "..tostring(IsInMenu()))
+            gui.text(0,48,"Item Wait Time: "..tostring(itemQueueCounter))
+            gui.text(0,64,itemState)
+            if itemQueued == nil then
+                gui.text(0,80,"No item queued")
+            else
+                gui.text(0,80,itemQueued["type"].." "..itemQueued["itemID"])
+            end
+            gui.text(0,96,"Item Index: "..loadItemIndexFromRAM())
+        end
+
+        emu.frameadvance()
+    end
+end
+
+main()

data/lua/connector_oot.lua

@@ -1862,7 +1862,7 @@ function receive()
 end
 
 function main()
-    if not checkBizhawkVersion() then
+    if not checkBizHawkVersion() then
         return
     end
     server, error = socket.bind('localhost', 28921)

data/lua/connector_pkmn_rb.lua

@@ -167,7 +167,7 @@ function receive()
 end
 
 function main()
-    if not checkBizhawkVersion() then
+    if not checkBizHawkVersion() then
         return
     end
     server, error = socket.bind('localhost', 17242)

data/lua/connector_tloz.lua

@@ -561,7 +561,7 @@ function receive()
 end
 
 function main()
-    if not checkBizhawkVersion() then
+    if not checkBizHawkVersion() then
         return
     end
     server, error = socket.bind('localhost', 52980)

data/lua/socket.lua

@@ -46,10 +46,10 @@ function get_socket_path()
     local pwd = (io.popen and io.popen("cd"):read'*l') or "."
 	return pwd .. "/" .. arch .. "/socket-" .. the_os .. "-" .. get_lua_version() .. "." .. ext
 end
-
+local lua_version = get_lua_version()
 local socket_path = get_socket_path()
 local socket = assert(package.loadlib(socket_path, "luaopen_socket_core"))()
-
+local event = event
 -- http://lua-users.org/wiki/ModulesTutorial
 local M = {}
 if setfenv then
@@ -59,6 +59,20 @@ else
 end
 
 M.socket = socket
+-- Bizhawk <= 2.8 has an issue where resetting the lua doesn't close the socket
+-- ...to get around this, we register an exit handler to close the socket first
+if lua_version == '5-1' then
+    local old_udp = socket.udp
+    function udp(self)
+        s = old_udp(self)
+        function close_socket(self)
+            s:close()
+        end
+        event.onexit(close_socket)
+        return s
+    end
+    socket.udp = udp
+end
 
 -----------------------------------------------------------------------------
 -- Exported auxiliar functions

docs/CODEOWNERS

@@ -0,0 +1,166 @@
+# Archipelago World Code Owners / Maintainers Document
+#
+# This file is used to notate the current "owners" or "maintainers" of any currently merged world folder. For any pull
+# requests that modify these worlds, a code owner must approve the PR in addition to a core maintainer. This is not to
+# be used for files/folders outside the /worlds folder, those will always need sign off from a core maintainer.
+#
+# All usernames must be GitHub usernames (and are case sensitive).
+
+###################
+## Active Worlds ##
+###################
+
+# Adventure
+/worlds/adventure/ @JusticePS
+
+# A Link to the Past
+/worlds/alttp/ @Berserker66
+
+# ArchipIDLE
+/worlds/archipidle/ @LegendaryLinux
+
+# Sudoku (BK Sudoku)
+/worlds/bk_sudoku/ @Jarno458
+
+# Blasphemous
+/worlds/blasphemous/ @TRPG0
+
+# Bumper Stickers
+/worlds/bumpstik/ @FelicitusNeko
+
+# ChecksFinder
+/worlds/checksfinder/ @jonloveslegos
+
+# Clique
+/worlds/clique/ @ThePhar
+
+# Dark Souls III
+/worlds/dark_souls_3/ @Marechal-L
+
+# Donkey Kong Country 3
+/worlds/dkc3/ @PoryGone
+
+# DLCQuest
+/worlds/dlcquest/ @axe-y @agilbert1412
+
+# DOOM 1993
+/worlds/doom_1993/ @Daivuk
+
+# Factorio
+/worlds/factorio/ @Berserker66
+
+# Final Fantasy
+/worlds/ff1/ @jtoyoda
+
+# Hollow Knight
+/worlds/hk/ @BadMagic100 @ThePhar
+
+# Hylics 2
+/worlds/hylics2/ @TRPG0
+
+# Kingdom Hearts 2
+/worlds/kh2/ @JaredWeakStrike
+
+# Links Awakening DX
+/worlds/ladx/ @zig-for
+
+# Lufia II Ancient Cave
+/worlds/lufia2ac/ @el-u
+/worlds/lufia2ac/docs/ @wordfcuk @el-u
+
+# Meritous
+/worlds/meritous/ @FelicitusNeko
+
+# The Messenger
+/worlds/messenger/ @alwaysintreble
+
+# Minecraft
+/worlds/minecraft/ @KonoTyran @espeon65536
+
+# MegaMan Battle Network 3
+/worlds/mmbn3/ @digiholic
+
+# Muse Dash
+/worlds/musedash/ @DeamonHunter
+
+# Noita
+/worlds/noita/ @ScipioWright @heinermann
+
+# Ocarina of Time
+/worlds/oot/ @espeon65536
+
+# Overcooked! 2
+/worlds/overcooked2/ @toasterparty
+
+# Pokemon Red and Blue
+/worlds/pokemon_rb/ @Alchav
+
+# Raft
+/worlds/raft/ @SunnyBat
+
+# Rogue Legacy
+/worlds/rogue_legacy/ @ThePhar
+
+# Risk of Rain 2
+/worlds/ror2/ @kindasneaki
+
+# Sonic Adventure 2 Battle
+/worlds/sa2b/ @PoryGone @RaspberrySpace
+
+# Starcraft 2 Wings of Liberty
+/worlds/sc2wol/ @Ziktofel
+
+# Super Metroid
+/worlds/sm/ @lordlou
+
+# Super Mario 64
+/worlds/sm64ex/ @N00byKing
+
+# Super Mario World
+/worlds/smw/ @PoryGone
+
+# SMZ3
+/worlds/smz3/ @lordlou
+
+# Secret of Evermore
+/worlds/soe/ @black-sliver
+
+# Slay the Spire
+/worlds/spire/ @KonoTyran
+
+# Stardew Valley
+/worlds/stardew_valley/ @agilbert1412
+
+# Subnautica
+/worlds/subnautica/ @Berserker66
+
+# Terraria
+/worlds/terraria/ @Seldom-SE
+
+# Timespinner
+/worlds/timespinner/ @Jarno458
+
+# The Legend of Zelda (1)
+/worlds/tloz/ @Rosalie-A @t3hf1gm3nt
+
+# Undertale
+/worlds/undertale/ @jonloveslegos
+
+# VVVVVV
+/worlds/v6/ @N00byKing
+
+# Wargroove
+/worlds/wargroove/ @FlySniper
+
+# The Witness
+/worlds/witness/ @NewSoupVi @blastron
+
+# Zillion
+/worlds/zillion/ @beauxq
+
+##################################
+## Disabled Unmaintained Worlds ##
+##################################
+
+# Ori and the Blind Forest
+# /worlds_disabled/oribf/ <Unmaintained>

docs/adding games.md

@@ -341,3 +341,4 @@ The various methods and attributes are documented in `/worlds/AutoWorld.py[World
 [world api.md](https://github.com/ArchipelagoMW/Archipelago/blob/main/docs/world%20api.md),
 though it is also recommended to look at existing implementations to see how all this works first-hand. 
 Once you get all that, all that remains to do is test the game and publish your work.
+Make sure to check out [world maintainer.md](./world%20maintainer.md) before publishing.

docs/apworld specification.md

@@ -1,7 +1,7 @@
 # apworld Specification
 
 Archipelago depends on worlds to provide game-specific details like items, locations and output generation.
-Those are located in the `worlds/` folder (source) or  `<insall dir>/lib/worlds/` (when installed).
+Those are located in the `worlds/` folder (source) or `<install dir>/lib/worlds/` (when installed).
 See [world api.md](world%20api.md) for details.
 
 apworld provides a way to package and ship a world that is not part of the main distribution by placing a `*.apworld`

docs/contributing.md

@@ -10,3 +10,5 @@ Otherwise, we tend to judge code on a case to case basis.
 For adding a new game to Archipelago and other documentation on how Archipelago functions, please see 
 [the docs folder](/docs/) for the relevant information and feel free to ask any questions in the #archipelago-dev 
 channel in our [Discord](https://archipelago.gg/discord).
+If you want to merge a new game, please make sure to read the responsibilities as
+[world maintainer](/docs/world%20maintainer.md).

docs/network diagram/network diagram.md

@@ -35,7 +35,7 @@ flowchart LR
     subgraph Final Fantasy 1
         FF1[FF1Client]
         FFLUA[Lua Connector]
-        BZFF[BizHawk with Final Fantasy Loaded]
+        BZFF[EmuHawk with Final Fantasy Loaded]
         FF1 <-- LuaSockets --> FFLUA
         FFLUA <--> BZFF
     end
@@ -45,7 +45,7 @@ flowchart LR
     subgraph Ocarina of Time
         OC[OoTClient] 
         LC[Lua Connector]
-        OCB[BizHawk with Ocarina of Time Loaded]
+        OCB[EmuHawk with Ocarina of Time Loaded]
         OC <-- LuaSockets --> LC
         LC <--> OCB
     end

docs/running from source.md

@@ -69,6 +69,19 @@ It should be dropped as "SNI" into the root folder of the project. Alternatively
 host.yaml at your SNI folder.
 
 
+## Optional: Git
+
+[Git](https://git-scm.com) is required to install some of the packages that Archipelago depends on.
+It may be possible to run Archipelago from source without it, at your own risk.
+
+It is also generally recommended to have Git installed and understand how to use it, especially if you're thinking about contributing.
+
+You can download the latest release of Git at [The downloads page on the Git website](https://git-scm.com/downloads).
+
+Beyond that, there are also graphical interfaces for Git that make it more accessible.
+For repositories on Github (such as this one), [Github Desktop](https://desktop.github.com) is one such option.
+PyCharm has a built-in version control integration that supports Git.
+
 ## Running tests
 
 Run `pip install pytest pytest-subtests`, then use your IDE to run tests or run `pytest` from the source folder.

docs/settings api.md

@@ -0,0 +1,187 @@
+# Archipelago Settings API
+
+The settings API describes how to use installation-wide config and let the user configure them, like paths, etc. using
+host.yaml. For the player settings / player yamls see [options api.md](options api.md).
+
+The settings API replaces `Utils.get_options()` and `Utils.get_default_options()`
+as well as the predefined `host.yaml` in the repository.
+
+For backwards compatibility with APWorlds, some interfaces are kept for now and will produce a warning when being used.
+
+
+## Config File
+
+Settings use options.yaml (manual override), if that exists, or host.yaml (the default) otherwise.
+The files are searched for in the current working directory, if different from install directory, and in `user_path`,
+which either points to the installation directory, if writable, or to %home%/Archipelago otherwise.
+
+**Examples:**
+* C:\Program Data\Archipelago\options.yaml
+* C:\Program Data\Archipelago\host.yaml
+* path\to\code\repository\host.yaml
+* ~/Archipelago/host.yaml
+
+Using the settings API, AP can update the config file or create a new one with default values and comments, 
+if it does not exist.
+
+
+## Global Settings
+
+All non-world-specific settings are defined directly in settings.py.
+Each value needs to have a default. If the default should be `None`, define it as `typing.Optional` and assign `None`.
+
+To access a "global" config value, with correct typing, use one of
+```python
+from settings import get_settings, GeneralOptions, FolderPath
+from typing import cast
+
+x = get_settings().general_options.output_path
+y = cast(GeneralOptions, get_settings()["general_options"]).output_path
+z = cast(FolderPath, get_settings()["general_options"]["output_path"])
+```
+
+
+## World Settings
+
+Worlds can define the top level key to use by defining `settings_key: ClassVar[str]` in their World class.
+It defaults to `{folder_name}_options` if undefined, i.e. `worlds/factorio/...` defaults to `factorio_options`.
+
+Worlds define the layout of their config section using type annotation of the variable `settings` in the class.
+The type has to inherit from `settings.Group`. Each value in the config can have a comment by subclassing a built-in
+type. Some helper types are defined in `settings.py`, see [Types](#Types) for a list.```
+
+Inside the class code, you can then simply use `self.settings.rom_file` to get the value.
+In case of paths they will automatically be read as absolute file paths. No need to use user_path or local_path.
+
+```python
+import settings
+from worlds.AutoWorld import World
+
+
+class MyGameSettings(settings.Group):
+    class RomFile(settings.SNESRomPath):
+        """Description that is put into host.yaml"""
+        description = "My Game US v1.0 ROM File"  # displayed in the file browser
+        copy_to = "MyGame.sfc"  # instead of storing the path, copy to AP dir
+        md5s = ["..."]
+
+    rom_file: RomFile = RomFile("MyGame.sfc")  # definition and default value
+
+
+class MyGameWorld(World):
+    ...
+    settings: MyGameSettings
+    ...
+
+    def something(self):
+        pass  # use self.settings.rom_file here
+```
+
+
+## Types
+
+When writing the host.yaml, the code will down cast the values to builtins.
+When reading the host.yaml, the code will upcast the values to what is defined in the type annotations.
+E.g. an IntEnum becomes int when saving and will construct the IntEnum when loading.
+
+Types that can not be down cast to / up cast from a builtin can not be used except for Group, which will be converted
+to/from a dict.
+`bool` is a special case, see settings.py: ServerOptions.disable_item_cheat for an example.
+
+Below are some predefined types that can be used if they match your requirements:
+
+
+### Group
+
+A section / dict in the config file. Behaves similar to a dataclass.
+Type annotation and default assignment define how loading, saving and default values behave.
+It can be accessed using attributes or as a dict: `group["a"]` is equivalent to `group.a`.
+
+In worlds, this should only be used for the top level to avoid issues when upgrading/migrating.
+
+
+### Bool
+
+Since `bool` can not be subclassed, use the `settings.Bool` helper in a `typing.Union` to get a comment in host.yaml.
+
+```python
+import settings
+import typing
+
+class MySettings(settings.Group):
+    class MyBool(settings.Bool):
+        """Doc string"""
+
+    my_value: typing.Union[MyBool, bool] = True
+```
+
+### UserFilePath
+
+Path to a single file. Automatically resolves as user_path:
+Source folder or AP install path on Windows. ~/Archipelago for the AppImage.
+Will open a file browser if the file is missing when in GUI mode.
+
+#### class method validate(cls, path: str)
+
+Override this and raise ValueError if validation fails.
+Checks the file against [md5s](#md5s) by default.
+
+#### is_exe: bool
+
+Resolves to an executable (varying file extension based on platform)
+
+#### description: Optional\[str\]
+
+Human-readable name to use in file browser
+
+#### copy_to: Optional\[str\]
+
+Instead of storing the path, copy the file.
+
+#### md5s: List[Union[str, bytes]]
+
+Provide md5 hashes as hex digests or raw bytes for automatic validation.
+
+
+### UserFolderPath
+
+Same as [UserFilePath](#UserFilePath), but for a folder instead of a file.
+
+
+### LocalFilePath
+
+Same as [UserFilePath](#UserFilePath), but resolves as local_path:
+path inside the AP dir or Appimage even if read-only.
+
+
+### LocalFolderPath
+
+Same as [LocalFilePath](#LocalFilePath), but for a folder instead of a file.
+
+
+### OptionalUserFilePath, OptionalUserFolderPath, OptionalLocalFilePath, OptionalLocalFolderPath
+
+Same as UserFilePath, UserFolderPath, LocalFilePath, LocalFolderPath but does not open a file browser if missing.
+
+
+### SNESRomPath
+
+Specialized [UserFilePath](#UserFilePath) that ignores an optional 512 byte header when validating.
+
+
+## Caveats
+
+### Circular Imports
+
+Because the settings are defined on import, code that runs on import can not use settings since that would result in
+circular / partial imports. Instead, the code should fetch from settings on demand during generation.
+
+"Global" settings are populated immediately, while worlds settings are lazy loaded, so if really necessary,
+"global" settings could be used in global scope of worlds.
+
+
+### APWorld Backwards Compatibility
+
+APWorlds that want to be compatible with both stable and dev versions, have two options:
+1. use the old Utils.get_options() API until Archipelago 0.4.2 is out
+2. add some sort of compatibility code to your world that mimics the new API

docs/world api.md

@@ -22,8 +22,8 @@ allows using WebSockets.
 
 ## Coding style
 
-AP follows all the PEPs. When in doubt use an IDE with coding style
-linter, for example PyCharm Community Edition.
+AP follows [style.md](https://github.com/ArchipelagoMW/Archipelago/blob/main/docs/style.md).
+When in doubt use an IDE with coding style linter, for example PyCharm Community Edition.
 
 
 ## Docstrings
@@ -44,7 +44,7 @@ class MyGameWorld(World):
 ## Definitions
 
 This section will cover various classes and objects you can use for your world.
-While some of the attributes and methods are mentioned here not all of them are,
+While some of the attributes and methods are mentioned here, not all of them are,
 but you can find them in `BaseClasses.py`.
 
 ### World Class
@@ -56,11 +56,12 @@ game.
 ### WebWorld Class
 
 A `WebWorld` class contains specific attributes and methods that can be modified
-for your world specifically on the webhost.
+for your world specifically on the webhost:
 
-`settings_page` which can be changed to a link instead of an AP generated settings page.
+`settings_page`, which can be changed to a link instead of an AP generated settings page.
 
 `theme` to be used for your game specific AP pages. Available themes:
+
 | dirt  | grass (default) | grassFlowers | ice  | jungle  | ocean | partyTime | stone |
 |---|---|---|---|---|---|---|---|
 | <img src="img/theme_dirt.JPG" width="100"> | <img src="img/theme_grass.JPG" width="100"> | <img src="img/theme_grassFlowers.JPG" width="100"> | <img src="img/theme_ice.JPG" width="100"> | <img src="img/theme_jungle.JPG" width="100"> | <img src="img/theme_ocean.JPG" width="100"> | <img src="img/theme_partyTime.JPG" width="100"> | <img src="img/theme_stone.JPG" width="100"> |
@@ -75,26 +76,30 @@ prefixed with the same string as defined here. Default already has 'en'.
 ### MultiWorld Object
 
 The `MultiWorld` object references the whole multiworld (all items and locations
-for all players) and is accessible through `self.world` inside a `World` object.
+for all players) and is accessible through `self.multiworld` inside a `World` object.
 
 ### Player
 
 The player is just an integer in AP and is accessible through `self.player`
-inside a World object.
+inside a `World` object.
 
 ### Player Options
 
 Players provide customized settings for their World in the form of yamls.
-Those are accessible through `self.world.<option_name>[self.player]`. A dict
+Those are accessible through `self.multiworld.<option_name>[self.player]`. A dict
 of valid options has to be provided in `self.option_definitions`. Options are automatically
 added to the `World` object for easy access.
 
-### World Options
+### World Settings
 
-Any AP installation can provide settings for a world, for example a ROM file,
-accessible through `Utils.get_options()['<world>_options']['<option>']`.
+Any AP installation can provide settings for a world, for example a ROM file, accessible through
+`self.settings.<setting_name>` or `cls.settings.<setting_name>` (new API)
+or `Utils.get_options()["<world>_options"]["<setting_name>"]` (deprecated).
 
-Users can set those in their `host.yaml` file.
+Users can set those in their `host.yaml` file. Some settings may automatically open a file browser if a file is missing.
+
+Refer to [settings api.md](https://github.com/ArchipelagoMW/Archipelago/blob/main/docs/settings%20api.md)
+for details.
 
 ### Locations
 
@@ -111,8 +116,8 @@ World-specific IDs are 1 to 2<sup>53</sup>-1, IDs ≤ 0 are global and reserved.
 Special locations with ID `None` can hold events.
 
 Classification is one of `LocationProgressType.DEFAULT`, `PRIORITY` or `EXCLUDED`.
-The Fill algorithm will fill priority first, giving higher chance of it being
-required, and not place progression or useful items in excluded locations.
+The Fill algorithm will force progression items to be placed at priority locations, giving a higher chance of them being
+required, and will prevent progression and useful items from being placed at excluded locations.
 
 ### Items
 
@@ -132,10 +137,13 @@ same ID. Name must not be numeric (has to contain at least 1 letter or symbol).
 Special items with ID `None` can mark events (read below).
 
 Other classifications include
-* filler: a regular item or trash item
-* useful: generally quite useful, but not required for anything logical
-* trap: negative impact on the player
-* skip_balancing: add to progression to skip balancing; e.g. currency or tokens
+* `filler`: a regular item or trash item
+* `useful`: generally quite useful, but not required for anything logical
+* `trap`: negative impact on the player
+* `skip_balancing`: denotes that an item should not be moved to an earlier sphere for the purpose of balancing (to be
+  combined with `progression`; see below)
+* `progression_skip_balancing`: the combination of `progression` and `skip_balancing`, i.e., a progression item that
+  will not be moved around by progression balancing; used, e.g., for currency or tokens
 
 ### Events
 
@@ -159,10 +167,10 @@ or more event locations based on player options.
 
 Regions are logical groups of locations that share some common access rules. If
 location logic is written from scratch, using regions greatly simplifies the
-definition and allow to somewhat easily implement things like entrance
+definition and allows to somewhat easily implement things like entrance
 randomizer in logic.
 
-Regions have a list called `exits` which are `Entrance` objects representing
+Regions have a list called `exits`, which are `Entrance` objects representing
 transitions to other regions.
 
 There has to be one special region "Menu" from which the logic unfolds. AP
@@ -179,7 +187,7 @@ They can be static (regular logic) or be defined/connected during generation
 ### Access Rules
 
 An access rule is a function that returns `True` or `False` for a `Location` or
-`Entrance` based on the the current `state` (items that can be collected).
+`Entrance` based on the current `state` (items that can be collected).
 
 ### Item Rules
 
@@ -192,18 +200,18 @@ on a single item. It can be used to reject placement of an item there.
 ### Your World
 
 All code for your world implementation should be placed in a python package in
-the `/worlds` directory. The starting point for the package is `__init.py__`.
+the `/worlds` directory. The starting point for the package is `__init__.py`.
 Conventionally, your world class is placed in that file.
 
 World classes must inherit from the `World` class in `/worlds/AutoWorld.py`,
-which can be imported as `worlds.AutoWorld.World` from your package.
+which can be imported as `from worlds.AutoWorld import World` from your package.
 
 AP will pick up your world automatically due to the `AutoWorld` implementation.
 
 ### Requirements
 
 If your world needs specific python packages, they can be listed in
-`world/[world_name]/requirements.txt`. ModuleUpdate.py will automatically
+`worlds/<world_name>/requirements.txt`. ModuleUpdate.py will automatically
 pick up and install them.
 
 See [pip documentation](https://pip.pypa.io/en/stable/cli/pip_install/#requirements-file-format).
@@ -214,7 +222,7 @@ AP will only import the `__init__.py`. Depending on code size it makes sense to
 use multiple files and use relative imports to access them.
 
 e.g. `from .Options import mygame_options` from your `__init__.py` will load
-`world/[world_name]/Options.py` and make its `mygame_options` accesible.
+`worlds/<world_name>/Options.py` and make its `mygame_options` accessible.
 
 When imported names pile up it may be easier to use `from . import Options`
 and access the variable as `Options.mygame_options`.
@@ -225,12 +233,12 @@ function, see [apworld specification.md](apworld%20specification.md).
 
 ### Your Item Type
 
-Each world uses its own subclass of `BaseClasses.Item`. The constuctor can be
+Each world uses its own subclass of `BaseClasses.Item`. The constructor can be
 overridden to attach additional data to it, e.g. "price in shop".
 Since the constructor is only ever called from your code, you can add whatever
 arguments you like to the constructor.
 
-In its simplest form we only set the game name and use the default constuctor
+In its simplest form we only set the game name and use the default constructor
 ```python
 from BaseClasses import Item
 
@@ -265,7 +273,7 @@ Each option has its own class, inherits from a base option type, has a docstring
 to describe it and a `display_name` property for display on the website and in
 spoiler logs.
 
-The actual name as used in the yaml is defined in a `dict[str, Option]`, that is
+The actual name as used in the yaml is defined in a `Dict[str, AssembleOptions]`, that is
 assigned to the world under `self.option_definitions`.
 
 Common option types are `Toggle`, `DefaultOnToggle`, `Choice`, `Range`.
@@ -326,10 +334,10 @@ class FixXYZGlitch(Toggle):
     display_name = "Fix XYZ Glitch"
 
 # By convention we call the options dict variable `<world>_options`.
-mygame_options: typing.Dict[str, type(Option)] = {
+mygame_options: typing.Dict[str, AssembleOptions] = {
     "difficulty": Difficulty,
     "final_boss_hp": FinalBossHP,
-    "fix_xyz_glitch": FixXYZGlitch
+    "fix_xyz_glitch": FixXYZGlitch,
 }
 ```
 ```python
@@ -349,27 +357,39 @@ class MyGameWorld(World):
 ```python
 # world/mygame/__init__.py
 
+import settings
+import typing
 from .Options import mygame_options  # the options we defined earlier
 from .Items import mygame_items  # data used below to add items to the World
 from .Locations import mygame_locations  # same as above
 from worlds.AutoWorld import World
 from BaseClasses import Region, Location, Entrance, Item, RegionType, ItemClassification
-from Utils import get_options, output_path
+
 
 class MyGameItem(Item):  # or from Items import MyGameItem
     game = "My Game"  # name of the game/world this item is from
 
+
 class MyGameLocation(Location):  # or from Locations import MyGameLocation
     game = "My Game"  # name of the game/world this location is in
 
+
+class MyGameSettings(settings.Group):
+    class RomFile(settings.SNESRomPath):
+        """Insert help text for host.yaml here."""
+
+    rom_file: RomFile = RomFile("MyGame.sfc")
+
+
 class MyGameWorld(World):
     """Insert description of the world/game here."""
     game = "My Game"  # name of the game/world
     option_definitions = mygame_options  # options the player can set
+    settings: typing.ClassVar[MyGameSettings]  # will be automatically assigned from type hint
     topology_present = True  # show path to required location checks in spoiler
 
     # ID of first item and location, could be hard-coded but code may be easier
-    # to read with this as a propery.
+    # to read with this as a property.
     base_id = 1234
     # Instead of dynamic numbering, IDs could be part of data.
 
@@ -384,7 +404,7 @@ class MyGameWorld(World):
     # Items can be grouped using their names to allow easy checking if any item
     # from that group has been collected. Group names can also be used for !hint
     item_name_groups = {
-        "weapons": {"sword", "lance"}
+        "weapons": {"sword", "lance"},
     }
 ```
 
@@ -398,7 +418,7 @@ The world has to provide the following things for generation
 * locations placed inside those regions
 * a `def create_item(self, item: str) -> MyGameItem` to create any item on demand
 * applying `self.multiworld.push_precollected` for start inventory
-* `required_client_version: Tuple(int, int, int)`
+* `required_client_version: Tuple[int, int, int]`
   Optional client version as tuple of 3 ints to make sure the client is compatible to
   this world (e.g. implements all required features) when connecting.
 
@@ -496,30 +516,28 @@ def create_items(self) -> None:
 def create_regions(self) -> None:
     # Add regions to the multiworld. "Menu" is the required starting point.
     # Arguments to Region() are name, player, world, and optionally hint_text
-    r = Region("Menu", self.player, self.multiworld)
-    # Set Region.exits to a list of entrances that are reachable from region
-    r.exits = [Entrance(self.player, "New game", r)]  # or use r.exits.append
-    # Append region to MultiWorld's regions
-    self.multiworld.regions.append(r)  # or use += [r...]
+    menu_region = Region("Menu", self.player, self.multiworld)
+    self.multiworld.regions.append(menu_region)  # or use += [menu_region...]
     
-    r = Region("Main Area", self.player, self.multiworld)
+    main_region = Region("Main Area", self.player, self.multiworld)
     # Add main area's locations to main area (all but final boss)
-    r.locations = [MyGameLocation(self.player, location.name,
-                   self.location_name_to_id[location.name], r)]
-    r.exits = [Entrance(self.player, "Boss Door", r)]
-    self.multiworld.regions.append(r)
+    main_region.add_locations(main_region_locations, MyGameLocation)
+    # or 
+    # main_region.locations = \
+    #   [MyGameLocation(self.player, location_name, self.location_name_to_id[location_name], main_region]
+    self.multiworld.regions.append(main_region)
     
-    r = Region("Boss Room", self.player, self.multiworld)
-    # add event to Boss Room
-    r.locations = [MyGameLocation(self.player, "Final Boss", None, r)]
-    self.multiworld.regions.append(r)
-    
-    # If entrances are not randomized, they should be connected here, otherwise
-    # they can also be connected at a later stage.
-    self.multiworld.get_entrance("New Game", self.player)
-        .connect(self.multiworld.get_region("Main Area", self.player))
-    self.multiworld.get_entrance("Boss Door", self.player)
-        .connect(self.multiworld.get_region("Boss Room", self.player))
+    boss_region = Region("Boss Room", self.player, self.multiworld)
+    # Add event to Boss Room
+    boss_region.locations.append(MyGameLocation(self.player, "Final Boss", None, boss_region))
+
+    # If entrances are not randomized, they should be connected here,
+    # otherwise they can also be connected at a later stage.
+    # Create Entrances and connect the Regions
+    menu_region.connect(main_region)  # connects the "Menu" and "Main Area", can also pass a rule
+    # or
+    main_region.add_exits({"Boss Room": "Boss Door"}, {"Boss Room": lambda state: state.has("Sword", self.player)})
+    # Connects the "Main Area" and "Boss Room" regions, and places a rule requiring the "Sword" item to traverse
     
     # If setting location access rules from data is easier here, set_rules can
     # possibly omitted.
@@ -573,7 +591,7 @@ def set_rules(self) -> None:
     # require one item from an item group
     add_rule(self.multiworld.get_location("Chest3", self.player),
              lambda state: state.has_group("weapons", self.player))
-    # state also has .item_count() for items, .has_any() and.has_all() for sets
+    # state also has .item_count() for items, .has_any() and .has_all() for sets
     # and .count_group() for groups
     # set_rule is likely to be a bit faster than add_rule
 
@@ -611,7 +629,7 @@ public members with `mygame_`.
 More advanced uses could be to add additional variables to the state object,
 override `World.collect(self, state, item)` and `remove(self, state, item)`
 to update the state object, and check those added variables in added methods.
-Please do this with caution and only when neccessary.
+Please do this with caution and only when necessary.
 
 #### Sample
 
@@ -623,7 +641,7 @@ from worlds.AutoWorld import LogicMixin
 class MyGameLogic(LogicMixin):
     def mygame_has_key(self, player: int):
         # Arguments above are free to choose
-        # MultiWorld can be accessed through self.world, explicitly passing in
+        # MultiWorld can be accessed through self.multiworld, explicitly passing in
         # MyGameWorld instance for easy options access is also a valid approach
         return self.has("key", player)  # or whatever
 ```
@@ -636,8 +654,8 @@ import .Logic  # apply the mixin by importing its file
 class MyGameWorld(World):
     # ...
     def set_rules(self):
-        set_rule(self.world.get_location("A Door", self.player),
-                 lamda state: state.mygame_has_key(self.player))
+        set_rule(self.multiworld.get_location("A Door", self.player),
+                 lambda state: state.mygame_has_key(self.player))
 ```
 
 ### Generate Output
@@ -665,14 +683,14 @@ def generate_output(self, output_directory: str):
         # store option name "easy", "normal" or "hard" for difficuly
         "difficulty": self.multiworld.difficulty[self.player].current_key,
         # store option value True or False for fixing a glitch
-        "fix_xyz_glitch": self.multiworld.fix_xyz_glitch[self.player].value
+        "fix_xyz_glitch": self.multiworld.fix_xyz_glitch[self.player].value,
     }
     # point to a ROM specified by the installation
-    src = Utils.get_options()["mygame_options"]["rom_file"]
+    src = self.settings.rom_file
     # or point to worlds/mygame/data/mod_template
     src = os.path.join(os.path.dirname(__file__), "data", "mod_template")
     # generate output path
-    mod_name = f"AP-{self.multiworld.seed_name}-P{self.player}-{self.multiworld.player_name[self.player]}"
+    mod_name = self.multiworld.get_out_file_name_base(self.player)
     out_file = os.path.join(output_directory, mod_name + ".zip")
     # generate the file
     generate_mod(src, out_file, data)
@@ -721,14 +739,14 @@ from . import MyGameTestBase
 
 
 class TestChestAccess(MyGameTestBase):
-    def testSwordChests(self):
+    def test_sword_chests(self):
         """Test locations that require a sword"""
         locations = ["Chest1", "Chest2"]
         items = [["Sword"]]
         # this will test that each location can't be accessed without the "Sword", but can be accessed once obtained.
         self.assertAccessDependency(locations, items)
-    
-    def testAnyWeaponChests(self):
+
+    def test_any_weapon_chests(self):
         """Test locations that require any weapon"""
         locations = [f"Chest{i}" for i in range(3, 6)]
         items = [["Sword"], ["Axe"], ["Spear"]]

docs/world maintainer.md

@@ -0,0 +1,60 @@
+# World Maintainer
+
+A world maintainer is a person responsible for a world or part of a world in Archipelago.
+
+If a world author does not want to take on the responsibilities of a world maintainer, they can release their world as
+an unofficial [APWorld](/docs/apworld%20specification.md) or maintain their own fork instead.
+
+All current world maintainers are listed in the [CODEOWNERS](/docs/CODEOWNERS) document.
+
+## Responsibilities
+
+Unless these are shared between multiple people, we expect the following from each world maintainer
+
+* Be on our Discord to get updates on problems with and suggestions for the world.
+* Decide if a feature (pull request) should be merged.
+* Review contents of such pull requests or organize peer reviews or post that you did not review the content.
+* Fix or point out issues when core changes break your code.
+* Use the watch function on GitHub, the #github-updates channel on Discord or check manually from time to time for new
+  pull requests. Core maintainers may also ping you if a pull request concerns your world.
+* Test (or have tested) the world on the main branch from time to time, especially during RC (release candidate) phases
+  of development.
+* Let us know of long periods of unavailability.
+
+## Becoming a World Maintainer
+
+### Adding a World
+
+When we merge your world into the core Archipelago repository, you automatically become world maintainer unless you
+nominate someone else (i.e. there are multiple devs). You can define who is allowed to approve changes to your world
+in the [CODEOWNERS](/docs/CODEOWNERS) document.
+
+### Getting Voted
+
+When a world is unmaintained, the [core maintainers](https://github.com/orgs/ArchipelagoMW/people)
+can vote for a new maintainer if there is a candidate.
+For a vote to pass, the majority of participating core maintainers must vote in the affirmative.
+The time limit is 1 week, but can end early if the majority is reached earlier.
+Voting shall be conducted on Discord in #archipelago-dev.
+
+## Dropping out
+
+### Resigning
+
+A world maintainer can resign and have their username removed from the [CODEOWNERS](/docs/CODEOWNERS) document. If no
+new maintainer takes over management of the world, the world becomes unmaintained.
+
+### Getting Voted out
+
+A world maintainer can be voted out by the [core maintainers](https://github.com/orgs/ArchipelagoMW/people),
+for example when they become unreachable.
+For a vote to pass, the majority of participating core maintainers must vote in the affirmative.
+The time limit is 2 weeks, but can end early if the majority is reached earlier AND the world maintainer was pinged and
+made their case or was pinged and has been unreachable for more than 2 weeks already.
+Voting shall be conducted on Discord in #archipelago-dev. Commits that are a direct result of the voting shall include
+date, voting members and final result in the commit message.
+
+## Handling of Unmaintained Worlds
+
+As long as worlds are known to work for the most part, they can stay included. Once a world becomes broken it shall be
+moved from `worlds/` to `worlds_disabled/`.

host.yaml

@@ -1,191 +0,0 @@
-general_options:
-  # Where to place output files
-  output_path: "output"
-# Options for MultiServer
-# Null means nothing, for the server this means to default the value
-# These overwrite command line arguments!
-server_options:
-  host: null
-  port: 38281
-  password: null
-  multidata: null
-  savefile: null
-  disable_save: false
-  loglevel: "info"
-  # Allows for clients to log on and manage the server.  If this is null, no remote administration is possible.
-  server_password: null
-  # Disallow !getitem.
-  disable_item_cheat: false
-  # Client hint system
-  # Points given to a player for each acquired item in their world
-  location_check_points: 1
-  # Relative point cost to receive a hint via !hint for players
-  # so for example hint_cost: 20 would mean that for every 20% of available checks, you get the ability to hint, for a total of 5
-  hint_cost: 10 # Set to 0 if you want free hints
-  # Release modes
-  # A Release sends out the remaining items *from* a world that releases
-  # "disabled" -> clients can't release,
-  # "enabled" -> clients can always release
-  # "auto" -> automatic release on goal completion
-  # "auto-enabled" -> automatic release on goal completion and manual release is also enabled
-  # "goal" -> release is allowed after goal completion
-  release_mode: "goal"
-  # Collect modes
-  # A Collect sends the remaining items *to* a world that collects
-  # "disabled" -> clients can't collect,
-  # "enabled" -> clients can always collect
-  # "auto" -> automatic collect on goal completion
-  # "auto-enabled" -> automatic collect on goal completion and manual collect is also enabled
-  # "goal" -> collect is allowed after goal completion
-  collect_mode: "goal"
-  # Remaining modes
-  # !remaining handling, that tells a client which items remain in their pool
-  # "enabled" -> Client can always ask for remaining items
-  # "disabled" -> Client can never ask for remaining items
-  # "goal" -> Client can ask for remaining items after goal completion
-  remaining_mode: "goal"
-  # Automatically shut down the server after this many seconds without new location checks, 0 to keep running
-  auto_shutdown: 0
-  # Compatibility handling
-  # 2 -> Recommended for casual/cooperative play, attempt to be compatible with everything across all versions
-  # 1 -> No longer in use, kept reserved in case of future use
-  # 0 -> Recommended for tournaments to force a level playing field, only allow an exact version match
-  compatibility: 2
-  # log all server traffic, mostly for dev use
-  log_network: 0
-# Options for Generation
-generator:
-  # Location of your Enemizer CLI, available here: https://github.com/Ijwu/Enemizer/releases
-  enemizer_path: "EnemizerCLI/EnemizerCLI.Core" # + ".exe" is implied on Windows
-  # Folder from which the player yaml files are pulled from
-  player_files_path: "Players"
-  #amount of players, 0 to infer from player files
-  players: 0
-  # general weights file, within the stated player_files_path location
-  # gets used if players is higher than the amount of per-player files found to fill remaining slots
-  weights_file_path: "weights.yaml"
-  # Meta file name, within the stated player_files_path location
-  meta_file_path: "meta.yaml"
-  # Create a spoiler file
-  # 0 -> None
-  # 1 -> Spoiler without playthrough or paths to playthrough required items
-  # 2 -> Spoiler with playthrough (viable solution to goals)
-  # 3 -> Spoiler with playthrough and traversal paths towards items
-  spoiler: 3
-  # Glitch to Triforce room from Ganon
-  # When disabled, you have to have a weapon that can hurt ganon (master sword or swordless/easy item functionality + hammer)
-  # and have completed the goal required for killing ganon to be able to access the triforce room.
-  # 1 -> Enabled.
-  # 0 -> Disabled (except in no-logic)
-  glitch_triforce_room: 1
-  # Create encrypted race roms and flag games as race mode
-  race: 0
-  # List of options that can be plando'd. Can be combined, for example "bosses, items"
-  # Available options: bosses, items, texts, connections
-  plando_options: "bosses"
-sni_options:
-  # Set this to your SNI folder location if you want the MultiClient to attempt an auto start, does nothing if not found
-  sni_path: "SNI"
-  # Set this to false to never autostart a rom (such as after patching)
-  # True for operating system default program
-  # Alternatively, a path to a program to open the .sfc file with
-  snes_rom_start: true
-lttp_options:
-  # File name of the v1.0 J rom
-  rom_file: "Zelda no Densetsu - Kamigami no Triforce (Japan).sfc"
-ladx_options:
-  # File name of the Link's Awakening DX rom
-  rom_file: "Legend of Zelda, The - Link's Awakening DX (USA, Europe) (SGB Enhanced).gbc"
-
-lufia2ac_options:
-  # File name of the US rom
-  rom_file: "Lufia II - Rise of the Sinistrals (USA).sfc"
-sm_options:
-  # File name of the v1.0 J rom
-  rom_file: "Super Metroid (JU).sfc"
-factorio_options:
-  executable: "factorio/bin/x64/factorio"
-  # by default, no settings are loaded if this file does not exist.  If this file does exist, then it will be used.
-  # server_settings: "factorio\\data\\server-settings.json"
-  # Whether to filter item send messages displayed in-game to only those that involve you.
-  filter_item_sends: false
-  # Whether to send chat messages from players on the Factorio server to Archipelago.
-  bridge_chat_out: true
-minecraft_options:
-  forge_directory: "Minecraft Forge server"
-  max_heap_size: "2G"
-  # release channel, currently "release", or "beta"
-  # any games played on the "beta" channel have a high likelihood of no longer working on the "release" channel.
-  release_channel: "release"
-oot_options:
-  # File name of the OoT v1.0 ROM
-  rom_file: "The Legend of Zelda - Ocarina of Time.z64"
-  # Set this to false to never autostart a rom (such as after patching)
-  #             true  for operating system default program
-  # Alternatively, a path to a program to open the .z64 file with
-  rom_start: true
-soe_options:
-  # File name of the SoE US ROM
-  rom_file: "Secret of Evermore (USA).sfc"
-ffr_options:
-  display_msgs: true
-tloz_options:
-  # File name of the Zelda 1
-  rom_file: "Legend of Zelda, The (U) (PRG0) [!].nes"
-  # Set this to false to never autostart a rom (such as after patching)
-  #             true  for operating system default program
-  # Alternatively, a path to a program to open the .nes file with
-  rom_start: true
-  # Display message inside of Bizhawk
-  display_msgs: true
-dkc3_options:
-  # File name of the DKC3 US rom
-  rom_file: "Donkey Kong Country 3 - Dixie Kong's Double Trouble! (USA) (En,Fr).sfc"
-smw_options:
-  # File name of the SMW US rom
-  rom_file: "Super Mario World (USA).sfc"
-pokemon_rb_options:
-  # File names of the Pokemon Red and Blue roms
-  red_rom_file: "Pokemon Red (UE) [S][!].gb"
-  blue_rom_file: "Pokemon Blue (UE) [S][!].gb"
-  # Set this to false to never autostart a rom (such as after patching)
-  # True for operating system default program
-  # Alternatively, a path to a program to open the .gb file with
-  rom_start: true
-
-wargroove_options:
-  # Locate the Wargroove root directory on your system.
-  # This is used by the Wargroove client, so it knows where to send communication files to
-  root_directory: "C:/Program Files (x86)/Steam/steamapps/common/Wargroove"
-
-zillion_options:
-  # File name of the Zillion US rom
-  rom_file: "Zillion (UE) [!].sms"
-  # Set this to false to never autostart a rom (such as after patching)
-  # True for operating system default program
-  # Alternatively, a path to a program to open the .sfc file with
-  # RetroArch doesn't make it easy to launch a game from the command line.
-  # You have to know the path to the emulator core library on the user's computer.
-  rom_start: "retroarch"
-
-adventure_options:
-  # File name of the standard NTSC Adventure rom.
-  # The licensed "The 80 Classic Games" CD-ROM contains this.
-  # It may also have a .a26 extension
-  rom_file: "ADVNTURE.BIN"
-  # Set this to false to never autostart a rom (such as after patching)
-  # True for operating system default program for '.a26'
-  # Alternatively, a path to a program to open the .a26 file with (generally EmuHawk for multiworld)
-  rom_start: true
-  # Optional, additional args passed into rom_start before the .bin file
-  # For example, this can be used to autoload the connector script in BizHawk
-  # (see BizHawk --lua= option)
-  # Windows example:
-  # rom_args: "--lua=C:/ProgramData/Archipelago/data/lua/connector_adventure.lua"
-  rom_args: " "
-  # Set this to true to display item received messages in Emuhawk
-  display_msgs: true
-
-
-
-

inno_setup.iss

@@ -2,10 +2,10 @@
 #define min_windows ReadIni(SourcePath + "\setup.ini", "Data", "min_windows")
 
 #define MyAppName "Archipelago"
-#define MyAppExeName "ArchipelagoServer.exe"
+#define MyAppExeName "ArchipelagoLauncher.exe"
 #define MyAppIcon "data/icon.ico"
 #dim VersionTuple[4]
-#define MyAppVersion GetVersionComponents(source_path + '\ArchipelagoServer.exe', VersionTuple[0], VersionTuple[1], VersionTuple[2], VersionTuple[3])
+#define MyAppVersion GetVersionComponents(source_path + '\ArchipelagoLauncher.exe', VersionTuple[0], VersionTuple[1], VersionTuple[2], VersionTuple[3])
 #define MyAppVersionText Str(VersionTuple[0])+"."+Str(VersionTuple[1])+"."+Str(VersionTuple[2])
 
 
@@ -63,6 +63,7 @@ Name: "generator/oot";    Description: "Ocarina of Time ROM Setup"; Types: full
 Name: "generator/zl";     Description: "Zillion ROM Setup"; Types: full hosting; ExtraDiskSpaceRequired: 150000; Flags: disablenouninstallwarning
 Name: "generator/pkmn_r"; Description: "Pokemon Red ROM Setup"; Types: full hosting
 Name: "generator/pkmn_b"; Description: "Pokemon Blue ROM Setup"; Types: full hosting
+Name: "generator/mmbn3";  Description: "MegaMan Battle Network 3"; Types: full hosting; ExtraDiskSpaceRequired: 8388608; Flags: disablenouninstallwarning
 Name: "generator/ladx";   Description: "Link's Awakening DX ROM Setup"; Types: full hosting
 Name: "generator/tloz";   Description: "The Legend of Zelda ROM Setup"; Types: full hosting; ExtraDiskSpaceRequired: 135168; Flags: disablenouninstallwarning
 Name: "server";           Description: "Server"; Types: full hosting
@@ -81,6 +82,7 @@ Name: "client/ff1";       Description: "Final Fantasy 1"; Types: full playing
 Name: "client/pkmn";      Description: "Pokemon Client"
 Name: "client/pkmn/red";  Description: "Pokemon Client - Pokemon Red Setup"; Types: full playing; ExtraDiskSpaceRequired: 1048576
 Name: "client/pkmn/blue"; Description: "Pokemon Client - Pokemon Blue Setup"; Types: full playing; ExtraDiskSpaceRequired: 1048576
+Name: "client/mmbn3";     Description: "MegaMan Battle Network 3 Client"; Types: full playing;
 Name: "client/ladx";      Description: "Link's Awakening Client"; Types: full playing; ExtraDiskSpaceRequired: 1048576
 Name: "client/cf";        Description: "ChecksFinder"; Types: full playing
 Name: "client/sc2";       Description: "Starcraft 2"; Types: full playing
@@ -88,6 +90,7 @@ Name: "client/wargroove"; Description: "Wargroove"; Types: full playing
 Name: "client/zl";        Description: "Zillion"; Types: full playing
 Name: "client/tloz";      Description: "The Legend of Zelda"; Types: full playing
 Name: "client/advn";      Description: "Adventure"; Types: full playing
+Name: "client/ut";        Description: "Undertale"; Types: full playing
 Name: "client/text";      Description: "Text, to !command and chat"; Types: full playing
 
 [Dirs]
@@ -104,6 +107,7 @@ Source: "{code:GetOoTROMPath}"; DestDir: "{app}"; DestName: "The Legend of Zelda
 Source: "{code:GetZlROMPath}"; DestDir: "{app}"; DestName: "Zillion (UE) [!].sms"; Flags: external; Components: client/zl or generator/zl
 Source: "{code:GetRedROMPath}"; DestDir: "{app}"; DestName: "Pokemon Red (UE) [S][!].gb"; Flags: external; Components: client/pkmn/red or generator/pkmn_r
 Source: "{code:GetBlueROMPath}"; DestDir: "{app}"; DestName: "Pokemon Blue (UE) [S][!].gb"; Flags: external; Components: client/pkmn/blue or generator/pkmn_b
+Source: "{code:GetBN3ROMPath}"; DestDir: "{app}"; DestName: "Mega Man Battle Network 3 - Blue Version (USA).gba"; Flags: external; Components: client/mmbn3
 Source: "{code:GetLADXROMPath}"; DestDir: "{app}"; DestName: "Legend of Zelda, The - Link's Awakening DX (USA, Europe) (SGB Enhanced).gbc"; Flags: external; Components: client/ladx or generator/ladx
 Source: "{code:GetTLoZROMPath}"; DestDir: "{app}"; DestName: "Legend of Zelda, The (U) (PRG0) [!].nes"; Flags: external; Components: client/tloz or generator/tloz
 Source: "{code:GetAdvnROMPath}"; DestDir: "{app}"; DestName: "ADVNTURE.BIN"; Flags: external; Components: client/advn
@@ -112,6 +116,7 @@ Source: "{#source_path}\SNI\*"; Excludes: "*.sfc, *.log"; DestDir: "{app}\SNI";
 Source: "{#source_path}\EnemizerCLI\*"; Excludes: "*.sfc, *.log"; DestDir: "{app}\EnemizerCLI"; Flags: ignoreversion recursesubdirs createallsubdirs; Components: generator/lttp
 
 Source: "{#source_path}\ArchipelagoLauncher.exe"; DestDir: "{app}"; Flags: ignoreversion;
+Source: "{#source_path}\ArchipelagoLauncher(DEBUG).exe"; DestDir: "{app}"; Flags: ignoreversion;
 Source: "{#source_path}\ArchipelagoGenerate.exe"; DestDir: "{app}"; Flags: ignoreversion; Components: generator
 Source: "{#source_path}\ArchipelagoServer.exe"; DestDir: "{app}"; Flags: ignoreversion; Components: server
 Source: "{#source_path}\ArchipelagoFactorioClient.exe";  DestDir: "{app}"; Flags: ignoreversion; Components: client/factorio
@@ -127,15 +132,18 @@ Source: "{#source_path}\ArchipelagoFF1Client.exe"; DestDir: "{app}"; Flags: igno
 Source: "{#source_path}\ArchipelagoPokemonClient.exe"; DestDir: "{app}"; Flags: ignoreversion; Components: client/pkmn
 Source: "{#source_path}\ArchipelagoChecksFinderClient.exe"; DestDir: "{app}"; Flags: ignoreversion; Components: client/cf
 Source: "{#source_path}\ArchipelagoStarcraft2Client.exe"; DestDir: "{app}"; Flags: ignoreversion; Components: client/sc2
+Source: "{#source_path}\ArchipelagoMMBN3Client.exe"; DestDir: "{app}"; Flags: ignoreversion; Components: client/mmbn3
 Source: "{#source_path}\ArchipelagoZelda1Client.exe"; DestDir: "{app}"; Flags: ignoreversion; Components: client/tloz
 Source: "{#source_path}\ArchipelagoWargrooveClient.exe"; DestDir: "{app}"; Flags: ignoreversion; Components: client/wargroove
 Source: "{#source_path}\ArchipelagoKH2Client.exe"; DestDir: "{app}"; Flags: ignoreversion; Components: client/kh2
 Source: "{#source_path}\ArchipelagoAdventureClient.exe"; DestDir: "{app}"; Flags: ignoreversion; Components: client/advn
+Source: "{#source_path}\ArchipelagoUndertaleClient.exe"; DestDir: "{app}"; Flags: ignoreversion; Components: client/ut
 Source: "vc_redist.x64.exe"; DestDir: {tmp}; Flags: deleteafterinstall
 
 [Icons]
 Name: "{group}\{#MyAppName} Folder"; Filename: "{app}";
-Name: "{group}\{#MyAppName} Server"; Filename: "{app}\{#MyAppExeName}"; Components: server
+Name: "{group}\{#MyAppName} Launcher"; Filename: "{app}\ArchipelagoLauncher.exe"
+Name: "{group}\{#MyAppName} Server"; Filename: "{app}\ArchipelagoServer"; Components: server
 Name: "{group}\{#MyAppName} Text Client"; Filename: "{app}\ArchipelagoTextClient.exe"; Components: client/text
 Name: "{group}\{#MyAppName} SNI Client"; Filename: "{app}\ArchipelagoSNIClient.exe"; Components: client/sni
 Name: "{group}\{#MyAppName} Factorio Client"; Filename: "{app}\ArchipelagoFactorioClient.exe"; Components: client/factorio
@@ -146,13 +154,17 @@ Name: "{group}\{#MyAppName} Final Fantasy 1 Client"; Filename: "{app}\Archipelag
 Name: "{group}\{#MyAppName} Pokemon Client"; Filename: "{app}\ArchipelagoPokemonClient.exe"; Components: client/pkmn
 Name: "{group}\{#MyAppName} ChecksFinder Client"; Filename: "{app}\ArchipelagoChecksFinderClient.exe"; Components: client/cf
 Name: "{group}\{#MyAppName} Starcraft 2 Client"; Filename: "{app}\ArchipelagoStarcraft2Client.exe"; Components: client/sc2
+Name: "{group}\{#MyAppName} MegaMan Battle Network 3 Client"; Filename: "{app}\ArchipelagoMMBN3Client.exe"; Components: client/mmbn3
 Name: "{group}\{#MyAppName} The Legend of Zelda Client"; Filename: "{app}\ArchipelagoZelda1Client.exe"; Components: client/tloz
 Name: "{group}\{#MyAppName} Kingdom Hearts 2 Client"; Filename: "{app}\ArchipelagoKH2Client.exe"; Components: client/kh2
+Name: "{group}\{#MyAppName} Link's Awakening Client"; Filename: "{app}\ArchipelagoLinksAwakeningClient.exe"; Components: client/ladx
 Name: "{group}\{#MyAppName} Adventure Client"; Filename: "{app}\ArchipelagoAdventureClient.exe"; Components: client/advn
 Name: "{group}\{#MyAppName} Wargroove Client"; Filename: "{app}\ArchipelagoWargrooveClient.exe"; Components: client/wargroove
+Name: "{group}\{#MyAppName} Undertale Client"; Filename: "{app}\ArchipelagoUndertaleClient.exe"; Components: client/ut
 
 Name: "{commondesktop}\{#MyAppName} Folder"; Filename: "{app}"; Tasks: desktopicon
-Name: "{commondesktop}\{#MyAppName} Server"; Filename: "{app}\{#MyAppExeName}"; Tasks: desktopicon; Components: server
+Name: "{commondesktop}\{#MyAppName} Launcher"; Filename: "{app}\ArchipelagoLauncher.exe"; Tasks: desktopicon
+Name: "{commondesktop}\{#MyAppName} Server"; Filename: "{app}\ArchipelagoServer"; Tasks: desktopicon; Components: server
 Name: "{commondesktop}\{#MyAppName} SNI Client"; Filename: "{app}\ArchipelagoSNIClient.exe"; Tasks: desktopicon; Components: client/sni
 Name: "{commondesktop}\{#MyAppName} Factorio Client"; Filename: "{app}\ArchipelagoFactorioClient.exe"; Tasks: desktopicon; Components: client/factorio
 Name: "{commondesktop}\{#MyAppName} Minecraft Client"; Filename: "{app}\ArchipelagoMinecraftClient.exe"; Tasks: desktopicon; Components: client/minecraft
@@ -162,16 +174,21 @@ Name: "{commondesktop}\{#MyAppName} Final Fantasy 1 Client"; Filename: "{app}\Ar
 Name: "{commondesktop}\{#MyAppName} Pokemon Client"; Filename: "{app}\ArchipelagoPokemonClient.exe"; Tasks: desktopicon; Components: client/pkmn
 Name: "{commondesktop}\{#MyAppName} ChecksFinder Client"; Filename: "{app}\ArchipelagoChecksFinderClient.exe"; Tasks: desktopicon; Components: client/cf
 Name: "{commondesktop}\{#MyAppName} Starcraft 2 Client"; Filename: "{app}\ArchipelagoStarcraft2Client.exe"; Tasks: desktopicon; Components: client/sc2
+Name: "{commondesktop}\{#MyAppName} MegaMan Battle Network 3 Client"; Filename: "{app}\ArchipelagoMMBN3Client.exe"; Tasks: desktopicon; Components: client/mmbn3
 Name: "{commondesktop}\{#MyAppName} The Legend of Zelda Client"; Filename: "{app}\ArchipelagoZelda1Client.exe"; Tasks: desktopicon; Components: client/tloz
 Name: "{commondesktop}\{#MyAppName} Wargroove Client"; Filename: "{app}\ArchipelagoWargrooveClient.exe"; Tasks: desktopicon; Components: client/wargroove
 Name: "{commondesktop}\{#MyAppName} Kingdom Hearts 2 Client"; Filename: "{app}\ArchipelagoKH2Client.exe"; Tasks: desktopicon; Components: client/kh2
+Name: "{commondesktop}\{#MyAppName} Link's Awakening Client"; Filename: "{app}\ArchipelagoLinksAwakeningClient.exe"; Tasks: desktopicon; Components: client/ladx
 Name: "{commondesktop}\{#MyAppName} Adventure Client"; Filename: "{app}\ArchipelagoAdventureClient.exe"; Tasks: desktopicon; Components: client/advn
+Name: "{commondesktop}\{#MyAppName} Undertale Client"; Filename: "{app}\ArchipelagoUndertaleClient.exe"; Tasks: desktopicon; Components: client/ut
 
 [Run]
 
 Filename: "{tmp}\vc_redist.x64.exe"; Parameters: "/passive /norestart"; Check: IsVCRedist64BitNeeded; StatusMsg: "Installing VC++ redistributable..."
 Filename: "{app}\ArchipelagoLttPAdjuster"; Parameters: "--update_sprites"; StatusMsg: "Updating Sprite Library..."; Components: client/sni/lttp or generator/lttp
 Filename: "{app}\ArchipelagoMinecraftClient.exe"; Parameters: "--install"; StatusMsg: "Installing Forge Server..."; Components: client/minecraft
+Filename: "{app}\ArchipelagoLauncher"; Parameters: "--update_settings"; StatusMsg: "Updating host.yaml..."; Flags: runasoriginaluser runhidden
+Filename: "{app}\ArchipelagoLauncher"; Description: "{cm:LaunchProgram,{#StringChange('Launcher', '&', '&&')}}"; Flags: nowait postinstall skipifsilent
 
 [UninstallDelete]
 Type: dirifempty; Name: "{app}"
@@ -179,6 +196,8 @@ Type: dirifempty; Name: "{app}"
 [InstallDelete]
 Type: files; Name: "{app}\ArchipelagoLttPClient.exe"
 Type: filesandordirs; Name: "{app}\lib\worlds\rogue-legacy*"
+Type: filesandordirs; Name: "{app}\SNI\lua*"
+Type: filesandordirs; Name: "{app}\EnemizerCLI*"
 #include "installdelete.iss"
 
 [Registry]
@@ -243,6 +262,11 @@ Root: HKCR; Subkey: "{#MyAppName}pkmnbpatch";                     ValueData: "Ar
 Root: HKCR; Subkey: "{#MyAppName}pkmnbpatch\DefaultIcon";         ValueData: "{app}\ArchipelagoPokemonClient.exe,0";                           ValueType: string;  ValueName: ""; Components: client/pkmn
 Root: HKCR; Subkey: "{#MyAppName}pkmnbpatch\shell\open\command";  ValueData: """{app}\ArchipelagoPokemonClient.exe"" ""%1""";                  ValueType: string;  ValueName: ""; Components: client/pkmn
 
+Root: HKCR; Subkey: ".apbn3";                                     ValueData: "{#MyAppName}bn3bpatch";        Flags: uninsdeletevalue; ValueType: string;  ValueName: ""; Components: client/mmbn3
+Root: HKCR; Subkey: "{#MyAppName}bn3bpatch";                      ValueData: "Archipelago MegaMan Battle Network 3 Patch"; Flags: uninsdeletekey;   ValueType: string;  ValueName: ""; Components: client/mmbn3
+Root: HKCR; Subkey: "{#MyAppName}bn3bpatch\DefaultIcon";          ValueData: "{app}\ArchipelagoMMBN3Client.exe,0";                           ValueType: string;  ValueName: ""; Components: client/mmbn3
+Root: HKCR; Subkey: "{#MyAppName}bn3bpatch\shell\open\command";   ValueData: """{app}\ArchipelagoMMBN3Client.exe"" ""%1""";                  ValueType: string;  ValueName: ""; Components: client/mmbn3
+
 Root: HKCR; Subkey: ".apladx";                                    ValueData: "{#MyAppName}ladxpatch";        Flags: uninsdeletevalue; ValueType: string;  ValueName: ""; Components: client/ladx
 Root: HKCR; Subkey: "{#MyAppName}ladxpatch";                     ValueData: "Archipelago Links Awakening DX Patch"; Flags: uninsdeletekey;   ValueType: string;  ValueName: ""; Components: client/ladx
 Root: HKCR; Subkey: "{#MyAppName}ladxpatch\DefaultIcon";         ValueData: "{app}\ArchipelagoLinksAwakeningClient.exe,0";                           ValueType: string;  ValueName: ""; Components: client/ladx
@@ -325,6 +349,9 @@ var RedROMFilePage:  TInputFileWizardPage;
 var bluerom: string;
 var BlueROMFilePage:  TInputFileWizardPage;
 
+var bn3rom: string;
+var BN3ROMFilePage: TInputFileWizardPage;
+
 var ladxrom: string;
 var LADXROMFilePage:  TInputFileWizardPage;
 
@@ -444,6 +471,20 @@ begin
     '.gb');
 end;
 
+function AddGBARomPage(name: string): TInputFileWizardPage;
+begin
+  Result :=
+    CreateInputFilePage(
+      wpSelectComponents,
+      'Select ROM File',
+      'Where is your ' + name + ' located?',
+      'Select the file, then click Next.');
+  Result.Add(
+    'Location of ROM file:',
+    'GBA ROM files|*.gba|All files|*.*',
+    '.gba');
+end;
+
 function AddSMSRomPage(name: string): TInputFileWizardPage;
 begin
   Result :=
@@ -452,7 +493,6 @@ begin
       'Select ROM File',
       'Where is your ' + name + ' located?',
       'Select the file, then click Next.');
-
   Result.Add(
     'Location of ROM file:',
     'SMS ROM files|*.sms|All files|*.*',
@@ -535,6 +575,8 @@ begin
     Result := not (L2ACROMFilePage.Values[0] = '')
   else if (assigned(OoTROMFilePage)) and (CurPageID = OoTROMFilePage.ID) then
     Result := not (OoTROMFilePage.Values[0] = '')
+  else if (assigned(BN3ROMFilePage)) and (CurPageID = BN3ROMFilePage.ID) then
+    Result := not (BN3ROMFilePage.Values[0] = '')
   else if (assigned(ZlROMFilePage)) and (CurPageID = ZlROMFilePage.ID) then
     Result := not (ZlROMFilePage.Values[0] = '')
   else if (assigned(RedROMFilePage)) and (CurPageID = RedROMFilePage.ID) then
@@ -759,6 +801,22 @@ begin
     Result := '';
 end;
 
+function GetBN3ROMPath(Param: string): string;
+begin
+  if Length(bn3rom) > 0 then
+    Result := bn3rom
+  else if Assigned(BN3ROMFilePage) then
+    begin
+      R := CompareStr(GetMD5OfFile(BN3ROMFilePage.Values[0]), '6fe31df0144759b34ad666badaacc442')
+      if R <> 0 then
+        MsgBox('MegaMan Battle Network 3 Blue ROM validation failed. Very likely wrong file.', mbInformation, MB_OK);
+
+      Result := BN3ROMFilePage.Values[0]
+    end
+  else
+    Result := '';
+ end;
+
 procedure InitializeWizard();
 begin
   AddOoTRomPage();
@@ -795,6 +853,10 @@ begin
   if Length(bluerom) = 0 then
     BlueROMFilePage:= AddGBRomPage('Pokemon Blue (UE) [S][!].gb');
 
+  bn3rom := CheckRom('Mega Man Battle Network 3 - Blue Version (USA).gba','6fe31df0144759b34ad666badaacc442');
+  if Length(bn3rom) = 0 then
+    BN3ROMFilePage:= AddGBARomPage('Mega Man Battle Network 3 - Blue Version (USA).gba');
+  
   ladxrom := CheckRom('Legend of Zelda, The - Link''s Awakening DX (USA, Europe) (SGB Enhanced).gbc','07c211479386825042efb4ad31bb525f');
   if Length(ladxrom) = 0 then
     LADXROMFilePage:= AddGBRomPage('Legend of Zelda, The - Link''s Awakening DX (USA, Europe) (SGB Enhanced).gbc');
@@ -836,6 +898,8 @@ begin
     Result := not (WizardIsComponentSelected('generator/pkmn_r') or WizardIsComponentSelected('client/pkmn/red'));
   if (assigned(BlueROMFilePage)) and (PageID = BlueROMFilePage.ID) then
     Result := not (WizardIsComponentSelected('generator/pkmn_b') or WizardIsComponentSelected('client/pkmn/blue'));
+  if (assigned(BN3ROMFilePage)) and (PageID = BN3ROMFilePage.ID) then
+    Result := not (WizardIsComponentSelected('generator/mmbn3') or WizardIsComponentSelected('client/mmbn3'));
   if (assigned(LADXROMFilePage)) and (PageID = LADXROMFilePage.ID) then
     Result := not (WizardIsComponentSelected('generator/ladx') or WizardIsComponentSelected('client/ladx'));
   if (assigned(TLoZROMFilePage)) and (PageID = TLoZROMFilePage.ID) then

requirements.txt

@@ -1,9 +1,12 @@
 colorama>=0.4.5
-websockets>=11.0.1
-PyYAML>=6.0
-jellyfish>=0.11.2
+websockets>=11.0.3
+PyYAML>=6.0.1
+jellyfish>=1.0.0
 jinja2>=3.1.2
 schema>=0.7.5
-kivy>=2.1.0
+kivy>=2.2.0
 bsdiff4>=1.2.3
-platformdirs>=3.2.0
+platformdirs>=3.9.1
+certifi>=2023.7.22
+cython>=0.29.35
+cymem>=2.0.7

settings.py

@@ -0,0 +1,836 @@
+"""
+Application settings / host.yaml interface using type hints.
+This is different from player settings.
+"""
+
+import os.path
+import shutil
+import sys
+import typing
+import warnings
+from enum import IntEnum
+from threading import Lock
+from typing import cast, Any, BinaryIO, ClassVar, Dict, Iterator, List, Optional, TextIO, Tuple, Union, TypeVar
+import os
+
+__all__ = [
+    "get_settings", "fmt_doc", "no_gui",
+    "Group", "Bool", "Path", "UserFilePath", "UserFolderPath", "LocalFilePath", "LocalFolderPath",
+    "OptionalUserFilePath", "OptionalUserFolderPath", "OptionalLocalFilePath", "OptionalLocalFolderPath",
+    "GeneralOptions", "ServerOptions", "GeneratorOptions", "SNIOptions", "Settings"
+]
+
+no_gui = False
+skip_autosave = False
+_world_settings_name_cache: Dict[str, str] = {}  # TODO: cache on disk and update when worlds change
+_world_settings_name_cache_updated = False
+_lock = Lock()
+
+
+def _update_cache() -> None:
+    """Load all worlds and update world_settings_name_cache"""
+    global _world_settings_name_cache_updated
+    if _world_settings_name_cache_updated:
+        return
+
+    try:
+        from worlds.AutoWorld import AutoWorldRegister
+        for world in AutoWorldRegister.world_types.values():
+            annotation = world.__annotations__.get("settings", None)
+            if annotation is None or annotation == "ClassVar[Optional['Group']]":
+                continue
+            _world_settings_name_cache[world.settings_key] = f"{world.__module__}.{world.__name__}"
+    finally:
+        _world_settings_name_cache_updated = True
+
+
+def fmt_doc(cls: type, level: int) -> str:
+    comment = cls.__doc__
+    assert comment, f"{cls} has no __doc__"
+    indent = level * 2 * " "
+    return "\n".join(map(lambda s: f"{indent}# {s}", filter(None, map(lambda s: s.strip(), comment.split("\n")))))
+
+
+class Group:
+    _type_cache: ClassVar[Optional[Dict[str, Any]]] = None
+    _dumping: bool = False
+    _has_attr: bool = False
+    _changed: bool = False
+    _dumper: ClassVar[type]
+
+    def __getitem__(self, key: str) -> Any:
+        try:
+            return getattr(self, key)
+        except NameError:
+            raise KeyError(key)
+
+    def __iter__(self) -> Iterator[str]:
+        cls_members = dir(self.__class__)
+        members = filter(lambda k: not k.startswith("_") and (k not in cls_members or k in self.__annotations__),
+                         list(self.__annotations__) +
+                         [name for name in dir(self) if name not in self.__annotations__])
+        return members.__iter__()
+
+    def __contains__(self, key: str) -> bool:
+        try:
+            self._has_attr = True
+            return hasattr(self, key)
+        finally:
+            self._has_attr = False
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        setattr(self, key, value)
+
+    def __getattribute__(self, item: str) -> Any:
+        attr = super().__getattribute__(item)
+        if isinstance(attr, Path) and not super().__getattribute__("_dumping"):
+            if attr.required and not attr.exists() and not super().__getattribute__("_has_attr"):
+                # if a file is required, and the one from settings does not exist, ask the user to provide it
+                # unless we are dumping the settings, because that would ask for each entry
+                with _lock:  # lock to avoid opening multiple
+                    new = None if no_gui else attr.browse()
+                    if new is None:
+                        raise FileNotFoundError(f"{attr} does not exist, but "
+                                                f"{self.__class__.__name__}.{item} is required")
+                    setattr(self, item, new)
+                    self._changed = True
+                    attr = new
+            # resolve the path immediately when accessing it
+            return attr.__class__(attr.resolve())
+        return attr
+
+    @property
+    def changed(self) -> bool:
+        return self._changed or any(map(lambda v: isinstance(v, Group) and v.changed,
+                                        self.__dict__.values()))
+
+    @classmethod
+    def get_type_hints(cls) -> Dict[str, Any]:
+        """Returns resolved type hints for the class"""
+        if cls._type_cache is None:
+            if not isinstance(next(iter(cls.__annotations__.values())), str):
+                # non-str: assume already resolved
+                cls._type_cache = cls.__annotations__
+            else:
+                # str: build dicts and resolve with eval
+                mod = sys.modules[cls.__module__]  # assume the module wasn't deleted
+                mod_dict = {k: getattr(mod, k) for k in dir(mod)}
+                cls._type_cache = typing.get_type_hints(cls, globalns=mod_dict, localns=cls.__dict__)
+        return cls._type_cache
+
+    def get(self, key: str, default: Any) -> Any:
+        if key in self:
+            return self[key]
+        return default
+
+    def items(self) -> List[Tuple[str, Any]]:
+        return [(key, getattr(self, key)) for key in self]
+
+    def update(self, dct: Dict[str, Any]) -> None:
+        assert isinstance(dct, dict), f"{self.__class__.__name__}.update called with " \
+                                      f"{dct.__class__.__name__} instead of dict."
+
+        for k in self.__annotations__:
+            if not k.startswith("_") and k not in dct:
+                self._changed = True  # key missing from host.yaml
+
+        for k, v in dct.items():
+            # don't do getattr to stay lazy with world group init/loading
+            # instead we assign unknown groups as dicts and a later getattr will upcast them
+            attr = self.__dict__[k] if k in self.__dict__ else \
+                self.__class__.__dict__[k] if k in self.__class__.__dict__ else None
+            if isinstance(attr, Group):
+                # update group
+                if k not in self.__dict__:
+                    attr = attr.__class__()  # make a copy of default
+                    setattr(self, k, attr)
+                if isinstance(v, dict):
+                    attr.update(v)
+                else:
+                    warnings.warn(f"{self.__class__.__name__}.{k} "
+                                  f"tried to update Group from {type(v)}")
+            elif isinstance(attr, dict):
+                # update dict
+                if k not in self.__dict__:
+                    attr = attr.copy()  # make a copy of default
+                    setattr(self, k, attr)
+                if isinstance(v, dict):
+                    attr.update(v)
+                else:
+                    warnings.warn(f"{self.__class__.__name__}.{k} "
+                                  f"tried to update dict from {type(v)}")
+            else:
+                # assign value, try to upcast to type hint
+                annotation = self.get_type_hints().get(k, None)
+                candidates = [] if annotation is None else \
+                    typing.get_args(annotation) if typing.get_origin(annotation) is Union else [annotation]
+                none_type = type(None)
+                for cls in candidates:
+                    assert isinstance(cls, type), f"{self.__class__.__name__}.{k}: type {cls} not supported in settings"
+                    if v is None and cls is none_type:
+                        # assign None, i.e. from Optional
+                        setattr(self, k, v)
+                        break
+                    if cls is bool and isinstance(v, bool):
+                        # assign bool - special handling because issubclass(int, bool) is True
+                        setattr(self, k, v)
+                        break
+                    if cls is not bool and issubclass(cls, type(v)):
+                        # upcast, i.e. int -> IntEnum, str -> Path
+                        setattr(self, k, cls.__call__(v))
+                        break
+                    if issubclass(cls, (tuple, set)) and isinstance(v, list):
+                        # convert or upcast from list
+                        setattr(self, k, cls.__call__(v))
+                        break
+                else:
+                    # assign scalar and hope for the best
+                    setattr(self, k, v)
+                    if annotation:
+                        warnings.warn(f"{self.__class__.__name__}.{k} "
+                                      f"assigned from incompatible type {type(v).__name__}")
+
+    def as_dict(self, *args: str, downcast: bool = True) -> Dict[str, Any]:
+        return {
+            name: _to_builtin(cast(object, getattr(self, name))) if downcast else getattr(self, name)
+            for name in self if not args or name in args
+        }
+
+    @classmethod
+    def _dump_value(cls, value: Any, f: TextIO, indent: str) -> None:
+        """Write a single yaml line to f"""
+        from Utils import dump, Dumper as BaseDumper
+        yaml_line: str = dump(value, Dumper=cast(BaseDumper, cls._dumper))
+        assert yaml_line.count("\n") == 1, f"Unexpected input for yaml dumper: {value}"
+        f.write(f"{indent}{yaml_line}")
+
+    @classmethod
+    def _dump_item(cls, name: Optional[str], attr: object, f: TextIO, level: int) -> None:
+        """Write a group, dict or sequence item to f, where attr can be a scalar or a collection"""
+
+        # lazy construction of yaml Dumper to avoid loading Utils early
+        from Utils import Dumper as BaseDumper
+        from yaml import ScalarNode, MappingNode
+        if not hasattr(cls, "_dumper"):
+            if cls is Group or not hasattr(Group, "_dumper"):
+                class Dumper(BaseDumper):
+                    def represent_mapping(self, tag: str, mapping: Any, flow_style: Any = None) -> MappingNode:
+                        from yaml import ScalarNode
+                        res: MappingNode = super().represent_mapping(tag, mapping, flow_style)
+                        pairs = cast(List[Tuple[ScalarNode, Any]], res.value)
+                        for k, v in pairs:
+                            k.style = None  # remove quotes from keys
+                        return res
+
+                    def represent_str(self, data: str) -> ScalarNode:
+                        # default double quote all strings
+                        return self.represent_scalar("tag:yaml.org,2002:str", data, style='"')
+
+                Dumper.add_representer(str, Dumper.represent_str)
+                Group._dumper = Dumper
+            if cls is not Group:
+                cls._dumper = Group._dumper
+
+        indent = "  " * level
+        start = f"{indent}-\n" if name is None else f"{indent}{name}:\n"
+        if isinstance(attr, Group):
+            # handle group
+            f.write(start)
+            attr.dump(f, level=level+1)
+        elif isinstance(attr, (list, tuple, set)) and attr:
+            # handle non-empty sequence; empty use one-line [] syntax
+            f.write(start)
+            for value in attr:
+                cls._dump_item(None, value, f, level=level + 1)
+        elif isinstance(attr, dict) and attr:
+            # handle non-empty dict; empty use one-line {} syntax
+            f.write(start)
+            for dict_key, value in attr.items():
+                # not dumping doc string here, since there is no way to upcast it after dumping
+                assert dict_key is not None, "Key None is reserved for sequences"
+                cls._dump_item(dict_key, value, f, level=level + 1)
+        else:
+            # dump scalar or empty sequence or mapping item
+            line = [_to_builtin(attr)] if name is None else {name: _to_builtin(attr)}
+            cls._dump_value(line, f, indent=indent)
+
+    def dump(self, f: TextIO, level: int = 0) -> None:
+        """Dump Group to stream f at given indentation level"""
+        # There is no easy way to generate extra lines into default yaml output,
+        # so we format part of it by hand using an odd recursion here and in _dump_*.
+
+        self._dumping = True
+        try:
+            # fetch class to avoid going through getattr
+            cls = self.__class__
+            type_hints = cls.get_type_hints()
+            # validate group
+            for name in cls.__annotations__.keys():
+                assert hasattr(cls, name), f"{cls}.{name} is missing a default value"
+            # dump ordered members
+            for name in self:
+                attr = cast(object, getattr(self, name))
+                attr_cls = type_hints[name] if name in type_hints else attr.__class__
+                attr_cls_origin = typing.get_origin(attr_cls)
+                while attr_cls_origin is Union:  # resolve to first type for doc string
+                    attr_cls = typing.get_args(attr_cls)[0]
+                    attr_cls_origin = typing.get_origin(attr_cls)
+                if attr_cls.__doc__ and attr_cls.__module__ != "builtins":
+                    f.write(fmt_doc(attr_cls, level=level) + "\n")
+                self._dump_item(name, attr, f, level=level)
+            self._changed = False
+        finally:
+            self._dumping = False
+
+
+class Bool:
+    # can't subclass bool, so we use this and Union or type: ignore
+    def __bool__(self) -> bool:
+        raise NotImplementedError()
+
+
+# Types for generic settings
+T = TypeVar("T", bound="Path")
+
+
+def _resolve_exe(s: str) -> str:
+    """Append exe file extension if the file is an executable"""
+    if isinstance(s, Path):
+        from Utils import is_windows
+        if s.is_exe and is_windows and not s.lower().endswith(".exe"):
+            return str(s + ".exe")
+    return str(s)
+
+
+def _to_builtin(o: object) -> Any:
+    """Downcast object to a builtin type for output"""
+    if o is None:
+        return None
+    c = o.__class__
+    while c.__module__ != "builtins":
+        c = c.__base__
+    return c.__call__(o)
+
+
+class Path(str):
+    # paths in host.yaml are str
+    required: bool = True
+    """Marks the file as required and opens a file browser when missing"""
+    is_exe: bool = False
+    """Special cross-platform handling for executables"""
+    description: Optional[str] = None
+    """Title to display when browsing for the file"""
+    copy_to: Optional[str] = None
+    """If not None, copy to AP folder instead of linking it"""
+
+    @classmethod
+    def validate(cls, path: str) -> None:
+        """Overload and raise to validate input files from browse"""
+        pass
+
+    def browse(self: T, **kwargs: Any) -> Optional[T]:
+        """Opens a file browser to search for the file"""
+        raise NotImplementedError(f"Please use a subclass of Path for {self.__class__.__name__}")
+
+    def resolve(self) -> str:
+        return _resolve_exe(self)
+
+    def exists(self) -> bool:
+        return os.path.exists(self.resolve())
+
+
+class _UserPath(str):
+    def resolve(self) -> str:
+        if os.path.isabs(self):
+            return str(self)
+        from Utils import user_path
+        return user_path(_resolve_exe(self))
+
+
+class _LocalPath(str):
+    def resolve(self) -> str:
+        if os.path.isabs(self):
+            return str(self)
+        from Utils import local_path
+        return local_path(_resolve_exe(self))
+
+
+class FilePath(Path):
+    # path to a file
+
+    md5s: ClassVar[List[Union[str, bytes]]] = []
+    """MD5 hashes for default validator."""
+
+    def browse(self: T,
+               filetypes: Optional[typing.Sequence[typing.Tuple[str, typing.Sequence[str]]]] = None, **kwargs: Any)\
+            -> Optional[T]:
+        from Utils import open_filename, is_windows
+        if not filetypes:
+            if self.is_exe:
+                name, ext = "Program", ".exe" if is_windows else ""
+            else:
+                ext = os.path.splitext(self)[1]
+                name = ext[1:] if ext else "File"
+            filetypes = [(name, [ext])]
+        res = open_filename(f"Select {self.description or self.__class__.__name__}", filetypes, self)
+        if res:
+            self.validate(res)
+            if self.copy_to:
+                # instead of linking the file, copy it
+                dst = self.__class__(self.copy_to).resolve()
+                shutil.copy(res, dst, follow_symlinks=True)
+                res = dst
+            try:
+                rel = os.path.relpath(res, self.__class__("").resolve())
+                if not rel.startswith(".."):
+                    res = rel
+            except ValueError:
+                pass
+            return self.__class__(res)
+        return None
+
+    @classmethod
+    def _validate_stream_hashes(cls, f: BinaryIO) -> None:
+        """Helper to efficiently validate stream against hashes"""
+        if not cls.md5s:
+            return  # no hashes to validate against
+
+        pos = f.tell()
+        try:
+            from hashlib import md5
+            file_md5 = md5()
+            block = bytearray(64*1024)
+            view = memoryview(block)
+            while n := f.readinto(view):  # type: ignore
+                file_md5.update(view[:n])
+            file_md5_hex = file_md5.hexdigest()
+            for valid_md5 in cls.md5s:
+                if isinstance(valid_md5, str):
+                    if valid_md5.lower() == file_md5_hex:
+                        break
+                elif valid_md5 == file_md5.digest():
+                    break
+            else:
+                raise ValueError(f"Hashes do not match for {cls.__name__}")
+        finally:
+            f.seek(pos)
+
+    @classmethod
+    def validate(cls, path: str) -> None:
+        """Try to open and validate file against hashes"""
+        with open(path, "rb", buffering=0) as f:
+            try:
+                cls._validate_stream_hashes(f)
+            except ValueError:
+                raise ValueError(f"File hash does not match for {path}")
+
+
+class FolderPath(Path):
+    # path to a folder
+
+    def browse(self: T, **kwargs: Any) -> Optional[T]:
+        from Utils import open_directory
+        res = open_directory(f"Select {self.description or self.__class__.__name__}", self)
+        if res:
+            try:
+                rel = os.path.relpath(res, self.__class__("").resolve())
+                if not rel.startswith(".."):
+                    res = rel
+            except ValueError:
+                pass
+            return self.__class__(res)
+        return None
+
+
+class UserFilePath(_UserPath, FilePath):
+    pass
+
+
+class UserFolderPath(_UserPath, FolderPath):
+    pass
+
+
+class OptionalUserFilePath(UserFilePath):
+    required = False
+
+
+class OptionalUserFolderPath(UserFolderPath):
+    required = False
+
+
+class LocalFilePath(_LocalPath, FilePath):
+    pass
+
+
+class LocalFolderPath(_LocalPath, FolderPath):
+    pass
+
+
+class OptionalLocalFilePath(LocalFilePath):
+    required = False
+
+
+class OptionalLocalFolderPath(LocalFolderPath):
+    required = False
+
+
+class SNESRomPath(UserFilePath):
+    # Special UserFilePath that ignores an optional header when validating
+
+    @classmethod
+    def validate(cls, path: str) -> None:
+        """Try to open and validate file against hashes"""
+        with open(path, "rb", buffering=0) as f:
+            f.seek(0, os.SEEK_END)
+            size = f.tell()
+            if size % 1024 == 512:
+                f.seek(512)  # skip header
+            elif size % 1024 == 0:
+                f.seek(0)  # header-less
+            else:
+                raise ValueError(f"Unexpected file size for {path}")
+
+            try:
+                cls._validate_stream_hashes(f)
+            except ValueError:
+                raise ValueError(f"File hash does not match for {path}")
+
+
+# World-independent setting groups
+
+class GeneralOptions(Group):
+    class OutputPath(OptionalUserFolderPath):
+        """
+        Where to place output files
+        """
+        # created on demand, so marked as optional
+
+    output_path: OutputPath = OutputPath("output")
+
+
+class ServerOptions(Group):
+    """
+    Options for MultiServer
+    Null means nothing, for the server this means to default the value
+    These overwrite command line arguments!
+    """
+
+    class ServerPassword(str):
+        """
+        Allows for clients to log on and manage the server.  If this is null, no remote administration is possible.
+        """
+
+    class DisableItemCheat(Bool):
+        """Disallow !getitem"""
+
+    class LocationCheckPoints(int):
+        """
+        Client hint system
+        Points given to a player for each acquired item in their world
+        """
+
+    class HintCost(int):
+        """
+        Relative point cost to receive a hint via !hint for players
+        so for example hint_cost: 20 would mean that for every 20% of available checks, you get the ability to hint,
+        for a total of 5
+        """
+
+    class ReleaseMode(str):
+        """
+        Release modes
+        A Release sends out the remaining items *from* a world that releases
+        "disabled" -> clients can't release,
+        "enabled" -> clients can always release
+        "auto" -> automatic release on goal completion
+        "auto-enabled" -> automatic release on goal completion and manual release is also enabled
+        "goal" -> release is allowed after goal completion
+        """
+
+    class CollectMode(str):
+        """
+        Collect modes
+        A Collect sends the remaining items *to* a world that collects
+        "disabled" -> clients can't collect,
+        "enabled" -> clients can always collect
+        "auto" -> automatic collect on goal completion
+        "auto-enabled" -> automatic collect on goal completion and manual collect is also enabled
+        "goal" -> collect is allowed after goal completion
+        """
+
+    class RemainingMode(str):
+        """
+        Remaining modes
+        !remaining handling, that tells a client which items remain in their pool
+        "enabled" -> Client can always ask for remaining items
+        "disabled" -> Client can never ask for remaining items
+        "goal" -> Client can ask for remaining items after goal completion
+        """
+
+    class AutoShutdown(int):
+        """Automatically shut down the server after this many seconds without new location checks, 0 to keep running"""
+
+    class Compatibility(IntEnum):
+        """
+        Compatibility handling
+        2 -> Recommended for casual/cooperative play, attempt to be compatible with everything across all versions
+        1 -> No longer in use, kept reserved in case of future use
+        0 -> Recommended for tournaments to force a level playing field, only allow an exact version match
+        """
+        OFF = 0
+        ON = 1
+        FULL = 2
+
+    class LogNetwork(IntEnum):
+        """log all server traffic, mostly for dev use"""
+        OFF = 0
+        ON = 1
+
+    host: Optional[str] = None
+    port: int = 38281
+    password: Optional[str] = None
+    multidata: Optional[str] = None
+    savefile: Optional[str] = None
+    disable_save: bool = False
+    loglevel: str = "info"
+    server_password: Optional[ServerPassword] = None
+    disable_item_cheat: Union[DisableItemCheat, bool] = False
+    location_check_points: LocationCheckPoints = LocationCheckPoints(1)
+    hint_cost: HintCost = HintCost(10)
+    release_mode: ReleaseMode = ReleaseMode("goal")
+    collect_mode: CollectMode = CollectMode("goal")
+    remaining_mode: RemainingMode = RemainingMode("goal")
+    auto_shutdown: AutoShutdown = AutoShutdown(0)
+    compatibility: Compatibility = Compatibility(2)
+    log_network: LogNetwork = LogNetwork(0)
+
+
+class GeneratorOptions(Group):
+    """Options for Generation"""
+
+    class EnemizerPath(LocalFilePath):
+        """Location of your Enemizer CLI, available here: https://github.com/Ijwu/Enemizer/releases"""
+        is_exe = True
+
+    class PlayerFilesPath(OptionalUserFolderPath):
+        """Folder from which the player yaml files are pulled from"""
+        # created on demand, so marked as optional
+
+    class Players(int):
+        """amount of players, 0 to infer from player files"""
+
+    class WeightsFilePath(str):
+        """
+        general weights file, within the stated player_files_path location
+        gets used if players is higher than the amount of per-player files found to fill remaining slots
+        """
+        # this is special because the path is relative to player_files_path
+
+    class MetaFilePath(str):
+        """Meta file name, within the stated player_files_path location"""
+        # this is special because the path is relative to player_files_path
+
+    class Spoiler(IntEnum):
+        """
+        Create a spoiler file
+        0 -> None
+        1 -> Spoiler without playthrough or paths to playthrough required items
+        2 -> Spoiler with playthrough (viable solution to goals)
+        3 -> Spoiler with playthrough and traversal paths towards items
+        """
+        NONE = 0
+        BASIC = 1
+        PLAYTHROUGH = 2
+        FULL = 3
+
+    class GlitchTriforceRoom(IntEnum):
+        """
+        Glitch to Triforce room from Ganon
+        When disabled, you have to have a weapon that can hurt ganon (master sword or swordless/easy item functionality
+        + hammer) and have completed the goal required for killing ganon to be able to access the triforce room.
+        1 -> Enabled.
+        0 -> Disabled (except in no-logic)
+        """
+        OFF = 0
+        ON = 1
+
+    class PlandoOptions(str):
+        """
+        List of options that can be plando'd. Can be combined, for example "bosses, items"
+        Available options: bosses, items, texts, connections
+        """
+
+    class Race(IntEnum):
+        """Create encrypted race roms and flag games as race mode"""
+        OFF = 0
+        ON = 1
+
+    enemizer_path: EnemizerPath = EnemizerPath("EnemizerCLI/EnemizerCLI.Core")  # + ".exe" is implied on Windows
+    player_files_path: PlayerFilesPath = PlayerFilesPath("Players")
+    players: Players = Players(0)
+    weights_file_path: WeightsFilePath = WeightsFilePath("weights.yaml")
+    meta_file_path: MetaFilePath = MetaFilePath("meta.yaml")
+    spoiler: Spoiler = Spoiler(3)
+    glitch_triforce_room: GlitchTriforceRoom = GlitchTriforceRoom(1)  # why is this here?
+    race: Race = Race(0)
+    plando_options: PlandoOptions = PlandoOptions("bosses")
+
+
+class SNIOptions(Group):
+    class SNIPath(LocalFolderPath):
+        """
+        Set this to your SNI folder location if you want the MultiClient to attempt an auto start, \
+does nothing if not found
+        """
+
+    class SnesRomStart(str):
+        """
+        Set this to false to never autostart a rom (such as after patching)
+        True for operating system default program
+        Alternatively, a path to a program to open the .sfc file with
+        """
+
+    sni_path: SNIPath = SNIPath("SNI")
+    snes_rom_start: Union[SnesRomStart, bool] = True
+
+
+# Top-level group with lazy loading of worlds
+
+class Settings(Group):
+    general_options: GeneralOptions = GeneralOptions()
+    server_options: ServerOptions = ServerOptions()
+    generator: GeneratorOptions = GeneratorOptions()
+    sni_options: SNIOptions = SNIOptions()
+
+    _filename: Optional[str] = None
+
+    def __getattribute__(self, key: str) -> Any:
+        if key.startswith("_") or key in self.__class__.__dict__:
+            # not a group or a hard-coded group
+            pass
+        elif key not in dir(self) or isinstance(super().__getattribute__(key), dict):
+            # settings class not loaded yet
+            if key not in _world_settings_name_cache:
+                # find world that provides the settings class
+                _update_cache()
+                # check for missing keys to update _changed
+                for world_settings_name in _world_settings_name_cache:
+                    if world_settings_name not in dir(self):
+                        self._changed = True
+            if key not in _world_settings_name_cache:
+                # not a world group
+                return super().__getattribute__(key)
+            # directly import world and grab settings class
+            world_mod, world_cls_name = _world_settings_name_cache[key].rsplit(".", 1)
+            world = cast(type, getattr(__import__(world_mod, fromlist=[world_cls_name]), world_cls_name))
+            assert getattr(world, "settings_key") == key
+            try:
+                cls_or_name = world.__annotations__["settings"]
+            except KeyError:
+                import warnings
+                warnings.warn(f"World {world_cls_name} does not define settings. Please consider upgrading the world.")
+                return super().__getattribute__(key)
+            if isinstance(cls_or_name, str):
+                # Try to resolve type. Sadly we can't use get_type_hints, see https://bugs.python.org/issue43463
+                cls_name = cls_or_name
+                if "[" in cls_name:  # resolve ClassVar[]
+                    cls_name = cls_name.split("[", 1)[1].rsplit("]", 1)[0]
+                cls = cast(type, getattr(__import__(world_mod, fromlist=[cls_name]), cls_name))
+            else:
+                type_args = typing.get_args(cls_or_name)  # resolve ClassVar[]
+                cls = type_args[0] if type_args else cast(type, cls_or_name)
+            impl: Group = cast(Group, cls())
+            assert isinstance(impl, Group), f"{world_cls_name}.settings has to inherit from settings.Group. " \
+                                            "If that's already the case, please avoid recursive partial imports."
+            # above assert fails for recursive partial imports
+            # upcast loaded data to settings class
+            try:
+                dct = super().__getattribute__(key)
+                if isinstance(dct, dict):
+                    impl.update(dct)
+                else:
+                    self._changed = True  # key is a class var -> new section
+            except AttributeError:
+                self._changed = True  # key is unknown -> new section
+            setattr(self, key, impl)
+
+        return super().__getattribute__(key)
+
+    def __init__(self, location: Optional[str]):  # change to PathLike[str] once we drop 3.8?
+        super().__init__()
+        if location:
+            from Utils import parse_yaml
+            with open(location, encoding="utf-8-sig") as f:
+                options = parse_yaml(f.read())
+                # TODO: detect if upgrade is required
+                # TODO: once we have a cache for _world_settings_name_cache, detect if any game section is missing
+                self.update(options or {})
+            self._filename = location
+
+        def autosave() -> None:
+            if __debug__:
+                import __main__
+                main_file = getattr(__main__, "__file__", "")
+                assert "pytest" not in main_file and "unittest" not in main_file, \
+                       f"Auto-saving {self._filename} during unittests"
+            if self._filename and self.changed and not skip_autosave:
+                self.save()
+
+        if not skip_autosave:
+            import atexit
+            atexit.register(autosave)
+
+    def save(self, location: Optional[str] = None) -> None:  # as above
+        location = location or self._filename
+        assert location, "No file specified"
+        temp_location = location + ".tmp"  # not using tempfile to test expected file access
+        # remove old temps
+        if os.path.exists(temp_location):
+            os.unlink(temp_location)
+        # can't use utf-8-sig because it breaks backward compat: pyyaml on Windows with bytes does not strip the BOM
+        with open(temp_location, "w", encoding="utf-8") as f:
+            self.dump(f)
+        # replace old with new
+        if os.path.exists(location):
+            os.unlink(location)
+        os.rename(temp_location, location)
+        self._filename = location
+
+    def dump(self, f: TextIO, level: int = 0) -> None:
+        # load all world setting classes
+        _update_cache()
+        for key in _world_settings_name_cache:
+            self.__getattribute__(key)  # load all worlds
+        super().dump(f, level)
+
+    @property
+    def filename(self) -> Optional[str]:
+        return self._filename
+
+
+# host.yaml loader
+
+def get_settings() -> Settings:
+    """Returns settings from the default host.yaml"""
+    with _lock:  # make sure we only have one instance
+        res = getattr(get_settings, "_cache", None)
+        if not res:
+            import os
+            from Utils import user_path, local_path
+            filenames = ("options.yaml", "host.yaml")
+            locations: List[str] = []
+            if os.path.join(os.getcwd()) != local_path():
+                locations += filenames  # use files from cwd only if it's not the local_path
+            locations += [user_path(filename) for filename in filenames]
+            for location in locations:
+                try:
+                    res = Settings(location)
+                    break
+                except FileNotFoundError:
+                    continue
+            else:
+                warnings.warn(f"Could not find {filenames[1]} to load options. Creating a new one.")
+                res = Settings(None)
+                res.save(user_path(filenames[1]))
+            setattr(get_settings, "_cache", res)
+        return res

setup.py

@@ -6,6 +6,7 @@ import shutil
 import sys
 import sysconfig
 import typing
+import warnings
 import zipfile
 import urllib.request
 import io
@@ -20,7 +21,7 @@ from pathlib import Path
 
 # This is a bit jank. We need cx-Freeze to be able to run anything from this script, so install it
 try:
-    requirement = 'cx-Freeze>=6.14.7'
+    requirement = 'cx-Freeze>=6.15.2'
     import pkg_resources
     try:
         pkg_resources.require(requirement)
@@ -57,25 +58,39 @@ if __name__ == "__main__":
 
 from worlds.LauncherComponents import components, icon_paths
 from Utils import version_tuple, is_windows, is_linux
+from Cython.Build import cythonize
 
 
 # On  Python < 3.10 LogicMixin is not currently supported.
-apworlds: set = {
-    "Subnautica",
-    "Factorio",
-    "Rogue Legacy",
-    "Sonic Adventure 2 Battle",
-    "Donkey Kong Country 3",
-    "Super Mario World",
-    "Stardew Valley",
-    "Timespinner",
-    "Minecraft",
-    "The Messenger",
-    "Links Awakening DX",
-    "Super Metroid",
-    "SMZ3",
+non_apworlds: set = {
+    "A Link to the Past",
+    "Adventure",
+    "ArchipIDLE",
+    "Archipelago",
+    "ChecksFinder",
+    "Clique",
+    "DLCQuest",
+    "Final Fantasy",
+    "Hylics 2",
+    "Kingdom Hearts 2",
+    "Lufia II Ancient Cave",
+    "Meritous",
+    "Ocarina of Time",
+    "Overcooked! 2",
+    "Raft",
+    "Secret of Evermore",
+    "Slay the Spire",
+    "Starcraft 2 Wings of Liberty",
+    "Sudoku",
+    "Super Mario 64",
+    "VVVVVV",
+    "Wargroove",
+    "Zillion",
 }
 
+# LogicMixin is broken before 3.10 import revamp
+if sys.version_info < (3,10):
+    non_apworlds.add("Hollow Knight")
 
 def download_SNI():
     print("Updating SNI")
@@ -170,14 +185,23 @@ def resolve_icon(icon_name: str):
 
 exes = [
     cx_Freeze.Executable(
-        script=f'{c.script_name}.py',
+        script=f"{c.script_name}.py",
         target_name=c.frozen_name + (".exe" if is_windows else ""),
         icon=resolve_icon(c.icon),
         base="Win32GUI" if is_windows and not c.cli else None
     ) for c in components if c.script_name and c.frozen_name
 ]
 
-extra_data = ["LICENSE", "data", "EnemizerCLI", "host.yaml", "SNI"]
+if is_windows:
+    # create a duplicate Launcher for Windows, which has a working stdout/stderr, for debugging and --help
+    c = next(component for component in components if component.script_name == "Launcher")
+    exes.append(cx_Freeze.Executable(
+        script=f"{c.script_name}.py",
+        target_name=f"{c.frozen_name}(DEBUG).exe",
+        icon=resolve_icon(c.icon),
+    ))
+
+extra_data = ["LICENSE", "data", "EnemizerCLI", "SNI"]
 extra_libs = ["libssl.so", "libcrypto.so"] if is_linux else []
 
 
@@ -279,17 +303,38 @@ class BuildExeCommand(cx_Freeze.command.build_exe.BuildEXE):
         sni_thread = threading.Thread(target=download_SNI, name="SNI Downloader")
         sni_thread.start()
 
-        # pre build steps
+        # pre-build steps
         print(f"Outputting to: {self.buildfolder}")
         os.makedirs(self.buildfolder, exist_ok=True)
         import ModuleUpdate
         ModuleUpdate.requirements_files.add(os.path.join("WebHostLib", "requirements.txt"))
         ModuleUpdate.update(yes=self.yes)
 
+        # auto-build cython modules
+        build_ext = self.distribution.get_command_obj("build_ext")
+        build_ext.inplace = False
+        self.run_command("build_ext")
+        # find remains of previous in-place builds, try to delete and warn otherwise
+        for path in build_ext.get_outputs():
+            parts = os.path.split(path)[-1].split(".")
+            pattern = parts[0] + ".*." + parts[-1]
+            for match in Path().glob(pattern):
+                try:
+                    match.unlink()
+                    print(f"Removed {match}")
+                except Exception as ex:
+                    warnings.warn(f"Could not delete old build output: {match}\n"
+                                  f"{ex}\nPlease close all AP instances and delete manually.")
+
         # regular cx build
         self.buildtime = datetime.datetime.utcnow()
         super().run()
 
+        # manually copy built modules to lib folder. cx_Freeze does not know they exist.
+        for src in build_ext.get_outputs():
+            print(f"copying {src} -> {self.libfolder}")
+            shutil.copy(src, self.libfolder, follow_symlinks=False)
+
         # need to finish download before copying
         sni_thread.join()
 
@@ -322,11 +367,12 @@ class BuildExeCommand(cx_Freeze.command.build_exe.BuildEXE):
         os.makedirs(self.buildfolder / "Players" / "Templates", exist_ok=True)
         from Options import generate_yaml_templates
         from worlds.AutoWorld import AutoWorldRegister
-        assert not apworlds - set(AutoWorldRegister.world_types), "Unknown world designated for .apworld"
+        assert not non_apworlds - set(AutoWorldRegister.world_types), \
+            f"Unknown world {non_apworlds - set(AutoWorldRegister.world_types)} designated for .apworld"
         folders_to_remove: typing.List[str] = []
         generate_yaml_templates(self.buildfolder / "Players" / "Templates", False)
         for worldname, worldtype in AutoWorldRegister.world_types.items():
-            if worldname in apworlds:
+            if worldname not in non_apworlds:
                 file_name = os.path.split(os.path.dirname(worldtype.__file__))[1]
                 world_directory = self.libfolder / "worlds" / file_name
                 # this method creates an apworld that cannot be moved to a different OS or minor python version,
@@ -381,14 +427,6 @@ class BuildExeCommand(cx_Freeze.command.build_exe.BuildEXE):
             for extra_exe in extra_exes:
                 if extra_exe.is_file():
                     extra_exe.chmod(0o755)
-            # rewrite windows-specific things in host.yaml
-            host_yaml = self.buildfolder / 'host.yaml'
-            with host_yaml.open('r+b') as f:
-                data = f.read()
-                data = data.replace(b'factorio\\\\bin\\\\x64\\\\factorio', b'factorio/bin/x64/factorio')
-                f.seek(0, os.SEEK_SET)
-                f.write(data)
-                f.truncate()
 
 
 class AppImageCommand(setuptools.Command):
@@ -571,10 +609,10 @@ cx_Freeze.setup(
     version=f"{version_tuple.major}.{version_tuple.minor}.{version_tuple.build}",
     description="Archipelago",
     executables=exes,
-    ext_modules=[],  # required to disable auto-discovery with setuptools>=61
+    ext_modules=cythonize("_speedups.pyx"),
     options={
         "build_exe": {
-            "packages": ["websockets", "worlds", "kivy"],
+            "packages": ["worlds", "kivy", "cymem", "websockets"],
             "includes": [],
             "excludes": ["numpy", "Cython", "PySide2", "PIL",
                          "pandas"],

test/TestBase.py

@@ -237,7 +237,8 @@ class WorldTestBase(unittest.TestCase):
             for location in self.multiworld.get_locations():
                 if location.name not in excluded:
                     with self.subTest("Location should be reached", location=location):
-                        self.assertTrue(location.can_reach(state), f"{location.name} unreachable")
+                        reachable = location.can_reach(state)
+                        self.assertTrue(reachable, f"{location.name} unreachable")
             with self.subTest("Beatable"):
                 self.multiworld.state = state
                 self.assertBeatable(True)

test/__init__.py

@@ -1,3 +1,7 @@
 import warnings
-warnings.simplefilter("always")
 
+import settings
+
+warnings.simplefilter("always")
+settings.no_gui = True
+settings.skip_autosave = True

test/general/TestFill.py

@@ -199,6 +199,41 @@ class TestFillRestrictive(unittest.TestCase):
         # Unnecessary unreachable Item
         self.assertEqual(locations[1].item, items[0])
 
+    def test_minimal_mixed_fill(self):
+        """
+        Test that fill for 1 minimal and 1 non-minimal player will correctly place items in a way that lets
+        the non-minimal player get all items.
+        """
+
+        multi_world = generate_multi_world(2)
+        player1 = generate_player_data(multi_world, 1, 3, 3)
+        player2 = generate_player_data(multi_world, 2, 3, 3)
+
+        multi_world.accessibility[player1.id].value = multi_world.accessibility[player1.id].option_minimal
+        multi_world.accessibility[player2.id].value = multi_world.accessibility[player2.id].option_locations
+
+        multi_world.completion_condition[player1.id] = lambda state: True
+        multi_world.completion_condition[player2.id] = lambda state: state.has(player2.prog_items[2].name, player2.id)
+
+        set_rule(player1.locations[1], lambda state: state.has(player1.prog_items[0].name, player1.id))
+        set_rule(player1.locations[2], lambda state: state.has(player1.prog_items[1].name, player1.id))
+        set_rule(player2.locations[1], lambda state: state.has(player2.prog_items[0].name, player2.id))
+        set_rule(player2.locations[2], lambda state: state.has(player2.prog_items[1].name, player2.id))
+
+        # force-place an item that makes it impossible to have all locations accessible
+        player1.locations[0].place_locked_item(player1.prog_items[2])
+
+        # fill remaining locations with remaining items
+        location_pool = player1.locations[1:] + player2.locations
+        item_pool = player1.prog_items[:-1] + player2.prog_items
+        fill_restrictive(multi_world, multi_world.state, location_pool, item_pool)
+        multi_world.state.sweep_for_events()  # collect everything
+
+        # all of player2's locations and items should be accessible (not all of player1's)
+        for item in player2.prog_items:
+            self.assertTrue(multi_world.state.has(item.name, player2.id),
+                            f'{item} is unreachable in {item.location}')
+
     def test_reversed_fill(self):
         multi_world = generate_multi_world()
         player1 = generate_player_data(multi_world, 1, 2, 2)
@@ -398,6 +433,20 @@ class TestFillRestrictive(unittest.TestCase):
         self.assertTrue(multi_world.state.prog_items[item.name, item.player], "Sweep did not collect - Test flawed")
         self.assertEqual(multi_world.state.prog_items[item.name, item.player], 1, "Sweep collected multiple times")
 
+    def test_correct_item_instance_removed_from_pool(self):
+        multi_world = generate_multi_world()
+        player1 = generate_player_data(multi_world, 1, 2, 2)
+
+        player1.prog_items[0].name = "Different_item_instance_but_same_item_name"
+        player1.prog_items[1].name = "Different_item_instance_but_same_item_name"
+        loc0 = player1.locations[0]
+
+        fill_restrictive(multi_world, multi_world.state,
+                         [loc0], player1.prog_items)
+
+        self.assertEqual(1, len(player1.prog_items))
+        self.assertIsNot(loc0.item, player1.prog_items[0], "Filled item was still present in item pool")
+
 
 class TestDistributeItemsRestrictive(unittest.TestCase):
     def test_basic_distribute(self):

test/general/TestHelpers.py

@@ -19,6 +19,7 @@ class TestHelpers(unittest.TestCase):
         regions: Dict[str, str] = {
             "TestRegion1": "I'm an apple",
             "TestRegion2": "I'm a banana",
+            "TestRegion3": "Empty Region",
         }
 
         locations: Dict[str, Dict[str, Optional[int]]] = {
@@ -38,6 +39,10 @@ class TestHelpers(unittest.TestCase):
             "TestRegion2": {"TestRegion1": None},
         }
         
+        reg_exit_set: Dict[str, set[str]] = {
+            "TestRegion1": {"TestRegion3"}
+        }
+        
         exit_rules: Dict[str, Callable[[CollectionState], bool]] = {
             "TestRegion1": lambda state: state.has("test_item", self.player)
         }
@@ -68,3 +73,10 @@ class TestHelpers(unittest.TestCase):
                         entrance_name = exit_name if exit_name else f"{parent} -> {exit_reg}"
                         self.assertEqual(exit_rules[exit_reg],
                                          self.multiworld.get_entrance(entrance_name, self.player).access_rule)
+            
+            for region in reg_exit_set:
+                current_region = self.multiworld.get_region(region, self.player)
+                current_region.add_exits(reg_exit_set[region])
+                exit_names = {_exit.name for _exit in current_region.exits}
+                for reg_exit in reg_exit_set[region]:
+                    self.assertTrue(f"{region} -> {reg_exit}" in exit_names, f"{region} -> {reg_exit} not in {exit_names}")

test/general/TestHostYAML.py

@@ -1,22 +1,27 @@
+import os
 import unittest
+from tempfile import TemporaryFile
 
+from settings import Settings
 import Utils
 
 
 class TestIDs(unittest.TestCase):
     @classmethod
     def setUpClass(cls) -> None:
-        with open(Utils.local_path("host.yaml")) as f:
+        with TemporaryFile("w+", encoding="utf-8") as f:
+            Settings(None).dump(f)
+            f.seek(0, os.SEEK_SET)
             cls.yaml_options = Utils.parse_yaml(f.read())
 
-    def testUtilsHasHost(self):
+    def test_utils_in_yaml(self) -> None:
         for option_key, option_set in Utils.get_default_options().items():
             with self.subTest(option_key):
                 self.assertIn(option_key, self.yaml_options)
                 for sub_option_key in option_set:
                     self.assertIn(sub_option_key, self.yaml_options[option_key])
 
-    def testHostHasUtils(self):
+    def test_yaml_in_utils(self) -> None:
         utils_options = Utils.get_default_options()
         for option_key, option_set in self.yaml_options.items():
             with self.subTest(option_key):

test/general/TestReachability.py

@@ -2,7 +2,6 @@ import unittest
 
 from BaseClasses import CollectionState
 from worlds.AutoWorld import AutoWorldRegister
-
 from . import setup_solo_multiworld
 
 
@@ -22,8 +21,9 @@ class TestBase(unittest.TestCase):
             "ZD Eyeball Frog Timeout",  # trade quest starts after this item
             "ZR Top of Waterfall",  # dummy region used for entrance shuffle
         },
-        # The following SM regions are only used when the corresponding StartLocation option is selected (so not with default settings).
-        # Also, those dont have any entrances as they serve as starting Region (that's why they have to be excluded for testAllStateCanReachEverything).
+        # The following SM regions are only used when the corresponding StartLocation option is selected (so not with
+        # default settings). Also, those don't have any entrances as they serve as starting Region (that's why they
+        # have to be excluded for testAllStateCanReachEverything).
         "Super Metroid": {
             "Ceres",
             "Gauntlet Top",
@@ -31,37 +31,38 @@ class TestBase(unittest.TestCase):
         }
     }
 
-    def testAllStateCanReachEverything(self):
+    def testDefaultAllStateCanReachEverything(self):
         for game_name, world_type in AutoWorldRegister.world_types.items():
-            # Final Fantasy logic is controlled by finalfantasyrandomizer.com
-            if game_name not in {"Ori and the Blind Forest"}:  # TODO: fix Ori Logic
-                unreachable_regions = self.default_settings_unreachable_regions.get(game_name, set())
-                with self.subTest("Game", game=game_name):
-                    world = setup_solo_multiworld(world_type)
-                    excluded = world.exclude_locations[1].value
-                    state = world.get_all_state(False)
-                    for location in world.get_locations():
-                        if location.name not in excluded:
-                            with self.subTest("Location should be reached", location=location):
-                                self.assertTrue(location.can_reach(state), f"{location.name} unreachable")
+            unreachable_regions = self.default_settings_unreachable_regions.get(game_name, set())
+            with self.subTest("Game", game=game_name):
+                world = setup_solo_multiworld(world_type)
+                excluded = world.exclude_locations[1].value
+                state = world.get_all_state(False)
+                for location in world.get_locations():
+                    if location.name not in excluded:
+                        with self.subTest("Location should be reached", location=location):
+                            self.assertTrue(location.can_reach(state), f"{location.name} unreachable")
 
-                    for region in world.get_regions():
-                        if region.name not in unreachable_regions:
-                            with self.subTest("Region should be reached", region=region):
-                                self.assertTrue(region.can_reach(state))
+                for region in world.get_regions():
+                    if region.name in unreachable_regions:
+                        with self.subTest("Region should be unreachable", region=region):
+                            self.assertFalse(region.can_reach(state))
+                    else:
+                        with self.subTest("Region should be reached", region=region):
+                            self.assertTrue(region.can_reach(state))
 
-                    with self.subTest("Completion Condition"):
-                        self.assertTrue(world.can_beat_game(state))
+                with self.subTest("Completion Condition"):
+                    self.assertTrue(world.can_beat_game(state))
 
-    def testEmptyStateCanReachSomething(self):
+    def testDefaultEmptyStateCanReachSomething(self):
         for game_name, world_type in AutoWorldRegister.world_types.items():
-            # Final Fantasy logic is controlled by finalfantasyrandomizer.com
-            if game_name not in {"Archipelago", "Sudoku"}:
-                with self.subTest("Game", game=game_name):
-                    world = setup_solo_multiworld(world_type)
-                    state = CollectionState(world)
+            with self.subTest("Game", game=game_name):
+                world = setup_solo_multiworld(world_type)
+                state = CollectionState(world)
+                all_locations = world.get_locations()
+                if all_locations:
                     locations = set()
-                    for location in world.get_locations():
+                    for location in all_locations:
                         if location.can_reach(state):
                             locations.add(location)
                     self.assertGreater(len(locations), 0,

test/netutils/TestLocationStore.py

@@ -0,0 +1,238 @@
+# Tests for _speedups.LocationStore and NetUtils._LocationStore
+import typing
+import unittest
+import warnings
+from NetUtils import LocationStore, _LocationStore
+
+State = typing.Dict[typing.Tuple[int, int], typing.Set[int]]
+RawLocations = typing.Dict[int, typing.Dict[int, typing.Tuple[int, int, int]]]
+
+sample_data: RawLocations = {
+    1: {
+        11: (21, 2, 7),
+        12: (22, 2, 0),
+        13: (13, 1, 0),
+    },
+    2: {
+        23: (11, 1, 0),
+        22: (12, 1, 0),
+        21: (23, 2, 0),
+    },
+    4: {
+        9: (99, 3, 0),
+    },
+    3: {
+        9: (99, 4, 0),
+    },
+}
+
+empty_state: State = {
+    (0, slot): set() for slot in sample_data
+}
+
+full_state: State = {
+    (0, slot): set(locations) for (slot, locations) in sample_data.items()
+}
+
+one_state: State = {
+    (0, 1): {12}
+}
+
+
+class Base:
+    class TestLocationStore(unittest.TestCase):
+        """Test method calls on a loaded store."""
+        store: typing.Union[LocationStore, _LocationStore]
+
+        def test_len(self) -> None:
+            self.assertEqual(len(self.store), 4)
+            self.assertEqual(len(self.store[1]), 3)
+
+        def test_key_error(self) -> None:
+            with self.assertRaises(KeyError):
+                _ = self.store[0]
+            with self.assertRaises(KeyError):
+                _ = self.store[5]
+            locations = self.store[1]  # no Exception
+            with self.assertRaises(KeyError):
+                _ = locations[7]
+            _ = locations[11]  # no Exception
+
+        def test_getitem(self) -> None:
+            self.assertEqual(self.store[1][11], (21, 2, 7))
+            self.assertEqual(self.store[1][13], (13, 1, 0))
+            self.assertEqual(self.store[2][22], (12, 1, 0))
+            self.assertEqual(self.store[4][9], (99, 3, 0))
+
+        def test_get(self) -> None:
+            self.assertEqual(self.store.get(1, None), self.store[1])
+            self.assertEqual(self.store.get(0, None), None)
+            self.assertEqual(self.store[1].get(11, (None, None, None)), self.store[1][11])
+            self.assertEqual(self.store[1].get(10, (None, None, None)), (None, None, None))
+
+        def test_iter(self) -> None:
+            self.assertEqual(sorted(self.store), [1, 2, 3, 4])
+            self.assertEqual(len(self.store), len(sample_data))
+            self.assertEqual(list(self.store[1]), [11, 12, 13])
+            self.assertEqual(len(self.store[1]), len(sample_data[1]))
+
+        def test_items(self) -> None:
+            self.assertEqual(sorted(p for p, _ in self.store.items()), sorted(self.store))
+            self.assertEqual(sorted(p for p, _ in self.store[1].items()), sorted(self.store[1]))
+            self.assertEqual(sorted(self.store.items())[0][0], 1)
+            self.assertEqual(sorted(self.store.items())[0][1], self.store[1])
+            self.assertEqual(sorted(self.store[1].items())[0][0], 11)
+            self.assertEqual(sorted(self.store[1].items())[0][1], self.store[1][11])
+
+        def test_find_item(self) -> None:
+            self.assertEqual(sorted(self.store.find_item(set(), 99)), [])
+            self.assertEqual(sorted(self.store.find_item({3}, 1)), [])
+            self.assertEqual(sorted(self.store.find_item({5}, 99)), [])
+            self.assertEqual(sorted(self.store.find_item({3}, 99)),
+                             [(4, 9, 99, 3, 0)])
+            self.assertEqual(sorted(self.store.find_item({3, 4}, 99)),
+                             [(3, 9, 99, 4, 0), (4, 9, 99, 3, 0)])
+
+        def test_get_for_player(self) -> None:
+            self.assertEqual(self.store.get_for_player(3), {4: {9}})
+            self.assertEqual(self.store.get_for_player(1), {1: {13}, 2: {22, 23}})
+
+        def test_get_checked(self) -> None:
+            self.assertEqual(self.store.get_checked(full_state, 0, 1), [11, 12, 13])
+            self.assertEqual(self.store.get_checked(one_state, 0, 1), [12])
+            self.assertEqual(self.store.get_checked(empty_state, 0, 1), [])
+            self.assertEqual(self.store.get_checked(full_state, 0, 3), [9])
+
+        def test_get_missing(self) -> None:
+            self.assertEqual(self.store.get_missing(full_state, 0, 1), [])
+            self.assertEqual(self.store.get_missing(one_state, 0, 1), [11, 13])
+            self.assertEqual(self.store.get_missing(empty_state, 0, 1), [11, 12, 13])
+            self.assertEqual(self.store.get_missing(empty_state, 0, 3), [9])
+
+        def test_get_remaining(self) -> None:
+            self.assertEqual(self.store.get_remaining(full_state, 0, 1), [])
+            self.assertEqual(self.store.get_remaining(one_state, 0, 1), [13, 21])
+            self.assertEqual(self.store.get_remaining(empty_state, 0, 1), [13, 21, 22])
+            self.assertEqual(self.store.get_remaining(empty_state, 0, 3), [99])
+
+        def test_location_set_intersection(self) -> None:
+            locations = {10, 11, 12}
+            locations.intersection_update(self.store[1])
+            self.assertEqual(locations, {11, 12})
+
+    class TestLocationStoreConstructor(unittest.TestCase):
+        """Test constructors for a given store type."""
+        type: type
+
+        def test_hole(self) -> None:
+            with self.assertRaises(Exception):
+                self.type({
+                    1: {1: (1, 1, 1)},
+                    3: {1: (1, 1, 1)},
+                })
+
+        def test_no_slot1(self) -> None:
+            with self.assertRaises(Exception):
+                self.type({
+                    2: {1: (1, 1, 1)},
+                    3: {1: (1, 1, 1)},
+                })
+
+        def test_slot0(self) -> None:
+            with self.assertRaises(ValueError):
+                self.type({
+                    0: {1: (1, 1, 1)},
+                    1: {1: (1, 1, 1)},
+                })
+            with self.assertRaises(ValueError):
+                self.type({
+                    0: {1: (1, 1, 1)},
+                    2: {1: (1, 1, 1)},
+                })
+
+        def test_no_players(self) -> None:
+            with self.assertRaises(Exception):
+                _ = self.type({})
+
+        def test_no_locations(self) -> None:
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                store = self.type({
+                    1: {},
+                })
+                self.assertEqual(len(store), 1)
+                self.assertEqual(len(store[1]), 0)
+
+        def test_no_locations_for_1(self) -> None:
+            store = self.type({
+                1: {},
+                2: {1: (1, 2, 3)},
+            })
+            self.assertEqual(len(store), 2)
+            self.assertEqual(len(store[1]), 0)
+            self.assertEqual(len(store[2]), 1)
+
+        def test_no_locations_for_last(self) -> None:
+            store = self.type({
+                1: {1: (1, 2, 3)},
+                2: {},
+            })
+            self.assertEqual(len(store), 2)
+            self.assertEqual(len(store[1]), 1)
+            self.assertEqual(len(store[2]), 0)
+
+
+class TestPurePythonLocationStore(Base.TestLocationStore):
+    """Run base method tests for pure python implementation."""
+    def setUp(self) -> None:
+        self.store = _LocationStore(sample_data)
+        super().setUp()
+
+
+class TestPurePythonLocationStoreConstructor(Base.TestLocationStoreConstructor):
+    """Run base constructor tests for the pure python implementation."""
+    def setUp(self) -> None:
+        self.type = _LocationStore
+        super().setUp()
+
+
+@unittest.skipIf(LocationStore is _LocationStore, "_speedups not available")
+class TestSpeedupsLocationStore(Base.TestLocationStore):
+    """Run base method tests for cython implementation."""
+    def setUp(self) -> None:
+        self.store = LocationStore(sample_data)
+        super().setUp()
+
+
+@unittest.skipIf(LocationStore is _LocationStore, "_speedups not available")
+class TestSpeedupsLocationStoreConstructor(Base.TestLocationStoreConstructor):
+    """Run base constructor tests and tests the additional constraints for cython implementation."""
+    def setUp(self) -> None:
+        self.type = LocationStore
+        super().setUp()
+
+    def test_float_key(self) -> None:
+        with self.assertRaises(Exception):
+            self.type({
+                1: {1: (1, 1, 1)},
+                1.1: {1: (1, 1, 1)},
+                3: {1: (1, 1, 1)}
+            })
+
+    def test_string_key(self) -> None:
+        with self.assertRaises(Exception):
+            self.type({
+                "1": {1: (1, 1, 1)},
+            })
+
+    def test_high_player_number(self) -> None:
+        with self.assertRaises(Exception):
+            self.type({
+                1 << 32: {1: (1, 1, 1)},
+            })
+
+    def test_not_a_tuple(self) -> None:
+        with self.assertRaises(Exception):
+            self.type({
+                1: {1: None},
+            })

test/programs/TestGenerate.py

@@ -73,13 +73,21 @@ class TestGenerateMain(unittest.TestCase):
 
     def test_generate_yaml(self):
         # override host.yaml
-        defaults = Generate.Utils.get_options()["generator"]
-        defaults["player_files_path"] = str(self.yaml_input_dir)
-        defaults["players"] = 0
-
-        sys.argv = [sys.argv[0], '--seed', '0',
-                    '--outputpath', self.output_tempdir.name]
-        print(f'Testing Generate.py {sys.argv} in {os.getcwd()}, player_files_path={self.yaml_input_dir}')
-        Generate.main()
+        from settings import get_settings
+        from Utils import user_path, local_path
+        settings = get_settings()
+        # NOTE: until/unless we override settings.Group's setattr, we have to upcast the input dir here
+        settings.generator.player_files_path = settings.generator.PlayerFilesPath(self.yaml_input_dir)
+        settings.generator.players = 0
+        settings._filename = None  # don't write to disk
+        user_path_backup = user_path.cached_path
+        user_path.cached_path = local_path()  # test yaml is actually in local_path
+        try:
+            sys.argv = [sys.argv[0], '--seed', '0',
+                        '--outputpath', self.output_tempdir.name]
+            print(f'Testing Generate.py {sys.argv} in {os.getcwd()}, player_files_path={self.yaml_input_dir}')
+            Generate.main()
+        finally:
+            user_path.cached_path = user_path_backup
 
         self.assertOutput(self.output_tempdir.name)

test/worlds/__init__.py

@@ -8,7 +8,13 @@ def load_tests(loader, standard_tests, pattern):
     suite.addTests(standard_tests)
     folders = [os.path.join(os.path.split(world.__file__)[0], "test")
                for world in AutoWorldRegister.world_types.values()]
-    for folder in folders:
-        if os.path.exists(folder):
-            suite.addTests(loader.discover(folder, top_level_dir=file_path))
+
+    all_tests = [
+        test_case for folder in folders if os.path.exists(folder)
+        for test_collection in loader.discover(folder, top_level_dir=file_path)
+        for test_suite in test_collection
+        for test_case in test_suite
+    ]
+
+    suite.addTests(sorted(all_tests, key=lambda test: test.__module__))
     return suite

worlds/AutoWorld.py

@@ -11,14 +11,26 @@ from BaseClasses import CollectionState
 from Options import AssembleOptions
 
 if TYPE_CHECKING:
+    import random
     from BaseClasses import MultiWorld, Item, Location, Tutorial
     from . import GamesPackage
+    from settings import Group
 
 
 class AutoWorldRegister(type):
     world_types: Dict[str, Type[World]] = {}
     __file__: str
     zip_path: Optional[str]
+    settings_key: str
+    __settings: Any
+
+    @property
+    def settings(cls) -> Any:  # actual type is defined in World
+        # lazy loading + caching to minimize runtime cost
+        if cls.__settings is None:
+            from settings import get_settings
+            cls.__settings = get_settings()[cls.settings_key]
+        return cls.__settings
 
     def __new__(mcs, name: str, bases: Tuple[type, ...], dct: Dict[str, Any]) -> AutoWorldRegister:
         if "web" in dct:
@@ -60,6 +72,11 @@ class AutoWorldRegister(type):
         new_class.__file__ = sys.modules[new_class.__module__].__file__
         if ".apworld" in new_class.__file__:
             new_class.zip_path = pathlib.Path(new_class.__file__).parents[1]
+        if "settings_key" not in dct:
+            mod_name = new_class.__module__
+            world_folder_name = mod_name[7:].lower() if mod_name.startswith("worlds.") else mod_name.lower()
+            new_class.settings_key = world_folder_name + "_options"
+        new_class.__settings = None
         return new_class
 
 
@@ -81,7 +98,17 @@ class AutoLogicRegister(type):
 
 def call_single(multiworld: "MultiWorld", method_name: str, player: int, *args: Any) -> Any:
     method = getattr(multiworld.worlds[player], method_name)
-    return method(*args)
+    try:
+        ret = method(*args)
+    except Exception as e:
+        message = f"Exception in {method} for player {player}, named {multiworld.player_name[player]}."
+        if sys.version_info >= (3, 11, 0):
+            e.add_note(message)  # PEP 678
+        else:
+            logging.error(message)
+        raise e
+    else:
+        return ret
 
 
 def call_all(multiworld: "MultiWorld", method_name: str, *args: Any) -> None:
@@ -203,6 +230,14 @@ class World(metaclass=AutoWorldRegister):
     location_names: ClassVar[Set[str]]
     """set of all potential location names"""
 
+    random: random.Random
+    """This world's random object. Should be used for any randomization needed in world for this player slot."""
+
+    settings_key: ClassVar[str]
+    """name of the section in host.yaml for world-specific settings, will default to {folder}_options"""
+    settings: ClassVar[Optional["Group"]]
+    """loaded settings from host.yaml"""
+
     zip_path: ClassVar[Optional[pathlib.Path]] = None
     """If loaded from a .apworld, this is the Path to it."""
     __file__: ClassVar[str]
@@ -212,6 +247,11 @@ class World(metaclass=AutoWorldRegister):
         self.multiworld = multiworld
         self.player = player
 
+    def __getattr__(self, item: str) -> Any:
+        if item == "settings":
+            return self.__class__.settings
+        raise AttributeError
+
     # overridable methods that get called by Main.py, sorted by execution order
     # can also be implemented as a classmethod and called "stage_<original_name>",
     # in that case the MultiWorld object is passed as an argument and it gets called once for the entire multiworld.
@@ -269,8 +309,8 @@ class World(metaclass=AutoWorldRegister):
         This happens before progression balancing, so the items may not be in their final locations yet."""
 
     def generate_output(self, output_directory: str) -> None:
-        """This method gets called from a threadpool, do not use world.random here.
-        If you need any last-second randomization, use MultiWorld.per_slot_randoms[slot] instead."""
+        """This method gets called from a threadpool, do not use multiworld.random here.
+        If you need any last-second randomization, use self.random instead."""
         pass
 
     def fill_slot_data(self) -> Dict[str, Any]:  # json of WebHostLib.models.Slot

worlds/LauncherComponents.py

@@ -1,19 +1,22 @@
+import weakref
 from enum import Enum, auto
 from typing import Optional, Callable, List, Iterable
 
-from Utils import local_path, is_windows
+from Utils import local_path
 
 
 class Type(Enum):
     TOOL = auto()
-    FUNC = auto()  # not a real component
+    MISC = auto()
     CLIENT = auto()
     ADJUSTER = auto()
+    FUNC = auto()  # do not use anymore
+    HIDDEN = auto()
 
 
 class Component:
     display_name: str
-    type: Optional[Type]
+    type: Type
     script_name: Optional[str]
     frozen_name: Optional[str]
     icon: str  # just the name, no suffix
@@ -22,18 +25,21 @@ class Component:
     file_identifier: Optional[Callable[[str], bool]]
 
     def __init__(self, display_name: str, script_name: Optional[str] = None, frozen_name: Optional[str] = None,
-                 cli: bool = False, icon: str = 'icon', component_type: Type = None, func: Optional[Callable] = None,
-                 file_identifier: Optional[Callable[[str], bool]] = None):
+                 cli: bool = False, icon: str = 'icon', component_type: Optional[Type] = None,
+                 func: Optional[Callable] = None, file_identifier: Optional[Callable[[str], bool]] = None):
         self.display_name = display_name
         self.script_name = script_name
         self.frozen_name = frozen_name or f'Archipelago{script_name}' if script_name else None
         self.icon = icon
         self.cli = cli
-        self.type = component_type or \
-            None if not display_name else \
-            Type.FUNC if func else \
-            Type.CLIENT if 'Client' in display_name else \
-            Type.ADJUSTER if 'Adjuster' in display_name else Type.TOOL
+        if component_type == Type.FUNC:
+            from Utils import deprecate
+            deprecate(f"Launcher Component {self.display_name} is using Type.FUNC Type, which is pending removal.")
+            component_type = Type.MISC
+
+        self.type = component_type or (
+            Type.CLIENT if "Client" in display_name else
+            Type.ADJUSTER if "Adjuster" in display_name else Type.MISC)
         self.func = func
         self.file_identifier = file_identifier
 
@@ -43,6 +49,14 @@ class Component:
     def __repr__(self):
         return f"{self.__class__.__name__}({self.display_name})"
 
+processes = weakref.WeakSet()
+
+def launch_subprocess(func: Callable, name: str = None):
+    global processes
+    import multiprocessing
+    process = multiprocessing.Process(target=func, name=name)
+    process.start()
+    processes.add(process)
 
 class SuffixIdentifier:
     suffixes: Iterable[str]
@@ -58,14 +72,19 @@ class SuffixIdentifier:
         return False
 
 
+def launch_textclient():
+    import CommonClient
+    launch_subprocess(CommonClient.run_as_textclient, name="TextClient")
+
+
 components: List[Component] = [
     # Launcher
-    Component('', 'Launcher'),
+    Component('Launcher', 'Launcher', component_type=Type.HIDDEN),
     # Core
     Component('Host', 'MultiServer', 'ArchipelagoServer', cli=True,
               file_identifier=SuffixIdentifier('.archipelago', '.zip')),
     Component('Generate', 'Generate', cli=True),
-    Component('Text Client', 'CommonClient', 'ArchipelagoTextClient'),
+    Component('Text Client', 'CommonClient', 'ArchipelagoTextClient', func=launch_textclient),
     # SNI
     Component('SNI Client', 'SNIClient',
               file_identifier=SuffixIdentifier('.apz3', '.apm3', '.apsoe', '.aplttp', '.apsm', '.apsmz3', '.apdkc3',
@@ -85,7 +104,7 @@ components: List[Component] = [
     # Pokémon
     Component('Pokemon Client', 'PokemonClient', file_identifier=SuffixIdentifier('.apred', '.apblue')),
     # TLoZ
-    Component('Zelda 1 Client', 'Zelda1Client'),
+    Component('Zelda 1 Client', 'Zelda1Client', file_identifier=SuffixIdentifier('.aptloz')),
     # ChecksFinder
     Component('ChecksFinder Client', 'ChecksFinderClient'),
     # Starcraft 2
@@ -97,6 +116,9 @@ components: List[Component] = [
               file_identifier=SuffixIdentifier('.apzl')),
     # Kingdom Hearts 2
     Component('KH2 Client', "KH2Client"),
+
+    #MegaMan Battle Network 3
+    Component('MMBN3 Client', 'MMBN3Client', file_identifier=SuffixIdentifier('.apbn3'))
 ]
 
 

worlds/__init__.py

@@ -39,9 +39,50 @@ class DataPackage(typing.TypedDict):
 class WorldSource(typing.NamedTuple):
     path: str  # typically relative path from this module
     is_zip: bool = False
+    relative: bool = True  # relative to regular world import folder
 
     def __repr__(self):
-        return f"{self.__class__.__name__}({self.path}, is_zip={self.is_zip})"
+        return f"{self.__class__.__name__}({self.path}, is_zip={self.is_zip}, relative={self.relative})"
+
+    @property
+    def resolved_path(self) -> str:
+        if self.relative:
+            return os.path.join(folder, self.path)
+        return self.path
+
+    def load(self) -> bool:
+        try:
+            if self.is_zip:
+                importer = zipimport.zipimporter(self.resolved_path)
+                if hasattr(importer, "find_spec"):  # new in Python 3.10
+                    spec = importer.find_spec(os.path.basename(self.path).rsplit(".", 1)[0])
+                    mod = importlib.util.module_from_spec(spec)
+                else:  # TODO: remove with 3.8 support
+                    mod = importer.load_module(os.path.basename(self.path).rsplit(".", 1)[0])
+
+                mod.__package__ = f"worlds.{mod.__package__}"
+                mod.__name__ = f"worlds.{mod.__name__}"
+                sys.modules[mod.__name__] = mod
+                with warnings.catch_warnings():
+                    warnings.filterwarnings("ignore", message="__package__ != __spec__.parent")
+                    # Found no equivalent for < 3.10
+                    if hasattr(importer, "exec_module"):
+                        importer.exec_module(mod)
+            else:
+                importlib.import_module(f".{self.path}", "worlds")
+            return True
+
+        except Exception as e:
+            # A single world failing can still mean enough is working for the user, log and carry on
+            import traceback
+            import io
+            file_like = io.StringIO()
+            print(f"Could not load world {self}:", file=file_like)
+            traceback.print_exc(file=file_like)
+            file_like.seek(0)
+            import logging
+            logging.exception(file_like.read())
+            return False
 
 
 # find potential world containers, currently folders and zip-importable .apworld's
@@ -58,35 +99,7 @@ for file in os.scandir(folder):
 # import all submodules to trigger AutoWorldRegister
 world_sources.sort()
 for world_source in world_sources:
-    try:
-        if world_source.is_zip:
-            importer = zipimport.zipimporter(os.path.join(folder, world_source.path))
-            if hasattr(importer, "find_spec"):  # new in Python 3.10
-                spec = importer.find_spec(world_source.path.split(".", 1)[0])
-                mod = importlib.util.module_from_spec(spec)
-            else:  # TODO: remove with 3.8 support
-                mod = importer.load_module(world_source.path.split(".", 1)[0])
-
-            mod.__package__ = f"worlds.{mod.__package__}"
-            mod.__name__ = f"worlds.{mod.__name__}"
-            sys.modules[mod.__name__] = mod
-            with warnings.catch_warnings():
-                warnings.filterwarnings("ignore", message="__package__ != __spec__.parent")
-                # Found no equivalent for < 3.10
-                if hasattr(importer, "exec_module"):
-                    importer.exec_module(mod)
-        else:
-            importlib.import_module(f".{world_source.path}", "worlds")
-    except Exception as e:
-        # A single world failing can still mean enough is working for the user, log and carry on
-        import traceback
-        import io
-        file_like = io.StringIO()
-        print(f"Could not load world {world_source}:", file=file_like)
-        traceback.print_exc(file=file_like)
-        file_like.seek(0)
-        import logging
-        logging.exception(file_like.read())
+    world_source.load()
 
 lookup_any_item_id_to_name = {}
 lookup_any_location_id_to_name = {}

worlds/_sc2common/bot/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Hannes Karppila
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

worlds/_sc2common/bot/README.md

@@ -0,0 +1,6 @@
+# SC2 Bot
+This is client library to communicate with Starcraft 2 game  
+It's based on `burnysc2` python package, see https://github.com/BurnySc2/python-sc2
+
+The base package is stripped down to clean up unneeded features and those not working outside a
+melee game.

worlds/_sc2common/bot/__init__.py

@@ -0,0 +1,16 @@
+from pathlib import Path
+from loguru import logger
+
+
+def is_submodule(path):
+    if path.is_file():
+        return path.suffix == ".py" and path.stem != "__init__"
+    if path.is_dir():
+        return (path / "__init__.py").exists()
+    return False
+
+
+__all__ = [p.stem for p in Path(__file__).parent.iterdir() if is_submodule(p)]
+
+
+logger = logger

worlds/_sc2common/bot/bot_ai.py

@@ -0,0 +1,476 @@
+# pylint: disable=W0212,R0916,R0904
+from __future__ import annotations
+
+import math
+from functools import cached_property
+from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Union
+
+from .bot_ai_internal import BotAIInternal
+from .cache import property_cache_once_per_frame
+from .data import Alert, Result
+from .position import Point2
+from .unit import Unit
+from .units import Units
+
+if TYPE_CHECKING:
+    from .game_info import Ramp
+
+
+class BotAI(BotAIInternal):
+    """Base class for bots."""
+
+    EXPANSION_GAP_THRESHOLD = 15
+
+    @property
+    def time(self) -> float:
+        """ Returns time in seconds, assumes the game is played on 'faster' """
+        return self.state.game_loop / 22.4  # / (1/1.4) * (1/16)
+
+    @property
+    def time_formatted(self) -> str:
+        """ Returns time as string in min:sec format """
+        t = self.time
+        return f"{int(t // 60):02}:{int(t % 60):02}"
+
+    @property
+    def step_time(self) -> Tuple[float, float, float, float]:
+        """Returns a tuple of step duration in milliseconds.
+        First value is the minimum step duration - the shortest the bot ever took
+        Second value is the average step duration
+        Third value is the maximum step duration - the longest the bot ever took (including on_start())
+        Fourth value is the step duration the bot took last iteration
+        If called in the first iteration, it returns (inf, 0, 0, 0)"""
+        avg_step_duration = (
+            (self._total_time_in_on_step / self._total_steps_iterations) if self._total_steps_iterations else 0
+        )
+        return (
+            self._min_step_time * 1000,
+            avg_step_duration * 1000,
+            self._max_step_time * 1000,
+            self._last_step_step_time * 1000,
+        )
+
+    def alert(self, alert_code: Alert) -> bool:
+        """
+        Check if alert is triggered in the current step.
+        Possible alerts are listed here https://github.com/Blizzard/s2client-proto/blob/e38efed74c03bec90f74b330ea1adda9215e655f/s2clientprotocol/sc2api.proto#L679-L702
+
+        Example use::
+
+            from sc2.data import Alert
+            if self.alert(Alert.AddOnComplete):
+                print("Addon Complete")
+
+        Alert codes::
+
+            AlertError
+            AddOnComplete
+            BuildingComplete
+            BuildingUnderAttack
+            LarvaHatched
+            MergeComplete
+            MineralsExhausted
+            MorphComplete
+            MothershipComplete
+            MULEExpired
+            NuclearLaunchDetected
+            NukeComplete
+            NydusWormDetected
+            ResearchComplete
+            TrainError
+            TrainUnitComplete
+            TrainWorkerComplete
+            TransformationComplete
+            UnitUnderAttack
+            UpgradeComplete
+            VespeneExhausted
+            WarpInComplete
+
+        :param alert_code:
+        """
+        assert isinstance(alert_code, Alert), f"alert_code {alert_code} is no Alert"
+        return alert_code.value in self.state.alerts
+
+    @property
+    def start_location(self) -> Point2:
+        """
+        Returns the spawn location of the bot, using the position of the first created townhall.
+        This will be None if the bot is run on an arcade or custom map that does not feature townhalls at game start.
+        """
+        return self.game_info.player_start_location
+
+    @property
+    def enemy_start_locations(self) -> List[Point2]:
+        """Possible start locations for enemies."""
+        return self.game_info.start_locations
+
+    @cached_property
+    def main_base_ramp(self) -> Ramp:
+        """Returns the Ramp instance of the closest main-ramp to start location.
+        Look in game_info.py for more information about the Ramp class
+
+        Example: See terran ramp wall bot
+        """
+        # The reason for len(ramp.upper) in {2, 5} is:
+        # ParaSite map has 5 upper points, and most other maps have 2 upper points at the main ramp.
+        # The map Acolyte has 4 upper points at the wrong ramp (which is closest to the start position).
+        try:
+            found_main_base_ramp = min(
+                (ramp for ramp in self.game_info.map_ramps if len(ramp.upper) in {2, 5}),
+                key=lambda r: self.start_location.distance_to(r.top_center),
+            )
+        except ValueError:
+            # Hardcoded hotfix for Honorgrounds LE map, as that map has a large main base ramp with inbase natural
+            found_main_base_ramp = min(
+                (ramp for ramp in self.game_info.map_ramps if len(ramp.upper) in {4, 9}),
+                key=lambda r: self.start_location.distance_to(r.top_center),
+            )
+        return found_main_base_ramp
+
+    @property_cache_once_per_frame
+    def expansion_locations_list(self) -> List[Point2]:
+        """ Returns a list of expansion positions, not sorted in any way. """
+        assert (
+            self._expansion_positions_list
+        ), "self._find_expansion_locations() has not been run yet, so accessing the list of expansion locations is pointless."
+        return self._expansion_positions_list
+
+    @property_cache_once_per_frame
+    def expansion_locations_dict(self) -> Dict[Point2, Units]:
+        """
+        Returns dict with the correct expansion position Point2 object as key,
+        resources as Units (mineral fields and vespene geysers) as value.
+
+        Caution: This function is slow. If you only need the expansion locations, use the property above.
+        """
+        assert (
+            self._expansion_positions_list
+        ), "self._find_expansion_locations() has not been run yet, so accessing the list of expansion locations is pointless."
+        expansion_locations: Dict[Point2, Units] = {pos: Units([], self) for pos in self._expansion_positions_list}
+        for resource in self.resources:
+            # It may be that some resources are not mapped to an expansion location
+            exp_position: Point2 = self._resource_location_to_expansion_position_dict.get(resource.position, None)
+            if exp_position:
+                assert exp_position in expansion_locations
+                expansion_locations[exp_position].append(resource)
+        return expansion_locations
+
+    async def get_next_expansion(self) -> Optional[Point2]:
+        """Find next expansion location."""
+
+        closest = None
+        distance = math.inf
+        for el in self.expansion_locations_list:
+
+            def is_near_to_expansion(t):
+                return t.distance_to(el) < self.EXPANSION_GAP_THRESHOLD
+
+            if any(map(is_near_to_expansion, self.townhalls)):
+                # already taken
+                continue
+
+            startp = self.game_info.player_start_location
+            d = await self.client.query_pathing(startp, el)
+            if d is None:
+                continue
+
+            if d < distance:
+                distance = d
+                closest = el
+
+        return closest
+
+    # pylint: disable=R0912
+    async def distribute_workers(self, resource_ratio: float = 2):
+        """
+        Distributes workers across all the bases taken.
+        Keyword `resource_ratio` takes a float. If the current minerals to gas
+        ratio is bigger than `resource_ratio`, this function prefer filling gas_buildings
+        first, if it is lower, it will prefer sending workers to minerals first.
+
+        NOTE: This function is far from optimal, if you really want to have
+        refined worker control, you should write your own distribution function.
+        For example long distance mining control and moving workers if a base was killed
+        are not being handled.
+
+        WARNING: This is quite slow when there are lots of workers or multiple bases.
+
+        :param resource_ratio:"""
+        if not self.mineral_field or not self.workers or not self.townhalls.ready:
+            return
+        worker_pool = self.workers.idle
+        bases = self.townhalls.ready
+        gas_buildings = self.gas_buildings.ready
+
+        # list of places that need more workers
+        deficit_mining_places = []
+
+        for mining_place in bases | gas_buildings:
+            difference = mining_place.surplus_harvesters
+            # perfect amount of workers, skip mining place
+            if not difference:
+                continue
+            if mining_place.has_vespene:
+                # get all workers that target the gas extraction site
+                # or are on their way back from it
+                local_workers = self.workers.filter(
+                    lambda unit: unit.order_target == mining_place.tag or
+                    (unit.is_carrying_vespene and unit.order_target == bases.closest_to(mining_place).tag)
+                )
+            else:
+                # get tags of minerals around expansion
+                local_minerals_tags = {
+                    mineral.tag
+                    for mineral in self.mineral_field if mineral.distance_to(mining_place) <= 8
+                }
+                # get all target tags a worker can have
+                # tags of the minerals he could mine at that base
+                # get workers that work at that gather site
+                local_workers = self.workers.filter(
+                    lambda unit: unit.order_target in local_minerals_tags or
+                    (unit.is_carrying_minerals and unit.order_target == mining_place.tag)
+                )
+            # too many workers
+            if difference > 0:
+                for worker in local_workers[:difference]:
+                    worker_pool.append(worker)
+            # too few workers
+            # add mining place to deficit bases for every missing worker
+            else:
+                deficit_mining_places += [mining_place for _ in range(-difference)]
+
+        # prepare all minerals near a base if we have too many workers
+        # and need to send them to the closest patch
+        if len(worker_pool) > len(deficit_mining_places):
+            all_minerals_near_base = [
+                mineral for mineral in self.mineral_field
+                if any(mineral.distance_to(base) <= 8 for base in self.townhalls.ready)
+            ]
+        # distribute every worker in the pool
+        for worker in worker_pool:
+            # as long as have workers and mining places
+            if deficit_mining_places:
+                # choose only mineral fields first if current mineral to gas ratio is less than target ratio
+                if self.vespene and self.minerals / self.vespene < resource_ratio:
+                    possible_mining_places = [place for place in deficit_mining_places if not place.vespene_contents]
+                # else prefer gas
+                else:
+                    possible_mining_places = [place for place in deficit_mining_places if place.vespene_contents]
+                # if preferred type is not available any more, get all other places
+                if not possible_mining_places:
+                    possible_mining_places = deficit_mining_places
+                # find closest mining place
+                current_place = min(deficit_mining_places, key=lambda place: place.distance_to(worker))
+                # remove it from the list
+                deficit_mining_places.remove(current_place)
+                # if current place is a gas extraction site, go there
+                if current_place.vespene_contents:
+                    worker.gather(current_place)
+                # if current place is a gas extraction site,
+                # go to the mineral field that is near and has the most minerals left
+                else:
+                    local_minerals = (
+                        mineral for mineral in self.mineral_field if mineral.distance_to(current_place) <= 8
+                    )
+                    # local_minerals can be empty if townhall is misplaced
+                    target_mineral = max(local_minerals, key=lambda mineral: mineral.mineral_contents, default=None)
+                    if target_mineral:
+                        worker.gather(target_mineral)
+            # more workers to distribute than free mining spots
+            # send to closest if worker is doing nothing
+            elif worker.is_idle and all_minerals_near_base:
+                target_mineral = min(all_minerals_near_base, key=lambda mineral: mineral.distance_to(worker))
+                worker.gather(target_mineral)
+            else:
+                # there are no deficit mining places and worker is not idle
+                # so dont move him
+                pass
+
+    @property_cache_once_per_frame
+    def owned_expansions(self) -> Dict[Point2, Unit]:
+        """Dict of expansions owned by the player with mapping {expansion_location: townhall_structure}."""
+        owned = {}
+        for el in self.expansion_locations_list:
+
+            def is_near_to_expansion(t):
+                return t.distance_to(el) < self.EXPANSION_GAP_THRESHOLD
+
+            th = next((x for x in self.townhalls if is_near_to_expansion(x)), None)
+            if th:
+                owned[el] = th
+        return owned
+
+    async def chat_send(self, message: str, team_only: bool = False):
+        """Send a chat message to the SC2 Client.
+
+        Example::
+
+            await self.chat_send("Hello, this is a message from my bot!")
+
+        :param message:
+        :param team_only:"""
+        assert isinstance(message, str), f"{message} is not a string"
+        await self.client.chat_send(message, team_only)
+
+    def in_map_bounds(self, pos: Union[Point2, tuple, list]) -> bool:
+        """Tests if a 2 dimensional point is within the map boundaries of the pixelmaps.
+
+        :param pos:"""
+        return (
+            self.game_info.playable_area.x <= pos[0] <
+            self.game_info.playable_area.x + self.game_info.playable_area.width and self.game_info.playable_area.y <=
+            pos[1] < self.game_info.playable_area.y + self.game_info.playable_area.height
+        )
+
+    # For the functions below, make sure you are inside the boundaries of the map size.
+    def get_terrain_height(self, pos: Union[Point2, Unit]) -> int:
+        """Returns terrain height at a position.
+        Caution: terrain height is different from a unit's z-coordinate.
+
+        :param pos:"""
+        assert isinstance(pos, (Point2, Unit)), "pos is not of type Point2 or Unit"
+        pos = pos.position.rounded
+        return self.game_info.terrain_height[pos]
+
+    def get_terrain_z_height(self, pos: Union[Point2, Unit]) -> float:
+        """Returns terrain z-height at a position.
+
+        :param pos:"""
+        assert isinstance(pos, (Point2, Unit)), "pos is not of type Point2 or Unit"
+        pos = pos.position.rounded
+        return -16 + 32 * self.game_info.terrain_height[pos] / 255
+
+    def in_placement_grid(self, pos: Union[Point2, Unit]) -> bool:
+        """Returns True if you can place something at a position.
+        Remember, buildings usually use 2x2, 3x3 or 5x5 of these grid points.
+        Caution: some x and y offset might be required, see ramp code in game_info.py
+
+        :param pos:"""
+        assert isinstance(pos, (Point2, Unit)), "pos is not of type Point2 or Unit"
+        pos = pos.position.rounded
+        return self.game_info.placement_grid[pos] == 1
+
+    def in_pathing_grid(self, pos: Union[Point2, Unit]) -> bool:
+        """Returns True if a ground unit can pass through a grid point.
+
+        :param pos:"""
+        assert isinstance(pos, (Point2, Unit)), "pos is not of type Point2 or Unit"
+        pos = pos.position.rounded
+        return self.game_info.pathing_grid[pos] == 1
+
+    def is_visible(self, pos: Union[Point2, Unit]) -> bool:
+        """Returns True if you have vision on a grid point.
+
+        :param pos:"""
+        # more info: https://github.com/Blizzard/s2client-proto/blob/9906df71d6909511907d8419b33acc1a3bd51ec0/s2clientprotocol/spatial.proto#L19
+        assert isinstance(pos, (Point2, Unit)), "pos is not of type Point2 or Unit"
+        pos = pos.position.rounded
+        return self.state.visibility[pos] == 2
+
+    def has_creep(self, pos: Union[Point2, Unit]) -> bool:
+        """Returns True if there is creep on the grid point.
+
+        :param pos:"""
+        assert isinstance(pos, (Point2, Unit)), "pos is not of type Point2 or Unit"
+        pos = pos.position.rounded
+        return self.state.creep[pos] == 1
+
+    async def on_unit_destroyed(self, unit_tag: int):
+        """
+        Override this in your bot class.
+        Note that this function uses unit tags and not the unit objects
+        because the unit does not exist any more.
+        This will event will be called when a unit (or structure, friendly or enemy) dies.
+        For enemy units, this only works if the enemy unit was in vision on death.
+
+        :param unit_tag:
+        """
+
+    async def on_unit_created(self, unit: Unit):
+        """Override this in your bot class. This function is called when a unit is created.
+
+        :param unit:"""
+
+    async def on_building_construction_started(self, unit: Unit):
+        """
+        Override this in your bot class.
+        This function is called when a building construction has started.
+
+        :param unit:
+        """
+
+    async def on_building_construction_complete(self, unit: Unit):
+        """
+        Override this in your bot class. This function is called when a building
+        construction is completed.
+
+        :param unit:
+        """
+
+    async def on_unit_took_damage(self, unit: Unit, amount_damage_taken: float):
+        """
+        Override this in your bot class. This function is called when your own unit (unit or structure) took damage.
+        It will not be called if the unit died this frame.
+
+        This may be called frequently for terran structures that are burning down, or zerg buildings that are off creep,
+        or terran bio units that just used stimpack ability.
+        TODO: If there is a demand for it, then I can add a similar event for when enemy units took damage
+
+        Examples::
+
+            print(f"My unit took damage: {unit} took {amount_damage_taken} damage")
+
+        :param unit:
+        :param amount_damage_taken:
+        """
+
+    async def on_enemy_unit_entered_vision(self, unit: Unit):
+        """
+        Override this in your bot class. This function is called when an enemy unit (unit or structure) entered vision (which was not visible last frame).
+
+        :param unit:
+        """
+
+    async def on_enemy_unit_left_vision(self, unit_tag: int):
+        """
+        Override this in your bot class. This function is called when an enemy unit (unit or structure) left vision (which was visible last frame).
+        Same as the self.on_unit_destroyed event, this function is called with the unit's tag because the unit is no longer visible anymore.
+        If you want to store a snapshot of the unit, use self._enemy_units_previous_map[unit_tag] for units or self._enemy_structures_previous_map[unit_tag] for structures.
+
+        Examples::
+
+            last_known_unit = self._enemy_units_previous_map.get(unit_tag, None) or self._enemy_structures_previous_map[unit_tag]
+            print(f"Enemy unit left vision, last known location: {last_known_unit.position}")
+
+        :param unit_tag:
+        """
+
+    async def on_before_start(self):
+        """
+        Override this in your bot class. This function is called before "on_start"
+        and before "prepare_first_step" that calculates expansion locations.
+        Not all data is available yet.
+        This function is useful in realtime=True mode to split your workers or start producing the first worker.
+        """
+
+    async def on_start(self):
+        """
+        Override this in your bot class.
+        At this point, game_data, game_info and the first iteration of game_state (self.state) are available.
+        """
+
+    async def on_step(self, iteration: int):
+        """
+        You need to implement this function!
+        Override this in your bot class.
+        This function is called on every game step (looped in realtime mode).
+
+        :param iteration:
+        """
+        raise NotImplementedError
+
+    async def on_end(self, game_result: Result):
+        """Override this in your bot class. This function is called at the end of a game.
+        Unsure if this function will be called on the laddermanager client as the bot process may forcefully be terminated.
+
+        :param game_result:"""

worlds/_sc2common/bot/bot_ai_internal.py

@@ -0,0 +1,490 @@
+# pylint: disable=W0201,W0212,R0912
+from __future__ import annotations
+
+import math
+import time
+import warnings
+from abc import ABC
+from collections import Counter
+from typing import TYPE_CHECKING, Any
+from typing import Dict, Generator, Iterable, List, Set, Tuple, Union, final
+
+from s2clientprotocol import sc2api_pb2 as sc_pb
+
+from .constants import (
+    IS_PLACEHOLDER,
+)
+from .data import Race
+from .game_data import GameData
+from .game_state import Blip, GameState
+from .pixel_map import PixelMap
+from .position import Point2
+from .unit import Unit
+from .units import Units
+
+# with warnings.catch_warnings():
+#     warnings.simplefilter("ignore")
+#     from scipy.spatial.distance import cdist, pdist
+
+if TYPE_CHECKING:
+    from .client import Client
+    from .game_info import GameInfo
+
+
+class BotAIInternal(ABC):
+    """Base class for bots."""
+
+    @final
+    def _initialize_variables(self):
+        """ Called from main.py internally """
+        self.cache: Dict[str, Any] = {}
+        # Specific opponent bot ID used in sc2ai ladder games http://sc2ai.net/ and on ai arena https://aiarena.net
+        # The bot ID will stay the same each game so your bot can "adapt" to the opponent
+        if not hasattr(self, "opponent_id"):
+            # Prevent overwriting the opponent_id which is set here https://github.com/Hannessa/python-sc2-ladderbot/blob/master/__init__.py#L40
+            # otherwise set it to None
+            self.opponent_id: str = None
+        # Select distance calculation method, see _distances_override_functions function
+        if not hasattr(self, "distance_calculation_method"):
+            self.distance_calculation_method: int = 2
+        # Select if the Unit.command should return UnitCommand objects. Set this to True if your bot uses 'self.do(unit(ability, target))'
+        if not hasattr(self, "unit_command_uses_self_do"):
+            self.unit_command_uses_self_do: bool = False
+        # This value will be set to True by main.py in self._prepare_start if game is played in realtime (if true, the bot will have limited time per step)
+        self.realtime: bool = False
+        self.base_build: int = -1
+        self.all_units: Units = Units([], self)
+        self.units: Units = Units([], self)
+        self.workers: Units = Units([], self)
+        self.larva: Units = Units([], self)
+        self.structures: Units = Units([], self)
+        self.townhalls: Units = Units([], self)
+        self.gas_buildings: Units = Units([], self)
+        self.all_own_units: Units = Units([], self)
+        self.enemy_units: Units = Units([], self)
+        self.enemy_structures: Units = Units([], self)
+        self.all_enemy_units: Units = Units([], self)
+        self.resources: Units = Units([], self)
+        self.destructables: Units = Units([], self)
+        self.watchtowers: Units = Units([], self)
+        self.mineral_field: Units = Units([], self)
+        self.vespene_geyser: Units = Units([], self)
+        self.placeholders: Units = Units([], self)
+        self.techlab_tags: Set[int] = set()
+        self.reactor_tags: Set[int] = set()
+        self.minerals: int = 50
+        self.vespene: int = 0
+        self.supply_army: float = 0
+        self.supply_workers: float = 12  # Doesn't include workers in production
+        self.supply_cap: float = 15
+        self.supply_used: float = 12
+        self.supply_left: float = 3
+        self.idle_worker_count: int = 0
+        self.army_count: int = 0
+        self.warp_gate_count: int = 0
+        self.blips: Set[Blip] = set()
+        self.race: Race = None
+        self.enemy_race: Race = None
+        self._generated_frame = -100
+        self._units_created: Counter = Counter()
+        self._unit_tags_seen_this_game: Set[int] = set()
+        self._units_previous_map: Dict[int, Unit] = {}
+        self._structures_previous_map: Dict[int, Unit] = {}
+        self._enemy_units_previous_map: Dict[int, Unit] = {}
+        self._enemy_structures_previous_map: Dict[int, Unit] = {}
+        self._all_units_previous_map: Dict[int, Unit] = {}
+        self._expansion_positions_list: List[Point2] = []
+        self._resource_location_to_expansion_position_dict: Dict[Point2, Point2] = {}
+        self._time_before_step: float = None
+        self._time_after_step: float = None
+        self._min_step_time: float = math.inf
+        self._max_step_time: float = 0
+        self._last_step_step_time: float = 0
+        self._total_time_in_on_step: float = 0
+        self._total_steps_iterations: int = 0
+        # Internally used to keep track which units received an action in this frame, so that self.train() function does not give the same larva two orders - cleared every frame
+        self.unit_tags_received_action: Set[int] = set()
+
+    @final
+    @property
+    def _game_info(self) -> GameInfo:
+        """ See game_info.py """
+        warnings.warn(
+            "Using self._game_info is deprecated and may be removed soon. Please use self.game_info directly.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self.game_info
+
+    @final
+    @property
+    def _game_data(self) -> GameData:
+        """ See game_data.py """
+        warnings.warn(
+            "Using self._game_data is deprecated and may be removed soon. Please use self.game_data directly.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self.game_data
+
+    @final
+    @property
+    def _client(self) -> Client:
+        """ See client.py """
+        warnings.warn(
+            "Using self._client is deprecated and may be removed soon. Please use self.client directly.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self.client
+
+    @final
+    def _prepare_start(self, client, player_id, game_info, game_data, realtime: bool = False, base_build: int = -1):
+        """
+        Ran until game start to set game and player data.
+
+        :param client:
+        :param player_id:
+        :param game_info:
+        :param game_data:
+        :param realtime:
+        """
+        self.client: Client = client
+        self.player_id: int = player_id
+        self.game_info: GameInfo = game_info
+        self.game_data: GameData = game_data
+        self.realtime: bool = realtime
+        self.base_build: int = base_build
+
+        self.race: Race = Race(self.game_info.player_races[self.player_id])
+
+        if len(self.game_info.player_races) == 2:
+            self.enemy_race: Race = Race(self.game_info.player_races[3 - self.player_id])
+
+
+    @final
+    def _prepare_first_step(self):
+        """First step extra preparations. Must not be called before _prepare_step."""
+        if self.townhalls:
+            self.game_info.player_start_location = self.townhalls.first.position
+            # Calculate and cache expansion locations forever inside 'self._cache_expansion_locations', this is done to prevent a bug when this is run and cached later in the game
+        self._time_before_step: float = time.perf_counter()
+
+    @final
+    def _prepare_step(self, state, proto_game_info):
+        """
+        :param state:
+        :param proto_game_info:
+        """
+        # Set attributes from new state before on_step."""
+        self.state: GameState = state  # See game_state.py
+        # update pathing grid, which unfortunately is in GameInfo instead of GameState
+        self.game_info.pathing_grid = PixelMap(proto_game_info.game_info.start_raw.pathing_grid, in_bits=True)
+        # Required for events, needs to be before self.units are initialized so the old units are stored
+        self._units_previous_map: Dict[int, Unit] = {unit.tag: unit for unit in self.units}
+        self._structures_previous_map: Dict[int, Unit] = {structure.tag: structure for structure in self.structures}
+        self._enemy_units_previous_map: Dict[int, Unit] = {unit.tag: unit for unit in self.enemy_units}
+        self._enemy_structures_previous_map: Dict[int, Unit] = {
+            structure.tag: structure
+            for structure in self.enemy_structures
+        }
+        self._all_units_previous_map: Dict[int, Unit] = {unit.tag: unit for unit in self.all_units}
+
+        self._prepare_units()
+        self.minerals: int = state.common.minerals
+        self.vespene: int = state.common.vespene
+        self.supply_army: int = state.common.food_army
+        self.supply_workers: int = state.common.food_workers  # Doesn't include workers in production
+        self.supply_cap: int = state.common.food_cap
+        self.supply_used: int = state.common.food_used
+        self.supply_left: int = self.supply_cap - self.supply_used
+
+        if self.race == Race.Zerg:
+            # Workaround Zerg supply rounding bug
+            pass
+            # self._correct_zerg_supply()
+        elif self.race == Race.Protoss:
+            self.warp_gate_count: int = state.common.warp_gate_count
+
+        self.idle_worker_count: int = state.common.idle_worker_count
+        self.army_count: int = state.common.army_count
+        self._time_before_step: float = time.perf_counter()
+
+        if self.enemy_race == Race.Random and self.all_enemy_units:
+            self.enemy_race = Race(self.all_enemy_units.first.race)
+
+    @final
+    def _prepare_units(self):
+        # Set of enemy units detected by own sensor tower, as blips have less unit information than normal visible units
+        self.blips: Set[Blip] = set()
+        self.all_units: Units = Units([], self)
+        self.units: Units = Units([], self)
+        self.workers: Units = Units([], self)
+        self.larva: Units = Units([], self)
+        self.structures: Units = Units([], self)
+        self.townhalls: Units = Units([], self)
+        self.gas_buildings: Units = Units([], self)
+        self.all_own_units: Units = Units([], self)
+        self.enemy_units: Units = Units([], self)
+        self.enemy_structures: Units = Units([], self)
+        self.all_enemy_units: Units = Units([], self)
+        self.resources: Units = Units([], self)
+        self.destructables: Units = Units([], self)
+        self.watchtowers: Units = Units([], self)
+        self.mineral_field: Units = Units([], self)
+        self.vespene_geyser: Units = Units([], self)
+        self.placeholders: Units = Units([], self)
+        self.techlab_tags: Set[int] = set()
+        self.reactor_tags: Set[int] = set()
+
+        index: int = 0
+        for unit in self.state.observation_raw.units:
+            if unit.is_blip:
+                self.blips.add(Blip(unit))
+            else:
+                unit_type: int = unit.unit_type
+                # Convert these units to effects: reaper grenade, parasitic bomb dummy, forcefield
+                unit_obj = Unit(unit, self, distance_calculation_index=index, base_build=self.base_build)
+                index += 1
+                self.all_units.append(unit_obj)
+                if unit.display_type == IS_PLACEHOLDER:
+                    self.placeholders.append(unit_obj)
+                    continue
+                alliance = unit.alliance
+                # Alliance.Neutral.value = 3
+                if alliance == 3:
+                    # XELNAGATOWER = 149
+                    if unit_type == 149:
+                        self.watchtowers.append(unit_obj)
+                    # all destructable rocks
+                    else:
+                        self.destructables.append(unit_obj)
+                # Alliance.Self.value = 1
+                elif alliance == 1:
+                    self.all_own_units.append(unit_obj)
+                    if unit_obj.is_structure:
+                        self.structures.append(unit_obj)
+                # Alliance.Enemy.value = 4
+                elif alliance == 4:
+                    self.all_enemy_units.append(unit_obj)
+                    if unit_obj.is_structure:
+                        self.enemy_structures.append(unit_obj)
+                    else:
+                        self.enemy_units.append(unit_obj)
+
+    @final
+    async def _after_step(self) -> int:
+        """ Executed by main.py after each on_step function. """
+        # Keep track of the bot on_step duration
+        self._time_after_step: float = time.perf_counter()
+        step_duration = self._time_after_step - self._time_before_step
+        self._min_step_time = min(step_duration, self._min_step_time)
+        self._max_step_time = max(step_duration, self._max_step_time)
+        self._last_step_step_time = step_duration
+        self._total_time_in_on_step += step_duration
+        self._total_steps_iterations += 1
+        # Clear set of unit tags that were given an order this frame by self.do()
+        self.unit_tags_received_action.clear()
+        # Commit debug queries
+        await self.client._send_debug()
+
+        return self.state.game_loop
+
+    @final
+    async def _advance_steps(self, steps: int):
+        """Advances the game loop by amount of 'steps'. This function is meant to be used as a debugging and testing tool only.
+        If you are using this, please be aware of the consequences, e.g. 'self.units' will be filled with completely new data."""
+        await self._after_step()
+        # Advance simulation by exactly "steps" frames
+        await self.client.step(steps)
+        state = await self.client.observation()
+        gs = GameState(state.observation)
+        proto_game_info = await self.client._execute(game_info=sc_pb.RequestGameInfo())
+        self._prepare_step(gs, proto_game_info)
+        await self.issue_events()
+
+    @final
+    async def issue_events(self):
+        """This function will be automatically run from main.py and triggers the following functions:
+        - on_unit_created
+        - on_unit_destroyed
+        - on_building_construction_started
+        - on_building_construction_complete
+        - on_upgrade_complete
+        """
+        await self._issue_unit_dead_events()
+        await self._issue_unit_added_events()
+        await self._issue_building_events()
+        await self._issue_upgrade_events()
+        await self._issue_vision_events()
+
+    @final
+    async def _issue_unit_added_events(self):
+        pass
+        # for unit in self.units:
+        #     if unit.tag not in self._units_previous_map and unit.tag not in self._unit_tags_seen_this_game:
+        #         self._unit_tags_seen_this_game.add(unit.tag)
+        #         self._units_created[unit.type_id] += 1
+        #         await self.on_unit_created(unit)
+        #     elif unit.tag in self._units_previous_map:
+        #         previous_frame_unit: Unit = self._units_previous_map[unit.tag]
+        #         # Check if a unit took damage this frame and then trigger event
+        #         if unit.health < previous_frame_unit.health or unit.shield < previous_frame_unit.shield:
+        #             damage_amount = previous_frame_unit.health - unit.health + previous_frame_unit.shield - unit.shield
+        #             await self.on_unit_took_damage(unit, damage_amount)
+        #         # Check if a unit type has changed
+        #         if previous_frame_unit.type_id != unit.type_id:
+        #             await self.on_unit_type_changed(unit, previous_frame_unit.type_id)
+
+    @final
+    async def _issue_upgrade_events(self):
+        pass
+        # difference = self.state.upgrades - self._previous_upgrades
+        # for upgrade_completed in difference:
+        #     await self.on_upgrade_complete(upgrade_completed)
+        # self._previous_upgrades = self.state.upgrades
+
+    @final
+    async def _issue_building_events(self):
+        pass
+        # for structure in self.structures:
+        #     if structure.tag not in self._structures_previous_map:
+        #         if structure.build_progress < 1:
+        #             await self.on_building_construction_started(structure)
+        #         else:
+        #             # Include starting townhall
+        #             self._units_created[structure.type_id] += 1
+        #             await self.on_building_construction_complete(structure)
+        #     elif structure.tag in self._structures_previous_map:
+        #         # Check if a structure took damage this frame and then trigger event
+        #         previous_frame_structure: Unit = self._structures_previous_map[structure.tag]
+        #         if (
+        #             structure.health < previous_frame_structure.health
+        #             or structure.shield < previous_frame_structure.shield
+        #         ):
+        #             damage_amount = (
+        #                 previous_frame_structure.health - structure.health + previous_frame_structure.shield -
+        #                 structure.shield
+        #             )
+        #             await self.on_unit_took_damage(structure, damage_amount)
+        #         # Check if a structure changed its type
+        #         if previous_frame_structure.type_id != structure.type_id:
+        #             await self.on_unit_type_changed(structure, previous_frame_structure.type_id)
+        #         # Check if structure completed
+        #         if structure.build_progress == 1 and previous_frame_structure.build_progress < 1:
+        #             self._units_created[structure.type_id] += 1
+        #             await self.on_building_construction_complete(structure)
+
+    @final
+    async def _issue_vision_events(self):
+        pass
+        # # Call events for enemy unit entered vision
+        # for enemy_unit in self.enemy_units:
+        #     if enemy_unit.tag not in self._enemy_units_previous_map:
+        #         await self.on_enemy_unit_entered_vision(enemy_unit)
+        # for enemy_structure in self.enemy_structures:
+        #     if enemy_structure.tag not in self._enemy_structures_previous_map:
+        #         await self.on_enemy_unit_entered_vision(enemy_structure)
+
+        # # Call events for enemy unit left vision
+        # enemy_units_left_vision: Set[int] = set(self._enemy_units_previous_map) - self.enemy_units.tags
+        # for enemy_unit_tag in enemy_units_left_vision:
+        #     await self.on_enemy_unit_left_vision(enemy_unit_tag)
+        # enemy_structures_left_vision: Set[int] = (set(self._enemy_structures_previous_map) - self.enemy_structures.tags)
+        # for enemy_structure_tag in enemy_structures_left_vision:
+        #     await self.on_enemy_unit_left_vision(enemy_structure_tag)
+
+    @final
+    async def _issue_unit_dead_events(self):
+        pass
+        # for unit_tag in self.state.dead_units & set(self._all_units_previous_map):
+        #     await self.on_unit_destroyed(unit_tag)
+
+    # DISTANCE CALCULATION
+
+    @final
+    @property
+    def _units_count(self) -> int:
+        return len(self.all_units)
+
+    # Helper functions
+
+    @final
+    def square_to_condensed(self, i, j) -> int:
+        # Converts indices of a square matrix to condensed matrix
+        # https://stackoverflow.com/a/36867493/10882657
+        assert i != j, "No diagonal elements in condensed matrix! Diagonal elements are zero"
+        if i < j:
+            i, j = j, i
+        return self._units_count * j - j * (j + 1) // 2 + i - 1 - j
+
+    # Fast and simple calculation functions
+
+    @final
+    @staticmethod
+    def distance_math_hypot(
+        p1: Union[Tuple[float, float], Point2],
+        p2: Union[Tuple[float, float], Point2],
+    ) -> float:
+        return math.hypot(p1[0] - p2[0], p1[1] - p2[1])
+
+    @final
+    @staticmethod
+    def distance_math_hypot_squared(
+        p1: Union[Tuple[float, float], Point2],
+        p2: Union[Tuple[float, float], Point2],
+    ) -> float:
+        return pow(p1[0] - p2[0], 2) + pow(p1[1] - p2[1], 2)
+
+    @final
+    def _distance_squared_unit_to_unit_method0(self, unit1: Unit, unit2: Unit) -> float:
+        return self.distance_math_hypot_squared(unit1.position_tuple, unit2.position_tuple)
+
+    # Distance calculation using the pre-calculated matrix above
+
+    @final
+    def _distance_squared_unit_to_unit_method1(self, unit1: Unit, unit2: Unit) -> float:
+        # If checked on units if they have the same tag, return distance 0 as these are not in the 1 dimensional pdist array - would result in an error otherwise
+        if unit1.tag == unit2.tag:
+            return 0
+        # Calculate index, needs to be after pdist has been calculated and cached
+        condensed_index = self.square_to_condensed(unit1.distance_calculation_index, unit2.distance_calculation_index)
+        assert condensed_index < len(
+            self._cached_pdist
+        ), f"Condensed index is larger than amount of calculated distances: {condensed_index} < {len(self._cached_pdist)}, units that caused the assert error: {unit1} and {unit2}"
+        distance = self._pdist[condensed_index]
+        return distance
+
+    @final
+    def _distance_squared_unit_to_unit_method2(self, unit1: Unit, unit2: Unit) -> float:
+        # Calculate index, needs to be after cdist has been calculated and cached
+        return self._cdist[unit1.distance_calculation_index, unit2.distance_calculation_index]
+
+    # Distance calculation using the fastest distance calculation functions
+
+    @final
+    def _distance_pos_to_pos(
+        self,
+        pos1: Union[Tuple[float, float], Point2],
+        pos2: Union[Tuple[float, float], Point2],
+    ) -> float:
+        return self.distance_math_hypot(pos1, pos2)
+
+    @final
+    def _distance_units_to_pos(
+        self,
+        units: Units,
+        pos: Union[Tuple[float, float], Point2],
+    ) -> Generator[float, None, None]:
+        """ This function does not scale well, if len(units) > 100 it gets fairly slow """
+        return (self.distance_math_hypot(u.position_tuple, pos) for u in units)
+
+    @final
+    def _distance_unit_to_points(
+        self,
+        unit: Unit,
+        points: Iterable[Tuple[float, float]],
+    ) -> Generator[float, None, None]:
+        """ This function does not scale well, if len(points) > 100 it gets fairly slow """
+        pos = unit.position_tuple
+        return (self.distance_math_hypot(p, pos) for p in points)

worlds/_sc2common/bot/cache.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Callable, Hashable, TypeVar
+
+if TYPE_CHECKING:
+    from .bot_ai import BotAI
+
+T = TypeVar("T")
+
+
+class CacheDict(dict):
+
+    def retrieve_and_set(self, key: Hashable, func: Callable[[], T]) -> T:
+        """ Either return the value at a certain key,
+        or set the return value of a function to that key, then return that value. """
+        if key not in self:
+            self[key] = func()
+        return self[key]
+
+
+class property_cache_once_per_frame(property):
+    """This decorator caches the return value for one game loop,
+    then clears it if it is accessed in a different game loop.
+    Only works on properties of the bot object, because it requires
+    access to self.state.game_loop
+
+    This decorator compared to the above runs a little faster, however you should only use this decorator if you are sure that you do not modify the mutable once it is calculated and cached.
+
+    Copied and modified from https://tedboy.github.io/flask/_modules/werkzeug/utils.html#cached_property
+    # """
+
+    def __init__(self, func: Callable[[BotAI], T], name=None):
+        # pylint: disable=W0231
+        self.__name__ = name or func.__name__
+        self.__frame__ = f"__frame__{self.__name__}"
+        self.func = func
+
+    def __set__(self, obj: BotAI, value: T):
+        obj.cache[self.__name__] = value
+        obj.cache[self.__frame__] = obj.state.game_loop
+
+    def __get__(self, obj: BotAI, _type=None) -> T:
+        value = obj.cache.get(self.__name__, None)
+        bot_frame = obj.state.game_loop
+        if value is None or obj.cache[self.__frame__] < bot_frame:
+            value = self.func(obj)
+            obj.cache[self.__name__] = value
+            obj.cache[self.__frame__] = bot_frame
+        return value

worlds/_sc2common/bot/client.py

@@ -0,0 +1,720 @@
+from __future__ import annotations
+
+from typing import Dict, Iterable, List, Optional, Set, Tuple, Union
+
+from worlds._sc2common.bot import logger
+
+from s2clientprotocol import debug_pb2 as debug_pb
+from s2clientprotocol import query_pb2 as query_pb
+from s2clientprotocol import raw_pb2 as raw_pb
+from s2clientprotocol import sc2api_pb2 as sc_pb
+from s2clientprotocol import spatial_pb2 as spatial_pb
+
+from .data import ActionResult, ChatChannel, Race, Result, Status
+from .game_data import AbilityData, GameData
+from .game_info import GameInfo
+from .position import Point2, Point3
+from .protocol import ConnectionAlreadyClosed, Protocol, ProtocolError
+from .renderer import Renderer
+from .unit import Unit
+from .units import Units
+
+
+# pylint: disable=R0904
+class Client(Protocol):
+
+    def __init__(self, ws, save_replay_path: str = None):
+        """
+        :param ws:
+        """
+        super().__init__(ws)
+        # How many frames will be waited between iterations before the next one is called
+        self.game_step: int = 4
+        self.save_replay_path: Optional[str] = save_replay_path
+        self._player_id = None
+        self._game_result = None
+        # Store a hash value of all the debug requests to prevent sending the same ones again if they haven't changed last frame
+        self._debug_hash_tuple_last_iteration: Tuple[int, int, int, int] = (0, 0, 0, 0)
+        self._debug_draw_last_frame = False
+        self._debug_texts = []
+        self._debug_lines = []
+        self._debug_boxes = []
+        self._debug_spheres = []
+
+        self._renderer = None
+        self.raw_affects_selection = False
+
+    @property
+    def in_game(self) -> bool:
+        return self._status in {Status.in_game, Status.in_replay}
+
+    async def join_game(self, name=None, race=None, observed_player_id=None, portconfig=None, rgb_render_config=None):
+        ifopts = sc_pb.InterfaceOptions(
+            raw=True,
+            score=True,
+            show_cloaked=True,
+            show_burrowed_shadows=True,
+            raw_affects_selection=self.raw_affects_selection,
+            raw_crop_to_playable_area=False,
+            show_placeholders=True,
+        )
+
+        if rgb_render_config:
+            assert isinstance(rgb_render_config, dict)
+            assert "window_size" in rgb_render_config and "minimap_size" in rgb_render_config
+            window_size = rgb_render_config["window_size"]
+            minimap_size = rgb_render_config["minimap_size"]
+            self._renderer = Renderer(self, window_size, minimap_size)
+            map_width, map_height = window_size
+            minimap_width, minimap_height = minimap_size
+
+            ifopts.render.resolution.x = map_width
+            ifopts.render.resolution.y = map_height
+            ifopts.render.minimap_resolution.x = minimap_width
+            ifopts.render.minimap_resolution.y = minimap_height
+
+        if race is None:
+            assert isinstance(observed_player_id, int), f"observed_player_id is of type {type(observed_player_id)}"
+            # join as observer
+            req = sc_pb.RequestJoinGame(observed_player_id=observed_player_id, options=ifopts)
+        else:
+            assert isinstance(race, Race)
+            req = sc_pb.RequestJoinGame(race=race.value, options=ifopts)
+
+        if portconfig:
+            req.server_ports.game_port = portconfig.server[0]
+            req.server_ports.base_port = portconfig.server[1]
+
+            for ppc in portconfig.players:
+                p = req.client_ports.add()
+                p.game_port = ppc[0]
+                p.base_port = ppc[1]
+
+        if name is not None:
+            assert isinstance(name, str), f"name is of type {type(name)}"
+            req.player_name = name
+
+        result = await self._execute(join_game=req)
+        self._game_result = None
+        self._player_id = result.join_game.player_id
+        return result.join_game.player_id
+
+    async def leave(self):
+        """ You can use 'await self.client.leave()' to surrender midst game. """
+        is_resign = self._game_result is None
+
+        if is_resign:
+            # For all clients that can leave, result of leaving the game either
+            # loss, or the client will ignore the result
+            self._game_result = {self._player_id: Result.Defeat}
+
+        try:
+            if self.save_replay_path is not None:
+                await self.save_replay(self.save_replay_path)
+                self.save_replay_path = None
+            await self._execute(leave_game=sc_pb.RequestLeaveGame())
+        except (ProtocolError, ConnectionAlreadyClosed):
+            if is_resign:
+                raise
+
+    async def save_replay(self, path):
+        logger.debug("Requesting replay from server")
+        result = await self._execute(save_replay=sc_pb.RequestSaveReplay())
+        with open(path, "wb") as f:
+            f.write(result.save_replay.data)
+        logger.info(f"Saved replay to {path}")
+
+    async def observation(self, game_loop: int = None):
+        if game_loop is not None:
+            result = await self._execute(observation=sc_pb.RequestObservation(game_loop=game_loop))
+        else:
+            result = await self._execute(observation=sc_pb.RequestObservation())
+        assert result.HasField("observation")
+
+        if not self.in_game or result.observation.player_result:
+            # Sometimes game ends one step before results are available
+            if not result.observation.player_result:
+                result = await self._execute(observation=sc_pb.RequestObservation())
+                assert result.observation.player_result
+
+            player_id_to_result = {}
+            for pr in result.observation.player_result:
+                player_id_to_result[pr.player_id] = Result(pr.result)
+            self._game_result = player_id_to_result
+        self._game_result = None
+
+        # if render_data is available, then RGB rendering was requested
+        if self._renderer and result.observation.observation.HasField("render_data"):
+            await self._renderer.render(result.observation)
+
+        return result
+
+    async def step(self, step_size: int = None):
+        """ EXPERIMENTAL: Change self._client.game_step during the step function to increase or decrease steps per second """
+        step_size = step_size or self.game_step
+        return await self._execute(step=sc_pb.RequestStep(count=step_size))
+
+    async def get_game_data(self) -> GameData:
+        result = await self._execute(
+            data=sc_pb.RequestData(ability_id=True, unit_type_id=True, upgrade_id=True, buff_id=True, effect_id=True)
+        )
+        return GameData(result.data)
+
+    async def dump_data(self, ability_id=True, unit_type_id=True, upgrade_id=True, buff_id=True, effect_id=True):
+        """
+        Dump the game data files
+        choose what data to dump in the keywords
+        this function writes to a text file
+        call it one time in on_step with:
+        await self._client.dump_data()
+        """
+        result = await self._execute(
+            data=sc_pb.RequestData(
+                ability_id=ability_id,
+                unit_type_id=unit_type_id,
+                upgrade_id=upgrade_id,
+                buff_id=buff_id,
+                effect_id=effect_id,
+            )
+        )
+        with open("data_dump.txt", "a") as file:
+            file.write(str(result.data))
+
+    async def get_game_info(self) -> GameInfo:
+        result = await self._execute(game_info=sc_pb.RequestGameInfo())
+        return GameInfo(result.game_info)
+
+    async def query_pathing(self, start: Union[Unit, Point2, Point3],
+                            end: Union[Point2, Point3]) -> Optional[Union[int, float]]:
+        """Caution: returns "None" when path not found
+        Try to combine queries with the function below because the pathing query is generally slow.
+
+        :param start:
+        :param end:"""
+        assert isinstance(start, (Point2, Unit))
+        assert isinstance(end, Point2)
+        if isinstance(start, Point2):
+            path = [query_pb.RequestQueryPathing(start_pos=start.as_Point2D, end_pos=end.as_Point2D)]
+        else:
+            path = [query_pb.RequestQueryPathing(unit_tag=start.tag, end_pos=end.as_Point2D)]
+        result = await self._execute(query=query_pb.RequestQuery(pathing=path))
+        distance = float(result.query.pathing[0].distance)
+        if distance <= 0.0:
+            return None
+        return distance
+
+    async def query_pathings(self, zipped_list: List[List[Union[Unit, Point2, Point3]]]) -> List[float]:
+        """Usage: await self.query_pathings([[unit1, target2], [unit2, target2]])
+        -> returns [distance1, distance2]
+        Caution: returns 0 when path not found
+
+        :param zipped_list:
+        """
+        assert zipped_list, "No zipped_list"
+        assert isinstance(zipped_list, list), f"{type(zipped_list)}"
+        assert isinstance(zipped_list[0], list), f"{type(zipped_list[0])}"
+        assert len(zipped_list[0]) == 2, f"{len(zipped_list[0])}"
+        assert isinstance(zipped_list[0][0], (Point2, Unit)), f"{type(zipped_list[0][0])}"
+        assert isinstance(zipped_list[0][1], Point2), f"{type(zipped_list[0][1])}"
+        if isinstance(zipped_list[0][0], Point2):
+            path = (
+                query_pb.RequestQueryPathing(start_pos=p1.as_Point2D, end_pos=p2.as_Point2D) for p1, p2 in zipped_list
+            )
+        else:
+            path = (query_pb.RequestQueryPathing(unit_tag=p1.tag, end_pos=p2.as_Point2D) for p1, p2 in zipped_list)
+        results = await self._execute(query=query_pb.RequestQuery(pathing=path))
+        return [float(d.distance) for d in results.query.pathing]
+
+    async def query_building_placement(
+        self,
+        ability: AbilityData,
+        positions: List[Union[Point2, Point3]],
+        ignore_resources: bool = True
+    ) -> List[ActionResult]:
+        """This function might be deleted in favor of the function above (_query_building_placement_fast).
+
+        :param ability:
+        :param positions:
+        :param ignore_resources:"""
+        assert isinstance(ability, AbilityData)
+        result = await self._execute(
+            query=query_pb.RequestQuery(
+                placements=(
+                    query_pb.RequestQueryBuildingPlacement(ability_id=ability.id.value, target_pos=position.as_Point2D)
+                    for position in positions
+                ),
+                ignore_resource_requirements=ignore_resources,
+            )
+        )
+        # Unnecessary converting to ActionResult?
+        return [ActionResult(p.result) for p in result.query.placements]
+
+    async def chat_send(self, message: str, team_only: bool):
+        """ Writes a message to the chat """
+        ch = ChatChannel.Team if team_only else ChatChannel.Broadcast
+        await self._execute(
+            action=sc_pb.RequestAction(
+                actions=[sc_pb.Action(action_chat=sc_pb.ActionChat(channel=ch.value, message=message))]
+            )
+        )
+
+    async def debug_kill_unit(self, unit_tags: Union[Unit, Units, List[int], Set[int]]):
+        """
+        :param unit_tags:
+        """
+        if isinstance(unit_tags, Units):
+            unit_tags = unit_tags.tags
+        if isinstance(unit_tags, Unit):
+            unit_tags = [unit_tags.tag]
+        assert unit_tags
+
+        await self._execute(
+            debug=sc_pb.RequestDebug(debug=[debug_pb.DebugCommand(kill_unit=debug_pb.DebugKillUnit(tag=unit_tags))])
+        )
+
+    async def move_camera(self, position: Union[Unit, Units, Point2, Point3]):
+        """Moves camera to the target position
+
+        :param position:"""
+        assert isinstance(position, (Unit, Units, Point2, Point3))
+        if isinstance(position, Units):
+            position = position.center
+        if isinstance(position, Unit):
+            position = position.position
+        await self._execute(
+            action=sc_pb.RequestAction(
+                actions=[
+                    sc_pb.Action(
+                        action_raw=raw_pb.ActionRaw(
+                            camera_move=raw_pb.ActionRawCameraMove(center_world_space=position.to3.as_Point)
+                        )
+                    )
+                ]
+            )
+        )
+
+    async def obs_move_camera(self, position: Union[Unit, Units, Point2, Point3]):
+        """Moves observer camera to the target position. Only works when observing (e.g. watching the replay).
+
+        :param position:"""
+        assert isinstance(position, (Unit, Units, Point2, Point3))
+        if isinstance(position, Units):
+            position = position.center
+        if isinstance(position, Unit):
+            position = position.position
+        await self._execute(
+            obs_action=sc_pb.RequestObserverAction(
+                actions=[
+                    sc_pb.ObserverAction(camera_move=sc_pb.ActionObserverCameraMove(world_pos=position.as_Point2D))
+                ]
+            )
+        )
+
+    async def move_camera_spatial(self, position: Union[Point2, Point3]):
+        """Moves camera to the target position using the spatial aciton interface
+
+        :param position:"""
+        assert isinstance(position, (Point2, Point3))
+        action = sc_pb.Action(
+            action_render=spatial_pb.ActionSpatial(
+                camera_move=spatial_pb.ActionSpatialCameraMove(center_minimap=position.as_PointI)
+            )
+        )
+        await self._execute(action=sc_pb.RequestAction(actions=[action]))
+
+    def debug_text_simple(self, text: str):
+        """ Draws a text in the top left corner of the screen (up to a max of 6 messages fit there). """
+        self._debug_texts.append(DrawItemScreenText(text=text, color=None, start_point=Point2((0, 0)), font_size=8))
+
+    def debug_text_screen(
+        self,
+        text: str,
+        pos: Union[Point2, Point3, tuple, list],
+        color: Union[tuple, list, Point3] = None,
+        size: int = 8,
+    ):
+        """
+        Draws a text on the screen (monitor / game window) with coordinates 0 <= x, y <= 1.
+
+        :param text:
+        :param pos:
+        :param color:
+        :param size:
+        """
+        assert len(pos) >= 2
+        assert 0 <= pos[0] <= 1
+        assert 0 <= pos[1] <= 1
+        pos = Point2((pos[0], pos[1]))
+        self._debug_texts.append(DrawItemScreenText(text=text, color=color, start_point=pos, font_size=size))
+
+    def debug_text_2d(
+        self,
+        text: str,
+        pos: Union[Point2, Point3, tuple, list],
+        color: Union[tuple, list, Point3] = None,
+        size: int = 8,
+    ):
+        return self.debug_text_screen(text, pos, color, size)
+
+    def debug_text_world(
+        self, text: str, pos: Union[Unit, Point3], color: Union[tuple, list, Point3] = None, size: int = 8
+    ):
+        """
+        Draws a text at Point3 position in the game world.
+        To grab a unit's 3d position, use unit.position3d
+        Usually the Z value of a Point3 is between 8 and 14 (except for flying units). Use self.get_terrain_z_height() from bot_ai.py to get the Z value (height) of the terrain at a 2D position.
+
+        :param text:
+        :param color:
+        :param size:
+        """
+        if isinstance(pos, Unit):
+            pos = pos.position3d
+        assert isinstance(pos, Point3)
+        self._debug_texts.append(DrawItemWorldText(text=text, color=color, start_point=pos, font_size=size))
+
+    def debug_text_3d(
+        self, text: str, pos: Union[Unit, Point3], color: Union[tuple, list, Point3] = None, size: int = 8
+    ):
+        return self.debug_text_world(text, pos, color, size)
+
+    def debug_line_out(
+        self, p0: Union[Unit, Point3], p1: Union[Unit, Point3], color: Union[tuple, list, Point3] = None
+    ):
+        """
+        Draws a line from p0 to p1.
+
+        :param p0:
+        :param p1:
+        :param color:
+        """
+        if isinstance(p0, Unit):
+            p0 = p0.position3d
+        assert isinstance(p0, Point3)
+        if isinstance(p1, Unit):
+            p1 = p1.position3d
+        assert isinstance(p1, Point3)
+        self._debug_lines.append(DrawItemLine(color=color, start_point=p0, end_point=p1))
+
+    def debug_box_out(
+        self,
+        p_min: Union[Unit, Point3],
+        p_max: Union[Unit, Point3],
+        color: Union[tuple, list, Point3] = None,
+    ):
+        """
+        Draws a box with p_min and p_max as corners of the box.
+
+        :param p_min:
+        :param p_max:
+        :param color:
+        """
+        if isinstance(p_min, Unit):
+            p_min = p_min.position3d
+        assert isinstance(p_min, Point3)
+        if isinstance(p_max, Unit):
+            p_max = p_max.position3d
+        assert isinstance(p_max, Point3)
+        self._debug_boxes.append(DrawItemBox(start_point=p_min, end_point=p_max, color=color))
+
+    def debug_box2_out(
+        self,
+        pos: Union[Unit, Point3],
+        half_vertex_length: float = 0.25,
+        color: Union[tuple, list, Point3] = None,
+    ):
+        """
+        Draws a box center at a position 'pos', with box side lengths (vertices) of two times 'half_vertex_length'.
+
+        :param pos:
+        :param half_vertex_length:
+        :param color:
+        """
+        if isinstance(pos, Unit):
+            pos = pos.position3d
+        assert isinstance(pos, Point3)
+        p0 = pos + Point3((-half_vertex_length, -half_vertex_length, -half_vertex_length))
+        p1 = pos + Point3((half_vertex_length, half_vertex_length, half_vertex_length))
+        self._debug_boxes.append(DrawItemBox(start_point=p0, end_point=p1, color=color))
+
+    def debug_sphere_out(self, p: Union[Unit, Point3], r: float, color: Union[tuple, list, Point3] = None):
+        """
+        Draws a sphere at point p with radius r.
+
+        :param p:
+        :param r:
+        :param color:
+        """
+        if isinstance(p, Unit):
+            p = p.position3d
+        assert isinstance(p, Point3)
+        self._debug_spheres.append(DrawItemSphere(start_point=p, radius=r, color=color))
+
+    async def _send_debug(self):
+        """Sends the debug draw execution. This is run by main.py now automatically, if there is any items in the list. You do not need to run this manually any longer.
+        Check examples/terran/ramp_wall.py for example drawing. Each draw request needs to be sent again in every single on_step iteration.
+        """
+        debug_hash = (
+            sum(hash(item) for item in self._debug_texts),
+            sum(hash(item) for item in self._debug_lines),
+            sum(hash(item) for item in self._debug_boxes),
+            sum(hash(item) for item in self._debug_spheres),
+        )
+        if debug_hash != (0, 0, 0, 0):
+            if debug_hash != self._debug_hash_tuple_last_iteration:
+                # Something has changed, either more or less is to be drawn, or a position of a drawing changed (e.g. when drawing on a moving unit)
+                self._debug_hash_tuple_last_iteration = debug_hash
+                try:
+                    await self._execute(
+                        debug=sc_pb.RequestDebug(
+                            debug=[
+                                debug_pb.DebugCommand(
+                                    draw=debug_pb.DebugDraw(
+                                        text=[text.to_proto()
+                                              for text in self._debug_texts] if self._debug_texts else None,
+                                        lines=[line.to_proto()
+                                               for line in self._debug_lines] if self._debug_lines else None,
+                                        boxes=[box.to_proto()
+                                               for box in self._debug_boxes] if self._debug_boxes else None,
+                                        spheres=[sphere.to_proto()
+                                                 for sphere in self._debug_spheres] if self._debug_spheres else None,
+                                    )
+                                )
+                            ]
+                        )
+                    )
+                except ProtocolError:
+                    return
+            self._debug_draw_last_frame = True
+            self._debug_texts.clear()
+            self._debug_lines.clear()
+            self._debug_boxes.clear()
+            self._debug_spheres.clear()
+        elif self._debug_draw_last_frame:
+            # Clear drawing if we drew last frame but nothing to draw this frame
+            self._debug_hash_tuple_last_iteration = (0, 0, 0, 0)
+            await self._execute(
+                debug=sc_pb.RequestDebug(
+                    debug=[
+                        debug_pb.DebugCommand(draw=debug_pb.DebugDraw(text=None, lines=None, boxes=None, spheres=None))
+                    ]
+                )
+            )
+            self._debug_draw_last_frame = False
+
+    async def debug_leave(self):
+        await self._execute(debug=sc_pb.RequestDebug(debug=[debug_pb.DebugCommand(end_game=debug_pb.DebugEndGame())]))
+
+    async def debug_set_unit_value(self, unit_tags: Union[Iterable[int], Units, Unit], unit_value: int, value: float):
+        """Sets a "unit value" (Energy, Life or Shields) of the given units to the given value.
+        Can't set the life of a unit to 0, use "debug_kill_unit" for that. Also can't set the life above the unit's maximum.
+        The following example sets the health of all your workers to 1:
+        await self.debug_set_unit_value(self.workers, 2, value=1)"""
+        if isinstance(unit_tags, Units):
+            unit_tags = unit_tags.tags
+        if isinstance(unit_tags, Unit):
+            unit_tags = [unit_tags.tag]
+        assert hasattr(
+            unit_tags, "__iter__"
+        ), f"unit_tags argument needs to be an iterable (list, dict, set, Units), given argument is {type(unit_tags).__name__}"
+        assert (
+            1 <= unit_value <= 3
+        ), f"unit_value needs to be between 1 and 3 (1 for energy, 2 for life, 3 for shields), given argument is {unit_value}"
+        assert all(tag > 0 for tag in unit_tags), f"Unit tags have invalid value: {unit_tags}"
+        assert isinstance(value, (int, float)), "Value needs to be of type int or float"
+        assert value >= 0, "Value can't be negative"
+        await self._execute(
+            debug=sc_pb.RequestDebug(
+                debug=(
+                    debug_pb.DebugCommand(
+                        unit_value=debug_pb.
+                        DebugSetUnitValue(unit_value=unit_value, value=float(value), unit_tag=unit_tag)
+                    ) for unit_tag in unit_tags
+                )
+            )
+        )
+
+    async def debug_hang(self, delay_in_seconds: float):
+        """ Freezes the SC2 client. Not recommended to be used. """
+        delay_in_ms = int(round(delay_in_seconds * 1000))
+        await self._execute(
+            debug=sc_pb.RequestDebug(
+                debug=[debug_pb.DebugCommand(test_process=debug_pb.DebugTestProcess(test=1, delay_ms=delay_in_ms))]
+            )
+        )
+
+    async def debug_show_map(self):
+        """ Reveals the whole map for the bot. Using it a second time disables it again. """
+        await self._execute(debug=sc_pb.RequestDebug(debug=[debug_pb.DebugCommand(game_state=1)]))
+
+    async def debug_control_enemy(self):
+        """ Allows control over enemy units and structures similar to team games control - does not allow the bot to spend the opponent's ressources. Using it a second time disables it again.  """
+        await self._execute(debug=sc_pb.RequestDebug(debug=[debug_pb.DebugCommand(game_state=2)]))
+
+    async def debug_food(self):
+        """ Should disable food usage (does not seem to work?). Using it a second time disables it again.  """
+        await self._execute(debug=sc_pb.RequestDebug(debug=[debug_pb.DebugCommand(game_state=3)]))
+
+    async def debug_free(self):
+        """ Units, structures and upgrades are free of mineral and gas cost. Using it a second time disables it again.  """
+        await self._execute(debug=sc_pb.RequestDebug(debug=[debug_pb.DebugCommand(game_state=4)]))
+
+    async def debug_all_resources(self):
+        """ Gives 5000 minerals and 5000 vespene to the bot. """
+        await self._execute(debug=sc_pb.RequestDebug(debug=[debug_pb.DebugCommand(game_state=5)]))
+
+    async def debug_god(self):
+        """ Your units and structures no longer take any damage. Using it a second time disables it again. """
+        await self._execute(debug=sc_pb.RequestDebug(debug=[debug_pb.DebugCommand(game_state=6)]))
+
+    async def debug_minerals(self):
+        """ Gives 5000 minerals to the bot. """
+        await self._execute(debug=sc_pb.RequestDebug(debug=[debug_pb.DebugCommand(game_state=7)]))
+
+    async def debug_gas(self):
+        """ Gives 5000 vespene to the bot. This does not seem to be working. """
+        await self._execute(debug=sc_pb.RequestDebug(debug=[debug_pb.DebugCommand(game_state=8)]))
+
+    async def debug_cooldown(self):
+        """ Disables cooldowns of unit abilities for the bot. Using it a second time disables it again. """
+        await self._execute(debug=sc_pb.RequestDebug(debug=[debug_pb.DebugCommand(game_state=9)]))
+
+    async def debug_tech_tree(self):
+        """ Removes all tech requirements (e.g. can build a factory without having a barracks). Using it a second time disables it again. """
+        await self._execute(debug=sc_pb.RequestDebug(debug=[debug_pb.DebugCommand(game_state=10)]))
+
+    async def debug_upgrade(self):
+        """ Researches all currently available upgrades. E.g. using it once unlocks combat shield, stimpack and 1-1. Using it a second time unlocks 2-2 and all other upgrades stay researched. """
+        await self._execute(debug=sc_pb.RequestDebug(debug=[debug_pb.DebugCommand(game_state=11)]))
+
+    async def debug_fast_build(self):
+        """ Sets the build time of units and structures and upgrades to zero. Using it a second time disables it again. """
+        await self._execute(debug=sc_pb.RequestDebug(debug=[debug_pb.DebugCommand(game_state=12)]))
+
+    async def quick_save(self):
+        """Saves the current game state to an in-memory bookmark.
+        See: https://github.com/Blizzard/s2client-proto/blob/eeaf5efaea2259d7b70247211dff98da0a2685a2/s2clientprotocol/sc2api.proto#L93"""
+        await self._execute(quick_save=sc_pb.RequestQuickSave())
+
+    async def quick_load(self):
+        """Loads the game state from the previously stored in-memory bookmark.
+        Caution:
+            - The SC2 Client will crash if the game wasn't quicksaved
+            - The bot step iteration counter will not reset
+            - self.state.game_loop will be set to zero after the quickload, and self.time is dependant on it"""
+        await self._execute(quick_load=sc_pb.RequestQuickLoad())
+
+
+class DrawItem:
+
+    @staticmethod
+    def to_debug_color(color: Union[tuple, Point3]):
+        """ Helper function for color conversion """
+        if color is None:
+            return debug_pb.Color(r=255, g=255, b=255)
+        # Need to check if not of type Point3 because Point3 inherits from tuple
+        if isinstance(color, (tuple, list)) and not isinstance(color, Point3) and len(color) == 3:
+            return debug_pb.Color(r=color[0], g=color[1], b=color[2])
+        # In case color is of type Point3
+        r = getattr(color, "r", getattr(color, "x", 255))
+        g = getattr(color, "g", getattr(color, "y", 255))
+        b = getattr(color, "b", getattr(color, "z", 255))
+        if max(r, g, b) <= 1:
+            r *= 255
+            g *= 255
+            b *= 255
+
+        return debug_pb.Color(r=int(r), g=int(g), b=int(b))
+
+
+class DrawItemScreenText(DrawItem):
+
+    def __init__(self, start_point: Point2 = None, color: Point3 = None, text: str = "", font_size: int = 8):
+        self._start_point: Point2 = start_point
+        self._color: Point3 = color
+        self._text: str = text
+        self._font_size: int = font_size
+
+    def to_proto(self):
+        return debug_pb.DebugText(
+            color=self.to_debug_color(self._color),
+            text=self._text,
+            virtual_pos=self._start_point.to3.as_Point,
+            world_pos=None,
+            size=self._font_size,
+        )
+
+    def __hash__(self):
+        return hash((self._start_point, self._color, self._text, self._font_size))
+
+
+class DrawItemWorldText(DrawItem):
+
+    def __init__(self, start_point: Point3 = None, color: Point3 = None, text: str = "", font_size: int = 8):
+        self._start_point: Point3 = start_point
+        self._color: Point3 = color
+        self._text: str = text
+        self._font_size: int = font_size
+
+    def to_proto(self):
+        return debug_pb.DebugText(
+            color=self.to_debug_color(self._color),
+            text=self._text,
+            virtual_pos=None,
+            world_pos=self._start_point.as_Point,
+            size=self._font_size,
+        )
+
+    def __hash__(self):
+        return hash((self._start_point, self._text, self._font_size, self._color))
+
+
+class DrawItemLine(DrawItem):
+
+    def __init__(self, start_point: Point3 = None, end_point: Point3 = None, color: Point3 = None):
+        self._start_point: Point3 = start_point
+        self._end_point: Point3 = end_point
+        self._color: Point3 = color
+
+    def to_proto(self):
+        return debug_pb.DebugLine(
+            line=debug_pb.Line(p0=self._start_point.as_Point, p1=self._end_point.as_Point),
+            color=self.to_debug_color(self._color),
+        )
+
+    def __hash__(self):
+        return hash((self._start_point, self._end_point, self._color))
+
+
+class DrawItemBox(DrawItem):
+
+    def __init__(self, start_point: Point3 = None, end_point: Point3 = None, color: Point3 = None):
+        self._start_point: Point3 = start_point
+        self._end_point: Point3 = end_point
+        self._color: Point3 = color
+
+    def to_proto(self):
+        return debug_pb.DebugBox(
+            min=self._start_point.as_Point,
+            max=self._end_point.as_Point,
+            color=self.to_debug_color(self._color),
+        )
+
+    def __hash__(self):
+        return hash((self._start_point, self._end_point, self._color))
+
+
+class DrawItemSphere(DrawItem):
+
+    def __init__(self, start_point: Point3 = None, radius: float = None, color: Point3 = None):
+        self._start_point: Point3 = start_point
+        self._radius: float = radius
+        self._color: Point3 = color
+
+    def to_proto(self):
+        return debug_pb.DebugSphere(
+            p=self._start_point.as_Point, r=self._radius, color=self.to_debug_color(self._color)
+        )
+
+    def __hash__(self):
+        return hash((self._start_point, self._radius, self._color))