Blasphemous: Fixed logic errors in WotHP

---------

Co-authored-by: Alchav <59858495+Alchav@users.noreply.github.com>

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -0,0 +1,35 @@
+name: Bug Report
+description: File a bug report.
+title: "Bug: "
+labels:
+  - bug / fix
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report! If this bug occurred during local generation check your 
+        Archipelago install for a log (probably `C:\ProgramData\Archipelago\logs`)
+        and upload it with this report, as well as all yaml files used.
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+    validations:
+      required: true
+  - type: textarea
+    id: expected-results
+    attributes:
+      label: What were the expected results?
+    validations:
+      required: true
+  - type: dropdown
+    id: version
+    attributes:
+      label: Software
+      description: Where did this bug occur?
+      options:
+        - Website
+        - Local generation
+        - While playing
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -0,0 +1,17 @@
+name: Feature Request
+description: Request a feature!
+title: "Category: "
+labels:
+  - enhancement
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Please replace `Category` in the title with what this feature will be targeting, such as Core generation,
+        website, documentation, or a game.
+        Note: this is not for requesting new games to be added. If you would like to request a game, the best place to
+        ask is about it is in the [discord](https://archipelago.gg/discord).
+  - type: textarea
+    id: feature
+    attributes:
+      label: What feature would you like to see?

.github/ISSUE_TEMPLATE/task.yaml

@@ -0,0 +1,10 @@
+name: Task
+description: Submit a task to be done. If this is not targeting core, it should likely be elsewhere.
+title: "Core: "
+labels:
+  - core
+  - enhancement
+body:
+  - type: textarea
+    attributes:
+      label: What task needs to be completed?

.github/pull_request_template.md

@@ -0,0 +1,12 @@
+Please format your title with what portion of the project this pull request is
+targeting and what it's changing.
+
+ex. "MyGame4: implement new game" or "Docs: add new guide for customizing MyGame3"
+
+## What is this fixing or adding?
+
+
+## How was this tested?
+
+
+## If this makes graphical changes, please attach screenshots.

.github/workflows/build.yml

@@ -2,7 +2,22 @@
 
 name: Build
 
-on: workflow_dispatch
+on:
+  push:
+    paths:
+      - '.github/workflows/build.yml'
+      - 'setup.py'
+      - 'requirements.txt'
+  pull_request:
+    paths:
+      - '.github/workflows/build.yml'
+      - 'setup.py'
+      - 'requirements.txt'
+  workflow_dispatch:
+
+env:
+  ENEMIZER_VERSION: 7.1
+  APPIMAGETOOL_VERSION: 13
 
 jobs:
   # build-release-macos: # LF volunteer
@@ -10,22 +25,19 @@ jobs:
   build-win-py38: # RCs will still be built and signed by hand
     runs-on: windows-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: '3.8'
       - name: Download run-time dependencies
         run: |
-          Invoke-WebRequest -Uri https://github.com/alttpo/sni/releases/download/v0.0.79/sni-v0.0.79-windows-amd64.zip -OutFile sni.zip
-          Expand-Archive -Path sni.zip -DestinationPath SNI -Force
-          Invoke-WebRequest -Uri https://github.com/Ijwu/Enemizer/releases/download/6.4/win-x64.zip -OutFile enemizer.zip
+          Invoke-WebRequest -Uri https://github.com/Ijwu/Enemizer/releases/download/${Env:ENEMIZER_VERSION}/win-x64.zip -OutFile enemizer.zip
           Expand-Archive -Path enemizer.zip -DestinationPath EnemizerCLI -Force
       - name: Build
         run: |
-          python -m pip install --upgrade pip setuptools==60.10.0  # 61 does not work with the current layout
-          pip install -r requirements.txt
-          python setup.py build --yes
+          python -m pip install --upgrade pip
+          python setup.py build_exe --yes
           $NAME="$(ls build)".Split('.',2)[1]
           $ZIP_NAME="Archipelago_$NAME.7z"
           echo "ZIP_NAME=$ZIP_NAME" >> $Env:GITHUB_ENV
@@ -34,49 +46,46 @@ jobs:
           Rename-Item exe.$NAME Archipelago
           7z a -mx=9 -mhe=on -ms "../dist/$ZIP_NAME" Archipelago
       - name: Store 7z
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ env.ZIP_NAME }}
           path: dist/${{ env.ZIP_NAME }}
           retention-days: 7  # keep for 7 days, should be enough
 
-  build-ubuntu1804:
-    runs-on: ubuntu-18.04
+  build-ubuntu2004:
+    runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
+      # - copy code below to release.yml -
+      - uses: actions/checkout@v3
       - name: Install base dependencies
         run: |
           sudo apt update
           sudo apt -y install build-essential p7zip xz-utils wget libglib2.0-0
           sudo apt -y install python3-gi libgirepository1.0-dev  # should pull dependencies for gi installation below
       - name: Get a recent python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: '3.9'
       - name: Install build-time dependencies
         run: |
           echo "PYTHON=python3.9" >> $GITHUB_ENV
-          wget -nv https://github.com/AppImage/AppImageKit/releases/download/13/appimagetool-x86_64.AppImage
+          wget -nv https://github.com/AppImage/AppImageKit/releases/download/$APPIMAGETOOL_VERSION/appimagetool-x86_64.AppImage
           chmod a+rx appimagetool-x86_64.AppImage
           ./appimagetool-x86_64.AppImage --appimage-extract
           echo -e '#/bin/sh\n./squashfs-root/AppRun "$@"' > appimagetool
           chmod a+rx appimagetool
       - name: Download run-time dependencies
         run: |
-          wget -nv https://github.com/black-sliver/sni/releases/download/v0.0.78-2/sni-v0.0.78-2-manylinux2014-amd64.tar.xz
-          tar xf sni-*.tar.xz
-          rm sni-*.tar.xz
-          mv sni-* SNI
-          wget -nv https://github.com/Ijwu/Enemizer/releases/download/6.4/ubuntu.16.04-x64.7z
+          wget -nv https://github.com/Ijwu/Enemizer/releases/download/$ENEMIZER_VERSION/ubuntu.16.04-x64.7z
           7za x -oEnemizerCLI/ ubuntu.16.04-x64.7z
       - name: Build
         run: |
           # pygobject is an optional dependency for kivy that's not in requirements
-          "${{ env.PYTHON }}" -m pip install --upgrade pip virtualenv PyGObject setuptools==60.10.0  # setuptools same as windows
+          # charset-normalizer was somehow incomplete in the github runner
           "${{ env.PYTHON }}" -m venv venv
           source venv/bin/activate
-          pip install -r requirements.txt
-          python setup.py build --yes bdist_appimage --yes
+          "${{ env.PYTHON }}" -m pip install --upgrade pip PyGObject charset-normalizer
+          python setup.py build_exe --yes bdist_appimage --yes
           echo -e "setup.py build output:\n `ls build`"
           echo -e "setup.py dist output:\n `ls dist`"
           cd dist && export APPIMAGE_NAME="`ls *.AppImage`" && cd ..
@@ -84,14 +93,19 @@ jobs:
           (cd build && DIR_NAME="`ls | grep exe`" && mv "$DIR_NAME" Archipelago && tar -czvf ../dist/$TAR_NAME Archipelago && mv Archipelago "$DIR_NAME")
           echo "APPIMAGE_NAME=$APPIMAGE_NAME" >> $GITHUB_ENV
           echo "TAR_NAME=$TAR_NAME" >> $GITHUB_ENV
+      # - copy code above to release.yml -
+      - name: Build Again
+        run: |
+          source venv/bin/activate
+          python setup.py build_exe --yes
       - name: Store AppImage
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ env.APPIMAGE_NAME }}
           path: dist/${{ env.APPIMAGE_NAME }}
           retention-days: 7
       - name: Store .tar.gz
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ env.TAR_NAME }}
           path: dist/${{ env.TAR_NAME }}

.github/workflows/codeql-analysis.yml

@@ -14,9 +14,17 @@ name: "CodeQL"
 on:
   push:
     branches: [ main ]
+    paths:
+      - '**.py'
+      - '**.js'
+      - '.github/workflows/codeql-analysis.yml'
   pull_request:
     # The branches below must be a subset of the branches above
     branches: [ main ]
+    paths:
+      - '**.py'
+      - '**.js'
+      - '.github/workflows/codeql-analysis.yml'
   schedule:
     - cron: '44 8 * * 1'
 
@@ -35,11 +43,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -50,7 +58,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -64,4 +72,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.github/workflows/lint.yml

@@ -3,23 +3,29 @@
 
 name: lint
 
-on: [push, pull_request]
+on:
+  push:
+    paths:
+      - '**.py'
+  pull_request:
+    paths:
+      - '**.py'
 
 jobs:
-  build:
+  flake8:
 
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python 3.9
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v4
       with:
         python-version: 3.9
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install flake8 pytest
+        python -m pip install --upgrade pip wheel
+        pip install flake8
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Lint with flake8
       run: |

.github/workflows/release.yml

@@ -7,6 +7,10 @@ on:
     tags:
       - '*.*.*'
 
+env:
+  ENEMIZER_VERSION: 7.1
+  APPIMAGETOOL_VERSION: 13
+
 jobs:
   create-release:
     runs-on: ubuntu-latest
@@ -25,45 +29,42 @@ jobs:
   # build-release-windows: # this is done by hand because of signing
   # build-release-macos: # LF volunteer
 
-  build-release-ubuntu1804:
-    runs-on: ubuntu-18.04
+  build-release-ubuntu2004:
+    runs-on: ubuntu-20.04
     steps:
       - name: Set env
         run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       # - code below copied from build.yml -
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install base dependencies
         run: |
           sudo apt update
           sudo apt -y install build-essential p7zip xz-utils wget libglib2.0-0
           sudo apt -y install python3-gi libgirepository1.0-dev  # should pull dependencies for gi installation below
       - name: Get a recent python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: '3.9'
       - name: Install build-time dependencies
         run: |
           echo "PYTHON=python3.9" >> $GITHUB_ENV
-          wget -nv https://github.com/AppImage/AppImageKit/releases/download/13/appimagetool-x86_64.AppImage
+          wget -nv https://github.com/AppImage/AppImageKit/releases/download/$APPIMAGETOOL_VERSION/appimagetool-x86_64.AppImage
           chmod a+rx appimagetool-x86_64.AppImage
           ./appimagetool-x86_64.AppImage --appimage-extract
           echo -e '#/bin/sh\n./squashfs-root/AppRun "$@"' > appimagetool
           chmod a+rx appimagetool
       - name: Download run-time dependencies
         run: |
-          wget -nv https://github.com/black-sliver/sni/releases/download/v0.0.78-2/sni-v0.0.78-2-manylinux2014-amd64.tar.xz
-          tar xf sni-*.tar.xz
-          rm sni-*.tar.xz
-          mv sni-* SNI
-          wget -nv https://github.com/Ijwu/Enemizer/releases/download/6.4/ubuntu.16.04-x64.7z
+          wget -nv https://github.com/Ijwu/Enemizer/releases/download/$ENEMIZER_VERSION/ubuntu.16.04-x64.7z
           7za x -oEnemizerCLI/ ubuntu.16.04-x64.7z
       - name: Build
         run: |
-          "${{ env.PYTHON }}" -m pip install --upgrade pip setuptools virtualenv PyGObject  # pygobject should probably move to requirements
+          # pygobject is an optional dependency for kivy that's not in requirements
+          # charset-normalizer was somehow incomplete in the github runner
           "${{ env.PYTHON }}" -m venv venv
           source venv/bin/activate
-          pip install -r requirements.txt
-          python setup.py build --yes bdist_appimage --yes
+          "${{ env.PYTHON }}" -m pip install --upgrade pip PyGObject charset-normalizer
+          python setup.py build_exe --yes bdist_appimage --yes
           echo -e "setup.py build output:\n `ls build`"
           echo -e "setup.py dist output:\n `ls dist`"
           cd dist && export APPIMAGE_NAME="`ls *.AppImage`" && cd ..

.github/workflows/unittests.yml

@@ -3,7 +3,25 @@
 
 name: unittests
 
-on: [push, pull_request]
+on:
+  push:
+    paths:
+      - '**'
+      - '!docs/**'
+      - '!setup.py'
+      - '!*.iss'
+      - '!.gitignore'
+      - '!.github/workflows/**'
+      - '.github/workflows/unittests.yml'
+  pull_request:
+    paths:
+      - '**'
+      - '!docs/**'
+      - '!setup.py'
+      - '!*.iss'
+      - '!.gitignore'
+      - '!.github/workflows/**'
+      - '.github/workflows/unittests.yml'
 
 jobs:
   build:
@@ -23,18 +41,20 @@ jobs:
             os: windows-latest
           - python: {version: '3.10'}  # current
             os: windows-latest
+          - python: {version: '3.10'}  # current
+            os: macos-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python.version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python.version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install flake8 pytest
+        pip install pytest pytest-subtests
         python ModuleUpdate.py --yes --force --append "WebHostLib/requirements.txt"
     - name: Unittests
       run: |
-        pytest test
+        pytest

.gitignore

@@ -4,15 +4,21 @@
 *_Spoiler.txt
 *.bmbp
 *.apbp
+*.apl2ac
 *.apm3
 *.apmc
 *.apz5
+*.aptloz
 *.pyc
 *.pyd
 *.sfc
 *.z64
 *.n64
 *.nes
+*.sms
+*.gb
+*.gbc
+*.gba
 *.wixobj
 *.lck
 *.db3
@@ -20,6 +26,7 @@
 *multisave
 *.archipelago
 *.apsave
+*.BIN
 
 build
 bundle/components.wxs
@@ -28,6 +35,7 @@ README.html
 .vs/
 EnemizerCLI/
 /Players/
+/SNI/
 /options.yaml
 /config.yaml
 /logs/
@@ -43,7 +51,9 @@ Output Logs/
 /freeze_requirements.txt
 /Archipelago.zip
 /setup.ini
-
+/installdelete.iss
+/data/user.kv
+/datapackage
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -116,17 +126,22 @@ target/
 profile_default/
 ipython_config.py
 
+# vim editor
+*.swp
+
 # SageMath parsed files
 *.sage.py
 
 # Environments
 .env
-.venv
+.venv*
 env/
 venv/
 ENV/
 env.bak/
 venv.bak/
+.code-workspace
+shell.nix
 
 # Spyder project settings
 .spyderproject
@@ -152,10 +167,18 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
-#minecraft server stuff
+# minecraft server stuff
 jdk*/
 minecraft*/
 minecraft_versions.json
+!worlds/minecraft/
 
-#pyenv
+# pyenv
 .python-version
+
+# OS General Files
+.DS_Store
+.AppleDouble
+.LSOverride
+Thumbs.db
+[Dd]esktop.ini

AdventureClient.py

@@ -0,0 +1,516 @@
+import asyncio
+import hashlib
+import json
+import time
+import os
+import bsdiff4
+import subprocess
+import zipfile
+from asyncio import StreamReader, StreamWriter, CancelledError
+from typing import List
+
+
+import Utils
+from NetUtils import ClientStatus
+from Utils import async_start
+from CommonClient import CommonContext, server_loop, gui_enabled, ClientCommandProcessor, logger, \
+    get_base_parser
+from worlds.adventure import AdventureDeltaPatch
+
+from worlds.adventure.Locations import base_location_id
+from worlds.adventure.Rom import AdventureForeignItemInfo, AdventureAutoCollectLocation, BatNoTouchLocation
+from worlds.adventure.Items import base_adventure_item_id, standard_item_max, item_table
+from worlds.adventure.Offsets import static_item_element_size, connector_port_offset
+
+SYSTEM_MESSAGE_ID = 0
+
+CONNECTION_TIMING_OUT_STATUS = \
+    "Connection timing out. Please restart your emulator, then restart connector_adventure.lua"
+CONNECTION_REFUSED_STATUS = \
+    "Connection Refused. Please start your emulator and make sure connector_adventure.lua is running"
+CONNECTION_RESET_STATUS = \
+    "Connection was reset. Please restart your emulator, then restart connector_adventure.lua"
+CONNECTION_TENTATIVE_STATUS = "Initial Connection Made"
+CONNECTION_CONNECTED_STATUS = "Connected"
+CONNECTION_INITIAL_STATUS = "Connection has not been initiated"
+
+SCRIPT_VERSION = 1
+
+
+class AdventureCommandProcessor(ClientCommandProcessor):
+    def __init__(self, ctx: CommonContext):
+        super().__init__(ctx)
+
+    def _cmd_2600(self):
+        """Check 2600 Connection State"""
+        if isinstance(self.ctx, AdventureContext):
+            logger.info(f"2600 Status: {self.ctx.atari_status}")
+
+    def _cmd_aconnect(self):
+        """Discard current atari 2600 connection state"""
+        if isinstance(self.ctx, AdventureContext):
+            self.ctx.atari_sync_task.cancel()
+
+
+class AdventureContext(CommonContext):
+    command_processor = AdventureCommandProcessor
+    game = 'Adventure'
+    lua_connector_port: int = 17242
+
+    def __init__(self, server_address, password):
+        super().__init__(server_address, password)
+        self.freeincarnates_used: int = -1
+        self.freeincarnate_pending: int = 0
+        self.foreign_items: [AdventureForeignItemInfo] = []
+        self.autocollect_items: [AdventureAutoCollectLocation] = []
+        self.atari_streams: (StreamReader, StreamWriter) = None
+        self.atari_sync_task = None
+        self.messages = {}
+        self.locations_array = None
+        self.atari_status = CONNECTION_INITIAL_STATUS
+        self.awaiting_rom = False
+        self.display_msgs = True
+        self.deathlink_pending = False
+        self.set_deathlink = False
+        self.client_compatibility_mode = 0
+        self.items_handling = 0b111
+        self.checked_locations_sent: bool = False
+        self.port_offset = 0
+        self.bat_no_touch_locations: [BatNoTouchLocation] = []
+        self.local_item_locations = {}
+        self.dragon_speed_info = {}
+
+        options = Utils.get_options()
+        self.display_msgs = options["adventure_options"]["display_msgs"]
+
+    async def server_auth(self, password_requested: bool = False):
+        if password_requested and not self.password:
+            await super(AdventureContext, self).server_auth(password_requested)
+        if not self.auth:
+            self.auth = self.player_name
+        if not self.auth:
+            self.awaiting_rom = True
+            logger.info('Awaiting connection to adventure_connector to get Player information')
+            return
+
+        await self.send_connect()
+
+    def _set_message(self, msg: str, msg_id: int):
+        if self.display_msgs:
+            self.messages[(time.time(), msg_id)] = msg
+
+    def on_package(self, cmd: str, args: dict):
+        if cmd == 'Connected':
+            self.locations_array = None
+            if Utils.get_options()["adventure_options"].get("death_link", False):
+                self.set_deathlink = True
+            async_start(self.get_freeincarnates_used())
+        elif cmd == "RoomInfo":
+            self.seed_name = args['seed_name']
+        elif cmd == 'Print':
+            msg = args['text']
+            if ': !' not in msg:
+                self._set_message(msg, SYSTEM_MESSAGE_ID)
+        elif cmd == "ReceivedItems":
+            msg = f"Received {', '.join([self.item_names[item.item] for item in args['items']])}"
+            self._set_message(msg, SYSTEM_MESSAGE_ID)
+        elif cmd == "Retrieved":
+            self.freeincarnates_used = args["keys"][f"adventure_{self.auth}_freeincarnates_used"]
+            if self.freeincarnates_used is None:
+                self.freeincarnates_used = 0
+            self.freeincarnates_used += self.freeincarnate_pending
+            self.send_pending_freeincarnates()
+        elif cmd == "SetReply":
+            if args["key"] == f"adventure_{self.auth}_freeincarnates_used":
+                self.freeincarnates_used = args["value"]
+                if self.freeincarnates_used is None:
+                    self.freeincarnates_used = 0
+                self.freeincarnates_used += self.freeincarnate_pending
+                self.send_pending_freeincarnates()
+
+    def on_deathlink(self, data: dict):
+        self.deathlink_pending = True
+        super().on_deathlink(data)
+
+    def run_gui(self):
+        from kvui import GameManager
+
+        class AdventureManager(GameManager):
+            logging_pairs = [
+                ("Client", "Archipelago")
+            ]
+            base_title = "Archipelago Adventure Client"
+
+        self.ui = AdventureManager(self)
+        self.ui_task = asyncio.create_task(self.ui.async_run(), name="UI")
+
+    async def get_freeincarnates_used(self):
+        if self.server and not self.server.socket.closed:
+            await self.send_msgs([{"cmd": "SetNotify", "keys": [f"adventure_{self.auth}_freeincarnates_used"]}])
+            await self.send_msgs([{"cmd": "Get", "keys": [f"adventure_{self.auth}_freeincarnates_used"]}])
+
+    def send_pending_freeincarnates(self):
+        if self.freeincarnate_pending > 0:
+            async_start(self.send_pending_freeincarnates_impl(self.freeincarnate_pending))
+            self.freeincarnate_pending = 0
+
+    async def send_pending_freeincarnates_impl(self, send_val: int) -> None:
+        await self.send_msgs([{"cmd": "Set", "key": f"adventure_{self.auth}_freeincarnates_used",
+                               "default": 0, "want_reply": False,
+                               "operations": [{"operation": "add", "value": send_val}]}])
+
+    async def used_freeincarnate(self) -> None:
+        if self.server and not self.server.socket.closed:
+            await self.send_msgs([{"cmd": "Set", "key": f"adventure_{self.auth}_freeincarnates_used",
+                                   "default": 0, "want_reply": True,
+                                   "operations": [{"operation": "add", "value": 1}]}])
+        else:
+            self.freeincarnate_pending = self.freeincarnate_pending + 1
+
+
+def convert_item_id(ap_item_id: int):
+    static_item_index = ap_item_id - base_adventure_item_id
+    return static_item_index * static_item_element_size
+
+
+def get_payload(ctx: AdventureContext):
+    current_time = time.time()
+    items = []
+    dragon_speed_update = {}
+    diff_a_locked = ctx.diff_a_mode > 0
+    diff_b_locked = ctx.diff_b_mode > 0
+    freeincarnate_count = 0
+    for item in ctx.items_received:
+        item_id_str = str(item.item)
+        if base_adventure_item_id < item.item <= standard_item_max:
+            items.append(convert_item_id(item.item))
+        elif item_id_str in ctx.dragon_speed_info:
+            if item.item in dragon_speed_update:
+                last_index = len(ctx.dragon_speed_info[item_id_str]) - 1
+                dragon_speed_update[item.item] = ctx.dragon_speed_info[item_id_str][last_index]
+            else:
+                dragon_speed_update[item.item] = ctx.dragon_speed_info[item_id_str][0]
+        elif item.item == item_table["Left Difficulty Switch"].id:
+            diff_a_locked = False
+        elif item.item == item_table["Right Difficulty Switch"].id:
+            diff_b_locked = False
+        elif item.item == item_table["Freeincarnate"].id:
+            freeincarnate_count = freeincarnate_count + 1
+    freeincarnates_available = 0
+
+    if ctx.freeincarnates_used >= 0:
+        freeincarnates_available = freeincarnate_count - (ctx.freeincarnates_used + ctx.freeincarnate_pending)
+    ret = json.dumps(
+        {
+            "items": items,
+            "messages": {f'{key[0]}:{key[1]}': value for key, value in ctx.messages.items()
+                         if key[0] > current_time - 10},
+            "deathlink": ctx.deathlink_pending,
+            "dragon_speeds": dragon_speed_update,
+            "difficulty_a_locked": diff_a_locked,
+            "difficulty_b_locked": diff_b_locked,
+            "freeincarnates_available": freeincarnates_available,
+            "bat_logic": ctx.bat_logic
+        }
+    )
+    ctx.deathlink_pending = False
+    return ret
+
+
+async def parse_locations(data: List, ctx: AdventureContext):
+    locations = data
+
+    # for loc_name, loc_data in location_table.items():
+
+    # if flags["EventFlag"][280] & 1 and not ctx.finished_game:
+    #    await ctx.send_msgs([
+    #                {"cmd": "StatusUpdate",
+    #                 "status": 30}
+    #            ])
+    #    ctx.finished_game = True
+    if locations == ctx.locations_array:
+        return
+    ctx.locations_array = locations
+    if locations is not None:
+        await ctx.send_msgs([{"cmd": "LocationChecks", "locations": locations}])
+
+
+def send_ap_foreign_items(adventure_context):
+    foreign_item_json_list = []
+    autocollect_item_json_list = []
+    bat_no_touch_locations_json_list = []
+    for fi in adventure_context.foreign_items:
+        foreign_item_json_list.append(fi.get_dict())
+    for fi in adventure_context.autocollect_items:
+        autocollect_item_json_list.append(fi.get_dict())
+    for ntl in adventure_context.bat_no_touch_locations:
+        bat_no_touch_locations_json_list.append(ntl.get_dict())
+    payload = json.dumps(
+        {
+            "foreign_items": foreign_item_json_list,
+            "autocollect_items": autocollect_item_json_list,
+            "local_item_locations": adventure_context.local_item_locations,
+            "bat_no_touch_locations": bat_no_touch_locations_json_list
+        }
+    )
+    print("sending foreign items")
+    msg = payload.encode()
+    (reader, writer) = adventure_context.atari_streams
+    writer.write(msg)
+    writer.write(b'\n')
+
+
+def send_checked_locations_if_needed(adventure_context):
+    if not adventure_context.checked_locations_sent and adventure_context.checked_locations is not None:
+        if len(adventure_context.checked_locations) == 0:
+            return
+        checked_short_ids = []
+        for location in adventure_context.checked_locations:
+            checked_short_ids.append(location - base_location_id)
+        print("Sending checked locations")
+        payload = json.dumps(
+            {
+                "checked_locations": checked_short_ids,
+            }
+        )
+        msg = payload.encode()
+        (reader, writer) = adventure_context.atari_streams
+        writer.write(msg)
+        writer.write(b'\n')
+        adventure_context.checked_locations_sent = True
+
+
+async def atari_sync_task(ctx: AdventureContext):
+    logger.info("Starting Atari 2600 connector. Use /2600 for status information")
+    while not ctx.exit_event.is_set():
+        try:
+            error_status = None
+            if ctx.atari_streams:
+                (reader, writer) = ctx.atari_streams
+                msg = get_payload(ctx).encode()
+                writer.write(msg)
+                writer.write(b'\n')
+                try:
+                    await asyncio.wait_for(writer.drain(), timeout=1.5)
+                    try:
+                        # Data will return a dict with 1+ fields
+                        # 1. A keepalive response of the Players Name (always)
+                        # 2. romhash field with sha256 hash of the ROM memory region
+                        # 3. locations, messages, and deathLink
+                        # 4. freeincarnate, to indicate a freeincarnate was used
+                        data = await asyncio.wait_for(reader.readline(), timeout=5)
+                        data_decoded = json.loads(data.decode())
+                        if 'scriptVersion' not in data_decoded or data_decoded['scriptVersion'] != SCRIPT_VERSION:
+                            msg = "You are connecting with an incompatible Lua script version. Ensure your connector " \
+                                  "Lua and AdventureClient are from the same Archipelago installation."
+                            logger.info(msg, extra={'compact_gui': True})
+                            ctx.gui_error('Error', msg)
+                            error_status = CONNECTION_RESET_STATUS
+                        if ctx.seed_name and bytes(ctx.seed_name, encoding='ASCII') != ctx.seed_name_from_data:
+                            msg = "The server is running a different multiworld than your client is. " \
+                                  "(invalid seed_name)"
+                            logger.info(msg, extra={'compact_gui': True})
+                            ctx.gui_error('Error', msg)
+                            error_status = CONNECTION_RESET_STATUS
+                        if 'romhash' in data_decoded:
+                            if ctx.rom_hash.upper() != data_decoded['romhash'].upper():
+                                msg = "The rom hash does not match the client rom hash data"
+                                print("got " + data_decoded['romhash'])
+                                print("expected " + str(ctx.rom_hash))
+                                logger.info(msg, extra={'compact_gui': True})
+                                ctx.gui_error('Error', msg)
+                                error_status = CONNECTION_RESET_STATUS
+                                if ctx.auth is None:
+                                    ctx.auth = ctx.player_name
+                            if ctx.awaiting_rom:
+                                await ctx.server_auth(False)
+                        if 'locations' in data_decoded and ctx.game and ctx.atari_status == CONNECTION_CONNECTED_STATUS \
+                                and not error_status and ctx.auth:
+                            # Not just a keep alive ping, parse
+                            async_start(parse_locations(data_decoded['locations'], ctx))
+                        if 'deathLink' in data_decoded and data_decoded['deathLink'] > 0 and 'DeathLink' in ctx.tags:
+                            dragon_name = "a dragon"
+                            if data_decoded['deathLink'] == 1:
+                                dragon_name = "Rhindle"
+                            elif data_decoded['deathLink'] == 2:
+                                dragon_name = "Yorgle"
+                            elif data_decoded['deathLink'] == 3:
+                                dragon_name = "Grundle"
+                            print (ctx.auth + " has been eaten by " + dragon_name )
+                            await ctx.send_death(ctx.auth + " has been eaten by " + dragon_name)
+                            # TODO - also if player reincarnates with a dragon onscreen ' dies to avoid being eaten by '
+                        if 'victory' in data_decoded and not ctx.finished_game:
+                            await ctx.send_msgs([{"cmd": "StatusUpdate", "status": ClientStatus.CLIENT_GOAL}])
+                            ctx.finished_game = True
+                        if 'freeincarnate' in data_decoded:
+                            await ctx.used_freeincarnate()
+                        if ctx.set_deathlink:
+                            await ctx.update_death_link(True)
+                        send_checked_locations_if_needed(ctx)
+                    except asyncio.TimeoutError:
+                        logger.debug("Read Timed Out, Reconnecting")
+                        error_status = CONNECTION_TIMING_OUT_STATUS
+                        writer.close()
+                        ctx.atari_streams = None
+                    except ConnectionResetError as e:
+                        logger.debug("Read failed due to Connection Lost, Reconnecting")
+                        error_status = CONNECTION_RESET_STATUS
+                        writer.close()
+                        ctx.atari_streams = None
+                except TimeoutError:
+                    logger.debug("Connection Timed Out, Reconnecting")
+                    error_status = CONNECTION_TIMING_OUT_STATUS
+                    writer.close()
+                    ctx.atari_streams = None
+                except ConnectionResetError:
+                    logger.debug("Connection Lost, Reconnecting")
+                    error_status = CONNECTION_RESET_STATUS
+                    writer.close()
+                    ctx.atari_streams = None
+                except CancelledError:
+                    logger.debug("Connection Cancelled, Reconnecting")
+                    error_status = CONNECTION_RESET_STATUS
+                    writer.close()
+                    ctx.atari_streams = None
+                    pass
+                except Exception as e:
+                    print("unknown exception " + e)
+                    raise
+                if ctx.atari_status == CONNECTION_TENTATIVE_STATUS:
+                    if not error_status:
+                        logger.info("Successfully Connected to 2600")
+                        ctx.atari_status = CONNECTION_CONNECTED_STATUS
+                        ctx.checked_locations_sent = False
+                        send_ap_foreign_items(ctx)
+                        send_checked_locations_if_needed(ctx)
+                    else:
+                        ctx.atari_status = f"Was tentatively connected but error occurred: {error_status}"
+                elif error_status:
+                    ctx.atari_status = error_status
+                    logger.info("Lost connection to 2600 and attempting to reconnect. Use /2600 for status updates")
+            else:
+                try:
+                    port = ctx.lua_connector_port + ctx.port_offset
+                    logger.debug(f"Attempting to connect to 2600 on port {port}")
+                    print(f"Attempting to connect to 2600 on port {port}")
+                    ctx.atari_streams = await asyncio.wait_for(
+                        asyncio.open_connection("localhost",
+                                                port),
+                                                timeout=10)
+                    ctx.atari_status = CONNECTION_TENTATIVE_STATUS
+                except TimeoutError:
+                    logger.debug("Connection Timed Out, Trying Again")
+                    ctx.atari_status = CONNECTION_TIMING_OUT_STATUS
+                    continue
+                except ConnectionRefusedError:
+                    logger.debug("Connection Refused, Trying Again")
+                    ctx.atari_status = CONNECTION_REFUSED_STATUS
+                    continue
+                except CancelledError:
+                    pass
+        except CancelledError:
+            pass
+    print("exiting atari sync task")
+
+
+async def run_game(romfile):
+    auto_start = Utils.get_options()["adventure_options"].get("rom_start", True)
+    rom_args = Utils.get_options()["adventure_options"].get("rom_args")
+    if auto_start is True:
+        import webbrowser
+        webbrowser.open(romfile)
+    elif os.path.isfile(auto_start):
+        open_args = [auto_start, romfile]
+        if rom_args is not None:
+            open_args.insert(1, rom_args)
+        subprocess.Popen(open_args,
+                         stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+
+async def patch_and_run_game(patch_file, ctx):
+    base_name = os.path.splitext(patch_file)[0]
+    comp_path = base_name + '.a26'
+    try:
+        base_rom = AdventureDeltaPatch.get_source_data()
+    except Exception as msg:
+        logger.info(msg, extra={'compact_gui': True})
+        ctx.gui_error('Error', msg)
+
+    with open(Utils.local_path("data", "adventure_basepatch.bsdiff4"), "rb") as file:
+        basepatch = bytes(file.read())
+
+    base_patched_rom_data = bsdiff4.patch(base_rom, basepatch)
+
+    with zipfile.ZipFile(patch_file, 'r') as patch_archive:
+        if not AdventureDeltaPatch.check_version(patch_archive):
+            logger.error("apadvn version doesn't match this client.  Make sure your generator and client are the same")
+            raise Exception("apadvn version doesn't match this client.")
+
+        ctx.foreign_items = AdventureDeltaPatch.read_foreign_items(patch_archive)
+        ctx.autocollect_items = AdventureDeltaPatch.read_autocollect_items(patch_archive)
+        ctx.local_item_locations = AdventureDeltaPatch.read_local_item_locations(patch_archive)
+        ctx.dragon_speed_info = AdventureDeltaPatch.read_dragon_speed_info(patch_archive)
+        ctx.seed_name_from_data, ctx.player_name = AdventureDeltaPatch.read_rom_info(patch_archive)
+        ctx.diff_a_mode, ctx.diff_b_mode = AdventureDeltaPatch.read_difficulty_switch_info(patch_archive)
+        ctx.bat_logic = AdventureDeltaPatch.read_bat_logic(patch_archive)
+        ctx.bat_no_touch_locations = AdventureDeltaPatch.read_bat_no_touch(patch_archive)
+        ctx.rom_deltas = AdventureDeltaPatch.read_rom_deltas(patch_archive)
+        ctx.auth = ctx.player_name
+
+    patched_rom_data = AdventureDeltaPatch.apply_rom_deltas(base_patched_rom_data, ctx.rom_deltas)
+    rom_hash = hashlib.sha256()
+    rom_hash.update(patched_rom_data)
+    ctx.rom_hash = rom_hash.hexdigest()
+    ctx.port_offset = patched_rom_data[connector_port_offset]
+
+    with open(comp_path, "wb") as patched_rom_file:
+        patched_rom_file.write(patched_rom_data)
+
+    async_start(run_game(comp_path))
+
+
+if __name__ == '__main__':
+
+    Utils.init_logging("AdventureClient")
+
+    async def main():
+        parser = get_base_parser()
+        parser.add_argument('patch_file', default="", type=str, nargs="?",
+                            help='Path to an ADVNTURE.BIN rom file')
+        parser.add_argument('port', default=17242, type=int, nargs="?",
+                            help='port for adventure_connector connection')
+        args = parser.parse_args()
+
+        ctx = AdventureContext(args.connect, args.password)
+        ctx.server_task = asyncio.create_task(server_loop(ctx), name="ServerLoop")
+        if gui_enabled:
+            ctx.run_gui()
+        ctx.run_cli()
+        ctx.atari_sync_task = asyncio.create_task(atari_sync_task(ctx), name="Adventure Sync")
+
+        if args.patch_file:
+            ext = args.patch_file.split(".")[len(args.patch_file.split(".")) - 1].lower()
+            if ext == "apadvn":
+                logger.info("apadvn file supplied, beginning patching process...")
+                async_start(patch_and_run_game(args.patch_file, ctx))
+            else:
+                logger.warning(f"Unknown patch file extension {ext}")
+        if args.port is int:
+            ctx.lua_connector_port = args.port
+
+        await ctx.exit_event.wait()
+        ctx.server_address = None
+
+        await ctx.shutdown()
+
+        if ctx.atari_sync_task:
+            await ctx.atari_sync_task
+            print("finished atari_sync_task (main)")
+
+
+    import colorama
+
+    colorama.init()
+
+    asyncio.run(main())
+    colorama.deinit()

ChecksFinderClient.py

@@ -1,229 +1,70 @@
 from __future__ import annotations
 import os
-import logging
-import asyncio
-import urllib.parse
 import sys
-import typing
-import time
+import asyncio
+import shutil
 
-import websockets
+import ModuleUpdate
+ModuleUpdate.update()
 
 import Utils
 
 if __name__ == "__main__":
     Utils.init_logging("ChecksFinderClient", exception_logger="Client")
 
-from MultiServer import CommandProcessor
-from NetUtils import Endpoint, decode, NetworkItem, encode, JSONtoTextParser, ClientStatus, Permission
-from Utils import Version, stream_input
-from worlds import network_data_package, AutoWorldRegister
-from CommonClient import gui_enabled, console_loop, logger, server_autoreconnect, get_base_parser, \
-    keep_alive
-from worlds.checksfinder import ChecksFinderWorld
+from NetUtils import NetworkItem, ClientStatus
+from CommonClient import gui_enabled, logger, get_base_parser, ClientCommandProcessor, \
+    CommonContext, server_loop
 
 
-class ClientCommandProcessor(CommandProcessor):
-    def __init__(self, ctx: CommonContext):
-        self.ctx = ctx
-
-    def output(self, text: str):
-        logger.info(text)
-
-    def _cmd_exit(self) -> bool:
-        """Close connections and client"""
-        self.ctx.exit_event.set()
-        return True
-
-    def _cmd_connect(self, address: str = "") -> bool:
-        """Connect to a MultiWorld Server"""
-        self.ctx.server_address = None
-        asyncio.create_task(self.ctx.connect(address if address else None), name="connecting")
-        return True
-
-    def _cmd_disconnect(self) -> bool:
-        """Disconnect from a MultiWorld Server"""
-        self.ctx.server_address = None
-        asyncio.create_task(self.ctx.disconnect(), name="disconnecting")
-        return True
-
-    def _cmd_received(self) -> bool:
-        """List all received items"""
-        logger.info(f'{len(self.ctx.items_received)} received items:')
-        for index, item in enumerate(self.ctx.items_received, 1):
-            self.output(f"{self.ctx.item_name_getter(item.item)} from {self.ctx.player_names[item.player]}")
-        return True
-
-    def _cmd_missing(self) -> bool:
-        """List all missing location checks, from your local game state"""
-        if not self.ctx.game:
-            self.output("No game set, cannot determine missing checks.")
-            return False
-        count = 0
-        checked_count = 0
-        for location, location_id in AutoWorldRegister.world_types[self.ctx.game].location_name_to_id.items():
-            if location_id < 0:
-                continue
-            if location_id not in self.ctx.locations_checked:
-                if location_id in self.ctx.missing_locations:
-                    self.output('Missing: ' + location)
-                    count += 1
-                elif location_id in self.ctx.checked_locations:
-                    self.output('Checked: ' + location)
-                    count += 1
-                    checked_count += 1
-
-        if count:
-            self.output(
-                f"Found {count} missing location checks{f'. {checked_count} location checks previously visited.' if checked_count else ''}")
-        else:
-            self.output("No missing location checks found.")
-        return True
-
-    def _cmd_items(self):
-        """List all item names for the currently running game."""
-        self.output(f"Item Names for {self.ctx.game}")
-        for item_name in AutoWorldRegister.world_types[self.ctx.game].item_name_to_id:
-            self.output(item_name)
-
-    def _cmd_locations(self):
-        """List all location names for the currently running game."""
-        self.output(f"Location Names for {self.ctx.game}")
-        for location_name in AutoWorldRegister.world_types[self.ctx.game].location_name_to_id:
-            self.output(location_name)
-
+class ChecksFinderClientCommandProcessor(ClientCommandProcessor):
     def _cmd_resync(self):
         """Manually trigger a resync."""
         self.output(f"Syncing items.")
         self.ctx.syncing = True
 
-    def _cmd_ready(self):
-        """Send ready status to server."""
-        self.ctx.ready = not self.ctx.ready
-        if self.ctx.ready:
-            state = ClientStatus.CLIENT_READY
-            self.output("Readied up.")
-        else:
-            state = ClientStatus.CLIENT_CONNECTED
-            self.output("Unreadied.")
-        asyncio.create_task(self.ctx.send_msgs([{"cmd": "StatusUpdate", "status": state}]), name="send StatusUpdate")
 
-    def default(self, raw: str):
-        raw = self.ctx.on_user_say(raw)
-        if raw:
-            asyncio.create_task(self.ctx.send_msgs([{"cmd": "Say", "text": raw}]), name="send Say")
-
-
-class CommonContext():
-    tags: typing.Set[str] = {"AP"}
-    starting_reconnect_delay: int = 5
-    current_reconnect_delay: int = starting_reconnect_delay
-    command_processor: int = ClientCommandProcessor
-    game = None
-    ui = None
-    keep_alive_task = None
-    items_handling: typing.Optional[int] = None
-    current_energy_link_value = 0  # to display in UI, gets set by server
+class ChecksFinderContext(CommonContext):
+    command_processor: int = ChecksFinderClientCommandProcessor
+    game = "ChecksFinder"
+    items_handling = 0b111  # full remote
 
     def __init__(self, server_address, password):
-        # server state
+        super(ChecksFinderContext, self).__init__(server_address, password)
         self.send_index: int = 0
-        self.server_address = server_address
-        self.password = password
         self.syncing = False
         self.awaiting_bridge = False
-        self.server_task = None
-        self.server: typing.Optional[Endpoint] = None
-        self.server_version = Version(0, 0, 0)
-        self.hint_cost: typing.Optional[int] = None
-        self.games: typing.Dict[int, str] = {}
-        self.permissions = {
-            "forfeit": "disabled",
-            "collect": "disabled",
-            "remaining": "disabled",
-        }
+        # self.game_communication_path: files go in this path to pass data between us and the actual game
+        if "localappdata" in os.environ:
+            self.game_communication_path = os.path.expandvars(r"%localappdata%/ChecksFinder")
+        else:
+            # not windows. game is an exe so let's see if wine might be around to run it
+            if "WINEPREFIX" in os.environ:
+                wineprefix = os.environ["WINEPREFIX"]
+            elif shutil.which("wine") or shutil.which("wine-stable"):
+                wineprefix = os.path.expanduser("~/.wine") # default root of wine system data, deep in which is app data
+            else:
+                msg = "ChecksFinderClient couldn't detect system type. Unable to infer required game_communication_path"
+                logger.error("Error: " + msg)
+                Utils.messagebox("Error", msg, error=True)
+                sys.exit(1)
+            self.game_communication_path = os.path.join(
+                wineprefix,
+                "drive_c",
+                os.path.expandvars("users/$USER/Local Settings/Application Data/ChecksFinder"))
 
-        # own state
-        self.finished_game = False
-        self.ready = False
-        self.team = None
-        self.slot = None
-        self.auth = None
-        self.seed_name = None
-
-        self.locations_checked: typing.Set[int] = set()  # local state
-        self.locations_scouted: typing.Set[int] = set()
-        self.items_received = []
-        self.missing_locations: typing.Set[int] = set()
-        self.checked_locations: typing.Set[int] = set()  # server state
-        self.locations_info = {}
-
-        self.input_queue = asyncio.Queue()
-        self.input_requests = 0
-
-        self.last_death_link: float = time.time()  # last send/received death link on AP layer
-
-        # game state
-        self.player_names: typing.Dict[int: str] = {0: "Archipelago"}
-        self.exit_event = asyncio.Event()
-        self.watcher_event = asyncio.Event()
-
-        self.slow_mode = False
-        self.jsontotextparser = JSONtoTextParser(self)
-        self.set_getters(network_data_package)
-
-        # execution
-        self.keep_alive_task = asyncio.create_task(keep_alive(self), name="Bouncy")
-
-    @property
-    def total_locations(self) -> typing.Optional[int]:
-        """Will return None until connected."""
-        if self.checked_locations or self.missing_locations:
-            return len(self.checked_locations | self.missing_locations)
+    async def server_auth(self, password_requested: bool = False):
+        if password_requested and not self.password:
+            await super(ChecksFinderContext, self).server_auth(password_requested)
+        await self.get_username()
+        await self.send_connect()
 
     async def connection_closed(self):
-        self.auth = None
-        self.items_received = []
-        self.locations_info = {}
-        self.server_version = Version(0, 0, 0)
-        if self.server and self.server.socket is not None:
-            await self.server.socket.close()
-        self.server = None
-        self.server_task = None
-        path = os.path.expandvars(r"%localappdata%/ChecksFinder")
-        for root, dirs, files in os.walk(path):
+        await super(ChecksFinderContext, self).connection_closed()
+        for root, dirs, files in os.walk(self.game_communication_path):
             for file in files:
                 if file.find("obtain") <= -1:
-                    os.remove(root+"/"+file)
-
-    # noinspection PyAttributeOutsideInit
-    def set_getters(self, data_package: dict, network=False):
-        if not network:  # local data; check if newer data was already downloaded
-            local_package = Utils.persistent_load().get("datapackage", {}).get("latest", {})
-            if local_package and local_package["version"] > network_data_package["version"]:
-                data_package: dict = local_package
-        elif network:  # check if data from server is newer
-
-            if data_package["version"] > network_data_package["version"]:
-                Utils.persistent_store("datapackage", "latest", network_data_package)
-
-        item_lookup: dict = {}
-        locations_lookup: dict = {}
-        for game, gamedata in data_package["games"].items():
-            for item_name, item_id in gamedata["item_name_to_id"].items():
-                item_lookup[item_id] = item_name
-            for location_name, location_id in gamedata["location_name_to_id"].items():
-                locations_lookup[location_id] = location_name
-
-        def get_item_name_from_id(code: int):
-            return item_lookup.get(code, f'Unknown item (ID:{code})')
-
-        self.item_name_getter = get_item_name_from_id
-
-        def get_location_name_from_address(address: int):
-            return locations_lookup.get(address, f'Unknown location (ID:{address})')
-
-        self.location_name_getter = get_location_name_from_address
+                    os.remove(root + "/" + file)
 
     @property
     def endpoints(self):
@@ -232,346 +73,52 @@ class CommonContext():
         else:
             return []
 
-    async def disconnect(self):
-        if self.server and not self.server.socket.closed:
-            await self.server.socket.close()
-        if self.server_task is not None:
-            await self.server_task
-
-    async def send_msgs(self, msgs):
-        if not self.server or not self.server.socket.open or self.server.socket.closed:
-            return
-        await self.server.socket.send(encode(msgs))
-
-    def consume_players_package(self, package: typing.List[tuple]):
-        self.player_names = {slot: name for team, slot, name, orig_name in package if self.team == team}
-        self.player_names[0] = "Archipelago"
-
-    def event_invalid_slot(self):
-        raise Exception('Invalid Slot; please verify that you have connected to the correct world.')
-
-    def event_invalid_game(self):
-        raise Exception('Invalid Game; please verify that you connected with the right game to the correct world.')
-
-    async def server_auth(self, password_requested: bool = False):
-        if password_requested and not self.password:
-            logger.info('Enter the password required to join this game:')
-            self.password = await self.console_input()
-            return self.password
-
-    async def send_connect(self, **kwargs):
-        payload = {
-            'cmd': 'Connect',
-            'password': self.password, 'name': self.auth, 'version': Utils.version_tuple,
-            'tags': self.tags, 'items_handling': self.items_handling,
-            'uuid': Utils.get_unique_identifier(), 'game': self.game
-        }
-        if kwargs:
-            payload.update(kwargs)
-        await self.send_msgs([payload])
-
-    async def console_input(self):
-        self.input_requests += 1
-        return await self.input_queue.get()
-
-    async def connect(self, address=None):
-        await self.disconnect()
-        self.server_task = asyncio.create_task(server_loop(self, address), name="server loop")
-
-    def on_print(self, args: dict):
-        logger.info(args["text"])
-
-    def on_print_json(self, args: dict):
-        if self.ui:
-            self.ui.print_json(args["data"])
-        else:
-            text = self.jsontotextparser(args["data"])
-            logger.info(text)
-
-    def on_package(self, cmd: str, args: dict):
-        pass
-
-    def on_user_say(self, text: str) -> typing.Optional[str]:
-        """Gets called before sending a Say to the server from the user.
-        Returned text is sent, or sending is aborted if None is returned."""
-        return text
-
-    def update_permissions(self, permissions: typing.Dict[str, int]):
-        for permission_name, permission_flag in permissions.items():
-            try:
-                flag = Permission(permission_flag)
-                logger.info(f"{permission_name.capitalize()} permission: {flag.name}")
-                self.permissions[permission_name] = flag.name
-            except Exception as e:  # safeguard against permissions that may be implemented in the future
-                logger.exception(e)
-
     async def shutdown(self):
-        self.server_address = None
-        if self.server and not self.server.socket.closed:
-            await self.server.socket.close()
-        if self.server_task:
-            await self.server_task
-
-        while self.input_requests > 0:
-            self.input_queue.put_nowait(None)
-            self.input_requests -= 1
-        self.keep_alive_task.cancel()
-        path = os.path.expandvars(r"%localappdata%/ChecksFinder")
-        for root, dirs, files in os.walk(path):
+        await super(ChecksFinderContext, self).shutdown()
+        for root, dirs, files in os.walk(self.game_communication_path):
             for file in files:
                 if file.find("obtain") <= -1:
                     os.remove(root+"/"+file)
 
-    # DeathLink hooks
-
-    def on_deathlink(self, data: dict):
-        """Gets dispatched when a new DeathLink is triggered by another linked player."""
-        self.last_death_link = max(data["time"], self.last_death_link)
-        text = data.get("cause", "")
-        if text:
-            logger.info(f"DeathLink: {text}")
-        else:
-            logger.info(f"DeathLink: Received from {data['source']}")
-
-    async def send_death(self, death_text: str = ""):
-        if self.server and self.server.socket:
-            logger.info("DeathLink: Sending death to your friends...")
-            self.last_death_link = time.time()
-            await self.send_msgs([{
-                "cmd": "Bounce", "tags": ["DeathLink"],
-                "data": {
-                    "time": self.last_death_link,
-                    "source": self.player_names[self.slot],
-                    "cause": death_text
-                }
-            }])
-
-    async def update_death_link(self, death_link):
-        old_tags = self.tags.copy()
-        if death_link:
-            self.tags.add("DeathLink")
-        else:
-            self.tags -= {"DeathLink"}
-        if old_tags != self.tags and self.server and not self.server.socket.closed:
-            await self.send_msgs([{"cmd": "ConnectUpdate", "tags": self.tags}])
-
-
-async def server_loop(ctx: CommonContext, address=None):
-    cached_address = None
-    if ctx.server and ctx.server.socket:
-        logger.error('Already connected')
-        return
-
-    if address is None:  # set through CLI or APBP
-        address = ctx.server_address
-
-    # Wait for the user to provide a multiworld server address
-    if not address:
-        logger.info('Please connect to an Archipelago server.')
-        return
-
-    address = f"ws://{address}" if "://" not in address else address
-    port = urllib.parse.urlparse(address).port or 38281
-    logger.info(f'Connecting to Archipelago server at {address}')
-    try:
-        socket = await websockets.connect(address, port=port, ping_timeout=None, ping_interval=None)
-        ctx.server = Endpoint(socket)
-        logger.info('Connected')
-        ctx.server_address = address
-        ctx.current_reconnect_delay = ctx.starting_reconnect_delay
-        async for data in ctx.server.socket:
-            for msg in decode(data):
-                await process_server_cmd(ctx, msg)
-        logger.warning('Disconnected from multiworld server, type /connect to reconnect')
-    except ConnectionRefusedError:
-        if cached_address:
-            logger.error('Unable to connect to multiworld server at cached address. '
-                         'Please use the connect button above.')
-        else:
-            logger.exception('Connection refused by the multiworld server')
-    except websockets.InvalidURI:
-        logger.exception('Failed to connect to the multiworld server (invalid URI)')
-    except (OSError, websockets.InvalidURI):
-        logger.exception('Failed to connect to the multiworld server')
-    except Exception as e:
-        logger.exception('Lost connection to the multiworld server, type /connect to reconnect')
-    finally:
-        await ctx.connection_closed()
-        if ctx.server_address:
-            logger.info(f"... reconnecting in {ctx.current_reconnect_delay}s")
-            asyncio.create_task(server_autoreconnect(ctx), name="server auto reconnect")
-        ctx.current_reconnect_delay *= 2
-
-
-async def process_server_cmd(ctx: CommonContext, args: dict):
-    try:
-        cmd = args["cmd"]
-    except:
-        logger.exception(f"Could not get command from {args}")
-        raise
-    if cmd == 'RoomInfo':
-        if ctx.seed_name and ctx.seed_name != args["seed_name"]:
-            logger.info("The server is running a different multiworld than your client is. (invalid seed_name)")
-        else:
-            logger.info('--------------------------------')
-            logger.info('Room Information:')
-            logger.info('--------------------------------')
-            version = args["version"]
-            ctx.server_version = tuple(version)
-            version = ".".join(str(item) for item in version)
-
-            logger.info(f'Server protocol version: {version}')
-            logger.info("Server protocol tags: " + ", ".join(args["tags"]))
-            if args['password']:
-                logger.info('Password required')
-            ctx.update_permissions(args.get("permissions", {}))
-            if "games" in args:
-                ctx.games = {x: game for x, game in enumerate(args["games"], start=1)}
-            logger.info(
-                f"A !hint costs {args['hint_cost']}% of your total location count as points"
-                f" and you get {args['location_check_points']}"
-                f" for each location checked. Use !hint for more information.")
-            ctx.hint_cost = int(args['hint_cost'])
-            ctx.check_points = int(args['location_check_points'])
-
-            if len(args['players']) < 1:
-                logger.info('No player connected')
-            else:
-                args['players'].sort()
-                current_team = -1
-                logger.info('Connected Players:')
-                for network_player in args['players']:
-                    if network_player.team != current_team:
-                        logger.info(f'  Team #{network_player.team + 1}')
-                        current_team = network_player.team
-                    logger.info('    %s (Player %d)' % (network_player.alias, network_player.slot))
-            if args["datapackage_version"] > network_data_package["version"] or args["datapackage_version"] == 0:
-                await ctx.send_msgs([{"cmd": "GetDataPackage"}])
-            await ctx.server_auth(args['password'])
-
-    elif cmd == 'DataPackage':
-        logger.info("Got new ID/Name Datapackage")
-        ctx.set_getters(args['data'], network=True)
-
-    elif cmd == 'ConnectionRefused':
-        errors = args["errors"]
-        if 'InvalidSlot' in errors:
-            ctx.event_invalid_slot()
-        elif 'InvalidGame' in errors:
-            ctx.event_invalid_game()
-        elif 'SlotAlreadyTaken' in errors:
-            raise Exception('Player slot already in use for that team')
-        elif 'IncompatibleVersion' in errors:
-            raise Exception('Server reported your client version as incompatible')
-        elif 'InvalidItemsHandling' in errors:
-            raise Exception('The item handling flags requested by the client are not supported')
-        # last to check, recoverable problem
-        elif 'InvalidPassword' in errors:
-            logger.error('Invalid password')
-            ctx.password = None
-            await ctx.server_auth(True)
-        elif errors:
-            raise Exception("Unknown connection errors: " + str(errors))
-        else:
-            raise Exception('Connection refused by the multiworld host, no reason provided')
-
-    elif cmd == 'Connected':
-        if not os.path.exists(os.path.expandvars(r"%localappdata%/ChecksFinder")):
-            os.mkdir(os.path.expandvars(r"%localappdata%/ChecksFinder"))
-        ctx.team = args["team"]
-        ctx.slot = args["slot"]
-        ctx.consume_players_package(args["players"])
-        msgs = []
-        if ctx.locations_checked:
-            msgs.append({"cmd": "LocationChecks",
-                         "locations": list(ctx.locations_checked)})
-        if ctx.locations_scouted:
-            msgs.append({"cmd": "LocationScouts",
-                         "locations": list(ctx.locations_scouted)})
-        if msgs:
-            await ctx.send_msgs(msgs)
-        if ctx.finished_game:
-            await ctx.send_msgs([{"cmd": "StatusUpdate", "status": ClientStatus.CLIENT_GOAL}])
-
-        # Get the server side view of missing as of time of connecting.
-        # This list is used to only send to the server what is reported as ACTUALLY Missing.
-        # This also serves to allow an easy visual of what locations were already checked previously
-        # when /missing is used for the client side view of what is missing.
-        ctx.missing_locations = set(args["missing_locations"])
-        ctx.checked_locations = set(args["checked_locations"])
-        for ss in ctx.checked_locations:
-            filename = f"send{ss}"
-            with open(os.path.expandvars(r"%localappdata%/ChecksFinder/"+filename), 'w') as f:
-                f.close()
-
-    elif cmd == 'ReceivedItems':
-        start_index = args["index"]
-
-        if start_index == 0:
-            ctx.items_received = []
-        elif start_index != len(ctx.items_received):
-            sync_msg = [{'cmd': 'Sync'}]
-            if ctx.locations_checked:
-                sync_msg.append({"cmd": "LocationChecks",
-                                 "locations": list(ctx.locations_checked)})
-            await ctx.send_msgs(sync_msg)
-        if start_index == len(ctx.items_received):
-            for item in args['items']:
-                filename = f"AP_{str(NetworkItem(*item).location)}PLR{str(NetworkItem(*item).player)}.item"
-                with open(os.path.expandvars(r"%localappdata%/ChecksFinder/"+filename), 'w') as f:
-                    f.write(str(NetworkItem(*item).item))
-                    f.close()
-                ctx.items_received.append(NetworkItem(*item))
-        ctx.watcher_event.set()
-
-    elif cmd == 'LocationInfo':
-        for item, location, player in args['locations']:
-            if location not in ctx.locations_info:
-                ctx.locations_info[location] = (item, player)
-        ctx.watcher_event.set()
-
-    elif cmd == "RoomUpdate":
-        if "players" in args:
-            ctx.consume_players_package(args["players"])
-        if "hint_points" in args:
-            ctx.hint_points = args['hint_points']
-        if "checked_locations" in args:
-            checked = set(args["checked_locations"])
-            ctx.checked_locations |= checked
-            ctx.missing_locations -= checked
-            for ss in ctx.checked_locations:
+    def on_package(self, cmd: str, args: dict):
+        if cmd in {"Connected"}:
+            if not os.path.exists(self.game_communication_path):
+                os.makedirs(self.game_communication_path)
+            for ss in self.checked_locations:
                 filename = f"send{ss}"
-                with open(os.path.expandvars(r"%localappdata%/ChecksFinder/"+filename), 'w') as f:
+                with open(os.path.join(self.game_communication_path, filename), 'w') as f:
                     f.close()
-        if "permissions" in args:
-            ctx.update_permissions(args["permissions"])
+        if cmd in {"ReceivedItems"}:
+            start_index = args["index"]
+            if start_index != len(self.items_received):
+                for item in args['items']:
+                    filename = f"AP_{str(NetworkItem(*item).location)}PLR{str(NetworkItem(*item).player)}.item"
+                    with open(os.path.join(self.game_communication_path, filename), 'w') as f:
+                        f.write(str(NetworkItem(*item).item))
+                        f.close()
 
-    elif cmd == 'Print':
-        ctx.on_print(args)
+        if cmd in {"RoomUpdate"}:
+            if "checked_locations" in args:
+                for ss in self.checked_locations:
+                    filename = f"send{ss}"
+                    with open(os.path.join(self.game_communication_path, filename), 'w') as f:
+                        f.close()
 
-    elif cmd == 'PrintJSON':
-        ctx.on_print_json(args)
+    def run_gui(self):
+        """Import kivy UI system and start running it as self.ui_task."""
+        from kvui import GameManager
 
-    elif cmd == 'InvalidPacket':
-        logger.warning(f"Invalid Packet of {args['type']}: {args['text']}")
+        class ChecksFinderManager(GameManager):
+            logging_pairs = [
+                ("Client", "Archipelago")
+            ]
+            base_title = "Archipelago ChecksFinder Client"
 
-    elif cmd == "Bounced":
-        tags = args.get("tags", [])
-        # we can skip checking "DeathLink" in ctx.tags, as otherwise we wouldn't have been send this
-        if "DeathLink" in tags and ctx.last_death_link != args["data"]["time"]:
-            ctx.on_deathlink(args["data"])
-    elif cmd == "SetReply":
-        if args["key"] == "EnergyLink":
-            ctx.current_energy_link_value = args["value"]
-            if ctx.ui:
-                ctx.ui.set_new_energy_link_value()
-    else:
-        logger.debug(f"unknown command {cmd}")
-
-    ctx.on_package(cmd, args)
+        self.ui = ChecksFinderManager(self)
+        self.ui_task = asyncio.create_task(self.ui.async_run(), name="UI")
 
 
-async def game_watcher(ctx: CommonContext):
+async def game_watcher(ctx: ChecksFinderContext):
     from worlds.checksfinder.Locations import lookup_id_to_name
     while not ctx.exit_event.is_set():
         if ctx.syncing == True:
@@ -580,10 +127,9 @@ async def game_watcher(ctx: CommonContext):
                 sync_msg.append({"cmd": "LocationChecks", "locations": list(ctx.locations_checked)})
             await ctx.send_msgs(sync_msg)
             ctx.syncing = False
-        path = os.path.expandvars(r"%localappdata%/ChecksFinder")
         sending = []
         victory = False
-        for root, dirs, files in os.walk(path):
+        for root, dirs, files in os.walk(ctx.game_communication_path):
             for file in files:
                 if file.find("send") > -1:
                     st = file.split("send", -1)[1]
@@ -600,38 +146,12 @@ async def game_watcher(ctx: CommonContext):
 
 
 if __name__ == '__main__':
-    # Text Mode to use !hint and such with games that have no text entry
-
-    class TextContext(CommonContext):
-        game = "ChecksFinder"
-        items_handling = 0b111  # full remote
-
-        async def server_auth(self, password_requested: bool = False):
-            if password_requested and not self.password:
-                await super(TextContext, self).server_auth(password_requested)
-            if not self.auth:
-                logger.info('Enter slot name:')
-                self.auth = await self.console_input()
-
-            await self.send_connect()
-
-        def on_package(self, cmd: str, args: dict):
-            if cmd == "Connected":
-                self.game = self.games.get(self.slot, None)
-
-
     async def main(args):
-        ctx = TextContext(args.connect, args.password)
+        ctx = ChecksFinderContext(args.connect, args.password)
         ctx.server_task = asyncio.create_task(server_loop(ctx), name="server loop")
-        input_task = None
         if gui_enabled:
-            from kvui import ChecksFinderManager
-            ctx.ui = ChecksFinderManager(ctx)
-            ui_task = asyncio.create_task(ctx.ui.async_run(), name="UI")
-        else:
-            ui_task = None
-        if sys.stdin:
-            input_task = asyncio.create_task(console_loop(ctx), name="Input")
+            ctx.run_gui()
+        ctx.run_cli()
         progression_watcher = asyncio.create_task(
             game_watcher(ctx), name="ChecksFinderProgressionWatcher")
 
@@ -641,11 +161,6 @@ if __name__ == '__main__':
         await progression_watcher
 
         await ctx.shutdown()
-        if ui_task:
-            await ui_task
-
-        if input_task:
-            input_task.cancel()
 
     import colorama
 
@@ -653,8 +168,5 @@ if __name__ == '__main__':
 
     args, rest = parser.parse_known_args()
     colorama.init()
-
-    loop = asyncio.get_event_loop()
-    loop.run_until_complete(main(args))
-    loop.close()
+    asyncio.run(main(args))
     colorama.deinit()

CommonClient.py

@@ -5,6 +5,7 @@ import urllib.parse
 import sys
 import typing
 import time
+import functools
 
 import ModuleUpdate
 ModuleUpdate.update()
@@ -17,11 +18,15 @@ if __name__ == "__main__":
     Utils.init_logging("TextClient", exception_logger="Client")
 
 from MultiServer import CommandProcessor
-from NetUtils import Endpoint, decode, NetworkItem, encode, JSONtoTextParser, ClientStatus, Permission, NetworkSlot
-from Utils import Version, stream_input
+from NetUtils import Endpoint, decode, NetworkItem, encode, JSONtoTextParser, \
+    ClientStatus, Permission, NetworkSlot, RawJSONtoTextParser
+from Utils import Version, stream_input, async_start
 from worlds import network_data_package, AutoWorldRegister
 import os
 
+if typing.TYPE_CHECKING:
+    import kvui
+
 logger = logging.getLogger("Client")
 
 # without terminal, we have to use gui mode
@@ -42,31 +47,38 @@ class ClientCommandProcessor(CommandProcessor):
 
     def _cmd_connect(self, address: str = "") -> bool:
         """Connect to a MultiWorld Server"""
-        self.ctx.server_address = None
-        asyncio.create_task(self.ctx.connect(address if address else None), name="connecting")
+        if address:
+            self.ctx.server_address = None
+            self.ctx.username = None
+        elif not self.ctx.server_address:
+            self.output("Please specify an address.")
+            return False
+        async_start(self.ctx.connect(address if address else None), name="connecting")
         return True
 
     def _cmd_disconnect(self) -> bool:
         """Disconnect from a MultiWorld Server"""
-        self.ctx.server_address = None
-        asyncio.create_task(self.ctx.disconnect(), name="disconnecting")
+        async_start(self.ctx.disconnect(), name="disconnecting")
         return True
 
     def _cmd_received(self) -> bool:
         """List all received items"""
-        logger.info(f'{len(self.ctx.items_received)} received items:')
+        self.output(f'{len(self.ctx.items_received)} received items:')
         for index, item in enumerate(self.ctx.items_received, 1):
-            self.output(f"{self.ctx.item_name_getter(item.item)} from {self.ctx.player_names[item.player]}")
+            self.output(f"{self.ctx.item_names[item.item]} from {self.ctx.player_names[item.player]}")
         return True
 
-    def _cmd_missing(self) -> bool:
-        """List all missing location checks, from your local game state"""
+    def _cmd_missing(self, filter_text = "") -> bool:
+        """List all missing location checks, from your local game state.
+        Can be given text, which will be used as filter."""
         if not self.ctx.game:
             self.output("No game set, cannot determine missing checks.")
             return False
         count = 0
         checked_count = 0
         for location, location_id in AutoWorldRegister.world_types[self.ctx.game].location_name_to_id.items():
+            if filter_text and filter_text not in location:
+                continue
             if location_id < 0:
                 continue
             if location_id not in self.ctx.locations_checked:
@@ -87,12 +99,18 @@ class ClientCommandProcessor(CommandProcessor):
 
     def _cmd_items(self):
         """List all item names for the currently running game."""
+        if not self.ctx.game:
+            self.output("No game set, cannot determine existing items.")
+            return False
         self.output(f"Item Names for {self.ctx.game}")
         for item_name in AutoWorldRegister.world_types[self.ctx.game].item_name_to_id:
             self.output(item_name)
 
     def _cmd_locations(self):
         """List all location names for the currently running game."""
+        if not self.ctx.game:
+            self.output("No game set, cannot determine existing locations.")
+            return False
         self.output(f"Location Names for {self.ctx.game}")
         for location_name in AutoWorldRegister.world_types[self.ctx.game].location_name_to_id:
             self.output(location_name)
@@ -106,40 +124,81 @@ class ClientCommandProcessor(CommandProcessor):
         else:
             state = ClientStatus.CLIENT_CONNECTED
             self.output("Unreadied.")
-        asyncio.create_task(self.ctx.send_msgs([{"cmd": "StatusUpdate", "status": state}]), name="send StatusUpdate")
+        async_start(self.ctx.send_msgs([{"cmd": "StatusUpdate", "status": state}]), name="send StatusUpdate")
 
     def default(self, raw: str):
         raw = self.ctx.on_user_say(raw)
         if raw:
-            asyncio.create_task(self.ctx.send_msgs([{"cmd": "Say", "text": raw}]), name="send Say")
+            async_start(self.ctx.send_msgs([{"cmd": "Say", "text": raw}]), name="send Say")
 
 
-class CommonContext():
+class CommonContext:
+    # Should be adjusted as needed in subclasses
     tags: typing.Set[str] = {"AP"}
+    game: typing.Optional[str] = None
+    items_handling: typing.Optional[int] = None
+    want_slot_data: bool = True  # should slot_data be retrieved via Connect
+
+    # data package
+    # Contents in flux until connection to server is made, to download correct data for this multiworld.
+    item_names: typing.Dict[int, str] = Utils.KeyedDefaultDict(lambda code: f'Unknown item (ID:{code})')
+    location_names: typing.Dict[int, str] = Utils.KeyedDefaultDict(lambda code: f'Unknown location (ID:{code})')
+
+    # defaults
     starting_reconnect_delay: int = 5
     current_reconnect_delay: int = starting_reconnect_delay
-    command_processor: int = ClientCommandProcessor
-    game: typing.Optional[str] = None
+    command_processor: typing.Type[CommandProcessor] = ClientCommandProcessor
     ui = None
-    ui_task: typing.Optional[asyncio.Task] = None
-    input_task: typing.Optional[asyncio.Task] = None
-    keep_alive_task: typing.Optional[asyncio.Task] = None
-    items_handling: typing.Optional[int] = None
-    slot_info: typing.Dict[int, NetworkSlot]
-    current_energy_link_value: int = 0  # to display in UI, gets set by server
+    ui_task: typing.Optional["asyncio.Task[None]"] = None
+    input_task: typing.Optional["asyncio.Task[None]"] = None
+    keep_alive_task: typing.Optional["asyncio.Task[None]"] = None
+    server_task: typing.Optional["asyncio.Task[None]"] = None
+    autoreconnect_task: typing.Optional["asyncio.Task[None]"] = None
+    disconnected_intentionally: bool = False
+    server: typing.Optional[Endpoint] = None
+    server_version: Version = Version(0, 0, 0)
+    generator_version: Version = Version(0, 0, 0)
+    current_energy_link_value: typing.Optional[int] = None  # to display in UI, gets set by server
 
-    def __init__(self, server_address, password):
+    last_death_link: float = time.time()  # last send/received death link on AP layer
+
+    # remaining type info
+    slot_info: typing.Dict[int, NetworkSlot]
+    server_address: typing.Optional[str]
+    password: typing.Optional[str]
+    hint_cost: typing.Optional[int]
+    hint_points: typing.Optional[int]
+    player_names: typing.Dict[int, str]
+
+    finished_game: bool
+    ready: bool
+    auth: typing.Optional[str]
+    seed_name: typing.Optional[str]
+
+    # locations
+    locations_checked: typing.Set[int]  # local state
+    locations_scouted: typing.Set[int]
+    items_received: typing.List[NetworkItem]
+    missing_locations: typing.Set[int]  # server state
+    checked_locations: typing.Set[int]  # server state
+    server_locations: typing.Set[int]  # all locations the server knows of, missing_location | checked_locations
+    locations_info: typing.Dict[int, NetworkItem]
+
+    # internals
+    # current message box through kvui
+    _messagebox: typing.Optional["kvui.MessageBox"] = None
+    # message box reporting a loss of connection
+    _messagebox_connection_loss: typing.Optional["kvui.MessageBox"] = None
+
+    def __init__(self, server_address: typing.Optional[str], password: typing.Optional[str]) -> None:
         # server state
         self.server_address = server_address
+        self.username = None
         self.password = password
-        self.server_task = None
-        self.server: typing.Optional[Endpoint] = None
-        self.server_version = Version(0, 0, 0)
-        self.hint_cost: typing.Optional[int] = None
-        self.games: typing.Dict[int, str] = {}
+        self.hint_cost = None
         self.slot_info = {}
         self.permissions = {
-            "forfeit": "disabled",
+            "release": "disabled",
             "collect": "disabled",
             "remaining": "disabled",
         }
@@ -152,30 +211,38 @@ class CommonContext():
         self.auth = None
         self.seed_name = None
 
-        self.locations_checked: typing.Set[int] = set()  # local state
-        self.locations_scouted: typing.Set[int] = set()
+        self.locations_checked = set()  # local state
+        self.locations_scouted = set()
         self.items_received = []
-        self.missing_locations: typing.Set[int] = set()
-        self.checked_locations: typing.Set[int] = set()  # server state
-        self.locations_info: typing.Dict[int, NetworkItem] = {}
+        self.missing_locations = set()  # server state
+        self.checked_locations = set()  # server state
+        self.server_locations = set()  # all locations the server knows of, missing_location | checked_locations
+        self.locations_info = {}
 
         self.input_queue = asyncio.Queue()
         self.input_requests = 0
 
-        self.last_death_link: float = time.time()  # last send/received death link on AP layer
-
         # game state
-        self.player_names: typing.Dict[int: str] = {0: "Archipelago"}
+        self.player_names = {0: "Archipelago"}
         self.exit_event = asyncio.Event()
         self.watcher_event = asyncio.Event()
 
-        self.slow_mode = False
         self.jsontotextparser = JSONtoTextParser(self)
-        self.set_getters(network_data_package)
+        self.update_data_package(network_data_package)
 
         # execution
         self.keep_alive_task = asyncio.create_task(keep_alive(self), name="Bouncy")
 
+    @property
+    def suggested_address(self) -> str:
+        if self.server_address:
+            return self.server_address
+        return Utils.persistent_load().get("client", {}).get("last_server_address", "")
+
+    @functools.cached_property
+    def raw_text_parser(self) -> RawJSONtoTextParser:
+        return RawJSONtoTextParser(self)
+
     @property
     def total_locations(self) -> typing.Optional[int]:
         """Will return None until connected."""
@@ -183,9 +250,9 @@ class CommonContext():
             return len(self.checked_locations | self.missing_locations)
 
     async def connection_closed(self):
-        self.reset_server_state()
         if self.server and self.server.socket is not None:
             await self.server.socket.close()
+        self.reset_server_state()
 
     def reset_server_state(self):
         self.auth = None
@@ -194,52 +261,28 @@ class CommonContext():
         self.items_received = []
         self.locations_info = {}
         self.server_version = Version(0, 0, 0)
+        self.generator_version = Version(0, 0, 0)
         self.server = None
         self.server_task = None
-        self.games = {}
         self.hint_cost = None
         self.permissions = {
-            "forfeit": "disabled",
+            "release": "disabled",
             "collect": "disabled",
             "remaining": "disabled",
         }
 
-    # noinspection PyAttributeOutsideInit
-    def set_getters(self, data_package: dict, network=False):
-        if not network:  # local data; check if newer data was already downloaded
-            local_package = Utils.persistent_load().get("datapackage", {}).get("latest", {})
-            if local_package and local_package["version"] > network_data_package["version"]:
-                data_package: dict = local_package
-        elif network:  # check if data from server is newer
-
-            if data_package["version"] > network_data_package["version"]:
-                Utils.persistent_store("datapackage", "latest", network_data_package)
-
-        item_lookup: dict = {}
-        locations_lookup: dict = {}
-        for game, gamedata in data_package["games"].items():
-            for item_name, item_id in gamedata["item_name_to_id"].items():
-                item_lookup[item_id] = item_name
-            for location_name, location_id in gamedata["location_name_to_id"].items():
-                locations_lookup[location_id] = location_name
-
-        def get_item_name_from_id(code: int) -> str:
-            return item_lookup.get(code, f'Unknown item (ID:{code})')
-
-        self.item_name_getter = get_item_name_from_id
-
-        def get_location_name_from_address(address: int) -> str:
-            return locations_lookup.get(address, f'Unknown location (ID:{address})')
-
-        self.location_name_getter = get_location_name_from_address
-
-    async def disconnect(self):
+    async def disconnect(self, allow_autoreconnect: bool = False):
+        if not allow_autoreconnect:
+            self.disconnected_intentionally = True
+            if self.cancel_autoreconnect():
+                logger.info("Cancelled auto-reconnect.")
         if self.server and not self.server.socket.closed:
             await self.server.socket.close()
         if self.server_task is not None:
             await self.server_task
 
-    async def send_msgs(self, msgs):
+    async def send_msgs(self, msgs: typing.List[typing.Any]) -> None:
+        """ `msgs` JSON serializable """
         if not self.server or not self.server.socket.open or self.server.socket.closed:
             return
         await self.server.socket.send(encode(msgs))
@@ -260,25 +303,61 @@ class CommonContext():
             self.password = await self.console_input()
             return self.password
 
-    async def send_connect(self, **kwargs):
+    async def get_username(self):
+        if not self.auth:
+            self.auth = self.username
+            if not self.auth:
+                logger.info('Enter slot name:')
+                self.auth = await self.console_input()
+
+    async def send_connect(self, **kwargs: typing.Any) -> None:
+        """ send `Connect` packet to log in to server """
         payload = {
             'cmd': 'Connect',
             'password': self.password, 'name': self.auth, 'version': Utils.version_tuple,
             'tags': self.tags, 'items_handling': self.items_handling,
-            'uuid': Utils.get_unique_identifier(), 'game': self.game
+            'uuid': Utils.get_unique_identifier(), 'game': self.game, "slot_data": self.want_slot_data,
         }
         if kwargs:
             payload.update(kwargs)
         await self.send_msgs([payload])
 
-    async def console_input(self):
+    async def console_input(self) -> str:
+        if self.ui:
+            self.ui.focus_textinput()
         self.input_requests += 1
         return await self.input_queue.get()
 
-    async def connect(self, address=None):
+    async def connect(self, address: typing.Optional[str] = None) -> None:
+        """ disconnect any previous connection, and open new connection to the server """
         await self.disconnect()
         self.server_task = asyncio.create_task(server_loop(self, address), name="server loop")
 
+    def cancel_autoreconnect(self) -> bool:
+        if self.autoreconnect_task:
+            self.autoreconnect_task.cancel()
+            self.autoreconnect_task = None
+            return True
+        return False
+
+    def slot_concerns_self(self, slot) -> bool:
+        if slot == self.slot:
+            return True
+        if slot in self.slot_info:
+            return self.slot in self.slot_info[slot].group_members
+        return False
+
+    def is_echoed_chat(self, print_json_packet: dict) -> bool:
+        return print_json_packet.get("type", "") == "Chat" \
+            and print_json_packet.get("team", None) == self.team \
+            and print_json_packet.get("slot", None) == self.slot
+
+    def is_uninteresting_item_send(self, print_json_packet: dict) -> bool:
+        """Helper function for filtering out ItemSend prints that do not concern the local player."""
+        return print_json_packet.get("type", "") == "ItemSend" \
+            and not self.slot_concerns_self(print_json_packet["receiving"]) \
+            and not self.slot_concerns_self(print_json_packet["item"].player)
+
     def on_print(self, args: dict):
         logger.info(args["text"])
 
@@ -308,7 +387,9 @@ class CommonContext():
                 logger.exception(e)
 
     async def shutdown(self):
-        self.server_address = None
+        self.server_address = ""
+        self.username = None
+        self.cancel_autoreconnect()
         if self.server and not self.server.socket.closed:
             await self.server.socket.close()
         if self.server_task:
@@ -323,9 +404,65 @@ class CommonContext():
         if self.input_task:
             self.input_task.cancel()
 
+    # DataPackage
+    async def prepare_data_package(self, relevant_games: typing.Set[str],
+                                   remote_date_package_versions: typing.Dict[str, int],
+                                   remote_data_package_checksums: typing.Dict[str, str]):
+        """Validate that all data is present for the current multiworld.
+        Download, assimilate and cache missing data from the server."""
+        # by documentation any game can use Archipelago locations/items -> always relevant
+        relevant_games.add("Archipelago")
+
+        needed_updates: typing.Set[str] = set()
+        for game in relevant_games:
+            if game not in remote_date_package_versions and game not in remote_data_package_checksums:
+                continue
+
+            remote_version: int = remote_date_package_versions.get(game, 0)
+            remote_checksum: typing.Optional[str] = remote_data_package_checksums.get(game)
+
+            if remote_version == 0 and not remote_checksum:  # custom data package and no checksum for this game
+                needed_updates.add(game)
+                continue
+
+            local_version: int = network_data_package["games"].get(game, {}).get("version", 0)
+            local_checksum: typing.Optional[str] = network_data_package["games"].get(game, {}).get("checksum")
+            # no action required if local version is new enough
+            if (not remote_checksum and (remote_version > local_version or remote_version == 0)) \
+                    or remote_checksum != local_checksum:
+                cached_game = Utils.load_data_package_for_checksum(game, remote_checksum)
+                cache_version: int = cached_game.get("version", 0)
+                cache_checksum: typing.Optional[str] = cached_game.get("checksum")
+                # download remote version if cache is not new enough
+                if (not remote_checksum and (remote_version > cache_version or remote_version == 0)) \
+                        or remote_checksum != cache_checksum:
+                    needed_updates.add(game)
+                else:
+                    self.update_game(cached_game)
+        if needed_updates:
+            await self.send_msgs([{"cmd": "GetDataPackage", "games": list(needed_updates)}])
+
+    def update_game(self, game_package: dict):
+        for item_name, item_id in game_package["item_name_to_id"].items():
+            self.item_names[item_id] = item_name
+        for location_name, location_id in game_package["location_name_to_id"].items():
+            self.location_names[location_id] = location_name
+
+    def update_data_package(self, data_package: dict):
+        for game, game_data in data_package["games"].items():
+            self.update_game(game_data)
+
+    def consume_network_data_package(self, data_package: dict):
+        self.update_data_package(data_package)
+        current_cache = Utils.persistent_load().get("datapackage", {}).get("games", {})
+        current_cache.update(data_package["games"])
+        Utils.persistent_store("datapackage", "games", current_cache)
+        for game, game_data in data_package["games"].items():
+            Utils.store_data_package_for_checksum(game, game_data)
+
     # DeathLink hooks
 
-    def on_deathlink(self, data: dict):
+    def on_deathlink(self, data: typing.Dict[str, typing.Any]) -> None:
         """Gets dispatched when a new DeathLink is triggered by another linked player."""
         self.last_death_link = max(data["time"], self.last_death_link)
         text = data.get("cause", "")
@@ -356,6 +493,34 @@ class CommonContext():
         if old_tags != self.tags and self.server and not self.server.socket.closed:
             await self.send_msgs([{"cmd": "ConnectUpdate", "tags": self.tags}])
 
+    def gui_error(self, title: str, text: typing.Union[Exception, str]) -> typing.Optional["kvui.MessageBox"]:
+        """Displays an error messagebox"""
+        if not self.ui:
+            return None
+        title = title or "Error"
+        from kvui import MessageBox
+        if self._messagebox:
+            self._messagebox.dismiss()
+        # make "Multiple exceptions" look nice
+        text = str(text).replace('[Errno', '\n[Errno').strip()
+        # split long messages into title and text
+        parts = title.split('. ', 1)
+        if len(parts) == 1:
+            parts = title.split(', ', 1)
+        if len(parts) > 1:
+            text = parts[1] + '\n\n' + text
+            title = parts[0]
+        # display error
+        self._messagebox = MessageBox(title, text, error=True)
+        self._messagebox.open()
+        return self._messagebox
+
+    def handle_connection_loss(self, msg: str) -> None:
+        """Helper for logging and displaying a loss of connection. Must be called from an except block."""
+        exc_info = sys.exc_info()
+        logger.exception(msg, exc_info=exc_info, extra={'compact_gui': True})
+        self._messagebox_connection_loss = self.gui_error(msg, exc_info[1])
+
     def run_gui(self):
         """Import kivy UI system and start running it as self.ui_task."""
         from kvui import GameManager
@@ -391,7 +556,7 @@ async def keep_alive(ctx: CommonContext, seconds_between_checks=100):
                 seconds_elapsed = 0
 
 
-async def server_loop(ctx: CommonContext, address=None):
+async def server_loop(ctx: CommonContext, address: typing.Optional[str] = None) -> None:
     if ctx.server and ctx.server.socket:
         logger.error('Already connected')
         return
@@ -404,33 +569,60 @@ async def server_loop(ctx: CommonContext, address=None):
         logger.info('Please connect to an Archipelago server.')
         return
 
-    address = f"ws://{address}" if "://" not in address else address
-    port = urllib.parse.urlparse(address).port or 38281
+    ctx.cancel_autoreconnect()
+    if ctx._messagebox_connection_loss:
+        ctx._messagebox_connection_loss.dismiss()
+        ctx._messagebox_connection_loss = None
+
+    address = f"ws://{address}" if "://" not in address \
+        else address.replace("archipelago://", "ws://")
+
+    server_url = urllib.parse.urlparse(address)
+    if server_url.username:
+        ctx.username = server_url.username
+    if server_url.password:
+        ctx.password = server_url.password
+    port = server_url.port or 38281
+
+    def reconnect_hint() -> str:
+        return ", type /connect to reconnect" if ctx.server_address else ""
 
     logger.info(f'Connecting to Archipelago server at {address}')
     try:
         socket = await websockets.connect(address, port=port, ping_timeout=None, ping_interval=None)
+        if ctx.ui is not None:
+            ctx.ui.update_address_bar(server_url.netloc)
         ctx.server = Endpoint(socket)
         logger.info('Connected')
         ctx.server_address = address
         ctx.current_reconnect_delay = ctx.starting_reconnect_delay
+        ctx.disconnected_intentionally = False
         async for data in ctx.server.socket:
             for msg in decode(data):
                 await process_server_cmd(ctx, msg)
-        logger.warning('Disconnected from multiworld server, type /connect to reconnect')
+        logger.warning(f"Disconnected from multiworld server{reconnect_hint()}")
+    except websockets.InvalidMessage:
+        # probably encrypted
+        if address.startswith("ws://"):
+            await server_loop(ctx, "ws" + address[1:])
+        else:
+            ctx.handle_connection_loss(f"Lost connection to the multiworld server due to InvalidMessage"
+                                       f"{reconnect_hint()}")
     except ConnectionRefusedError:
-        logger.exception('Connection refused by the server. May not be running Archipelago on that address or port.')
+        ctx.handle_connection_loss("Connection refused by the server. "
+                                   "May not be running Archipelago on that address or port.")
     except websockets.InvalidURI:
-        logger.exception('Failed to connect to the multiworld server (invalid URI)')
+        ctx.handle_connection_loss("Failed to connect to the multiworld server (invalid URI)")
     except OSError:
-        logger.exception('Failed to connect to the multiworld server')
+        ctx.handle_connection_loss("Failed to connect to the multiworld server")
     except Exception:
-        logger.exception('Lost connection to the multiworld server, type /connect to reconnect')
+        ctx.handle_connection_loss(f"Lost connection to the multiworld server{reconnect_hint()}")
     finally:
         await ctx.connection_closed()
-        if ctx.server_address:
-            logger.info(f"... reconnecting in {ctx.current_reconnect_delay}s")
-            asyncio.create_task(server_autoreconnect(ctx), name="server auto reconnect")
+        if ctx.server_address and ctx.username and not ctx.disconnected_intentionally:
+            logger.info(f"... automatically reconnecting in {ctx.current_reconnect_delay} seconds")
+            assert ctx.autoreconnect_task is None
+            ctx.autoreconnect_task = asyncio.create_task(server_autoreconnect(ctx), name="server auto reconnect")
         ctx.current_reconnect_delay *= 2
 
 
@@ -448,22 +640,27 @@ async def process_server_cmd(ctx: CommonContext, args: dict):
         raise
     if cmd == 'RoomInfo':
         if ctx.seed_name and ctx.seed_name != args["seed_name"]:
-            logger.info("The server is running a different multiworld than your client is. (invalid seed_name)")
+            msg = "The server is running a different multiworld than your client is. (invalid seed_name)"
+            logger.info(msg, extra={'compact_gui': True})
+            ctx.gui_error('Error', msg)
         else:
             logger.info('--------------------------------')
             logger.info('Room Information:')
             logger.info('--------------------------------')
             version = args["version"]
-            ctx.server_version = tuple(version)
-            version = ".".join(str(item) for item in version)
+            ctx.server_version = Version(*version)
 
-            logger.info(f'Server protocol version: {version}')
-            logger.info("Server protocol tags: " + ", ".join(args["tags"]))
+            if "generator_version" in args:
+                ctx.generator_version = Version(*args["generator_version"])
+                logger.info(f'Server protocol version: {ctx.server_version.as_simple_string()}, '
+                            f'generator version: {ctx.generator_version.as_simple_string()}, '
+                            f'tags: {", ".join(args["tags"])}')
+            else:
+                logger.info(f'Server protocol version: {ctx.server_version.as_simple_string()}, '
+                            f'tags: {", ".join(args["tags"])}')
             if args['password']:
                 logger.info('Password required')
             ctx.update_permissions(args.get("permissions", {}))
-            if "games" in args:
-                ctx.games = {x: game for x, game in enumerate(args["games"], start=1)}
             logger.info(
                 f"A !hint costs {args['hint_cost']}% of your total location count as points"
                 f" and you get {args['location_check_points']}"
@@ -471,24 +668,30 @@ async def process_server_cmd(ctx: CommonContext, args: dict):
             ctx.hint_cost = int(args['hint_cost'])
             ctx.check_points = int(args['location_check_points'])
 
-            if len(args['players']) < 1:
-                logger.info('No player connected')
-            else:
-                args['players'].sort()
-                current_team = -1
-                logger.info('Connected Players:')
-                for network_player in args['players']:
-                    if network_player.team != current_team:
-                        logger.info(f'  Team #{network_player.team + 1}')
-                        current_team = network_player.team
-                    logger.info('    %s (Player %d)' % (network_player.alias, network_player.slot))
-            if args["datapackage_version"] > network_data_package["version"] or args["datapackage_version"] == 0:
-                await ctx.send_msgs([{"cmd": "GetDataPackage"}])
+            if "players" in args:  # TODO remove when servers sending this are outdated
+                players = args.get("players", [])
+                if len(players) < 1:
+                    logger.info('No player connected')
+                else:
+                    players.sort()
+                    current_team = -1
+                    logger.info('Connected Players:')
+                    for network_player in players:
+                        if network_player.team != current_team:
+                            logger.info(f'  Team #{network_player.team + 1}')
+                            current_team = network_player.team
+                        logger.info('    %s (Player %d)' % (network_player.alias, network_player.slot))
+
+            # update data package
+            data_package_versions = args.get("datapackage_versions", {})
+            data_package_checksums = args.get("datapackage_checksums", {})
+            await ctx.prepare_data_package(set(args["games"]), data_package_versions, data_package_checksums)
+
             await ctx.server_auth(args['password'])
 
     elif cmd == 'DataPackage':
-        logger.info("Got new ID/Name Datapackage")
-        ctx.set_getters(args['data'], network=True)
+        logger.info("Got new ID/Name DataPackage")
+        ctx.consume_network_data_package(args['data'])
 
     elif cmd == 'ConnectionRefused':
         errors = args["errors"]
@@ -511,10 +714,12 @@ async def process_server_cmd(ctx: CommonContext, args: dict):
             raise Exception('Connection refused by the multiworld host, no reason provided')
 
     elif cmd == 'Connected':
+        ctx.username = ctx.auth
         ctx.team = args["team"]
         ctx.slot = args["slot"]
         # int keys get lost in JSON transfer
         ctx.slot_info = {int(pid): data for pid, data in args["slot_info"].items()}
+        ctx.hint_points = args.get("hint_points", 0)
         ctx.consume_players_package(args["players"])
         msgs = []
         if ctx.locations_checked:
@@ -534,6 +739,10 @@ async def process_server_cmd(ctx: CommonContext, args: dict):
         # when /missing is used for the client side view of what is missing.
         ctx.missing_locations = set(args["missing_locations"])
         ctx.checked_locations = set(args["checked_locations"])
+        ctx.server_locations = ctx.missing_locations | ctx. checked_locations
+
+        server_url = urllib.parse.urlparse(ctx.server_address)
+        Utils.persistent_store("client", "last_server_address", server_url.netloc)
 
     elif cmd == 'ReceivedItems':
         start_index = args["index"]
@@ -613,7 +822,7 @@ async def console_loop(ctx: CommonContext):
             logger.exception(e)
 
 
-def get_base_parser(description=None):
+def get_base_parser(description: typing.Optional[str] = None):
     import argparse
     parser = argparse.ArgumentParser(description=description)
     parser.add_argument('--connect', default=None, help='Address of the multiworld host.')
@@ -627,22 +836,24 @@ if __name__ == '__main__':
     # Text Mode to use !hint and such with games that have no text entry
 
     class TextContext(CommonContext):
-        tags = {"AP", "IgnoreGame", "TextOnly"}
+        tags = {"AP", "TextOnly"}
         game = ""  # empty matches any game since 0.3.2
-        items_handling = 0  # don't receive any NetworkItems
+        items_handling = 0b111  # receive all items for /received
+        want_slot_data = False  # Can't use game specific slot_data
 
         async def server_auth(self, password_requested: bool = False):
             if password_requested and not self.password:
                 await super(TextContext, self).server_auth(password_requested)
-            if not self.auth:
-                logger.info('Enter slot name:')
-                self.auth = await self.console_input()
-
+            await self.get_username()
             await self.send_connect()
 
         def on_package(self, cmd: str, args: dict):
             if cmd == "Connected":
-                self.game = self.games.get(self.slot, None)
+                self.game = self.slot_info[self.slot].game
+        
+        async def disconnect(self, allow_autoreconnect: bool = False):
+            self.game = ""
+            await super().disconnect(allow_autoreconnect)
 
 
     async def main(args):

FF1Client.py

@@ -1,4 +1,5 @@
 import asyncio
+import copy
 import json
 import time
 from asyncio import StreamReader, StreamWriter
@@ -6,14 +7,15 @@ from typing import List
 
 
 import Utils
-from CommonClient import CommonContext, server_loop, gui_enabled, console_loop, ClientCommandProcessor, logger, \
+from Utils import async_start
+from CommonClient import CommonContext, server_loop, gui_enabled, ClientCommandProcessor, logger, \
     get_base_parser
 
 SYSTEM_MESSAGE_ID = 0
 
-CONNECTION_TIMING_OUT_STATUS = "Connection timing out. Please restart your emulator, then restart ff1_connector.lua"
-CONNECTION_REFUSED_STATUS = "Connection Refused. Please start your emulator and make sure ff1_connector.lua is running"
-CONNECTION_RESET_STATUS = "Connection was reset. Please restart your emulator, then restart ff1_connector.lua"
+CONNECTION_TIMING_OUT_STATUS = "Connection timing out. Please restart your emulator, then restart connector_ff1.lua"
+CONNECTION_REFUSED_STATUS = "Connection Refused. Please start your emulator and make sure connector_ff1.lua is running"
+CONNECTION_RESET_STATUS = "Connection was reset. Please restart your emulator, then restart connector_ff1.lua"
 CONNECTION_TENTATIVE_STATUS = "Initial Connection Made"
 CONNECTION_CONNECTED_STATUS = "Connected"
 CONNECTION_INITIAL_STATUS = "Connection has not been initiated"
@@ -39,6 +41,7 @@ class FF1CommandProcessor(ClientCommandProcessor):
 
 class FF1Context(CommonContext):
     command_processor = FF1CommandProcessor
+    game = 'Final Fantasy'
     items_handling = 0b111  # full remote
 
     def __init__(self, server_address, password):
@@ -48,7 +51,6 @@ class FF1Context(CommonContext):
         self.messages = {}
         self.locations_array = None
         self.nes_status = CONNECTION_INITIAL_STATUS
-        self.game = 'Final Fantasy'
         self.awaiting_rom = False
         self.display_msgs = True
 
@@ -64,42 +66,37 @@ class FF1Context(CommonContext):
 
     def _set_message(self, msg: str, msg_id: int):
         if DISPLAY_MSGS:
-            self.messages[(time.time(), msg_id)] = msg
+            self.messages[time.time(), msg_id] = msg
 
     def on_package(self, cmd: str, args: dict):
         if cmd == 'Connected':
-            self.game = self.games.get(self.slot, None)
-            asyncio.create_task(parse_locations(self.locations_array, self, True))
+            async_start(parse_locations(self.locations_array, self, True))
         elif cmd == 'Print':
             msg = args['text']
             if ': !' not in msg:
                 self._set_message(msg, SYSTEM_MESSAGE_ID)
-        elif cmd == "ReceivedItems":
-            msg = f"Received {', '.join([self.item_name_getter(item.item) for item in args['items']])}"
-            self._set_message(msg, SYSTEM_MESSAGE_ID)
-        elif cmd == 'PrintJSON':
-            print_type = args['type']
-            item = args['item']
-            receiving_player_id = args['receiving']
-            receiving_player_name = self.player_names[receiving_player_id]
-            sending_player_id = item.player
-            sending_player_name = self.player_names[item.player]
-            if print_type == 'Hint':
-                msg = f"Hint: Your {self.item_name_getter(item.item)} is at" \
-                      f" {self.player_names[item.player]}'s {self.location_name_getter(item.location)}"
-                self._set_message(msg, item.item)
-            elif print_type == 'ItemSend' and receiving_player_id != self.slot:
-                if sending_player_id == self.slot:
-                    if receiving_player_id == self.slot:
-                        msg = f"You found your own {self.item_name_getter(item.item)}"
-                    else:
-                        msg = f"You sent {self.item_name_getter(item.item)} to {receiving_player_name}"
-                else:
-                    if receiving_player_id == sending_player_id:
-                        msg = f"{sending_player_name} found their {self.item_name_getter(item.item)}"
-                    else:
-                        msg = f"{sending_player_name} sent {self.item_name_getter(item.item)} to " \
-                              f"{receiving_player_name}"
+
+    def on_print_json(self, args: dict):
+        if self.ui:
+            self.ui.print_json(copy.deepcopy(args["data"]))
+        else:
+            text = self.jsontotextparser(copy.deepcopy(args["data"]))
+            logger.info(text)
+        relevant = args.get("type", None) in {"Hint", "ItemSend"}
+        if relevant:
+            item = args["item"]
+            # goes to this world
+            if self.slot_concerns_self(args["receiving"]):
+                relevant = True
+            # found in this world
+            elif self.slot_concerns_self(item.player):
+                relevant = True
+            # not related
+            else:
+                relevant = False
+            if relevant:
+                item = args["item"]
+                msg = self.raw_text_parser(copy.deepcopy(args["data"]))
                 self._set_message(msg, item.item)
 
     def run_gui(self):
@@ -151,13 +148,13 @@ async def parse_locations(locations_array: List[int], ctx: FF1Context, force: bo
                 index -= 0x200
                 flag = 0x02
 
-            # print(f"Location: {ctx.location_name_getter(location)}")
+            # print(f"Location: {ctx.location_names[location]}")
             # print(f"Index: {str(hex(index))}")
             # print(f"value: {locations_array[index] & flag != 0}")
             if locations_array[index] & flag != 0:
                 locations_checked.append(location)
         if locations_checked:
-            # print([ctx.location_name_getter(location) for location in locations_checked])
+            # print([ctx.location_names[location] for location in locations_checked])
             await ctx.send_msgs([
                 {"cmd": "LocationChecks",
                  "locations": locations_checked}
@@ -184,7 +181,7 @@ async def nes_sync_task(ctx: FF1Context):
                     # print(data_decoded)
                     if ctx.game is not None and 'locations' in data_decoded:
                         # Not just a keep alive ping, parse
-                        asyncio.create_task(parse_locations(data_decoded['locations'], ctx, False))
+                        async_start(parse_locations(data_decoded['locations'], ctx, False))
                     if not ctx.auth:
                         ctx.auth = ''.join([chr(i) for i in data_decoded['playerName'] if i != 0])
                         if ctx.auth == '':

FactorioClient.py

@@ -4,9 +4,12 @@ import logging
 import json
 import string
 import copy
+import re
 import subprocess
+import sys
 import time
 import random
+import typing
 
 import ModuleUpdate
 ModuleUpdate.update()
@@ -17,13 +20,18 @@ import asyncio
 from queue import Queue
 import Utils
 
+def check_stdin() -> None:
+    if Utils.is_windows and sys.stdin:
+        print("WARNING: Console input is not routed reliably on Windows, use the GUI instead.")
+
 if __name__ == "__main__":
     Utils.init_logging("FactorioClient", exception_logger="Client")
+    check_stdin()
 
-from CommonClient import CommonContext, server_loop, console_loop, ClientCommandProcessor, logger, gui_enabled, \
-    get_base_parser
+from CommonClient import CommonContext, server_loop, ClientCommandProcessor, logger, gui_enabled, get_base_parser
 from MultiServer import mark_raw
 from NetUtils import NetworkItem, ClientStatus, JSONtoTextParser, JSONMessagePart
+from Utils import async_start
 
 from worlds.factorio import Factorio
 
@@ -31,6 +39,10 @@ from worlds.factorio import Factorio
 class FactorioCommandProcessor(ClientCommandProcessor):
     ctx: FactorioContext
 
+    def _cmd_energy_link(self):
+        """Print the status of the energy link."""
+        self.output(f"Energy Link: {self.ctx.energy_link_status}")
+
     @mark_raw
     def _cmd_factorio(self, text: str) -> bool:
         """Send the following command to the bound Factorio Server."""
@@ -47,6 +59,13 @@ class FactorioCommandProcessor(ClientCommandProcessor):
         """Manually trigger a resync."""
         self.ctx.awaiting_bridge = True
 
+    def _cmd_toggle_send_filter(self):
+        """Toggle filtering of item sends that get displayed in-game to only those that involve you."""
+        self.ctx.toggle_filter_item_sends()
+
+    def _cmd_toggle_chat(self):
+        """Toggle sending of chat messages from players on the Factorio server to Archipelago."""
+        self.ctx.toggle_bridge_chat_out()
 
 class FactorioContext(CommonContext):
     command_processor = FactorioCommandProcessor
@@ -66,6 +85,9 @@ class FactorioContext(CommonContext):
         self.factorio_json_text_parser = FactorioJSONtoTextParser(self)
         self.energy_link_increment = 0
         self.last_deplete = 0
+        self.filter_item_sends: bool = False
+        self.multiplayer: bool = False  # whether multiple different players have connected
+        self.bridge_chat_out: bool = True
 
     async def server_auth(self, password_requested: bool = False):
         if password_requested and not self.password:
@@ -82,12 +104,16 @@ class FactorioContext(CommonContext):
     def on_print(self, args: dict):
         super(FactorioContext, self).on_print(args)
         if self.rcon_client:
-            self.print_to_game(args['text'])
+            if not args['text'].startswith(self.player_names[self.slot] + ":"):
+                self.print_to_game(args['text'])
 
     def on_print_json(self, args: dict):
         if self.rcon_client:
-            text = self.factorio_json_text_parser(copy.deepcopy(args["data"]))
-            self.print_to_game(text)
+            if (not self.filter_item_sends or not self.is_uninteresting_item_send(args)) \
+                    and not self.is_echoed_chat(args):
+                text = self.factorio_json_text_parser(copy.deepcopy(args["data"]))
+                if not text.startswith(self.player_names[self.slot] + ":"): # TODO: Remove string heuristic in the future.
+                    self.print_to_game(text)
         super(FactorioContext, self).on_print_json(args)
 
     @property
@@ -98,6 +124,15 @@ class FactorioContext(CommonContext):
         self.rcon_client.send_command(f"/ap-print [font=default-large-bold]Archipelago:[/font] "
                                       f"{text}")
 
+    @property
+    def energy_link_status(self) -> str:
+        if not self.energy_link_increment:
+            return "Disabled"
+        elif self.current_energy_link_value is None:
+            return "Standby"
+        else:
+            return f"{Utils.format_SI_prefix(self.current_energy_link_value)}J"
+
     def on_deathlink(self, data: dict):
         if self.rcon_client:
             self.rcon_client.send_command(f"/ap-deathlink {data['source']}")
@@ -110,7 +145,7 @@ class FactorioContext(CommonContext):
                 self.rcon_client.send_commands({item_name: f'/ap-get-technology ap-{item_name}-\t-1' for
                                                 item_name in args["checked_locations"]})
             if cmd == "Connected" and self.energy_link_increment:
-                asyncio.create_task(self.send_msgs([{
+                async_start(self.send_msgs([{
                     "cmd": "SetNotify", "keys": ["EnergyLink"]
                 }]))
         elif cmd == "SetReply":
@@ -124,6 +159,45 @@ class FactorioContext(CommonContext):
                                      f"{Utils.format_SI_prefix(args['value'])}J remaining.")
                         self.rcon_client.send_command(f"/ap-energylink {gained}")
 
+    def on_user_say(self, text: str) -> typing.Optional[str]:
+        # Mirror chat sent from the UI to the Factorio server.
+        self.print_to_game(f"{self.player_names[self.slot]}: {text}")
+        return text
+
+    async def chat_from_factorio(self, user: str, message: str) -> None:
+        if not self.bridge_chat_out:
+            return
+
+        # Pass through commands
+        if message.startswith("!"):
+            await self.send_msgs([{"cmd": "Say", "text": message}])
+            return
+
+        # Omit messages that contain local coordinates
+        if "[gps=" in message:
+            return
+
+        prefix = f"({user}) " if self.multiplayer else ""
+        await self.send_msgs([{"cmd": "Say", "text": f"{prefix}{message}"}])
+
+    def toggle_filter_item_sends(self) -> None:
+        self.filter_item_sends = not self.filter_item_sends
+        if self.filter_item_sends:
+            announcement = "Item sends are now filtered."
+        else:
+            announcement = "Item sends are no longer filtered."
+        logger.info(announcement)
+        self.print_to_game(announcement)
+
+    def toggle_bridge_chat_out(self) -> None:
+        self.bridge_chat_out = not self.bridge_chat_out
+        if self.bridge_chat_out:
+            announcement = "Chat is now bridged to Archipelago."
+        else:
+            announcement = "Chat is no longer bridged to Archipelago."
+        logger.info(announcement)
+        self.print_to_game(announcement)
+
     def run_gui(self):
         from kvui import GameManager
 
@@ -141,7 +215,6 @@ class FactorioContext(CommonContext):
 
 async def game_watcher(ctx: FactorioContext):
     bridge_logger = logging.getLogger("FactorioWatcher")
-    from worlds.factorio.Technologies import lookup_id_to_name
     next_bridge = time.perf_counter() + 1
     try:
         while not ctx.exit_event.is_set():
@@ -150,7 +223,9 @@ async def game_watcher(ctx: FactorioContext):
                 next_bridge = time.perf_counter() + 1
                 ctx.awaiting_bridge = False
                 data = json.loads(ctx.rcon_client.send_command("/ap-sync"))
-                if data["slot_name"] != ctx.auth:
+                if not ctx.auth:
+                    pass  # auth failed, wait for new attempt
+                elif data["slot_name"] != ctx.auth:
                     bridge_logger.warning(f"Connected World is not the expected one {data['slot_name']} != {ctx.auth}")
                 elif data["seed_name"] != ctx.seed_name:
                     bridge_logger.warning(
@@ -161,6 +236,7 @@ async def game_watcher(ctx: FactorioContext):
                     research_data = {int(tech_name.split("-")[1]) for tech_name in research_data}
                     victory = data["victory"]
                     await ctx.update_death_link(data["death_link"])
+                    ctx.multiplayer = data.get("multiplayer", False)
 
                     if not ctx.finished_game and victory:
                         await ctx.send_msgs([{"cmd": "StatusUpdate", "status": ClientStatus.CLIENT_GOAL}])
@@ -169,14 +245,14 @@ async def game_watcher(ctx: FactorioContext):
                     if ctx.locations_checked != research_data:
                         bridge_logger.debug(
                             f"New researches done: "
-                            f"{[lookup_id_to_name[rid] for rid in research_data - ctx.locations_checked]}")
+                            f"{[ctx.location_names[rid] for rid in research_data - ctx.locations_checked]}")
                         ctx.locations_checked = research_data
                         await ctx.send_msgs([{"cmd": 'LocationChecks', "locations": tuple(research_data)}])
                     death_link_tick = data.get("death_link_tick", 0)
                     if death_link_tick != ctx.death_link_tick:
                         ctx.death_link_tick = death_link_tick
                         if "DeathLink" in ctx.tags:
-                            asyncio.create_task(ctx.send_death())
+                            async_start(ctx.send_death())
                     if ctx.energy_link_increment:
                         in_world_bridges = data["energy_bridges"]
                         if in_world_bridges:
@@ -184,7 +260,7 @@ async def game_watcher(ctx: FactorioContext):
                             if in_world_energy < (ctx.energy_link_increment * in_world_bridges):
                                 # attempt to refill
                                 ctx.last_deplete = time.time()
-                                asyncio.create_task(ctx.send_msgs([{
+                                async_start(ctx.send_msgs([{
                                     "cmd": "Set", "key": "EnergyLink", "operations":
                                         [{"operation": "add", "value": -ctx.energy_link_increment * in_world_bridges},
                                          {"operation": "max", "value": 0}],
@@ -194,7 +270,7 @@ async def game_watcher(ctx: FactorioContext):
                             elif in_world_energy > (in_world_bridges * ctx.energy_link_increment * 5) - \
                                 ctx.energy_link_increment*in_world_bridges:
                                 value = ctx.energy_link_increment * in_world_bridges
-                                asyncio.create_task(ctx.send_msgs([{
+                                async_start(ctx.send_msgs([{
                                     "cmd": "Set", "key": "EnergyLink", "operations":
                                         [{"operation": "add", "value": value}]
                                 }]))
@@ -210,6 +286,8 @@ async def game_watcher(ctx: FactorioContext):
 
 
 def stream_factorio_output(pipe, queue, process):
+    pipe.reconfigure(errors="replace")
+
     def queuer():
         while process.poll() is None:
             text = pipe.readline().strip()
@@ -242,7 +320,7 @@ async def factorio_server_watcher(ctx: FactorioContext):
     stream_factorio_output(factorio_process.stderr, factorio_queue, factorio_process)
     try:
         while not ctx.exit_event.is_set():
-            if factorio_process.poll():
+            if factorio_process.poll() is not None:
                 factorio_server_logger.info("Factorio server has exited.")
                 ctx.exit_event.set()
 
@@ -255,12 +333,25 @@ async def factorio_server_watcher(ctx: FactorioContext):
                     if not ctx.server:
                         logger.info("Established bridge to Factorio Server. "
                                     "Ready to connect to Archipelago via /connect")
+                        check_stdin()
 
                 if not ctx.awaiting_bridge and "Archipelago Bridge Data available for game tick " in msg:
                     ctx.awaiting_bridge = True
                     factorio_server_logger.debug(msg)
+                elif re.match(r"^[0-9.]+ Script @[^ ]+\.lua:\d+: Player command energy-link$", msg):
+                    factorio_server_logger.debug(msg)
+                    ctx.print_to_game(f"Energy Link: {ctx.energy_link_status}")
+                elif re.match(r"^[0-9.]+ Script @[^ ]+\.lua:\d+: Player command toggle-ap-send-filter$", msg):
+                    factorio_server_logger.debug(msg)
+                    ctx.toggle_filter_item_sends()
+                elif re.match(r"^[0-9.]+ Script @[^ ]+\.lua:\d+: Player command toggle-ap-chat$", msg):
+                    factorio_server_logger.debug(msg)
+                    ctx.toggle_bridge_chat_out()
                 else:
                     factorio_server_logger.info(msg)
+                    match = re.match(r"^\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d \[CHAT\] ([^:]+): (.*)$", msg)
+                    if match:
+                        await ctx.chat_from_factorio(match.group(1), match.group(2))
             if ctx.rcon_client:
                 commands = {}
                 while ctx.send_index < len(ctx.items_received):
@@ -281,12 +372,34 @@ async def factorio_server_watcher(ctx: FactorioContext):
     except Exception as e:
         logging.exception(e)
         logging.error("Aborted Factorio Server Bridge")
-        ctx.rcon_client = None
         ctx.exit_event.set()
 
     finally:
-        factorio_process.terminate()
-        factorio_process.wait(5)
+        if factorio_process.poll() is not None:
+            if ctx.rcon_client:
+                ctx.rcon_client.close()
+                ctx.rcon_client = None
+            return
+
+        sent_quit = False
+        if ctx.rcon_client:
+            # Attempt clean quit through RCON.
+            try:
+                ctx.rcon_client.send_command("/quit")
+            except factorio_rcon.RCONNetworkError:
+                pass
+            else:
+                sent_quit = True
+            ctx.rcon_client.close()
+            ctx.rcon_client = None
+        if not sent_quit:
+            # Attempt clean quit using SIGTERM. (Note that on Windows this kills the process instead.)
+            factorio_process.terminate()
+
+        try:
+            factorio_process.wait(10)
+        except subprocess.TimeoutExpired:
+            factorio_process.kill()
 
 
 async def get_info(ctx: FactorioContext, rcon_client: factorio_rcon.RCONClient):
@@ -342,8 +455,10 @@ async def factorio_spinup_server(ctx: FactorioContext) -> bool:
             await asyncio.sleep(0.01)
 
     except Exception as e:
-        logger.exception(e)
-        logger.error("Aborted Factorio Server Bridge")
+        logger.exception(e, extra={"compact_gui": True})
+        msg = "Aborted Factorio Server Bridge"
+        logger.error(msg)
+        ctx.gui_error(msg, e)
         ctx.exit_event.set()
 
     else:
@@ -358,6 +473,8 @@ async def factorio_spinup_server(ctx: FactorioContext) -> bool:
 
 async def main(args):
     ctx = FactorioContext(args.connect, args.password)
+    ctx.filter_item_sends = initial_filter_item_sends
+    ctx.bridge_chat_out = initial_bridge_chat_out
     ctx.server_task = asyncio.create_task(server_loop(ctx), name="ServerLoop")
 
     if gui_enabled:
@@ -396,6 +513,7 @@ if __name__ == '__main__':
                                          "Refer to Factorio --help for those.")
     parser.add_argument('--rcon-port', default='24242', type=int, help='Port to use to communicate with Factorio')
     parser.add_argument('--rcon-password', help='Password to authenticate with RCON.')
+    parser.add_argument('--server-settings', help='Factorio server settings configuration file.')
 
     args, rest = parser.parse_known_args()
     colorama.init()
@@ -406,6 +524,15 @@ if __name__ == '__main__':
     factorio_server_logger = logging.getLogger("FactorioServer")
     options = Utils.get_options()
     executable = options["factorio_options"]["executable"]
+    server_settings = args.server_settings if args.server_settings else options["factorio_options"].get("server_settings", None)
+    if server_settings:
+        server_settings = os.path.abspath(server_settings)
+    if not isinstance(options["factorio_options"]["filter_item_sends"], bool):
+        logging.warning(f"Warning: Option filter_item_sends should be a bool.")
+    initial_filter_item_sends = bool(options["factorio_options"]["filter_item_sends"])
+    if not isinstance(options["factorio_options"]["bridge_chat_out"], bool):
+        logging.warning(f"Warning: Option bridge_chat_out should be a bool.")
+    initial_bridge_chat_out = bool(options["factorio_options"]["bridge_chat_out"])
 
     if not os.path.exists(os.path.dirname(executable)):
         raise FileNotFoundError(f"Path {os.path.dirname(executable)} does not exist or could not be accessed.")
@@ -417,7 +544,10 @@ if __name__ == '__main__':
         else:
             raise FileNotFoundError(f"Path {executable} is not an executable file.")
 
-    server_args = ("--rcon-port", rcon_port, "--rcon-password", rcon_password, *rest)
+    if server_settings and os.path.isfile(server_settings):
+        server_args = ("--rcon-port", rcon_port, "--rcon-password", rcon_password, "--server-settings", server_settings, *rest)
+    else:
+        server_args = ("--rcon-port", rcon_port, "--rcon-password", rcon_password, *rest)
 
     asyncio.run(main(args))
     colorama.deinit()

Fill.py

@@ -1,12 +1,12 @@
-import logging
-import typing
 import collections
 import itertools
+import logging
+import typing
 from collections import Counter, deque
 
-from BaseClasses import CollectionState, Location, LocationProgressType, MultiWorld, Item
-
+from BaseClasses import CollectionState, Item, Location, LocationProgressType, MultiWorld
 from worlds.AutoWorld import call_all
+from worlds.generic.Rules import add_item_rule
 
 
 class FillError(RuntimeError):
@@ -22,13 +22,27 @@ def sweep_from_pool(base_state: CollectionState, itempool: typing.Sequence[Item]
 
 
 def fill_restrictive(world: MultiWorld, base_state: CollectionState, locations: typing.List[Location],
-                     itempool: typing.List[Item], single_player_placement: bool = False, lock: bool = False) -> None:
+                     item_pool: typing.List[Item], single_player_placement: bool = False, lock: bool = False,
+                     swap: bool = True, on_place: typing.Optional[typing.Callable[[Location], None]] = None,
+                     allow_partial: bool = False, allow_excluded: bool = False) -> None:
+    """
+    :param world: Multiworld to be filled.
+    :param base_state: State assumed before fill.
+    :param locations: Locations to be filled with item_pool
+    :param item_pool: Items to fill into the locations
+    :param single_player_placement: if true, can speed up placement if everything belongs to a single player
+    :param lock: locations are set to locked as they are filled
+    :param swap: if true, swaps of already place items are done in the event of a dead end
+    :param on_place: callback that is called when a placement happens
+    :param allow_partial: only place what is possible. Remaining items will be in the item_pool list.
+    :param allow_excluded: if true and placement fails, it is re-attempted while ignoring excluded on Locations
+    """
     unplaced_items: typing.List[Item] = []
     placements: typing.List[Location] = []
 
     swapped_items: typing.Counter[typing.Tuple[int, str]] = Counter()
     reachable_items: typing.Dict[int, typing.Deque[Item]] = {}
-    for item in itempool:
+    for item in item_pool:
         reachable_items.setdefault(item.player, deque()).append(item)
 
     while any(reachable_items.values()) and locations:
@@ -36,14 +50,22 @@ def fill_restrictive(world: MultiWorld, base_state: CollectionState, locations:
         items_to_place = [items.pop()
                           for items in reachable_items.values() if items]
         for item in items_to_place:
-            itempool.remove(item)
+            item_pool.remove(item)
         maximum_exploration_state = sweep_from_pool(
-            base_state, itempool + unplaced_items)
+            base_state, item_pool + unplaced_items)
 
         has_beaten_game = world.has_beaten_game(maximum_exploration_state)
 
-        for item_to_place in items_to_place:
+        while items_to_place:
+            # if we have run out of locations to fill,break out of this loop
+            if not locations:
+                unplaced_items += items_to_place
+                break
+            item_to_place = items_to_place.pop(0)
+
             spot_to_fill: typing.Optional[Location] = None
+
+            # if minimal accessibility, only check whether location is reachable if game not beatable
             if world.accessibility[item_to_place.player] == 'minimal':
                 perform_access_check = not world.has_beaten_game(maximum_exploration_state,
                                                                  item_to_place.player) \
@@ -54,69 +76,90 @@ def fill_restrictive(world: MultiWorld, base_state: CollectionState, locations:
             for i, location in enumerate(locations):
                 if (not single_player_placement or location.player == item_to_place.player) \
                         and location.can_fill(maximum_exploration_state, item_to_place, perform_access_check):
-                    # poping by index is faster than removing by content,
+                    # popping by index is faster than removing by content,
                     spot_to_fill = locations.pop(i)
                     # skipping a scan for the element
                     break
 
             else:
                 # we filled all reachable spots.
-                # try swapping this item with previously placed items
-                for (i, location) in enumerate(placements):
-                    placed_item = location.item
-                    # Unplaceable items can sometimes be swapped infinitely. Limit the
-                    # number of times we will swap an individual item to prevent this
-                    swap_count = swapped_items[placed_item.player,
-                                               placed_item.name]
-                    if swap_count > 1:
+                if swap:
+                    # try swapping this item with previously placed items
+                    for (i, location) in enumerate(placements):
+                        placed_item = location.item
+                        # Unplaceable items can sometimes be swapped infinitely. Limit the
+                        # number of times we will swap an individual item to prevent this
+                        swap_count = swapped_items[placed_item.player,
+                                                   placed_item.name]
+                        if swap_count > 1:
+                            continue
+
+                        location.item = None
+                        placed_item.location = None
+                        swap_state = sweep_from_pool(base_state, [placed_item])
+                        # swap_state assumes we can collect placed item before item_to_place
+                        if (not single_player_placement or location.player == item_to_place.player) \
+                                and location.can_fill(swap_state, item_to_place, perform_access_check):
+
+                            # Verify that placing this item won't reduce available locations, which could happen with rules
+                            # that want to not have both items. Left in until removal is proven useful.
+                            prev_state = swap_state.copy()
+                            prev_loc_count = len(
+                                world.get_reachable_locations(prev_state))
+
+                            swap_state.collect(item_to_place, True)
+                            new_loc_count = len(
+                                world.get_reachable_locations(swap_state))
+
+                            if new_loc_count >= prev_loc_count:
+                                # Add this item to the existing placement, and
+                                # add the old item to the back of the queue
+                                spot_to_fill = placements.pop(i)
+
+                                swap_count += 1
+                                swapped_items[placed_item.player,
+                                              placed_item.name] = swap_count
+
+                                reachable_items[placed_item.player].appendleft(
+                                    placed_item)
+                                item_pool.append(placed_item)
+
+                                break
+
+                        # Item can't be placed here, restore original item
+                        location.item = placed_item
+                        placed_item.location = location
+
+                    if spot_to_fill is None:
+                        # Can't place this item, move on to the next
+                        unplaced_items.append(item_to_place)
                         continue
-
-                    location.item = None
-                    placed_item.location = None
-                    swap_state = sweep_from_pool(base_state)
-                    if (not single_player_placement or location.player == item_to_place.player) \
-                            and location.can_fill(swap_state, item_to_place, perform_access_check):
-
-                        # Verify that placing this item won't reduce available locations
-                        prev_state = swap_state.copy()
-                        prev_state.collect(placed_item)
-                        prev_loc_count = len(
-                            world.get_reachable_locations(prev_state))
-
-                        swap_state.collect(item_to_place, True)
-                        new_loc_count = len(
-                            world.get_reachable_locations(swap_state))
-
-                        if new_loc_count >= prev_loc_count:
-                            # Add this item to the existing placement, and
-                            # add the old item to the back of the queue
-                            spot_to_fill = placements.pop(i)
-
-                            swap_count += 1
-                            swapped_items[placed_item.player,
-                                          placed_item.name] = swap_count
-
-                            reachable_items[placed_item.player].appendleft(
-                                placed_item)
-                            itempool.append(placed_item)
-
-                            break
-
-                    # Item can't be placed here, restore original item
-                    location.item = placed_item
-                    placed_item.location = location
-
-                if spot_to_fill is None:
-                    # Can't place this item, move on to the next
+                else:
                     unplaced_items.append(item_to_place)
                     continue
-
             world.push_item(spot_to_fill, item_to_place, False)
             spot_to_fill.locked = lock
             placements.append(spot_to_fill)
             spot_to_fill.event = item_to_place.advancement
+            if on_place:
+                on_place(spot_to_fill)
 
-    if len(unplaced_items) > 0 and len(locations) > 0:
+    if allow_excluded:
+        # check if partial fill is the result of excluded locations, in which case retry
+        excluded_locations = [
+            location for location in locations
+            if location.progress_type == location.progress_type.EXCLUDED and not location.item
+        ]
+        if excluded_locations:
+            for location in excluded_locations:
+                location.progress_type = location.progress_type.DEFAULT
+            fill_restrictive(world, base_state, excluded_locations, unplaced_items, single_player_placement, lock,
+                             swap, on_place, allow_partial, False)
+            for location in excluded_locations:
+                if not location.item:
+                    location.progress_type = location.progress_type.EXCLUDED
+
+    if not allow_partial and len(unplaced_items) > 0 and len(locations) > 0:
         # There are leftover unplaceable items and locations that won't accept them
         if world.can_beat_game():
             logging.warning(
@@ -125,36 +168,219 @@ def fill_restrictive(world: MultiWorld, base_state: CollectionState, locations:
             raise FillError(f'No more spots to place {unplaced_items}, locations {locations} are invalid. '
                             f'Already placed {len(placements)}: {", ".join(str(place) for place in placements)}')
 
+    item_pool.extend(unplaced_items)
+
+
+def remaining_fill(world: MultiWorld,
+                   locations: typing.List[Location],
+                   itempool: typing.List[Item]) -> None:
+    unplaced_items: typing.List[Item] = []
+    placements: typing.List[Location] = []
+    swapped_items: typing.Counter[typing.Tuple[int, str]] = Counter()
+    while locations and itempool:
+        item_to_place = itempool.pop()
+        spot_to_fill: typing.Optional[Location] = None
+
+        for i, location in enumerate(locations):
+            if location.item_rule(item_to_place):
+                # popping by index is faster than removing by content,
+                spot_to_fill = locations.pop(i)
+                # skipping a scan for the element
+                break
+
+        else:
+            # we filled all reachable spots.
+            # try swapping this item with previously placed items
+
+            for (i, location) in enumerate(placements):
+                placed_item = location.item
+                # Unplaceable items can sometimes be swapped infinitely. Limit the
+                # number of times we will swap an individual item to prevent this
+
+                if swapped_items[placed_item.player,
+                                 placed_item.name] > 1:
+                    continue
+
+                location.item = None
+                placed_item.location = None
+                if location.item_rule(item_to_place):
+                    # Add this item to the existing placement, and
+                    # add the old item to the back of the queue
+                    spot_to_fill = placements.pop(i)
+
+                    swapped_items[placed_item.player,
+                                  placed_item.name] += 1
+
+                    itempool.append(placed_item)
+
+                    break
+
+                # Item can't be placed here, restore original item
+                location.item = placed_item
+                placed_item.location = location
+
+            if spot_to_fill is None:
+                # Can't place this item, move on to the next
+                unplaced_items.append(item_to_place)
+                continue
+
+        world.push_item(spot_to_fill, item_to_place, False)
+        placements.append(spot_to_fill)
+
+    if unplaced_items and locations:
+        # There are leftover unplaceable items and locations that won't accept them
+        raise FillError(f'No more spots to place {unplaced_items}, locations {locations} are invalid. '
+                        f'Already placed {len(placements)}: {", ".join(str(place) for place in placements)}')
+
     itempool.extend(unplaced_items)
 
 
+def fast_fill(world: MultiWorld,
+              item_pool: typing.List[Item],
+              fill_locations: typing.List[Location]) -> typing.Tuple[typing.List[Item], typing.List[Location]]:
+    placing = min(len(item_pool), len(fill_locations))
+    for item, location in zip(item_pool, fill_locations):
+        world.push_item(location, item, False)
+    return item_pool[placing:], fill_locations[placing:]
+
+
+def accessibility_corrections(world: MultiWorld, state: CollectionState, locations, pool=[]):
+    maximum_exploration_state = sweep_from_pool(state, pool)
+    minimal_players = {player for player in world.player_ids if world.accessibility[player] == "minimal"}
+    unreachable_locations = [location for location in world.get_locations() if location.player in minimal_players and
+                             not location.can_reach(maximum_exploration_state)]
+    for location in unreachable_locations:
+        if (location.item is not None and location.item.advancement and location.address is not None and not
+                location.locked and location.item.player not in minimal_players):
+            pool.append(location.item)
+            state.remove(location.item)
+            location.item = None
+            location.event = False
+            if location in state.events:
+                state.events.remove(location)
+            locations.append(location)
+    if pool and locations:
+        locations.sort(key=lambda loc: loc.progress_type != LocationProgressType.PRIORITY)
+        fill_restrictive(world, state, locations, pool)
+
+
+def inaccessible_location_rules(world: MultiWorld, state: CollectionState, locations):
+    maximum_exploration_state = sweep_from_pool(state)
+    unreachable_locations = [location for location in locations if not location.can_reach(maximum_exploration_state)]
+    if unreachable_locations:
+        def forbid_important_item_rule(item: Item):
+            return not ((item.classification & 0b0011) and world.accessibility[item.player] != 'minimal')
+
+        for location in unreachable_locations:
+            add_item_rule(location, forbid_important_item_rule)
+
+
+def distribute_early_items(world: MultiWorld,
+                           fill_locations: typing.List[Location],
+                           itempool: typing.List[Item]) -> typing.Tuple[typing.List[Location], typing.List[Item]]:
+    """ returns new fill_locations and itempool """
+    early_items_count: typing.Dict[typing.Tuple[str, int], typing.List[int]] = {}
+    for player in world.player_ids:
+        items = itertools.chain(world.early_items[player], world.local_early_items[player])
+        for item in items:
+            early_items_count[item, player] = [world.early_items[player].get(item, 0),
+                                               world.local_early_items[player].get(item, 0)]
+    if early_items_count:
+        early_locations: typing.List[Location] = []
+        early_priority_locations: typing.List[Location] = []
+        loc_indexes_to_remove: typing.Set[int] = set()
+        base_state = world.state.copy()
+        base_state.sweep_for_events(locations=(loc for loc in world.get_filled_locations() if loc.address is None))
+        for i, loc in enumerate(fill_locations):
+            if loc.can_reach(base_state):
+                if loc.progress_type == LocationProgressType.PRIORITY:
+                    early_priority_locations.append(loc)
+                else:
+                    early_locations.append(loc)
+                loc_indexes_to_remove.add(i)
+        fill_locations = [loc for i, loc in enumerate(fill_locations) if i not in loc_indexes_to_remove]
+
+        early_prog_items: typing.List[Item] = []
+        early_rest_items: typing.List[Item] = []
+        early_local_prog_items: typing.Dict[int, typing.List[Item]] = {player: [] for player in world.player_ids}
+        early_local_rest_items: typing.Dict[int, typing.List[Item]] = {player: [] for player in world.player_ids}
+        item_indexes_to_remove: typing.Set[int] = set()
+        for i, item in enumerate(itempool):
+            if (item.name, item.player) in early_items_count:
+                if item.advancement:
+                    if early_items_count[item.name, item.player][1]:
+                        early_local_prog_items[item.player].append(item)
+                        early_items_count[item.name, item.player][1] -= 1
+                    else:
+                        early_prog_items.append(item)
+                        early_items_count[item.name, item.player][0] -= 1
+                else:
+                    if early_items_count[item.name, item.player][1]:
+                        early_local_rest_items[item.player].append(item)
+                        early_items_count[item.name, item.player][1] -= 1
+                    else:
+                        early_rest_items.append(item)
+                        early_items_count[item.name, item.player][0] -= 1
+                item_indexes_to_remove.add(i)
+                if early_items_count[item.name, item.player] == [0, 0]:
+                    del early_items_count[item.name, item.player]
+                    if len(early_items_count) == 0:
+                        break
+        itempool = [item for i, item in enumerate(itempool) if i not in item_indexes_to_remove]
+        for player in world.player_ids:
+            player_local = early_local_rest_items[player]
+            fill_restrictive(world, base_state,
+                             [loc for loc in early_locations if loc.player == player],
+                             player_local, lock=True, allow_partial=True)
+            if player_local:
+                logging.warning(f"Could not fulfill rules of early items: {player_local}")
+                early_rest_items.extend(early_local_rest_items[player])
+        early_locations = [loc for loc in early_locations if not loc.item]
+        fill_restrictive(world, base_state, early_locations, early_rest_items, lock=True, allow_partial=True)
+        early_locations += early_priority_locations
+        for player in world.player_ids:
+            player_local = early_local_prog_items[player]
+            fill_restrictive(world, base_state,
+                             [loc for loc in early_locations if loc.player == player],
+                             player_local, lock=True, allow_partial=True)
+            if player_local:
+                logging.warning(f"Could not fulfill rules of early items: {player_local}")
+                early_prog_items.extend(player_local)
+        early_locations = [loc for loc in early_locations if not loc.item]
+        fill_restrictive(world, base_state, early_locations, early_prog_items, lock=True, allow_partial=True)
+        unplaced_early_items = early_rest_items + early_prog_items
+        if unplaced_early_items:
+            logging.warning("Ran out of early locations for early items. Failed to place "
+                            f"{unplaced_early_items} early.")
+            itempool += unplaced_early_items
+
+        fill_locations.extend(early_locations)
+        world.random.shuffle(fill_locations)
+    return fill_locations, itempool
+
+
 def distribute_items_restrictive(world: MultiWorld) -> None:
     fill_locations = sorted(world.get_unfilled_locations())
     world.random.shuffle(fill_locations)
-
     # get items to distribute
     itempool = sorted(world.itempool)
     world.random.shuffle(itempool)
+
+    fill_locations, itempool = distribute_early_items(world, fill_locations, itempool)
+
     progitempool: typing.List[Item] = []
-    nonexcludeditempool: typing.List[Item] = []
-    localrestitempool: typing.Dict[int, typing.List[Item]] = {player: [] for player in range(1, world.players + 1)}
-    nonlocalrestitempool: typing.List[Item] = []
-    restitempool: typing.List[Item] = []
+    usefulitempool: typing.List[Item] = []
+    filleritempool: typing.List[Item] = []
 
     for item in itempool:
         if item.advancement:
             progitempool.append(item)
-        elif item.never_exclude:  # this only gets nonprogression items which should not appear in excluded locations
-            nonexcludeditempool.append(item)
-        elif item.name in world.local_items[item.player].value:
-            localrestitempool[item.player].append(item)
-        elif item.name in world.non_local_items[item.player].value:
-            nonlocalrestitempool.append(item)
+        elif item.useful:
+            usefulitempool.append(item)
         else:
-            restitempool.append(item)
+            filleritempool.append(item)
 
-    call_all(world, "fill_hook", progitempool, nonexcludeditempool,
-             localrestitempool, nonlocalrestitempool, restitempool, fill_locations)
+    call_all(world, "fill_hook", progitempool, usefulitempool, filleritempool, fill_locations)
 
     locations: typing.Dict[LocationProgressType, typing.List[Location]] = {
         loc_type: [] for loc_type in LocationProgressType}
@@ -166,60 +392,44 @@ def distribute_items_restrictive(world: MultiWorld) -> None:
     defaultlocations = locations[LocationProgressType.DEFAULT]
     excludedlocations = locations[LocationProgressType.EXCLUDED]
 
-    fill_restrictive(world, world.state, prioritylocations, progitempool, lock=True)
+    # can't lock due to accessibility corrections touching things, so we remember which ones got placed and lock later
+    lock_later = []
+
+    def mark_for_locking(location: Location):
+        nonlocal lock_later
+        lock_later.append(location)
+
     if prioritylocations:
+        # "priority fill"
+        fill_restrictive(world, world.state, prioritylocations, progitempool, swap=False, on_place=mark_for_locking)
+        accessibility_corrections(world, world.state, prioritylocations, progitempool)
         defaultlocations = prioritylocations + defaultlocations
 
     if progitempool:
+        # "progression fill"
         fill_restrictive(world, world.state, defaultlocations, progitempool)
         if progitempool:
             raise FillError(
                 f'Not enough locations for progress items. There are {len(progitempool)} more items than locations')
+        accessibility_corrections(world, world.state, defaultlocations)
 
-    if nonexcludeditempool:
-        world.random.shuffle(defaultlocations)
-        # needs logical fill to not conflict with local items
-        fill_restrictive(
-            world, world.state, defaultlocations, nonexcludeditempool)
-        if nonexcludeditempool:
-            raise FillError(
-                f'Not enough locations for non-excluded items. There are {len(nonexcludeditempool)} more items than locations')
+    for location in lock_later:
+        if location.item:
+            location.locked = True
+    del mark_for_locking, lock_later
 
-    defaultlocations = defaultlocations + excludedlocations
-    world.random.shuffle(defaultlocations)
+    inaccessible_location_rules(world, world.state, defaultlocations)
 
-    if any(localrestitempool.values()):  # we need to make sure some fills are limited to certain worlds
-        local_locations: typing.Dict[int, typing.List[Location]] = {player: [] for player in world.player_ids}
-        for location in defaultlocations:
-            local_locations[location.player].append(location)
-        for player_locations in local_locations.values():
-            world.random.shuffle(player_locations)
+    remaining_fill(world, excludedlocations, filleritempool)
+    if excludedlocations:
+        raise FillError(
+            f"Not enough filler items for excluded locations. There are {len(excludedlocations)} more locations than items")
 
-        for player, items in localrestitempool.items():  # items already shuffled
-            player_local_locations = local_locations[player]
-            for item_to_place in items:
-                if not player_local_locations:
-                    logging.warning(f"Ran out of local locations for player {player}, "
-                                    f"cannot place {item_to_place}.")
-                    break
-                spot_to_fill = player_local_locations.pop()
-                world.push_item(spot_to_fill, item_to_place, False)
-                defaultlocations.remove(spot_to_fill)
+    restitempool = usefulitempool + filleritempool
 
-    for item_to_place in nonlocalrestitempool:
-        for i, location in enumerate(defaultlocations):
-            if location.player != item_to_place.player:
-                world.push_item(defaultlocations.pop(i), item_to_place, False)
-                break
-        else:
-            logging.warning(
-                f"Could not place non_local_item {item_to_place} among {defaultlocations}, tossing.")
+    remaining_fill(world, defaultlocations, restitempool)
 
-    world.random.shuffle(defaultlocations)
-
-    restitempool, defaultlocations = fast_fill(
-        world, restitempool, defaultlocations)
-    unplaced = progitempool + restitempool
+    unplaced = restitempool
     unfilled = defaultlocations
 
     if unplaced or unfilled:
@@ -233,15 +443,6 @@ def distribute_items_restrictive(world: MultiWorld) -> None:
         logging.info(f'Per-Player counts: {print_data})')
 
 
-def fast_fill(world: MultiWorld,
-              item_pool: typing.List[Item],
-              fill_locations: typing.List[Location]) -> typing.Tuple[typing.List[Item], typing.List[Location]]:
-    placing = min(len(item_pool), len(fill_locations))
-    for item, location in zip(item_pool, fill_locations):
-        world.push_item(location, item, False)
-    return item_pool[placing:], fill_locations[placing:]
-
-
 def flood_items(world: MultiWorld) -> None:
     # get items to distribute
     world.random.shuffle(world.itempool)
@@ -324,16 +525,16 @@ def balance_multiworld_progression(world: MultiWorld) -> None:
         checked_locations: typing.Set[Location] = set()
         unchecked_locations: typing.Set[Location] = set(world.get_locations())
 
-        reachable_locations_count: typing.Dict[int, int] = {
-            player: 0
-            for player in world.player_ids
-            if len(world.get_filled_locations(player)) != 0
-        }
         total_locations_count: typing.Counter[int] = Counter(
             location.player
             for location in world.get_locations()
             if not location.locked
         )
+        reachable_locations_count: typing.Dict[int, int] = {
+            player: 0
+            for player in world.player_ids
+            if total_locations_count[player] and len(world.get_filled_locations(player)) != 0
+        }
         balanceable_players = {
             player: balanceable_players[player]
             for player in balanceable_players
@@ -350,6 +551,10 @@ def balance_multiworld_progression(world: MultiWorld) -> None:
         def item_percentage(player: int, num: int) -> float:
             return num / total_locations_count[player]
 
+        # If there are no locations that aren't locked, there's no point in attempting to balance progression.
+        if len(total_locations_count) == 0:
+            return
+
         while True:
             # Gather non-locked locations.
             # This ensures that only shuffled locations get counted for progression balancing,
@@ -518,6 +723,17 @@ def distribute_planned(world: MultiWorld) -> None:
         else:
             warn(warning, force)
 
+    swept_state = world.state.copy()
+    swept_state.sweep_for_events()
+    reachable = frozenset(world.get_reachable_locations(swept_state))
+    early_locations: typing.Dict[int, typing.List[str]] = collections.defaultdict(list)
+    non_early_locations: typing.Dict[int, typing.List[str]] = collections.defaultdict(list)
+    for loc in world.get_unfilled_locations():
+        if loc in reachable:
+            early_locations[loc.player].append(loc.name)
+        else:  # not reachable with swept state
+            non_early_locations[loc.player].append(loc.name)
+
     # TODO: remove. Preferably by implementing key drop
     from worlds.alttp.Regions import key_drop_data
     world_name_lookup = world.world_name_lookup
@@ -533,7 +749,39 @@ def distribute_planned(world: MultiWorld) -> None:
             if 'from_pool' not in block:
                 block['from_pool'] = True
             if 'world' not in block:
-                block['world'] = False
+                target_world = False
+            else:
+                target_world = block['world']
+
+            if target_world is False or world.players == 1:  # target own world
+                worlds: typing.Set[int] = {player}
+            elif target_world is True:  # target any worlds besides own
+                worlds = set(world.player_ids) - {player}
+            elif target_world is None:  # target all worlds
+                worlds = set(world.player_ids)
+            elif type(target_world) == list:  # list of target worlds
+                worlds = set()
+                for listed_world in target_world:
+                    if listed_world not in world_name_lookup:
+                        failed(f"Cannot place item to {target_world}'s world as that world does not exist.",
+                               block['force'])
+                        continue
+                    worlds.add(world_name_lookup[listed_world])
+            elif type(target_world) == int:  # target world by slot number
+                if target_world not in range(1, world.players + 1):
+                    failed(
+                        f"Cannot place item in world {target_world} as it is not in range of (1, {world.players})",
+                        block['force'])
+                    continue
+                worlds = {target_world}
+            else:  # target world by slot name
+                if target_world not in world_name_lookup:
+                    failed(f"Cannot place item to {target_world}'s world as that world does not exist.",
+                           block['force'])
+                    continue
+                worlds = {world_name_lookup[target_world]}
+            block['world'] = worlds
+
             items: block_value = []
             if "items" in block:
                 items = block["items"]
@@ -570,6 +818,16 @@ def distribute_planned(world: MultiWorld) -> None:
                 for key, value in locations.items():
                     location_list += [key] * value
                 locations = location_list
+
+            if "early_locations" in locations:
+                locations.remove("early_locations")
+                for player in worlds:
+                    locations += early_locations[player]
+            if "non_early_locations" in locations:
+                locations.remove("non_early_locations")
+                for player in worlds:
+                    locations += non_early_locations[player]
+
             block['locations'] = locations
 
             if not block['count']:
@@ -605,41 +863,13 @@ def distribute_planned(world: MultiWorld) -> None:
     for placement in plando_blocks:
         player = placement['player']
         try:
-            target_world = placement['world']
+            worlds = placement['world']
             locations = placement['locations']
             items = placement['items']
             maxcount = placement['count']['target']
             from_pool = placement['from_pool']
-            if target_world is False or world.players == 1:  # target own world
-                worlds: typing.Set[int] = {player}
-            elif target_world is True:  # target any worlds besides own
-                worlds = set(world.player_ids) - {player}
-            elif target_world is None:  # target all worlds
-                worlds = set(world.player_ids)
-            elif type(target_world) == list:  # list of target worlds
-                worlds = set()
-                for listed_world in target_world:
-                    if listed_world not in world_name_lookup:
-                        failed(f"Cannot place item to {target_world}'s world as that world does not exist.",
-                               placement['force'])
-                        continue
-                    worlds.add(world_name_lookup[listed_world])
-            elif type(target_world) == int:  # target world by slot number
-                if target_world not in range(1, world.players + 1):
-                    failed(
-                        f"Cannot place item in world {target_world} as it is not in range of (1, {world.players})",
-                        placement['force'])
-                    continue
-                worlds = {target_world}
-            else:  # target world by slot name
-                if target_world not in world_name_lookup:
-                    failed(f"Cannot place item to {target_world}'s world as that world does not exist.",
-                           placement['force'])
-                    continue
-                worlds = {world_name_lookup[target_world]}
 
-            candidates = list(location for location in world.get_unfilled_locations_for_players(locations,
-                                                                                                worlds))
+            candidates = list(world.get_unfilled_locations_for_players(locations, sorted(worlds)))
             world.random.shuffle(candidates)
             world.random.shuffle(items)
             count = 0

Generate.py

@@ -1,12 +1,14 @@
+from __future__ import annotations
+
 import argparse
 import logging
-import random
-import urllib.request
-import urllib.parse
-from typing import Set, Dict, Tuple, Callable, Any, Union
 import os
-from collections import Counter
+import random
 import string
+import urllib.parse
+import urllib.request
+from collections import Counter, ChainMap
+from typing import Dict, Tuple, Callable, Any, Union
 
 import ModuleUpdate
 
@@ -15,17 +17,17 @@ ModuleUpdate.update()
 import Utils
 from worlds.alttp import Options as LttPOptions
 from worlds.generic import PlandoConnection
-from Utils import parse_yamls, version_tuple, __version__, tuplize_version, get_options, local_path, user_path
+from Utils import parse_yamls, version_tuple, __version__, tuplize_version, get_options, user_path
 from worlds.alttp.EntranceRandomizer import parse_arguments
 from Main import main as ERmain
-from BaseClasses import seeddigits, get_seed
+from BaseClasses import seeddigits, get_seed, PlandoOptions
 import Options
-from worlds.alttp import Bosses
 from worlds.alttp.Text import TextTable
 from worlds.AutoWorld import AutoWorldRegister
 import copy
 
-categories = set(AutoWorldRegister.world_types)
+
+
 
 
 def mystery_argparse():
@@ -45,11 +47,6 @@ def mystery_argparse():
     parser.add_argument('--seed', help='Define seed number to generate.', type=int)
     parser.add_argument('--multi', default=defaults["players"], type=lambda value: max(int(value), 1))
     parser.add_argument('--spoiler', type=int, default=defaults["spoiler"])
-    parser.add_argument('--lttp_rom', default=options["lttp_options"]["rom_file"],
-                        help="Path to the 1.0 JP LttP Baserom.")  # absolute, relative to cwd or relative to app path
-    parser.add_argument('--sm_rom', default=options["sm_options"]["rom_file"],
-                        help="Path to the 1.0 JP SM Baserom.")
-    parser.add_argument('--enemizercli', default=resolve_path(defaults["enemizer_path"], local_path))
     parser.add_argument('--outputpath', default=resolve_path(options["general_options"]["output_path"], user_path),
                         help="Path to output folder. Absolute or relative to cwd.")  # absolute or relative to cwd
     parser.add_argument('--race', action='store_true', default=defaults["race"])
@@ -64,7 +61,7 @@ def mystery_argparse():
         args.weights_file_path = os.path.join(args.player_files_path, args.weights_file_path)
     if not os.path.isabs(args.meta_file_path):
         args.meta_file_path = os.path.join(args.player_files_path, args.meta_file_path)
-    args.plando: Set[str] = {arg.strip().lower() for arg in args.plando.split(",")}
+    args.plando: PlandoOptions = PlandoOptions.from_option_string(args.plando)
     return args, options
 
 
@@ -94,12 +91,14 @@ def main(args=None, callback=ERmain):
 
     if args.meta_file_path and os.path.exists(args.meta_file_path):
         try:
-            weights_cache[args.meta_file_path] = read_weights_yamls(args.meta_file_path)
+            meta_weights = read_weights_yamls(args.meta_file_path)[-1]
         except Exception as e:
             raise ValueError(f"File {args.meta_file_path} is destroyed. Please fix your yaml.") from e
-        meta_weights = weights_cache[args.meta_file_path][-1]
         print(f"Meta: {args.meta_file_path} >> {get_choice('meta_description', meta_weights)}")
-        del(meta_weights["meta_description"])
+        try:  # meta description allows us to verify that the file named meta.yaml is intentionally a meta file
+            del(meta_weights["meta_description"])
+        except Exception as e:
+            raise ValueError("No meta description found for meta.yaml. Unable to verify.") from e
         if args.samesettings:
             raise Exception("Cannot mix --samesettings with --meta")
     else:
@@ -108,28 +107,34 @@ def main(args=None, callback=ERmain):
     player_files = {}
     for file in os.scandir(args.player_files_path):
         fname = file.name
-        if file.is_file() and os.path.join(args.player_files_path, fname) not in {args.meta_file_path, args.weights_file_path}:
+        if file.is_file() and not fname.startswith(".") and \
+                os.path.join(args.player_files_path, fname) not in {args.meta_file_path, args.weights_file_path}:
             path = os.path.join(args.player_files_path, fname)
             try:
                 weights_cache[fname] = read_weights_yamls(path)
             except Exception as e:
                 raise ValueError(f"File {fname} is destroyed. Please fix your yaml.") from e
-            else:
-                for yaml in weights_cache[fname]:
-                    print(f"P{player_id} Weights: {fname} >> "
-                          f"{get_choice('description', yaml, 'No description specified')}")
-                    player_files[player_id] = fname
-                    player_id += 1
 
-    args.multi = max(player_id-1, args.multi)
+    # sort dict for consistent results across platforms:
+    weights_cache = {key: value for key, value in sorted(weights_cache.items())}
+    for filename, yaml_data in weights_cache.items():
+        if filename not in {args.meta_file_path, args.weights_file_path}:
+            for yaml in yaml_data:
+                print(f"P{player_id} Weights: {filename} >> "
+                      f"{get_choice('description', yaml, 'No description specified')}")
+                player_files[player_id] = filename
+                player_id += 1
+
+    args.multi = max(player_id - 1, args.multi)
     print(f"Generating for {args.multi} player{'s' if args.multi > 1 else ''}, {seed_name} Seed {seed} with plando: "
-          f"{', '.join(args.plando)}")
+          f"{args.plando}")
 
     if not weights_cache:
         raise Exception(f"No weights found. Provide a general weights file ({args.weights_file_path}) or individual player files. "
                         f"A mix is also permitted.")
     erargs = parse_arguments(['--multi', str(args.multi)])
     erargs.seed = seed
+    erargs.plando_options = args.plando
     erargs.glitch_triforce = options["generator"]["glitch_triforce_room"]
     erargs.spoiler = args.spoiler
     erargs.race = args.race
@@ -138,31 +143,29 @@ def main(args=None, callback=ERmain):
 
     Utils.init_logging(f"Generate_{seed}", loglevel=args.log_level)
 
-    erargs.lttp_rom = args.lttp_rom
-    erargs.sm_rom = args.sm_rom
-    erargs.enemizercli = args.enemizercli
-
     settings_cache: Dict[str, Tuple[argparse.Namespace, ...]] = \
-        {fname: ( tuple(roll_settings(yaml, args.plando) for yaml in yamls) if args.samesettings else None)
-                for fname, yamls in weights_cache.items()}
-    player_path_cache = {}
-    for player in range(1, args.multi + 1):
-        player_path_cache[player] = player_files.get(player, args.weights_file_path)
+        {fname: (tuple(roll_settings(yaml, args.plando) for yaml in yamls) if args.samesettings else None)
+         for fname, yamls in weights_cache.items()}
 
     if meta_weights:
         for category_name, category_dict in meta_weights.items():
             for key in category_dict:
-                option = get_choice(key, category_dict)
+                option = roll_meta_option(key, category_name, category_dict)
                 if option is not None:
-                    for player, path in player_path_cache.items():
+                    for path in weights_cache:
                         for yaml in weights_cache[path]:
                             if category_name is None:
-                                yaml[key] = option
+                                for category in yaml:
+                                    if category in AutoWorldRegister.world_types and key in Options.common_options:
+                                        yaml[category][key] = option
                             elif category_name not in yaml:
                                 logging.warning(f"Meta: Category {category_name} is not present in {path}.")
                             else:
-                               yaml[category_name][key] = option
+                                yaml[category_name][key] = option
 
+    player_path_cache = {}
+    for player in range(1, args.multi + 1):
+        player_path_cache[player] = player_files.get(player, args.weights_file_path)
     name_counter = Counter()
     erargs.player_settings = {}
 
@@ -188,15 +191,15 @@ def main(args=None, callback=ERmain):
                     elif not erargs.name[player]:  # if name was not specified, generate it from filename
                         erargs.name[player] = os.path.splitext(os.path.split(path)[-1])[0]
                     erargs.name[player] = handle_name(erargs.name[player], player, name_counter)
-                    
+
                     player += 1
             except Exception as e:
                 raise ValueError(f"File {path} is destroyed. Please fix your yaml.") from e
         else:
             raise RuntimeError(f'No weights specified for player {player}')
 
-    if len(set(erargs.name.values())) != len(erargs.name):
-        raise Exception(f"Names have to be unique. Names: {Counter(erargs.name.values())}")
+    if len(set(name.lower() for name in erargs.name.values())) != len(erargs.name):
+        raise Exception(f"Names have to be unique. Names: {Counter(name.lower() for name in erargs.name.values())}")
 
     if args.yaml_output:
         import yaml
@@ -279,11 +282,11 @@ class SafeDict(dict):
 
 
 def handle_name(name: str, player: int, name_counter: Counter):
-    name_counter[name] += 1
+    name_counter[name.lower()] += 1
+    number = name_counter[name.lower()]
     new_name = "%".join([x.replace("%number%", "{number}").replace("%player%", "{player}") for x in name.split("%%")])
-    new_name = string.Formatter().vformat(new_name, (), SafeDict(number=name_counter[name],
-                                                                 NUMBER=(name_counter[name] if name_counter[
-                                                                                                   name] > 1 else ''),
+    new_name = string.Formatter().vformat(new_name, (), SafeDict(number=number,
+                                                                 NUMBER=(number if number > 1 else ''),
                                                                  player=player,
                                                                  PLAYER=(player if player > 1 else '')))
     new_name = new_name.strip()[:16]
@@ -299,19 +302,6 @@ def prefer_int(input_data: str) -> Union[str, int]:
         return input_data
 
 
-available_boss_names: Set[str] = {boss.lower() for boss in Bosses.boss_table if boss not in
-                                         {'Agahnim', 'Agahnim2', 'Ganon'}}
-available_boss_locations: Set[str] = {f"{loc.lower()}{f' {level}' if level else ''}" for loc, level in
-                                             Bosses.boss_location_table}
-
-boss_shuffle_options = {None: 'none',
-                        'none': 'none',
-                        'basic': 'basic',
-                        'full': 'full',
-                        'chaos': 'chaos',
-                        'singularity': 'singularity'
-                        }
-
 goals = {
     'ganon': 'ganon',
     'crystals': 'crystals',
@@ -344,6 +334,28 @@ def update_weights(weights: dict, new_weights: dict, type: str, name: str) -> di
     return weights
 
 
+def roll_meta_option(option_key, game: str, category_dict: Dict) -> Any:
+    if not game:
+        return get_choice(option_key, category_dict)
+    if game in AutoWorldRegister.world_types:
+        game_world = AutoWorldRegister.world_types[game]
+        options = ChainMap(game_world.option_definitions, Options.per_game_common_options)
+        if option_key in options:
+            if options[option_key].supports_weighting:
+                return get_choice(option_key, category_dict)
+            return category_dict[option_key]
+    if game == "A Link to the Past":  # TODO wow i hate this
+        if option_key in {"glitches_required", "dark_room_logic", "entrance_shuffle", "goals", "triforce_pieces_mode",
+                          "triforce_pieces_percentage", "triforce_pieces_available", "triforce_pieces_extra",
+                          "triforce_pieces_required", "shop_shuffle", "mode", "item_pool", "item_functionality",
+                          "boss_shuffle", "enemy_damage", "enemy_health", "timer", "countdown_start_time",
+                          "red_clock_time", "blue_clock_time", "green_clock_time", "dungeon_counters", "shuffle_prizes",
+                          "misery_mire_medallion", "turtle_rock_medallion", "sprite_pool", "sprite",
+                          "random_sprite_on_event"}:
+            return get_choice(option_key, category_dict)
+    raise Exception(f"Error generating meta option {option_key} for {game}.")
+
+
 def roll_linked_options(weights: dict) -> dict:
     weights = copy.deepcopy(weights)  # make sure we don't write back to other weights sets in same_settings
     for option_set in weights["linked_options"]:
@@ -396,42 +408,7 @@ def roll_triggers(weights: dict, triggers: list) -> dict:
     return weights
 
 
-def get_plando_bosses(boss_shuffle: str, plando_options: Set[str]) -> str:
-    if boss_shuffle in boss_shuffle_options:
-        return boss_shuffle_options[boss_shuffle]
-    elif "bosses" in plando_options:
-        options = boss_shuffle.lower().split(";")
-        remainder_shuffle = "none"  # vanilla
-        bosses = []
-        for boss in options:
-            if boss in boss_shuffle_options:
-                remainder_shuffle = boss_shuffle_options[boss]
-            elif "-" in boss:
-                loc, boss_name = boss.split("-")
-                if boss_name not in available_boss_names:
-                    raise ValueError(f"Unknown Boss name {boss_name}")
-                if loc not in available_boss_locations:
-                    raise ValueError(f"Unknown Boss Location {loc}")
-                level = ''
-                if loc.split(" ")[-1] in {"top", "middle", "bottom"}:
-                    # split off level
-                    loc = loc.split(" ")
-                    level = f" {loc[-1]}"
-                    loc = " ".join(loc[:-1])
-                loc = loc.title().replace("Of", "of")
-                if not Bosses.can_place_boss(boss_name.title(), loc, level):
-                    raise ValueError(f"Cannot place {boss_name} at {loc}{level}")
-                bosses.append(boss)
-            elif boss not in available_boss_names:
-                raise ValueError(f"Unknown Boss name or Boss shuffle option {boss}.")
-            else:
-                bosses.append(boss)
-        return ";".join(bosses + [remainder_shuffle])
-    else:
-        raise Exception(f"Boss Shuffle {boss_shuffle} is unknown and boss plando is turned off.")
-
-
-def handle_option(ret: argparse.Namespace, game_weights: dict, option_key: str, option: type(Options.Option)):
+def handle_option(ret: argparse.Namespace, game_weights: dict, option_key: str, option: type(Options.Option), plando_options: PlandoOptions):
     if option_key in game_weights:
         try:
             if not option.supports_weighting:
@@ -442,13 +419,12 @@ def handle_option(ret: argparse.Namespace, game_weights: dict, option_key: str,
         except Exception as e:
             raise Exception(f"Error generating option {option_key} in {ret.game}") from e
         else:
-            if hasattr(player_option, "verify"):
-                player_option.verify(AutoWorldRegister.world_types[ret.game])
+            player_option.verify(AutoWorldRegister.world_types[ret.game], ret.name, plando_options)
     else:
-        setattr(ret, option_key, option(option.default))
+        setattr(ret, option_key, option.from_any(option.default))  # call the from_any here to support default "random"
 
 
-def roll_settings(weights: dict, plando_options: Set[str] = frozenset(("bosses",))):
+def roll_settings(weights: dict, plando_options: PlandoOptions = PlandoOptions.bosses):
     if "linked_options" in weights:
         weights = roll_linked_options(weights)
 
@@ -461,17 +437,11 @@ def roll_settings(weights: dict, plando_options: Set[str] = frozenset(("bosses",
         if tuplize_version(version) > version_tuple:
             raise Exception(f"Settings reports required version of generator is at least {version}, "
                             f"however generator is of version {__version__}")
-        required_plando_options = requirements.get("plando", "")
-        if required_plando_options:
-            required_plando_options = set(option.strip() for option in required_plando_options.split(","))
-            required_plando_options -= plando_options
+        required_plando_options = PlandoOptions.from_option_string(requirements.get("plando", ""))
+        if required_plando_options not in plando_options:
             if required_plando_options:
-                if len(required_plando_options) == 1:
-                    raise Exception(f"Settings reports required plando module {', '.join(required_plando_options)}, "
-                                    f"which is not enabled.")
-                else:
-                    raise Exception(f"Settings reports required plando modules {', '.join(required_plando_options)}, "
-                                    f"which are not enabled.")
+                raise Exception(f"Settings reports required plando module {str(required_plando_options)}, "
+                                f"which is not enabled.")
 
     ret = argparse.Namespace()
     for option_key in Options.per_game_common_options:
@@ -494,18 +464,19 @@ def roll_settings(weights: dict, plando_options: Set[str] = frozenset(("bosses",
         setattr(ret, option_key, option.from_any(get_choice(option_key, weights, option.default)))
 
     if ret.game in AutoWorldRegister.world_types:
-        for option_key, option in world_type.options.items():
-            handle_option(ret, game_weights, option_key, option)
+        for option_key, option in world_type.option_definitions.items():
+            handle_option(ret, game_weights, option_key, option, plando_options)
         for option_key, option in Options.per_game_common_options.items():
             # skip setting this option if already set from common_options, defaulting to root option
-            if not (option_key in Options.common_options and option_key not in game_weights):
-                handle_option(ret, game_weights, option_key, option)
-        if "items" in plando_options:
+            if option_key not in world_type.option_definitions and \
+                    (option_key not in Options.common_options or option_key in game_weights):
+                handle_option(ret, game_weights, option_key, option, plando_options)
+        if PlandoOptions.items in plando_options:
             ret.plando_items = game_weights.get("plando_items", [])
         if ret.game == "Minecraft" or ret.game == "Ocarina of Time":
             # bad hardcoded behavior to make this work for now
             ret.plando_connections = []
-            if "connections" in plando_options:
+            if PlandoOptions.connections in plando_options:
                 options = game_weights.get("plando_connections", [])
                 for placement in options:
                     if roll_percentage(get_choice("percentage", placement, 100)):
@@ -551,9 +522,6 @@ def roll_alttp_settings(ret: argparse.Namespace, weights, plando_options):
 
     ret.goal = goals[goal]
 
-    # TODO consider moving open_pyramid to an automatic variable in the core roller, set to True when
-    # fast ganon + ganon at hole
-    ret.open_pyramid = get_choice_legacy('open_pyramid', weights, 'goal')
 
     extra_pieces = get_choice_legacy('triforce_pieces_mode', weights, 'available')
 
@@ -585,8 +553,6 @@ def roll_alttp_settings(ret: argparse.Namespace, weights, plando_options):
 
     ret.item_functionality = get_choice_legacy('item_functionality', weights)
 
-    boss_shuffle = get_choice_legacy('boss_shuffle', weights)
-    ret.shufflebosses = get_plando_bosses(boss_shuffle, plando_options)
 
     ret.enemy_damage = {None: 'default',
                         'default': 'default',
@@ -625,7 +591,7 @@ def roll_alttp_settings(ret: argparse.Namespace, weights, plando_options):
             raise Exception(f"unknown Medallion {medallion} for {'misery mire' if index == 0 else 'turtle rock'}")
 
     ret.plando_texts = {}
-    if "texts" in plando_options:
+    if PlandoOptions.texts in plando_options:
         tt = TextTable()
         tt.removeUnwantedText()
         options = weights.get("plando_texts", [])
@@ -637,7 +603,7 @@ def roll_alttp_settings(ret: argparse.Namespace, weights, plando_options):
                 ret.plando_texts[at] = str(get_choice_legacy("text", placement))
 
     ret.plando_connections = []
-    if "connections" in plando_options:
+    if PlandoOptions.connections in plando_options:
         options = weights.get("plando_connections", [])
         for placement in options:
             if roll_percentage(get_choice_legacy("percentage", placement, 100)):

KH2Client.py

@@ -0,0 +1,894 @@
+import os
+import asyncio
+import ModuleUpdate
+import json
+import Utils
+from pymem import pymem
+from worlds.kh2.Items import exclusionItem_table, CheckDupingItems
+from worlds.kh2 import all_locations, item_dictionary_table, exclusion_table
+
+from worlds.kh2.WorldLocations import *
+
+from worlds import network_data_package
+
+if __name__ == "__main__":
+    Utils.init_logging("KH2Client", exception_logger="Client")
+
+from NetUtils import ClientStatus
+from CommonClient import gui_enabled, logger, get_base_parser, ClientCommandProcessor, \
+    CommonContext, server_loop
+
+ModuleUpdate.update()
+
+kh2_loc_name_to_id = network_data_package["games"]["Kingdom Hearts 2"]["location_name_to_id"]
+
+
+# class KH2CommandProcessor(ClientCommandProcessor):
+
+
+class KH2Context(CommonContext):
+    # command_processor: int = KH2CommandProcessor
+    game = "Kingdom Hearts 2"
+    items_handling = 0b101  # Indicates you get items sent from other worlds.
+
+    def __init__(self, server_address, password):
+        super(KH2Context, self).__init__(server_address, password)
+        self.kh2LocalItems = None
+        self.ability = None
+        self.growthlevel = None
+        self.KH2_sync_task = None
+        self.syncing = False
+        self.kh2connected = False
+        self.serverconneced = False
+        self.item_name_to_data = {name: data for name, data, in item_dictionary_table.items()}
+        self.location_name_to_data = {name: data for name, data, in all_locations.items()}
+        self.lookup_id_to_item: typing.Dict[int, str] = {data.code: item_name for item_name, data in
+                                                         item_dictionary_table.items() if data.code}
+        self.lookup_id_to_Location: typing.Dict[int, str] = {data.code: item_name for item_name, data in
+                                                             all_locations.items() if data.code}
+        self.location_name_to_worlddata = {name: data for name, data, in all_world_locations.items()}
+
+        self.location_table = {}
+        self.collectible_table = {}
+        self.collectible_override_flags_address = 0
+        self.collectible_offsets = {}
+        self.sending = []
+        # list used to keep track of locations+items player has. Used for disoneccting
+        self.kh2seedsave = None
+        self.slotDataProgressionNames = {}
+        self.kh2seedname = None
+        self.kh2slotdata = None
+        self.itemamount = {}
+        # sora equipped, valor equipped, master equipped, final equipped
+        self.keybladeAnchorList = (0x24F0, 0x32F4, 0x339C, 0x33D4)
+        if "localappdata" in os.environ:
+            self.game_communication_path = os.path.expandvars(r"%localappdata%\KH2AP")
+        self.amountOfPieces = 0
+        # hooked object
+        self.kh2 = None
+        self.ItemIsSafe = False
+        self.game_connected = False
+        self.finalxemnas = False
+        self.worldid = {
+            #  1:   {},  # world of darkness (story cutscenes)
+            2:  TT_Checks,
+            #  3:   {},  # destiny island doesn't have checks to ima put tt checks here
+            4:  HB_Checks,
+            5:  BC_Checks,
+            6:  Oc_Checks,
+            7:  AG_Checks,
+            8:  LoD_Checks,
+            9:  HundredAcreChecks,
+            10: PL_Checks,
+            11: DC_Checks,  # atlantica isn't a supported world. if you go in atlantica it will check dc
+            12: DC_Checks,
+            13: TR_Checks,
+            14: HT_Checks,
+            15: HB_Checks,  # world map, but you only go to the world map while on the way to goa so checking hb
+            16: PR_Checks,
+            17: SP_Checks,
+            18: TWTNW_Checks,
+            #  255: {},  # starting screen
+        }
+        # 0x2A09C00+0x40 is the sve anchor. +1 is the last saved room
+        self.sveroom = 0x2A09C00 + 0x41
+        # 0 not in battle 1 in yellow battle 2 red battle #short
+        self.inBattle = 0x2A0EAC4 + 0x40
+        self.onDeath = 0xAB9078
+        # PC Address anchors
+        self.Now = 0x0714DB8
+        self.Save = 0x09A70B0
+        self.Sys3 = 0x2A59DF0
+        self.Bt10 = 0x2A74880
+        self.BtlEnd = 0x2A0D3E0
+        self.Slot1 = 0x2A20C98
+
+        self.chest_set = set(exclusion_table["Chests"])
+
+        self.keyblade_set = set(CheckDupingItems["Weapons"]["Keyblades"])
+        self.staff_set = set(CheckDupingItems["Weapons"]["Staffs"])
+        self.shield_set = set(CheckDupingItems["Weapons"]["Shields"])
+
+        self.all_weapons = self.keyblade_set.union(self.staff_set).union(self.shield_set)
+
+        self.equipment_categories = CheckDupingItems["Equipment"]
+        self.armor_set = set(self.equipment_categories["Armor"])
+        self.accessories_set = set(self.equipment_categories["Accessories"])
+        self.all_equipment = self.armor_set.union(self.accessories_set)
+
+        self.Equipment_Anchor_Dict = {
+            "Armor":       [0x2504, 0x2506, 0x2508, 0x250A],
+            "Accessories": [0x2514, 0x2516, 0x2518, 0x251A]}
+
+        self.AbilityQuantityDict = {}
+        self.ability_categories = CheckDupingItems["Abilities"]
+
+        self.sora_ability_set = set(self.ability_categories["Sora"])
+        self.donald_ability_set = set(self.ability_categories["Donald"])
+        self.goofy_ability_set = set(self.ability_categories["Goofy"])
+
+        self.all_abilities = self.sora_ability_set.union(self.donald_ability_set).union(self.goofy_ability_set)
+
+        self.boost_set = set(CheckDupingItems["Boosts"])
+        self.stat_increase_set = set(CheckDupingItems["Stat Increases"])
+        self.AbilityQuantityDict = {item: self.item_name_to_data[item].quantity for item in self.all_abilities}
+        #  Growth:[level 1,level 4,slot]
+        self.growth_values_dict = {"High Jump":    [0x05E, 0x061, 0x25DA],
+                                   "Quick Run":    [0x62, 0x65, 0x25DC],
+                                   "Dodge Roll":   [0x234, 0x237, 0x25DE],
+                                   "Aerial Dodge": [0x066, 0x069, 0x25E0],
+                                   "Glide":        [0x6A, 0x6D, 0x25E2]}
+        self.boost_to_anchor_dict = {
+            "Power Boost":   0x24F9,
+            "Magic Boost":   0x24FA,
+            "Defense Boost": 0x24FB,
+            "AP Boost":      0x24F8}
+
+        self.AbilityCodeList = [self.item_name_to_data[item].code for item in exclusionItem_table["Ability"]]
+        self.master_growth = {"High Jump", "Quick Run", "Dodge Roll", "Aerial Dodge", "Glide"}
+
+        self.bitmask_item_code = [
+            0x130000, 0x130001, 0x130002, 0x130003, 0x130004, 0x130005, 0x130006, 0x130007
+            , 0x130008, 0x130009, 0x13000A, 0x13000B, 0x13000C
+            , 0x13001F, 0x130020, 0x130021, 0x130022, 0x130023
+            , 0x13002A, 0x13002B, 0x13002C, 0x13002D]
+
+    async def server_auth(self, password_requested: bool = False):
+        if password_requested and not self.password:
+            await super(KH2Context, self).server_auth(password_requested)
+        await self.get_username()
+        await self.send_connect()
+
+    async def connection_closed(self):
+        self.kh2connected = False
+        self.serverconneced = False
+        if self.kh2seedname is not None and self.auth is not None:
+            with open(os.path.join(self.game_communication_path, f"kh2save{self.kh2seedname}{self.auth}.json"),
+                      'w') as f:
+                f.write(json.dumps(self.kh2seedsave, indent=4))
+        await super(KH2Context, self).connection_closed()
+
+    async def disconnect(self, allow_autoreconnect: bool = False):
+        self.kh2connected = False
+        self.serverconneced = False
+        if self.kh2seedname not in {None} and self.auth not in {None}:
+            with open(os.path.join(self.game_communication_path, f"kh2save{self.kh2seedname}{self.auth}.json"),
+                      'w') as f:
+                f.write(json.dumps(self.kh2seedsave, indent=4))
+        await super(KH2Context, self).disconnect()
+
+    @property
+    def endpoints(self):
+        if self.server:
+            return [self.server]
+        else:
+            return []
+
+    async def shutdown(self):
+        if self.kh2seedname not in {None} and self.auth not in {None}:
+            with open(os.path.join(self.game_communication_path, f"kh2save{self.kh2seedname}{self.auth}.json"),
+                      'w') as f:
+                f.write(json.dumps(self.kh2seedsave, indent=4))
+        await super(KH2Context, self).shutdown()
+
+    def on_package(self, cmd: str, args: dict):
+        if cmd in {"RoomInfo"}:
+            self.kh2seedname = args['seed_name']
+            if not os.path.exists(self.game_communication_path):
+                os.makedirs(self.game_communication_path)
+            if not os.path.exists(self.game_communication_path + f"\kh2save{self.kh2seedname}{self.auth}.json"):
+                self.kh2seedsave = {"itemIndex":        -1,
+                                    # back of soras invo is 0x25E2. Growth should be moved there
+                                    #  Character: [back of invo, front of invo]
+                                    "SoraInvo":         [0x25D8, 0x2546],
+                                    "DonaldInvo":       [0x26F4, 0x2658],
+                                    "GoofyInvo":        [0x280A, 0x276C],
+                                    "AmountInvo":       {
+                                        "ServerItems": {
+                                            "Ability":      {},
+                                            "Amount":       {},
+                                            "Growth":       {"High Jump":    0, "Quick Run": 0, "Dodge Roll": 0,
+                                                             "Aerial Dodge": 0,
+                                                             "Glide":        0},
+                                            "Bitmask":      [],
+                                            "Weapon":       {"Sora": [], "Donald": [], "Goofy": []},
+                                            "Equipment":    [],
+                                            "Magic":        {},
+                                            "StatIncrease": {},
+                                            "Boost":        {},
+                                        },
+                                        "LocalItems":  {
+                                            "Ability":      {},
+                                            "Amount":       {},
+                                            "Growth":       {"High Jump":    0, "Quick Run": 0, "Dodge Roll": 0,
+                                                             "Aerial Dodge": 0, "Glide": 0},
+                                            "Bitmask":      [],
+                                            "Weapon":       {"Sora": [], "Donald": [], "Goofy": []},
+                                            "Equipment":    [],
+                                            "Magic":        {},
+                                            "StatIncrease": {},
+                                            "Boost":        {},
+                                        }},
+                                    #  1,3,255 are in this list in case the player gets locations in those "worlds" and I need to still have them checked
+                                    "LocationsChecked": [],
+                                    "Levels":           {
+                                        "SoraLevel":   0,
+                                        "ValorLevel":  0,
+                                        "WisdomLevel": 0,
+                                        "LimitLevel":  0,
+                                        "MasterLevel": 0,
+                                        "FinalLevel":  0,
+                                    },
+                                    "SoldEquipment":    [],
+                                    "SoldBoosts":       {"Power Boost":   0,
+                                                         "Magic Boost":   0,
+                                                         "Defense Boost": 0,
+                                                         "AP Boost":      0}
+                                    }
+                with open(os.path.join(self.game_communication_path, f"kh2save{self.kh2seedname}{self.auth}.json"),
+                          'wt') as f:
+                    pass
+                self.locations_checked = set()
+            elif os.path.exists(self.game_communication_path + f"\kh2save{self.kh2seedname}{self.auth}.json"):
+                with open(self.game_communication_path + f"\kh2save{self.kh2seedname}{self.auth}.json", 'r') as f:
+                    self.kh2seedsave = json.load(f)
+                    self.locations_checked = set(self.kh2seedsave["LocationsChecked"])
+            self.serverconneced = True
+
+        if cmd in {"Connected"}:
+            self.kh2slotdata = args['slot_data']
+            self.kh2LocalItems = {int(location): item for location, item in self.kh2slotdata["LocalItems"].items()}
+            try:
+                self.kh2 = pymem.Pymem(process_name="KINGDOM HEARTS II FINAL MIX")
+                logger.info("You are now auto-tracking")
+                self.kh2connected = True
+            except Exception as e:
+                logger.info("Line 247")
+                if self.kh2connected:
+                    logger.info("Connection Lost")
+                    self.kh2connected = False
+                logger.info(e)
+
+        if cmd in {"ReceivedItems"}:
+            start_index = args["index"]
+            if start_index == 0:
+                # resetting everything that were sent from the server
+                self.kh2seedsave["SoraInvo"][0] = 0x25D8
+                self.kh2seedsave["DonaldInvo"][0] = 0x26F4
+                self.kh2seedsave["GoofyInvo"][0] = 0x280A
+                self.kh2seedsave["itemIndex"] = - 1
+                self.kh2seedsave["AmountInvo"]["ServerItems"] = {
+                    "Ability":      {},
+                    "Amount":       {},
+                    "Growth":       {"High Jump":    0, "Quick Run": 0, "Dodge Roll": 0,
+                                     "Aerial Dodge": 0,
+                                     "Glide":        0},
+                    "Bitmask":      [],
+                    "Weapon":       {"Sora": [], "Donald": [], "Goofy": []},
+                    "Equipment":    [],
+                    "Magic":        {},
+                    "StatIncrease": {},
+                    "Boost":        {},
+                }
+            if start_index > self.kh2seedsave["itemIndex"]:
+                self.kh2seedsave["itemIndex"] = start_index
+                for item in args['items']:
+                    asyncio.create_task(self.give_item(item.item))
+
+        if cmd in {"RoomUpdate"}:
+            if "checked_locations" in args:
+                new_locations = set(args["checked_locations"])
+                # TODO: make this take locations from other players on the same slot so proper coop happens
+                #  items_to_give = [self.kh2slotdata["LocalItems"][str(location_id)] for location_id in new_locations if
+                #                 location_id in self.kh2LocalItems.keys()]
+                self.checked_locations |= new_locations
+
+    async def checkWorldLocations(self):
+        try:
+            currentworldint = int.from_bytes(self.kh2.read_bytes(self.kh2.base_address + 0x0714DB8, 1), "big")
+            if currentworldint in self.worldid:
+                curworldid = self.worldid[currentworldint]
+                for location, data in curworldid.items():
+                    locationId = kh2_loc_name_to_id[location]
+                    if locationId not in self.locations_checked \
+                            and (int.from_bytes(
+                            self.kh2.read_bytes(self.kh2.base_address + self.Save + data.addrObtained, 1),
+                            "big") & 0x1 << data.bitIndex) > 0:
+                        self.sending = self.sending + [(int(locationId))]
+        except Exception as e:
+            logger.info("Line 285")
+            if self.kh2connected:
+                logger.info("Connection Lost.")
+                self.kh2connected = False
+            logger.info(e)
+
+    async def checkLevels(self):
+        try:
+            for location, data in SoraLevels.items():
+                currentLevel = int.from_bytes(
+                        self.kh2.read_bytes(self.kh2.base_address + self.Save + 0x24FF, 1), "big")
+                locationId = kh2_loc_name_to_id[location]
+                if locationId not in self.locations_checked \
+                        and currentLevel >= data.bitIndex:
+                    if self.kh2seedsave["Levels"]["SoraLevel"] < currentLevel:
+                        self.kh2seedsave["Levels"]["SoraLevel"] = currentLevel
+                    self.sending = self.sending + [(int(locationId))]
+            formDict = {
+                0: ["ValorLevel", ValorLevels], 1: ["WisdomLevel", WisdomLevels], 2: ["LimitLevel", LimitLevels],
+                3: ["MasterLevel", MasterLevels], 4: ["FinalLevel", FinalLevels]}
+            for i in range(5):
+                for location, data in formDict[i][1].items():
+                    formlevel = int.from_bytes(
+                            self.kh2.read_bytes(self.kh2.base_address + self.Save + data.addrObtained, 1), "big")
+                    locationId = kh2_loc_name_to_id[location]
+                    if locationId not in self.locations_checked \
+                            and formlevel >= data.bitIndex:
+                        if formlevel > self.kh2seedsave["Levels"][formDict[i][0]]:
+                            self.kh2seedsave["Levels"][formDict[i][0]] = formlevel
+                        self.sending = self.sending + [(int(locationId))]
+        except Exception as e:
+            logger.info("Line 312")
+            if self.kh2connected:
+                logger.info("Connection Lost.")
+                self.kh2connected = False
+            logger.info(e)
+
+    async def checkSlots(self):
+        try:
+            for location, data in weaponSlots.items():
+                locationId = kh2_loc_name_to_id[location]
+                if locationId not in self.locations_checked:
+                    if int.from_bytes(self.kh2.read_bytes(self.kh2.base_address + self.Save + data.addrObtained, 1),
+                                      "big") > 0:
+                        self.sending = self.sending + [(int(locationId))]
+
+            for location, data in formSlots.items():
+                locationId = kh2_loc_name_to_id[location]
+                if locationId not in self.locations_checked:
+                    if int.from_bytes(self.kh2.read_bytes(self.kh2.base_address + self.Save + data.addrObtained, 1),
+                                      "big") & 0x1 << data.bitIndex > 0:
+                        # self.locations_checked
+                        self.sending = self.sending + [(int(locationId))]
+
+        except Exception as e:
+            if self.kh2connected:
+                logger.info("Line 333")
+                logger.info("Connection Lost.")
+                self.kh2connected = False
+            logger.info(e)
+
+    async def verifyChests(self):
+        try:
+            for location in self.locations_checked:
+                locationName = self.lookup_id_to_Location[location]
+                if locationName in self.chest_set:
+                    if locationName in self.location_name_to_worlddata.keys():
+                        locationData = self.location_name_to_worlddata[locationName]
+                        if int.from_bytes(
+                                self.kh2.read_bytes(self.kh2.base_address + self.Save + locationData.addrObtained, 1),
+                                "big") & 0x1 << locationData.bitIndex == 0:
+                            roomData = int.from_bytes(
+                                    self.kh2.read_bytes(self.kh2.base_address + self.Save + locationData.addrObtained,
+                                                        1), "big")
+                            self.kh2.write_bytes(self.kh2.base_address + self.Save + locationData.addrObtained,
+                                                 (roomData | 0x01 << locationData.bitIndex).to_bytes(1, 'big'), 1)
+
+        except Exception as e:
+            if self.kh2connected:
+                logger.info("Line 350")
+                logger.info("Connection Lost.")
+                self.kh2connected = False
+            logger.info(e)
+
+    async def verifyLevel(self):
+        for leveltype, anchor in {"SoraLevel":   0x24FF,
+                                  "ValorLevel":  0x32F6,
+                                  "WisdomLevel": 0x332E,
+                                  "LimitLevel":  0x3366,
+                                  "MasterLevel": 0x339E,
+                                  "FinalLevel":  0x33D6}.items():
+            if int.from_bytes(self.kh2.read_bytes(self.kh2.base_address + self.Save + anchor, 1), "big") < \
+                    self.kh2seedsave["Levels"][leveltype]:
+                self.kh2.write_bytes(self.kh2.base_address + self.Save + anchor,
+                                     (self.kh2seedsave["Levels"][leveltype]).to_bytes(1, 'big'), 1)
+
+    async def give_item(self, item, ItemType="ServerItems"):
+        try:
+            itemname = self.lookup_id_to_item[item]
+            itemcode = self.item_name_to_data[itemname]
+            if itemcode.ability:
+                abilityInvoType = 0
+                TwilightZone = 2
+                if ItemType == "LocalItems":
+                    abilityInvoType = 1
+                    TwilightZone = -2
+                if itemname in {"High Jump", "Quick Run", "Dodge Roll", "Aerial Dodge", "Glide"}:
+                    self.kh2seedsave["AmountInvo"][ItemType]["Growth"][itemname] += 1
+                    return
+
+                if itemname not in self.kh2seedsave["AmountInvo"][ItemType]["Ability"]:
+                    self.kh2seedsave["AmountInvo"][ItemType]["Ability"][itemname] = []
+                    #  appending the slot that the ability should be in
+
+                if len(self.kh2seedsave["AmountInvo"][ItemType]["Ability"][itemname]) < \
+                        self.AbilityQuantityDict[itemname]:
+                    if itemname in self.sora_ability_set:
+                        self.kh2seedsave["AmountInvo"][ItemType]["Ability"][itemname].append(
+                                self.kh2seedsave["SoraInvo"][abilityInvoType])
+                        self.kh2seedsave["SoraInvo"][abilityInvoType] -= TwilightZone
+                    elif itemname in self.donald_ability_set:
+                        self.kh2seedsave["AmountInvo"][ItemType]["Ability"][itemname].append(
+                                self.kh2seedsave["DonaldInvo"][abilityInvoType])
+                        self.kh2seedsave["DonaldInvo"][abilityInvoType] -= TwilightZone
+                    else:
+                        self.kh2seedsave["AmountInvo"][ItemType]["Ability"][itemname].append(
+                                self.kh2seedsave["GoofyInvo"][abilityInvoType])
+                        self.kh2seedsave["GoofyInvo"][abilityInvoType] -= TwilightZone
+
+            elif itemcode.code in self.bitmask_item_code:
+
+                if itemname not in self.kh2seedsave["AmountInvo"][ItemType]["Bitmask"]:
+                    self.kh2seedsave["AmountInvo"][ItemType]["Bitmask"].append(itemname)
+
+            elif itemcode.memaddr in {0x3594, 0x3595, 0x3596, 0x3597, 0x35CF, 0x35D0}:
+
+                if itemname in self.kh2seedsave["AmountInvo"][ItemType]["Magic"]:
+                    self.kh2seedsave["AmountInvo"][ItemType]["Magic"][itemname] += 1
+                else:
+                    self.kh2seedsave["AmountInvo"][ItemType]["Magic"][itemname] = 1
+            elif itemname in self.all_equipment:
+
+                self.kh2seedsave["AmountInvo"][ItemType]["Equipment"].append(itemname)
+
+            elif itemname in self.all_weapons:
+                if itemname in self.keyblade_set:
+                    self.kh2seedsave["AmountInvo"][ItemType]["Weapon"]["Sora"].append(itemname)
+                elif itemname in self.staff_set:
+                    self.kh2seedsave["AmountInvo"][ItemType]["Weapon"]["Donald"].append(itemname)
+                else:
+                    self.kh2seedsave["AmountInvo"][ItemType]["Weapon"]["Goofy"].append(itemname)
+
+            elif itemname in self.boost_set:
+                if itemname in self.kh2seedsave["AmountInvo"][ItemType]["Boost"]:
+                    self.kh2seedsave["AmountInvo"][ItemType]["Boost"][itemname] += 1
+                else:
+                    self.kh2seedsave["AmountInvo"][ItemType]["Boost"][itemname] = 1
+
+            elif itemname in self.stat_increase_set:
+
+                if itemname in self.kh2seedsave["AmountInvo"][ItemType]["StatIncrease"]:
+                    self.kh2seedsave["AmountInvo"][ItemType]["StatIncrease"][itemname] += 1
+                else:
+                    self.kh2seedsave["AmountInvo"][ItemType]["StatIncrease"][itemname] = 1
+
+            else:
+                if itemname in self.kh2seedsave["AmountInvo"][ItemType]["Amount"]:
+                    self.kh2seedsave["AmountInvo"][ItemType]["Amount"][itemname] += 1
+                else:
+                    self.kh2seedsave["AmountInvo"][ItemType]["Amount"][itemname] = 1
+
+        except Exception as e:
+            if self.kh2connected:
+                logger.info("Line 398")
+                logger.info("Connection Lost.")
+                self.kh2connected = False
+            logger.info(e)
+
+    def run_gui(self):
+        """Import kivy UI system and start running it as self.ui_task."""
+        from kvui import GameManager
+
+        class KH2Manager(GameManager):
+            logging_pairs = [
+                ("Client", "Archipelago")
+            ]
+            base_title = "Archipelago KH2 Client"
+
+        self.ui = KH2Manager(self)
+        self.ui_task = asyncio.create_task(self.ui.async_run(), name="UI")
+
+    async def IsInShop(self, sellable, master_boost):
+        # journal = 0x741230 shop = 0x741320
+        # if journal=-1 and shop = 5 then in shop
+        # if journam !=-1 and shop = 10 then journal
+        journal = self.kh2.read_short(self.kh2.base_address + 0x741230)
+        shop = self.kh2.read_short(self.kh2.base_address + 0x741320)
+        if (journal == -1 and shop == 5) or (journal != -1 and shop == 10):
+            # print("your in the shop")
+            sellable_dict = {}
+            for itemName in sellable:
+                itemdata = self.item_name_to_data[itemName]
+                amount = int.from_bytes(
+                        self.kh2.read_bytes(self.kh2.base_address + self.Save + itemdata.memaddr, 1), "big")
+                sellable_dict[itemName] = amount
+            while (journal == -1 and shop == 5) or (journal != -1 and shop == 10):
+                journal = self.kh2.read_short(self.kh2.base_address + 0x741230)
+                shop = self.kh2.read_short(self.kh2.base_address + 0x741320)
+                await asyncio.sleep(0.5)
+            for item, amount in sellable_dict.items():
+                itemdata = self.item_name_to_data[item]
+                afterShop = int.from_bytes(
+                        self.kh2.read_bytes(self.kh2.base_address + self.Save + itemdata.memaddr, 1), "big")
+                if afterShop < amount:
+                    if item in master_boost:
+                        self.kh2seedsave["SoldBoosts"][item] += (amount - afterShop)
+                    else:
+                        self.kh2seedsave["SoldEquipment"].append(item)
+
+    async def verifyItems(self):
+        try:
+            local_amount = set(self.kh2seedsave["AmountInvo"]["LocalItems"]["Amount"].keys())
+            server_amount = set(self.kh2seedsave["AmountInvo"]["ServerItems"]["Amount"].keys())
+            master_amount = local_amount | server_amount
+
+            local_ability = set(self.kh2seedsave["AmountInvo"]["LocalItems"]["Ability"].keys())
+            server_ability = set(self.kh2seedsave["AmountInvo"]["ServerItems"]["Ability"].keys())
+            master_ability = local_ability | server_ability
+
+            local_bitmask = set(self.kh2seedsave["AmountInvo"]["LocalItems"]["Bitmask"])
+            server_bitmask = set(self.kh2seedsave["AmountInvo"]["ServerItems"]["Bitmask"])
+            master_bitmask = local_bitmask | server_bitmask
+
+            local_keyblade = set(self.kh2seedsave["AmountInvo"]["LocalItems"]["Weapon"]["Sora"])
+            local_staff = set(self.kh2seedsave["AmountInvo"]["LocalItems"]["Weapon"]["Donald"])
+            local_shield = set(self.kh2seedsave["AmountInvo"]["LocalItems"]["Weapon"]["Goofy"])
+
+            server_keyblade = set(self.kh2seedsave["AmountInvo"]["ServerItems"]["Weapon"]["Sora"])
+            server_staff = set(self.kh2seedsave["AmountInvo"]["ServerItems"]["Weapon"]["Donald"])
+            server_shield = set(self.kh2seedsave["AmountInvo"]["ServerItems"]["Weapon"]["Goofy"])
+
+            master_keyblade = local_keyblade | server_keyblade
+            master_staff = local_staff | server_staff
+            master_shield = local_shield | server_shield
+
+            local_equipment = set(self.kh2seedsave["AmountInvo"]["LocalItems"]["Equipment"])
+            server_equipment = set(self.kh2seedsave["AmountInvo"]["ServerItems"]["Equipment"])
+            master_equipment = local_equipment | server_equipment
+
+            local_magic = set(self.kh2seedsave["AmountInvo"]["LocalItems"]["Magic"].keys())
+            server_magic = set(self.kh2seedsave["AmountInvo"]["ServerItems"]["Magic"].keys())
+            master_magic = local_magic | server_magic
+
+            local_stat = set(self.kh2seedsave["AmountInvo"]["LocalItems"]["StatIncrease"].keys())
+            server_stat = set(self.kh2seedsave["AmountInvo"]["ServerItems"]["StatIncrease"].keys())
+            master_stat = local_stat | server_stat
+
+            local_boost = set(self.kh2seedsave["AmountInvo"]["LocalItems"]["Boost"].keys())
+            server_boost = set(self.kh2seedsave["AmountInvo"]["ServerItems"]["Boost"].keys())
+            master_boost = local_boost | server_boost
+
+            master_sell = master_equipment | master_staff | master_shield | master_boost
+            await asyncio.create_task(self.IsInShop(master_sell, master_boost))
+            for itemName in master_amount:
+                itemData = self.item_name_to_data[itemName]
+                amountOfItems = 0
+                if itemName in local_amount:
+                    amountOfItems += self.kh2seedsave["AmountInvo"]["LocalItems"]["Amount"][itemName]
+                if itemName in server_amount:
+                    amountOfItems += self.kh2seedsave["AmountInvo"]["ServerItems"]["Amount"][itemName]
+
+                if itemName == "Torn Page":
+                    # Torn Pages are handled differently because they can be consumed.
+                    # Will check the progression in 100 acre and - the amount of visits
+                    # amountofitems-amount of visits done
+                    for location, data in tornPageLocks.items():
+                        if int.from_bytes(self.kh2.read_bytes(self.kh2.base_address + self.Save + data.addrObtained, 1),
+                                          "big") & 0x1 << data.bitIndex > 0:
+                            amountOfItems -= 1
+                if int.from_bytes(self.kh2.read_bytes(self.kh2.base_address + self.Save + itemData.memaddr, 1),
+                                  "big") != amountOfItems and amountOfItems >= 0:
+                    self.kh2.write_bytes(self.kh2.base_address + self.Save + itemData.memaddr,
+                                         amountOfItems.to_bytes(1, 'big'), 1)
+
+            for itemName in master_keyblade:
+                itemData = self.item_name_to_data[itemName]
+                # if the inventory slot for that keyblade is less than the amount they should have
+                if int.from_bytes(self.kh2.read_bytes(self.kh2.base_address + self.Save + itemData.memaddr, 1),
+                                  "big") != 1 and int.from_bytes(self.kh2.read_bytes(self.kh2.base_address + 0x1CFF, 1),
+                                                                 "big") != 13:
+                    # Checking form anchors for the keyblade
+                    if self.kh2.read_short(self.kh2.base_address + self.Save + 0x24F0) == itemData.kh2id \
+                            or self.kh2.read_short(self.kh2.base_address + self.Save + 0x32F4) == itemData.kh2id \
+                            or self.kh2.read_short(self.kh2.base_address + self.Save + 0x339C) == itemData.kh2id \
+                            or self.kh2.read_short(self.kh2.base_address + self.Save + 0x33D4) == itemData.kh2id:
+                        self.kh2.write_bytes(self.kh2.base_address + self.Save + itemData.memaddr,
+                                             (0).to_bytes(1, 'big'), 1)
+                    else:
+                        self.kh2.write_bytes(self.kh2.base_address + self.Save + itemData.memaddr,
+                                             (1).to_bytes(1, 'big'), 1)
+            for itemName in master_staff:
+                itemData = self.item_name_to_data[itemName]
+                if int.from_bytes(self.kh2.read_bytes(self.kh2.base_address + self.Save + itemData.memaddr, 1),
+                                  "big") != 1 \
+                        and self.kh2.read_short(self.kh2.base_address + self.Save + 0x2604) != itemData.kh2id \
+                        and itemName not in self.kh2seedsave["SoldEquipment"]:
+                    self.kh2.write_bytes(self.kh2.base_address + self.Save + itemData.memaddr,
+                                         (1).to_bytes(1, 'big'), 1)
+
+            for itemName in master_shield:
+                itemData = self.item_name_to_data[itemName]
+                if int.from_bytes(self.kh2.read_bytes(self.kh2.base_address + self.Save + itemData.memaddr, 1),
+                                  "big") != 1 \
+                        and self.kh2.read_short(self.kh2.base_address + self.Save + 0x2718) != itemData.kh2id \
+                        and itemName not in self.kh2seedsave["SoldEquipment"]:
+                    self.kh2.write_bytes(self.kh2.base_address + self.Save + itemData.memaddr,
+                                         (1).to_bytes(1, 'big'), 1)
+
+            for itemName in master_ability:
+                itemData = self.item_name_to_data[itemName]
+                ability_slot = []
+                if itemName in local_ability:
+                    ability_slot += self.kh2seedsave["AmountInvo"]["LocalItems"]["Ability"][itemName]
+                if itemName in server_ability:
+                    ability_slot += self.kh2seedsave["AmountInvo"]["ServerItems"]["Ability"][itemName]
+                for slot in ability_slot:
+                    current = self.kh2.read_short(self.kh2.base_address + self.Save + slot)
+                    ability = current & 0x0FFF
+                    if ability | 0x8000 != (0x8000 + itemData.memaddr):
+                        if current - 0x8000 > 0:
+                            self.kh2.write_short(self.kh2.base_address + self.Save + slot, (0x8000 + itemData.memaddr))
+                        else:
+                            self.kh2.write_short(self.kh2.base_address + self.Save + slot, itemData.memaddr)
+            # removes the duped ability if client gave faster than the game.
+            for charInvo in {"SoraInvo", "DonaldInvo", "GoofyInvo"}:
+                if self.kh2.read_short(self.kh2.base_address + self.Save + self.kh2seedsave[charInvo][1]) != 0 and \
+                        self.kh2seedsave[charInvo][1] + 2 < self.kh2seedsave[charInvo][0]:
+                    self.kh2.write_short(self.kh2.base_address + self.Save + self.kh2seedsave[charInvo][1], 0)
+            # remove the dummy level 1 growths if they are in these invo slots.
+            for inventorySlot in {0x25CE, 0x25D0, 0x25D2, 0x25D4, 0x25D6, 0x25D8}:
+                current = self.kh2.read_short(self.kh2.base_address + self.Save + inventorySlot)
+                ability = current & 0x0FFF
+                if 0x05E <= ability <= 0x06D:
+                    self.kh2.write_short(self.kh2.base_address + self.Save + inventorySlot, 0)
+
+            for itemName in self.master_growth:
+                growthLevel = self.kh2seedsave["AmountInvo"]["ServerItems"]["Growth"][itemName] \
+                              + self.kh2seedsave["AmountInvo"]["LocalItems"]["Growth"][itemName]
+                if growthLevel > 0:
+                    slot = self.growth_values_dict[itemName][2]
+                    min_growth = self.growth_values_dict[itemName][0]
+                    max_growth = self.growth_values_dict[itemName][1]
+                    if growthLevel > 4:
+                        growthLevel = 4
+                    current_growth_level = self.kh2.read_short(self.kh2.base_address + self.Save + slot)
+                    ability = current_growth_level & 0x0FFF
+                    # if the player should be getting a growth ability
+                    if ability | 0x8000 != 0x8000 + min_growth - 1 + growthLevel:
+                        # if it should be level one of that growth
+                        if 0x8000 + min_growth - 1 + growthLevel <= 0x8000 + min_growth or ability < min_growth:
+                            self.kh2.write_short(self.kh2.base_address + self.Save + slot, min_growth)
+                        # if it is already in the inventory
+                        elif ability | 0x8000 < (0x8000 + max_growth):
+                            self.kh2.write_short(self.kh2.base_address + self.Save + slot, current_growth_level + 1)
+
+            for itemName in master_bitmask:
+                itemData = self.item_name_to_data[itemName]
+                itemMemory = int.from_bytes(
+                        self.kh2.read_bytes(self.kh2.base_address + self.Save + itemData.memaddr, 1), "big")
+                if (int.from_bytes(self.kh2.read_bytes(self.kh2.base_address + self.Save + itemData.memaddr, 1),
+                                   "big") & 0x1 << itemData.bitmask) == 0:
+                    # when getting a form anti points should be reset to 0 but bit-shift doesn't trigger the game.
+                    if itemName in {"Valor Form", "Wisdom Form", "Limit Form", "Master Form", "Final Form"}:
+                        self.kh2.write_bytes(self.kh2.base_address + self.Save + 0x3410,
+                                             (0).to_bytes(1, 'big'), 1)
+                    self.kh2.write_bytes(self.kh2.base_address + self.Save + itemData.memaddr,
+                                         (itemMemory | 0x01 << itemData.bitmask).to_bytes(1, 'big'), 1)
+
+            for itemName in master_equipment:
+                itemData = self.item_name_to_data[itemName]
+                isThere = False
+                if itemName in self.accessories_set:
+                    Equipment_Anchor_List = self.Equipment_Anchor_Dict["Accessories"]
+                else:
+                    Equipment_Anchor_List = self.Equipment_Anchor_Dict["Armor"]
+                    # Checking form anchors for the equipment
+                for slot in Equipment_Anchor_List:
+                    if self.kh2.read_short(self.kh2.base_address + self.Save + slot) == itemData.kh2id:
+                        isThere = True
+                        if int.from_bytes(self.kh2.read_bytes(self.kh2.base_address + self.Save + itemData.memaddr, 1),
+                                          "big") != 0:
+                            self.kh2.write_bytes(self.kh2.base_address + self.Save + itemData.memaddr,
+                                                 (0).to_bytes(1, 'big'), 1)
+                        break
+                if not isThere and itemName not in self.kh2seedsave["SoldEquipment"]:
+                    if int.from_bytes(self.kh2.read_bytes(self.kh2.base_address + self.Save + itemData.memaddr, 1),
+                                      "big") != 1:
+                        self.kh2.write_bytes(self.kh2.base_address + self.Save + itemData.memaddr,
+                                             (1).to_bytes(1, 'big'), 1)
+
+            for itemName in master_magic:
+                itemData = self.item_name_to_data[itemName]
+                amountOfItems = 0
+                if itemName in local_magic:
+                    amountOfItems += self.kh2seedsave["AmountInvo"]["LocalItems"]["Magic"][itemName]
+                if itemName in server_magic:
+                    amountOfItems += self.kh2seedsave["AmountInvo"]["ServerItems"]["Magic"][itemName]
+                if int.from_bytes(self.kh2.read_bytes(self.kh2.base_address + self.Save + itemData.memaddr, 1),
+                                  "big") != amountOfItems \
+                        and int.from_bytes(self.kh2.read_bytes(self.kh2.base_address + 0x741320, 1), "big") in {10, 8}:
+                    self.kh2.write_bytes(self.kh2.base_address + self.Save + itemData.memaddr,
+                                         amountOfItems.to_bytes(1, 'big'), 1)
+
+            for itemName in master_stat:
+                itemData = self.item_name_to_data[itemName]
+                amountOfItems = 0
+                if itemName in local_stat:
+                    amountOfItems += self.kh2seedsave["AmountInvo"]["LocalItems"]["StatIncrease"][itemName]
+                if itemName in server_stat:
+                    amountOfItems += self.kh2seedsave["AmountInvo"]["ServerItems"]["StatIncrease"][itemName]
+
+                # 0x130293 is Crit_1's location id for touching the computer
+                if int.from_bytes(self.kh2.read_bytes(self.kh2.base_address + self.Save + itemData.memaddr, 1),
+                                  "big") != amountOfItems \
+                        and int.from_bytes(self.kh2.read_bytes(self.kh2.base_address + self.Slot1 + 0x1B2, 1),
+                                           "big") >= 5 and int.from_bytes(
+                        self.kh2.read_bytes(self.kh2.base_address + self.Save + 0x23DF, 1),
+                        "big") > 0:
+                    self.kh2.write_bytes(self.kh2.base_address + self.Save + itemData.memaddr,
+                                         amountOfItems.to_bytes(1, 'big'), 1)
+
+            for itemName in master_boost:
+                itemData = self.item_name_to_data[itemName]
+                amountOfItems = 0
+                if itemName in local_boost:
+                    amountOfItems += self.kh2seedsave["AmountInvo"]["LocalItems"]["Boost"][itemName]
+                if itemName in server_boost:
+                    amountOfItems += self.kh2seedsave["AmountInvo"]["ServerItems"]["Boost"][itemName]
+                amountOfBoostsInInvo = int.from_bytes(
+                        self.kh2.read_bytes(self.kh2.base_address + self.Save + itemData.memaddr, 1),
+                        "big")
+                amountOfUsedBoosts = int.from_bytes(
+                        self.kh2.read_bytes(self.kh2.base_address + self.Save + self.boost_to_anchor_dict[itemName], 1),
+                        "big")
+                # Ap Boots start at +50 for some reason
+                if itemName == "AP Boost":
+                    amountOfUsedBoosts -= 50
+                totalBoosts = (amountOfBoostsInInvo + amountOfUsedBoosts)
+                if totalBoosts <= amountOfItems - self.kh2seedsave["SoldBoosts"][
+                    itemName] and amountOfBoostsInInvo < 255:
+                    self.kh2.write_bytes(self.kh2.base_address + self.Save + itemData.memaddr,
+                                         (amountOfBoostsInInvo + 1).to_bytes(1, 'big'), 1)
+
+        except Exception as e:
+            logger.info("Line 573")
+            if self.kh2connected:
+                logger.info("Connection Lost.")
+                self.kh2connected = False
+            logger.info(e)
+
+
+def finishedGame(ctx: KH2Context, message):
+    if ctx.kh2slotdata['FinalXemnas'] == 1:
+        if 0x1301ED in message[0]["locations"]:
+            ctx.finalxemnas = True
+    # three proofs
+    if ctx.kh2slotdata['Goal'] == 0:
+        if int.from_bytes(ctx.kh2.read_bytes(ctx.kh2.base_address + ctx.Save + 0x36B2, 1), "big") > 0 \
+                and int.from_bytes(ctx.kh2.read_bytes(ctx.kh2.base_address + ctx.Save + 0x36B3, 1), "big") > 0 \
+                and int.from_bytes(ctx.kh2.read_bytes(ctx.kh2.base_address + ctx.Save + 0x36B4, 1), "big") > 0:
+            if ctx.kh2slotdata['FinalXemnas'] == 1:
+                if ctx.finalxemnas:
+                    return True
+                else:
+                    return False
+            else:
+                return True
+        else:
+            return False
+    elif ctx.kh2slotdata['Goal'] == 1:
+        if int.from_bytes(ctx.kh2.read_bytes(ctx.kh2.base_address + ctx.Save + 0x3641, 1), "big") >= \
+                ctx.kh2slotdata['LuckyEmblemsRequired']:
+            ctx.kh2.write_bytes(ctx.kh2.base_address + ctx.Save + 0x36B2, (1).to_bytes(1, 'big'), 1)
+            ctx.kh2.write_bytes(ctx.kh2.base_address + ctx.Save + 0x36B3, (1).to_bytes(1, 'big'), 1)
+            ctx.kh2.write_bytes(ctx.kh2.base_address + ctx.Save + 0x36B4, (1).to_bytes(1, 'big'), 1)
+            if ctx.kh2slotdata['FinalXemnas'] == 1:
+                if ctx.finalxemnas:
+                    return True
+                else:
+                    return False
+            else:
+                return True
+        else:
+            return False
+    elif ctx.kh2slotdata['Goal'] == 2:
+        for boss in ctx.kh2slotdata["hitlist"]:
+            if boss in message[0]["locations"]:
+                ctx.amountOfPieces += 1
+        if ctx.amountOfPieces >= ctx.kh2slotdata["BountyRequired"]:
+            ctx.kh2.write_bytes(ctx.kh2.base_address + ctx.Save + 0x36B2, (1).to_bytes(1, 'big'), 1)
+            ctx.kh2.write_bytes(ctx.kh2.base_address + ctx.Save + 0x36B3, (1).to_bytes(1, 'big'), 1)
+            ctx.kh2.write_bytes(ctx.kh2.base_address + ctx.Save + 0x36B4, (1).to_bytes(1, 'big'), 1)
+            if ctx.kh2slotdata['FinalXemnas'] == 1:
+                if ctx.finalxemnas:
+                    return True
+                else:
+                    return False
+            else:
+                return True
+        else:
+            return False
+
+
+async def kh2_watcher(ctx: KH2Context):
+    while not ctx.exit_event.is_set():
+        try:
+            if ctx.kh2connected and ctx.serverconneced:
+                ctx.sending = []
+                await asyncio.create_task(ctx.checkWorldLocations())
+                await asyncio.create_task(ctx.checkLevels())
+                await asyncio.create_task(ctx.checkSlots())
+                await asyncio.create_task(ctx.verifyChests())
+                await asyncio.create_task(ctx.verifyItems())
+                await asyncio.create_task(ctx.verifyLevel())
+                message = [{"cmd": 'LocationChecks', "locations": ctx.sending}]
+                if finishedGame(ctx, message):
+                    await ctx.send_msgs([{"cmd": "StatusUpdate", "status": ClientStatus.CLIENT_GOAL}])
+                    ctx.finished_game = True
+                location_ids = []
+                location_ids = [location for location in message[0]["locations"] if location not in location_ids]
+                for location in location_ids:
+                    if location not in ctx.locations_checked:
+                        ctx.locations_checked.add(location)
+                        ctx.kh2seedsave["LocationsChecked"].append(location)
+                        if location in ctx.kh2LocalItems:
+                            item = ctx.kh2slotdata["LocalItems"][str(location)]
+                            await asyncio.create_task(ctx.give_item(item, "LocalItems"))
+                await ctx.send_msgs(message)
+            elif not ctx.kh2connected and ctx.serverconneced:
+                logger.info("Game is not open. Disconnecting from Server.")
+                await ctx.disconnect()
+        except Exception as e:
+            logger.info("Line 661")
+            if ctx.kh2connected:
+                logger.info("Connection Lost.")
+                ctx.kh2connected = False
+            logger.info(e)
+        await asyncio.sleep(0.5)
+
+
+if __name__ == '__main__':
+    async def main(args):
+        ctx = KH2Context(args.connect, args.password)
+        ctx.server_task = asyncio.create_task(server_loop(ctx), name="server loop")
+        if gui_enabled:
+            ctx.run_gui()
+        ctx.run_cli()
+        progression_watcher = asyncio.create_task(
+                kh2_watcher(ctx), name="KH2ProgressionWatcher")
+
+        await ctx.exit_event.wait()
+        ctx.server_address = None
+
+        await progression_watcher
+
+        await ctx.shutdown()
+
+
+    import colorama
+
+    parser = get_base_parser(description="KH2 Client, for text interfacing.")
+
+    args, rest = parser.parse_known_args()
+    colorama.init()
+    asyncio.run(main(args))
+    colorama.deinit()

Launcher.py

@@ -10,21 +10,25 @@ Scroll down to components= to add components to the launcher as well as setup.py
 
 
 import argparse
-from os.path import isfile
-import sys
-from typing import Iterable, Sequence, Callable, Union, Optional
-import subprocess
 import itertools
-from Utils import is_frozen, user_path, local_path, init_logging
-from shutil import which
+import multiprocessing
 import shlex
-from enum import Enum, auto
-import logging
+import subprocess
+import sys
+import webbrowser
+from os.path import isfile
+from shutil import which
+from typing import Sequence, Union, Optional
 
+import Utils
+from worlds.LauncherComponents import Component, components, Type, SuffixIdentifier, icon_paths
 
-is_linux = sys.platform.startswith('linux')
-is_macos = sys.platform == 'darwin'
-is_windows = sys.platform in ("win32", "cygwin", "msys")
+if __name__ == "__main__":
+    import ModuleUpdate
+    ModuleUpdate.update()
+
+from Utils import is_frozen, user_path, local_path, init_logging, open_filename, messagebox, \
+    is_windows, is_macos, is_linux
 
 
 def open_host_yaml():
@@ -37,132 +41,57 @@ def open_host_yaml():
         exe = which("open")
         subprocess.Popen([exe, file])
     else:
-        import webbrowser
         webbrowser.open(file)
 
 
 def open_patch():
+    suffixes = []
+    for c in components:
+        if isfile(get_exe(c)[-1]):
+            suffixes += c.file_identifier.suffixes if c.type == Type.CLIENT and \
+                                                      isinstance(c.file_identifier, SuffixIdentifier) else []
     try:
-        import tkinter
-        import tkinter.filedialog
+        filename = open_filename('Select patch', (('Patches', suffixes),))
     except Exception as e:
-        logging.error("Could not load tkinter, which is likely not installed. "
-                      "This attempt was made because Launcher.open_patch was used.")
-        raise e
+        messagebox('Error', str(e), error=True)
     else:
-        root = tkinter.Tk()
-        root.withdraw()
-        suffixes = []
-        for c in components:
-            if isfile(get_exe(c)[-1]):
-                suffixes += c.file_identifier.suffixes if c.type == Type.CLIENT and \
-                                                          isinstance(c.file_identifier, SuffixIdentifier) else []
-        filename = tkinter.filedialog.askopenfilename(filetypes=(('Patches', ' '.join(suffixes)),))
         file, _, component = identify(filename)
         if file and component:
             launch([*get_exe(component), file], component.cli)
 
 
+def generate_yamls():
+    from Options import generate_yaml_templates
+
+    target = Utils.user_path("Players", "Templates")
+    generate_yaml_templates(target, False)
+    open_folder(target)
+
+
 def browse_files():
-    file = user_path()
+    open_folder(user_path())
+
+
+def open_folder(folder_path):
     if is_linux:
         exe = which('xdg-open') or which('gnome-open') or which('kde-open')
-        subprocess.Popen([exe, file])
+        subprocess.Popen([exe, folder_path])
     elif is_macos:
         exe = which("open")
-        subprocess.Popen([exe, file])
+        subprocess.Popen([exe, folder_path])
     else:
-        import webbrowser
-        webbrowser.open(file)
+        webbrowser.open(folder_path)
 
 
-class Type(Enum):
-    TOOL = auto()
-    FUNC = auto()  # not a real component
-    CLIENT = auto()
-    ADJUSTER = auto()
-
-
-class SuffixIdentifier:
-    suffixes: Iterable[str]
-
-    def __init__(self, *args: str):
-        self.suffixes = args
-
-    def __call__(self, path: str):
-        if isinstance(path, str):
-            for suffix in self.suffixes:
-                if path.endswith(suffix):
-                    return True
-        return False
-
-
-class Component:
-    display_name: str
-    type: Optional[Type]
-    script_name: Optional[str]
-    frozen_name: Optional[str]
-    icon: str  # just the name, no suffix
-    cli: bool
-    func: Optional[Callable]
-    file_identifier: Optional[Callable[[str], bool]]
-
-    def __init__(self, display_name: str, script_name: Optional[str] = None, frozen_name: Optional[str] = None,
-                 cli: bool = False, icon: str = 'icon', component_type: Type = None, func: Optional[Callable] = None,
-                 file_identifier: Optional[Callable[[str], bool]] = None):
-        self.display_name = display_name
-        self.script_name = script_name
-        self.frozen_name = frozen_name or f'Archipelago{script_name}' if script_name else None
-        self.icon = icon
-        self.cli = cli
-        self.type = component_type or \
-            None if not display_name else \
-            Type.FUNC if func else \
-            Type.CLIENT if 'Client' in display_name else \
-            Type.ADJUSTER if 'Adjuster' in display_name else Type.TOOL
-        self.func = func
-        self.file_identifier = file_identifier
-
-    def handles_file(self, path: str):
-        return self.file_identifier(path) if self.file_identifier else False
-
-
-components: Iterable[Component] = (
-    # Launcher
-    Component('', 'Launcher'),
-    # Core
-    Component('Host', 'MultiServer', 'ArchipelagoServer', cli=True,
-              file_identifier=SuffixIdentifier('.archipelago', '.zip')),
-    Component('Generate', 'Generate', cli=True),
-    Component('Text Client', 'CommonClient', 'ArchipelagoTextClient'),
-    # SNI
-    Component('SNI Client', 'SNIClient',
-              file_identifier=SuffixIdentifier('.apz3', '.apm3', '.apsoe', '.aplttp', '.apsm', '.apsmz3')),
-    Component('LttP Adjuster', 'LttPAdjuster'),
-    # Factorio
-    Component('Factorio Client', 'FactorioClient'),
-    # Minecraft
-    Component('Minecraft Client', 'MinecraftClient', icon='mcicon', cli=True,
-              file_identifier=SuffixIdentifier('.apmc')),
-    # Ocarina of Time
-    Component('OoT Client', 'OoTClient',
-              file_identifier=SuffixIdentifier('.apz5')),
-    Component('OoT Adjuster', 'OoTAdjuster'),
-    # FF1
-    Component('FF1 Client', 'FF1Client'),
-    # ChecksFinder
-    Component('ChecksFinder Client', 'ChecksFinderClient'),
-    # Starcraft 2
-    Component('Starcraft 2 Client', 'Starcraft2Client'),
+components.extend([
     # Functions
-    Component('Open host.yaml', func=open_host_yaml),
-    Component('Open Patch', func=open_patch),
-    Component('Browse Files', func=browse_files),
-)
-icon_paths = {
-    'icon': local_path('data', 'icon.ico' if is_windows else 'icon.png'),
-    'mcicon': local_path('data', 'mcicon.ico')
-}
+    Component("Open host.yaml", func=open_host_yaml),
+    Component("Open Patch", func=open_patch),
+    Component("Generate Template Settings", func=generate_yamls),
+    Component("Discord Server", icon="discord", func=lambda: webbrowser.open("https://discord.gg/8Z65BR2")),
+    Component("18+ Discord Server", icon="discord", func=lambda: webbrowser.open("https://discord.gg/fqvNCCRsu4")),
+    Component("Browse Files", func=browse_files),
+])
 
 
 def identify(path: Union[None, str]):
@@ -217,14 +146,9 @@ def launch(exe, in_terminal=False):
 
 
 def run_gui():
-    if not sys.stdout:
-        from kvui import App, ContainerLayout, GridLayout, Button, Label  # this kills stdout
-    else:
-        from kivy.app import App
-        from kivy.uix.button import Button
-        from kivy.uix.floatlayout import FloatLayout as ContainerLayout
-        from kivy.uix.gridlayout import GridLayout
-        from kivy.uix.label import Label
+    from kvui import App, ContainerLayout, GridLayout, Button, Label
+    from kivy.uix.image import AsyncImage
+    from kivy.uix.relativelayout import RelativeLayout
 
     class Launcher(App):
         base_title: str = "Archipelago Launcher"
@@ -246,24 +170,44 @@ def run_gui():
             self.container = ContainerLayout()
             self.grid = GridLayout(cols=2)
             self.container.add_widget(self.grid)
-
+            self.grid.add_widget(Label(text="General"))
+            self.grid.add_widget(Label(text="Clients"))
             button_layout = self.grid  # make buttons fill the window
+
+            def build_button(component: Component):
+                """
+                Builds a button widget for a given component.
+
+                Args:
+                    component (Component): The component associated with the button.
+
+                Returns:
+                    None. The button is added to the parent grid layout.
+
+                """
+                button = Button(text=component.display_name)
+                button.component = component
+                button.bind(on_release=self.component_action)
+                if component.icon != "icon":
+                    image = AsyncImage(source=icon_paths[component.icon],
+                                       size=(38, 38), size_hint=(None, 1), pos=(5, 0))
+                    box_layout = RelativeLayout()
+                    box_layout.add_widget(button)
+                    box_layout.add_widget(image)
+                    button_layout.add_widget(box_layout)
+                else:
+                    button_layout.add_widget(button)
+
             for (tool, client) in itertools.zip_longest(itertools.chain(
                     self._tools.items(), self._funcs.items(), self._adjusters.items()), self._clients.items()):
                 # column 1
                 if tool:
-                    button = Button(text=tool[0])
-                    button.component = tool[1]
-                    button.bind(on_release=self.component_action)
-                    button_layout.add_widget(button)
+                    build_button(tool[1])
                 else:
                     button_layout.add_widget(Label())
                 # column 2
                 if client:
-                    button = Button(text=client[0])
-                    button.component = client[1]
-                    button.bind(on_press=self.component_action)
-                    button_layout.add_widget(button)
+                    build_button(client[1])
                 else:
                     button_layout.add_widget(Label())
 
@@ -302,6 +246,7 @@ def main(args: Optional[Union[argparse.Namespace, dict]] = None):
 
 if __name__ == '__main__':
     init_logging('Launcher')
+    multiprocessing.freeze_support()
     parser = argparse.ArgumentParser(description='Archipelago Launcher')
     parser.add_argument('Patch|Game|Component', type=str, nargs='?',
                         help="Pass either a patch file, a generated game or the name of a component to run.")

LinksAwakeningClient.py

@@ -0,0 +1,609 @@
+import ModuleUpdate
+ModuleUpdate.update()
+
+import Utils
+
+if __name__ == "__main__":
+    Utils.init_logging("LinksAwakeningContext", exception_logger="Client")
+
+import asyncio
+import base64
+import binascii
+import io
+import logging
+import select
+import socket
+import time
+import typing
+import urllib
+
+import colorama
+
+
+from CommonClient import (CommonContext, get_base_parser, gui_enabled, logger,
+                          server_loop)
+from NetUtils import ClientStatus
+from worlds.ladx.Common import BASE_ID as LABaseID
+from worlds.ladx.GpsTracker import GpsTracker
+from worlds.ladx.ItemTracker import ItemTracker
+from worlds.ladx.LADXR.checkMetadata import checkMetadataTable
+from worlds.ladx.Locations import get_locations_to_id, meta_to_name
+from worlds.ladx.Tracker import LocationTracker, MagpieBridge
+
+class GameboyException(Exception):
+    pass
+
+
+class RetroArchDisconnectError(GameboyException):
+    pass
+
+
+class InvalidEmulatorStateError(GameboyException):
+    pass
+
+
+class BadRetroArchResponse(GameboyException):
+    pass
+
+
+def magpie_logo():
+    from kivy.uix.image import CoreImage
+    binary_data = """
+iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAAAXN
+SR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA
+7DAcdvqGQAAADGSURBVDhPhVLBEcIwDHOYhjHCBuXHj2OTbAL8+
+MEGZIxOQ1CinOOk0Op0bmo7tlXXeR9FJMYDLOD9mwcLjQK7+hSZ
+wgcWMZJOAGeGKtChNHFL0j+FZD3jSCuo0w7l03wDrWdg00C4/aW
+eDEYNenuzPOfPspBnxf0kssE80vN0L8361j10P03DK4x6FHabuV
+ear8fHme+b17rwSjbAXeUMLb+EVTV2QHm46MWQanmnydA98KsVS
+XkV+qFpGQXrLhT/fqraQeQLuplpNH5g+WkAAAAASUVORK5CYII="""
+    binary_data = base64.b64decode(binary_data)
+    data = io.BytesIO(binary_data)
+    return CoreImage(data, ext="png").texture
+
+
+class LAClientConstants:
+    # Connector version
+    VERSION = 0x01
+    #
+    # Memory locations of LADXR
+    ROMGameID = 0x0051  # 4 bytes
+    SlotName = 0x0134
+    # Unused
+    # ROMWorldID = 0x0055
+    # ROMConnectorVersion = 0x0056
+    # RO: We should only act if this is higher then 6, as it indicates that the game is running normally
+    wGameplayType = 0xDB95
+    # RO: Starts at 0, increases every time an item is received from the server and processed
+    wLinkSyncSequenceNumber = 0xDDF6
+    wLinkStatusBits = 0xDDF7          # RW:
+    #      Bit0: wLinkGive* contains valid data, set from script cleared from ROM.
+    wLinkHealth = 0xDB5A
+    wLinkGiveItem = 0xDDF8  # RW
+    wLinkGiveItemFrom = 0xDDF9  # RW
+    # All of these six bytes are unused, we can repurpose
+    # wLinkSendItemRoomHigh = 0xDDFA  # RO
+    # wLinkSendItemRoomLow = 0xDDFB  # RO
+    # wLinkSendItemTarget = 0xDDFC  # RO
+    # wLinkSendItemItem = 0xDDFD  # RO
+    # wLinkSendShopItem = 0xDDFE # RO, which item to send (1 based, order of the shop items)
+    # RO, which player to send to, but it's just the X position of the NPC used, so 0x18 is player 0
+    # wLinkSendShopTarget = 0xDDFF
+
+
+    wRecvIndex = 0xDDFE  # 0xDB58
+    wCheckAddress = 0xC0FF - 0x4
+    WRamCheckSize = 0x4
+    WRamSafetyValue = bytearray([0]*WRamCheckSize)
+
+    MinGameplayValue = 0x06
+    MaxGameplayValue = 0x1A
+    VictoryGameplayAndSub = 0x0102
+
+
+class RAGameboy():
+    cache = []
+    cache_start = 0
+    cache_size = 0
+    last_cache_read = None
+    socket = None
+
+    def __init__(self, address, port) -> None:
+        self.address = address
+        self.port = port
+        self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        assert (self.socket)
+        self.socket.setblocking(False)
+
+    def get_retroarch_version(self):
+        self.send(b'VERSION\n')
+        select.select([self.socket], [], [])
+        response_str, addr = self.socket.recvfrom(16)
+        return response_str.rstrip()
+
+    def get_retroarch_status(self, timeout):
+        self.send(b'GET_STATUS\n')
+        select.select([self.socket], [], [], timeout)
+        response_str, addr = self.socket.recvfrom(1000, )
+        return response_str.rstrip()
+
+    def set_cache_limits(self, cache_start, cache_size):
+        self.cache_start = cache_start
+        self.cache_size = cache_size
+
+    def send(self, b):
+        if type(b) is str:
+            b = b.encode('ascii')
+        self.socket.sendto(b, (self.address, self.port))
+
+    def recv(self):
+        select.select([self.socket], [], [])
+        response, _ = self.socket.recvfrom(4096)
+        return response
+
+    async def async_recv(self):
+        response = await asyncio.get_event_loop().sock_recv(self.socket, 4096)
+        return response
+
+    async def check_safe_gameplay(self, throw=True):
+        async def check_wram():
+            check_values = await self.async_read_memory(LAClientConstants.wCheckAddress, LAClientConstants.WRamCheckSize)
+
+            if check_values != LAClientConstants.WRamSafetyValue:
+                if throw:
+                    raise InvalidEmulatorStateError()
+                return False
+            return True
+
+        if not await check_wram():
+            if throw:
+                raise InvalidEmulatorStateError()
+            return False
+
+        gameplay_value = await self.async_read_memory(LAClientConstants.wGameplayType)
+        gameplay_value = gameplay_value[0]
+        # In gameplay or credits
+        if not (LAClientConstants.MinGameplayValue <= gameplay_value <= LAClientConstants.MaxGameplayValue) and gameplay_value != 0x1:
+            if throw:
+                logger.info("invalid emu state")
+                raise InvalidEmulatorStateError()
+            return False
+        if not await check_wram():
+            return False
+        return True
+
+    # We're sadly unable to update the whole cache at once
+    # as RetroArch only gives back some number of bytes at a time
+    # So instead read as big as chunks at a time as we can manage
+    async def update_cache(self):
+        # First read the safety address - if it's invalid, bail
+        self.cache = []
+
+        if not await self.check_safe_gameplay():
+            return
+
+        cache = []
+        remaining_size = self.cache_size
+        while remaining_size:
+            block = await self.async_read_memory(self.cache_start + len(cache), remaining_size)
+            remaining_size -= len(block)
+            cache += block
+
+        if not await self.check_safe_gameplay():
+            return
+
+        self.cache = cache
+        self.last_cache_read = time.time()
+
+    async def read_memory_cache(self, addresses):
+        # TODO: can we just update once per frame?
+        if not self.last_cache_read or self.last_cache_read + 0.1 < time.time():
+            await self.update_cache()
+        if not self.cache:
+            return None
+        assert (len(self.cache) == self.cache_size)
+        for address in addresses:
+            assert self.cache_start <= address <= self.cache_start + self.cache_size
+        r = {address: self.cache[address - self.cache_start]
+             for address in addresses}
+        return r
+
+    async def async_read_memory_safe(self, address, size=1):
+        # whenever we do a read for a check, we need to make sure that we aren't reading
+        # garbage memory values - we also need to protect against reading a value, then the emulator resetting
+        #
+        # ...actually, we probably _only_ need the post check
+
+        # Check before read
+        if not await self.check_safe_gameplay():
+            return None
+
+        # Do read
+        r = await self.async_read_memory(address, size)
+
+        # Check after read
+        if not await self.check_safe_gameplay():
+            return None
+
+        return r
+
+    def read_memory(self, address, size=1):
+        command = "READ_CORE_MEMORY"
+
+        self.send(f'{command} {hex(address)} {size}\n')
+        response = self.recv()
+
+        splits = response.decode().split(" ", 2)
+
+        assert (splits[0] == command)
+        # Ignore the address for now
+
+        # TODO: transform to bytes
+        if splits[2][:2] == "-1" or splits[0] != "READ_CORE_MEMORY":
+            raise BadRetroArchResponse()
+        return bytearray.fromhex(splits[2])
+
+    async def async_read_memory(self, address, size=1):
+        command = "READ_CORE_MEMORY"
+
+        self.send(f'{command} {hex(address)} {size}\n')
+        response = await self.async_recv()
+        response = response[:-1]
+        splits = response.decode().split(" ", 2)
+
+        assert (splits[0] == command)
+        # Ignore the address for now
+
+        # TODO: transform to bytes
+        return bytearray.fromhex(splits[2])
+
+    def write_memory(self, address, bytes):
+        command = "WRITE_CORE_MEMORY"
+
+        self.send(f'{command} {hex(address)} {" ".join(hex(b) for b in bytes)}')
+        select.select([self.socket], [], [])
+        response, _ = self.socket.recvfrom(4096)
+
+        splits = response.decode().split(" ", 3)
+
+        assert (splits[0] == command)
+
+        if splits[2] == "-1":
+            logger.info(splits[3])
+
+
+class LinksAwakeningClient():
+    socket = None
+    gameboy = None
+    tracker = None
+    auth = None
+    game_crc = None
+    pending_deathlink = False
+    deathlink_debounce = True
+    recvd_checks = {}
+
+    def msg(self, m):
+        logger.info(m)
+        s = f"SHOW_MSG {m}\n"
+        self.gameboy.send(s)
+
+    def __init__(self, retroarch_address="127.0.0.1", retroarch_port=55355):
+        self.gameboy = RAGameboy(retroarch_address, retroarch_port)
+
+    async def wait_for_retroarch_connection(self):
+        logger.info("Waiting on connection to Retroarch...")
+        while True:
+            try:
+                version = self.gameboy.get_retroarch_version()
+                NO_CONTENT = b"GET_STATUS CONTENTLESS"
+                status = NO_CONTENT
+                core_type = None
+                GAME_BOY = b"game_boy"
+                while status == NO_CONTENT or core_type != GAME_BOY:
+                    try:
+                        status = self.gameboy.get_retroarch_status(0.1)
+                        if status.count(b" ") < 2:
+                            await asyncio.sleep(1.0)
+                            continue
+                        
+                        GET_STATUS, PLAYING, info = status.split(b" ", 2)
+                        if status.count(b",") < 2:
+                            await asyncio.sleep(1.0)
+                            continue
+                        core_type, rom_name, self.game_crc = info.split(b",", 2)
+                        if core_type != GAME_BOY:
+                            logger.info(
+                                f"Core type should be '{GAME_BOY}', found {core_type} instead - wrong type of ROM?")
+                            await asyncio.sleep(1.0)
+                            continue
+                    except (BlockingIOError, TimeoutError) as e:
+                        await asyncio.sleep(0.1)
+                        pass
+                logger.info(f"Connected to Retroarch {version} {info}")
+                self.gameboy.read_memory(0x1000)
+                return
+            except ConnectionResetError:
+                await asyncio.sleep(1.0)
+                pass
+
+    def reset_auth(self):
+        auth = binascii.hexlify(self.gameboy.read_memory(0x0134, 12)).decode()
+
+        if self.auth:
+            assert (auth == self.auth)
+
+        self.auth = auth
+
+    async def wait_and_init_tracker(self):
+        await self.wait_for_game_ready()
+        self.tracker = LocationTracker(self.gameboy)
+        self.item_tracker = ItemTracker(self.gameboy)
+        self.gps_tracker = GpsTracker(self.gameboy)
+
+    async def recved_item_from_ap(self, item_id, from_player, next_index):
+        # Don't allow getting an item until you've got your first check
+        if not self.tracker.has_start_item():
+            return
+
+        # Spin until we either:
+        # get an exception from a bad read (emu shut down or reset)
+        # beat the game
+        # the client handles the last pending item
+        status = (await self.gameboy.async_read_memory_safe(LAClientConstants.wLinkStatusBits))[0]
+        while not (await self.is_victory()) and status & 1 == 1:
+            time.sleep(0.1)
+            status = (await self.gameboy.async_read_memory_safe(LAClientConstants.wLinkStatusBits))[0]
+
+        item_id -= LABaseID
+        # The player name table only goes up to 100, so don't go past that
+        # Even if it didn't, the remote player _index_ byte is just a byte, so 255 max
+        if from_player > 100:
+            from_player = 100
+
+        next_index += 1
+        self.gameboy.write_memory(LAClientConstants.wLinkGiveItem, [
+                                  item_id, from_player])
+        status |= 1
+        status = self.gameboy.write_memory(LAClientConstants.wLinkStatusBits, [status])
+        self.gameboy.write_memory(LAClientConstants.wRecvIndex, [next_index])
+
+    async def wait_for_game_ready(self):
+        logger.info("Waiting on game to be in valid state...")
+        while not await self.gameboy.check_safe_gameplay(throw=False):
+            pass
+        logger.info("Ready!")
+    last_index = 0
+
+    async def is_victory(self):
+        return (await self.gameboy.read_memory_cache([LAClientConstants.wGameplayType]))[LAClientConstants.wGameplayType] == 1
+
+    async def main_tick(self, item_get_cb, win_cb, deathlink_cb):
+        await self.tracker.readChecks(item_get_cb)
+        await self.item_tracker.readItems()
+        await self.gps_tracker.read_location()
+
+        next_index = self.gameboy.read_memory(LAClientConstants.wRecvIndex)[0]
+        if next_index != self.last_index:
+            self.last_index = next_index
+            # logger.info(f"Got new index {next_index}")
+
+        current_health = (await self.gameboy.read_memory_cache([LAClientConstants.wLinkHealth]))[LAClientConstants.wLinkHealth]
+        if self.deathlink_debounce and current_health != 0:
+            self.deathlink_debounce = False
+        elif not self.deathlink_debounce and current_health == 0:
+            # logger.info("YOU DIED.")
+            await deathlink_cb()
+            self.deathlink_debounce = True
+
+        if self.pending_deathlink:
+            logger.info("Got a deathlink")
+            self.gameboy.write_memory(LAClientConstants.wLinkHealth, [0])
+            self.pending_deathlink = False
+            self.deathlink_debounce = True
+
+        if await self.is_victory():
+            await win_cb()
+
+        recv_index = (await self.gameboy.async_read_memory_safe(LAClientConstants.wRecvIndex))[0]
+
+        # Play back one at a time
+        if recv_index in self.recvd_checks:
+            item = self.recvd_checks[recv_index]
+            await self.recved_item_from_ap(item.item, item.player, recv_index)
+
+
+all_tasks = set()
+
+def create_task_log_exception(awaitable) -> asyncio.Task:
+    async def _log_exception(awaitable):
+        try:
+            return await awaitable
+        except Exception as e:
+            logger.exception(e)
+            pass
+        finally:
+            all_tasks.remove(task)
+    task = asyncio.create_task(_log_exception(awaitable))
+    all_tasks.add(task)
+
+
+class LinksAwakeningContext(CommonContext):
+    tags = {"AP"}
+    game = "Links Awakening DX"
+    items_handling = 0b101
+    want_slot_data = True
+    la_task = None
+    client = None
+    # TODO: does this need to re-read on reset?
+    found_checks = []
+    last_resend = time.time()
+
+    magpie = MagpieBridge()
+    magpie_task = None
+    won = False
+
+    def __init__(self, server_address: typing.Optional[str], password: typing.Optional[str]) -> None:
+        self.client = LinksAwakeningClient()
+        super().__init__(server_address, password)
+
+    def run_gui(self) -> None:
+        import webbrowser
+        import kvui
+        from kvui import Button, GameManager
+        from kivy.uix.image import Image
+
+        class LADXManager(GameManager):
+            logging_pairs = [
+                ("Client", "Archipelago"),
+                ("Tracker", "Tracker"),
+            ]
+            base_title = "Archipelago Links Awakening DX Client"
+
+            def build(self):
+                b = super().build()
+
+                button = Button(text="", size=(30, 30), size_hint_x=None,
+                                on_press=lambda _: webbrowser.open('https://magpietracker.us/?enable_autotracker=1'))
+                image = Image(size=(16, 16), texture=magpie_logo())
+                button.add_widget(image)
+
+                def set_center(_, center):
+                    image.center = center
+                button.bind(center=set_center)
+
+                self.connect_layout.add_widget(button)
+                return b
+
+        self.ui = LADXManager(self)
+        self.ui_task = asyncio.create_task(self.ui.async_run(), name="UI")
+
+    async def send_checks(self):
+        message = [{"cmd": 'LocationChecks', "locations": self.found_checks}]
+        await self.send_msgs(message)
+
+    ENABLE_DEATHLINK = False
+    async def send_deathlink(self):
+        if self.ENABLE_DEATHLINK:
+            message = [{"cmd": 'Deathlink',
+                        'time': time.time(),
+                        'cause': 'Had a nightmare',
+                        # 'source': self.slot_info[self.slot].name,
+                        }]
+            await self.send_msgs(message)
+
+    async def send_victory(self):
+        if not self.won:
+            message = [{"cmd": "StatusUpdate",
+                        "status": ClientStatus.CLIENT_GOAL}]
+            logger.info("victory!")
+            await self.send_msgs(message)
+            self.won = True
+
+    async def on_deathlink(self, data: typing.Dict[str, typing.Any]) -> None:
+        if self.ENABLE_DEATHLINK:
+            self.client.pending_deathlink = True
+
+    def new_checks(self, item_ids, ladxr_ids):
+        self.found_checks += item_ids
+        create_task_log_exception(self.send_checks())
+        create_task_log_exception(self.magpie.send_new_checks(ladxr_ids))
+
+    async def server_auth(self, password_requested: bool = False):
+        if password_requested and not self.password:
+            await super(LinksAwakeningContext, self).server_auth(password_requested)
+        self.auth = self.client.auth
+        await self.get_username()
+        await self.send_connect()
+
+    def on_package(self, cmd: str, args: dict):
+        if cmd == "Connected":
+            self.game = self.slot_info[self.slot].game
+        # TODO - use watcher_event
+        if cmd == "ReceivedItems":
+            for index, item in enumerate(args["items"], args["index"]):
+                self.client.recvd_checks[index] = item
+
+    item_id_lookup = get_locations_to_id()
+
+    async def run_game_loop(self):
+        def on_item_get(ladxr_checks):
+            checks = [self.item_id_lookup[meta_to_name(
+                checkMetadataTable[check.id])] for check in ladxr_checks]
+            self.new_checks(checks, [check.id for check in ladxr_checks])
+
+        async def victory():
+            await self.send_victory()
+
+        async def deathlink():
+            await self.send_deathlink()
+
+        self.magpie_task = asyncio.create_task(self.magpie.serve())
+        
+        # yield to allow UI to start
+        await asyncio.sleep(0)
+
+        while True:
+            try:
+                # TODO: cancel all client tasks
+                logger.info("(Re)Starting game loop")
+                self.found_checks.clear()
+                await self.client.wait_for_retroarch_connection()
+                self.client.reset_auth()
+                await self.client.wait_and_init_tracker()
+
+                while True:
+                    await self.client.main_tick(on_item_get, victory, deathlink)
+                    await asyncio.sleep(0.1)
+                    now = time.time()
+                    if self.last_resend + 5.0 < now:
+                        self.last_resend = now
+                        await self.send_checks()
+                    self.magpie.set_checks(self.client.tracker.all_checks)
+                    await self.magpie.set_item_tracker(self.client.item_tracker)
+                    await self.magpie.send_gps(self.client.gps_tracker)
+
+            except GameboyException:
+                time.sleep(1.0)
+                pass
+
+
+async def main():
+    parser = get_base_parser(description="Link's Awakening Client.")
+    parser.add_argument("--url", help="Archipelago connection url")
+
+    parser.add_argument('diff_file', default="", type=str, nargs="?",
+                        help='Path to a .apladx Archipelago Binary Patch file')
+    args = parser.parse_args()
+    logger.info(args)
+
+    if args.diff_file:
+        import Patch
+        logger.info("patch file was supplied - creating rom...")
+        meta, rom_file = Patch.create_rom_file(args.diff_file)
+        if "server" in meta:
+            args.url = meta["server"]
+        logger.info(f"wrote rom file to {rom_file}")
+
+    if args.url:
+        url = urllib.parse.urlparse(args.url)
+        args.connect = url.netloc
+        if url.password:
+            args.password = urllib.parse.unquote(url.password)
+
+    ctx = LinksAwakeningContext(args.connect, args.password)
+
+    ctx.server_task = asyncio.create_task(server_loop(ctx), name="server loop")
+
+    # TODO: nothing about the lambda about has to be in a lambda
+    ctx.la_task = create_task_log_exception(ctx.run_game_loop())
+    if gui_enabled:
+        ctx.run_gui()
+    ctx.run_cli()
+
+    await ctx.exit_event.wait()
+    await ctx.shutdown()
+
+if __name__ == '__main__':
+    colorama.init()
+    asyncio.run(main())
+    colorama.deinit()

LttPAdjuster.py

@@ -26,14 +26,16 @@ ModuleUpdate.update()
 from worlds.alttp.Rom import Sprite, LocalRom, apply_rom_settings, get_base_rom_bytes
 from Utils import output_path, local_path, user_path, open_file, get_cert_none_ssl_context, persistent_store, \
     get_adjuster_settings, tkinter_center_window, init_logging
-from Patch import GAME_ALTTP
+
+
+GAME_ALTTP = "A Link to the Past"
 
 
 class AdjusterWorld(object):
     def __init__(self, sprite_pool):
         import random
         self.sprite_pool = {1: sprite_pool}
-        self.slot_seeds = {1: random}
+        self.per_slot_randoms = {1: random}
 
 
 class ArgumentDefaultsHelpFormatter(argparse.RawTextHelpFormatter):
@@ -47,7 +49,7 @@ def main():
 
     parser.add_argument('rom', nargs="?", default='AP_LttP.sfc', help='Path to an ALttP rom to adjust.')
     parser.add_argument('--baserom', default='Zelda no Densetsu - Kamigami no Triforce (Japan).sfc',
-                        help='Path to an ALttP JAP(1.0) rom to use as a base.')
+                        help='Path to an ALttP Japan(1.0) rom to use as a base.')
     parser.add_argument('--loglevel', default='info', const='info', nargs='?',
                         choices=['error', 'info', 'warning', 'debug'], help='Select level of logging for output.')
     parser.add_argument('--menuspeed', default='normal', const='normal', nargs='?',
@@ -83,9 +85,9 @@ def main():
     parser.add_argument('--ow_palettes', default='default',
                         choices=['default', 'random', 'blackout', 'puke', 'classic', 'grayscale', 'negative', 'dizzy',
                                  'sick'])
-    parser.add_argument('--link_palettes', default='default',
-                        choices=['default', 'random', 'blackout', 'puke', 'classic', 'grayscale', 'negative', 'dizzy',
-                                 'sick'])
+    # parser.add_argument('--link_palettes', default='default',
+    #                     choices=['default', 'random', 'blackout', 'puke', 'classic', 'grayscale', 'negative', 'dizzy',
+    #                              'sick'])
     parser.add_argument('--shield_palettes', default='default',
                         choices=['default', 'random', 'blackout', 'puke', 'classic', 'grayscale', 'negative', 'dizzy',
                                  'sick'])
@@ -105,6 +107,12 @@ def main():
                              Alternatively, can be a ALttP Rom patched with a Link
                              sprite that will be extracted.
                              ''')
+    parser.add_argument('--oof', help='''\
+                             Path to a sound effect to replace Link's "oof" sound.
+                             Needs to be in a .brr format and have a length of no
+                             more than 2673 bytes, created from a 16-bit signed PCM
+                             .wav at 12khz. https://github.com/boldowa/snesbrr
+                             ''')
     parser.add_argument('--names', default='', type=str)
     parser.add_argument('--update_sprites', action='store_true', help='Update Sprite Database, then exit.')
     args = parser.parse_args()
@@ -124,6 +132,13 @@ def main():
         if args.sprite is not None and not os.path.isfile(args.sprite) and not Sprite.get_sprite_from_name(args.sprite):
             input('Could not find link sprite sheet at given location. \nPress Enter to exit.')
             sys.exit(1)
+        if args.oof is not None and not os.path.isfile(args.oof):
+            input('Could not find oof sound effect at given location. \nPress Enter to exit.')
+            sys.exit(1)
+        if args.oof is not None and os.path.getsize(args.oof) > 2673:
+            input('"oof" sound effect cannot exceed 2673 bytes. \nPress Enter to exit.')
+            sys.exit(1)
+            
 
         args, path = adjust(args=args)
         if isinstance(args.sprite, Sprite):
@@ -139,7 +154,7 @@ def adjust(args):
     vanillaRom = args.baserom
     if not os.path.exists(vanillaRom) and not os.path.isabs(vanillaRom):
         vanillaRom = local_path(vanillaRom)
-    if os.path.splitext(args.rom)[-1].lower() in {'.apbp', '.aplttp'}:
+    if os.path.splitext(args.rom)[-1].lower() == '.aplttp':
         import Patch
         meta, args.rom = Patch.create_rom_file(args.rom)
 
@@ -163,7 +178,7 @@ def adjust(args):
         world = getattr(args, "world")
 
     apply_rom_settings(rom, args.heartbeep, args.heartcolor, args.quickswap, args.menuspeed, args.music,
-                       args.sprite, palettes_options, reduceflashing=args.reduceflashing or racerom, world=world,
+                       args.sprite, args.oof, palettes_options, reduceflashing=args.reduceflashing or racerom, world=world,
                        deathlink=args.deathlink, allowcollect=args.allowcollect)
     path = output_path(f'{os.path.basename(args.rom)[:-4]}_adjusted.sfc')
     rom.write_to_file(path)
@@ -195,7 +210,7 @@ def adjustGUI():
     romEntry2 = Entry(romDialogFrame, textvariable=romVar2)
 
     def RomSelect2():
-        rom = filedialog.askopenfilename(filetypes=[("Rom Files", (".sfc", ".smc", ".apbp")), ("All Files", "*")])
+        rom = filedialog.askopenfilename(filetypes=[("Rom Files", (".sfc", ".smc", ".aplttp")), ("All Files", "*")])
         romVar2.set(rom)
 
     romSelectButton2 = Button(romDialogFrame, text='Select Rom', command=RomSelect2)
@@ -225,6 +240,7 @@ def adjustGUI():
         guiargs.sprite = rom_vars.sprite
         if rom_vars.sprite_pool:
             guiargs.world = AdjusterWorld(rom_vars.sprite_pool)
+        guiargs.oof = rom_vars.oof
 
         try:
             guiargs, path = adjust(args=guiargs)
@@ -263,6 +279,7 @@ def adjustGUI():
         else:
             guiargs.sprite = rom_vars.sprite
         guiargs.sprite_pool = rom_vars.sprite_pool
+        guiargs.oof = rom_vars.oof
         persistent_store("adjuster", GAME_ALTTP, guiargs)
         messagebox.showinfo(title="Success", message="Settings saved to persistent storage")
 
@@ -289,7 +306,7 @@ def run_sprite_update():
     else:
         top.withdraw()
         task = BackgroundTaskProgress(top, update_sprites, "Updating Sprites", lambda succesful, resultmessage: done.set())
-    while not done.isSet():
+    while not done.is_set():
         task.do_events()
     logging.info("Done updating sprites")
 
@@ -300,6 +317,7 @@ def update_sprites(task, on_finish=None):
     sprite_dir = user_path("data", "sprites", "alttpr")
     os.makedirs(sprite_dir, exist_ok=True)
     ctx = get_cert_none_ssl_context()
+
     def finished():
         task.close_window()
         if on_finish:
@@ -478,6 +496,36 @@ class BackgroundTaskProgressNullWindow(BackgroundTask):
         self.stop()
 
 
+class AttachTooltip(object):
+
+    def __init__(self, parent, text):
+        self._parent = parent
+        self._text = text
+        self._window = None
+        parent.bind('<Enter>', lambda event : self.show())
+        parent.bind('<Leave>', lambda event : self.hide())
+
+    def show(self):
+        if self._window or not self._text:
+            return
+        self._window = Toplevel(self._parent)
+        #remove window bar controls
+        self._window.wm_overrideredirect(1)
+        #adjust positioning
+        x, y, *_ = self._parent.bbox("insert")
+        x = x + self._parent.winfo_rootx() + 20
+        y = y + self._parent.winfo_rooty() + 20
+        self._window.wm_geometry("+{0}+{1}".format(x,y))
+        #show text
+        label = Label(self._window, text=self._text, justify=LEFT)
+        label.pack(ipadx=1)
+
+    def hide(self):
+        if self._window:
+            self._window.destroy()
+            self._window = None
+
+
 def get_rom_frame(parent=None):
     adjuster_settings = get_adjuster_settings(GAME_ALTTP)
     if not adjuster_settings:
@@ -519,6 +567,7 @@ def get_rom_options_frame(parent=None):
         "reduceflashing": True,
         "deathlink": False,
         "sprite": None,
+        "oof": None,
         "quickswap": True,
         "menuspeed": 'normal',
         "heartcolor": 'red',
@@ -595,12 +644,50 @@ def get_rom_options_frame(parent=None):
     spriteEntry.pack(side=LEFT)
     spriteSelectButton.pack(side=LEFT)
 
+    oofDialogFrame = Frame(romOptionsFrame)
+    oofDialogFrame.grid(row=1, column=1)
+    baseOofLabel = Label(oofDialogFrame, text='"OOF" Sound:')
+
+    vars.oofNameVar = StringVar()
+    vars.oof = adjuster_settings.oof
+
+    def set_oof(oof_param):
+        nonlocal vars
+        if isinstance(oof_param, str) and os.path.isfile(oof_param) and os.path.getsize(oof_param) <= 2673:
+            vars.oof = oof_param
+            vars.oofNameVar.set(oof_param.rsplit('/',1)[-1])
+        else:
+            vars.oof = None
+            vars.oofNameVar.set('(unchanged)')
+
+    set_oof(adjuster_settings.oof)
+    oofEntry = Label(oofDialogFrame, textvariable=vars.oofNameVar)
+
+    def OofSelect():
+        nonlocal vars
+        oof_file = filedialog.askopenfilename(
+            filetypes=[("BRR files", ".brr"),
+                       ("All Files", "*")])
+        try:
+            set_oof(oof_file)
+        except Exception:
+            set_oof(None)
+
+    oofSelectButton = Button(oofDialogFrame, text='...', command=OofSelect)
+    AttachTooltip(oofSelectButton,
+                  text="Select a .brr file no more than 2673 bytes.\n" + \
+                  "This can be created from a <=0.394s 16-bit signed PCM .wav file at 12khz using snesbrr.")
+
+    baseOofLabel.pack(side=LEFT)
+    oofEntry.pack(side=LEFT)
+    oofSelectButton.pack(side=LEFT)
+
     vars.quickSwapVar = IntVar(value=adjuster_settings.quickswap)
     quickSwapCheckbutton = Checkbutton(romOptionsFrame, text="L/R Quickswapping", variable=vars.quickSwapVar)
     quickSwapCheckbutton.grid(row=1, column=0, sticky=E)
 
     menuspeedFrame = Frame(romOptionsFrame)
-    menuspeedFrame.grid(row=1, column=1, sticky=E)
+    menuspeedFrame.grid(row=6, column=1, sticky=E)
     menuspeedLabel = Label(menuspeedFrame, text='Menu speed')
     menuspeedLabel.pack(side=LEFT)
     vars.menuspeedVar = StringVar()
@@ -724,7 +811,7 @@ def get_rom_options_frame(parent=None):
     vars.auto_apply = StringVar(value=adjuster_settings.auto_apply)
     autoApplyFrame = Frame(romOptionsFrame)
     autoApplyFrame.grid(row=9, column=0, columnspan=2, sticky=W)
-    filler = Label(autoApplyFrame, text="Automatically apply last used settings on opening .apbp files")
+    filler = Label(autoApplyFrame, text="Automatically apply last used settings on opening .aplttp files")
     filler.pack(side=TOP, expand=True, fill=X)
     askRadio = Radiobutton(autoApplyFrame, text='Ask', variable=vars.auto_apply, value='ask')
     askRadio.pack(side=LEFT, padx=5, pady=5)
@@ -751,6 +838,7 @@ class SpriteSelector():
         self.window['pady'] = 5
         self.spritesPerRow = 32
         self.all_sprites = []
+        self.invalid_sprites = []
         self.sprite_pool = spritePool
 
         def open_custom_sprite_dir(_evt):
@@ -832,6 +920,13 @@ class SpriteSelector():
         self.window.focus()
         tkinter_center_window(self.window)
 
+        if self.invalid_sprites:
+            invalid = sorted(self.invalid_sprites)
+            logging.warning(f"The following sprites are invalid: {', '.join(invalid)}")
+            msg = f"{invalid[0]} "
+            msg += f"and {len(invalid)-1} more are invalid" if len(invalid) > 1 else "is invalid"
+            messagebox.showerror("Invalid sprites detected", msg, parent=self.window)
+
     def remove_from_sprite_pool(self, button, spritename):
         self.callback(("remove", spritename))
         self.spritePoolButtons.buttons.remove(button)
@@ -896,7 +991,13 @@ class SpriteSelector():
         sprites = []
 
         for file in os.listdir(path):
-            sprites.append((file, Sprite(os.path.join(path, file))))
+            if file == '.gitignore':
+                continue
+            sprite = Sprite(os.path.join(path, file))
+            if sprite.valid:
+                sprites.append((file, sprite))
+            else:
+                self.invalid_sprites.append(file)
 
         sprites.sort(key=lambda s: str.lower(s[1].name or "").strip())
 
@@ -1039,7 +1140,6 @@ class SpriteSelector():
     def custom_sprite_dir(self):
         return user_path("data", "sprites", "custom")
 
-
 def get_image_for_sprite(sprite, gif_only: bool = False):
     if not sprite.valid:
         return None
@@ -1263,4 +1363,4 @@ class ToolTips(object):
 
 
 if __name__ == '__main__':
-    main()
+    main()

Main.py

@@ -1,24 +1,24 @@
-import copy
 import collections
-from itertools import zip_longest, chain
+import concurrent.futures
 import logging
 import os
-import time
-import zlib
-import concurrent.futures
 import pickle
 import tempfile
+import time
 import zipfile
-from typing import Dict, Tuple, Optional, Set
+import zlib
+from typing import Dict, List, Optional, Set, Tuple
 
-from BaseClasses import MultiWorld, CollectionState, Region, RegionType, LocationProgressType, Location
-from worlds.alttp.Items import item_name_groups
-from worlds.alttp.Regions import lookup_vanilla_location_to_entrance
-from Fill import distribute_items_restrictive, flood_items, balance_multiworld_progression, distribute_planned
-from worlds.alttp.Shops import SHOP_ID_START, total_shop_slots, FillDisabledShopSlots
-from Utils import output_path, get_options, __version__, version_tuple
-from worlds.generic.Rules import locality_rules, exclusion_rules, group_locality_rules
+import worlds
+from BaseClasses import CollectionState, Item, Location, LocationProgressType, MultiWorld, Region
+from Fill import balance_multiworld_progression, distribute_items_restrictive, distribute_planned, flood_items
+from Options import StartInventoryPool
+from Utils import __version__, get_options, output_path, version_tuple
 from worlds import AutoWorld
+from worlds.alttp.Regions import is_main_entrance
+from worlds.alttp.Shops import FillDisabledShopSlots
+from worlds.alttp.SubClasses import LTTPRegionType
+from worlds.generic.Rules import exclusion_rules, locality_rules
 
 ordered_areas = (
     'Light World', 'Dark World', 'Hyrule Castle', 'Agahnims Tower', 'Eastern Palace', 'Desert Palace',
@@ -39,7 +39,8 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
     world = MultiWorld(args.multi)
 
     logger = logging.getLogger()
-    world.set_seed(seed, args.race, str(args.outputname if args.outputname else world.seed))
+    world.set_seed(seed, args.race, str(args.outputname) if args.outputname else None)
+    world.plando_options = args.plando_options
 
     world.shuffle = args.shuffle.copy()
     world.logic = args.logic.copy()
@@ -48,13 +49,11 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
     world.item_functionality = args.item_functionality.copy()
     world.timer = args.timer.copy()
     world.goal = args.goal.copy()
-    world.open_pyramid = args.open_pyramid.copy()
     world.boss_shuffle = args.shufflebosses.copy()
     world.enemy_health = args.enemy_health.copy()
     world.enemy_damage = args.enemy_damage.copy()
     world.beemizer_total_chance = args.beemizer_total_chance.copy()
     world.beemizer_trap_chance = args.beemizer_trap_chance.copy()
-    world.timer = args.timer.copy()
     world.countdown_start_time = args.countdown_start_time.copy()
     world.red_clock_time = args.red_clock_time.copy()
     world.blue_clock_time = args.blue_clock_time.copy()
@@ -72,7 +71,6 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
     world.required_medallions = args.required_medallions.copy()
     world.game = args.game.copy()
     world.player_name = args.name.copy()
-    world.enemizer = args.enemizercli
     world.sprite = args.sprite.copy()
     world.glitch_triforce = args.glitch_triforce  # This is enabled/disabled globally, no per player option.
 
@@ -81,17 +79,32 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
     world.state = CollectionState(world)
     logger.info('Archipelago Version %s  -  Seed: %s\n', __version__, world.seed)
 
-    logger.info("Found World Types:")
+    logger.info(f"Found {len(AutoWorld.AutoWorldRegister.world_types)} World Types:")
     longest_name = max(len(text) for text in AutoWorld.AutoWorldRegister.world_types)
-    numlength = 8
+
+    max_item = 0
+    max_location = 0
+    for cls in AutoWorld.AutoWorldRegister.world_types.values():
+        if cls.item_id_to_name:
+            max_item = max(max_item, max(cls.item_id_to_name))
+            max_location = max(max_location, max(cls.location_id_to_name))
+
+    item_digits = len(str(max_item))
+    location_digits = len(str(max_location))
+    item_count = len(str(max(len(cls.item_names) for cls in AutoWorld.AutoWorldRegister.world_types.values())))
+    location_count = len(str(max(len(cls.location_names) for cls in AutoWorld.AutoWorldRegister.world_types.values())))
+    del max_item, max_location
+
     for name, cls in AutoWorld.AutoWorldRegister.world_types.items():
-        if not cls.hidden:
-            logger.info(f"  {name:{longest_name}}: {len(cls.item_names):3} "
-                        f"Items (IDs: {min(cls.item_id_to_name):{numlength}} - "
-                        f"{max(cls.item_id_to_name):{numlength}}) | "
-                        f"{len(cls.location_names):3} "
-                        f"Locations (IDs: {min(cls.location_id_to_name):{numlength}} - "
-                        f"{max(cls.location_id_to_name):{numlength}})")
+        if not cls.hidden and len(cls.item_names) > 0:
+            logger.info(f" {name:{longest_name}}: {len(cls.item_names):{item_count}} "
+                        f"Items (IDs: {min(cls.item_id_to_name):{item_digits}} - "
+                        f"{max(cls.item_id_to_name):{item_digits}}) | "
+                        f"{len(cls.location_names):{location_count}} "
+                        f"Locations (IDs: {min(cls.location_id_to_name):{location_digits}} - "
+                        f"{max(cls.location_id_to_name):{location_digits}})")
+
+    del item_digits, location_digits, item_count, location_count
 
     AutoWorld.call_stage(world, "assert_generate")
 
@@ -104,18 +117,9 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
             for _ in range(count):
                 world.push_precollected(world.create_item(item_name, player))
 
-    for player in world.player_ids:
-        if player in world.get_game_players("A Link to the Past"):
-            # enforce pre-defined local items.
-            if world.goal[player] in ["localtriforcehunt", "localganontriforcehunt"]:
-                world.local_items[player].value.add('Triforce Piece')
-
-            # Not possible to place pendants/crystals out side of boss prizes yet.
-            world.non_local_items[player].value -= item_name_groups['Pendants']
-            world.non_local_items[player].value -= item_name_groups['Crystals']
-
-        # items can't be both local and non-local, prefer local
-        world.non_local_items[player].value -= world.local_items[player].value
+        for item_name, count in world.start_inventory_from_pool.setdefault(player, StartInventoryPool({})).value.items():
+            for _ in range(count):
+                world.push_precollected(world.create_item(item_name, player))
 
     logger.info('Creating World.')
     AutoWorld.call_all(world, "create_regions")
@@ -123,11 +127,19 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
     logger.info('Creating Items.')
     AutoWorld.call_all(world, "create_items")
 
+    # All worlds should have finished creating all regions, locations, and entrances.
+    # Recache to ensure that they are all visible for locality rules.
+    world._recache()
+
     logger.info('Calculating Access Rules.')
+
+    for player in world.player_ids:
+        # items can't be both local and non-local, prefer local
+        world.non_local_items[player].value -= world.local_items[player].value
+        world.non_local_items[player].value -= set(world.local_early_items[player])
+
     if world.players > 1:
-        for player in world.player_ids:
-            locality_rules(world, player)
-        group_locality_rules(world)
+        locality_rules(world)
     else:
         world.non_local_items[1].value = set()
         world.local_items[1].value = set()
@@ -142,21 +154,53 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
 
     AutoWorld.call_all(world, "generate_basic")
 
+    # remove starting inventory from pool items.
+    # Because some worlds don't actually create items during create_items this has to be as late as possible.
+    if any(world.start_inventory_from_pool[player].value for player in world.player_ids):
+        new_items: List[Item] = []
+        depletion_pool: Dict[int, Dict[str, int]] = {
+            player: world.start_inventory_from_pool[player].value.copy() for player in world.player_ids}
+        for player, items in depletion_pool.items():
+            player_world: AutoWorld.World = world.worlds[player]
+            for count in items.values():
+                new_items.append(player_world.create_filler())
+        target: int = sum(sum(items.values()) for items in depletion_pool.values())
+        for i, item in enumerate(world.itempool):
+            if depletion_pool[item.player].get(item.name, 0):
+                target -= 1
+                depletion_pool[item.player][item.name] -= 1
+                # quick abort if we have found all items
+                if not target:
+                    new_items.extend(world.itempool[i+1:])
+                    break
+            else:
+                new_items.append(item)
+
+        # leftovers?
+        if target:
+            for player, remaining_items in depletion_pool.items():
+                remaining_items = {name: count for name, count in remaining_items.items() if count}
+                if remaining_items:
+                    raise Exception(f"{world.get_player_name(player)}"
+                                    f" is trying to remove items from their pool that don't exist: {remaining_items}")
+        world.itempool[:] = new_items
+
     # temporary home for item links, should be moved out of Main
     for group_id, group in world.groups.items():
-        def find_common_pool(players: Set[int], shared_pool: Set[str]):
-            advancement = set()
+        def find_common_pool(players: Set[int], shared_pool: Set[str]) -> Tuple[
+            Optional[Dict[int, Dict[str, int]]], Optional[Dict[str, int]]
+        ]:
+            classifications: Dict[str, int] = collections.defaultdict(int)
             counters = {player: {name: 0 for name in shared_pool} for player in players}
             for item in world.itempool:
                 if item.player in counters and item.name in shared_pool:
                     counters[item.player][item.name] += 1
-                    if item.advancement:
-                        advancement.add(item.name)
+                    classifications[item.name] |= item.classification
 
             for player in players.copy():
                 if all([counters[player][item] == 0 for item in shared_pool]):
                     players.remove(player)
-                    del(counters[player])
+                    del (counters[player])
 
             if not players:
                 return None, None
@@ -168,22 +212,22 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
                         counters[player][item] = count
                 else:
                     for player in players:
-                        del(counters[player][item])
-            return counters, advancement
+                        del (counters[player][item])
+            return counters, classifications
 
-        common_item_count, common_advancement_items = find_common_pool(group["players"], group["item_pool"])
+        common_item_count, classifications = find_common_pool(group["players"], group["item_pool"])
         if not common_item_count:
             continue
 
-        new_itempool = []
+        new_itempool: List[Item] = []
         for item_name, item_count in next(iter(common_item_count.values())).items():
-            advancement = item_name in common_advancement_items
             for _ in range(item_count):
                 new_item = group["world"].create_item(item_name)
-                new_item.advancement = advancement
+                # mangle together all original classification bits
+                new_item.classification |= classifications[item_name]
                 new_itempool.append(new_item)
 
-        region = Region("Menu", RegionType.Generic, "ItemLink", group_id, world)
+        region = Region("Menu", group_id, world, "ItemLink")
         world.regions.append(region)
         locations = region.locations = []
         for item in world.itempool:
@@ -206,11 +250,15 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
         while itemcount > len(world.itempool):
             items_to_add = []
             for player in group["players"]:
+                if group["link_replacement"]:
+                    item_player = group_id
+                else:
+                    item_player = player
                 if group["replacement_items"][player]:
-                    items_to_add.append(AutoWorld.call_single(world, "create_item", player,
+                    items_to_add.append(AutoWorld.call_single(world, "create_item", item_player,
                                                                 group["replacement_items"][player]))
                 else:
-                    items_to_add.append(AutoWorld.call_single(world, "create_filler", player))
+                    items_to_add.append(AutoWorld.call_single(world, "create_filler", item_player))
             world.random.shuffle(items_to_add)
             world.itempool.extend(items_to_add[:itemcount - len(world.itempool)])
 
@@ -220,9 +268,6 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
 
     logger.info("Running Item Plando")
 
-    for item in world.itempool:
-        item.world = world
-
     distribute_planned(world)
 
     logger.info('Running Pre Main Fill.')
@@ -242,6 +287,10 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
         balance_multiworld_progression(world)
 
     logger.info(f'Beginning output...')
+
+    # we're about to output using multithreading, so we're removing the global random state to prevent accidental use
+    world.random.passthrough = False
+
     outfilebase = 'AP_' + world.seed_name
 
     output = tempfile.TemporaryDirectory()
@@ -256,24 +305,9 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
                     output_file_futures.append(
                         pool.submit(AutoWorld.call_single, world, "generate_output", player, temp_dir))
 
-            def get_entrance_to_region(region: Region):
-                for entrance in region.entrances:
-                    if entrance.parent_region.type in (RegionType.DarkWorld, RegionType.LightWorld, RegionType.Generic):
-                        return entrance
-                for entrance in region.entrances:  # BFS might be better here, trying DFS for now.
-                    return get_entrance_to_region(entrance.parent_region)
-
             # collect ER hint info
-            er_hint_data = {player: {} for player in world.get_game_players("A Link to the Past") if
-                            world.shuffle[player] != "vanilla" or world.retro[player]}
-
-            for region in world.regions:
-                if region.player in er_hint_data and region.locations:
-                    main_entrance = get_entrance_to_region(region)
-                    for location in region.locations:
-                        if type(location.address) == int:  # skips events and crystals
-                            if lookup_vanilla_location_to_entrance[location.address] != main_entrance.name:
-                                er_hint_data[region.player][location.address] = main_entrance.name
+            er_hint_data: Dict[int, Dict[int, str]] = {}
+            AutoWorld.call_all(world, 'extend_hint_information', er_hint_data)
 
             checks_in_area = {player: {area: list() for area in ordered_areas}
                               for player in range(1, world.players + 1)}
@@ -283,44 +317,24 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
 
             for location in world.get_filled_locations():
                 if type(location.address) is int:
-                    main_entrance = get_entrance_to_region(location.parent_region)
                     if location.game != "A Link to the Past":
                         checks_in_area[location.player]["Light World"].append(location.address)
-                    elif location.parent_region.dungeon:
-                        dungeonname = {'Inverted Agahnims Tower': 'Agahnims Tower',
-                                       'Inverted Ganons Tower': 'Ganons Tower'} \
-                            .get(location.parent_region.dungeon.name, location.parent_region.dungeon.name)
-                        checks_in_area[location.player][dungeonname].append(location.address)
-                    elif location.parent_region.type == RegionType.LightWorld:
-                        checks_in_area[location.player]["Light World"].append(location.address)
-                    elif location.parent_region.type == RegionType.DarkWorld:
-                        checks_in_area[location.player]["Dark World"].append(location.address)
-                    elif main_entrance.parent_region.type == RegionType.LightWorld:
-                        checks_in_area[location.player]["Light World"].append(location.address)
-                    elif main_entrance.parent_region.type == RegionType.DarkWorld:
-                        checks_in_area[location.player]["Dark World"].append(location.address)
-                    checks_in_area[location.player]["Total"] += 1
-
-            oldmancaves = []
-            takeanyregions = ["Old Man Sword Cave", "Take-Any #1", "Take-Any #2", "Take-Any #3", "Take-Any #4"]
-            for index, take_any in enumerate(takeanyregions):
-                for region in [world.get_region(take_any, player) for player in
-                               world.get_game_players("A Link to the Past") if world.retro[player]]:
-                    item = world.create_item(
-                        region.shop.inventory[(0 if take_any == "Old Man Sword Cave" else 1)]['item'],
-                        region.player)
-                    player = region.player
-                    location_id = SHOP_ID_START + total_shop_slots + index
-
-                    main_entrance = get_entrance_to_region(region)
-                    if main_entrance.parent_region.type == RegionType.LightWorld:
-                        checks_in_area[player]["Light World"].append(location_id)
                     else:
-                        checks_in_area[player]["Dark World"].append(location_id)
-                    checks_in_area[player]["Total"] += 1
-
-                    er_hint_data[player][location_id] = main_entrance.name
-                    oldmancaves.append(((location_id, player), (item.code, player)))
+                        main_entrance = location.parent_region.get_connecting_entrance(is_main_entrance)
+                        if location.parent_region.dungeon:
+                            dungeonname = {'Inverted Agahnims Tower': 'Agahnims Tower',
+                                           'Inverted Ganons Tower': 'Ganons Tower'} \
+                                .get(location.parent_region.dungeon.name, location.parent_region.dungeon.name)
+                            checks_in_area[location.player][dungeonname].append(location.address)
+                        elif location.parent_region.type == LTTPRegionType.LightWorld:
+                            checks_in_area[location.player]["Light World"].append(location.address)
+                        elif location.parent_region.type == LTTPRegionType.DarkWorld:
+                            checks_in_area[location.player]["Dark World"].append(location.address)
+                        elif main_entrance.parent_region.type == LTTPRegionType.LightWorld:
+                            checks_in_area[location.player]["Light World"].append(location.address)
+                        elif main_entrance.parent_region.type == LTTPRegionType.DarkWorld:
+                            checks_in_area[location.player]["Dark World"].append(location.address)
+                    checks_in_area[location.player]["Total"] += 1
 
             FillDisabledShopSlots(world)
 
@@ -347,7 +361,6 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
                                       for player, world_precollected in world.precollected_items.items()}
                 precollected_hints = {player: set() for player in range(1, world.players + 1 + len(world.groups))}
 
-
                 for slot in world.player_ids:
                     slot_data[slot] = world.worlds[slot].fill_slot_data()
 
@@ -366,7 +379,8 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
                 for location in world.get_filled_locations():
                     if type(location.address) == int:
                         assert location.item.code is not None, "item code None should be event, " \
-                                                               "location.address should then also be None"
+                                                               "location.address should then also be None. Location: " \
+                                                               f" {location}"
                         locations_data[location.player][location.address] = \
                             location.item.code, location.item.player, location.item.flags
                         if location.name in world.start_location_hints[location.player]:
@@ -377,16 +391,17 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
                                   for player in world.groups.get(location.item.player, {}).get("players", [])]):
                             precollect_hint(location)
 
+                # embedded data package
+                data_package = {
+                    game_world.game: worlds.network_data_package["games"][game_world.game]
+                    for game_world in world.worlds.values()
+                }
+
                 multidata = {
                     "slot_data": slot_data,
                     "slot_info": slot_info,
-                    "names": names,  # TODO: remove around 0.2.5 in favor of slot_info
-                    "games": games,  # TODO: remove around 0.2.5 in favor of slot_info
+                    "names": names,  # TODO: remove after 0.3.9
                     "connect_names": {name: (0, player) for player, name in world.player_name.items()},
-                    "remote_items": {player for player in world.player_ids if
-                                     world.worlds[player].remote_items},
-                    "remote_start_inventory": {player for player in world.player_ids if
-                                               world.worlds[player].remote_start_inventory},
                     "locations": locations_data,
                     "checks_in_area": checks_in_area,
                     "server_options": baked_server_options,
@@ -396,7 +411,8 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
                     "version": tuple(version_tuple),
                     "tags": ["AP"],
                     "minimum_versions": minimum_versions,
-                    "seed_name": world.seed_name
+                    "seed_name": world.seed_name,
+                    "datapackage": data_package,
                 }
                 AutoWorld.call_all(world, "modify_multidata", multidata)
 
@@ -422,13 +438,13 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
 
         if args.spoiler > 1:
             logger.info('Calculating playthrough.')
-            create_playthrough(world)
+            world.spoiler.create_playthrough(create_paths=args.spoiler > 2)
 
         if args.spoiler:
             world.spoiler.to_file(os.path.join(temp_dir, '%s_Spoiler.txt' % outfilebase))
 
         zipfilename = output_path(f"AP_{world.seed_name}.zip")
-        logger.info(f'Creating final archive at {zipfilename}.')
+        logger.info(f"Creating final archive at {zipfilename}")
         with zipfile.ZipFile(zipfilename, mode="w", compression=zipfile.ZIP_DEFLATED,
                              compresslevel=9) as zf:
             for file in os.scandir(temp_dir):
@@ -436,143 +452,3 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No
 
     logger.info('Done. Enjoy. Total Time: %s', time.perf_counter() - start)
     return world
-
-
-def create_playthrough(world):
-    """Destructive to the world while it is run, damage gets repaired afterwards."""
-    # get locations containing progress items
-    prog_locations = {location for location in world.get_filled_locations() if location.item.advancement}
-    state_cache = [None]
-    collection_spheres = []
-    state = CollectionState(world)
-    sphere_candidates = set(prog_locations)
-    logging.debug('Building up collection spheres.')
-    while sphere_candidates:
-
-        # build up spheres of collection radius.
-        # Everything in each sphere is independent from each other in dependencies and only depends on lower spheres
-
-        sphere = {location for location in sphere_candidates if state.can_reach(location)}
-
-        for location in sphere:
-            state.collect(location.item, True, location)
-
-        sphere_candidates -= sphere
-        collection_spheres.append(sphere)
-        state_cache.append(state.copy())
-
-        logging.debug('Calculated sphere %i, containing %i of %i progress items.', len(collection_spheres), len(sphere),
-                      len(prog_locations))
-        if not sphere:
-            logging.debug('The following items could not be reached: %s', ['%s (Player %d) at %s (Player %d)' % (
-                location.item.name, location.item.player, location.name, location.player) for location in
-                                                                           sphere_candidates])
-            if any([world.accessibility[location.item.player] != 'minimal' for location in sphere_candidates]):
-                raise RuntimeError(f'Not all progression items reachable ({sphere_candidates}). '
-                                   f'Something went terribly wrong here.')
-            else:
-                world.spoiler.unreachables = sphere_candidates
-                break
-
-    # in the second phase, we cull each sphere such that the game is still beatable,
-    # reducing each range of influence to the bare minimum required inside it
-    restore_later = {}
-    for num, sphere in reversed(tuple(enumerate(collection_spheres))):
-        to_delete = set()
-        for location in sphere:
-            # we remove the item at location and check if game is still beatable
-            logging.debug('Checking if %s (Player %d) is required to beat the game.', location.item.name,
-                          location.item.player)
-            old_item = location.item
-            location.item = None
-            if world.can_beat_game(state_cache[num]):
-                to_delete.add(location)
-                restore_later[location] = old_item
-            else:
-                # still required, got to keep it around
-                location.item = old_item
-
-        # cull entries in spheres for spoiler walkthrough at end
-        sphere -= to_delete
-
-    # second phase, sphere 0
-    removed_precollected = []
-    for item in (i for i in chain.from_iterable(world.precollected_items.values()) if i.advancement):
-        logging.debug('Checking if %s (Player %d) is required to beat the game.', item.name, item.player)
-        world.precollected_items[item.player].remove(item)
-        world.state.remove(item)
-        if not world.can_beat_game():
-            world.push_precollected(item)
-        else:
-            removed_precollected.append(item)
-
-    # we are now down to just the required progress items in collection_spheres. Unfortunately
-    # the previous pruning stage could potentially have made certain items dependant on others
-    # in the same or later sphere (because the location had 2 ways to access but the item originally
-    # used to access it was deemed not required.) So we need to do one final sphere collection pass
-    # to build up the correct spheres
-
-    required_locations = {item for sphere in collection_spheres for item in sphere}
-    state = CollectionState(world)
-    collection_spheres = []
-    while required_locations:
-        state.sweep_for_events(key_only=True)
-
-        sphere = set(filter(state.can_reach, required_locations))
-
-        for location in sphere:
-            state.collect(location.item, True, location)
-
-        required_locations -= sphere
-
-        collection_spheres.append(sphere)
-
-        logging.debug('Calculated final sphere %i, containing %i of %i progress items.', len(collection_spheres),
-                      len(sphere), len(required_locations))
-        if not sphere:
-            raise RuntimeError(f'Not all required items reachable. Unreachable locations: {required_locations}')
-
-    def flist_to_iter(node):
-        while node:
-            value, node = node
-            yield value
-
-    def get_path(state, region):
-        reversed_path_as_flist = state.path.get(region, (region, None))
-        string_path_flat = reversed(list(map(str, flist_to_iter(reversed_path_as_flist))))
-        # Now we combine the flat string list into (region, exit) pairs
-        pathsiter = iter(string_path_flat)
-        pathpairs = zip_longest(pathsiter, pathsiter)
-        return list(pathpairs)
-
-    world.spoiler.paths = {}
-    topology_worlds = (player for player in world.player_ids if world.worlds[player].topology_present)
-    for player in topology_worlds:
-        world.spoiler.paths.update(
-            {str(location): get_path(state, location.parent_region) for sphere in collection_spheres for location in
-             sphere if location.player == player})
-        if player in world.get_game_players("A Link to the Past"):
-            # If Pyramid Fairy Entrance needs to be reached, also path to Big Bomb Shop
-            # Maybe move the big bomb over to the Event system instead?
-            if any(exit_path == 'Pyramid Fairy' for path in world.spoiler.paths.values() for (_, exit_path) in path):
-                if world.mode[player] != 'inverted':
-                    world.spoiler.paths[str(world.get_region('Big Bomb Shop', player))] = \
-                        get_path(state, world.get_region('Big Bomb Shop', player))
-                else:
-                    world.spoiler.paths[str(world.get_region('Inverted Big Bomb Shop', player))] = \
-                        get_path(state, world.get_region('Inverted Big Bomb Shop', player))
-
-    # we can finally output our playthrough
-    world.spoiler.playthrough = {"0": sorted([str(item) for item in
-                                              chain.from_iterable(world.precollected_items.values())
-                                              if item.advancement])}
-
-    for i, sphere in enumerate(collection_spheres):
-        world.spoiler.playthrough[str(i + 1)] = {str(location): str(location.item) for location in sorted(sphere)}
-
-    # repair the world again
-    for location, item in restore_later.items():
-        location.item = item
-
-    for item in removed_precollected:
-        world.push_precollected(item)

MinecraftClient.py

@@ -13,12 +13,12 @@ import logging
 import requests
 
 import Utils
+from Utils import is_windows
 
 atexit.register(input, "Press enter to exit.")
 
 # 1 or more digits followed by m or g, then optional b
 max_heap_re = re.compile(r"^\d+[mMgG][bB]?$")
-is_windows = sys.platform in ("win32", "cygwin", "msys")
 
 
 def prompt_yes_no(prompt):
@@ -77,49 +77,34 @@ def read_apmc_file(apmc_file):
         return json.loads(b64decode(f.read()))
 
 
-def update_mod(forge_dir, minecraft_version: str, get_prereleases=False):
+def update_mod(forge_dir, url: str):
     """Check mod version, download new mod from GitHub releases page if needed. """
     ap_randomizer = find_ap_randomizer_jar(forge_dir)
-
-    client_releases_endpoint = "https://api.github.com/repos/KonoTyran/Minecraft_AP_Randomizer/releases"
-    resp = requests.get(client_releases_endpoint)
-    if resp.status_code == 200:  # OK
-        try:
-            latest_release = next(filter(lambda release: (not release['prerelease'] or get_prereleases) and
-                (minecraft_version in release['assets'][0]['name']),
-                resp.json()))
-            if ap_randomizer != latest_release['assets'][0]['name']:
-                logging.info(f"A new release of the Minecraft AP randomizer mod was found: "
-                             f"{latest_release['assets'][0]['name']}")
-                if ap_randomizer is not None:
-                    logging.info(f"Your current mod is {ap_randomizer}.")
-                else:
-                    logging.info(f"You do not have the AP randomizer mod installed.")
-                if prompt_yes_no("Would you like to update?"):
-                    old_ap_mod = os.path.join(forge_dir, 'mods', ap_randomizer) if ap_randomizer is not None else None
-                    new_ap_mod = os.path.join(forge_dir, 'mods', latest_release['assets'][0]['name'])
-                    logging.info("Downloading AP randomizer mod. This may take a moment...")
-                    apmod_resp = requests.get(latest_release['assets'][0]['browser_download_url'])
-                    if apmod_resp.status_code == 200: 
-                        with open(new_ap_mod, 'wb') as f:
-                            f.write(apmod_resp.content)
-                            logging.info(f"Wrote new mod file to {new_ap_mod}")
-                        if old_ap_mod is not None:
-                            os.remove(old_ap_mod)
-                            logging.info(f"Removed old mod file from {old_ap_mod}")
-                    else:
-                        logging.error(f"Error retrieving the randomizer mod (status code {apmod_resp.status_code}).")
-                        logging.error(f"Please report this issue on the Archipelago Discord server.")
-                        sys.exit(1)
-        except StopIteration:
-            logging.warning(f"No compatible mod version found for {minecraft_version}.")
-            if not prompt_yes_no("Run server anyway?"):
-                sys.exit(0)
+    os.path.basename(url)
+    if ap_randomizer is not None:
+        logging.info(f"Your current mod is {ap_randomizer}.")
     else:
-        logging.error(f"Error checking for randomizer mod updates (status code {resp.status_code}).")
-        logging.error(f"If this was not expected, please report this issue on the Archipelago Discord server.")
-        if not prompt_yes_no("Continue anyways?"):
-            sys.exit(0)
+        logging.info(f"You do not have the AP randomizer mod installed.")
+
+    if ap_randomizer != os.path.basename(url):
+        logging.info(f"A new release of the Minecraft AP randomizer mod was found: "
+                     f"{os.path.basename(url)}")
+        if prompt_yes_no("Would you like to update?"):
+            old_ap_mod = os.path.join(forge_dir, 'mods', ap_randomizer) if ap_randomizer is not None else None
+            new_ap_mod = os.path.join(forge_dir, 'mods', os.path.basename(url))
+            logging.info("Downloading AP randomizer mod. This may take a moment...")
+            apmod_resp = requests.get(url)
+            if apmod_resp.status_code == 200:
+                with open(new_ap_mod, 'wb') as f:
+                    f.write(apmod_resp.content)
+                    logging.info(f"Wrote new mod file to {new_ap_mod}")
+                if old_ap_mod is not None:
+                    os.remove(old_ap_mod)
+                    logging.info(f"Removed old mod file from {old_ap_mod}")
+            else:
+                logging.error(f"Error retrieving the randomizer mod (status code {apmod_resp.status_code}).")
+                logging.error(f"Please report this issue on the Archipelago Discord server.")
+                sys.exit(1)
 
 
 def check_eula(forge_dir):
@@ -196,8 +181,8 @@ def download_java(java: str):
 def install_forge(directory: str, forge_version: str, java_version: str):
     """download and install forge"""
 
-    jdk = find_jdk(java_version)
-    if jdk is not None:
+    java_exe = find_jdk(java_version)
+    if java_exe is not None:
         print(f"Downloading Forge {forge_version}...")
         forge_url = f"https://maven.minecraftforge.net/net/minecraftforge/forge/{forge_version}/forge-{forge_version}-installer.jar"
         resp = requests.get(forge_url)
@@ -208,8 +193,7 @@ def install_forge(directory: str, forge_version: str, java_version: str):
             with open(forge_install_jar, 'wb') as f:
                 f.write(resp.content)
             print(f"Installing Forge...")
-            argstring = ' '.join([jdk, "-jar", "\"" + forge_install_jar + "\"", "--installServer", "\"" + directory + "\""])
-            install_process = Popen(argstring, shell=not is_windows)
+            install_process = Popen([java_exe, "-jar", forge_install_jar, "--installServer", directory])
             install_process.wait()
             os.remove(forge_install_jar)
 
@@ -228,15 +212,15 @@ def run_forge_server(forge_dir: str, java_version: str, heap_arg: str) -> Popen:
 
     os_args = "win_args.txt" if is_windows else "unix_args.txt"
     args_file = os.path.join(forge_dir, "libraries", "net", "minecraftforge", "forge", forge_version, os_args)
-    win_args = []
+    forge_args = []
     with open(args_file) as argfile:
         for line in argfile:
-            win_args.append(line.strip())
+            forge_args.extend(line.strip().split(" "))
 
-    argstring = ' '.join([java_exe, heap_arg] + win_args + ["-nogui"])
-    logging.info(f"Running Forge server: {argstring}")
+    args = [java_exe, heap_arg, *forge_args, "-nogui"]
+    logging.info(f"Running Forge server: {args}")
     os.chdir(forge_dir)
-    return Popen(argstring, shell=not is_windows)
+    return Popen(args)
 
 
 def get_minecraft_versions(version, release_channel="release"):
@@ -254,10 +238,10 @@ def get_minecraft_versions(version, release_channel="release"):
         local = True
 
     if local:
-        with open(Utils.local_path("minecraft_versions.json"), 'r') as f:
+        with open(Utils.user_path("minecraft_versions.json"), 'r') as f:
             data = json.load(f)
     else:
-        with open(Utils.local_path("minecraft_versions.json"), 'w') as f:
+        with open(Utils.user_path("minecraft_versions.json"), 'w') as f:
             json.dump(data, f)
 
     try:
@@ -265,8 +249,13 @@ def get_minecraft_versions(version, release_channel="release"):
             return next(filter(lambda entry: entry["version"] == version, data[release_channel]))
         else:
             return resp.json()[release_channel][0]
-    except StopIteration:
-        logging.error(f"No compatible mod version found for client version {version}.")
+    except (StopIteration, KeyError):
+        logging.error(f"No compatible mod version found for client version {version} on \"{release_channel}\" channel.")
+        if release_channel != "release":
+            logging.error("Consider switching \"release_channel\" to \"release\" in your Host.yaml file")
+        else:
+            logging.error("No suitable mod found on the \"release\" channel. Please Contact us on discord to report this error.")
+        sys.exit(0)
 
 
 def is_correct_forge(forge_dir) -> bool:
@@ -287,6 +276,8 @@ if __name__ == '__main__':
                         help="specify java version.")
     parser.add_argument('--forge', '-f', metavar='1.18.2-40.1.0', dest='forge', type=str, default=False, action='store',
                         help="specify forge version. (Minecraft Version-Forge Version)")
+    parser.add_argument('--version', '-v', metavar='9', dest='data_version', type=int, action='store',
+                        help="specify Mod data version to download.")
 
     args = parser.parse_args()
     apmc_file = os.path.abspath(args.apmc_file) if args.apmc_file else None
@@ -297,27 +288,33 @@ if __name__ == '__main__':
     options = Utils.get_options()
     channel = args.channel or options["minecraft_options"]["release_channel"]
     apmc_data = None
-    data_version = None
+    data_version = args.data_version or None
 
-    if apmc_file is not None:
+    if apmc_file is None and not args.install:
+        apmc_file = Utils.open_filename('Select APMC file', (('APMC File', ('.apmc',)),))
+
+    if apmc_file is not None and data_version is None:
         apmc_data = read_apmc_file(apmc_file)
         data_version = apmc_data.get('client_version', '')
 
     versions = get_minecraft_versions(data_version, channel)
 
-    forge_dir = options["minecraft_options"]["forge_directory"]
+    forge_dir = Utils.user_path(options["minecraft_options"]["forge_directory"])
     max_heap = options["minecraft_options"]["max_heap_size"]
     forge_version = args.forge or versions["forge"]
     java_version = args.java or versions["java"]
+    mod_url = versions["url"]
     java_dir = find_jdk_dir(java_version)
 
     if args.install:
         if is_windows:
-            print("Installing Java and Minecraft Forge")
+            print("Installing Java")
             download_java(java_version)
-        else:
+        if not is_correct_forge(forge_dir):
             print("Installing Minecraft Forge")
-        install_forge(forge_dir, forge_version, java_version)
+            install_forge(forge_dir, forge_version, java_version)
+        else:
+            print("Correct Forge version already found, skipping install.")
         sys.exit(0)
 
     if apmc_data is None:
@@ -340,7 +337,7 @@ if __name__ == '__main__':
     if not max_heap_re.match(max_heap):
         raise Exception(f"Max heap size {max_heap} in incorrect format. Use a number followed by M or G, e.g. 512M or 2G.")
 
-    update_mod(forge_dir, f"MC{forge_version.split('-')[0]}", channel != "release")
+    update_mod(forge_dir, mod_url)
     replace_apmc_files(forge_dir, apmc_file)
     check_eula(forge_dir)
     server_process = run_forge_server(forge_dir, java_version, max_heap)

ModuleUpdate.py

@@ -1,7 +1,8 @@
 import os
 import sys
 import subprocess
-import pkg_resources
+import multiprocessing
+import warnings
 
 local_dir = os.path.dirname(__file__)
 requirements_files = {os.path.join(local_dir, 'requirements.txt')}
@@ -9,49 +10,109 @@ requirements_files = {os.path.join(local_dir, 'requirements.txt')}
 if sys.version_info < (3, 8, 6):
     raise RuntimeError("Incompatible Python Version. 3.8.7+ is supported.")
 
-update_ran = getattr(sys, "frozen", False)  # don't run update if environment is frozen/compiled
+# don't run update if environment is frozen/compiled or if not the parent process (skip in subprocess)
+update_ran = getattr(sys, "frozen", False) or multiprocessing.parent_process()
 
 if not update_ran:
     for entry in os.scandir(os.path.join(local_dir, "worlds")):
-        if entry.is_dir():
-            req_file = os.path.join(entry.path, "requirements.txt")
-            if os.path.exists(req_file):
-                requirements_files.add(req_file)
+        # skip .* (hidden / disabled) folders
+        if not entry.name.startswith("."):
+            if entry.is_dir():
+                req_file = os.path.join(entry.path, "requirements.txt")
+                if os.path.exists(req_file):
+                    requirements_files.add(req_file)
+
+
+def check_pip():
+    # detect if pip is available
+    try:
+        import pip  # noqa: F401
+    except ImportError:
+        raise RuntimeError("pip not available. Please install pip.")
+
+
+def confirm(msg: str):
+    try:
+        input(f"\n{msg}")
+    except KeyboardInterrupt:
+        print("\nAborting")
+        sys.exit(1)
 
 
 def update_command():
+    check_pip()
     for file in requirements_files:
-        subprocess.call([sys.executable, '-m', 'pip', 'install', '-r', file, '--upgrade'])
+        subprocess.call([sys.executable, "-m", "pip", "install", "-r", file, "--upgrade"])
+
+
+def install_pkg_resources(yes=False):
+    try:
+        import pkg_resources  # noqa: F401
+    except ImportError:
+        check_pip()
+        if not yes:
+            confirm("pkg_resources not found, press enter to install it")
+        subprocess.call([sys.executable, "-m", "pip", "install", "--upgrade", "setuptools"])
 
 
 def update(yes=False, force=False):
     global update_ran
     if not update_ran:
         update_ran = True
+
         if force:
             update_command()
             return
+
+        install_pkg_resources(yes=yes)
+        import pkg_resources
+
         for req_file in requirements_files:
             path = os.path.join(os.path.dirname(sys.argv[0]), req_file)
             if not os.path.exists(path):
                 path = os.path.join(os.path.dirname(__file__), req_file)
             with open(path) as requirementsfile:
                 for line in requirementsfile:
-                    if line.startswith('https://'):
-                        # extract name and version from url
-                        wheel = line.split('/')[-1]
-                        name, version, _ = wheel.split('-', 2)
-                        line = f'{name}=={version}'
+                    if not line or line[0] == "#":
+                        continue  # ignore comments
+                    if line.startswith(("https://", "git+https://")):
+                        # extract name and version for url
+                        rest = line.split('/')[-1]
+                        line = ""
+                        if "#egg=" in rest:
+                            # from egg info
+                            rest, egg = rest.split("#egg=", 1)
+                            egg = egg.split(";", 1)[0].rstrip()
+                            if any(compare in egg for compare in ("==", ">=", ">", "<", "<=", "!=")):
+                                warnings.warn(f"Specifying version as #egg={egg} will become unavailable in pip 25.0. "
+                                              "Use name @ url#version instead.", DeprecationWarning)
+                                line = egg
+                        else:
+                            egg = ""
+                        if "@" in rest and not line:
+                            raise ValueError("Can't deduce version from requirement")
+                        elif not line:
+                            # from filename
+                            rest = rest.replace(".zip", "-").replace(".tar.gz", "-")
+                            name, version, _ = rest.split("-", 2)
+                            line = f'{egg or name}=={version}'
+                    elif "@" in line and "#" in line:
+                        # PEP 508 does not allow us to specify a version, so we use custom syntax
+                        # name @ url#version ; marker
+                        name, rest = line.split("@", 1)
+                        version = rest.split("#", 1)[1].split(";", 1)[0].rstrip()
+                        line = f"{name.rstrip()}=={version}"
+                        if ";" in rest:  # keep marker
+                            line += rest[rest.find(";"):]
                     requirements = pkg_resources.parse_requirements(line)
-                    for requirement in requirements:
-                        requirement = str(requirement)
+                    for requirement in map(str, requirements):
                         try:
                             pkg_resources.require(requirement)
                         except pkg_resources.ResolutionError:
                             if not yes:
                                 import traceback
                                 traceback.print_exc()
-                                input(f'Requirement {requirement} is not satisfied, press enter to install it')
+                                confirm(f"Requirement {requirement} is not satisfied, press enter to install it")
                             update_command()
                             return
 

NetUtils.py

@@ -6,7 +6,7 @@ from json import JSONEncoder, JSONDecoder
 
 import websockets
 
-from Utils import Version
+from Utils import ByValue, Version
 
 
 class JSONMessagePart(typing.TypedDict, total=False):
@@ -20,7 +20,7 @@ class JSONMessagePart(typing.TypedDict, total=False):
     flags: int
 
 
-class ClientStatus(enum.IntEnum):
+class ClientStatus(ByValue, enum.IntEnum):
     CLIENT_UNKNOWN = 0
     CLIENT_CONNECTED = 5
     CLIENT_READY = 10
@@ -28,22 +28,22 @@ class ClientStatus(enum.IntEnum):
     CLIENT_GOAL = 30
 
 
-class SlotType(enum.IntFlag):
+class SlotType(ByValue, enum.IntFlag):
     spectator = 0b00
     player = 0b01
     group = 0b10
 
     @property
     def always_goal(self) -> bool:
-        """Mark this slot has having reached its goal instantly."""
+        """Mark this slot as having reached its goal instantly."""
         return self.value != 0b01
 
 
-class Permission(enum.IntFlag):
+class Permission(ByValue, enum.IntFlag):
     disabled = 0b000  # 0, completely disables access
     enabled = 0b001  # 1, allows manual use
     goal = 0b010  # 2, allows manual use after goal completion
-    auto = 0b110  # 6, forces use after goal completion, only works for forfeit
+    auto = 0b110  # 6, forces use after goal completion, only works for release
     auto_enabled = 0b111  # 7, forces use after goal completion, allows manual use any time
 
     @staticmethod
@@ -86,7 +86,7 @@ def _scan_for_TypedTuples(obj: typing.Any) -> typing.Any:
         data = obj._asdict()
         data["class"] = obj.__class__.__name__
         return data
-    if isinstance(obj, (tuple, list, set)):
+    if isinstance(obj, (tuple, list, set, frozenset)):
         return tuple(_scan_for_TypedTuples(o) for o in obj)
     if isinstance(obj, dict):
         return {key: _scan_for_TypedTuples(value) for key, value in obj.items()}
@@ -96,10 +96,11 @@ def _scan_for_TypedTuples(obj: typing.Any) -> typing.Any:
 _encode = JSONEncoder(
     ensure_ascii=False,
     check_circular=False,
+    separators=(',', ':'),
 ).encode
 
 
-def encode(obj):
+def encode(obj: typing.Any) -> str:
     return _encode(_scan_for_TypedTuples(obj))
 
 
@@ -108,7 +109,7 @@ def get_any_version(data: dict) -> Version:
     return Version(int(data["major"]), int(data["minor"]), int(data["build"]))
 
 
-whitelist = {
+allowlist = {
     "NetworkPlayer": NetworkPlayer,
     "NetworkItem": NetworkItem,
     "NetworkSlot": NetworkSlot
@@ -124,7 +125,7 @@ def _object_hook(o: typing.Any) -> typing.Any:
         hook = custom_hooks.get(o.get("class", None), None)
         if hook:
             return hook(o)
-        cls = whitelist.get(o.get("class", None), None)
+        cls = allowlist.get(o.get("class", None), None)
         if cls:
             for key in tuple(o):
                 if key not in cls._fields:
@@ -235,7 +236,7 @@ class JSONtoTextParser(metaclass=HandlerMeta):
             node["color"] = 'cyan'
         elif flags & 0b001:  # advancement
             node["color"] = 'plum'
-        elif flags & 0b010:  # never_exclude
+        elif flags & 0b010:  # useful
             node["color"] = 'slateblue'
         elif flags & 0b100:  # trap
             node["color"] = 'salmon'
@@ -245,7 +246,7 @@ class JSONtoTextParser(metaclass=HandlerMeta):
 
     def _handle_item_id(self, node: JSONMessagePart):
         item_id = int(node["text"])
-        node["text"] = self.ctx.item_name_getter(item_id)
+        node["text"] = self.ctx.item_names[item_id]
         return self._handle_item_name(node)
 
     def _handle_location_name(self, node: JSONMessagePart):
@@ -254,7 +255,7 @@ class JSONtoTextParser(metaclass=HandlerMeta):
 
     def _handle_location_id(self, node: JSONMessagePart):
         item_id = int(node["text"])
-        node["text"] = self.ctx.location_name_getter(item_id)
+        node["text"] = self.ctx.location_names[item_id]
         return self._handle_location_name(node)
 
     def _handle_entrance_name(self, node: JSONMessagePart):
@@ -269,7 +270,7 @@ class RawJSONtoTextParser(JSONtoTextParser):
 
 color_codes = {'reset': 0, 'bold': 1, 'underline': 4, 'black': 30, 'red': 31, 'green': 32, 'yellow': 33, 'blue': 34,
                'magenta': 35, 'cyan': 36, 'white': 37, 'black_bg': 40, 'red_bg': 41, 'green_bg': 42, 'yellow_bg': 43,
-               'blue_bg': 44, 'purple_bg': 45, 'cyan_bg': 46, 'white_bg': 47}
+               'blue_bg': 44, 'magenta_bg': 45, 'cyan_bg': 46, 'white_bg': 47}
 
 
 def color_code(*args):

OoTAdjuster.py

@@ -3,6 +3,7 @@ import argparse
 import logging
 import random
 import os
+import zipfile
 from itertools import chain
 
 from BaseClasses import MultiWorld
@@ -196,7 +197,7 @@ def set_icon(window):
 def adjust(args):
     # Create a fake world and OOTWorld to use as a base
     world = MultiWorld(1)
-    world.slot_seeds = {1: random}
+    world.per_slot_randoms = {1: random}
     ootworld = OOTWorld(world, 1)
     # Set options in the fake OOTWorld
     for name, option in chain(cosmetic_options.items(), sfx_options.items()):
@@ -217,13 +218,18 @@ def adjust(args):
         # Load up the ROM
         rom = Rom(file=args.rom, force_use=True)
         delete_zootdec = True
-    elif os.path.splitext(args.rom)[-1] == '.apz5':
+    elif os.path.splitext(args.rom)[-1] in ['.apz5', '.zpf']:
         # Load vanilla ROM
         rom = Rom(file=args.vanilla_rom, force_use=True)
+        apz5_file = args.rom
+        base_name = os.path.splitext(apz5_file)[0]
         # Patch file
-        apply_patch_file(rom, args.rom)
+        apply_patch_file(rom, apz5_file,
+            sub_file=(os.path.basename(base_name) + '.zpf'
+                if zipfile.is_zipfile(apz5_file)
+                else None))
     else:
-        raise Exception("Invalid file extension; requires .n64, .z64, .apz5")
+        raise Exception("Invalid file extension; requires .n64, .z64, .apz5, .zpf")
     # Call patch_cosmetics
     try:
         patch_cosmetics(ootworld, rom)

OoTClient.py

@@ -3,20 +3,23 @@ import json
 import os
 import multiprocessing
 import subprocess
+import zipfile
 from asyncio import StreamReader, StreamWriter
 
-from CommonClient import CommonContext, server_loop, gui_enabled, console_loop, \
+# CommonClient import first to trigger ModuleUpdater
+from CommonClient import CommonContext, server_loop, gui_enabled, \
     ClientCommandProcessor, logger, get_base_parser
 import Utils
+from Utils import async_start
 from worlds import network_data_package
 from worlds.oot.Rom import Rom, compress_rom_file
 from worlds.oot.N64Patch import apply_patch_file
 from worlds.oot.Utils import data_path
 
 
-CONNECTION_TIMING_OUT_STATUS = "Connection timing out. Please restart your emulator, then restart oot_connector.lua"
-CONNECTION_REFUSED_STATUS = "Connection refused. Please start your emulator and make sure oot_connector.lua is running"
-CONNECTION_RESET_STATUS = "Connection was reset. Please restart your emulator, then restart oot_connector.lua"
+CONNECTION_TIMING_OUT_STATUS = "Connection timing out. Please restart your emulator, then restart connector_oot.lua"
+CONNECTION_REFUSED_STATUS = "Connection refused. Please start your emulator and make sure connector_oot.lua is running"
+CONNECTION_RESET_STATUS = "Connection was reset. Please restart your emulator, then restart connector_oot.lua"
 CONNECTION_TENTATIVE_STATUS = "Initial Connection Made"
 CONNECTION_CONNECTED_STATUS = "Connected"
 CONNECTION_INITIAL_STATUS = "Connection has not been initiated"
@@ -48,7 +51,7 @@ deathlink_sent_this_death: we interacted with the multiworld on this death, wait
 
 oot_loc_name_to_id = network_data_package["games"]["Ocarina of Time"]["location_name_to_id"]
 
-script_version: int = 1
+script_version: int = 3
 
 def get_item_value(ap_id):
     return ap_id - 66000
@@ -68,7 +71,7 @@ class OoTCommandProcessor(ClientCommandProcessor):
         if isinstance(self.ctx, OoTContext):
             self.ctx.deathlink_client_override = True
             self.ctx.deathlink_enabled = not self.ctx.deathlink_enabled
-            asyncio.create_task(self.ctx.update_death_link(self.ctx.deathlink_enabled), name="Update Deathlink")
+            async_start(self.ctx.update_death_link(self.ctx.deathlink_enabled), name="Update Deathlink")
 
 
 class OoTContext(CommonContext):
@@ -83,6 +86,9 @@ class OoTContext(CommonContext):
         self.n64_status = CONNECTION_INITIAL_STATUS
         self.awaiting_rom = False
         self.location_table = {}
+        self.collectible_table = {}
+        self.collectible_override_flags_address = 0
+        self.collectible_offsets = {}
         self.deathlink_enabled = False
         self.deathlink_pending = False
         self.deathlink_sent_this_death = False
@@ -115,6 +121,13 @@ class OoTContext(CommonContext):
         self.ui = OoTManager(self)
         self.ui_task = asyncio.create_task(self.ui.async_run(), name="UI")
 
+    def on_package(self, cmd, args):
+        if cmd == 'Connected':
+            slot_data = args.get('slot_data', None)
+            if slot_data:
+                self.collectible_override_flags_address = slot_data.get('collectible_override_flags', 0)
+                self.collectible_offsets = slot_data.get('collectible_flag_offsets', {})
+
 
 def get_payload(ctx: OoTContext):
     if ctx.deathlink_enabled and ctx.deathlink_pending:
@@ -123,15 +136,32 @@ def get_payload(ctx: OoTContext):
     else:
         trigger_death = False
 
-    return json.dumps({
+    payload = json.dumps({
             "items": [get_item_value(item.item) for item in ctx.items_received],
             "playerNames": [name for (i, name) in ctx.player_names.items() if i != 0],
-            "triggerDeath": trigger_death
+            "triggerDeath": trigger_death,
+            "collectibleOverrides": ctx.collectible_override_flags_address,
+            "collectibleOffsets": ctx.collectible_offsets
         })
+    return payload
 
 
 async def parse_payload(payload: dict, ctx: OoTContext, force: bool):
 
+    # Refuse to do anything if ROM is detected as changed
+    if ctx.auth and payload['playerName'] != ctx.auth:
+        logger.warning("ROM change detected. Disconnecting and reconnecting...")
+        ctx.deathlink_enabled = False
+        ctx.deathlink_client_override = False
+        ctx.finished_game = False
+        ctx.location_table = {}
+        ctx.collectible_table = {}
+        ctx.deathlink_pending = False
+        ctx.deathlink_sent_this_death = False
+        ctx.auth = payload['playerName']
+        await ctx.send_connect()
+        return
+
     # Turn on deathlink if it is on, and if the client hasn't overriden it
     if payload['deathlinkActive'] and not ctx.deathlink_enabled and not ctx.deathlink_client_override:
         await ctx.update_death_link(True)
@@ -146,11 +176,23 @@ async def parse_payload(payload: dict, ctx: OoTContext, force: bool):
         ctx.finished_game = True
 
     # Locations handling
-    if ctx.location_table != payload['locations']:
-        ctx.location_table = payload['locations']
+    locations = payload['locations']
+    collectibles = payload['collectibles']
+
+    # The Lua JSON library serializes an empty table into a list instead of a dict. Verify types for safety:
+    if isinstance(locations, list):
+        locations = {}
+    if isinstance(collectibles, list):
+        collectibles = {}
+
+    if ctx.location_table != locations or ctx.collectible_table != collectibles:
+        ctx.location_table = locations
+        ctx.collectible_table = collectibles
+        locs1 = [oot_loc_name_to_id[loc] for loc, b in ctx.location_table.items() if b]
+        locs2 = [int(loc) for loc, b in ctx.collectible_table.items() if b]
         await ctx.send_msgs([{
             "cmd": "LocationChecks",
-            "locations": [oot_loc_name_to_id[loc] for loc in ctx.location_table if ctx.location_table[loc]]
+            "locations": locs1 + locs2
         }])
 
     # Deathlink handling
@@ -176,20 +218,13 @@ async def n64_sync_task(ctx: OoTContext):
             try:
                 await asyncio.wait_for(writer.drain(), timeout=1.5)
                 try:
-                    # Data will return a dict with up to six fields:
-                    # 1. str: player name (always)
-                    # 2. int: script version (always)
-                    # 3. bool: deathlink active (always)
-                    # 4. dict[str, bool]: checked locations
-                    # 5. bool: whether Link is currently at 0 HP
-                    # 6. bool: whether the game currently registers as complete
                     data = await asyncio.wait_for(reader.readline(), timeout=10)
                     data_decoded = json.loads(data.decode())
                     reported_version = data_decoded.get('scriptVersion', 0)
-                    if reported_version == script_version:
+                    if reported_version >= script_version:
                         if ctx.game is not None and 'locations' in data_decoded:
                             # Not just a keep alive ping, parse
-                            asyncio.create_task(parse_payload(data_decoded, ctx, False))
+                            async_start(parse_payload(data_decoded, ctx, False))
                         if not ctx.auth:
                             ctx.auth = data_decoded['playerName']
                             if ctx.awaiting_rom:
@@ -255,17 +290,29 @@ async def run_game(romfile):
 
 
 async def patch_and_run_game(apz5_file):
+    apz5_file = os.path.abspath(apz5_file)
     base_name = os.path.splitext(apz5_file)[0]
     decomp_path = base_name + '-decomp.z64'
     comp_path = base_name + '.z64'
     # Load vanilla ROM, patch file, compress ROM
-    rom = Rom(Utils.local_path(Utils.get_options()["oot_options"]["rom_file"]))
-    apply_patch_file(rom, apz5_file)
+    rom_file_name = Utils.get_options()["oot_options"]["rom_file"]
+    if not os.path.exists(rom_file_name):
+        rom_file_name = Utils.user_path(rom_file_name)
+    rom = Rom(rom_file_name)
+
+    sub_file = None
+    if zipfile.is_zipfile(apz5_file):
+        for name in zipfile.ZipFile(apz5_file).namelist():
+            if name.endswith('.zpf'):
+                sub_file = name
+                break
+
+    apply_patch_file(rom, apz5_file, sub_file=sub_file)
     rom.write_to_file(decomp_path)
     os.chdir(data_path("Compress"))
     compress_rom_file(decomp_path, comp_path)
     os.remove(decomp_path)
-    asyncio.create_task(run_game(comp_path))
+    async_start(run_game(comp_path))
 
 
 if __name__ == '__main__':
@@ -281,7 +328,7 @@ if __name__ == '__main__':
 
         if args.apz5_file:
             logger.info("APZ5 file supplied, beginning patching process...")
-            asyncio.create_task(patch_and_run_game(args.apz5_file))
+            async_start(patch_and_run_game(args.apz5_file))
 
         ctx = OoTContext(args.connect, args.password)
         ctx.server_task = asyncio.create_task(server_loop(ctx), name="Server Loop")

Options.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 import abc
+import logging
+from copy import deepcopy
 import math
 import numbers
 import typing
@@ -8,6 +10,11 @@ import random
 from schema import Schema, And, Or, Optional
 from Utils import get_fuzzy_results
 
+if typing.TYPE_CHECKING:
+    from BaseClasses import PlandoOptions
+    from worlds.AutoWorld import World
+    import pathlib
+
 
 class AssembleOptions(abc.ABCMeta):
     def __new__(mcs, name, bases, attrs):
@@ -26,10 +33,30 @@ class AssembleOptions(abc.ABCMeta):
 
         attrs["name_lookup"].update({option_id: name for name, option_id in new_options.items()})
         options.update(new_options)
-
         # apply aliases, without name_lookup
-        options.update({name[6:].lower(): option_id for name, option_id in attrs.items() if
-                        name.startswith("alias_")})
+        aliases = {name[6:].lower(): option_id for name, option_id in attrs.items() if
+                   name.startswith("alias_")}
+
+        assert "random" not in aliases, "Choice option 'random' cannot be manually assigned."
+
+        # auto-alias Off and On being parsed as True and False
+        if "off" in options:
+            options["false"] = options["off"]
+        if "on" in options:
+            options["true"] = options["on"]
+
+        options.update(aliases)
+
+        if "verify" not in attrs:
+            # not overridden by class -> look up bases
+            verifiers = [f for f in (getattr(base, "verify", None) for base in bases) if f]
+            if len(verifiers) > 1:  # verify multiple bases/mixins
+                def verify(self, *args, **kwargs) -> None:
+                    for f in verifiers:
+                        f(self, *args, **kwargs)
+                attrs["verify"] = verify
+            else:
+                assert verifiers, "class Option is supposed to implement def verify"
 
         # auto-validate schema on __init__
         if "schema" in attrs.keys():
@@ -74,11 +101,11 @@ class Option(typing.Generic[T], metaclass=AssembleOptions):
     supports_weighting = True
 
     # filled by AssembleOptions:
-    name_lookup: typing.Dict[int, str]
+    name_lookup: typing.Dict[T, str]
     options: typing.Dict[str, int]
 
     def __repr__(self) -> str:
-        return f"{self.__class__.__name__}({self.get_current_option_name()})"
+        return f"{self.__class__.__name__}({self.current_option_name})"
 
     def __hash__(self) -> int:
         return hash(self.value)
@@ -88,7 +115,14 @@ class Option(typing.Generic[T], metaclass=AssembleOptions):
         return self.name_lookup[self.value]
 
     def get_current_option_name(self) -> str:
-        """For display purposes."""
+        """Deprecated. use current_option_name instead. TODO remove around 0.4"""
+        logging.warning(DeprecationWarning(f"get_current_option_name for {self.__class__.__name__} is deprecated."
+                                           f" use current_option_name instead. Worlds should use {self}.current_key"))
+        return self.current_option_name
+
+    @property
+    def current_option_name(self) -> str:
+        """For display purposes. Worlds should be using current_key."""
         return self.get_option_name(self.value)
 
     @classmethod
@@ -105,11 +139,45 @@ class Option(typing.Generic[T], metaclass=AssembleOptions):
         return bool(self.value)
 
     @classmethod
+    @abc.abstractmethod
     def from_any(cls, data: typing.Any) -> Option[T]:
-        raise NotImplementedError
+        ...
+
+    if typing.TYPE_CHECKING:
+        def verify(self, world: typing.Type[World], player_name: str, plando_options: PlandoOptions) -> None:
+            pass
+    else:
+        def verify(self, *args, **kwargs) -> None:
+            pass
 
 
-class NumericOption(Option[int], numbers.Integral):
+class FreeText(Option[str]):
+    """Text option that allows users to enter strings.
+    Needs to be validated by the world or option definition."""
+
+    def __init__(self, value: str):
+        assert isinstance(value, str), "value of FreeText must be a string"
+        self.value = value
+
+    @property
+    def current_key(self) -> str:
+        return self.value
+
+    @classmethod
+    def from_text(cls, text: str) -> FreeText:
+        return cls(text)
+
+    @classmethod
+    def from_any(cls, data: typing.Any) -> FreeText:
+        return cls.from_text(str(data))
+
+    @classmethod
+    def get_option_name(cls, value: str) -> str:
+        return value
+
+
+class NumericOption(Option[int], numbers.Integral, abc.ABC):
+    default = 0
     # note: some of the `typing.Any`` here is a result of unresolved issue in python standards
     # `int` is not a `numbers.Integral` according to the official typestubs
     # (even though isinstance(5, numbers.Integral) == True)
@@ -294,7 +362,7 @@ class Toggle(NumericOption):
         if type(data) == str:
             return cls.from_text(data)
         else:
-            return cls(data)
+            return cls(int(data))
 
     @classmethod
     def get_option_name(cls, value):
@@ -364,6 +432,169 @@ class Choice(NumericOption):
     __hash__ = Option.__hash__  # see https://docs.python.org/3/reference/datamodel.html#object.__hash__
 
 
+class TextChoice(Choice):
+    """Allows custom string input and offers choices. Choices will resolve to int and text will resolve to string"""
+    value: typing.Union[str, int]
+
+    def __init__(self, value: typing.Union[str, int]):
+        assert isinstance(value, str) or isinstance(value, int), \
+            f"{value} is not a valid option for {self.__class__.__name__}"
+        self.value = value
+
+    @property
+    def current_key(self) -> str:
+        if isinstance(self.value, str):
+            return self.value
+        return super().current_key
+
+    @classmethod
+    def from_text(cls, text: str) -> TextChoice:
+        if text.lower() == "random":  # chooses a random defined option but won't use any free text options
+            return cls(random.choice(list(cls.name_lookup)))
+        for option_name, value in cls.options.items():
+            if option_name.lower() == text.lower():
+                return cls(value)
+        return cls(text)
+
+    @classmethod
+    def get_option_name(cls, value: T) -> str:
+        if isinstance(value, str):
+            return value
+        return super().get_option_name(value)
+
+    def __eq__(self, other: typing.Any):
+        if isinstance(other, self.__class__):
+            return other.value == self.value
+        elif isinstance(other, str):
+            if other in self.options:
+                return other == self.current_key
+            return other == self.value
+        elif isinstance(other, int):
+            assert other in self.name_lookup, f"compared against an int that could never be equal. {self} == {other}"
+            return other == self.value
+        elif isinstance(other, bool):
+            return other == bool(self.value)
+        else:
+            raise TypeError(f"Can't compare {self.__class__.__name__} with {other.__class__.__name__}")
+
+
+class BossMeta(AssembleOptions):
+    def __new__(mcs, name, bases, attrs):
+        if name != "PlandoBosses":
+            assert "bosses" in attrs, f"Please define valid bosses for {name}"
+            attrs["bosses"] = frozenset((boss.lower() for boss in attrs["bosses"]))
+            assert "locations" in attrs, f"Please define valid locations for {name}"
+            attrs["locations"] = frozenset((location.lower() for location in attrs["locations"]))
+        cls = super().__new__(mcs, name, bases, attrs)
+        assert not cls.duplicate_bosses or "singularity" in cls.options, f"Please define option_singularity for {name}"
+        return cls
+
+
+class PlandoBosses(TextChoice, metaclass=BossMeta):
+    """Generic boss shuffle option that supports plando. Format expected is
+    'location1-boss1;location2-boss2;shuffle_mode'.
+    If shuffle_mode is not provided in the string, this will be the default shuffle mode. Must override can_place_boss,
+    which passes a plando boss and location. Check if the placement is valid for your game here."""
+    bosses: typing.ClassVar[typing.Union[typing.Set[str], typing.FrozenSet[str]]]
+    locations: typing.ClassVar[typing.Union[typing.Set[str], typing.FrozenSet[str]]]
+
+    duplicate_bosses: bool = False
+
+    @classmethod
+    def from_text(cls, text: str):
+        # set all of our text to lower case for name checking
+        text = text.lower()
+        if text == "random":
+            return cls(random.choice(list(cls.options.values())))
+        for option_name, value in cls.options.items():
+            if option_name == text:
+                return cls(value)
+        options = text.split(";")
+
+        # since plando exists in the option verify the plando values given are valid
+        cls.validate_plando_bosses(options)
+        return cls.get_shuffle_mode(options)
+
+    @classmethod
+    def get_shuffle_mode(cls, option_list: typing.List[str]):
+        # find out what mode of boss shuffle we should use for placing bosses after plando
+        # and add as a string to look nice in the spoiler
+        if "random" in option_list:
+            shuffle = random.choice(list(cls.options))
+            option_list.remove("random")
+            options = ";".join(option_list) + f";{shuffle}"
+            boss_class = cls(options)
+        else:
+            for option in option_list:
+                if option in cls.options:
+                    options = ";".join(option_list)
+                    break
+            else:
+                if cls.duplicate_bosses and len(option_list) == 1:
+                    if cls.valid_boss_name(option_list[0]):
+                        # this doesn't exist in this class but it's a forced option for classes where this is called
+                        options = option_list[0] + ";singularity"
+                    else:
+                        options = option_list[0] + f";{cls.name_lookup[cls.default]}"
+                else:
+                    options = ";".join(option_list) + f";{cls.name_lookup[cls.default]}"
+            boss_class = cls(options)
+        return boss_class
+
+    @classmethod
+    def validate_plando_bosses(cls, options: typing.List[str]) -> None:
+        used_locations = []
+        used_bosses = []
+        for option in options:
+            # check if a shuffle mode was provided in the incorrect location
+            if option == "random" or option in cls.options:
+                if option != options[-1]:
+                    raise ValueError(f"{option} option must be at the end of the boss_shuffle options!")
+            elif "-" in option:
+                location, boss = option.split("-")
+                if location in used_locations:
+                    raise ValueError(f"Duplicate Boss Location {location} not allowed.")
+                if not cls.duplicate_bosses and boss in used_bosses:
+                    raise ValueError(f"Duplicate Boss {boss} not allowed.")
+                used_locations.append(location)
+                used_bosses.append(boss)
+                if not cls.valid_boss_name(boss):
+                    raise ValueError(f"{boss.title()} is not a valid boss name.")
+                if not cls.valid_location_name(location):
+                    raise ValueError(f"{location.title()} is not a valid boss location name.")
+                if not cls.can_place_boss(boss, location):
+                    raise ValueError(f"{location.title()} is not a valid location for {boss.title()} to be placed.")
+            else:
+                if cls.duplicate_bosses:
+                    if not cls.valid_boss_name(option):
+                        raise ValueError(f"{option} is not a valid boss name.")
+                else:
+                    raise ValueError(f"{option.title()} is not formatted correctly.")
+
+    @classmethod
+    def can_place_boss(cls, boss: str, location: str) -> bool:
+        raise NotImplementedError
+
+    @classmethod
+    def valid_boss_name(cls, value: str) -> bool:
+        return value in cls.bosses
+
+    @classmethod
+    def valid_location_name(cls, value: str) -> bool:
+        return value in cls.locations
+
+    def verify(self, world: typing.Type[World], player_name: str, plando_options: "PlandoOptions") -> None:
+        if isinstance(self.value, int):
+            return
+        from BaseClasses import PlandoOptions
+        if not(PlandoOptions.bosses & plando_options):
+            # plando is disabled but plando options were given so pull the option and change it to an int
+            option = self.value.split(";")[-1]
+            self.value = self.options[option]
+            logging.warning(f"The plando bosses module is turned off, so {self.name_lookup[self.value].title()} "
+                            f"boss shuffle will be used for player {player_name}.")
+
+
 class Range(NumericOption):
     range_start = 0
     range_end = 1
@@ -379,37 +610,9 @@ class Range(NumericOption):
     def from_text(cls, text: str) -> Range:
         text = text.lower()
         if text.startswith("random"):
-            if text == "random-low":
-                return cls(int(round(random.triangular(cls.range_start, cls.range_end, cls.range_start), 0)))
-            elif text == "random-high":
-                return cls(int(round(random.triangular(cls.range_start, cls.range_end, cls.range_end), 0)))
-            elif text == "random-middle":
-                return cls(int(round(random.triangular(cls.range_start, cls.range_end), 0)))
-            elif text.startswith("random-range-"):
-                textsplit = text.split("-")
-                try:
-                    random_range = [int(textsplit[len(textsplit) - 2]), int(textsplit[len(textsplit) - 1])]
-                except ValueError:
-                    raise ValueError(f"Invalid random range {text} for option {cls.__name__}")
-                random_range.sort()
-                if random_range[0] < cls.range_start or random_range[1] > cls.range_end:
-                    raise Exception(
-                        f"{random_range[0]}-{random_range[1]} is outside allowed range "
-                        f"{cls.range_start}-{cls.range_end} for option {cls.__name__}")
-                if text.startswith("random-range-low"):
-                    return cls(int(round(random.triangular(random_range[0], random_range[1], random_range[0]))))
-                elif text.startswith("random-range-middle"):
-                    return cls(int(round(random.triangular(random_range[0], random_range[1]))))
-                elif text.startswith("random-range-high"):
-                    return cls(int(round(random.triangular(random_range[0], random_range[1], random_range[1]))))
-                else:
-                    return cls(int(round(random.randint(random_range[0], random_range[1]))))
-            elif text == "random":
-                return cls(random.randint(cls.range_start, cls.range_end))
-            else:
-                raise Exception(f"random text \"{text}\" did not resolve to a recognized pattern. Acceptable values are: random, random-high, random-middle, random-low, random-range-low-<min>-<max>, random-range-middle-<min>-<max>, random-range-high-<min>-<max>, or random-range-<min>-<max>.")
+            return cls.weighted_range(text)
         elif text == "default" and hasattr(cls, "default"):
-            return cls(cls.default)
+            return cls.from_any(cls.default)
         elif text == "high":
             return cls(cls.range_end)
         elif text == "low":
@@ -420,11 +623,50 @@ class Range(NumericOption):
                 and text in ("true", "false"):
             # these are the conditions where "true" and "false" make sense
             if text == "true":
-                return cls(cls.default)
+                return cls.from_any(cls.default)
             else:  # "false"
                 return cls(0)
         return cls(int(text))
 
+    @classmethod
+    def weighted_range(cls, text) -> Range:
+        if text == "random-low":
+            return cls(cls.triangular(cls.range_start, cls.range_end, cls.range_start))
+        elif text == "random-high":
+            return cls(cls.triangular(cls.range_start, cls.range_end, cls.range_end))
+        elif text == "random-middle":
+            return cls(cls.triangular(cls.range_start, cls.range_end))
+        elif text.startswith("random-range-"):
+            return cls.custom_range(text)
+        elif text == "random":
+            return cls(random.randint(cls.range_start, cls.range_end))
+        else:
+            raise Exception(f"random text \"{text}\" did not resolve to a recognized pattern. "
+                            f"Acceptable values are: random, random-high, random-middle, random-low, "
+                            f"random-range-low-<min>-<max>, random-range-middle-<min>-<max>, "
+                            f"random-range-high-<min>-<max>, or random-range-<min>-<max>.")
+
+    @classmethod
+    def custom_range(cls, text) -> Range:
+        textsplit = text.split("-")
+        try:
+            random_range = [int(textsplit[len(textsplit) - 2]), int(textsplit[len(textsplit) - 1])]
+        except ValueError:
+            raise ValueError(f"Invalid random range {text} for option {cls.__name__}")
+        random_range.sort()
+        if random_range[0] < cls.range_start or random_range[1] > cls.range_end:
+            raise Exception(
+                f"{random_range[0]}-{random_range[1]} is outside allowed range "
+                f"{cls.range_start}-{cls.range_end} for option {cls.__name__}")
+        if text.startswith("random-range-low"):
+            return cls(cls.triangular(random_range[0], random_range[1], random_range[0]))
+        elif text.startswith("random-range-middle"):
+            return cls(cls.triangular(random_range[0], random_range[1]))
+        elif text.startswith("random-range-high"):
+            return cls(cls.triangular(random_range[0], random_range[1], random_range[1]))
+        else:
+            return cls(random.randint(random_range[0], random_range[1]))
+
     @classmethod
     def from_any(cls, data: typing.Any) -> Range:
         if type(data) == int:
@@ -438,9 +680,52 @@ class Range(NumericOption):
     def __str__(self) -> str:
         return str(self.value)
 
+    @staticmethod
+    def triangular(lower: int, end: int, tri: typing.Optional[int] = None) -> int:
+        return int(round(random.triangular(lower, end, tri), 0))
 
-class VerifyKeys:
-    valid_keys = frozenset()
+
+class SpecialRange(Range):
+    special_range_cutoff = 0
+    special_range_names: typing.Dict[str, int] = {}
+    """Special Range names have to be all lowercase as matching is done with text.lower()"""
+
+    @classmethod
+    def from_text(cls, text: str) -> Range:
+        text = text.lower()
+        if text in cls.special_range_names:
+            return cls(cls.special_range_names[text])
+        return super().from_text(text)
+
+    @classmethod
+    def weighted_range(cls, text) -> Range:
+        if text == "random-low":
+            return cls(cls.triangular(cls.special_range_cutoff, cls.range_end, cls.special_range_cutoff))
+        elif text == "random-high":
+            return cls(cls.triangular(cls.special_range_cutoff, cls.range_end, cls.range_end))
+        elif text == "random-middle":
+            return cls(cls.triangular(cls.special_range_cutoff, cls.range_end))
+        elif text.startswith("random-range-"):
+            return cls.custom_range(text)
+        elif text == "random":
+            return cls(random.randint(cls.special_range_cutoff, cls.range_end))
+        else:
+            raise Exception(f"random text \"{text}\" did not resolve to a recognized pattern. "
+                            f"Acceptable values are: random, random-high, random-middle, random-low, "
+                            f"random-range-low-<min>-<max>, random-range-middle-<min>-<max>, "
+                            f"random-range-high-<min>-<max>, or random-range-<min>-<max>.")
+
+
+class FreezeValidKeys(AssembleOptions):
+    def __new__(mcs, name, bases, attrs):
+        if "valid_keys" in attrs:
+            attrs["_valid_keys"] = frozenset(attrs["valid_keys"])
+        return super(FreezeValidKeys, mcs).__new__(mcs, name, bases, attrs)
+
+
+class VerifyKeys(metaclass=FreezeValidKeys):
+    valid_keys: typing.Iterable = []
+    _valid_keys: frozenset  # gets created by AssembleOptions from valid_keys
     valid_keys_casefold: bool = False
     convert_name_groups: bool = False
     verify_item_name: bool = False
@@ -448,21 +733,26 @@ class VerifyKeys:
     value: typing.Any
 
     @classmethod
-    def verify_keys(cls, data):
+    def verify_keys(cls, data: typing.List[str]):
         if cls.valid_keys:
             data = set(data)
             dataset = set(word.casefold() for word in data) if cls.valid_keys_casefold else set(data)
-            extra = dataset - cls.valid_keys
+            extra = dataset - cls._valid_keys
             if extra:
                 raise Exception(f"Found unexpected key {', '.join(extra)} in {cls}. "
-                                f"Allowed keys: {cls.valid_keys}.")
+                                f"Allowed keys: {cls._valid_keys}.")
 
-    def verify(self, world):
+    def verify(self, world: typing.Type[World], player_name: str, plando_options: "PlandoOptions") -> None:
         if self.convert_name_groups and self.verify_item_name:
             new_value = type(self.value)()  # empty container of whatever value is
             for item_name in self.value:
                 new_value |= world.item_name_groups.get(item_name, {item_name})
             self.value = new_value
+        elif self.convert_name_groups and self.verify_location_name:
+            new_value = type(self.value)()
+            for loc_name in self.value:
+                new_value |= world.location_name_groups.get(loc_name, {loc_name})
+            self.value = new_value
         if self.verify_item_name:
             for item_name in self.value:
                 if item_name not in world.item_names:
@@ -480,11 +770,11 @@ class VerifyKeys:
 
 
 class OptionDict(Option[typing.Dict[str, typing.Any]], VerifyKeys):
-    default = {}
+    default: typing.Dict[str, typing.Any] = {}
     supports_weighting = False
 
     def __init__(self, value: typing.Dict[str, typing.Any]):
-        self.value = value
+        self.value = deepcopy(value)
 
     @classmethod
     def from_any(cls, data: typing.Dict[str, typing.Any]) -> OptionDict:
@@ -511,11 +801,15 @@ class ItemDict(OptionDict):
 
 
 class OptionList(Option[typing.List[typing.Any]], VerifyKeys):
-    default = []
+    # Supports duplicate entries and ordering.
+    # If only unique entries are needed and input order of elements does not matter, OptionSet should be used instead.
+    # Not a docstring so it doesn't get grabbed by the options system.
+
+    default: typing.List[typing.Any] = []
     supports_weighting = False
 
     def __init__(self, value: typing.List[typing.Any]):
-        self.value = value or []
+        self.value = deepcopy(value)
         super(OptionList, self).__init__()
 
     @classmethod
@@ -537,11 +831,11 @@ class OptionList(Option[typing.List[typing.Any]], VerifyKeys):
 
 
 class OptionSet(Option[typing.Set[str]], VerifyKeys):
-    default = frozenset()
+    default: typing.Union[typing.Set[str], typing.FrozenSet[str]] = frozenset()
     supports_weighting = False
 
-    def __init__(self, value: typing.Union[typing.Set[str, typing.Any], typing.List[str, typing.Any]]):
-        self.value = set(value)
+    def __init__(self, value: typing.Iterable[str]):
+        self.value = set(deepcopy(value))
         super(OptionSet, self).__init__()
 
     @classmethod
@@ -550,10 +844,7 @@ class OptionSet(Option[typing.Set[str]], VerifyKeys):
 
     @classmethod
     def from_any(cls, data: typing.Any):
-        if type(data) == list:
-            cls.verify_keys(data)
-            return cls(data)
-        elif type(data) == set:
+        if isinstance(data, (list, set, frozenset)):
             cls.verify_keys(data)
             return cls(data)
         return cls.from_text(str(data))
@@ -565,7 +856,9 @@ class OptionSet(Option[typing.Set[str]], VerifyKeys):
         return item in self.value
 
 
-local_objective = Toggle  # local triforce pieces, local dungeon prizes etc.
+class ItemSet(OptionSet):
+    verify_item_name = True
+    convert_name_groups = True
 
 
 class Accessibility(Choice):
@@ -581,13 +874,18 @@ class Accessibility(Choice):
     default = 1
 
 
-class ProgressionBalancing(Range):
+class ProgressionBalancing(SpecialRange):
     """A system that can move progression earlier, to try and prevent the player from getting stuck and bored early.
-    [0-99, default 50] A lower setting means more getting stuck. A higher setting means less getting stuck."""
+    A lower setting means more getting stuck. A higher setting means less getting stuck."""
     default = 50
     range_start = 0
     range_end = 99
     display_name = "Progression Balancing"
+    special_range_names = {
+        "disabled": 0,
+        "normal": 50,
+        "extreme": 99,
+    }
 
 
 common_options = {
@@ -596,11 +894,6 @@ common_options = {
 }
 
 
-class ItemSet(OptionSet):
-    verify_item_name = True
-    convert_name_groups = True
-
-
 class LocalItems(ItemSet):
     """Forces these items to be in their native world."""
     display_name = "Local Items"
@@ -617,27 +910,36 @@ class StartInventory(ItemDict):
     display_name = "Start Inventory"
 
 
+class StartInventoryPool(StartInventory):
+    """Start with these items and don't place them in the world.
+    The game decides what the replacement items will be."""
+    verify_item_name = True
+    display_name = "Start Inventory from Pool"
+
+
 class StartHints(ItemSet):
     """Start with these item's locations prefilled into the !hint command."""
     display_name = "Start Hints"
 
 
-class StartLocationHints(OptionSet):
+class LocationSet(OptionSet):
+    verify_location_name = True
+    convert_name_groups = True
+
+
+class StartLocationHints(LocationSet):
     """Start with these locations and their item prefilled into the !hint command"""
     display_name = "Start Location Hints"
-    verify_location_name = True
 
 
-class ExcludeLocations(OptionSet):
+class ExcludeLocations(LocationSet):
     """Prevent these locations from having an important item"""
     display_name = "Excluded Locations"
-    verify_location_name = True
 
 
-class PriorityLocations(OptionSet):
+class PriorityLocations(LocationSet):
     """Prevent these locations from having an unimportant item"""
     display_name = "Priority Locations"
-    verify_location_name = True
 
 
 class DeathLink(Toggle):
@@ -655,7 +957,8 @@ class ItemLinks(OptionList):
             Optional("exclude"): [And(str, len)],
             "replacement_item": Or(And(str, len), None),
             Optional("local_items"): [And(str, len)],
-            Optional("non_local_items"): [And(str, len)]
+            Optional("non_local_items"): [And(str, len)],
+            Optional("link_replacement"): Or(None, bool),
         }
     ])
 
@@ -677,8 +980,9 @@ class ItemLinks(OptionList):
                 pool |= {item_name}
         return pool
 
-    def verify(self, world):
-        super(ItemLinks, self).verify(world)
+    def verify(self, world: typing.Type[World], player_name: str, plando_options: "PlandoOptions") -> None:
+        link: dict
+        super(ItemLinks, self).verify(world, player_name, plando_options)
         existing_links = set()
         for link in self.value:
             if link["name"] in existing_links:
@@ -702,9 +1006,9 @@ class ItemLinks(OptionList):
 
             intersection = local_items.intersection(non_local_items)
             if intersection:
-                raise Exception(f"item_link {link['name']} has {intersection} items in both its local_items and non_local_items pool.")
-
-
+                raise Exception(f"item_link {link['name']} has {intersection} "
+                                f"items in both its local_items and non_local_items pool.")
+            link.setdefault("link_replacement", None)
 
 
 per_game_common_options = {
@@ -719,6 +1023,64 @@ per_game_common_options = {
     "item_links": ItemLinks
 }
 
+
+def generate_yaml_templates(target_folder: typing.Union[str, "pathlib.Path"], generate_hidden: bool = True):
+    import os
+
+    import yaml
+    from jinja2 import Template
+
+    from worlds import AutoWorldRegister
+    from Utils import local_path, __version__
+
+    full_path: str
+
+    os.makedirs(target_folder, exist_ok=True)
+
+    # clean out old
+    for file in os.listdir(target_folder):
+        full_path = os.path.join(target_folder, file)
+        if os.path.isfile(full_path) and full_path.endswith(".yaml"):
+            os.unlink(full_path)
+
+    def dictify_range(option: typing.Union[Range, SpecialRange]):
+        data = {option.default: 50}
+        for sub_option in ["random", "random-low", "random-high"]:
+            if sub_option != option.default:
+                data[sub_option] = 0
+
+        notes = {}
+        for name, number in getattr(option, "special_range_names", {}).items():
+            notes[name] = f"equivalent to {number}"
+            if number in data:
+                data[name] = data[number]
+                del data[number]
+            else:
+                data[name] = 0
+
+        return data, notes
+
+    for game_name, world in AutoWorldRegister.world_types.items():
+        if not world.hidden or generate_hidden:
+            all_options: typing.Dict[str, AssembleOptions] = {
+                **per_game_common_options,
+                **world.option_definitions
+            }
+
+            with open(local_path("data", "options.yaml")) as f:
+                file_data = f.read()
+            res = Template(file_data).render(
+                options=all_options,
+                __version__=__version__, game=game_name, yaml_dump=yaml.dump,
+                dictify_range=dictify_range,
+            )
+
+            del file_data
+
+            with open(os.path.join(target_folder, game_name + ".yaml"), "w", encoding="utf-8-sig") as f:
+                f.write(res)
+
+
 if __name__ == "__main__":
 
     from worlds.alttp.Options import Logic

Patch.py

@@ -1,256 +1,23 @@
 from __future__ import annotations
 
-import shutil
-import json
-import bsdiff4
-import yaml
 import os
-import lzma
-import threading
-import concurrent.futures
-import zipfile
 import sys
-from typing import Tuple, Optional, Dict, Any, Union, BinaryIO
+from typing import Tuple, Optional, TypedDict
 
-import ModuleUpdate
-ModuleUpdate.update()
+if __name__ == "__main__":
+    import ModuleUpdate
+    ModuleUpdate.update()
 
-import Utils
-
-current_patch_version = 4
+from worlds.Files import AutoPatchRegister, APDeltaPatch
 
 
-class AutoPatchRegister(type):
-    patch_types: Dict[str, APDeltaPatch] = {}
-    file_endings: Dict[str, APDeltaPatch] = {}
-
-    def __new__(cls, name: str, bases, dct: Dict[str, Any]):
-        # construct class
-        new_class = super().__new__(cls, name, bases, dct)
-        if "game" in dct:
-            AutoPatchRegister.patch_types[dct["game"]] = new_class
-            if not dct["patch_file_ending"]:
-                raise Exception(f"Need an expected file ending for {name}")
-            AutoPatchRegister.file_endings[dct["patch_file_ending"]] = new_class
-        return new_class
-
-    @staticmethod
-    def get_handler(file: str) -> Optional[type(APDeltaPatch)]:
-        for file_ending, handler in AutoPatchRegister.file_endings.items():
-            if file.endswith(file_ending):
-                return handler
-
-
-class APContainer:
-    """A zipfile containing at least archipelago.json"""
-    version: int = current_patch_version
-    compression_level: int = 9
-    compression_method: int = zipfile.ZIP_DEFLATED
-    game: Optional[str] = None
-
-    # instance attributes:
-    path: Optional[str]
+class RomMeta(TypedDict):
+    server: str
     player: Optional[int]
     player_name: str
-    server: str
-
-    def __init__(self, path: Optional[str] = None, player: Optional[int] = None,
-                 player_name: str = "", server: str = ""):
-        self.path = path
-        self.player = player
-        self.player_name = player_name
-        self.server = server
-
-    def write(self, file: Optional[Union[str, BinaryIO]] = None):
-        if not self.path and not file:
-            raise FileNotFoundError(f"Cannot write {self.__class__.__name__} due to no path provided.")
-        with zipfile.ZipFile(file if file else self.path, "w", self.compression_method, True, self.compression_level) \
-                as zf:
-            if file:
-                self.path = zf.filename
-            self.write_contents(zf)
-
-    def write_contents(self, opened_zipfile: zipfile.ZipFile):
-        manifest = self.get_manifest()
-        try:
-            manifest = json.dumps(manifest)
-        except Exception as e:
-            raise Exception(f"Manifest {manifest} did not convert to json.") from e
-        else:
-            opened_zipfile.writestr("archipelago.json", manifest)
-
-    def read(self, file: Optional[Union[str, BinaryIO]] = None):
-        """Read data into patch object. file can be file-like, such as an outer zip file's stream."""
-        if not self.path and not file:
-            raise FileNotFoundError(f"Cannot read {self.__class__.__name__} due to no path provided.")
-        with zipfile.ZipFile(file if file else self.path, "r") as zf:
-            if file:
-                self.path = zf.filename
-            self.read_contents(zf)
-
-    def read_contents(self, opened_zipfile: zipfile.ZipFile):
-        with opened_zipfile.open("archipelago.json", "r") as f:
-            manifest = json.load(f)
-        if manifest["compatible_version"] > self.version:
-            raise Exception(f"File (version: {manifest['compatible_version']}) too new "
-                            f"for this handler (version: {self.version})")
-        self.player = manifest["player"]
-        self.server = manifest["server"]
-        self.player_name = manifest["player_name"]
-
-    def get_manifest(self) -> dict:
-        return {
-            "server": self.server,  # allow immediate connection to server in multiworld. Empty string otherwise
-            "player": self.player,
-            "player_name": self.player_name,
-            "game": self.game,
-            # minimum version of patch system expected for patching to be successful
-            "compatible_version": 4,
-            "version": current_patch_version,
-        }
 
 
-class APDeltaPatch(APContainer, metaclass=AutoPatchRegister):
-    """An APContainer that additionally has delta.bsdiff4
-    containing a delta patch to get the desired file, often a rom."""
-
-    hash = Optional[str]  # base checksum of source file
-    patch_file_ending: str = ""
-    delta: Optional[bytes] = None
-    result_file_ending: str = ".sfc"
-    source_data: bytes
-
-    def __init__(self, *args, patched_path: str = "", **kwargs):
-        self.patched_path = patched_path
-        super(APDeltaPatch, self).__init__(*args, **kwargs)
-
-    def get_manifest(self) -> dict:
-        manifest = super(APDeltaPatch, self).get_manifest()
-        manifest["base_checksum"] = self.hash
-        manifest["result_file_ending"] = self.result_file_ending
-        return manifest
-
-    @classmethod
-    def get_source_data(cls) -> bytes:
-        """Get Base data"""
-        raise NotImplementedError()
-
-    @classmethod
-    def get_source_data_with_cache(cls) -> bytes:
-        if not hasattr(cls, "source_data"):
-            cls.source_data = cls.get_source_data()
-        return cls.source_data
-
-    def write_contents(self, opened_zipfile: zipfile.ZipFile):
-        super(APDeltaPatch, self).write_contents(opened_zipfile)
-        # write Delta
-        opened_zipfile.writestr("delta.bsdiff4",
-                                bsdiff4.diff(self.get_source_data_with_cache(), open(self.patched_path, "rb").read()),
-                                compress_type=zipfile.ZIP_STORED)  # bsdiff4 is a format with integrated compression
-
-    def read_contents(self, opened_zipfile: zipfile.ZipFile):
-        super(APDeltaPatch, self).read_contents(opened_zipfile)
-        self.delta = opened_zipfile.read("delta.bsdiff4")
-
-    def patch(self, target: str):
-        """Base + Delta -> Patched"""
-        if not self.delta:
-            self.read()
-        result = bsdiff4.patch(self.get_source_data_with_cache(), self.delta)
-        with open(target, "wb") as f:
-            f.write(result)
-
-
-# legacy patch handling follows:
-GAME_ALTTP = "A Link to the Past"
-GAME_SM = "Super Metroid"
-GAME_SOE = "Secret of Evermore"
-GAME_SMZ3 = "SMZ3"
-supported_games = {"A Link to the Past", "Super Metroid", "Secret of Evermore", "SMZ3"}
-
-preferred_endings = {
-    GAME_ALTTP: "apbp",
-    GAME_SM: "apm3",
-    GAME_SOE: "apsoe",
-    GAME_SMZ3: "apsmz"
-}
-
-
-def generate_yaml(patch: bytes, metadata: Optional[dict] = None, game: str = GAME_ALTTP) -> bytes:
-    if game == GAME_ALTTP:
-        from worlds.alttp.Rom import JAP10HASH as HASH
-    elif game == GAME_SM:
-        from worlds.sm.Rom import JAP10HASH as HASH
-    elif game == GAME_SOE:
-        from worlds.soe.Patch import USHASH as HASH
-    elif game == GAME_SMZ3:
-        from worlds.alttp.Rom import JAP10HASH as ALTTPHASH
-        from worlds.sm.Rom import JAP10HASH as SMHASH
-        HASH = ALTTPHASH + SMHASH
-    else:
-        raise RuntimeError(f"Selected game {game} for base rom not found.")
-
-    patch = yaml.dump({"meta": metadata,
-                       "patch": patch,
-                       "game": game,
-                       # minimum version of patch system expected for patching to be successful
-                       "compatible_version": 3,
-                       "version": current_patch_version,
-                       "base_checksum": HASH})
-    return patch.encode(encoding="utf-8-sig")
-
-
-def generate_patch(rom: bytes, metadata: Optional[dict] = None, game: str = GAME_ALTTP) -> bytes:
-    if metadata is None:
-        metadata = {}
-    patch = bsdiff4.diff(get_base_rom_data(game), rom)
-    return generate_yaml(patch, metadata, game)
-
-
-def create_patch_file(rom_file_to_patch: str, server: str = "", destination: str = None,
-                      player: int = 0, player_name: str = "", game: str = GAME_ALTTP) -> str:
-    meta = {"server": server,  # allow immediate connection to server in multiworld. Empty string otherwise
-            "player_id": player,
-            "player_name": player_name}
-    bytes = generate_patch(load_bytes(rom_file_to_patch),
-                           meta,
-                           game)
-    target = destination if destination else os.path.splitext(rom_file_to_patch)[0] + (
-        ".apbp" if game == GAME_ALTTP else ".apsmz" if game == GAME_SMZ3 else ".apm3")
-    write_lzma(bytes, target)
-    return target
-
-
-def create_rom_bytes(patch_file: str, ignore_version: bool = False) -> Tuple[dict, str, bytearray]:
-    data = Utils.parse_yaml(lzma.decompress(load_bytes(patch_file)).decode("utf-8-sig"))
-    game_name = data["game"]
-    if not ignore_version and data["compatible_version"] > current_patch_version:
-        raise RuntimeError("Patch file is incompatible with this patcher, likely an update is required.")
-    patched_data = bsdiff4.patch(get_base_rom_data(game_name), data["patch"])
-    rom_hash = patched_data[int(0x7FC0):int(0x7FD5)]
-    data["meta"]["hash"] = "".join(chr(x) for x in rom_hash)
-    target = os.path.splitext(patch_file)[0] + ".sfc"
-    return data["meta"], target, patched_data
-
-
-def get_base_rom_data(game: str):
-    if game == GAME_ALTTP:
-        from worlds.alttp.Rom import get_base_rom_bytes
-    elif game == "alttp":  # old version for A Link to the Past
-        from worlds.alttp.Rom import get_base_rom_bytes
-    elif game == GAME_SM:
-        from worlds.sm.Rom import get_base_rom_bytes
-    elif game == GAME_SOE:
-        from worlds.soe.Patch import get_base_rom_path
-        get_base_rom_bytes = lambda: bytes(read_rom(open(get_base_rom_path(), "rb")))
-    elif game == GAME_SMZ3:
-        from worlds.smz3.Rom import get_base_rom_bytes
-    else:
-        raise RuntimeError("Selected game for base rom not found.")
-    return get_base_rom_bytes()
-
-
-def create_rom_file(patch_file: str) -> Tuple[dict, str]:
+def create_rom_file(patch_file: str) -> Tuple[RomMeta, str]:
     auto_handler = AutoPatchRegister.get_handler(patch_file)
     if auto_handler:
         handler: APDeltaPatch = auto_handler(patch_file)
@@ -259,162 +26,10 @@ def create_rom_file(patch_file: str) -> Tuple[dict, str]:
         return {"server": handler.server,
                 "player": handler.player,
                 "player_name": handler.player_name}, target
-    else:
-        data, target, patched_data = create_rom_bytes(patch_file)
-        with open(target, "wb") as f:
-            f.write(patched_data)
-        return data, target
-
-
-def update_patch_data(patch_data: bytes, server: str = "") -> bytes:
-    data = Utils.parse_yaml(lzma.decompress(patch_data).decode("utf-8-sig"))
-    data["meta"]["server"] = server
-    bytes = generate_yaml(data["patch"], data["meta"], data["game"])
-    return lzma.compress(bytes)
-
-
-def load_bytes(path: str) -> bytes:
-    with open(path, "rb") as f:
-        return f.read()
-
-
-def write_lzma(data: bytes, path: str):
-    with lzma.LZMAFile(path, 'wb') as f:
-        f.write(data)
-
-
-def read_rom(stream, strip_header=True) -> bytearray:
-    """Reads rom into bytearray and optionally strips off any smc header"""
-    buffer = bytearray(stream.read())
-    if strip_header and len(buffer) % 0x400 == 0x200:
-        return buffer[0x200:]
-    return buffer
+    raise NotImplementedError(f"No Handler for {patch_file} found.")
 
 
 if __name__ == "__main__":
-    host = Utils.get_public_ipv4()
-    options = Utils.get_options()['server_options']
-    if options['host']:
-        host = options['host']
-
-    address = f"{host}:{options['port']}"
-    ziplock = threading.Lock()
-    print(f"Host for patches to be created is {address}")
-    with concurrent.futures.ThreadPoolExecutor() as pool:
-        for rom in sys.argv:
-            try:
-                if rom.endswith(".sfc"):
-                    print(f"Creating patch for {rom}")
-                    result = pool.submit(create_patch_file, rom, address)
-                    result.add_done_callback(lambda task: print(f"Created patch {task.result()}"))
-
-                elif rom.endswith(".apbp"):
-                    print(f"Applying patch {rom}")
-                    data, target = create_rom_file(rom)
-                    #romfile, adjusted = Utils.get_adjuster_settings(target)
-                    adjuster_settings = Utils.get_adjuster_settings(GAME_ALTTP)
-                    adjusted = False
-                    if adjuster_settings:
-                        import pprint
-                        from worlds.alttp.Rom import get_base_rom_path
-                        adjuster_settings.rom = target
-                        adjuster_settings.baserom = get_base_rom_path()
-                        adjuster_settings.world = None
-                        whitelist = {"music", "menuspeed", "heartbeep", "heartcolor", "ow_palettes", "quickswap",
-                                        "uw_palettes", "sprite", "sword_palettes", "shield_palettes", "hud_palettes",
-                                        "reduceflashing", "deathlink"}
-                        printed_options = {name: value for name, value in vars(adjuster_settings).items() if name in whitelist}
-                        if hasattr(adjuster_settings, "sprite_pool"):
-                            sprite_pool = {}
-                            for sprite in getattr(adjuster_settings, "sprite_pool"):
-                                if sprite in sprite_pool:
-                                    sprite_pool[sprite] += 1
-                                else:
-                                    sprite_pool[sprite] = 1
-                            if sprite_pool:
-                                printed_options["sprite_pool"] = sprite_pool
-
-                        adjust_wanted = str('no')
-                        if not hasattr(adjuster_settings, 'auto_apply') or 'ask' in adjuster_settings.auto_apply:
-                            adjust_wanted = input(f"Last used adjuster settings were found. Would you like to apply these? \n"
-                                                  f"{pprint.pformat(printed_options)}\n"
-                                                  f"Enter yes, no, always or never: ")
-                        if adjuster_settings.auto_apply == 'never':  # never adjust, per user request
-                            adjust_wanted = 'no'
-                        elif adjuster_settings.auto_apply == 'always':
-                            adjust_wanted = 'yes'
-                        
-                        if adjust_wanted and "never" in adjust_wanted:
-                            adjuster_settings.auto_apply = 'never'
-                            Utils.persistent_store("adjuster", GAME_ALTTP, adjuster_settings)
-
-                        elif adjust_wanted and "always" in adjust_wanted:
-                            adjuster_settings.auto_apply = 'always'
-                            Utils.persistent_store("adjuster", GAME_ALTTP, adjuster_settings)
-
-                        if adjust_wanted and adjust_wanted.startswith("y"):
-                            if hasattr(adjuster_settings, "sprite_pool"):
-                                from LttPAdjuster import AdjusterWorld
-                                adjuster_settings.world = AdjusterWorld(getattr(adjuster_settings, "sprite_pool"))
-
-                            adjusted = True
-                            import LttPAdjuster
-                            _, romfile = LttPAdjuster.adjust(adjuster_settings)
-
-                            if hasattr(adjuster_settings, "world"):
-                                delattr(adjuster_settings, "world")
-                        else:
-                            adjusted = False
-                    if adjusted:
-                        try:
-                            shutil.move(romfile, target)
-                            romfile = target
-                        except Exception as e:
-                            print(e)
-                    print(f"Created rom {romfile if adjusted else target}.")
-                    if 'server' in data:
-                        Utils.persistent_store("servers", data['hash'], data['server'])
-                        print(f"Host is {data['server']}")
-                elif rom.endswith(".apm3"):
-                    print(f"Applying patch {rom}")
-                    data, target = create_rom_file(rom)
-                    print(f"Created rom {target}.")
-                    if 'server' in data:
-                        Utils.persistent_store("servers", data['hash'], data['server'])
-                        print(f"Host is {data['server']}")
-                elif rom.endswith(".apsmz"):
-                    print(f"Applying patch {rom}")
-                    data, target = create_rom_file(rom)
-                    print(f"Created rom {target}.")
-                    if 'server' in data:
-                        Utils.persistent_store("servers", data['hash'], data['server'])
-                        print(f"Host is {data['server']}")
-
-                elif rom.endswith(".zip"):
-                    print(f"Updating host in patch files contained in {rom}")
-
-
-                    def _handle_zip_file_entry(zfinfo: zipfile.ZipInfo, server: str):
-                        data = zfr.read(zfinfo)
-                        if zfinfo.filename.endswith(".apbp") or zfinfo.filename.endswith(".apm3"):
-                            data = update_patch_data(data, server)
-                        with ziplock:
-                            zfw.writestr(zfinfo, data)
-                        return zfinfo.filename
-
-
-                    futures = []
-                    with zipfile.ZipFile(rom, "r") as zfr:
-                        updated_zip = os.path.splitext(rom)[0] + "_updated.zip"
-                        with zipfile.ZipFile(updated_zip, "w", compression=zipfile.ZIP_DEFLATED,
-                                             compresslevel=9) as zfw:
-                            for zfname in zfr.namelist():
-                                futures.append(pool.submit(_handle_zip_file_entry, zfr.getinfo(zfname), address))
-                            for future in futures:
-                                print(f"File {future.result()} added to {os.path.split(updated_zip)[1]}")
-
-            except:
-                import traceback
-
-                traceback.print_exc()
-                input("Press enter to close.")
+    for file in sys.argv[1:]:
+        meta_data, result_file = create_rom_file(file)
+        print(f"Patch with meta-data {meta_data} was written to {result_file}")

PokemonClient.py

@@ -0,0 +1,351 @@
+import asyncio
+import json
+import time
+import os
+import bsdiff4
+import subprocess
+import zipfile
+from asyncio import StreamReader, StreamWriter
+from typing import List
+
+
+import Utils
+from Utils import async_start
+from CommonClient import CommonContext, server_loop, gui_enabled, ClientCommandProcessor, logger, \
+    get_base_parser
+
+from worlds.pokemon_rb.locations import location_data
+from worlds.pokemon_rb.rom import RedDeltaPatch, BlueDeltaPatch
+
+location_map = {"Rod": {}, "EventFlag": {}, "Missable": {}, "Hidden": {}, "list": {}, "DexSanityFlag": {}}
+location_bytes_bits = {}
+for location in location_data:
+    if location.ram_address is not None:
+        if type(location.ram_address) == list:
+            location_map[type(location.ram_address).__name__][(location.ram_address[0].flag, location.ram_address[1].flag)] = location.address
+            location_bytes_bits[location.address] = [{'byte': location.ram_address[0].byte, 'bit': location.ram_address[0].bit},
+                                                     {'byte': location.ram_address[1].byte, 'bit': location.ram_address[1].bit}]
+        else:
+            location_map[type(location.ram_address).__name__][location.ram_address.flag] = location.address
+            location_bytes_bits[location.address] = {'byte': location.ram_address.byte, 'bit': location.ram_address.bit}
+
+SYSTEM_MESSAGE_ID = 0
+
+CONNECTION_TIMING_OUT_STATUS = "Connection timing out. Please restart your emulator, then restart pkmn_rb.lua"
+CONNECTION_REFUSED_STATUS = "Connection Refused. Please start your emulator and make sure pkmn_rb.lua is running"
+CONNECTION_RESET_STATUS = "Connection was reset. Please restart your emulator, then restart pkmn_rb.lua"
+CONNECTION_TENTATIVE_STATUS = "Initial Connection Made"
+CONNECTION_CONNECTED_STATUS = "Connected"
+CONNECTION_INITIAL_STATUS = "Connection has not been initiated"
+
+DISPLAY_MSGS = True
+
+SCRIPT_VERSION = 3
+
+
+class GBCommandProcessor(ClientCommandProcessor):
+    def __init__(self, ctx: CommonContext):
+        super().__init__(ctx)
+
+    def _cmd_gb(self):
+        """Check Gameboy Connection State"""
+        if isinstance(self.ctx, GBContext):
+            logger.info(f"Gameboy Status: {self.ctx.gb_status}")
+
+
+class GBContext(CommonContext):
+    command_processor = GBCommandProcessor
+    game = 'Pokemon Red and Blue'
+
+    def __init__(self, server_address, password):
+        super().__init__(server_address, password)
+        self.gb_streams: (StreamReader, StreamWriter) = None
+        self.gb_sync_task = None
+        self.messages = {}
+        self.locations_array = None
+        self.gb_status = CONNECTION_INITIAL_STATUS
+        self.awaiting_rom = False
+        self.display_msgs = True
+        self.deathlink_pending = False
+        self.set_deathlink = False
+        self.client_compatibility_mode = 0
+        self.items_handling = 0b001
+        self.sent_release = False
+        self.sent_collect = False
+
+    async def server_auth(self, password_requested: bool = False):
+        if password_requested and not self.password:
+            await super(GBContext, self).server_auth(password_requested)
+        if not self.auth:
+            self.awaiting_rom = True
+            logger.info('Awaiting connection to Bizhawk to get Player information')
+            return
+
+        await self.send_connect()
+
+    def _set_message(self, msg: str, msg_id: int):
+        if DISPLAY_MSGS:
+            self.messages[(time.time(), msg_id)] = msg
+
+    def on_package(self, cmd: str, args: dict):
+        if cmd == 'Connected':
+            self.locations_array = None
+            if 'death_link' in args['slot_data'] and args['slot_data']['death_link']:
+                self.set_deathlink = True
+        elif cmd == "RoomInfo":
+            self.seed_name = args['seed_name']
+        elif cmd == 'Print':
+            msg = args['text']
+            if ': !' not in msg:
+                self._set_message(msg, SYSTEM_MESSAGE_ID)
+        elif cmd == "ReceivedItems":
+            msg = f"Received {', '.join([self.item_names[item.item] for item in args['items']])}"
+            self._set_message(msg, SYSTEM_MESSAGE_ID)
+
+    def on_deathlink(self, data: dict):
+        self.deathlink_pending = True
+        super().on_deathlink(data)
+
+    def run_gui(self):
+        from kvui import GameManager
+
+        class GBManager(GameManager):
+            logging_pairs = [
+                ("Client", "Archipelago")
+            ]
+            base_title = "Archipelago Pokémon Client"
+
+        self.ui = GBManager(self)
+        self.ui_task = asyncio.create_task(self.ui.async_run(), name="UI")
+
+
+def get_payload(ctx: GBContext):
+    current_time = time.time()
+    ret = json.dumps(
+        {
+            "items": [item.item for item in ctx.items_received],
+            "messages": {f'{key[0]}:{key[1]}': value for key, value in ctx.messages.items()
+                         if key[0] > current_time - 10},
+            "deathlink": ctx.deathlink_pending,
+            "options": ((ctx.permissions['release'] in ('goal', 'enabled')) * 2) + (ctx.permissions['collect'] in ('goal', 'enabled'))
+        }
+    )
+    ctx.deathlink_pending = False
+    return ret
+
+
+async def parse_locations(data: List, ctx: GBContext):
+    locations = []
+    flags = {"EventFlag": data[:0x140], "Missable": data[0x140:0x140 + 0x20],
+             "Hidden": data[0x140 + 0x20: 0x140 + 0x20 + 0x0E],
+             "Rod": data[0x140 + 0x20 + 0x0E:0x140 + 0x20 + 0x0E + 0x01]}
+
+    if len(data) > 0x140 + 0x20 + 0x0E + 0x01:
+        flags["DexSanityFlag"] = data[0x140 + 0x20 + 0x0E + 0x01:]
+    else:
+        flags["DexSanityFlag"] = [0] * 19
+
+    for flag_type, loc_map in location_map.items():
+        for flag, loc_id in loc_map.items():
+            if flag_type == "list":
+                if (flags["EventFlag"][location_bytes_bits[loc_id][0]['byte']] & 1 << location_bytes_bits[loc_id][0]['bit']
+                        and flags["Missable"][location_bytes_bits[loc_id][1]['byte']] & 1 << location_bytes_bits[loc_id][1]['bit']):
+                    locations.append(loc_id)
+            elif flags[flag_type][location_bytes_bits[loc_id]['byte']] & 1 << location_bytes_bits[loc_id]['bit']:
+                locations.append(loc_id)
+    if flags["EventFlag"][280] & 1 and not ctx.finished_game:
+        await ctx.send_msgs([
+                    {"cmd": "StatusUpdate",
+                     "status": 30}
+                ])
+        ctx.finished_game = True
+    if locations == ctx.locations_array:
+        return
+    ctx.locations_array = locations
+    if locations is not None:
+        await ctx.send_msgs([{"cmd": "LocationChecks", "locations": locations}])
+
+
+async def gb_sync_task(ctx: GBContext):
+    logger.info("Starting GB connector. Use /gb for status information")
+    while not ctx.exit_event.is_set():
+        error_status = None
+        if ctx.gb_streams:
+            (reader, writer) = ctx.gb_streams
+            msg = get_payload(ctx).encode()
+            writer.write(msg)
+            writer.write(b'\n')
+            try:
+                await asyncio.wait_for(writer.drain(), timeout=1.5)
+                try:
+                    # Data will return a dict with up to two fields:
+                    # 1. A keepalive response of the Players Name (always)
+                    # 2. An array representing the memory values of the locations area (if in game)
+                    data = await asyncio.wait_for(reader.readline(), timeout=5)
+                    data_decoded = json.loads(data.decode())
+                    if 'scriptVersion' not in data_decoded or data_decoded['scriptVersion'] != SCRIPT_VERSION:
+                        msg = "You are connecting with an incompatible Lua script version. Ensure your connector Lua " \
+                            "and PokemonClient are from the same Archipelago installation."
+                        logger.info(msg, extra={'compact_gui': True})
+                        ctx.gui_error('Error', msg)
+                        error_status = CONNECTION_RESET_STATUS
+                    ctx.client_compatibility_mode = data_decoded['clientCompatibilityVersion']
+                    if ctx.client_compatibility_mode == 0:
+                        ctx.items_handling = 0b101  # old patches will not have local start inventory, must be requested
+                    if ctx.seed_name and ctx.seed_name != ''.join([chr(i) for i in data_decoded['seedName'] if i != 0]):
+                        msg = "The server is running a different multiworld than your client is. (invalid seed_name)"
+                        logger.info(msg, extra={'compact_gui': True})
+                        ctx.gui_error('Error', msg)
+                        error_status = CONNECTION_RESET_STATUS
+                    ctx.seed_name = ''.join([chr(i) for i in data_decoded['seedName'] if i != 0])
+                    if not ctx.auth:
+                        ctx.auth = ''.join([chr(i) for i in data_decoded['playerName'] if i != 0])
+                        if ctx.auth == '':
+                            msg = "Invalid ROM detected. No player name built into the ROM."
+                            logger.info(msg, extra={'compact_gui': True})
+                            ctx.gui_error('Error', msg)
+                            error_status = CONNECTION_RESET_STATUS
+                        if ctx.awaiting_rom:
+                            await ctx.server_auth(False)
+                    if 'locations' in data_decoded and ctx.game and ctx.gb_status == CONNECTION_CONNECTED_STATUS \
+                            and not error_status and ctx.auth:
+                        # Not just a keep alive ping, parse
+                        async_start(parse_locations(data_decoded['locations'], ctx))
+                    if 'deathLink' in data_decoded and data_decoded['deathLink'] and 'DeathLink' in ctx.tags:
+                        await ctx.send_death(ctx.auth + " is out of usable Pokémon! " + ctx.auth + " blacked out!")
+                    if 'options' in data_decoded:
+                        msgs = []
+                        if data_decoded['options'] & 4 and not ctx.sent_release:
+                            ctx.sent_release = True
+                            msgs.append({"cmd": "Say", "text": "!release"})
+                        if data_decoded['options'] & 8 and not ctx.sent_collect:
+                            ctx.sent_collect = True
+                            msgs.append({"cmd": "Say", "text": "!collect"})
+                        if msgs:
+                            await ctx.send_msgs(msgs)
+                    if ctx.set_deathlink:
+                        await ctx.update_death_link(True)
+                except asyncio.TimeoutError:
+                    logger.debug("Read Timed Out, Reconnecting")
+                    error_status = CONNECTION_TIMING_OUT_STATUS
+                    writer.close()
+                    ctx.gb_streams = None
+                except ConnectionResetError as e:
+                    logger.debug("Read failed due to Connection Lost, Reconnecting")
+                    error_status = CONNECTION_RESET_STATUS
+                    writer.close()
+                    ctx.gb_streams = None
+            except TimeoutError:
+                logger.debug("Connection Timed Out, Reconnecting")
+                error_status = CONNECTION_TIMING_OUT_STATUS
+                writer.close()
+                ctx.gb_streams = None
+            except ConnectionResetError:
+                logger.debug("Connection Lost, Reconnecting")
+                error_status = CONNECTION_RESET_STATUS
+                writer.close()
+                ctx.gb_streams = None
+            if ctx.gb_status == CONNECTION_TENTATIVE_STATUS:
+                if not error_status:
+                    logger.info("Successfully Connected to Gameboy")
+                    ctx.gb_status = CONNECTION_CONNECTED_STATUS
+                else:
+                    ctx.gb_status = f"Was tentatively connected but error occured: {error_status}"
+            elif error_status:
+                ctx.gb_status = error_status
+                logger.info("Lost connection to Gameboy and attempting to reconnect. Use /gb for status updates")
+        else:
+            try:
+                logger.debug("Attempting to connect to Gameboy")
+                ctx.gb_streams = await asyncio.wait_for(asyncio.open_connection("localhost", 17242), timeout=10)
+                ctx.gb_status = CONNECTION_TENTATIVE_STATUS
+            except TimeoutError:
+                logger.debug("Connection Timed Out, Trying Again")
+                ctx.gb_status = CONNECTION_TIMING_OUT_STATUS
+                continue
+            except ConnectionRefusedError:
+                logger.debug("Connection Refused, Trying Again")
+                ctx.gb_status = CONNECTION_REFUSED_STATUS
+                continue
+
+
+async def run_game(romfile):
+    auto_start = Utils.get_options()["pokemon_rb_options"].get("rom_start", True)
+    if auto_start is True:
+        import webbrowser
+        webbrowser.open(romfile)
+    elif os.path.isfile(auto_start):
+        subprocess.Popen([auto_start, romfile],
+                         stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+
+async def patch_and_run_game(game_version, patch_file, ctx):
+    base_name = os.path.splitext(patch_file)[0]
+    comp_path = base_name + '.gb'
+    if game_version == "blue":
+        delta_patch = BlueDeltaPatch
+    else:
+        delta_patch = RedDeltaPatch
+
+    try:
+        base_rom = delta_patch.get_source_data()
+    except Exception as msg:
+        logger.info(msg, extra={'compact_gui': True})
+        ctx.gui_error('Error', msg)
+
+    with zipfile.ZipFile(patch_file, 'r') as patch_archive:
+        with patch_archive.open('delta.bsdiff4', 'r') as stream:
+            patch = stream.read()
+    patched_rom_data = bsdiff4.patch(base_rom, patch)
+
+    with open(comp_path, "wb") as patched_rom_file:
+        patched_rom_file.write(patched_rom_data)
+
+    async_start(run_game(comp_path))
+
+
+if __name__ == '__main__':
+
+    Utils.init_logging("PokemonClient")
+
+    options = Utils.get_options()
+
+    async def main():
+        parser = get_base_parser()
+        parser.add_argument('patch_file', default="", type=str, nargs="?",
+                            help='Path to an APRED or APBLUE patch file')
+        args = parser.parse_args()
+
+        ctx = GBContext(args.connect, args.password)
+        ctx.server_task = asyncio.create_task(server_loop(ctx), name="ServerLoop")
+        if gui_enabled:
+            ctx.run_gui()
+        ctx.run_cli()
+        ctx.gb_sync_task = asyncio.create_task(gb_sync_task(ctx), name="GB Sync")
+
+        if args.patch_file:
+            ext = args.patch_file.split(".")[len(args.patch_file.split(".")) - 1].lower()
+            if ext == "apred":
+                logger.info("APRED file supplied, beginning patching process...")
+                async_start(patch_and_run_game("red", args.patch_file, ctx))
+            elif ext == "apblue":
+                logger.info("APBLUE file supplied, beginning patching process...")
+                async_start(patch_and_run_game("blue", args.patch_file, ctx))
+            else:
+                logger.warning(f"Unknown patch file extension {ext}")
+
+        await ctx.exit_event.wait()
+        ctx.server_address = None
+
+        await ctx.shutdown()
+
+        if ctx.gb_sync_task:
+            await ctx.gb_sync_task
+
+
+    import colorama
+
+    colorama.init()
+
+    asyncio.run(main())
+    colorama.deinit()

README.md

@@ -26,6 +26,25 @@ Currently, the following games are supported:
 * The Witness
 * Sonic Adventure 2: Battle
 * Starcraft 2: Wings of Liberty
+* Donkey Kong Country 3
+* Dark Souls 3
+* Super Mario World
+* Pokémon Red and Blue
+* Hylics 2
+* Overcooked! 2
+* Zillion
+* Lufia II Ancient Cave
+* Blasphemous
+* Wargroove
+* Stardew Valley
+* The Legend of Zelda
+* The Messenger
+* Kingdom Hearts 2
+* The Legend of Zelda: Link's Awakening DX
+* Clique
+* Adventure
+* DLC Quest
+* Noita
 
 For setup and instructions check out our [tutorials page](https://archipelago.gg/tutorial/).
 Downloads can be found at [Releases](https://github.com/ArchipelagoMW/Archipelago/releases), including compiled
@@ -49,7 +68,7 @@ Archipelago was directly forked from bonta0's `multiworld_31` branch of ALttPEnt
 ## Running Archipelago
 For most people all you need to do is head over to the [releases](https://github.com/ArchipelagoMW/Archipelago/releases) page then download and run the appropriate installer. The installers function on Windows only.
 
-If you are running Archipelago from a non-Windows system then the likely scenario is that you are comfortable running source code directly. Please see our wiki page on [running Archipelago from source](https://github.com/ArchipelagoMW/Archipelago/wiki/Running-from-source).
+If you are running Archipelago from a non-Windows system then the likely scenario is that you are comfortable running source code directly. Please see our doc on [running Archipelago from source](docs/running%20from%20source.md).
 
 ## Related Repositories
 This project makes use of multiple other projects. We wouldn't be here without these other repositories and the contributions of their developers, past and present.
@@ -59,23 +78,10 @@ This project makes use of multiple other projects. We wouldn't be here without t
 * [Ocarina of Time Randomizer](https://github.com/TestRunnerSRL/OoT-Randomizer)
 
 ## Contributing
-Contributions are welcome. We have a few asks of any new contributors.
+For contribution guidelines, please see our [Contributing doc.](/docs/contributing.md)
 
-* Ensure that all changes which affect logic are covered by unit tests. 
-* Do not introduce any unit test failures/regressions.
-
-Otherwise, we tend to judge code on a case to case basis. It is a generally good idea to stick to PEP-8 guidelines to ensure consistency with existing code. (And to make the linter happy.)
-
-For adding a new game to Archipelago please see the docs folder for the relevant information and feel free to ask any questions in the #archipelago-dev channel in our discord.
+## FAQ
+For Frequently asked questions, please see the website's [FAQ Page.](https://archipelago.gg/faq/en/)
 
 ## Code of Conduct
-We conduct ourselves openly and inclusively here. Please do not contribute to an environment which makes other people uncomfortable. This means that we expect all contributors or participants here to:
-
-* Be welcoming and inclusive in tone and language.
-* Be respectful of others and their abilities.
-* Show empathy when speaking with others.
-* Be gracious and accept feedback and constructive criticism.
-
-These guidelines apply to all channels of communication within this GitHub repository. Please be respectful in both public channels, such as issues, and private, such as private messaging or emails.
-
-Any incidents of abuse may be reported directly to Ijwu at hmfarran@gmail.com.
+Please refer to our [code of conduct.](/docs/code_of_conduct.md)

Utils.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
-import shutil
+import asyncio
+import json
 import typing
 import builtins
 import os
@@ -12,11 +13,20 @@ import io
 import collections
 import importlib
 import logging
+from typing import BinaryIO, Coroutine, Optional, Set, Dict, Any, Union
+
+from yaml import load, load_all, dump, SafeLoader
+
+try:
+    from yaml import CLoader as UnsafeLoader
+    from yaml import CDumper as Dumper
+except ImportError:
+    from yaml import Loader as UnsafeLoader
+    from yaml import Dumper
 
 if typing.TYPE_CHECKING:
-    from tkinter import Tk
-else:
-    Tk = typing.Any
+    import tkinter
+    import pathlib
 
 
 def tuplize_version(version: str) -> Version:
@@ -28,17 +38,16 @@ class Version(typing.NamedTuple):
     minor: int
     build: int
 
+    def as_simple_string(self) -> str:
+        return ".".join(str(item) for item in self)
 
-__version__ = "0.3.2"
+
+__version__ = "0.4.1"
 version_tuple = tuplize_version(__version__)
 
-import jellyfish
-from yaml import load, load_all, dump, SafeLoader
-
-try:
-    from yaml import CLoader as Loader
-except ImportError:
-    from yaml import Loader
+is_linux = sys.platform.startswith("linux")
+is_macos = sys.platform == "darwin"
+is_windows = sys.platform in ("win32", "cygwin", "msys")
 
 
 def int16_as_bytes(value: int) -> typing.List[int]:
@@ -82,7 +91,10 @@ def is_frozen() -> bool:
 
 
 def local_path(*path: str) -> str:
-    """Returns path to a file in the local Archipelago installation or source."""
+    """
+    Returns path to a file in the local Archipelago installation or source.
+    This might be read-only and user_path should be used instead for ROMs, configuration, etc.
+    """
     if hasattr(local_path, 'cached_path'):
         pass
     elif is_frozen():
@@ -94,7 +106,7 @@ def local_path(*path: str) -> str:
             local_path.cached_path = os.path.dirname(os.path.abspath(sys.argv[0]))
     else:
         import __main__
-        if hasattr(__main__, "__file__"):
+        if hasattr(__main__, "__file__") and os.path.isfile(__main__.__file__):
             # we are running in a normal Python environment
             local_path.cached_path = os.path.dirname(os.path.abspath(__main__.__file__))
         else:
@@ -120,23 +132,35 @@ def home_path(*path: str) -> str:
 
 def user_path(*path: str) -> str:
     """Returns either local_path or home_path based on write permissions."""
-    if hasattr(user_path, 'cached_path'):
+    if hasattr(user_path, "cached_path"):
         pass
     elif os.access(local_path(), os.W_OK):
         user_path.cached_path = local_path()
     else:
         user_path.cached_path = home_path()
         # populate home from local - TODO: upgrade feature
-        if user_path.cached_path != local_path() and not os.path.exists(user_path('host.yaml')):
-            for dn in ('Players', 'data/sprites'):
+        if user_path.cached_path != local_path() and not os.path.exists(user_path("host.yaml")):
+            import shutil
+            for dn in ("Players", "data/sprites"):
                 shutil.copytree(local_path(dn), user_path(dn), dirs_exist_ok=True)
-            for fn in ('manifest.json', 'host.yaml'):
+            for fn in ("manifest.json", "host.yaml"):
                 shutil.copy2(local_path(fn), user_path(fn))
 
     return os.path.join(user_path.cached_path, *path)
 
 
-def output_path(*path: str):
+def cache_path(*path: str) -> str:
+    """Returns path to a file in the user's Archipelago cache directory."""
+    if hasattr(cache_path, "cached_path"):
+        pass
+    else:
+        import platformdirs
+        cache_path.cached_path = platformdirs.user_cache_dir("Archipelago", False)
+
+    return os.path.join(cache_path.cached_path, *path)
+
+
+def output_path(*path: str) -> str:
     if hasattr(output_path, 'cached_path'):
         return os.path.join(output_path.cached_path, *path)
     output_path.cached_path = user_path(get_options()["general_options"]["output_path"])
@@ -145,11 +169,12 @@ def output_path(*path: str):
     return path
 
 
-def open_file(filename):
-    if sys.platform == 'win32':
+def open_file(filename: typing.Union[str, "pathlib.Path"]) -> None:
+    if is_windows:
         os.startfile(filename)
     else:
-        open_command = 'open' if sys.platform == 'darwin' else 'xdg-open'
+        from shutil import which
+        open_command = which("open") if is_macos else (which("xdg-open") or which("gnome-open") or which("kde-open"))
         subprocess.call([open_command, filename])
 
 
@@ -168,7 +193,9 @@ class UniqueKeyLoader(SafeLoader):
 
 parse_yaml = functools.partial(load, Loader=UniqueKeyLoader)
 parse_yamls = functools.partial(load_all, Loader=UniqueKeyLoader)
-unsafe_parse_yaml = functools.partial(load, Loader=Loader)
+unsafe_parse_yaml = functools.partial(load, Loader=UnsafeLoader)
+
+del load, load_all  # should not be used. don't leak their names
 
 
 def get_cert_none_ssl_context():
@@ -186,11 +213,12 @@ def get_public_ipv4() -> str:
     ip = socket.gethostbyname(socket.gethostname())
     ctx = get_cert_none_ssl_context()
     try:
-        ip = urllib.request.urlopen('https://checkip.amazonaws.com/', context=ctx).read().decode('utf8').strip()
+        ip = urllib.request.urlopen("https://checkip.amazonaws.com/", context=ctx, timeout=10).read().decode("utf8").strip()
     except Exception as e:
+        # noinspection PyBroadException
         try:
-            ip = urllib.request.urlopen('https://v4.ident.me', context=ctx).read().decode('utf8').strip()
-        except:
+            ip = urllib.request.urlopen("https://v4.ident.me", context=ctx, timeout=10).read().decode("utf8").strip()
+        except Exception:
             logging.exception(e)
             pass  # we could be offline, in a local game, so no point in erroring out
     return ip
@@ -203,15 +231,18 @@ def get_public_ipv6() -> str:
     ip = socket.gethostbyname(socket.gethostname())
     ctx = get_cert_none_ssl_context()
     try:
-        ip = urllib.request.urlopen('https://v6.ident.me', context=ctx).read().decode('utf8').strip()
+        ip = urllib.request.urlopen("https://v6.ident.me", context=ctx, timeout=10).read().decode("utf8").strip()
     except Exception as e:
         logging.exception(e)
         pass  # we could be offline, in a local game, or ipv6 may not be available
     return ip
 
 
+OptionsType = typing.Dict[str, typing.Dict[str, typing.Any]]
+
+
 @cache_argsless
-def get_default_options() -> dict:
+def get_default_options() -> OptionsType:
     # Refer to host.yaml for comments as to what all these options mean.
     options = {
         "general_options": {
@@ -219,20 +250,24 @@ def get_default_options() -> dict:
         },
         "factorio_options": {
             "executable": os.path.join("factorio", "bin", "x64", "factorio"),
+            "filter_item_sends": False,
+            "bridge_chat_out": True,
+        },
+        "sni_options": {
+            "sni_path": "SNI",
+            "snes_rom_start": True,
         },
         "sm_options": {
             "rom_file": "Super Metroid (JU).sfc",
-            "sni": "SNI",
-            "rom_start": True,
         },
         "soe_options": {
             "rom_file": "Secret of Evermore (USA).sfc",
         },
         "lttp_options": {
             "rom_file": "Zelda no Densetsu - Kamigami no Triforce (Japan).sfc",
-            "sni": "SNI",
-            "rom_start": True,
-
+        },
+        "ladx_options": {
+            "rom_file": "Legend of Zelda, The - Link's Awakening DX (USA, Europe) (SGB Enhanced).gbc",
         },
         "server_options": {
             "host": None,
@@ -246,7 +281,7 @@ def get_default_options() -> dict:
             "disable_item_cheat": False,
             "location_check_points": 1,
             "hint_cost": 10,
-            "forfeit_mode": "goal",
+            "release_mode": "goal",
             "collect_mode": "disabled",
             "remaining_mode": "goal",
             "auto_shutdown": 0,
@@ -254,13 +289,12 @@ def get_default_options() -> dict:
             "log_network": 0
         },
         "generator": {
-            "teams": 1,
-            "enemizer_path": os.path.join("EnemizerCLI", "EnemizerCLI.Core.exe"),
+            "enemizer_path": os.path.join("EnemizerCLI", "EnemizerCLI.Core"),
             "player_files_path": "Players",
             "players": 0,
             "weights_file_path": "weights.yaml",
             "meta_file_path": "meta.yaml",
-            "spoiler": 2,
+            "spoiler": 3,
             "glitch_triforce_room": 1,
             "race": 0,
             "plando_options": "bosses",
@@ -272,13 +306,50 @@ def get_default_options() -> dict:
         },
         "oot_options": {
             "rom_file": "The Legend of Zelda - Ocarina of Time.z64",
-        }
+            "rom_start": True
+        },
+        "dkc3_options": {
+            "rom_file": "Donkey Kong Country 3 - Dixie Kong's Double Trouble! (USA) (En,Fr).sfc",
+        },
+        "smw_options": {
+            "rom_file": "Super Mario World (USA).sfc",
+        },
+        "zillion_options": {
+            "rom_file": "Zillion (UE) [!].sms",
+            # RetroArch doesn't make it easy to launch a game from the command line.
+            # You have to know the path to the emulator core library on the user's computer.
+            "rom_start": "retroarch",
+        },
+        "pokemon_rb_options": {
+            "red_rom_file": "Pokemon Red (UE) [S][!].gb",
+            "blue_rom_file": "Pokemon Blue (UE) [S][!].gb",
+            "rom_start": True
+        },
+        "ffr_options": {
+            "display_msgs": True,
+        },
+        "lufia2ac_options": {
+            "rom_file": "Lufia II - Rise of the Sinistrals (USA).sfc",
+        },
+        "tloz_options": {
+            "rom_file": "Legend of Zelda, The (U) (PRG0) [!].nes",
+            "rom_start": True,
+            "display_msgs": True,
+        },
+        "wargroove_options": {
+            "root_directory": "C:/Program Files (x86)/Steam/steamapps/common/Wargroove"
+        },
+        "adventure_options": {
+            "rom_file": "ADVNTURE.BIN",
+            "display_msgs": True,
+            "rom_start": True,
+            "rom_args": ""
+        },
     }
-
     return options
 
 
-def update_options(src: dict, dest: dict, filename: str, keys: list) -> dict:
+def update_options(src: dict, dest: dict, filename: str, keys: list) -> OptionsType:
     for key, value in src.items():
         new_keys = keys.copy()
         new_keys.append(key)
@@ -298,34 +369,20 @@ def update_options(src: dict, dest: dict, filename: str, keys: list) -> dict:
 
 
 @cache_argsless
-def get_options() -> dict:
-    if not hasattr(get_options, "options"):
-        filenames = ("options.yaml", "host.yaml")
-        locations = []
-        if os.path.join(os.getcwd()) != local_path():
-            locations += filenames  # use files from cwd only if it's not the local_path
-        locations += [user_path(filename) for filename in filenames]
+def get_options() -> OptionsType:
+    filenames = ("options.yaml", "host.yaml")
+    locations: typing.List[str] = []
+    if os.path.join(os.getcwd()) != local_path():
+        locations += filenames  # use files from cwd only if it's not the local_path
+    locations += [user_path(filename) for filename in filenames]
 
-        for location in locations:
-            if os.path.exists(location):
-                with open(location) as f:
-                    options = parse_yaml(f.read())
+    for location in locations:
+        if os.path.exists(location):
+            with open(location) as f:
+                options = parse_yaml(f.read())
+            return update_options(get_default_options(), options, location, list())
 
-                get_options.options = update_options(get_default_options(), options, location, list())
-                break
-        else:
-            raise FileNotFoundError(f"Could not find {filenames[1]} to load options.")
-    return get_options.options
-
-
-def get_item_name_from_id(code: int) -> str:
-    from worlds import lookup_any_item_id_to_name
-    return lookup_any_item_id_to_name.get(code, f'Unknown item (ID:{code})')
-
-
-def get_location_name_from_id(code: int) -> str:
-    from worlds import lookup_any_location_id_to_name
-    return lookup_any_location_id_to_name.get(code, f'Unknown location (ID:{code})')
+    raise FileNotFoundError(f"Could not find {filenames[1]} to load options.")
 
 
 def persistent_store(category: str, key: typing.Any, value: typing.Any):
@@ -334,10 +391,10 @@ def persistent_store(category: str, key: typing.Any, value: typing.Any):
     category = storage.setdefault(category, {})
     category[key] = value
     with open(path, "wt") as f:
-        f.write(dump(storage))
+        f.write(dump(storage, Dumper=Dumper))
 
 
-def persistent_load() -> typing.Dict[dict]:
+def persistent_load() -> typing.Dict[str, dict]:
     storage = getattr(persistent_load, "storage", None)
     if storage:
         return storage
@@ -355,8 +412,47 @@ def persistent_load() -> typing.Dict[dict]:
     return storage
 
 
-def get_adjuster_settings(gameName: str):
-    adjuster_settings = persistent_load().get("adjuster", {}).get(gameName, {})
+def get_file_safe_name(name: str) -> str:
+    return "".join(c for c in name if c not in '<>:"/\\|?*')
+
+
+def load_data_package_for_checksum(game: str, checksum: typing.Optional[str]) -> Dict[str, Any]:
+    if checksum and game:
+        if checksum != get_file_safe_name(checksum):
+            raise ValueError(f"Bad symbols in checksum: {checksum}")
+        path = cache_path("datapackage", get_file_safe_name(game), f"{checksum}.json")
+        if os.path.exists(path):
+            try:
+                with open(path, "r", encoding="utf-8-sig") as f:
+                    return json.load(f)
+            except Exception as e:
+                logging.debug(f"Could not load data package: {e}")
+
+    # fall back to old cache
+    cache = persistent_load().get("datapackage", {}).get("games", {}).get(game, {})
+    if cache.get("checksum") == checksum:
+        return cache
+
+    # cache does not match
+    return {}
+
+
+def store_data_package_for_checksum(game: str, data: typing.Dict[str, Any]) -> None:
+    checksum = data.get("checksum")
+    if checksum and game:
+        if checksum != get_file_safe_name(checksum):
+            raise ValueError(f"Bad symbols in checksum: {checksum}")
+        game_folder = cache_path("datapackage", get_file_safe_name(game))
+        os.makedirs(game_folder, exist_ok=True)
+        try:
+            with open(os.path.join(game_folder, f"{checksum}.json"), "w", encoding="utf-8-sig") as f:
+                json.dump(data, f, ensure_ascii=False, separators=(",", ":"))
+        except Exception as e:
+            logging.debug(f"Could not store data package: {e}")
+
+
+def get_adjuster_settings(game_name: str) -> typing.Dict[str, typing.Any]:
+    adjuster_settings = persistent_load().get("adjuster", {}).get(game_name, {})
     return adjuster_settings
 
 
@@ -372,10 +468,10 @@ def get_unique_identifier():
     return uuid
 
 
-safe_builtins = {
+safe_builtins = frozenset((
     'set',
     'frozenset',
-}
+))
 
 
 class RestrictedUnpickler(pickle.Unpickler):
@@ -394,7 +490,8 @@ class RestrictedUnpickler(pickle.Unpickler):
         # Options and Plando are unpickled by WebHost -> Generate
         if module == "worlds.generic" and name in {"PlandoItem", "PlandoConnection"}:
             return getattr(self.generic_properties_module, name)
-        if module.endswith("Options"):
+        # pep 8 specifies that modules should have "all-lowercase names" (options, not Options)
+        if module.lower().endswith("options"):
             if module == "Options":
                 mod = self.options_module
             else:
@@ -403,8 +500,7 @@ class RestrictedUnpickler(pickle.Unpickler):
             if issubclass(obj, self.options_module.Option):
                 return obj
         # Forbid everything else.
-        raise pickle.UnpicklingError("global '%s.%s' is forbidden" %
-                                     (module, name))
+        raise pickle.UnpicklingError(f"global '{module}.{name}' is forbidden")
 
 
 def restricted_loads(s):
@@ -412,7 +508,19 @@ def restricted_loads(s):
     return RestrictedUnpickler(io.BytesIO(s)).load()
 
 
+class ByValue:
+    """
+    Mixin for enums to pickle value instead of name (restores pre-3.11 behavior). Use as left-most parent.
+    See https://github.com/python/cpython/pull/26658 for why this exists.
+    """
+    def __reduce_ex__(self, prot):
+        return self.__class__, (self._value_, )
+
+
 class KeyedDefaultDict(collections.defaultdict):
+    """defaultdict variant that uses the missing key as argument to default_factory"""
+    default_factory: typing.Callable[[typing.Any], typing.Any]
+
     def __missing__(self, key):
         self[key] = value = self.default_factory(key)
         return value
@@ -422,11 +530,17 @@ def get_text_between(text: str, start: str, end: str) -> str:
     return text[text.index(start) + len(start): text.rindex(end)]
 
 
+def get_text_after(text: str, start: str) -> str:
+    return text[text.index(start) + len(start):]
+
+
 loglevel_mapping = {'error': logging.ERROR, 'info': logging.INFO, 'warning': logging.WARNING, 'debug': logging.DEBUG}
 
 
 def init_logging(name: str, loglevel: typing.Union[str, int] = logging.INFO, write_mode: str = "w",
-                 log_format: str = "[%(name)s at %(asctime)s]: %(message)s", exception_logger: str = ""):
+                 log_format: str = "[%(name)s at %(asctime)s]: %(message)s",
+                 exception_logger: typing.Optional[str] = None):
+    import datetime
     loglevel: int = loglevel_mapping.get(loglevel, loglevel)
     log_folder = user_path("logs")
     os.makedirs(log_folder, exist_ok=True)
@@ -435,6 +549,8 @@ def init_logging(name: str, loglevel: typing.Union[str, int] = logging.INFO, wri
         root_logger.removeHandler(handler)
         handler.close()
     root_logger.setLevel(loglevel)
+    if "a" not in write_mode:
+        name += f"_{datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S')}"
     file_handler = logging.FileHandler(
         os.path.join(log_folder, f"{name}.txt"),
         write_mode,
@@ -462,13 +578,37 @@ def init_logging(name: str, loglevel: typing.Union[str, int] = logging.INFO, wri
 
         sys.excepthook = handle_exception
 
+    def _cleanup():
+        for file in os.scandir(log_folder):
+            if file.name.endswith(".txt"):
+                last_change = datetime.datetime.fromtimestamp(file.stat().st_mtime)
+                if datetime.datetime.now() - last_change > datetime.timedelta(days=7):
+                    try:
+                        os.unlink(file.path)
+                    except Exception as e:
+                        logging.exception(e)
+                    else:
+                        logging.debug(f"Deleted old logfile {file.path}")
+    import threading
+    threading.Thread(target=_cleanup, name="LogCleaner").start()
+    import platform
+    logging.info(
+        f"Archipelago ({__version__}) logging initialized"
+        f" on {platform.platform()}"
+        f" running Python {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
+    )
+
 
 def stream_input(stream, queue):
     def queuer():
         while 1:
-            text = stream.readline().strip()
-            if text:
-                queue.put_nowait(text)
+            try:
+                text = stream.readline().strip()
+            except UnicodeDecodeError as e:
+                logging.exception(e)
+            else:
+                if text:
+                    queue.put_nowait(text)
 
     from threading import Thread
     thread = Thread(target=queuer, name=f"Stream handler for {stream.name}", daemon=True)
@@ -476,37 +616,48 @@ def stream_input(stream, queue):
     return thread
 
 
-def tkinter_center_window(window: Tk):
+def tkinter_center_window(window: "tkinter.Tk") -> None:
     window.update()
-    xPos = int(window.winfo_screenwidth() / 2 - window.winfo_reqwidth() / 2)
-    yPos = int(window.winfo_screenheight() / 2 - window.winfo_reqheight() / 2)
-    window.geometry("+{}+{}".format(xPos, yPos))
+    x = int(window.winfo_screenwidth() / 2 - window.winfo_reqwidth() / 2)
+    y = int(window.winfo_screenheight() / 2 - window.winfo_reqheight() / 2)
+    window.geometry(f"+{x}+{y}")
 
 
 class VersionException(Exception):
     pass
 
 
-# noinspection PyPep8Naming
-def format_SI_prefix(value, power=1000, power_labels=('', 'k', 'M', 'G', 'T', "P", "E", "Z", "Y")) -> str:
-    n = 0
+def chaining_prefix(index: int, labels: typing.Tuple[str]) -> str:
+    text = ""
+    max_label = len(labels) - 1
+    while index > max_label:
+        text += labels[-1]
+        index -= max_label
+    return labels[index] + text
 
-    while value > power:
+
+# noinspection PyPep8Naming
+def format_SI_prefix(value, power=1000, power_labels=("", "k", "M", "G", "T", "P", "E", "Z", "Y")) -> str:
+    """Formats a value into a value + metric/si prefix. More info at https://en.wikipedia.org/wiki/Metric_prefix"""
+    import decimal
+    n = 0
+    value = decimal.Decimal(value)
+    limit = power - decimal.Decimal("0.005")
+    while value >= limit:
         value /= power
         n += 1
-    if type(value) == int:
-        return f"{value} {power_labels[n]}"
-    else:
-        return f"{value:0.3f} {power_labels[n]}"
 
-
-def get_fuzzy_ratio(word1: str, word2: str) -> float:
-    return (1 - jellyfish.damerau_levenshtein_distance(word1.lower(), word2.lower())
-            / max(len(word1), len(word2)))
+    return f"{value.quantize(decimal.Decimal('1.00'))} {chaining_prefix(n, power_labels)}"
 
 
 def get_fuzzy_results(input_word: str, wordlist: typing.Sequence[str], limit: typing.Optional[int] = None) \
         -> typing.List[typing.Tuple[str, int]]:
+    import jellyfish
+
+    def get_fuzzy_ratio(word1: str, word2: str) -> float:
+        return (1 - jellyfish.damerau_levenshtein_distance(word1.lower(), word2.lower())
+                / max(len(word1), len(word2)))
+
     limit: int = limit if limit else len(wordlist)
     return list(
         map(
@@ -519,3 +670,117 @@ def get_fuzzy_results(input_word: str, wordlist: typing.Sequence[str], limit: ty
                 reverse=True)[0:limit]
         )
     )
+
+
+def open_filename(title: str, filetypes: typing.Sequence[typing.Tuple[str, typing.Sequence[str]]]) \
+        -> typing.Optional[str]:
+    def run(*args: str):
+        return subprocess.run(args, capture_output=True, text=True).stdout.split("\n", 1)[0] or None
+
+    if is_linux:
+        # prefer native dialog
+        from shutil import which
+        kdialog = which("kdialog")
+        if kdialog:
+            k_filters = '|'.join((f'{text} (*{" *".join(ext)})' for (text, ext) in filetypes))
+            return run(kdialog, f"--title={title}", "--getopenfilename", ".", k_filters)
+        zenity = which("zenity")
+        if zenity:
+            z_filters = (f'--file-filter={text} ({", ".join(ext)}) | *{" *".join(ext)}' for (text, ext) in filetypes)
+            return run(zenity, f"--title={title}", "--file-selection", *z_filters)
+
+    # fall back to tk
+    try:
+        import tkinter
+        import tkinter.filedialog
+    except Exception as e:
+        logging.error('Could not load tkinter, which is likely not installed. '
+                      f'This attempt was made because open_filename was used for "{title}".')
+        raise e
+    else:
+        root = tkinter.Tk()
+        root.withdraw()
+        return tkinter.filedialog.askopenfilename(title=title, filetypes=((t[0], ' '.join(t[1])) for t in filetypes))
+
+
+def messagebox(title: str, text: str, error: bool = False) -> None:
+    def run(*args: str):
+        return subprocess.run(args, capture_output=True, text=True).stdout.split("\n", 1)[0] or None
+
+    def is_kivy_running():
+        if "kivy" in sys.modules:
+            from kivy.app import App
+            return App.get_running_app() is not None
+        return False
+
+    if is_kivy_running():
+        from kvui import MessageBox
+        MessageBox(title, text, error).open()
+        return
+
+    if is_linux and "tkinter" not in sys.modules:
+        # prefer native dialog
+        from shutil import which
+        kdialog = which("kdialog")
+        if kdialog:
+            return run(kdialog, f"--title={title}", "--error" if error else "--msgbox", text)
+        zenity = which("zenity")
+        if zenity:
+            return run(zenity, f"--title={title}", f"--text={text}", "--error" if error else "--info")
+
+    # fall back to tk
+    try:
+        import tkinter
+        from tkinter.messagebox import showerror, showinfo
+    except Exception as e:
+        logging.error('Could not load tkinter, which is likely not installed. '
+                      f'This attempt was made because messagebox was used for "{title}".')
+        raise e
+    else:
+        root = tkinter.Tk()
+        root.withdraw()
+        showerror(title, text) if error else showinfo(title, text)
+        root.update()
+
+
+def title_sorted(data: typing.Sequence, key=None, ignore: typing.Set = frozenset(("a", "the"))):
+    """Sorts a sequence of text ignoring typical articles like "a" or "the" in the beginning."""
+    def sorter(element: Union[str, Dict[str, Any]]) -> str:
+        if (not isinstance(element, str)):
+            element = element["title"]
+
+        parts = element.split(maxsplit=1)
+        if parts[0].lower() in ignore:
+            return parts[1].lower()
+        else:
+            return element.lower()
+    return sorted(data, key=lambda i: sorter(key(i)) if key else sorter(i))
+
+
+def read_snes_rom(stream: BinaryIO, strip_header: bool = True) -> bytearray:
+    """Reads rom into bytearray and optionally strips off any smc header"""
+    buffer = bytearray(stream.read())
+    if strip_header and len(buffer) % 0x400 == 0x200:
+        return buffer[0x200:]
+    return buffer
+
+
+_faf_tasks: "Set[asyncio.Task[None]]" = set()
+
+
+def async_start(co: Coroutine[typing.Any, typing.Any, bool], name: Optional[str] = None) -> None:
+    """
+    Use this to start a task when you don't keep a reference to it or immediately await it,
+    to prevent early garbage collection. "fire-and-forget"
+    """
+    # https://docs.python.org/3.10/library/asyncio-task.html#asyncio.create_task
+    # Python docs:
+    # ```
+    # Important: Save a reference to the result of [asyncio.create_task],
+    # to avoid a task disappearing mid-execution.
+    # ```
+    # This implementation follows the pattern given in that documentation.
+
+    task = asyncio.create_task(co, name=name)
+    _faf_tasks.add(task)
+    task.add_done_callback(_faf_tasks.discard)

WargrooveClient.py

@@ -0,0 +1,445 @@
+from __future__ import annotations
+
+import atexit
+import os
+import sys
+import asyncio
+import random
+import shutil
+from typing import Tuple, List, Iterable, Dict
+
+from worlds.wargroove import WargrooveWorld
+from worlds.wargroove.Items import item_table, faction_table, CommanderData, ItemData
+
+import ModuleUpdate
+ModuleUpdate.update()
+
+import Utils
+import json
+import logging
+
+if __name__ == "__main__":
+    Utils.init_logging("WargrooveClient", exception_logger="Client")
+
+from NetUtils import NetworkItem, ClientStatus
+from CommonClient import gui_enabled, logger, get_base_parser, ClientCommandProcessor, \
+    CommonContext, server_loop
+
+wg_logger = logging.getLogger("WG")
+
+
+class WargrooveClientCommandProcessor(ClientCommandProcessor):
+    def _cmd_resync(self):
+        """Manually trigger a resync."""
+        self.output(f"Syncing items.")
+        self.ctx.syncing = True
+
+    def _cmd_commander(self, *commander_name: Iterable[str]):
+        """Set the current commander to the given commander."""
+        if commander_name:
+            self.ctx.set_commander(' '.join(commander_name))
+        else:
+            if self.ctx.can_choose_commander:
+                commanders = self.ctx.get_commanders()
+                wg_logger.info('Unlocked commanders: ' +
+                               ', '.join((commander.name for commander, unlocked in commanders if unlocked)))
+                wg_logger.info('Locked commanders: ' +
+                               ', '.join((commander.name for commander, unlocked in commanders if not unlocked)))
+            else:
+                wg_logger.error('Cannot set commanders in this game mode.')
+
+
+class WargrooveContext(CommonContext):
+    command_processor: int = WargrooveClientCommandProcessor
+    game = "Wargroove"
+    items_handling = 0b111  # full remote
+    current_commander: CommanderData = faction_table["Starter"][0]
+    can_choose_commander: bool = False
+    commander_defense_boost_multiplier: int = 0
+    income_boost_multiplier: int = 0
+    starting_groove_multiplier: float
+    faction_item_ids = {
+        'Starter': 0,
+        'Cherrystone': 52025,
+        'Felheim': 52026,
+        'Floran': 52027,
+        'Heavensong': 52028,
+        'Requiem': 52029,
+        'Outlaw': 52030
+    }
+    buff_item_ids = {
+        'Income Boost': 52023,
+        'Commander Defense Boost': 52024,
+    }
+
+    def __init__(self, server_address, password):
+        super(WargrooveContext, self).__init__(server_address, password)
+        self.send_index: int = 0
+        self.syncing = False
+        self.awaiting_bridge = False
+        # self.game_communication_path: files go in this path to pass data between us and the actual game
+        if "appdata" in os.environ:
+            options = Utils.get_options()
+            root_directory = os.path.join(options["wargroove_options"]["root_directory"])
+            data_directory = os.path.join("lib", "worlds", "wargroove", "data")
+            dev_data_directory = os.path.join("worlds", "wargroove", "data")
+            appdata_wargroove = os.path.expandvars(os.path.join("%APPDATA%", "Chucklefish", "Wargroove"))
+            if not os.path.isfile(os.path.join(root_directory, "win64_bin", "wargroove64.exe")):
+                print_error_and_close("WargrooveClient couldn't find wargroove64.exe. "
+                                      "Unable to infer required game_communication_path")
+            self.game_communication_path = os.path.join(root_directory, "AP")
+            if not os.path.exists(self.game_communication_path):
+                os.makedirs(self.game_communication_path)
+            self.remove_communication_files()
+            atexit.register(self.remove_communication_files)
+            if not os.path.isdir(appdata_wargroove):
+                print_error_and_close("WargrooveClient couldn't find Wargoove in appdata!"
+                                      "Boot Wargroove and then close it to attempt to fix this error")
+            if not os.path.isdir(data_directory):
+                data_directory = dev_data_directory
+            if not os.path.isdir(data_directory):
+                print_error_and_close("WargrooveClient couldn't find Wargoove mod and save files in install!")
+            shutil.copytree(data_directory, appdata_wargroove, dirs_exist_ok=True)
+        else:
+            print_error_and_close("WargrooveClient couldn't detect system type. "
+                                  "Unable to infer required game_communication_path")
+
+    async def server_auth(self, password_requested: bool = False):
+        if password_requested and not self.password:
+            await super(WargrooveContext, self).server_auth(password_requested)
+        await self.get_username()
+        await self.send_connect()
+
+    async def connection_closed(self):
+        await super(WargrooveContext, self).connection_closed()
+        self.remove_communication_files()
+
+    @property
+    def endpoints(self):
+        if self.server:
+            return [self.server]
+        else:
+            return []
+
+    async def shutdown(self):
+        await super(WargrooveContext, self).shutdown()
+        self.remove_communication_files()
+
+    def remove_communication_files(self):
+        for root, dirs, files in os.walk(self.game_communication_path):
+            for file in files:
+                os.remove(root + "/" + file)
+
+    def on_package(self, cmd: str, args: dict):
+        if cmd in {"Connected"}:
+            filename = f"AP_settings.json"
+            with open(os.path.join(self.game_communication_path, filename), 'w') as f:
+                slot_data = args["slot_data"]
+                json.dump(args["slot_data"], f)
+                self.can_choose_commander = slot_data["can_choose_commander"]
+                print('can choose commander:', self.can_choose_commander)
+                self.starting_groove_multiplier = slot_data["starting_groove_multiplier"]
+                self.income_boost_multiplier = slot_data["income_boost"]
+                self.commander_defense_boost_multiplier = slot_data["commander_defense_boost"]
+                f.close()
+            for ss in self.checked_locations:
+                filename = f"send{ss}"
+                with open(os.path.join(self.game_communication_path, filename), 'w') as f:
+                    f.close()
+            self.update_commander_data()
+            self.ui.update_tracker()
+
+            random.seed(self.seed_name + str(self.slot))
+            # Our indexes start at 1 and we have 24 levels
+            for i in range(1, 25):
+                filename = f"seed{i}"
+                with open(os.path.join(self.game_communication_path, filename), 'w') as f:
+                    f.write(str(random.randint(0, 4294967295)))
+                    f.close()
+
+        if cmd in {"RoomInfo"}:
+            self.seed_name = args["seed_name"]
+
+        if cmd in {"ReceivedItems"}:
+            received_ids = [item.item for item in self.items_received]
+            for network_item in self.items_received:
+                filename = f"AP_{str(network_item.item)}.item"
+                path = os.path.join(self.game_communication_path, filename)
+
+                # Newly-obtained items
+                if not os.path.isfile(path):
+                    open(path, 'w').close()
+                    # Announcing commander unlocks
+                    item_name = self.item_names[network_item.item]
+                    if item_name in faction_table.keys():
+                        for commander in faction_table[item_name]:
+                            logger.info(f"{commander.name} has been unlocked!")
+
+                with open(path, 'w') as f:
+                    item_count = received_ids.count(network_item.item)
+                    if self.buff_item_ids["Income Boost"] == network_item.item:
+                        f.write(f"{item_count * self.income_boost_multiplier}")
+                    elif self.buff_item_ids["Commander Defense Boost"] == network_item.item:
+                        f.write(f"{item_count * self.commander_defense_boost_multiplier}")
+                    else:
+                        f.write(f"{item_count}")
+                    f.close()
+
+                print_filename = f"AP_{str(network_item.item)}.item.print"
+                print_path = os.path.join(self.game_communication_path, print_filename)
+                if not os.path.isfile(print_path):
+                    open(print_path, 'w').close()
+                    with open(print_path, 'w') as f:
+                        f.write("Received " +
+                                self.item_names[network_item.item] +
+                                " from " +
+                                self.player_names[network_item.player])
+                        f.close()
+            self.update_commander_data()
+            self.ui.update_tracker()
+
+        if cmd in {"RoomUpdate"}:
+            if "checked_locations" in args:
+                for ss in self.checked_locations:
+                    filename = f"send{ss}"
+                    with open(os.path.join(self.game_communication_path, filename), 'w') as f:
+                        f.close()
+
+    def run_gui(self):
+        """Import kivy UI system and start running it as self.ui_task."""
+        from kvui import GameManager, HoverBehavior, ServerToolTip
+        from kivy.uix.tabbedpanel import TabbedPanelItem
+        from kivy.lang import Builder
+        from kivy.uix.button import Button
+        from kivy.uix.togglebutton import ToggleButton
+        from kivy.uix.boxlayout import BoxLayout
+        from kivy.uix.gridlayout import GridLayout
+        from kivy.uix.image import AsyncImage, Image
+        from kivy.uix.stacklayout import StackLayout
+        from kivy.uix.label import Label
+        from kivy.properties import ColorProperty
+        from kivy.uix.image import Image
+        import pkgutil
+
+        class TrackerLayout(BoxLayout):
+            pass
+
+        class CommanderSelect(BoxLayout):
+            pass
+
+        class CommanderButton(ToggleButton):
+            pass
+
+        class FactionBox(BoxLayout):
+            pass
+
+        class CommanderGroup(BoxLayout):
+            pass
+
+        class ItemTracker(BoxLayout):
+            pass
+
+        class ItemLabel(Label):
+            pass
+
+        class WargrooveManager(GameManager):
+            logging_pairs = [
+                ("Client", "Archipelago"),
+                ("WG", "WG Console"),
+            ]
+            base_title = "Archipelago Wargroove Client"
+            ctx: WargrooveContext
+            unit_tracker: ItemTracker
+            trigger_tracker: BoxLayout
+            boost_tracker: BoxLayout
+            commander_buttons: Dict[int, List[CommanderButton]]
+            tracker_items = {
+                "Swordsman": ItemData(None, "Unit", False),
+                "Dog": ItemData(None, "Unit", False),
+                **item_table
+            }
+
+            def build(self):
+                container = super().build()
+                panel = TabbedPanelItem(text="Wargroove")
+                panel.content = self.build_tracker()
+                self.tabs.add_widget(panel)
+                return container
+
+            def build_tracker(self) -> TrackerLayout:
+                try:
+                    tracker = TrackerLayout(orientation="horizontal")
+                    commander_select = CommanderSelect(orientation="vertical")
+                    self.commander_buttons = {}
+
+                    for faction, commanders in faction_table.items():
+                        faction_box = FactionBox(size_hint=(None, None), width=100 * len(commanders), height=70)
+                        commander_group = CommanderGroup()
+                        commander_buttons = []
+                        for commander in commanders:
+                            commander_button = CommanderButton(text=commander.name, group="commanders")
+                            if faction == "Starter":
+                                commander_button.disabled = False
+                            commander_button.bind(on_press=lambda instance: self.ctx.set_commander(instance.text))
+                            commander_buttons.append(commander_button)
+                            commander_group.add_widget(commander_button)
+                        self.commander_buttons[faction] = commander_buttons
+                        faction_box.add_widget(Label(text=faction, size_hint_x=None, pos_hint={'left': 1}, size_hint_y=None, height=10))
+                        faction_box.add_widget(commander_group)
+                        commander_select.add_widget(faction_box)
+                    item_tracker = ItemTracker(padding=[0,20])
+                    self.unit_tracker = BoxLayout(orientation="vertical")
+                    other_tracker = BoxLayout(orientation="vertical")
+                    self.trigger_tracker = BoxLayout(orientation="vertical")
+                    self.boost_tracker = BoxLayout(orientation="vertical")
+                    other_tracker.add_widget(self.trigger_tracker)
+                    other_tracker.add_widget(self.boost_tracker)
+                    item_tracker.add_widget(self.unit_tracker)
+                    item_tracker.add_widget(other_tracker)
+                    tracker.add_widget(commander_select)
+                    tracker.add_widget(item_tracker)
+                    self.update_tracker()
+                    return tracker
+                except Exception as e:
+                    print(e)
+
+            def update_tracker(self):
+                received_ids = [item.item for item in self.ctx.items_received]
+                for faction, item_id in self.ctx.faction_item_ids.items():
+                    for commander_button in self.commander_buttons[faction]:
+                        commander_button.disabled = not (faction == "Starter" or item_id in received_ids)
+                self.unit_tracker.clear_widgets()
+                self.trigger_tracker.clear_widgets()
+                for name, item in self.tracker_items.items():
+                    if item.type in ("Unit", "Trigger"):
+                        status_color = (1, 1, 1, 1) if item.code is None or item.code in received_ids else (0.6, 0.2, 0.2, 1)
+                        label = ItemLabel(text=name, color=status_color)
+                        if item.type == "Unit":
+                            self.unit_tracker.add_widget(label)
+                        else:
+                            self.trigger_tracker.add_widget(label)
+                self.boost_tracker.clear_widgets()
+                extra_income = received_ids.count(52023) * self.ctx.income_boost_multiplier
+                extra_defense = received_ids.count(52024) * self.ctx.commander_defense_boost_multiplier
+                income_boost = ItemLabel(text="Extra Income: " + str(extra_income))
+                defense_boost = ItemLabel(text="Comm Defense: " + str(100 + extra_defense))
+                self.boost_tracker.add_widget(income_boost)
+                self.boost_tracker.add_widget(defense_boost)
+
+        self.ui = WargrooveManager(self)
+        data = pkgutil.get_data(WargrooveWorld.__module__, "Wargroove.kv").decode()
+        Builder.load_string(data)
+        self.ui_task = asyncio.create_task(self.ui.async_run(), name="UI")
+
+    def update_commander_data(self):
+        if self.can_choose_commander:
+            faction_items = 0
+            faction_item_names = [faction + ' Commanders' for faction in faction_table.keys()]
+            for network_item in self.items_received:
+                if self.item_names[network_item.item] in faction_item_names:
+                    faction_items += 1
+            starting_groove = (faction_items - 1) * self.starting_groove_multiplier
+            # Must be an integer larger than 0
+            starting_groove = int(max(starting_groove, 0))
+            data = {
+                "commander": self.current_commander.internal_name,
+                "starting_groove": starting_groove
+            }
+        else:
+            data = {
+                "commander": "seed",
+                "starting_groove": 0
+            }
+        filename = 'commander.json'
+        with open(os.path.join(self.game_communication_path, filename), 'w') as f:
+            json.dump(data, f)
+        if self.ui:
+            self.ui.update_tracker()
+
+    def set_commander(self, commander_name: str) -> bool:
+        """Sets the current commander to the given one, if possible"""
+        if not self.can_choose_commander:
+            wg_logger.error("Cannot set commanders in this game mode.")
+            return
+        match_name = commander_name.lower()
+        for commander, unlocked in self.get_commanders():
+            if commander.name.lower() == match_name or commander.alt_name and commander.alt_name.lower() == match_name:
+                if unlocked:
+                    self.current_commander = commander
+                    self.syncing = True
+                    wg_logger.info(f"Commander set to {commander.name}.")
+                    self.update_commander_data()
+                    return True
+                else:
+                    wg_logger.error(f"Commander {commander.name} has not been unlocked.")
+                    return False
+        else:
+            wg_logger.error(f"{commander_name} is not a recognized Wargroove commander.")
+
+    def get_commanders(self) -> List[Tuple[CommanderData, bool]]:
+        """Gets a list of commanders with their unlocked status"""
+        commanders = []
+        received_ids = [item.item for item in self.items_received]
+        for faction in faction_table.keys():
+            unlocked = faction == 'Starter' or self.faction_item_ids[faction] in received_ids
+            commanders += [(commander, unlocked) for commander in faction_table[faction]]
+        return commanders
+
+
+async def game_watcher(ctx: WargrooveContext):
+    from worlds.wargroove.Locations import location_table
+    while not ctx.exit_event.is_set():
+        if ctx.syncing == True:
+            sync_msg = [{'cmd': 'Sync'}]
+            if ctx.locations_checked:
+                sync_msg.append({"cmd": "LocationChecks", "locations": list(ctx.locations_checked)})
+            await ctx.send_msgs(sync_msg)
+            ctx.syncing = False
+        sending = []
+        victory = False
+        for root, dirs, files in os.walk(ctx.game_communication_path):
+            for file in files:
+                if file.find("send") > -1:
+                    st = file.split("send", -1)[1]
+                    sending = sending+[(int(st))]
+                if file.find("victory") > -1:
+                    victory = True
+        ctx.locations_checked = sending
+        message = [{"cmd": 'LocationChecks', "locations": sending}]
+        await ctx.send_msgs(message)
+        if not ctx.finished_game and victory:
+            await ctx.send_msgs([{"cmd": "StatusUpdate", "status": ClientStatus.CLIENT_GOAL}])
+            ctx.finished_game = True
+        await asyncio.sleep(0.1)
+
+
+def print_error_and_close(msg):
+    logger.error("Error: " + msg)
+    Utils.messagebox("Error", msg, error=True)
+    sys.exit(1)
+
+if __name__ == '__main__':
+    async def main(args):
+        ctx = WargrooveContext(args.connect, args.password)
+        ctx.server_task = asyncio.create_task(server_loop(ctx), name="server loop")
+        if gui_enabled:
+            ctx.run_gui()
+        ctx.run_cli()
+        progression_watcher = asyncio.create_task(
+            game_watcher(ctx), name="WargrooveProgressionWatcher")
+
+        await ctx.exit_event.wait()
+        ctx.server_address = None
+
+        await progression_watcher
+
+        await ctx.shutdown()
+
+    import colorama
+
+    parser = get_base_parser(description="Wargroove Client, for text interfacing.")
+
+    args, rest = parser.parse_known_args()
+    colorama.init()
+    asyncio.run(main(args))
+    colorama.deinit()

WebHost.py

@@ -1,5 +1,4 @@
 import os
-import sys
 import multiprocessing
 import logging
 import typing
@@ -12,9 +11,9 @@ ModuleUpdate.update()
 # in case app gets imported by something like gunicorn
 import Utils
 
-Utils.local_path.cached_path = os.path.dirname(__file__)
+Utils.local_path.cached_path = os.path.dirname(__file__) or "."  # py3.8 is not abs. remove "." when dropping 3.8
 
-from WebHostLib import app as raw_app
+from WebHostLib import register, app as raw_app
 from waitress import serve
 
 from WebHostLib.models import db
@@ -22,19 +21,23 @@ from WebHostLib.autolauncher import autohost, autogen
 from WebHostLib.lttpsprites import update_sprites_lttp
 from WebHostLib.options import create as create_options_files
 
-from worlds.AutoWorld import AutoWorldRegister, WebWorld
-
 configpath = os.path.abspath("config.yaml")
 if not os.path.exists(configpath):  # fall back to config.yaml in home
     configpath = os.path.abspath(Utils.user_path('config.yaml'))
 
 
 def get_app():
+    register()
     app = raw_app
-    if os.path.exists(configpath):
+    if os.path.exists(configpath) and not app.config["TESTING"]:
         import yaml
         app.config.from_file(configpath, yaml.safe_load)
         logging.info(f"Updated config from {configpath}")
+    if not app.config["HOST_ADDRESS"]:
+        logging.info("Getting public IP, as HOST_ADDRESS is empty.")
+        app.config["HOST_ADDRESS"] = Utils.get_public_ipv4()
+        logging.info(f"HOST_ADDRESS was set to {app.config['HOST_ADDRESS']}")
+
     db.bind(**app.config["PONY"])
     db.generate_mapping(create_tables=True)
     return app
@@ -43,19 +46,39 @@ def get_app():
 def create_ordered_tutorials_file() -> typing.List[typing.Dict[str, typing.Any]]:
     import json
     import shutil
+    import zipfile
+
+    zfile: zipfile.ZipInfo
+
+    from worlds.AutoWorld import AutoWorldRegister
     worlds = {}
     data = []
     for game, world in AutoWorldRegister.world_types.items():
-        if hasattr(world.web, 'tutorials'):
+        if hasattr(world.web, 'tutorials') and (not world.hidden or game == 'Archipelago'):
             worlds[game] = world
+
+    base_target_path = Utils.local_path("WebHostLib", "static", "generated", "docs")
     for game, world in worlds.items():
         # copy files from world's docs folder to the generated folder
-        source_path = Utils.local_path(os.path.dirname(sys.modules[world.__module__].__file__), 'docs')
-        target_path = Utils.local_path("WebHostLib", "static", "generated", "docs", game)
-        files = os.listdir(source_path)
-        for file in files:
-            os.makedirs(os.path.dirname(Utils.local_path(target_path, file)), exist_ok=True)
-            shutil.copyfile(Utils.local_path(source_path, file), Utils.local_path(target_path, file))
+        target_path = os.path.join(base_target_path, game)
+        os.makedirs(target_path, exist_ok=True)
+
+        if world.zip_path:
+            zipfile_path = world.zip_path
+
+            assert os.path.isfile(zipfile_path), f"{zipfile_path} is not a valid file(path)."
+            assert zipfile.is_zipfile(zipfile_path), f"{zipfile_path} is not a valid zipfile."
+
+            with zipfile.ZipFile(zipfile_path) as zf:
+                for zfile in zf.infolist():
+                    if not zfile.is_dir() and "/docs/" in zfile.filename:
+                        zf.extract(zfile, target_path)
+        else:
+            source_path = Utils.local_path(os.path.dirname(world.__file__), "docs")
+            files = os.listdir(source_path)
+            for file in files:
+                shutil.copyfile(Utils.local_path(source_path, file), Utils.local_path(target_path, file))
+
         # build a json tutorial dict per game
         game_data = {'gameTitle': game, 'tutorials': []}
         for tutorial in world.web.tutorials:
@@ -67,7 +90,7 @@ def create_ordered_tutorials_file() -> typing.List[typing.Dict[str, typing.Any]]
                     'language': tutorial.language,
                     'filename': game + '/' + tutorial.file_name,
                     'link': f'{game}/{tutorial.link}',
-                    'authors': tutorial.author
+                    'authors': tutorial.authors
                 }]
             }
 
@@ -75,7 +98,6 @@ def create_ordered_tutorials_file() -> typing.List[typing.Dict[str, typing.Any]]
             for guide in game_data['tutorials']:
                 if guide and tutorial.tutorial_name == guide['name']:
                     guide['files'].append(current_tutorial['files'][0])
-                    added = True
                     break
             else:
                 game_data['tutorials'].append(current_tutorial)
@@ -86,7 +108,7 @@ def create_ordered_tutorials_file() -> typing.List[typing.Dict[str, typing.Any]]
         for games in data:
             if 'Archipelago' in games['gameTitle']:
                 generic_data = data.pop(data.index(games))
-        sorted_data = [generic_data] + sorted(data, key=lambda entry: entry["gameTitle"].lower())
+        sorted_data = [generic_data] + Utils.title_sorted(data, key=lambda entry: entry["gameTitle"])
         json.dump(sorted_data, json_target, indent=2, ensure_ascii=False)
     return sorted_data
 
@@ -109,7 +131,6 @@ if __name__ == "__main__":
         autogen(app.config)
     if app.config["SELFHOST"]:  # using WSGI, you just want to run get_app()
         if app.config["DEBUG"]:
-            autohost(app.config)
             app.run(debug=True, port=app.config["PORT"])
         else:
             serve(app, port=app.config["PORT"], threads=app.config["WAITRESS_THREADS"])

WebHostLib/README.md

@@ -0,0 +1,46 @@
+# WebHost
+
+## Contribution Guidelines
+**Thank you for your interest in contributing to the Archipelago website!**  
+Much of the content on the website is generated automatically, but there are some things
+that need a personal touch. For those things, we rely on contributions from both the core
+team and the community. The current primary maintainer of the website is Farrak Kilhn.
+He may be found on Discord as `Farrak Kilhn#0418`, or on GitHub as `LegendaryLinux`.
+
+### Small Changes
+Little changes like adding a button or a couple new select elements are perfectly fine.
+Tweaks to style specific to a PR's content are also probably not a problem. For example, if
+you build a new page which needs two side by side tables, and you need to write a CSS file
+specific to your page, that is perfectly reasonable.
+
+### Content Additions
+Once you develop a new feature or add new content the website, make a pull request. It will
+be reviewed by the community and there will probably be some discussion around it. Depending
+on the size of the feature, and if new styles are required, there may be an additional step
+before the PR is accepted wherein Farrak works with the designer to implement styles.
+
+### Restrictions on Style Changes
+A professional designer is paid to develop the styles and assets for the Archipelago website.
+In an effort to maintain a consistent look and feel, pull requests which *exclusively*
+change site styles are rejected. Please note this applies to code which changes the overall
+look and feel of the site, not to small tweaks to CSS for your custom page. The intention
+behind these restrictions is to maintain a curated feel for the design of the site. If
+any PR affects the overall feel of the site but includes additive changes, there will
+likely be a conversation about how to implement those changes without compromising the
+curated site style. It is therefore worth noting there are a couple files which, if
+changed in your pull request, will cause it to draw additional scrutiny.
+
+These closely guarded files are:
+- `globalStyles.css`
+- `islandFooter.css`
+- `landing.css`
+- `markdown.css`
+- `tooltip.css`
+
+### Site Themes
+There are several themes available for game pages. It is possible to request a new theme in
+the `#art-and-design` channel on Discord. Because themes are created by the designer, they
+are not free, and take some time to create. Farrak works closely with the designer to implement
+these themes, and pays for the assets out of pocket. Therefore, only a couple themes per year
+are added. If a proposed theme seems like a cool idea and the community likes it, there is a
+good chance it will become a reality.

WebHostLib/__init__.py

@@ -1,16 +1,15 @@
-import os
-import uuid
 import base64
+import os
 import socket
+import uuid
 
-import jinja2.exceptions
-from pony.flask import Pony
-from flask import Flask, request, redirect, url_for, render_template, Response, session, abort, send_from_directory
+from flask import Flask
 from flask_caching import Cache
 from flask_compress import Compress
-from worlds.AutoWorld import AutoWorldRegister
+from pony.flask import Pony
+from werkzeug.routing import BaseConverter
 
-from .models import *
+from Utils import title_sorted
 
 UPLOAD_FOLDER = os.path.relpath('uploads')
 LOGS_FOLDER = os.path.relpath('logs')
@@ -25,6 +24,8 @@ app.jinja_env.filters['all'] = all
 app.config["SELFHOST"] = True  # application process is in charge of running the websites
 app.config["GENERATORS"] = 8  # maximum concurrent world gens
 app.config["SELFLAUNCH"] = True  # application process is in charge of launching Rooms.
+app.config["SELFLAUNCHCERT"] = None  # can point to a SSL Certificate to encrypt Room websocket connections
+app.config["SELFLAUNCHKEY"] = None  # can point to a SSL Certificate Key to encrypt Room websocket connections
 app.config["SELFGEN"] = True  # application process is in charge of scheduling Generations.
 app.config["DEBUG"] = False
 app.config["PORT"] = 80
@@ -32,8 +33,10 @@ app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
 app.config['MAX_CONTENT_LENGTH'] = 64 * 1024 * 1024  # 64 megabyte limit
 # if you want to deploy, make sure you have a non-guessable secret key
 app.config["SECRET_KEY"] = bytes(socket.gethostname(), encoding="utf-8")
-# at what amount of worlds should scheduling be used, instead of rolling in the webthread
-app.config["JOB_THRESHOLD"] = 2
+# at what amount of worlds should scheduling be used, instead of rolling in the web-thread
+app.config["JOB_THRESHOLD"] = 1
+# after what time in seconds should generation be aborted, freeing the queue slot. Can be set to None to disable.
+app.config["JOB_TIME"] = 600
 app.config['SESSION_PERMANENT'] = True
 
 # waitress uses one thread for I/O, these are for processing of views that then get sent
@@ -46,15 +49,13 @@ app.config["PONY"] = {
     'create_db': True
 }
 app.config["MAX_ROLL"] = 20
-app.config["CACHE_TYPE"] = "simple"
+app.config["CACHE_TYPE"] = "flask_caching.backends.SimpleCache"
 app.config["JSON_AS_ASCII"] = False
-app.config["PATCH_TARGET"] = "archipelago.gg"
+app.config["HOST_ADDRESS"] = ""
 
 cache = Cache(app)
 Compress(app)
 
-from werkzeug.routing import BaseConverter
-
 
 class B64UUIDConverter(BaseConverter):
 
@@ -68,160 +69,20 @@ class B64UUIDConverter(BaseConverter):
 # short UUID
 app.url_map.converters["suuid"] = B64UUIDConverter
 app.jinja_env.filters['suuid'] = lambda value: base64.urlsafe_b64encode(value.bytes).rstrip(b'=').decode('ascii')
+app.jinja_env.filters["title_sorted"] = title_sorted
 
 
-def get_world_theme(game_name: str):
-    if game_name in AutoWorldRegister.world_types:
-        return AutoWorldRegister.world_types[game_name].web.theme
-    return 'grass'
+def register():
+    """Import submodules, triggering their registering on flask routing.
+    Note: initializes worlds subsystem."""
+    # has automatic patch integration
+    import worlds.AutoWorld
+    import worlds.Files
+    app.jinja_env.filters['supports_apdeltapatch'] = lambda game_name: \
+        game_name in worlds.Files.AutoPatchRegister.patch_types
 
+    from WebHostLib.customserver import run_server_process
+    # to trigger app routing picking up on it
+    from . import tracker, upload, landing, check, generate, downloads, api, stats, misc
 
-@app.before_request
-def register_session():
-    session.permanent = True  # technically 31 days after the last visit
-    if not session.get("_id", None):
-        session["_id"] = uuid4()  # uniquely identify each session without needing a login
-
-
-@app.errorhandler(404)
-@app.errorhandler(jinja2.exceptions.TemplateNotFound)
-def page_not_found(err):
-    return render_template('404.html'), 404
-
-
-# Start Playing Page
-@app.route('/start-playing')
-def start_playing():
-    return render_template(f"startPlaying.html")
-
-
-@app.route('/weighted-settings')
-def weighted_settings():
-    return render_template(f"weighted-settings.html")
-
-
-# Player settings pages
-@app.route('/games/<string:game>/player-settings')
-def player_settings(game):
-    return render_template(f"player-settings.html", game=game, theme=get_world_theme(game))
-
-
-# Game Info Pages
-@app.route('/games/<string:game>/info/<string:lang>')
-def game_info(game, lang):
-    return render_template('gameInfo.html', game=game, lang=lang, theme=get_world_theme(game))
-
-
-# List of supported games
-@app.route('/games')
-def games():
-    worlds = {}
-    for game, world in AutoWorldRegister.world_types.items():
-        if not world.hidden:
-            worlds[game] = world
-    return render_template("supportedGames.html", worlds=worlds)
-
-
-@app.route('/tutorial/<string:game>/<string:file>/<string:lang>')
-def tutorial(game, file, lang):
-    return render_template("tutorial.html", game=game, file=file, lang=lang, theme=get_world_theme(game))
-
-
-@app.route('/tutorial/')
-def tutorial_landing():
-    worlds = {}
-    for game, world in AutoWorldRegister.world_types.items():
-        if not world.hidden:
-            worlds[game] = world
-    return render_template("tutorialLanding.html")
-
-
-@app.route('/faq/<string:lang>/')
-def faq(lang):
-    return render_template("faq.html", lang=lang)
-
-
-@app.route('/seed/<suuid:seed>')
-def view_seed(seed: UUID):
-    seed = Seed.get(id=seed)
-    if not seed:
-        abort(404)
-    return render_template("viewSeed.html", seed=seed, slot_count=count(seed.slots))
-
-
-@app.route('/new_room/<suuid:seed>')
-def new_room(seed: UUID):
-    seed = Seed.get(id=seed)
-    if not seed:
-        abort(404)
-    room = Room(seed=seed, owner=session["_id"], tracker=uuid4())
-    commit()
-    return redirect(url_for("host_room", room=room.id))
-
-
-def _read_log(path: str):
-    if os.path.exists(path):
-        with open(path, encoding="utf-8-sig") as log:
-            yield from log
-    else:
-        yield f"Logfile {path} does not exist. " \
-              f"Likely a crash during spinup of multiworld instance or it is still spinning up."
-
-
-@app.route('/log/<suuid:room>')
-def display_log(room: UUID):
-    return Response(_read_log(os.path.join("logs", str(room) + ".txt")), mimetype="text/plain;charset=UTF-8")
-
-
-@app.route('/room/<suuid:room>', methods=['GET', 'POST'])
-def host_room(room: UUID):
-    room = Room.get(id=room)
-    if room is None:
-        return abort(404)
-    if request.method == "POST":
-        if room.owner == session["_id"]:
-            cmd = request.form["cmd"]
-            if cmd:
-                Command(room=room, commandtext=cmd)
-                commit()
-
-    with db_session:
-        room.last_activity = datetime.utcnow()  # will trigger a spinup, if it's not already running
-
-    return render_template("hostRoom.html", room=room)
-
-
-@app.route('/favicon.ico')
-def favicon():
-    return send_from_directory(os.path.join(app.root_path, 'static/static'),
-                               'favicon.ico', mimetype='image/vnd.microsoft.icon')
-
-
-@app.route('/discord')
-def discord():
-    return redirect("https://discord.gg/archipelago")
-
-
-@app.route('/datapackage')
-@cache.cached()
-def get_datapackge():
-    """A pretty print version of /api/datapackage"""
-    from worlds import network_data_package
-    import json
-    return Response(json.dumps(network_data_package, indent=4), mimetype="text/plain")
-
-
-@app.route('/index')
-@app.route('/sitemap')
-def get_sitemap():
-    available_games = []
-    for game, world in AutoWorldRegister.world_types.items():
-        if not world.hidden:
-            available_games.append(game)
-    return render_template("siteMap.html", games=available_games)
-
-
-from WebHostLib.customserver import run_server_process
-from . import tracker, upload, landing, check, generate, downloads, api, stats  # to trigger app routing picking up on it
-
-app.register_blueprint(api.api_endpoints)
+    app.register_blueprint(api.api_endpoints)

WebHostLib/api/__init__.py

@@ -1,11 +1,11 @@
 """API endpoints package."""
-from uuid import UUID
 from typing import List, Tuple
+from uuid import UUID
 
 from flask import Blueprint, abort
 
-from ..models import Room, Seed
 from .. import cache
+from ..models import Room, Seed
 
 api_endpoints = Blueprint('api', __name__, url_prefix="/api")
 
@@ -32,17 +32,27 @@ def room_info(room: UUID):
 
 @api_endpoints.route('/datapackage')
 @cache.cached()
-def get_datapackge():
+def get_datapackage():
     from worlds import network_data_package
     return network_data_package
 
 
 @api_endpoints.route('/datapackage_version')
 @cache.cached()
-def get_datapackge_versions():
-    from worlds import network_data_package, AutoWorldRegister
+def get_datapackage_versions():
+    from worlds import AutoWorldRegister
+
     version_package = {game: world.data_version for game, world in AutoWorldRegister.world_types.items()}
-    version_package["version"] = network_data_package["version"]
+    return version_package
+
+
+@api_endpoints.route('/datapackage_checksum')
+@cache.cached()
+def get_datapackage_checksums():
+    from worlds import network_data_package
+    version_package = {
+        game: game_data["checksum"] for game, game_data in network_data_package["games"].items()
+    }
     return version_package
 
 

WebHostLib/api/generate.py

@@ -1,15 +1,15 @@
 import json
 import pickle
-
 from uuid import UUID
 
-from . import api_endpoints
-from flask import request, session, url_for
+from flask import request, session, url_for, Markup
 from pony.orm import commit
 
-from WebHostLib import app, Generation, STATE_QUEUED, Seed, STATE_ERROR
+from WebHostLib import app
 from WebHostLib.check import get_yaml_data, roll_options
 from WebHostLib.generate import get_meta
+from WebHostLib.models import Generation, STATE_QUEUED, Seed, STATE_ERROR
+from . import api_endpoints
 
 
 @api_endpoints.route('/generate', methods=['POST'])
@@ -21,13 +21,18 @@ def generate_api():
         if 'file' in request.files:
             file = request.files['file']
             options = get_yaml_data(file)
-            if type(options) == str:
+            if isinstance(options, Markup):
+                return {"text": options.striptags()}, 400
+            if isinstance(options, str):
                 return {"text": options}, 400
             if "race" in request.form:
                 race = bool(0 if request.form["race"] in {"false"} else int(request.form["race"]))
             meta_options_source = request.form
 
-        json_data = request.get_json()
+        # json_data is optional, we can have it silently fall to None as it used to do.
+        # See https://flask.palletsprojects.com/en/2.2.x/api/#flask.Request.get_json -> Changelog -> 2.1
+        json_data = request.get_json(silent=True)
+
         if json_data:
             meta_options_source = json_data
             if 'weights' in json_data:
@@ -43,9 +48,8 @@ def generate_api():
         if len(options) > app.config["MAX_ROLL"]:
             return {"text": "Max size of multiworld exceeded",
                     "detail": app.config["MAX_ROLL"]}, 409
-        meta = get_meta(meta_options_source)
-        meta["race"] = race
-        results, gen_options = roll_options(options, meta["plando_options"])
+        meta = get_meta(meta_options_source, race)
+        results, gen_options = roll_options(options, set(meta["plando_options"]))
         if any(type(result) == str for result in results.values()):
             return {"text": str(results),
                     "detail": results}, 400

WebHostLib/api/user.py

@@ -1,6 +1,7 @@
 from flask import session, jsonify
+from pony.orm import select
 
-from WebHostLib.models import *
+from WebHostLib.models import Room, Seed
 from . import api_endpoints, get_players
 
 

WebHostLib/autolauncher.py

@@ -1,13 +1,14 @@
 from __future__ import annotations
-import logging
+
 import json
+import logging
 import multiprocessing
-from datetime import timedelta, datetime
-import concurrent.futures
-import sys
-import typing
-import time
 import os
+import sys
+import threading
+import time
+import typing
+from datetime import timedelta, datetime
 
 from pony.orm import db_session, select, commit
 
@@ -17,6 +18,7 @@ from Utils import restricted_loads
 class CommonLocker():
     """Uses a file lock to signal that something is already running"""
     lock_folder = "file_locks"
+
     def __init__(self, lockname: str, folder=None):
         if folder:
             self.lock_folder = folder
@@ -53,7 +55,7 @@ else:  # unix
         def __enter__(self):
             try:
                 self.fp = open(self.lockfile, "wb")
-                fcntl.flock(self.fp.fileno(), fcntl.LOCK_EX)
+                fcntl.flock(self.fp.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
             except OSError as e:
                 raise AlreadyRunningException() from e
 
@@ -110,6 +112,7 @@ def autohost(config: dict):
     def keep_running():
         try:
             with Locker("autohost"):
+                run_guardian()
                 while 1:
                     time.sleep(0.1)
                     with db_session:
@@ -132,7 +135,7 @@ def autogen(config: dict):
             with Locker("autogen"):
 
                 with multiprocessing.Pool(config["GENERATORS"], initializer=init_db,
-                                          initargs=(config["PONY"],)) as generator_pool:
+                                          initargs=(config["PONY"],), maxtasksperchild=10) as generator_pool:
                     with db_session:
                         to_start = select(generation for generation in Generation if generation.state == STATE_STARTED)
 
@@ -151,8 +154,10 @@ def autogen(config: dict):
                     while 1:
                         time.sleep(0.1)
                         with db_session:
+                            # for update locks the database row(s) during transaction, preventing writes from elsewhere
                             to_start = select(
-                                generation for generation in Generation if generation.state == STATE_QUEUED)
+                                generation for generation in Generation
+                                if generation.state == STATE_QUEUED).for_update()
                             for generation in to_start:
                                 launch_generator(generator_pool, generation)
         except AlreadyRunningException:
@@ -162,40 +167,80 @@ def autogen(config: dict):
     threading.Thread(target=keep_running, name="AP_Autogen").start()
 
 
-multiworlds = {}
-
-guardians = concurrent.futures.ThreadPoolExecutor(2, thread_name_prefix="Guardian")
+multiworlds: typing.Dict[type(Room.id), MultiworldInstance] = {}
 
 
 class MultiworldInstance():
     def __init__(self, room: Room, config: dict):
         self.room_id = room.id
         self.process: typing.Optional[multiprocessing.Process] = None
-        multiworlds[self.room_id] = self
+        with guardian_lock:
+            multiworlds[self.room_id] = self
         self.ponyconfig = config["PONY"]
+        self.cert = config["SELFLAUNCHCERT"]
+        self.key = config["SELFLAUNCHKEY"]
+        self.host = config["HOST_ADDRESS"]
 
     def start(self):
         if self.process and self.process.is_alive():
             return False
 
         logging.info(f"Spinning up {self.room_id}")
-        self.process = multiprocessing.Process(group=None, target=run_server_process,
-                                               args=(self.room_id, self.ponyconfig),
-                                               name="MultiHost")
-        self.process.start()
-        self.guardian = guardians.submit(self._collect)
+        process = multiprocessing.Process(group=None, target=run_server_process,
+                                          args=(self.room_id, self.ponyconfig, get_static_server_data(),
+                                                self.cert, self.key, self.host),
+                                          name="MultiHost")
+        process.start()
+        # bind after start to prevent thread sync issues with guardian.
+        self.process = process
 
     def stop(self):
         if self.process:
             self.process.terminate()
             self.process = None
 
-    def _collect(self):
+    def done(self):
+        return self.process and not self.process.is_alive()
+
+    def collect(self):
         self.process.join()  # wait for process to finish
         self.process = None
-        self.guardian = None
+
+
+guardian = None
+guardian_lock = threading.Lock()
+
+
+def run_guardian():
+    global guardian
+    global multiworlds
+    with guardian_lock:
+        if not guardian:
+            try:
+                import resource
+            except ModuleNotFoundError:
+                pass  # unix only module
+            else:
+                # Each Server is another file handle, so request as many as we can from the system
+                file_limit = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
+                # set soft limit to hard limit
+                resource.setrlimit(resource.RLIMIT_NOFILE, (file_limit, file_limit))
+
+            def guard():
+                while 1:
+                    time.sleep(1)
+                    done = []
+                    with guardian_lock:
+                        for key, instance in multiworlds.items():
+                            if instance.done():
+                                instance.collect()
+                                done.append(key)
+                        for key in done:
+                            del (multiworlds[key])
+
+            guardian = threading.Thread(name="Guardian", target=guard)
 
 
 from .models import Room, Generation, STATE_QUEUED, STATE_STARTED, STATE_ERROR, db, Seed
-from .customserver import run_server_process
+from .customserver import run_server_process, get_static_server_data
 from .generate import gen_game

WebHostLib/check.py

@@ -1,7 +1,7 @@
 import zipfile
 from typing import *
 
-from flask import request, flash, redirect, url_for, session, render_template
+from flask import request, flash, redirect, url_for, render_template, Markup
 
 from WebHostLib import app
 
@@ -12,7 +12,7 @@ def allowed_file(filename):
     return filename.endswith(('.txt', ".yaml", ".zip"))
 
 
-from Generate import roll_settings
+from Generate import roll_settings, PlandoOptions
 from Utils import parse_yamls
 
 
@@ -25,7 +25,7 @@ def check():
         else:
             file = request.files['file']
             options = get_yaml_data(file)
-            if type(options) == str:
+            if isinstance(options, str):
                 flash(options)
             else:
                 results, _ = roll_options(options)
@@ -38,7 +38,7 @@ def mysterycheck():
     return redirect(url_for("check"), 301)
 
 
-def get_yaml_data(file) -> Union[Dict[str, str], str]:
+def get_yaml_data(file) -> Union[Dict[str, str], str, Markup]:
     options = {}
     # if user does not select file, browser also
     # submit an empty part without filename
@@ -50,9 +50,14 @@ def get_yaml_data(file) -> Union[Dict[str, str], str]:
             with zipfile.ZipFile(file, 'r') as zfile:
                 infolist = zfile.infolist()
 
+                if any(file.filename.endswith(".archipelago") for file in infolist):
+                    return Markup("Error: Your .zip file contains an .archipelago file. "
+                                  'Did you mean to <a href="/uploads">host a game</a>?')
+
                 for file in infolist:
                     if file.filename.endswith(banned_zip_contents):
-                        return "Uploaded data contained a rom file, which is likely to contain copyrighted material. Your file was deleted."
+                        return "Uploaded data contained a rom file, which is likely to contain copyrighted material. " \
+                               "Your file was deleted."
                     elif file.filename.endswith((".yaml", ".json", ".yml", ".txt")):
                         options[file.filename] = zfile.open(file, "r").read()
         else:
@@ -65,7 +70,7 @@ def get_yaml_data(file) -> Union[Dict[str, str], str]:
 def roll_options(options: Dict[str, Union[dict, str]],
                  plando_options: Set[str] = frozenset({"bosses", "items", "connections", "texts"})) -> \
         Tuple[Dict[str, Union[str, bool]], Dict[str, dict]]:
-    plando_options = set(plando_options)
+    plando_options = PlandoOptions.from_set(set(plando_options))
     results = {}
     rolled_results = {}
     for filename, text in options.items():

WebHostLib/customserver.py

@@ -1,20 +1,25 @@
 from __future__ import annotations
 
+import asyncio
+import collections
+import datetime
 import functools
 import logging
-import websockets
-import asyncio
+import pickle
+import random
 import socket
 import threading
 import time
-import random
-import pickle
+import typing
+
+import websockets
+from pony.orm import commit, db_session, select
 
 import Utils
-from .models import *
 
-from MultiServer import Context, server, auto_shutdown, ServerCommandProcessor, ClientMessageProcessor
-from Utils import get_public_ipv4, get_public_ipv6, restricted_loads
+from MultiServer import Context, server, auto_shutdown, ServerCommandProcessor, ClientMessageProcessor, load_server_cert
+from Utils import get_public_ipv4, get_public_ipv6, restricted_loads, cache_argsless
+from .models import Command, GameDataPackage, Room, db
 
 
 class CustomClientMessageProcessor(ClientMessageProcessor):
@@ -39,7 +44,7 @@ class CustomClientMessageProcessor(ClientMessageProcessor):
 import MultiServer
 
 MultiServer.client_message_processor = CustomClientMessageProcessor
-del (MultiServer)
+del MultiServer
 
 
 class DBCommandProcessor(ServerCommandProcessor):
@@ -48,12 +53,23 @@ class DBCommandProcessor(ServerCommandProcessor):
 
 
 class WebHostContext(Context):
-    def __init__(self):
+    room_id: int
+
+    def __init__(self, static_server_data: dict):
+        # static server data is used during _load_game_data to load required data,
+        # without needing to import worlds system, which takes quite a bit of memory
+        self.static_server_data = static_server_data
         super(WebHostContext, self).__init__("", 0, "", "", 1, 40, True, "enabled", "enabled", "enabled", 0, 2)
+        del self.static_server_data
         self.main_loop = asyncio.get_running_loop()
         self.video = {}
         self.tags = ["AP", "WebHost"]
 
+    def _load_game_data(self):
+        for key, value in self.static_server_data.items():
+            setattr(self, key, value)
+        self.non_hintable_names = collections.defaultdict(frozenset, self.non_hintable_names)
+
     def listen_to_db_commands(self):
         cmdprocessor = DBCommandProcessor(self)
 
@@ -76,7 +92,21 @@ class WebHostContext(Context):
         else:
             self.port = get_random_port()
 
-        return self._load(self.decompress(room.seed.multidata), True)
+        multidata = self.decompress(room.seed.multidata)
+        game_data_packages = {}
+        for game in list(multidata.get("datapackage", {})):
+            game_data = multidata["datapackage"][game]
+            if "checksum" in game_data:
+                if self.gamespackage.get(game, {}).get("checksum") == game_data["checksum"]:
+                    # non-custom. remove from multidata
+                    # games package could be dropped from static data once all rooms embed data package
+                    del multidata["datapackage"][game]
+                else:
+                    row = GameDataPackage.get(checksum=game_data["checksum"])
+                    if row:  # None if rolled on >= 0.3.9 but uploaded to <= 0.3.8. multidata should be complete
+                        game_data_packages[game] = Utils.restricted_loads(row.data)
+
+        return self._load(multidata, game_data_packages, True)
 
     @db_session
     def init_save(self, enabled: bool = True):
@@ -94,7 +124,7 @@ class WebHostContext(Context):
         room.multisave = pickle.dumps(self.get_save())
         # saving only occurs on activity, so we can "abuse" this information to mark this as last_activity
         if not exit_save:  # we don't want to count a shutdown as activity, which would restart the server again
-            room.last_activity = datetime.utcnow()
+            room.last_activity = datetime.datetime.utcnow()
         return True
 
     def get_save(self) -> dict:
@@ -107,36 +137,63 @@ def get_random_port():
     return random.randint(49152, 65535)
 
 
-def run_server_process(room_id, ponyconfig: dict):
+@cache_argsless
+def get_static_server_data() -> dict:
+    import worlds
+    data = {
+        "non_hintable_names": {},
+        "gamespackage": worlds.network_data_package["games"],
+        "item_name_groups": {world_name: world.item_name_groups for world_name, world in
+                             worlds.AutoWorldRegister.world_types.items()},
+        "location_name_groups": {world_name: world.location_name_groups for world_name, world in
+                                 worlds.AutoWorldRegister.world_types.items()},
+    }
+
+    for world_name, world in worlds.AutoWorldRegister.world_types.items():
+        data["non_hintable_names"][world_name] = world.hint_blacklist
+
+    return data
+
+
+def run_server_process(room_id, ponyconfig: dict, static_server_data: dict,
+                       cert_file: typing.Optional[str], cert_key_file: typing.Optional[str],
+                       host: str):
     # establish DB connection for multidata and multisave
     db.bind(**ponyconfig)
     db.generate_mapping(check_tables=False)
 
     async def main():
         Utils.init_logging(str(room_id), write_mode="a")
-        ctx = WebHostContext()
+        ctx = WebHostContext(static_server_data)
         ctx.load(room_id)
         ctx.init_save()
-
+        ssl_context = load_server_cert(cert_file, cert_key_file) if cert_file else None
         try:
             ctx.server = websockets.serve(functools.partial(server, ctx=ctx), ctx.host, ctx.port, ping_timeout=None,
-                                          ping_interval=None)
+                                          ping_interval=None, ssl=ssl_context)
 
             await ctx.server
         except Exception:  # likely port in use - in windows this is OSError, but I didn't check the others
             ctx.server = websockets.serve(functools.partial(server, ctx=ctx), ctx.host, 0, ping_timeout=None,
-                                          ping_interval=None)
+                                          ping_interval=None, ssl=ssl_context)
 
             await ctx.server
+        port = 0
         for wssocket in ctx.server.ws_server.sockets:
             socketname = wssocket.getsockname()
             if wssocket.family == socket.AF_INET6:
-                logging.info(f'Hosting game at [{get_public_ipv6()}]:{socketname[1]}')
-                with db_session:
-                    room = Room.get(id=ctx.room_id)
-                    room.last_port = socketname[1]
+                # Prefer IPv4, as most users seem to not have working ipv6 support
+                if not port:
+                    port = socketname[1]
             elif wssocket.family == socket.AF_INET:
-                logging.info(f'Hosting game at {get_public_ipv4()}:{socketname[1]}')
+                port = socketname[1]
+        if port:
+            logging.info(f'Hosting game at {host}:{port}')
+            with db_session:
+                room = Room.get(id=ctx.room_id)
+                room.last_port = port
+        else:
+            logging.exception("Could not determine port. Likely hosting failure.")
         with db_session:
             ctx.auto_shutdown = Room.get(id=room_id).timeout
         ctx.shutdown_task = asyncio.create_task(auto_shutdown(ctx, []))
@@ -145,7 +202,17 @@ def run_server_process(room_id, ponyconfig: dict):
 
     from .autolauncher import Locker
     with Locker(room_id):
-        asyncio.run(main())
-
-
-from WebHostLib import LOGS_FOLDER
+        try:
+            asyncio.run(main())
+        except KeyboardInterrupt:
+            with db_session:
+                room = Room.get(id=room_id)
+                # ensure the Room does not spin up again on its own, minute of safety buffer
+                room.last_activity = datetime.datetime.utcnow() - datetime.timedelta(minutes=1, seconds=room.timeout)
+        except:
+            with db_session:
+                room = Room.get(id=room_id)
+                room.last_port = -1
+                # ensure the Room does not spin up again on its own, minute of safety buffer
+                room.last_activity = datetime.datetime.utcnow() - datetime.timedelta(minutes=1, seconds=room.timeout)
+            raise

WebHostLib/downloads.py

@@ -1,12 +1,13 @@
-import zipfile
 import json
+import zipfile
 from io import BytesIO
 
 from flask import send_file, Response, render_template
 from pony.orm import select
 
-from Patch import update_patch_data, preferred_endings, AutoPatchRegister
-from WebHostLib import app, Slot, Room, Seed, cache
+from worlds.Files import AutoPatchRegister
+from . import app, cache
+from .models import Slot, Room, Seed
 
 
 @app.route("/dl_patch/<suuid:room_id>/<int:patch_id>")
@@ -25,25 +26,23 @@ def download_patch(room_id, patch_id):
             with zipfile.ZipFile(filelike, "a") as zf:
                 with zf.open("archipelago.json", "r") as f:
                     manifest = json.load(f)
-                manifest["server"] = f"{app.config['PATCH_TARGET']}:{last_port}"
+                manifest["server"] = f"{app.config['HOST_ADDRESS']}:{last_port}" if last_port else None
                 with zipfile.ZipFile(new_file, "w") as new_zip:
                     for file in zf.infolist():
                         if file.filename == "archipelago.json":
                             new_zip.writestr("archipelago.json", json.dumps(manifest))
                         else:
                             new_zip.writestr(file.filename, zf.read(file), file.compress_type, 9)
-
+            if "patch_file_ending" in manifest:
+                patch_file_ending = manifest["patch_file_ending"]
+            else:
+                patch_file_ending = AutoPatchRegister.patch_types[patch.game].patch_file_ending
             fname = f"P{patch.player_id}_{patch.player_name}_{app.jinja_env.filters['suuid'](room_id)}" \
-                    f"{AutoPatchRegister.patch_types[patch.game].patch_file_ending}"
+                    f"{patch_file_ending}"
             new_file.seek(0)
-            return send_file(new_file, as_attachment=True, attachment_filename=fname)
+            return send_file(new_file, as_attachment=True, download_name=fname)
         else:
-            patch_data = update_patch_data(patch.data, server=f"{app.config['PATCH_TARGET']}:{last_port}")
-            patch_data = BytesIO(patch_data)
-
-            fname = f"P{patch.player_id}_{patch.player_name}_{app.jinja_env.filters['suuid'](room_id)}." \
-                    f"{preferred_endings[patch.game]}"
-        return send_file(patch_data, as_attachment=True, attachment_filename=fname)
+            return "Old Patch file, no longer compatible."
 
 
 @app.route("/dl_spoiler/<suuid:seed_id>")
@@ -55,7 +54,7 @@ def download_spoiler(seed_id):
 def download_slot_file(room_id, player_id: int):
     room = Room.get(id=room_id)
     slot_data: Slot = select(patch for patch in room.seed.slots if
-                   patch.player_id == player_id).first()
+                             patch.player_id == player_id).first()
 
     if not slot_data:
         return "Slot Data not found"
@@ -65,22 +64,36 @@ def download_slot_file(room_id, player_id: int):
         if slot_data.game == "Minecraft":
             from worlds.minecraft import mc_update_output
             fname = f"AP_{app.jinja_env.filters['suuid'](room_id)}_P{slot_data.player_id}_{slot_data.player_name}.apmc"
-            data = mc_update_output(slot_data.data, server=app.config['PATCH_TARGET'], port=room.last_port)
-            return send_file(io.BytesIO(data), as_attachment=True, attachment_filename=fname)
+            data = mc_update_output(slot_data.data, server=app.config['HOST_ADDRESS'], port=room.last_port)
+            return send_file(io.BytesIO(data), as_attachment=True, download_name=fname)
         elif slot_data.game == "Factorio":
             with zipfile.ZipFile(io.BytesIO(slot_data.data)) as zf:
                 for name in zf.namelist():
                     if name.endswith("info.json"):
-                        fname = name.rsplit("/", 1)[0]+".zip"
+                        fname = name.rsplit("/", 1)[0] + ".zip"
         elif slot_data.game == "Ocarina of Time":
-            fname = f"AP_{app.jinja_env.filters['suuid'](room_id)}_P{slot_data.player_id}_{slot_data.player_name}.apz5"
+            stream = io.BytesIO(slot_data.data)
+            if zipfile.is_zipfile(stream):
+                with zipfile.ZipFile(stream) as zf:
+                    for name in zf.namelist():
+                        if name.endswith(".zpf"):
+                            fname = name.rsplit(".", 1)[0] + ".apz5"
+            else: # pre-ootr-7.0 support
+                fname = f"AP_{app.jinja_env.filters['suuid'](room_id)}_P{slot_data.player_id}_{slot_data.player_name}.apz5"
         elif slot_data.game == "VVVVVV":
             fname = f"AP_{app.jinja_env.filters['suuid'](room_id)}_SP.apv6"
+        elif slot_data.game == "Zillion":
+            fname = f"AP_{app.jinja_env.filters['suuid'](room_id)}_SP.apzl"
         elif slot_data.game == "Super Mario 64":
             fname = f"AP_{app.jinja_env.filters['suuid'](room_id)}_SP.apsm64ex"
+        elif slot_data.game == "Dark Souls III":
+            fname = f"AP_{app.jinja_env.filters['suuid'](room_id)}.json"
+        elif slot_data.game == "Kingdom Hearts 2":
+            fname = f"AP_{app.jinja_env.filters['suuid'](room_id)}_P{slot_data.player_id}_{slot_data.player_name}.zip"
         else:
             return "Game download not supported."
-        return send_file(io.BytesIO(slot_data.data), as_attachment=True, attachment_filename=fname)
+        return send_file(io.BytesIO(slot_data.data), as_attachment=True, download_name=fname)
+
 
 @app.route("/templates")
 @cache.cached()
@@ -90,4 +103,4 @@ def list_yaml_templates():
     for world_name, world in AutoWorldRegister.world_types.items():
         if not world.hidden:
             files.append(world_name)
-    return render_template("templates.html", files=files)
+    return render_template("templates.html", files=files)

WebHostLib/generate.py

@@ -1,27 +1,28 @@
-import os
-import tempfile
-import random
 import json
+import os
+import pickle
+import random
+import tempfile
 import zipfile
+import concurrent.futures
 from collections import Counter
-from typing import Dict, Optional as TypeOptional
-from Utils import __version__
+from typing import Dict, Optional, Any, Union, List
 
 from flask import request, flash, redirect, url_for, session, render_template
+from pony.orm import commit, db_session
 
-from worlds.alttp.EntranceRandomizer import parse_arguments
-from Main import main as ERmain
 from BaseClasses import seeddigits, get_seed
-from Generate import handle_name
-import pickle
-
-from .models import *
+from Generate import handle_name, PlandoOptions
+from Main import main as ERmain
+from Utils import __version__
 from WebHostLib import app
+from worlds.alttp.EntranceRandomizer import parse_arguments
 from .check import get_yaml_data, roll_options
+from .models import Generation, STATE_ERROR, STATE_QUEUED, Seed, UUID
 from .upload import upload_zip_to_db
 
 
-def get_meta(options_source: dict) -> dict:
+def get_meta(options_source: dict, race: bool = False) -> Dict[str, Union[List[str], Dict[str, Any]]]:
     plando_options = {
         options_source.get("plando_bosses", ""),
         options_source.get("plando_items", ""),
@@ -30,16 +31,29 @@ def get_meta(options_source: dict) -> dict:
     }
     plando_options -= {""}
 
-    meta = {
+    server_options = {
         "hint_cost": int(options_source.get("hint_cost", 10)),
-        "forfeit_mode": options_source.get("forfeit_mode", "goal"),
+        "release_mode": options_source.get("release_mode", "goal"),
         "remaining_mode": options_source.get("remaining_mode", "disabled"),
         "collect_mode": options_source.get("collect_mode", "disabled"),
         "item_cheat": bool(int(options_source.get("item_cheat", 1))),
         "server_password": options_source.get("server_password", None),
-        "plando_options": list(plando_options)
     }
-    return meta
+    generator_options = {
+        "spoiler": int(options_source.get("spoiler", 0)),
+        "race": race
+    }
+
+    if race:
+        server_options["item_cheat"] = False
+        server_options["remaining_mode"] = "disabled"
+        generator_options["spoiler"] = 0
+
+    return {
+        "server_options": server_options,
+        "plando_options": list(plando_options),
+        "generator_options": generator_options,
+    }
 
 
 @app.route('/generate', methods=['GET', 'POST'])
@@ -52,21 +66,16 @@ def generate(race=False):
         else:
             file = request.files['file']
             options = get_yaml_data(file)
-            if type(options) == str:
+            if isinstance(options, str):
                 flash(options)
             else:
-                meta = get_meta(request.form)
-                meta["race"] = race
-                results, gen_options = roll_options(options, meta["plando_options"])
-
-                if race:
-                    meta["item_cheat"] = False
-                    meta["remaining_mode"] = "disabled"
+                meta = get_meta(request.form, race)
+                results, gen_options = roll_options(options, set(meta["plando_options"]))
 
                 if any(type(result) == str for result in results.values()):
                     return render_template("checkResult.html", results=results)
                 elif len(gen_options) > app.config["MAX_ROLL"]:
-                    flash(f"Sorry, generating of multiworlds is limited to {app.config['MAX_ROLL']} players for now. "
+                    flash(f"Sorry, generating of multiworlds is limited to {app.config['MAX_ROLL']} players. "
                           f"If you have a larger group, please generate it yourself and upload it.")
                 elif len(gen_options) >= app.config["JOB_THRESHOLD"]:
                     gen = Generation(
@@ -92,35 +101,35 @@ def generate(race=False):
     return render_template("generate.html", race=race, version=__version__)
 
 
-def gen_game(gen_options, meta: TypeOptional[Dict[str, object]] = None, owner=None, sid=None):
+def gen_game(gen_options: dict, meta: Optional[Dict[str, Any]] = None, owner=None, sid=None):
     if not meta:
-        meta: Dict[str, object] = {}
+        meta: Dict[str, Any] = {}
 
-    meta.setdefault("hint_cost", 10)
-    race = meta.get("race", False)
-    del (meta["race"])
-    plando_options = meta.get("plando", {"bosses", "items", "connections", "texts"})
-    del (meta["plando_options"])
-    try:
+    meta.setdefault("server_options", {}).setdefault("hint_cost", 10)
+    race = meta["generator_options"].setdefault("race", False)
+
+    def task():
         target = tempfile.TemporaryDirectory()
         playercount = len(gen_options)
         seed = get_seed()
-        random.seed(seed)
 
         if race:
-            random.seed()  # reset to time-based random source
+            random.seed()  # use time-based random source
+        else:
+            random.seed(seed)
 
         seedname = "W" + (f"{random.randint(0, pow(10, seeddigits) - 1)}".zfill(seeddigits))
 
         erargs = parse_arguments(['--multi', str(playercount)])
         erargs.seed = seed
-        erargs.name = {x: "" for x in range(1, playercount + 1)}  # only so it can be overwrittin in mystery
-        erargs.spoiler = 0 if race else 2
+        erargs.name = {x: "" for x in range(1, playercount + 1)}  # only so it can be overwritten in mystery
+        erargs.spoiler = meta["generator_options"]["spoiler"]
         erargs.race = race
         erargs.outputname = seedname
         erargs.outputpath = target.name
         erargs.teams = 1
-        erargs.plando_options = ", ".join(plando_options)
+        erargs.plando_options = PlandoOptions.from_set(meta.setdefault("plando_options",
+                                                                       {"bosses", "items", "connections", "texts"}))
 
         name_counter = Counter()
         for player, (playerfile, settings) in enumerate(gen_options.items(), 1):
@@ -136,9 +145,26 @@ def gen_game(gen_options, meta: TypeOptional[Dict[str, object]] = None, owner=No
             erargs.name[player] = handle_name(erargs.name[player], player, name_counter)
         if len(set(erargs.name.values())) != len(erargs.name):
             raise Exception(f"Names have to be unique. Names: {Counter(erargs.name.values())}")
-        ERmain(erargs, seed, baked_server_options=meta)
+        ERmain(erargs, seed, baked_server_options=meta["server_options"])
 
         return upload_to_db(target.name, sid, owner, race)
+    thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1)
+    thread = thread_pool.submit(task)
+
+    try:
+        return thread.result(app.config["JOB_TIME"])
+    except concurrent.futures.TimeoutError as e:
+        if sid:
+            with db_session:
+                gen = Generation.get(id=sid)
+                if gen is not None:
+                    gen.state = STATE_ERROR
+                    meta = json.loads(gen.meta)
+                    meta["error"] = (
+                            "Allowed time for Generation exceeded, please consider generating locally instead. " +
+                            e.__class__.__name__ + ": " + str(e))
+                    gen.meta = json.dumps(meta)
+                    commit()
     except BaseException as e:
         if sid:
             with db_session:
@@ -148,7 +174,6 @@ def gen_game(gen_options, meta: TypeOptional[Dict[str, object]] = None, owner=No
                     meta = json.loads(gen.meta)
                     meta["error"] = (e.__class__.__name__ + ": " + str(e))
                     gen.meta = json.dumps(meta)
-
                     commit()
         raise
 

WebHostLib/landing.py

@@ -1,7 +1,11 @@
+from datetime import timedelta, datetime
+
 from flask import render_template
+from pony.orm import count
+
 from WebHostLib import app, cache
-from .models import *
-from datetime import timedelta
+from .models import Room, Seed
+
 
 @app.route('/', methods=['GET', 'POST'])
 @cache.cached(timeout=300)  # cache has to appear under app route for caching to work

WebHostLib/lttpsprites.py

@@ -32,7 +32,7 @@ def update_sprites_lttp():
 
     spriteData = []
 
-    for file in os.listdir(input_dir):
+    for file in (file for file in os.listdir(input_dir) if not file.startswith(".")):
         sprite = Sprite(os.path.join(input_dir, file))
 
         if not sprite.name:

WebHostLib/misc.py

@@ -0,0 +1,176 @@
+import datetime
+import os
+from typing import List, Dict, Union
+
+import jinja2.exceptions
+from flask import request, redirect, url_for, render_template, Response, session, abort, send_from_directory
+from pony.orm import count, commit, db_session
+
+from worlds.AutoWorld import AutoWorldRegister
+from . import app, cache
+from .models import Seed, Room, Command, UUID, uuid4
+
+
+def get_world_theme(game_name: str):
+    if game_name in AutoWorldRegister.world_types:
+        return AutoWorldRegister.world_types[game_name].web.theme
+    return 'grass'
+
+
+@app.before_request
+def register_session():
+    session.permanent = True  # technically 31 days after the last visit
+    if not session.get("_id", None):
+        session["_id"] = uuid4()  # uniquely identify each session without needing a login
+
+
+@app.errorhandler(404)
+@app.errorhandler(jinja2.exceptions.TemplateNotFound)
+def page_not_found(err):
+    return render_template('404.html'), 404
+
+
+# Start Playing Page
+@app.route('/start-playing')
+def start_playing():
+    return render_template(f"startPlaying.html")
+
+
+@app.route('/weighted-settings')
+def weighted_settings():
+    return render_template(f"weighted-settings.html")
+
+
+# Player settings pages
+@app.route('/games/<string:game>/player-settings')
+def player_settings(game):
+    return render_template(f"player-settings.html", game=game, theme=get_world_theme(game))
+
+
+# Game Info Pages
+@app.route('/games/<string:game>/info/<string:lang>')
+def game_info(game, lang):
+    return render_template('gameInfo.html', game=game, lang=lang, theme=get_world_theme(game))
+
+
+# List of supported games
+@app.route('/games')
+def games():
+    worlds = {}
+    for game, world in AutoWorldRegister.world_types.items():
+        if not world.hidden:
+            worlds[game] = world
+    return render_template("supportedGames.html", worlds=worlds)
+
+
+@app.route('/tutorial/<string:game>/<string:file>/<string:lang>')
+def tutorial(game, file, lang):
+    return render_template("tutorial.html", game=game, file=file, lang=lang, theme=get_world_theme(game))
+
+
+@app.route('/tutorial/')
+def tutorial_landing():
+    return render_template("tutorialLanding.html")
+
+
+@app.route('/faq/<string:lang>/')
+def faq(lang):
+    return render_template("faq.html", lang=lang)
+
+
+@app.route('/glossary/<string:lang>/')
+def terms(lang):
+    return render_template("glossary.html", lang=lang)
+
+
+@app.route('/seed/<suuid:seed>')
+def view_seed(seed: UUID):
+    seed = Seed.get(id=seed)
+    if not seed:
+        abort(404)
+    return render_template("viewSeed.html", seed=seed, slot_count=count(seed.slots))
+
+
+@app.route('/new_room/<suuid:seed>')
+def new_room(seed: UUID):
+    seed = Seed.get(id=seed)
+    if not seed:
+        abort(404)
+    room = Room(seed=seed, owner=session["_id"], tracker=uuid4())
+    commit()
+    return redirect(url_for("host_room", room=room.id))
+
+
+def _read_log(path: str):
+    if os.path.exists(path):
+        with open(path, encoding="utf-8-sig") as log:
+            yield from log
+    else:
+        yield f"Logfile {path} does not exist. " \
+              f"Likely a crash during spinup of multiworld instance or it is still spinning up."
+
+
+@app.route('/log/<suuid:room>')
+def display_log(room: UUID):
+    room = Room.get(id=room)
+    if room is None:
+        return abort(404)
+    if room.owner == session["_id"]:
+        file_path = os.path.join("logs", str(room.id) + ".txt")
+        if os.path.exists(file_path):
+            return Response(_read_log(file_path), mimetype="text/plain;charset=UTF-8")
+        return "Log File does not exist."
+
+    return "Access Denied", 403
+
+
+@app.route('/room/<suuid:room>', methods=['GET', 'POST'])
+def host_room(room: UUID):
+    room: Room = Room.get(id=room)
+    if room is None:
+        return abort(404)
+    if request.method == "POST":
+        if room.owner == session["_id"]:
+            cmd = request.form["cmd"]
+            if cmd:
+                Command(room=room, commandtext=cmd)
+                commit()
+
+    now = datetime.datetime.utcnow()
+    # indicate that the page should reload to get the assigned port
+    should_refresh = not room.last_port and now - room.creation_time < datetime.timedelta(seconds=3)
+    with db_session:
+        room.last_activity = now  # will trigger a spinup, if it's not already running
+
+    return render_template("hostRoom.html", room=room, should_refresh=should_refresh)
+
+
+@app.route('/favicon.ico')
+def favicon():
+    return send_from_directory(os.path.join(app.root_path, 'static/static'),
+                               'favicon.ico', mimetype='image/vnd.microsoft.icon')
+
+
+@app.route('/discord')
+def discord():
+    return redirect("https://discord.gg/8Z65BR2")
+
+
+@app.route('/datapackage')
+@cache.cached()
+def get_datapackage():
+    """A pretty print version of /api/datapackage"""
+    from worlds import network_data_package
+    import json
+    return Response(json.dumps(network_data_package, indent=4), mimetype="text/plain")
+
+
+@app.route('/index')
+@app.route('/sitemap')
+def get_sitemap():
+    available_games: List[Dict[str, Union[str, bool]]] = []
+    for game, world in AutoWorldRegister.world_types.items():
+        if not world.hidden:
+            has_settings: bool = isinstance(world.web.settings_page, bool) and world.web.settings_page
+            available_games.append({ 'title': game, 'has_settings': has_settings })
+    return render_template("siteMap.html", games=available_games)

WebHostLib/models.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 from uuid import UUID, uuid4
-from pony.orm import *
+from pony.orm import Database, PrimaryKey, Required, Set, Optional, buffer, LongStr
 
 db = Database()
 
@@ -27,8 +27,9 @@ class Room(db.Entity):
     seed = Required('Seed', index=True)
     multisave = Optional(buffer, lazy=True)
     show_spoiler = Required(int, default=0)  # 0 -> never, 1 -> after completion, -> 2 always
-    timeout = Required(int, default=lambda: 6 * 60 * 60)  # seconds since last activity to shutdown
+    timeout = Required(int, default=lambda: 2 * 60 * 60)  # seconds since last activity to shutdown
     tracker = Optional(UUID, index=True)
+    # Port special value -1 means the server errored out. Another attempt can be made with a page refresh
     last_port = Optional(int, default=lambda: 0)
 
 
@@ -55,3 +56,8 @@ class Generation(db.Entity):
     options = Required(buffer, lazy=True)
     meta = Required(LongStr, default=lambda: "{\"race\": false}")
     state = Required(int, default=0, index=True)
+
+
+class GameDataPackage(db.Entity):
+    checksum = PrimaryKey(str)
+    data = Required(bytes)

WebHostLib/options.py

@@ -1,35 +1,29 @@
+import json
 import logging
 import os
-from Utils import __version__
-from jinja2 import Template
+import typing
+
 import yaml
-import json
+from jinja2 import Template
 
-from worlds.AutoWorld import AutoWorldRegister
 import Options
-
-target_folder = os.path.join("WebHostLib", "static", "generated")
+from Utils import __version__, local_path
+from worlds.AutoWorld import AutoWorldRegister
 
 handled_in_js = {"start_inventory", "local_items", "non_local_items", "start_hints", "start_location_hints",
-                 "exclude_locations"}
+                 "exclude_locations", "priority_locations"}
 
 
 def create():
-    os.makedirs(os.path.join(target_folder, 'configs'), exist_ok=True)
+    target_folder = local_path("WebHostLib", "static", "generated")
+    yaml_folder = os.path.join(target_folder, "configs")
 
-    def dictify_range(option):
-        data = {option.range_start: 0, option.range_end: 0, "random": 0, "random-low": 0, "random-high": 0,
-                option.default: 50}
-        notes = {
-            option.range_start: "minimum value",
-            option.range_end: "maximum value"
-        }
-        return data, notes
+    Options.generate_yaml_templates(yaml_folder)
 
-    def default_converter(default_value):
-        if isinstance(default_value, (set, frozenset)):
-            return list(default_value)
-        return default_value
+    def get_html_doc(option_type: type(Options.Option)) -> str:
+        if not option_type.__doc__:
+            return "Please document me!"
+        return "\n".join(line.strip() for line in option_type.__doc__.split("\n")).strip()
 
     weighted_settings = {
         "baseOptions": {
@@ -42,15 +36,10 @@ def create():
 
     for game_name, world in AutoWorldRegister.world_types.items():
 
-        all_options = {**world.options, **Options.per_game_common_options}
-        res = Template(open(os.path.join("WebHostLib", "templates", "options.yaml")).read()).render(
-            options=all_options,
-            __version__=__version__, game=game_name, yaml_dump=yaml.dump,
-            dictify_range=dictify_range, default_converter=default_converter,
-        )
-
-        with open(os.path.join(target_folder, 'configs', game_name + ".yaml"), "w") as f:
-            f.write(res)
+        all_options: typing.Dict[str, Options.AssembleOptions] = {
+            **Options.per_game_common_options,
+            **world.option_definitions
+        }
 
         # Generate JSON files for player-settings pages
         player_settings = {
@@ -66,60 +55,68 @@ def create():
             if option_name in handled_in_js:
                 pass
 
-            elif option.options:
+            elif issubclass(option, Options.Choice) or issubclass(option, Options.Toggle):
                 game_options[option_name] = this_option = {
                     "type": "select",
                     "displayName": option.display_name if hasattr(option, "display_name") else option_name,
-                    "description": option.__doc__ if option.__doc__ else "Please document me!",
+                    "description": get_html_doc(option),
                     "defaultValue": None,
                     "options": []
                 }
 
                 for sub_option_id, sub_option_name in option.name_lookup.items():
-                    this_option["options"].append({
-                        "name": option.get_option_name(sub_option_id),
-                        "value": sub_option_name,
-                    })
-
+                    if sub_option_name != "random":
+                        this_option["options"].append({
+                            "name": option.get_option_name(sub_option_id),
+                            "value": sub_option_name,
+                        })
                     if sub_option_id == option.default:
                         this_option["defaultValue"] = sub_option_name
 
-                this_option["options"].append({
-                    "name": "Random",
-                    "value": "random",
-                })
+                if not this_option["defaultValue"]:
+                    this_option["defaultValue"] = "random"
 
-            elif hasattr(option, "range_start") and hasattr(option, "range_end"):
+            elif issubclass(option, Options.Range):
                 game_options[option_name] = {
                     "type": "range",
                     "displayName": option.display_name if hasattr(option, "display_name") else option_name,
-                    "description": option.__doc__ if option.__doc__ else "Please document me!",
-                    "defaultValue": option.default if hasattr(option, "default") else option.range_start,
+                    "description": get_html_doc(option),
+                    "defaultValue": option.default if hasattr(
+                        option, "default") and option.default != "random" else option.range_start,
                     "min": option.range_start,
                     "max": option.range_end,
                 }
 
-            elif getattr(option, "verify_item_name", False):
+                if issubclass(option, Options.SpecialRange):
+                    game_options[option_name]["type"] = 'special_range'
+                    game_options[option_name]["value_names"] = {}
+                    for key, val in option.special_range_names.items():
+                        game_options[option_name]["value_names"][key] = val
+
+            elif issubclass(option, Options.ItemSet):
                 game_options[option_name] = {
                     "type": "items-list",
                     "displayName": option.display_name if hasattr(option, "display_name") else option_name,
-                    "description": option.__doc__ if option.__doc__ else "Please document me!",
+                    "description": get_html_doc(option),
+                    "defaultValue": list(option.default)
                 }
 
-            elif getattr(option, "verify_location_name", False):
+            elif issubclass(option, Options.LocationSet):
                 game_options[option_name] = {
                     "type": "locations-list",
                     "displayName": option.display_name if hasattr(option, "display_name") else option_name,
-                    "description": option.__doc__ if option.__doc__ else "Please document me!",
+                    "description": get_html_doc(option),
+                    "defaultValue": list(option.default)
                 }
 
-            elif hasattr(option, "valid_keys"):
+            elif issubclass(option, Options.VerifyKeys) and not issubclass(option, Options.OptionDict):
                 if option.valid_keys:
                     game_options[option_name] = {
                         "type": "custom-list",
                         "displayName": option.display_name if hasattr(option, "display_name") else option_name,
-                        "description": option.__doc__ if option.__doc__ else "Please document me!",
+                        "description": get_html_doc(option),
                         "options": list(option.valid_keys),
+                        "defaultValue": list(option.default) if hasattr(option, "default") else []
                     }
 
             else:
@@ -133,6 +130,14 @@ def create():
             json.dump(player_settings, f, indent=2, separators=(',', ': '))
 
         if not world.hidden and world.web.settings_page is True:
+            # Add the random option to Choice, TextChoice, and Toggle settings
+            for option in game_options.values():
+                if option["type"] == "select":
+                    option["options"].append({"name": "Random", "value": "random"})
+
+                    if not option["defaultValue"]:
+                        option["defaultValue"] = "random"
+
             weighted_settings["baseOptions"]["game"][game_name] = 0
             weighted_settings["games"][game_name] = {}
             weighted_settings["games"][game_name]["gameSettings"] = game_options

WebHostLib/requirements.txt

@@ -1,7 +1,7 @@
-flask>=2.1.2
+flask>=2.2.3
 pony>=0.7.16
-waitress>=2.1.1
-flask-caching>=1.10.1
-Flask-Compress>=1.12
-Flask-Limiter>=2.4.5.1
-bokeh>=2.4.3
+waitress>=2.1.2
+Flask-Caching>=2.0.2
+Flask-Compress>=1.13
+Flask-Limiter>=3.3.0
+bokeh>=3.1.0

WebHostLib/static/assets/autodatatable.js

@@ -4,6 +4,7 @@ window.addEventListener('load', () => {
         "ordering": true,
         "info": false,
         "dom": "t",
+        "stateSave": true,
     });
   console.log(tables);
 });

WebHostLib/static/assets/baseHeader.js

@@ -0,0 +1,40 @@
+window.addEventListener('load', () => {
+  // Mobile menu handling
+  const menuButton = document.getElementById('base-header-mobile-menu-button');
+  const mobileMenu = document.getElementById('base-header-mobile-menu');
+
+  menuButton.addEventListener('click', (evt) => {
+    evt.preventDefault();
+    evt.stopPropagation();
+
+    if (!mobileMenu.style.display || mobileMenu.style.display === 'none') {
+      return mobileMenu.style.display = 'flex';
+    }
+
+    mobileMenu.style.display = 'none';
+  });
+
+  window.addEventListener('resize', () => {
+    mobileMenu.style.display = 'none';
+  });
+
+  // Popover handling
+  const popoverText = document.getElementById('base-header-popover-text');
+  const popoverMenu = document.getElementById('base-header-popover-menu');
+
+  popoverText.addEventListener('click', (evt) => {
+    evt.preventDefault();
+    evt.stopPropagation();
+
+    if (!popoverMenu.style.display || popoverMenu.style.display === 'none') {
+      return popoverMenu.style.display = 'flex';
+    }
+
+    popoverMenu.style.display = 'none';
+  });
+
+  document.body.addEventListener('click', () => {
+    mobileMenu.style.display = 'none';
+    popoverMenu.style.display = 'none';
+  });
+});

WebHostLib/static/assets/checksfinderTracker.js

@@ -0,0 +1,49 @@
+window.addEventListener('load', () => {
+  // Reload tracker every 60 seconds
+  const url = window.location;
+  setInterval(() => {
+    const ajax = new XMLHttpRequest();
+    ajax.onreadystatechange = () => {
+      if (ajax.readyState !== 4) { return; }
+
+      // Create a fake DOM using the returned HTML
+      const domParser = new DOMParser();
+      const fakeDOM = domParser.parseFromString(ajax.responseText, 'text/html');
+
+      // Update item tracker
+      document.getElementById('inventory-table').innerHTML = fakeDOM.getElementById('inventory-table').innerHTML;
+      // Update only counters in the location-table
+      let counters = document.getElementsByClassName('counter');
+      const fakeCounters = fakeDOM.getElementsByClassName('counter');
+      for (let i = 0; i < counters.length; i++) {
+        counters[i].innerHTML = fakeCounters[i].innerHTML;
+      }
+    };
+    ajax.open('GET', url);
+    ajax.send();
+}, 60000)
+
+  // Collapsible advancement sections
+  const categories = document.getElementsByClassName("location-category");
+  for (let i = 0; i < categories.length; i++) {
+    let hide_id = categories[i].id.split('-')[0];
+    if (hide_id == 'Total') {
+      continue;
+    }
+    categories[i].addEventListener('click', function() {
+      // Toggle the advancement list
+      document.getElementById(hide_id).classList.toggle("hide");
+      // Change text of the header
+      const tab_header = document.getElementById(hide_id+'-header').children[0];
+      const orig_text = tab_header.innerHTML;
+      let new_text;
+      if (orig_text.includes("▼")) {
+        new_text = orig_text.replace("▼", "▲");
+      }
+      else {
+        new_text = orig_text.replace("▲", "▼");
+      }
+      tab_header.innerHTML = new_text;
+    });
+  }
+});

WebHostLib/static/assets/faq.js

@@ -26,24 +26,22 @@ window.addEventListener('load', () => {
         adjustHeaderWidth();
 
         // Reset the id of all header divs to something nicer
-        const headers = Array.from(document.querySelectorAll('h1, h2, h3, h4, h5, h6'));
-        const scrollTargetIndex = window.location.href.search(/#[A-z0-9-_]*$/);
-        for (let i=0; i < headers.length; i++){
-            const headerId = headers[i].innerText.replace(/[ ]/g,'-').toLowerCase()
-            headers[i].setAttribute('id', headerId);
-            headers[i].addEventListener('click', () =>
-                window.location.href = window.location.href.substring(0, scrollTargetIndex) + `#${headerId}`);
+        for (const header of document.querySelectorAll('h1, h2, h3, h4, h5, h6')) {
+            const headerId = header.innerText.replace(/\s+/g, '-').toLowerCase();
+            header.setAttribute('id', headerId);
+            header.addEventListener('click', () => {
+                window.location.hash = `#${headerId}`;
+                header.scrollIntoView();
+            });
         }
 
         // Manually scroll the user to the appropriate header if anchor navigation is used
-        if (scrollTargetIndex > -1) {
-            try{
-                const scrollTarget = window.location.href.substring(scrollTargetIndex + 1);
-                document.getElementById(scrollTarget).scrollIntoView({ behavior: "smooth" });
-            } catch(error) {
-                console.error(error);
+        document.fonts.ready.finally(() => {
+            if (window.location.hash) {
+                const scrollTarget = document.getElementById(window.location.hash.substring(1));
+                scrollTarget?.scrollIntoView();
             }
-        }
+        });
     }).catch((error) => {
         console.error(error);
         tutorialWrapper.innerHTML =

WebHostLib/static/assets/faq/faq_en.md

@@ -20,7 +20,7 @@ comfortable exploiting certain glitches in the game.
 ## What is a multi-world?
 
 While a randomizer shuffles a game, a multi-world randomizer shuffles that game for multiple players. For example, in a
-two player multi-world, players A and B each get their own randomized version of a game, called seeds. In each player's
+two player multi-world, players A and B each get their own randomized version of a game, called a world. In each player's
 game, they may find items which belong to the other player. If player A finds an item which belongs to player B, the
 item will be sent to player B's world over the internet.
 
@@ -29,7 +29,7 @@ their game.
 
 ## What happens if a person has to leave early?
 
-If a player must leave early, they can use Archipelago's forfeit system. When a player forfeits their game, all the
+If a player must leave early, they can use Archipelago's release system. When a player releases their game, all the
 items in that game which belong to other players are sent out automatically, so other players can continue to play.
 
 ## What does multi-game mean?
@@ -46,9 +46,15 @@ the website is not required to generate them.
 ## How do I get started?
 
 If you are ready to start randomizing games, or want to start playing your favorite randomizer with others, please join
-our discord server at the [Archipelago Discord](https://discord.gg/archipelago). There are always people ready to answer
+our discord server at the [Archipelago Discord](https://discord.gg/8Z65BR2). There are always people ready to answer
 any questions you might have.
 
+## What are some common terms I should know?
+
+As randomizers and multiworld randomizers have been around for a while now there are quite a lot of common terms
+and jargon that is used in conjunction by the communities surrounding them. For a lot of the terms that are more common
+to Archipelago and its specific systems please see the [Glossary](/glossary/en).
+
 ## I want to add a game to the Archipelago randomizer. How do I do that?
 
 The best way to get started is to take a look at our code on GitHub

WebHostLib/static/assets/faq/glossary_en.md

@@ -0,0 +1,94 @@
+# Multiworld Glossary
+
+There are a lot of common terms used when playing in different game randomizer communities and in multiworld as well.
+This document serves as a lookup for common terms that may be used by users in the community or in various other
+documentation.
+
+## Item
+Items are what get shuffled around in your world or other worlds that you then receive. This could be a sword, a stat 
+upgrade, a spell, or any other potential receivable for your game.
+
+## Location
+Locations are where items are placed in your game. Whenever you interact with a location, you or another player will
+then receive an item. A location could be a chest, an enemy drop, a shop purchase, or any other interactable that can
+contain items in your game.
+
+## Check
+A check is a common term for when you "check", or pick up, a location. In terms of Archipelago this is usually used for
+when a player goes to a location and sends its item, or "checks" the location. Players will often reference their now
+randomized locations as checks.
+
+## Slot
+A slot is the player name and number assigned during generation. The number of slots is equal to the number of players,
+or "worlds", created. Each name must be unique as these are used to identify the slot user.
+
+## World
+World in terms of Archipelago can mean multiple things and is used interchangeably in many situations.
+* During gameplay, a world is a single instance of a game, occupying one player "slot". However, 
+Archipelago allows multiple players to connect to the same slot; then those players can share a world 
+and complete it cooperatively. For games with native cooperative play, you can also play together and
+share a world that way, usually with only one player connected to the multiworld.
+* On the programming side, a world typically represents the package that integrates Archipelago with a
+particular game. For example this could be the entire `worlds/factorio` directory.
+
+## RNG
+Acronym for "Random Number Generator." Archipelago uses its own custom Random object with a unique seed per generation,
+or, if running from source, a seed can be supplied and this seed will control all randomization during generation as all
+game worlds will have access to it.
+
+## Seed
+A "seed" is a number used to initialize a pseudorandom number generator. Whenever you generate a new game on Archipelago
+this is a new "seed" as it has unique item placement, and you can create multiple "rooms" on the Archipelago site from a
+single seed. Using the same seed results in the random placement being the same.
+
+## Room
+Whenever you generate a seed on the Archipelago website you will be put on a seed page that contains all the seed info
+with a link to the spoiler if one exists and will show how many unique rooms exist per seed. Each room has its own 
+unique identifier that is separate from the seed. The room page is where you can find information to connect to the
+multiworld and download any patches if necessary. If you have a particularly fun or interesting seed, and you want to 
+share it with somebody you can link them to this seed page, where they can generate a new room to play it! For seeds 
+generated with race mode enabled, the seed page will only show rooms created by the unique user so the seed page is 
+perfectly safe to share for racing purposes.
+
+## Logic
+Base behavior of all seeds generated by Archipelago is they are expected to be completable based on the requirements of
+the settings. This is done by using "logic" in order to determine valid locations to place items while still being able
+to reach said location without this item. For the purposes of the randomizer a location is considered "in logic" if you 
+can reach it with your current toolset of items or skills based on settings. Some players are able to obtain locations
+"out of logic" by performing various glitches or tricks that the settings may not account for and tend to mention this
+when sending out an item they obtained this way.
+
+## Progression
+Certain items will allow access to more locations and are considered progression items as they "progress" the seed.
+
+## Trash
+A term used for "filler" items that have no bearing on the generation and are either marginally useful for the player
+or useless. These items can be very useful depending on the player but are never very important and as such are usually
+termed trash.
+
+## Burger King / BK Mode
+A term used in multiworlds when a player is unable to continue to progress and is awaiting an item. The term came to be 
+after a player, allegedly, was unable to progress during a multiworld and went to Burger King while waiting to receive
+items from other players.
+
+* "Logical BK" is when the player is unable to progress according to the settings of their game but may still be able to do
+things that would be "out of logic" by the generation.
+
+* "Hard / full BK" is when the player is completely unable to progress even with tricks they may know and are unable to
+continue to play, aside from doing something like killing enemies for experience or money.
+
+## Sphere
+Archipelago calculates the game playthrough by using a "sphere" system where it has a state for each player and checks
+to see what the players are able to reach with their current items. Any location that is reachable with the current
+state of items is a "sphere." For the purposes of Archipelago it starts playthrough calculation by distributing sphere 0
+items which are items that are either forced in the player's inventory by the game or placed in the `start_inventory` in
+their settings. Sphere 1 is then all accessible locations the players can reach with all the items they received from
+sphere 0, or their starting inventory. The playthrough continues in this fashion calculating a number of spheres until
+all players have completed their goal.
+
+## Scouts / Scouting
+In some games there are locations that have visible items even if the item itself is unobtainable at the current time.
+Some games utilize a scouting feature where when the player "sees" the item it will give a free hint for the item in the
+client letting the players know what the exact item is, since if the item was for that game it would know but the item
+being foreign is a lot harder to represent visually.
+

WebHostLib/static/assets/gameInfo.js

@@ -26,24 +26,22 @@ window.addEventListener('load', () => {
         adjustHeaderWidth();
 
         // Reset the id of all header divs to something nicer
-        const headers = Array.from(document.querySelectorAll('h1, h2, h3, h4, h5, h6'));
-        const scrollTargetIndex = window.location.href.search(/#[A-z0-9-_]*$/);
-        for (let i=0; i < headers.length; i++){
-            const headerId = headers[i].innerText.replace(/[ ]/g,'-').toLowerCase()
-            headers[i].setAttribute('id', headerId);
-            headers[i].addEventListener('click', () =>
-                window.location.href = window.location.href.substring(0, scrollTargetIndex) + `#${headerId}`);
+        for (const header of document.querySelectorAll('h1, h2, h3, h4, h5, h6')) {
+            const headerId = header.innerText.replace(/\s+/g, '-').toLowerCase();
+            header.setAttribute('id', headerId);
+            header.addEventListener('click', () => {
+                window.location.hash = `#${headerId}`;
+                header.scrollIntoView();
+            });
         }
 
         // Manually scroll the user to the appropriate header if anchor navigation is used
-        if (scrollTargetIndex > -1) {
-            try{
-                const scrollTarget = window.location.href.substring(scrollTargetIndex + 1);
-                document.getElementById(scrollTarget).scrollIntoView({ behavior: "smooth" });
-            } catch(error) {
-                console.error(error);
+        document.fonts.ready.finally(() => {
+            if (window.location.hash) {
+                const scrollTarget = document.getElementById(window.location.hash.substring(1));
+                scrollTarget?.scrollIntoView();
             }
-        }
+        });
     }).catch((error) => {
         console.error(error);
         gameInfo.innerHTML =

WebHostLib/static/assets/glossary.js

@@ -0,0 +1,51 @@
+window.addEventListener('load', () => {
+    const tutorialWrapper = document.getElementById('glossary-wrapper');
+    new Promise((resolve, reject) => {
+        const ajax = new XMLHttpRequest();
+        ajax.onreadystatechange = () => {
+            if (ajax.readyState !== 4) { return; }
+            if (ajax.status === 404) {
+                reject("Sorry, the glossary page is not available in that language yet.");
+                return;
+            }
+            if (ajax.status !== 200) {
+                reject("Something went wrong while loading the glossary.");
+                return;
+            }
+            resolve(ajax.responseText);
+        };
+        ajax.open('GET', `${window.location.origin}/static/assets/faq/` +
+          `glossary_${tutorialWrapper.getAttribute('data-lang')}.md`, true);
+        ajax.send();
+    }).then((results) => {
+        // Populate page with HTML generated from markdown
+        showdown.setOption('tables', true);
+        showdown.setOption('strikethrough', true);
+        showdown.setOption('literalMidWordUnderscores', true);
+        tutorialWrapper.innerHTML += (new showdown.Converter()).makeHtml(results);
+        adjustHeaderWidth();
+
+        // Reset the id of all header divs to something nicer
+        for (const header of document.querySelectorAll('h1, h2, h3, h4, h5, h6')) {
+            const headerId = header.innerText.replace(/\s+/g, '-').toLowerCase();
+            header.setAttribute('id', headerId);
+            header.addEventListener('click', () => {
+                window.location.hash = `#${headerId}`;
+                header.scrollIntoView();
+            });
+        }
+
+        // Manually scroll the user to the appropriate header if anchor navigation is used
+        document.fonts.ready.finally(() => {
+            if (window.location.hash) {
+                const scrollTarget = document.getElementById(window.location.hash.substring(1));
+                scrollTarget?.scrollIntoView();
+            }
+        });
+    }).catch((error) => {
+        console.error(error);
+        tutorialWrapper.innerHTML =
+            `<h2>This page is out of logic!</h2>
+            <h3>Click <a href="${window.location.origin}">here</a> to return to safety.</h3>`;
+    });
+});

WebHostLib/static/assets/lttpMultiTracker.js

@@ -0,0 +1,6 @@
+window.addEventListener('load', () => {
+    $(".table-wrapper").scrollsync({
+        y_sync: true,
+        x_sync: true
+    });
+});

WebHostLib/static/assets/player-settings.js

@@ -36,7 +36,8 @@ window.addEventListener('load', () => {
     const nameInput = document.getElementById('player-name');
     nameInput.addEventListener('keyup', (event) => updateBaseSetting(event));
     nameInput.value = playerSettings.name;
-  }).catch(() => {
+  }).catch((e) => {
+    console.error(e);
     const url = new URL(window.location.href);
     window.location.replace(`${url.protocol}//${url.hostname}/page-not-found`);
   })
@@ -101,9 +102,15 @@ const buildOptionsTable = (settings, romOpts = false) => {
     // td Left
     const tdl = document.createElement('td');
     const label = document.createElement('label');
+    label.textContent = `${settings[setting].displayName}: `;
     label.setAttribute('for', setting);
-    label.setAttribute('data-tooltip', settings[setting].description);
-    label.innerText = `${settings[setting].displayName}:`;
+
+    const questionSpan = document.createElement('span');
+    questionSpan.classList.add('interactive');
+    questionSpan.setAttribute('data-tooltip', settings[setting].description);
+    questionSpan.innerText = '(?)';
+
+    label.appendChild(questionSpan);
     tdl.appendChild(label);
     tr.appendChild(tdl);
 
@@ -111,6 +118,8 @@ const buildOptionsTable = (settings, romOpts = false) => {
     const tdr = document.createElement('td');
     let element = null;
 
+    const randomButton = document.createElement('button');
+
     switch(settings[setting].type){
       case 'select':
         element = document.createElement('div');
@@ -131,8 +140,21 @@ const buildOptionsTable = (settings, romOpts = false) => {
           }
           select.appendChild(option);
         });
-        select.addEventListener('change', (event) => updateGameSetting(event));
+        select.addEventListener('change', (event) => updateGameSetting(event.target));
         element.appendChild(select);
+
+        // Randomize button
+        randomButton.innerText = '🎲';
+        randomButton.classList.add('randomize-button');
+        randomButton.setAttribute('data-key', setting);
+        randomButton.setAttribute('data-tooltip', 'Toggle randomization for this option!');
+        randomButton.addEventListener('click', (event) => toggleRandomize(event, [select]));
+        if (currentSettings[gameName][setting] === 'random') {
+          randomButton.classList.add('active');
+          select.disabled = true;
+        }
+
+        element.appendChild(randomButton);
         break;
 
       case 'range':
@@ -147,15 +169,115 @@ const buildOptionsTable = (settings, romOpts = false) => {
         range.value = currentSettings[gameName][setting];
         range.addEventListener('change', (event) => {
           document.getElementById(`${setting}-value`).innerText = event.target.value;
-          updateGameSetting(event);
+          updateGameSetting(event.target);
         });
         element.appendChild(range);
 
         let rangeVal = document.createElement('span');
         rangeVal.classList.add('range-value');
         rangeVal.setAttribute('id', `${setting}-value`);
-        rangeVal.innerText = currentSettings[gameName][setting] ?? settings[setting].defaultValue;
+        rangeVal.innerText = currentSettings[gameName][setting] !== 'random' ?
+          currentSettings[gameName][setting] : settings[setting].defaultValue;
         element.appendChild(rangeVal);
+
+        // Randomize button
+        randomButton.innerText = '🎲';
+        randomButton.classList.add('randomize-button');
+        randomButton.setAttribute('data-key', setting);
+        randomButton.setAttribute('data-tooltip', 'Toggle randomization for this option!');
+        randomButton.addEventListener('click', (event) => toggleRandomize(event, [range]));
+        if (currentSettings[gameName][setting] === 'random') {
+          randomButton.classList.add('active');
+          range.disabled = true;
+        }
+
+        element.appendChild(randomButton);
+        break;
+
+      case 'special_range':
+        element = document.createElement('div');
+        element.classList.add('special-range-container');
+
+        // Build the select element
+        let specialRangeSelect = document.createElement('select');
+        specialRangeSelect.setAttribute('data-key', setting);
+        Object.keys(settings[setting].value_names).forEach((presetName) => {
+          let presetOption = document.createElement('option');
+          presetOption.innerText = presetName;
+          presetOption.value = settings[setting].value_names[presetName];
+          const words = presetOption.innerText.split("_");
+          for (let i = 0; i < words.length; i++) {
+            words[i] = words[i][0].toUpperCase() + words[i].substring(1);
+          }
+          presetOption.innerText = words.join(" ");
+          specialRangeSelect.appendChild(presetOption);
+        });
+        let customOption = document.createElement('option');
+        customOption.innerText = 'Custom';
+        customOption.value = 'custom';
+        customOption.selected = true;
+        specialRangeSelect.appendChild(customOption);
+        if (Object.values(settings[setting].value_names).includes(Number(currentSettings[gameName][setting]))) {
+          specialRangeSelect.value = Number(currentSettings[gameName][setting]);
+        }
+
+        // Build range element
+        let specialRangeWrapper = document.createElement('div');
+        specialRangeWrapper.classList.add('special-range-wrapper');
+        let specialRange = document.createElement('input');
+        specialRange.setAttribute('type', 'range');
+        specialRange.setAttribute('data-key', setting);
+        specialRange.setAttribute('min', settings[setting].min);
+        specialRange.setAttribute('max', settings[setting].max);
+        specialRange.value = currentSettings[gameName][setting];
+
+        // Build rage value element
+        let specialRangeVal = document.createElement('span');
+        specialRangeVal.classList.add('range-value');
+        specialRangeVal.setAttribute('id', `${setting}-value`);
+        specialRangeVal.innerText = currentSettings[gameName][setting] !== 'random' ?
+          currentSettings[gameName][setting] : settings[setting].defaultValue;
+
+        // Configure select event listener
+        specialRangeSelect.addEventListener('change', (event) => {
+          if (event.target.value === 'custom') { return; }
+
+          // Update range slider
+          specialRange.value = event.target.value;
+          document.getElementById(`${setting}-value`).innerText = event.target.value;
+          updateGameSetting(event.target);
+        });
+
+        // Configure range event handler
+        specialRange.addEventListener('change', (event) => {
+          // Update select element
+          specialRangeSelect.value =
+            (Object.values(settings[setting].value_names).includes(parseInt(event.target.value))) ?
+            parseInt(event.target.value) : 'custom';
+          document.getElementById(`${setting}-value`).innerText = event.target.value;
+          updateGameSetting(event.target);
+        });
+
+        element.appendChild(specialRangeSelect);
+        specialRangeWrapper.appendChild(specialRange);
+        specialRangeWrapper.appendChild(specialRangeVal);
+        element.appendChild(specialRangeWrapper);
+
+        // Randomize button
+        randomButton.innerText = '🎲';
+        randomButton.classList.add('randomize-button');
+        randomButton.setAttribute('data-key', setting);
+        randomButton.setAttribute('data-tooltip', 'Toggle randomization for this option!');
+        randomButton.addEventListener('click', (event) => toggleRandomize(
+            event, [specialRange, specialRangeSelect])
+        );
+        if (currentSettings[gameName][setting] === 'random') {
+          randomButton.classList.add('active');
+          specialRange.disabled = true;
+          specialRangeSelect.disabled = true;
+        }
+
+        specialRangeWrapper.appendChild(randomButton);
         break;
 
       default:
@@ -172,6 +294,25 @@ const buildOptionsTable = (settings, romOpts = false) => {
   return table;
 };
 
+const toggleRandomize = (event, inputElements) => {
+  const active = event.target.classList.contains('active');
+  const randomButton = event.target;
+
+  if (active) {
+    randomButton.classList.remove('active');
+    for (const element of inputElements) {
+      element.disabled = undefined;
+      updateGameSetting(element);
+    }
+  } else {
+    randomButton.classList.add('active');
+    for (const element of inputElements) {
+      element.disabled = true;
+      updateGameSetting(randomButton);
+    }
+  }
+};
+
 const updateBaseSetting = (event) => {
   const options = JSON.parse(localStorage.getItem(gameName));
   options[event.target.getAttribute('data-key')] = isNaN(event.target.value) ?
@@ -179,10 +320,17 @@ const updateBaseSetting = (event) => {
   localStorage.setItem(gameName, JSON.stringify(options));
 };
 
-const updateGameSetting = (event) => {
+const updateGameSetting = (settingElement) => {
   const options = JSON.parse(localStorage.getItem(gameName));
-  options[gameName][event.target.getAttribute('data-key')] = isNaN(event.target.value) ?
-      event.target.value : parseInt(event.target.value, 10);
+
+  if (settingElement.classList.contains('randomize-button')) {
+    // If the event passed in is the randomize button, then we know what we must do.
+    options[gameName][settingElement.getAttribute('data-key')] = 'random';
+  } else {
+    options[gameName][settingElement.getAttribute('data-key')] = isNaN(settingElement.value) ?
+      settingElement.value : parseInt(settingElement.value, 10);
+  }
+
   localStorage.setItem(gameName, JSON.stringify(options));
 };
 

WebHostLib/static/assets/sc2wolTracker.js

@@ -0,0 +1,49 @@
+window.addEventListener('load', () => {
+  // Reload tracker every 15 seconds
+  const url = window.location;
+  setInterval(() => {
+    const ajax = new XMLHttpRequest();
+    ajax.onreadystatechange = () => {
+      if (ajax.readyState !== 4) { return; }
+
+      // Create a fake DOM using the returned HTML
+      const domParser = new DOMParser();
+      const fakeDOM = domParser.parseFromString(ajax.responseText, 'text/html');
+
+      // Update item tracker
+      document.getElementById('inventory-table').innerHTML = fakeDOM.getElementById('inventory-table').innerHTML;
+      // Update only counters in the location-table
+      let counters = document.getElementsByClassName('counter');
+      const fakeCounters = fakeDOM.getElementsByClassName('counter');
+      for (let i = 0; i < counters.length; i++) {
+        counters[i].innerHTML = fakeCounters[i].innerHTML;
+      }
+    };
+    ajax.open('GET', url);
+    ajax.send();
+  }, 15000)
+
+  // Collapsible advancement sections
+  const categories = document.getElementsByClassName("location-category");
+  for (let i = 0; i < categories.length; i++) {
+    let hide_id = categories[i].id.split('-')[0];
+    if (hide_id == 'Total') {
+      continue;
+    }
+    categories[i].addEventListener('click', function() {
+      // Toggle the advancement list
+      document.getElementById(hide_id).classList.toggle("hide");
+      // Change text of the header
+      const tab_header = document.getElementById(hide_id+'-header').children[0];
+      const orig_text = tab_header.innerHTML;
+      let new_text;
+      if (orig_text.includes("▼")) {
+        new_text = orig_text.replace("▼", "▲");
+      }
+      else {
+        new_text = orig_text.replace("▲", "▼");
+      }
+      tab_header.innerHTML = new_text;
+    });
+  }
+});

WebHostLib/static/assets/trackerCommon.js

@@ -1,5 +1,7 @@
 const adjustTableHeight = () => {
     const tablesContainer = document.getElementById('tables-container');
+    if (!tablesContainer)
+        return;
     const upperDistance = tablesContainer.getBoundingClientRect().top;
 
     const containerHeight = window.innerHeight - upperDistance;
@@ -17,6 +19,14 @@ window.addEventListener('load', () => {
         paging: false,
         info: false,
         dom: "t",
+        stateSave: true,
+        stateSaveCallback: function(settings, data) {
+            delete data.search;
+            localStorage.setItem(`DataTables_${settings.sInstance}_/tracker`, JSON.stringify(data));
+        },
+        stateLoadCallback: function(settings) {
+            return JSON.parse(localStorage.getItem(`DataTables_${settings.sInstance}_/tracker`));
+        },
         columnDefs: [
             {
                 targets: 'hours',
@@ -63,16 +73,44 @@ window.addEventListener('load', () => {
         // the tbody and render two separate tables.
     });
 
-    document.getElementById('search').addEventListener('keyup', (event) => {
-        tables.search(event.target.value);
-        console.info(tables.search());
+    const searchBox = document.getElementById("search");
+    searchBox.value = tables.search();
+    searchBox.focus();
+    searchBox.select();
+    const doSearch = () => {
+        tables.search(searchBox.value);
         tables.draw();
+    };
+    searchBox.addEventListener("keyup", doSearch);
+    window.addEventListener("keydown", (event) => {
+        if (!event.ctrlKey && !event.altKey && event.key.length === 1 && document.activeElement !== searchBox) {
+            searchBox.focus();
+            searchBox.select();
+        }
+        if (!event.ctrlKey && !event.altKey && !event.shiftKey && event.key === "Escape") {
+            if (searchBox.value !== "") {
+                searchBox.value = "";
+                doSearch();
+            }
+            searchBox.blur();
+            if (!document.getElementById("tables-container"))
+                window.scroll(0, 0);
+            event.preventDefault();
+        }
     });
+    const tracker = document.getElementById('tracker-wrapper').getAttribute('data-tracker');
+    const target_second = document.getElementById('tracker-wrapper').getAttribute('data-second') + 3;
+
+    function getSleepTimeSeconds(){
+        // -40 % 60 is -40, which is absolutely wrong and should burn
+        var sleepSeconds = (((target_second - new Date().getSeconds()) % 60) + 60) % 60;
+        return sleepSeconds || 60;
+    }
 
     const update = () => {
         const target = $("<div></div>");
-        const tracker = document.getElementById('tracker-wrapper').getAttribute('data-tracker');
-        target.load("/tracker/" + tracker, function (response, status) {
+        console.log("Updating Tracker...");
+        target.load(location.href, function (response, status) {
             if (status === "success") {
                 target.find(".table").each(function (i, new_table) {
                     const new_trs = $(new_table).find("tbody>tr");
@@ -90,19 +128,14 @@ window.addEventListener('load', () => {
                 console.log(response);
             }
         })
+        setTimeout(update, getSleepTimeSeconds()*1000);
     }
-
-    setInterval(update, 30000);
+    setTimeout(update, getSleepTimeSeconds()*1000);
 
     window.addEventListener('resize', () => {
         adjustTableHeight();
         tables.draw();
     });
 
-    $(".table-wrapper").scrollsync({
-        y_sync: true,
-        x_sync: true
-    });
-
     adjustTableHeight();
 });

WebHostLib/static/assets/tutorial.js

@@ -27,25 +27,28 @@ window.addEventListener('load', () => {
         tutorialWrapper.innerHTML += (new showdown.Converter()).makeHtml(results);
         adjustHeaderWidth();
 
+        const title = document.querySelector('h1')
+        if (title) {
+            document.title = title.textContent;
+        }
+
         // Reset the id of all header divs to something nicer
-        const headers = Array.from(document.querySelectorAll('h1, h2, h3, h4, h5, h6'));
-        const scrollTargetIndex = window.location.href.search(/#[A-z0-9-_]*$/);
-        for (let i=0; i < headers.length; i++){
-            const headerId = headers[i].innerText.replace(/[ ]/g,'-').toLowerCase()
-            headers[i].setAttribute('id', headerId);
-            headers[i].addEventListener('click', () =>
-                window.location.href = window.location.href.substring(0, scrollTargetIndex) + `#${headerId}`);
+        for (const header of document.querySelectorAll('h1, h2, h3, h4, h5, h6')) {
+            const headerId = header.innerText.replace(/\s+/g, '-').toLowerCase();
+            header.setAttribute('id', headerId);
+            header.addEventListener('click', () => {
+                window.location.hash = `#${headerId}`;
+                header.scrollIntoView();
+            });
         }
 
         // Manually scroll the user to the appropriate header if anchor navigation is used
-        if (scrollTargetIndex > -1) {
-            try{
-                const scrollTarget = window.location.href.substring(scrollTargetIndex + 1);
-                document.getElementById(scrollTarget).scrollIntoView({ behavior: "smooth" });
-            } catch(error) {
-                console.error(error);
+        document.fonts.ready.finally(() => {
+            if (window.location.hash) {
+                const scrollTarget = document.getElementById(window.location.hash.substring(1));
+                scrollTarget?.scrollIntoView();
             }
-        }
+        });
     }).catch((error) => {
         console.error(error);
         tutorialWrapper.innerHTML =

WebHostLib/static/assets/tutorialLanding.js

@@ -23,6 +23,7 @@ window.addEventListener('load', () => {
       games.forEach((game) => {
         const gameTitle = document.createElement('h2');
         gameTitle.innerText = game.gameTitle;
+        gameTitle.id = `${encodeURIComponent(game.gameTitle)}`;
         tutorialDiv.appendChild(gameTitle);
 
         game.tutorials.forEach((tutorial) => {
@@ -65,6 +66,15 @@ window.addEventListener('load', () => {
       showError();
       console.error(error);
     }
+
+    // Check if we are on an anchor when coming in, and scroll to it.
+    const hash = window.location.hash;
+    if (hash) {
+      const offset = 128;  // To account for navbar banner at top of page.
+      window.scrollTo(0, 0);
+      const rect = document.getElementById(hash.slice(1)).getBoundingClientRect();
+      window.scrollTo(rect.left, rect.top - offset);
+    }
   };
   ajax.open('GET', `${window.location.origin}/static/generated/tutorials.json`, true);
   ajax.send();

WebHostLib/static/assets/userContent.js

@@ -6,6 +6,7 @@ window.addEventListener('load', () => {
         "order": [[ 3, "desc" ]],
         "info": false,
         "dom": "t",
+        "stateSave": true,
     });
   $("#seeds-table").DataTable({
         "paging": false,
@@ -13,5 +14,6 @@ window.addEventListener('load', () => {
         "order": [[ 2, "desc" ]],
         "info": false,
         "dom": "t",
+        "stateSave": true,
     });
 });

WebHostLib/static/assets/weighted-settings.js

@@ -77,19 +77,21 @@ const createDefaultSettings = (settingData) => {
             });
             break;
           case 'range':
-            for (let i = setting.min; i <= setting.max; ++i){
-              newSettings[game][gameSetting][i] =
-                (setting.hasOwnProperty('defaultValue') && setting.defaultValue === i) ? 25 : 0;
-            }
+          case 'special_range':
             newSettings[game][gameSetting]['random'] = 0;
             newSettings[game][gameSetting]['random-low'] = 0;
             newSettings[game][gameSetting]['random-high'] = 0;
+            if (setting.hasOwnProperty('defaultValue')) {
+              newSettings[game][gameSetting][setting.defaultValue] = 25;
+            } else {
+              newSettings[game][gameSetting][setting.min] = 25;
+            }
             break;
 
           case 'items-list':
           case 'locations-list':
           case 'custom-list':
-            newSettings[game][gameSetting] = [];
+            newSettings[game][gameSetting] = setting.defaultValue;
             break;
 
           default:
@@ -99,6 +101,7 @@ const createDefaultSettings = (settingData) => {
 
       newSettings[game].start_inventory = {};
       newSettings[game].exclude_locations = [];
+      newSettings[game].priority_locations = [];
       newSettings[game].local_items = [];
       newSettings[game].non_local_items = [];
       newSettings[game].start_hints = [];
@@ -134,21 +137,28 @@ const buildUI = (settingData) => {
     expandButton.classList.add('invisible');
     gameDiv.appendChild(expandButton);
 
-    const weightedSettingsDiv = buildWeightedSettingsDiv(game, settingData.games[game].gameSettings);
+    settingData.games[game].gameItems.sort((a, b) => (a > b ? 1 : (a < b ? -1 : 0)));
+    settingData.games[game].gameLocations.sort((a, b) => (a > b ? 1 : (a < b ? -1 : 0)));
+
+    const weightedSettingsDiv = buildWeightedSettingsDiv(game, settingData.games[game].gameSettings,
+      settingData.games[game].gameItems, settingData.games[game].gameLocations);
     gameDiv.appendChild(weightedSettingsDiv);
 
-    const itemsDiv = buildItemsDiv(game, settingData.games[game].gameItems);
-    gameDiv.appendChild(itemsDiv);
+    const itemPoolDiv = buildItemsDiv(game, settingData.games[game].gameItems);
+    gameDiv.appendChild(itemPoolDiv);
 
     const hintsDiv = buildHintsDiv(game, settingData.games[game].gameItems, settingData.games[game].gameLocations);
     gameDiv.appendChild(hintsDiv);
 
+    const locationsDiv = buildLocationsDiv(game, settingData.games[game].gameLocations);
+    gameDiv.appendChild(locationsDiv);
+
     gamesWrapper.appendChild(gameDiv);
 
     collapseButton.addEventListener('click', () => {
       collapseButton.classList.add('invisible');
       weightedSettingsDiv.classList.add('invisible');
-      itemsDiv.classList.add('invisible');
+      itemPoolDiv.classList.add('invisible');
       hintsDiv.classList.add('invisible');
       expandButton.classList.remove('invisible');
     });
@@ -156,7 +166,7 @@ const buildUI = (settingData) => {
     expandButton.addEventListener('click', () => {
       collapseButton.classList.remove('invisible');
       weightedSettingsDiv.classList.remove('invisible');
-      itemsDiv.classList.remove('invisible');
+      itemPoolDiv.classList.remove('invisible');
       hintsDiv.classList.remove('invisible');
       expandButton.classList.add('invisible');
     });
@@ -224,7 +234,7 @@ const buildGameChoice = (games) => {
   gameChoiceDiv.appendChild(table);
 };
 
-const buildWeightedSettingsDiv = (game, settings) => {
+const buildWeightedSettingsDiv = (game, settings, gameItems, gameLocations) => {
   const currentSettings = JSON.parse(localStorage.getItem('weighted-settings'));
   const settingsWrapper = document.createElement('div');
   settingsWrapper.classList.add('settings-wrapper');
@@ -266,7 +276,7 @@ const buildWeightedSettingsDiv = (game, settings) => {
           range.setAttribute('data-type', setting.type);
           range.setAttribute('min', 0);
           range.setAttribute('max', 50);
-          range.addEventListener('change', updateGameSetting);
+          range.addEventListener('change', updateRangeSetting);
           range.value = currentSettings[game][settingName][option.value];
           tdMiddle.appendChild(range);
           tr.appendChild(tdMiddle);
@@ -285,39 +295,40 @@ const buildWeightedSettingsDiv = (game, settings) => {
         break;
 
       case 'range':
+      case 'special_range':
         const rangeTable = document.createElement('table');
         const rangeTbody = document.createElement('tbody');
 
         if (((setting.max - setting.min) + 1) < 11) {
           for (let i=setting.min; i <= setting.max; ++i) {
             const tr = document.createElement('tr');
-              const tdLeft = document.createElement('td');
-              tdLeft.classList.add('td-left');
-              tdLeft.innerText = i;
-              tr.appendChild(tdLeft);
+            const tdLeft = document.createElement('td');
+            tdLeft.classList.add('td-left');
+            tdLeft.innerText = i;
+            tr.appendChild(tdLeft);
 
-              const tdMiddle = document.createElement('td');
-              tdMiddle.classList.add('td-middle');
-              const range = document.createElement('input');
-              range.setAttribute('type', 'range');
-              range.setAttribute('id', `${game}-${settingName}-${i}-range`);
-              range.setAttribute('data-game', game);
-              range.setAttribute('data-setting', settingName);
-              range.setAttribute('data-option', i);
-              range.setAttribute('min', 0);
-              range.setAttribute('max', 50);
-              range.addEventListener('change', updateGameSetting);
-              range.value = currentSettings[game][settingName][i];
-              tdMiddle.appendChild(range);
-              tr.appendChild(tdMiddle);
+            const tdMiddle = document.createElement('td');
+            tdMiddle.classList.add('td-middle');
+            const range = document.createElement('input');
+            range.setAttribute('type', 'range');
+            range.setAttribute('id', `${game}-${settingName}-${i}-range`);
+            range.setAttribute('data-game', game);
+            range.setAttribute('data-setting', settingName);
+            range.setAttribute('data-option', i);
+            range.setAttribute('min', 0);
+            range.setAttribute('max', 50);
+            range.addEventListener('change', updateRangeSetting);
+            range.value = currentSettings[game][settingName][i] || 0;
+            tdMiddle.appendChild(range);
+            tr.appendChild(tdMiddle);
 
-              const tdRight = document.createElement('td');
-              tdRight.setAttribute('id', `${game}-${settingName}-${i}`)
-              tdRight.classList.add('td-right');
-              tdRight.innerText = range.value;
-              tr.appendChild(tdRight);
+            const tdRight = document.createElement('td');
+            tdRight.setAttribute('id', `${game}-${settingName}-${i}`)
+            tdRight.classList.add('td-right');
+            tdRight.innerText = range.value;
+            tr.appendChild(tdRight);
 
-              rangeTbody.appendChild(tr);
+            rangeTbody.appendChild(tr);
           }
         } else {
           const hintText = document.createElement('p');
@@ -325,6 +336,14 @@ const buildWeightedSettingsDiv = (game, settings) => {
           hintText.innerHTML = 'This is a range option. You may enter a valid numerical value in the text box ' +
             `below, then press the "Add" button to add a weight for it.<br />Minimum value: ${setting.min}<br />` +
             `Maximum value: ${setting.max}`;
+
+          if (setting.hasOwnProperty('value_names')) {
+            hintText.innerHTML += '<br /><br />Certain values have special meaning:';
+            Object.keys(setting.value_names).forEach((specialName) => {
+              hintText.innerHTML += `<br />${specialName}: ${setting.value_names[specialName]}`;
+            });
+          }
+
           settingWrapper.appendChild(hintText);
 
           const addOptionDiv = document.createElement('div');
@@ -366,7 +385,7 @@ const buildWeightedSettingsDiv = (game, settings) => {
             range.setAttribute('data-option', option);
             range.setAttribute('min', 0);
             range.setAttribute('max', 50);
-            range.addEventListener('change', updateGameSetting);
+            range.addEventListener('change', updateRangeSetting);
             range.value = currentSettings[game][settingName][parseInt(option, 10)];
             tdMiddle.appendChild(range);
             tr.appendChild(tdMiddle);
@@ -391,11 +410,17 @@ const buildWeightedSettingsDiv = (game, settings) => {
             tr.appendChild(tdDelete);
 
             rangeTbody.appendChild(tr);
+
+            // Save new option to settings
+            range.dispatchEvent(new Event('change'));
           });
 
           Object.keys(currentSettings[game][settingName]).forEach((option) => {
-            if (currentSettings[game][settingName][option] > 0) {
-              const tr = document.createElement('tr');
+            // These options are statically generated below, and should always appear even if they are deleted
+            // from localStorage
+            if (['random-low', 'random', 'random-high'].includes(option)) { return; }
+
+            const tr = document.createElement('tr');
               const tdLeft = document.createElement('td');
               tdLeft.classList.add('td-left');
               tdLeft.innerText = option;
@@ -411,7 +436,7 @@ const buildWeightedSettingsDiv = (game, settings) => {
               range.setAttribute('data-option', option);
               range.setAttribute('min', 0);
               range.setAttribute('max', 50);
-              range.addEventListener('change', updateGameSetting);
+              range.addEventListener('change', updateRangeSetting);
               range.value = currentSettings[game][settingName][parseInt(option, 10)];
               tdMiddle.appendChild(range);
               tr.appendChild(tdMiddle);
@@ -429,14 +454,15 @@ const buildWeightedSettingsDiv = (game, settings) => {
               deleteButton.innerText = '❌';
               deleteButton.addEventListener('click', () => {
                 range.value = 0;
-                range.dispatchEvent(new Event('change'));
+                const changeEvent = new Event('change');
+                changeEvent.action = 'rangeDelete';
+                range.dispatchEvent(changeEvent);
                 rangeTbody.removeChild(tr);
               });
               tdDelete.appendChild(deleteButton);
               tr.appendChild(tdDelete);
 
               rangeTbody.appendChild(tr);
-            }
           });
         }
 
@@ -444,7 +470,17 @@ const buildWeightedSettingsDiv = (game, settings) => {
           const tr = document.createElement('tr');
             const tdLeft = document.createElement('td');
             tdLeft.classList.add('td-left');
-            tdLeft.innerText = option;
+            switch(option){
+              case 'random':
+                tdLeft.innerText = 'Random';
+                break;
+              case 'random-low':
+                tdLeft.innerText = "Random (Low)";
+                break;
+              case 'random-high':
+                tdLeft.innerText = "Random (High)";
+                break;
+            }
             tr.appendChild(tdLeft);
 
             const tdMiddle = document.createElement('td');
@@ -457,7 +493,7 @@ const buildWeightedSettingsDiv = (game, settings) => {
             range.setAttribute('data-option', option);
             range.setAttribute('min', 0);
             range.setAttribute('max', 50);
-            range.addEventListener('change', updateGameSetting);
+            range.addEventListener('change', updateRangeSetting);
             range.value = currentSettings[game][settingName][option];
             tdMiddle.appendChild(range);
             tr.appendChild(tdMiddle);
@@ -475,19 +511,112 @@ const buildWeightedSettingsDiv = (game, settings) => {
         break;
 
       case 'items-list':
-        // TODO
+        const itemsList = document.createElement('div');
+        itemsList.classList.add('simple-list');
+
+        Object.values(gameItems).forEach((item) => {
+          const itemRow = document.createElement('div');
+          itemRow.classList.add('list-row');
+
+          const itemLabel = document.createElement('label');
+          itemLabel.setAttribute('for', `${game}-${settingName}-${item}`)
+
+          const itemCheckbox = document.createElement('input');
+          itemCheckbox.setAttribute('id', `${game}-${settingName}-${item}`);
+          itemCheckbox.setAttribute('type', 'checkbox');
+          itemCheckbox.setAttribute('data-game', game);
+          itemCheckbox.setAttribute('data-setting', settingName);
+          itemCheckbox.setAttribute('data-option', item.toString());
+          itemCheckbox.addEventListener('change', updateListSetting);
+          if (currentSettings[game][settingName].includes(item)) {
+            itemCheckbox.setAttribute('checked', '1');
+          }
+
+          const itemName = document.createElement('span');
+          itemName.innerText = item.toString();
+
+          itemLabel.appendChild(itemCheckbox);
+          itemLabel.appendChild(itemName);
+
+          itemRow.appendChild(itemLabel);
+          itemsList.appendChild((itemRow));
+        });
+
+        settingWrapper.appendChild(itemsList);
         break;
 
       case 'locations-list':
-        // TODO
+        const locationsList = document.createElement('div');
+        locationsList.classList.add('simple-list');
+
+        Object.values(gameLocations).forEach((location) => {
+          const locationRow = document.createElement('div');
+          locationRow.classList.add('list-row');
+
+          const locationLabel = document.createElement('label');
+          locationLabel.setAttribute('for', `${game}-${settingName}-${location}`)
+
+          const locationCheckbox = document.createElement('input');
+          locationCheckbox.setAttribute('id', `${game}-${settingName}-${location}`);
+          locationCheckbox.setAttribute('type', 'checkbox');
+          locationCheckbox.setAttribute('data-game', game);
+          locationCheckbox.setAttribute('data-setting', settingName);
+          locationCheckbox.setAttribute('data-option', location.toString());
+          locationCheckbox.addEventListener('change', updateListSetting);
+          if (currentSettings[game][settingName].includes(location)) {
+            locationCheckbox.setAttribute('checked', '1');
+          }
+
+          const locationName = document.createElement('span');
+          locationName.innerText = location.toString();
+
+          locationLabel.appendChild(locationCheckbox);
+          locationLabel.appendChild(locationName);
+
+          locationRow.appendChild(locationLabel);
+          locationsList.appendChild((locationRow));
+        });
+
+        settingWrapper.appendChild(locationsList);
         break;
 
       case 'custom-list':
-        // TODO
+        const customList = document.createElement('div');
+        customList.classList.add('simple-list');
+
+        Object.values(settings[settingName].options).forEach((listItem) => {
+          const customListRow = document.createElement('div');
+          customListRow.classList.add('list-row');
+
+          const customItemLabel = document.createElement('label');
+          customItemLabel.setAttribute('for', `${game}-${settingName}-${listItem}`)
+
+          const customItemCheckbox = document.createElement('input');
+          customItemCheckbox.setAttribute('id', `${game}-${settingName}-${listItem}`);
+          customItemCheckbox.setAttribute('type', 'checkbox');
+          customItemCheckbox.setAttribute('data-game', game);
+          customItemCheckbox.setAttribute('data-setting', settingName);
+          customItemCheckbox.setAttribute('data-option', listItem.toString());
+          customItemCheckbox.addEventListener('change', updateListSetting);
+          if (currentSettings[game][settingName].includes(listItem)) {
+            customItemCheckbox.setAttribute('checked', '1');
+          }
+
+          const customItemName = document.createElement('span');
+          customItemName.innerText = listItem.toString();
+
+          customItemLabel.appendChild(customItemCheckbox);
+          customItemLabel.appendChild(customItemName);
+
+          customListRow.appendChild(customItemLabel);
+          customList.appendChild((customListRow));
+        });
+
+        settingWrapper.appendChild(customList);
         break;
 
       default:
-        console.error(`Unknown setting type for ${game} setting ${setting}: ${settings[setting].type}`);
+        console.error(`Unknown setting type for ${game} setting ${settingName}: ${setting.type}`);
         return;
     }
 
@@ -709,21 +838,22 @@ const buildHintsDiv = (game, items, locations) => {
   const hintsDescription = document.createElement('p');
   hintsDescription.classList.add('setting-description');
   hintsDescription.innerText = 'Choose any items or locations to begin the game with the knowledge of where those ' +
-    ' items are, or what those locations contain. Excluded locations will not contain progression items.';
+    ' items are, or what those locations contain.';
   hintsDiv.appendChild(hintsDescription);
 
   const itemHintsContainer = document.createElement('div');
   itemHintsContainer.classList.add('hints-container');
 
+  // Item Hints
   const itemHintsWrapper = document.createElement('div');
   itemHintsWrapper.classList.add('hints-wrapper');
   itemHintsWrapper.innerText = 'Starting Item Hints';
 
   const itemHintsDiv = document.createElement('div');
-  itemHintsDiv.classList.add('item-container');
+  itemHintsDiv.classList.add('simple-list');
   items.forEach((item) => {
-    const itemDiv = document.createElement('div');
-    itemDiv.classList.add('hint-div');
+    const itemRow = document.createElement('div');
+    itemRow.classList.add('list-row');
 
     const itemLabel = document.createElement('label');
     itemLabel.setAttribute('for', `${game}-start_hints-${item}`);
@@ -737,29 +867,30 @@ const buildHintsDiv = (game, items, locations) => {
     if (currentSettings[game].start_hints.includes(item)) {
       itemCheckbox.setAttribute('checked', 'true');
     }
-    itemCheckbox.addEventListener('change', hintChangeHandler);
+    itemCheckbox.addEventListener('change', updateListSetting);
     itemLabel.appendChild(itemCheckbox);
 
     const itemName = document.createElement('span');
     itemName.innerText = item;
     itemLabel.appendChild(itemName);
 
-    itemDiv.appendChild(itemLabel);
-    itemHintsDiv.appendChild(itemDiv);
+    itemRow.appendChild(itemLabel);
+    itemHintsDiv.appendChild(itemRow);
   });
 
   itemHintsWrapper.appendChild(itemHintsDiv);
   itemHintsContainer.appendChild(itemHintsWrapper);
 
+  // Starting Location Hints
   const locationHintsWrapper = document.createElement('div');
   locationHintsWrapper.classList.add('hints-wrapper');
   locationHintsWrapper.innerText = 'Starting Location Hints';
 
   const locationHintsDiv = document.createElement('div');
-  locationHintsDiv.classList.add('item-container');
+  locationHintsDiv.classList.add('simple-list');
   locations.forEach((location) => {
-    const locationDiv = document.createElement('div');
-    locationDiv.classList.add('hint-div');
+    const locationRow = document.createElement('div');
+    locationRow.classList.add('list-row');
 
     const locationLabel = document.createElement('label');
     locationLabel.setAttribute('for', `${game}-start_location_hints-${location}`);
@@ -773,29 +904,89 @@ const buildHintsDiv = (game, items, locations) => {
     if (currentSettings[game].start_location_hints.includes(location)) {
       locationCheckbox.setAttribute('checked', '1');
     }
-    locationCheckbox.addEventListener('change', hintChangeHandler);
+    locationCheckbox.addEventListener('change', updateListSetting);
     locationLabel.appendChild(locationCheckbox);
 
     const locationName = document.createElement('span');
     locationName.innerText = location;
     locationLabel.appendChild(locationName);
 
-    locationDiv.appendChild(locationLabel);
-    locationHintsDiv.appendChild(locationDiv);
+    locationRow.appendChild(locationLabel);
+    locationHintsDiv.appendChild(locationRow);
   });
 
   locationHintsWrapper.appendChild(locationHintsDiv);
   itemHintsContainer.appendChild(locationHintsWrapper);
 
+  hintsDiv.appendChild(itemHintsContainer);
+  return hintsDiv;
+};
+
+const buildLocationsDiv = (game, locations) => {
+  const currentSettings = JSON.parse(localStorage.getItem('weighted-settings'));
+  locations.sort(); // Sort alphabetical, in-place
+
+  const locationsDiv = document.createElement('div');
+  locationsDiv.classList.add('locations-div');
+  const locationsHeader = document.createElement('h3');
+  locationsHeader.innerText = 'Priority & Exclusion Locations';
+  locationsDiv.appendChild(locationsHeader);
+  const locationsDescription = document.createElement('p');
+  locationsDescription.classList.add('setting-description');
+  locationsDescription.innerText = 'Priority locations guarantee a progression item will be placed there while ' +
+    'excluded locations will not contain progression or useful items.';
+  locationsDiv.appendChild(locationsDescription);
+
+  const locationsContainer = document.createElement('div');
+  locationsContainer.classList.add('locations-container');
+
+  // Priority Locations
+  const priorityLocationsWrapper = document.createElement('div');
+  priorityLocationsWrapper.classList.add('locations-wrapper');
+  priorityLocationsWrapper.innerText = 'Priority Locations';
+
+  const priorityLocationsDiv = document.createElement('div');
+  priorityLocationsDiv.classList.add('simple-list');
+  locations.forEach((location) => {
+    const locationRow = document.createElement('div');
+    locationRow.classList.add('list-row');
+
+    const locationLabel = document.createElement('label');
+    locationLabel.setAttribute('for', `${game}-priority_locations-${location}`);
+
+    const locationCheckbox = document.createElement('input');
+    locationCheckbox.setAttribute('type', 'checkbox');
+    locationCheckbox.setAttribute('id', `${game}-priority_locations-${location}`);
+    locationCheckbox.setAttribute('data-game', game);
+    locationCheckbox.setAttribute('data-setting', 'priority_locations');
+    locationCheckbox.setAttribute('data-option', location);
+    if (currentSettings[game].priority_locations.includes(location)) {
+      locationCheckbox.setAttribute('checked', '1');
+    }
+    locationCheckbox.addEventListener('change', updateListSetting);
+    locationLabel.appendChild(locationCheckbox);
+
+    const locationName = document.createElement('span');
+    locationName.innerText = location;
+    locationLabel.appendChild(locationName);
+
+    locationRow.appendChild(locationLabel);
+    priorityLocationsDiv.appendChild(locationRow);
+  });
+
+  priorityLocationsWrapper.appendChild(priorityLocationsDiv);
+  locationsContainer.appendChild(priorityLocationsWrapper);
+
+  // Exclude Locations
   const excludeLocationsWrapper = document.createElement('div');
-  excludeLocationsWrapper.classList.add('hints-wrapper');
+  excludeLocationsWrapper.classList.add('locations-wrapper');
   excludeLocationsWrapper.innerText = 'Exclude Locations';
 
   const excludeLocationsDiv = document.createElement('div');
-  excludeLocationsDiv.classList.add('item-container');
+  excludeLocationsDiv.classList.add('simple-list');
   locations.forEach((location) => {
-    const locationDiv = document.createElement('div');
-    locationDiv.classList.add('hint-div');
+    const locationRow = document.createElement('div');
+    locationRow.classList.add('list-row');
 
     const locationLabel = document.createElement('label');
     locationLabel.setAttribute('for', `${game}-exclude_locations-${location}`);
@@ -809,40 +1000,22 @@ const buildHintsDiv = (game, items, locations) => {
     if (currentSettings[game].exclude_locations.includes(location)) {
       locationCheckbox.setAttribute('checked', '1');
     }
-    locationCheckbox.addEventListener('change', hintChangeHandler);
+    locationCheckbox.addEventListener('change', updateListSetting);
     locationLabel.appendChild(locationCheckbox);
 
     const locationName = document.createElement('span');
     locationName.innerText = location;
     locationLabel.appendChild(locationName);
 
-    locationDiv.appendChild(locationLabel);
-    excludeLocationsDiv.appendChild(locationDiv);
+    locationRow.appendChild(locationLabel);
+    excludeLocationsDiv.appendChild(locationRow);
   });
 
   excludeLocationsWrapper.appendChild(excludeLocationsDiv);
-  itemHintsContainer.appendChild(excludeLocationsWrapper);
+  locationsContainer.appendChild(excludeLocationsWrapper);
 
-  hintsDiv.appendChild(itemHintsContainer);
-  return hintsDiv;
-};
-
-const hintChangeHandler = (evt) => {
-  const currentSettings = JSON.parse(localStorage.getItem('weighted-settings'));
-  const game = evt.target.getAttribute('data-game');
-  const setting = evt.target.getAttribute('data-setting');
-  const option = evt.target.getAttribute('data-option');
-
-  if (evt.target.checked) {
-    if (!currentSettings[game][setting].includes(option)) {
-      currentSettings[game][setting].push(option);
-    }
-  } else {
-    if (currentSettings[game][setting].includes(option)) {
-      currentSettings[game][setting].splice(currentSettings[game][setting].indexOf(option), 1);
-    }
-  }
-  localStorage.setItem('weighted-settings', JSON.stringify(currentSettings));
+  locationsDiv.appendChild(locationsContainer);
+  return locationsDiv;
 };
 
 const updateVisibleGames = () => {
@@ -888,14 +1061,37 @@ const updateBaseSetting = (event) => {
   localStorage.setItem('weighted-settings', JSON.stringify(settings));
 };
 
-const updateGameSetting = (evt) => {
+const updateRangeSetting = (evt) => {
   const options = JSON.parse(localStorage.getItem('weighted-settings'));
   const game = evt.target.getAttribute('data-game');
   const setting = evt.target.getAttribute('data-setting');
   const option = evt.target.getAttribute('data-option');
   document.getElementById(`${game}-${setting}-${option}`).innerText = evt.target.value;
-  options[game][setting][option] = isNaN(evt.target.value) ?
-      evt.target.value : parseInt(evt.target.value, 10);
+  if (evt.action && evt.action === 'rangeDelete') {
+    delete options[game][setting][option];
+  } else {
+    options[game][setting][option] = parseInt(evt.target.value, 10);
+  }
+  localStorage.setItem('weighted-settings', JSON.stringify(options));
+};
+
+const updateListSetting = (evt) => {
+  const options = JSON.parse(localStorage.getItem('weighted-settings'));
+  const game = evt.target.getAttribute('data-game');
+  const setting = evt.target.getAttribute('data-setting');
+  const option = evt.target.getAttribute('data-option');
+
+  if (evt.target.checked) {
+    // If the option is to be enabled and it is already enabled, do nothing
+    if (options[game][setting].includes(option)) { return; }
+
+    options[game][setting].push(option);
+  } else {
+    // If the option is to be disabled and it is already disabled, do nothing
+    if (!options[game][setting].includes(option)) { return; }
+
+    options[game][setting].splice(options[game][setting].indexOf(option), 1);
+  }
   localStorage.setItem('weighted-settings', JSON.stringify(options));
 };
 

WebHostLib/static/static/backgrounds/LICENSE

@@ -1,4 +1,3 @@
-Copyright 2022 Berserker66 (Fabian Dill)
 Copyright 2022 LegendaryLinux (Chris Wilson)
 
 All rights reserved.

WebHostLib/static/static/branding/LICENSE

@@ -0,0 +1,3 @@
+Copyright 2022 LegendaryLinux (Chris Wilson)
+
+All rights reserved.

WebHostLib/static/static/button-images/LICENSE

@@ -1,4 +1,3 @@
-Copyright 2022 Berserker66 (Fabian Dill)
 Copyright 2022 LegendaryLinux (Chris Wilson)
 
 All rights reserved.

WebHostLib/static/static/decorations/LICENSE

@@ -1,4 +1,3 @@
-Copyright 2022 Berserker66 (Fabian Dill)
 Copyright 2022 LegendaryLinux (Chris Wilson)
 
 All rights reserved.

WebHostLib/static/styles/checksfinderTracker.css

@@ -0,0 +1,30 @@
+#player-tracker-wrapper{
+    margin: 0;
+}
+
+#inventory-table{
+    padding: 8px 10px 2px 6px;
+    background-color: #42b149;
+    border-radius: 4px;
+    border: 2px solid black;
+}
+
+#inventory-table tr.column-headers td {
+    font-size: 1rem;
+    padding: 0 5rem 0 0;
+}
+
+#inventory-table td{
+    padding: 0 0.5rem 0.5rem;
+    font-family: LexendDeca-Light, monospace;
+    font-size: 2.5rem;
+    color: #ffffff;
+}
+
+#inventory-table td img{
+    vertical-align: middle;
+}
+
+.hide {
+    display: none;
+}

WebHostLib/static/styles/generate.css

@@ -56,7 +56,3 @@
 #file-input{
     display: none;
 }
-
-.interactive{
-    color: #ffef00;
-}

WebHostLib/static/styles/globalStyles.css

@@ -105,3 +105,10 @@ h5, h6{
     margin-bottom: 20px;
     background-color: #ffff00;
 }
+.user-message a{
+    color: #ff7700;
+}
+
+.interactive{
+    color: #ffef00;
+}

WebHostLib/static/styles/hostRoom.css

@@ -55,4 +55,6 @@
     border: 1px solid #2a6c2f;
     border-radius: 6px;
     color: #000000;
+    overflow-y: auto;
+    max-height: 400px;
 }

WebHostLib/static/styles/islandFooter.css

@@ -15,3 +15,33 @@
     padding-left: 0.5rem;
     color: #dfedc6;
 }
+@media all and (max-width: 900px) {
+    #island-footer{
+        font-size: 17px;
+        font-size: 2vw;
+    }
+}
+@media all and (max-width: 768px) {
+    #island-footer{
+        font-size: 15px;
+        font-size: 2vw;
+    }
+}
+@media all and (max-width: 650px) {
+    #island-footer{
+        font-size: 13px;
+        font-size: 2vw;
+    }
+}
+@media all and (max-width: 580px) {
+    #island-footer{
+        font-size: 11px;
+        font-size: 2vw;
+    }
+}
+@media all and (max-width: 512px) {
+    #island-footer{
+        font-size: 9px;
+        font-size: 2vw;
+    }
+}

WebHostLib/static/styles/landing.css

@@ -21,7 +21,6 @@ html{
     margin-right: auto;
     margin-top: 10px;
     height: 140px;
-    z-index: 10;
 }
 
 #landing-header h4{
@@ -223,7 +222,7 @@ html{
 }
 
 #landing{
-    width: 700px;
+    max-width: 700px;
     min-height: 280px;
     margin-left: auto;
     margin-right: auto;

WebHostLib/static/styles/ootTracker.css

@@ -9,7 +9,7 @@
     border-top-left-radius: 4px;
     border-top-right-radius: 4px;
     padding: 3px 3px 10px;
-    width: 448px;
+    width: 480px;
     background-color: rgb(60, 114, 157);
 }
 
@@ -46,7 +46,7 @@
 }
 
 #location-table{
-    width: 448px;
+    width: 480px;
     border-left: 2px solid #000000;
     border-right: 2px solid #000000;
     border-bottom: 2px solid #000000;
@@ -108,7 +108,7 @@
 }
 
 #location-table td:first-child {
-    width: 272px;
+    width: 300px;
 }
 
 .location-category td:first-child {

WebHostLib/static/styles/player-settings.css

@@ -49,7 +49,6 @@ html{
     font-weight: normal;
     width: 100%;
     margin-bottom: 0.5rem;
-    color: #ffffff;
     text-shadow: 1px 1px 4px #000000;
 }
 
@@ -58,20 +57,14 @@ html{
     font-weight: normal;
     width: 100%;
     margin-bottom: 0.5rem;
-    color: #ffe993;
     text-transform: lowercase;
     text-shadow: 1px 1px 2px #000000;
 }
 
 #player-settings h3, #player-settings h4, #player-settings h5, #player-settings h6{
-    color: #ffffff;
     text-shadow: 1px 1px 1px rgba(0, 0, 0, 0.5);
 }
 
-#player-settings a{
-    color: #ffef00;
-}
-
 #player-settings input:not([type]){
     border: 1px solid #000000;
     padding: 3px;
@@ -123,6 +116,10 @@ html{
     flex-grow: 1;
 }
 
+#player-settings table select:disabled{
+    background-color: lightgray;
+}
+
 #player-settings table .range-container{
     display: flex;
     flex-direction: row;
@@ -137,6 +134,35 @@ html{
     margin-left: 0.25rem;
 }
 
+#player-settings table .special-range-container{
+    display: flex;
+    flex-direction: column;
+}
+
+#player-settings table .special-range-wrapper{
+    display: flex;
+    flex-direction: row;
+    margin-top: 0.25rem;
+}
+
+#player-settings table .special-range-wrapper input[type=range]{
+    flex-grow: 1;
+}
+
+#player-settings table .randomize-button {
+    max-height: 24px;
+    line-height: 16px;
+    padding: 2px 8px;
+    margin: 0 0 0 0.25rem;
+    font-size: 12px;
+    border: 1px solid black;
+    border-radius: 3px;
+}
+
+#player-settings table .randomize-button.active {
+    background-color: #ffef00; /* Same as .interactive in globalStyles.css */
+}
+
 #player-settings table label{
     display: block;
     min-width: 200px;
@@ -148,7 +174,7 @@ html{
     border: none;
     padding: 3px;
     font-size: 17px;
-    vertical-align: middle;
+    vertical-align: top;
 }
 
 @media all and (max-width: 1000px), all and (orientation: portrait){

WebHostLib/static/styles/sc2wolTracker.css

@@ -0,0 +1,110 @@
+#player-tracker-wrapper{
+    margin: 0;
+}
+
+#inventory-table{
+    border-top: 2px solid #000000;
+    border-left: 2px solid #000000;
+    border-right: 2px solid #000000;
+    border-top-left-radius: 4px;
+    border-top-right-radius: 4px;
+    padding: 3px 3px 10px;
+    width: 500px;
+    background-color: #525494;
+}
+
+#inventory-table td{
+    width: 40px;
+    height: 40px;
+    text-align: center;
+    vertical-align: middle;
+}
+
+#inventory-table td.title{
+    padding-top: 10px;
+    height: 20px;
+    font-family: "JuraBook", monospace;
+    font-size: 16px;
+    font-weight: bold;
+}
+
+#inventory-table img{
+    height: 100%;
+    max-width: 40px;
+    max-height: 40px;
+    border: 1px solid #000000;
+    filter: grayscale(100%) contrast(75%) brightness(20%);
+}
+
+#inventory-table img.acquired{
+    filter: none;
+}
+
+#inventory-table div.counted-item {
+    position: relative;
+}
+
+#inventory-table div.item-count {
+    text-align: left;
+    color: black;
+    font-family: "JuraBook", monospace;
+    font-weight: bold;
+}
+
+#location-table{
+    width: 500px;
+    border-left: 2px solid #000000;
+    border-right: 2px solid #000000;
+    border-bottom: 2px solid #000000;
+    border-bottom-left-radius: 4px;
+    border-bottom-right-radius: 4px;
+    background-color: #525494;
+    padding: 10px 3px 3px;
+    font-family: "JuraBook", monospace;
+    font-size: 16px;
+    font-weight: bold;
+    cursor: default;
+}
+
+#location-table th{
+    vertical-align: middle;
+    text-align: left;
+    padding-right: 10px;
+}
+
+#location-table td{
+    padding-top: 2px;
+    padding-bottom: 2px;
+    line-height: 20px;
+}
+
+#location-table td.counter {
+    text-align: right;
+    font-size: 14px;
+}
+
+#location-table td.toggle-arrow {
+    text-align: right;
+}
+
+#location-table tr#Total-header {
+    font-weight: bold;
+}
+
+#location-table img{
+    height: 100%;
+    max-width: 30px;
+    max-height: 30px;
+}
+
+#location-table tbody.locations {
+    font-size: 16px;
+}
+
+#location-table td.location-name {
+    padding-left: 16px;
+}
+
+.hide {
+    display: none;
+}

WebHostLib/static/styles/themes/base.css

@@ -1,5 +1,7 @@
 html{
     padding-top: 110px;
+    scroll-padding-top: 100px;
+    scroll-behavior: smooth;
 }
 
 #base-header{
@@ -28,6 +30,8 @@ html{
 }
 
 #base-header-right{
+    display: flex;
+    flex-direction: row;
     margin-top: 4px;
 }
 
@@ -40,7 +44,7 @@ html{
     margin-top: 4px;
 }
 
-#base-header a{
+#base-header a, #base-header-mobile-menu a, #base-header-popover-text{
     color: #2f6b83;
     text-decoration: none;
     cursor: pointer;
@@ -49,3 +53,126 @@ html{
     font-family: LondrinaSolid-Light, sans-serif;
     text-transform: uppercase;
 }
+
+#base-header-right-mobile{
+    display: none;
+    margin-top: 2rem;
+    margin-right: 1rem;
+}
+
+#base-header-mobile-menu{
+    display: none;
+    flex-direction: column;
+    background-color: #ffffff;
+    text-align: center;
+    overflow-y: auto;
+    z-index: 10000;
+    width: 100vw;
+    border-bottom-left-radius: 20px;
+    border-bottom-right-radius: 20px;
+
+    position: absolute;
+    top: 7rem;
+    right: 0;
+}
+
+#base-header-mobile-menu a{
+    padding: 3rem 1.5rem;
+    font-size: 4rem;
+    line-height: 5rem;
+    color: #699ca8;
+    border-top: 1px solid #d3d3d3;
+}
+
+#base-header-mobile-menu :first-child, #base-header-popover-menu :first-child{
+    border-top: none;
+}
+
+#base-header-right-mobile img{
+    height: 3rem;
+}
+
+#base-header-popover-menu{
+    display: none;
+    flex-direction: column;
+    position: absolute;
+    background-color: #fff;
+    margin-left: -108px;
+    margin-top: 2.25rem;
+    border-radius: 10px;
+    border-left: 2px solid #d0ebe6;
+    border-bottom: 2px solid #d0ebe6;
+    border-right: 1px solid #d0ebe6;
+    filter: drop-shadow(-6px 6px 2px #2e3e83);
+}
+
+#base-header-popover-menu a{
+    color: #699ca8;
+    border-top: 1px solid #d3d3d3;
+    text-align: center;
+    font-size: 1.5rem;
+    line-height: 3rem;
+    margin-right: 2px;
+    padding: 0.25rem 1rem;
+}
+
+#base-header-popover-icon {
+    width: 14px;
+    margin-bottom: 3px;
+    margin-left: 2px;
+}
+
+@media all and (max-width: 960px), only screen and (max-device-width: 768px) {
+    #base-header-right{
+        display: none;
+    }
+
+    #base-header-right-mobile{
+        display: unset;
+    }
+}
+
+@media all and (max-width: 960px){
+    #base-header-right-mobile{
+        margin-top: 0.5rem;
+        margin-right: 0;
+    }
+
+    #base-header-right-mobile img{
+        height: 1.5rem;
+    }
+
+    #base-header-mobile-menu{
+        top: 3.3rem;
+        width: unset;
+        border-left: 2px solid #d0ebe6;
+        border-bottom: 2px solid #d0ebe6;
+        filter: drop-shadow(-6px 6px 2px #2e3e83);
+        border-top-left-radius: 10px;
+    }
+
+    #base-header-mobile-menu a{
+        font-size: 1.5rem;
+        line-height: 3rem;
+        margin: 0;
+        padding: 0.25rem 1rem;
+    }
+}
+
+@media only screen and (max-device-width: 768px){
+    html{
+        padding-top: 260px;
+        scroll-padding-top: 230px;
+    }
+
+    #base-header{
+        height: 200px;
+        background-size: auto 200px;
+    }
+
+    #base-header #site-title img{
+        height: calc(38px * 2);
+        margin-top: 30px;
+        margin-left: 20px;
+    }
+}

WebHostLib/static/styles/themes/dirt.css

@@ -52,6 +52,7 @@ pre{
 
 pre code{
     border: none;
+    display: block;
 }
 
 code{

WebHostLib/static/styles/themes/grass-flowers.css

@@ -52,6 +52,7 @@ pre{
 
 pre code{
     border: none;
+    display: block;
 }
 
 code{

WebHostLib/static/styles/themes/grass.css

@@ -52,6 +52,7 @@ pre{
 
 pre code{
     border: none;
+    display: block;
 }
 
 code{

WebHostLib/static/styles/themes/ice.css

@@ -52,6 +52,7 @@ pre{
 
 pre code{
     border: none;
+    display: block;
 }
 
 code{

WebHostLib/static/styles/themes/jungle.css

@@ -52,6 +52,7 @@ pre{
 
 pre code{
     border: none;
+    display: block;
 }
 
 code{

WebHostLib/static/styles/themes/ocean.css

@@ -53,6 +53,7 @@ pre{
 
 pre code{
     border: none;
+    display: block;
 }
 
 code{

WebHostLib/static/styles/themes/party-time.css

@@ -52,6 +52,7 @@ pre{
 
 pre code{
     border: none;
+    display: block;
 }
 
 code{

WebHostLib/static/styles/themes/stone.css

@@ -0,0 +1,66 @@
+html{
+    background-image: url('../../static/backgrounds/stone.png');
+    background-repeat: repeat;
+    background-size: 275px 275px;
+}
+
+body{
+    color: #ffffff;
+}
+
+#base-header {
+    background: url('../../static/backgrounds/header/stone-header.png') repeat-x;
+}
+
+.markdown {
+    background-color: rgba(0, 0, 0, 0.66) !important;
+}
+
+h1{
+    color: #cccbc3;
+}
+
+h2{
+    color: #aad79c;
+}
+
+h3, h4, h5,h6{
+    color: #ffffff;
+    text-shadow: 1px 1px 1px rgba(0, 0, 0, 0.5);
+}
+
+table th{
+
+}
+
+table td{
+
+}
+
+a{
+    color: #96e2ff;
+}
+
+pre{
+    margin-top: 0;
+    padding: 0.5rem 0.25rem;
+    border-radius: 6px;
+    color: #000000;
+}
+
+pre code{
+    border: none;
+    display: block;
+}
+
+code{
+    border-radius: 4px;
+    padding-left: 0.25rem;
+    padding-right: 0.25rem;
+    color: #000000;
+}
+
+pre, code{
+    background-color: #e4ffdb;
+    border: 1px solid #2d3435;
+}

WebHostLib/static/styles/timespinnerTracker.css

@@ -9,19 +9,54 @@
     border-top-left-radius: 4px;
     border-top-right-radius: 4px;
     padding: 3px 3px 10px;
-    width: 384px;
+    width: 374px;
     background-color: #8d60a7;
-}
 
-#inventory-table td{
-    width: 40px;
-    height: 40px;
-    text-align: center;
-    vertical-align: middle;
+    display: grid;
+    grid-template-rows: repeat(5, 48px);
+}
+
+#inventory-table img{
+    display: block;
+}
+
+#inventory-table div.table-row{
+    display: grid;
+    grid-template-columns: repeat(5, 1fr);
+}
+
+#inventory-table div.C1{
+    grid-column: 1;
+    place-content: center;
+    place-items: center;
+    display: flex;
+}
+#inventory-table div.C2{
+    grid-column: 2;
+    place-content: center;
+    place-items: center;
+    display: flex;
+}
+#inventory-table div.C3{
+    grid-column: 3;
+    place-content: center;
+    place-items: center;
+    display: flex;
+}
+#inventory-table div.C4{
+    grid-column: 4;
+    place-content: center;
+    place-items: center;
+    display: flex;
+}
+#inventory-table div.C5{
+    grid-column: 5;
+    place-content: center;
+    place-items: center;
+    display: flex;
 }
 
 #inventory-table img{
-    height: 100%;
     max-width: 40px;
     max-height: 40px;
     filter: grayscale(100%) contrast(75%) brightness(30%);
@@ -31,11 +66,70 @@
     filter: none;
 }
 
-#inventory-table div.counted-item {
+#inventory-table img.acquired.purple{ /*00FFFF*/
+    filter: hue-rotate(270deg) saturate(6) brightness(0.8);
+}
+#inventory-table img.acquired.cyan{ /*FF00FF*/
+    filter: hue-rotate(138deg) saturate(10) brightness(0.8);
+}
+#inventory-table img.acquired.green{ /*32CD32*/
+    filter: hue-rotate(84deg) saturate(10) brightness(0.7);
+}
+
+#inventory-table div.image-stack{
+    display: grid;
+    position: relative;
+    grid-template-columns: 1fr;
+    grid-template-rows: 1fr;
+}
+
+#inventory-table div.image-stack div.stack-back{
+    grid-column: 1;
+    grid-row: 1;
+}
+
+#inventory-table div.image-stack div.stack-front{
+    grid-column: 1;
+    grid-row: 1;
+    display: grid;
+    grid-template-columns: 20px 20px;
+    grid-template-rows: 20px 20px;
+}
+
+#inventory-table div.image-stack div.stack-top-left{
+    grid-column: 1;
+    grid-row: 1;
+    z-index: 1;
+}
+
+#inventory-table div.image-stack div.stack-top-right{
+    grid-column: 2;
+    grid-row: 1;
+    z-index: 1;
+}
+
+#inventory-table div.image-stack div.stack-bottum-left{
+    grid-column: 1;
+    grid-row: 2;
+    z-index: 1;
+}
+
+#inventory-table div.image-stack div.stack-bottum-right{
+    grid-column: 2;
+    grid-row: 2;
+    z-index: 1;
+}
+
+#inventory-table div.image-stack div.stack-front img{
+    width: 20px;
+    height: 20px;
+}
+
+#inventory-table div.counted-item{
     position: relative;
 }
 
-#inventory-table div.item-count {
+#inventory-table div.item-count{
     position: absolute;
     color: white;
     font-family: "Minecraftia", monospace;
@@ -69,16 +163,16 @@
     line-height: 20px;
 }
 
-#location-table td.counter {
+#location-table td.counter{
     text-align: right;
     font-size: 14px;
 }
 
-#location-table td.toggle-arrow {
+#location-table td.toggle-arrow{
     text-align: right;
 }
 
-#location-table tr#Total-header {
+#location-table tr#Total-header{
     font-weight: bold;
 }
 
@@ -88,14 +182,14 @@
     max-height: 30px;
 }
 
-#location-table tbody.locations {
+#location-table tbody.locations{
     font-size: 12px;
 }
 
-#location-table td.location-name {
+#location-table td.location-name{
     padding-left: 16px;
 }
 
-.hide {
+.hide{
     display: none;
 }

WebHostLib/static/styles/tooltip.css

@@ -14,7 +14,6 @@ give it one of the following classes: tooltip-left, tooltip-right, tooltip-top,
 /* Base styles for the element that has a tooltip */
 [data-tooltip], .tooltip {
 	position: relative;
-	cursor: pointer;
 }
 
 /* Base styles for the entire tooltip */
@@ -55,14 +54,15 @@ give it one of the following classes: tooltip-left, tooltip-right, tooltip-top,
 
 /** Content styles */
 .tooltip:after, [data-tooltip]:after {
+	width: 260px;
 	z-index: 10000;
 	padding: 8px;
-	width: 160px;
     border-radius: 4px;
 	background-color: #000;
 	background-color: hsla(0, 0%, 20%, 0.9);
 	color: #fff;
 	content: attr(data-tooltip);
+	white-space: pre-wrap;
 	font-size: 14px;
 	line-height: 1.2;
 }

WebHostLib/static/styles/tracker.css

@@ -119,6 +119,33 @@ img.alttp-sprite {
     background-color: #d3c97d;
 }
 
+#tracker-navigation {
+    display: inline-flex;
+    background-color: #b0a77d;
+    margin: 0.5rem;
+    border-radius: 4px;
+}
+
+.tracker-navigation-button {
+    display: block;
+    margin: 4px;
+    padding-left: 12px;
+    padding-right: 12px;
+    border-radius: 4px;
+    text-align: center;
+    font-size: 14px;
+    color: #000;
+    font-weight: lighter;
+}
+
+.tracker-navigation-button:hover {
+    background-color: #e2eabb !important;
+}
+
+.tracker-navigation-button.selected {
+    background-color: rgb(220, 226, 189);
+}
+
 @media all and (max-width: 1700px) {
     table.dataTable thead th.upper-row{
         position: -webkit-sticky;

WebHostLib/static/styles/weighted-settings.css

@@ -157,41 +157,29 @@ html{
     background-color: rgba(0, 0, 0, 0.1);
 }
 
-#weighted-settings .hints-div{
+#weighted-settings .hints-div, #weighted-settings .locations-div{
     margin-top: 2rem;
 }
 
-#weighted-settings .hints-div h3{
+#weighted-settings .hints-div h3, #weighted-settings .locations-div h3{
     margin-bottom: 0.5rem;
 }
 
-#weighted-settings .hints-div .hints-container{
+#weighted-settings .hints-container, #weighted-settings .locations-container{
     display: flex;
     flex-direction: row;
     justify-content: space-between;
+}
+
+#weighted-settings .hints-wrapper, #weighted-settings .locations-wrapper{
+    width: calc(50% - 0.5rem);
     font-weight: bold;
 }
 
-#weighted-settings .hints-div .hints-wrapper{
-    width: 32.5%;
-}
-
-#weighted-settings .hints-div .hints-wrapper .hint-div{
-    display: flex;
-    flex-direction: row;
-    cursor: pointer;
-    user-select: none;
-    -moz-user-select: none;
-}
-
-#weighted-settings .hints-div .hints-wrapper .hint-div:hover{
-    background-color: rgba(0, 0, 0, 0.1);
-}
-
-#weighted-settings .hints-div .hints-wrapper .hint-div label{
-    flex-grow: 1;
-    padding: 0.125rem 0.5rem;
-    cursor: pointer;
+#weighted-settings .hints-wrapper .simple-list, #weighted-settings .locations-wrapper .simple-list{
+    margin-top: 0.25rem;
+    height: 300px;
+    font-weight: normal;
 }
 
 #weighted-settings #weighted-settings-button-row{
@@ -280,6 +268,30 @@ html{
     flex-direction: column;
 }
 
+#weighted-settings .simple-list{
+    display: flex;
+    flex-direction: column;
+
+    max-height: 300px;
+    overflow-y: auto;
+    border: 1px solid #ffffff;
+    border-radius: 4px;
+}
+
+#weighted-settings .simple-list .list-row label{
+    display: block;
+    width: calc(100% - 0.5rem);
+    padding: 0.0625rem 0.25rem;
+}
+
+#weighted-settings .simple-list .list-row label:hover{
+    background-color: rgba(0, 0, 0, 0.1);
+}
+
+#weighted-settings .simple-list .list-row label input[type=checkbox]{
+    margin-right: 0.5rem;
+}
+
 #weighted-settings .invisible{
     display: none;
 }