Update test_options.py

* WebHost: use new safe yaml template filename

this mirrors the change in ArchipelagoMW/#4106 in WebHost

* WebHost: install docs into safe filename and require docs to be named safe

* Test: update doc test for safe name

* WebHost: fix import order to not break ModuleUpdate

.github/workflows/codeql-analysis.yml

@@ -47,7 +47,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -58,7 +58,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -72,4 +72,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/unittests.yml

@@ -89,4 +89,4 @@ jobs:
       run: |
         source venv/bin/activate
         export PYTHONPATH=$(pwd)
-        python test/hosting/__main__.py
+        timeout 600 python test/hosting/__main__.py

BaseClasses.py

@@ -341,7 +341,7 @@ class MultiWorld():
                     new_item.classification |= classifications[item_name]
                     new_itempool.append(new_item)
 
-            region = Region("Menu", group_id, self, "ItemLink")
+            region = Region(group["world"].origin_region_name, group_id, self, "ItemLink")
             self.regions.append(region)
             locations = region.locations
             # ensure that progression items are linked first, then non-progression
@@ -1264,6 +1264,10 @@ class Item:
     def trap(self) -> bool:
         return ItemClassification.trap in self.classification
 
+    @property
+    def excludable(self) -> bool:
+        return not (self.advancement or self.useful)
+
     @property
     def flags(self) -> int:
         return self.classification.as_flag()

Generate.py

@@ -110,7 +110,7 @@ def main(args=None) -> Tuple[argparse.Namespace, int]:
     player_files = {}
     for file in os.scandir(args.player_files_path):
         fname = file.name
-        if file.is_file() and not fname.startswith(".") and \
+        if file.is_file() and not fname.startswith(".") and not fname.lower().endswith(".ini") and \
                 os.path.join(args.player_files_path, fname) not in {args.meta_file_path, args.weights_file_path}:
             path = os.path.join(args.player_files_path, fname)
             try:

Launcher.py

@@ -104,6 +104,7 @@ components.extend([
     Component("Open host.yaml", func=open_host_yaml),
     Component("Open Patch", func=open_patch),
     Component("Generate Template Options", func=generate_yamls),
+    Component("Archipelago Website", func=lambda: webbrowser.open("https://archipelago.gg/")),
     Component("Discord Server", icon="discord", func=lambda: webbrowser.open("https://discord.gg/8Z65BR2")),
     Component("Unrated/18+ Discord Server", icon="discord", func=lambda: webbrowser.open("https://discord.gg/fqvNCCRsu4")),
     Component("Browse Files", func=browse_files),
@@ -254,7 +255,7 @@ def run_gui():
         _client_layout: Optional[ScrollBox] = None
 
         def __init__(self, ctx=None):
-            self.title = self.base_title
+            self.title = self.base_title + " " + Utils.__version__
             self.ctx = ctx
             self.icon = r"data/icon.png"
             super().__init__()

MultiServer.py

@@ -1960,8 +1960,10 @@ class ServerCommandProcessor(CommonCommandProcessor):
 
     def _cmd_exit(self) -> bool:
         """Shutdown the server"""
-        self.ctx.server.ws_server.close()
-        self.ctx.exit_event.set()
+        try:
+            self.ctx.server.ws_server.close()
+        finally:
+            self.ctx.exit_event.set()
         return True
 
     @mark_raw

Options.py

@@ -15,7 +15,7 @@ from dataclasses import dataclass
 from schema import And, Optional, Or, Schema
 from typing_extensions import Self
 
-from Utils import get_fuzzy_results, is_iterable_except_str, output_path
+from Utils import get_file_safe_name, get_fuzzy_results, is_iterable_except_str, output_path
 
 if typing.TYPE_CHECKING:
     from BaseClasses import MultiWorld, PlandoOptions
@@ -1531,7 +1531,7 @@ def generate_yaml_templates(target_folder: typing.Union[str, "pathlib.Path"], ge
 
             del file_data
 
-            with open(os.path.join(target_folder, game_name + ".yaml"), "w", encoding="utf-8-sig") as f:
+            with open(os.path.join(target_folder, get_file_safe_name(game_name) + ".yaml"), "w", encoding="utf-8-sig") as f:
                 f.write(res)
 
 

SNIClient.py

@@ -633,7 +633,13 @@ async def game_watcher(ctx: SNIContext) -> None:
         if not ctx.client_handler:
             continue
 
-        rom_validated = await ctx.client_handler.validate_rom(ctx)
+        try:
+            rom_validated = await ctx.client_handler.validate_rom(ctx)
+        except Exception as e:
+            snes_logger.error(f"An error occurred, see logs for details: {e}")
+            text_file_logger = logging.getLogger()
+            text_file_logger.exception(e)
+            rom_validated = False
 
         if not rom_validated or (ctx.auth and ctx.auth != ctx.rom):
             snes_logger.warning("ROM change detected, please reconnect to the multiworld server")
@@ -649,7 +655,13 @@ async def game_watcher(ctx: SNIContext) -> None:
 
         perf_counter = time.perf_counter()
 
-        await ctx.client_handler.game_watcher(ctx)
+        try:
+            await ctx.client_handler.game_watcher(ctx)
+        except Exception as e:
+            snes_logger.error(f"An error occurred, see logs for details: {e}")
+            text_file_logger = logging.getLogger()
+            text_file_logger.exception(e)
+            await snes_disconnect(ctx)
 
 
 async def run_game(romfile: str) -> None:

Utils.py

@@ -31,6 +31,7 @@ if typing.TYPE_CHECKING:
     import tkinter
     import pathlib
     from BaseClasses import Region
+    import multiprocessing
 
 
 def tuplize_version(version: str) -> Version:
@@ -423,7 +424,7 @@ class RestrictedUnpickler(pickle.Unpickler):
         if module == "NetUtils" and name in {"NetworkItem", "ClientStatus", "Hint", "SlotType", "NetworkSlot"}:
             return getattr(self.net_utils_module, name)
         # Options and Plando are unpickled by WebHost -> Generate
-        if module == "worlds.generic" and name in {"PlandoItem", "PlandoConnection"}:
+        if module == "worlds.generic" and name == "PlandoItem":
             if not self.generic_properties_module:
                 self.generic_properties_module = importlib.import_module("worlds.generic")
             return getattr(self.generic_properties_module, name)
@@ -434,7 +435,7 @@ class RestrictedUnpickler(pickle.Unpickler):
             else:
                 mod = importlib.import_module(module)
             obj = getattr(mod, name)
-            if issubclass(obj, self.options_module.Option):
+            if issubclass(obj, (self.options_module.Option, self.options_module.PlandoConnection)):
                 return obj
         # Forbid everything else.
         raise pickle.UnpicklingError(f"global '{module}.{name}' is forbidden")
@@ -664,6 +665,19 @@ def get_input_text_from_response(text: str, command: str) -> typing.Optional[str
     return None
 
 
+def is_kivy_running() -> bool:
+    if "kivy" in sys.modules:
+        from kivy.app import App
+        return App.get_running_app() is not None
+    return False
+
+
+def _mp_open_filename(res: "multiprocessing.Queue[typing.Optional[str]]", *args: Any) -> None:
+    if is_kivy_running():
+        raise RuntimeError("kivy should not be running in multiprocess")
+    res.put(open_filename(*args))
+
+
 def open_filename(title: str, filetypes: typing.Iterable[typing.Tuple[str, typing.Iterable[str]]], suggest: str = "") \
         -> typing.Optional[str]:
     logging.info(f"Opening file input dialog for {title}.")
@@ -693,6 +707,13 @@ def open_filename(title: str, filetypes: typing.Iterable[typing.Tuple[str, typin
                       f'This attempt was made because open_filename was used for "{title}".')
         raise e
     else:
+        if is_macos and is_kivy_running():
+            # on macOS, mixing kivy and tk does not work, so spawn a new process
+            # FIXME: performance of this is pretty bad, and we should (also) look into alternatives
+            from multiprocessing import Process, Queue
+            res: "Queue[typing.Optional[str]]" = Queue()
+            Process(target=_mp_open_filename, args=(res, title, filetypes, suggest)).start()
+            return res.get()
         try:
             root = tkinter.Tk()
         except tkinter.TclError:
@@ -702,6 +723,12 @@ def open_filename(title: str, filetypes: typing.Iterable[typing.Tuple[str, typin
                                                   initialfile=suggest or None)
 
 
+def _mp_open_directory(res: "multiprocessing.Queue[typing.Optional[str]]", *args: Any) -> None:
+    if is_kivy_running():
+        raise RuntimeError("kivy should not be running in multiprocess")
+    res.put(open_directory(*args))
+
+
 def open_directory(title: str, suggest: str = "") -> typing.Optional[str]:
     def run(*args: str):
         return subprocess.run(args, capture_output=True, text=True).stdout.split("\n", 1)[0] or None
@@ -725,9 +752,16 @@ def open_directory(title: str, suggest: str = "") -> typing.Optional[str]:
         import tkinter.filedialog
     except Exception as e:
         logging.error('Could not load tkinter, which is likely not installed. '
-                      f'This attempt was made because open_filename was used for "{title}".')
+                      f'This attempt was made because open_directory was used for "{title}".')
         raise e
     else:
+        if is_macos and is_kivy_running():
+            # on macOS, mixing kivy and tk does not work, so spawn a new process
+            # FIXME: performance of this is pretty bad, and we should (also) look into alternatives
+            from multiprocessing import Process, Queue
+            res: "Queue[typing.Optional[str]]" = Queue()
+            Process(target=_mp_open_directory, args=(res, title, suggest)).start()
+            return res.get()
         try:
             root = tkinter.Tk()
         except tkinter.TclError:
@@ -740,12 +774,6 @@ def messagebox(title: str, text: str, error: bool = False) -> None:
     def run(*args: str):
         return subprocess.run(args, capture_output=True, text=True).stdout.split("\n", 1)[0] or None
 
-    def is_kivy_running():
-        if "kivy" in sys.modules:
-            from kivy.app import App
-            return App.get_running_app() is not None
-        return False
-
     if is_kivy_running():
         from kvui import MessageBox
         MessageBox(title, text, error).open()

WebHost.py

@@ -12,6 +12,7 @@ ModuleUpdate.update()
 # in case app gets imported by something like gunicorn
 import Utils
 import settings
+from Utils import get_file_safe_name
 
 if typing.TYPE_CHECKING:
     from flask import Flask
@@ -71,7 +72,7 @@ def create_ordered_tutorials_file() -> typing.List[typing.Dict[str, typing.Any]]
     shutil.rmtree(base_target_path, ignore_errors=True)
     for game, world in worlds.items():
         # copy files from world's docs folder to the generated folder
-        target_path = os.path.join(base_target_path, game)
+        target_path = os.path.join(base_target_path, get_file_safe_name(game))
         os.makedirs(target_path, exist_ok=True)
 
         if world.zip_path:

WebHostLib/__init__.py

@@ -9,7 +9,7 @@ from flask_compress import Compress
 from pony.flask import Pony
 from werkzeug.routing import BaseConverter
 
-from Utils import title_sorted
+from Utils import title_sorted, get_file_safe_name
 
 UPLOAD_FOLDER = os.path.relpath('uploads')
 LOGS_FOLDER = os.path.relpath('logs')
@@ -20,6 +20,7 @@ Pony(app)
 
 app.jinja_env.filters['any'] = any
 app.jinja_env.filters['all'] = all
+app.jinja_env.filters['get_file_safe_name'] = get_file_safe_name
 
 app.config["SELFHOST"] = True  # application process is in charge of running the websites
 app.config["GENERATORS"] = 8  # maximum concurrent world gens

WebHostLib/misc.py

@@ -77,7 +77,13 @@ def faq(lang: str):
     return render_template(
         "markdown_document.html",
         title="Frequently Asked Questions",
-        html_from_markdown=markdown.markdown(document,  extensions=["mdx_breakless_lists"]),
+        html_from_markdown=markdown.markdown(
+            document,
+            extensions=["toc", "mdx_breakless_lists"],
+            extension_configs={
+                "toc": {"anchorlink": True}
+            }
+        ),
     )
 
 
@@ -90,7 +96,13 @@ def glossary(lang: str):
     return render_template(
         "markdown_document.html",
         title="Glossary",
-        html_from_markdown=markdown.markdown(document,  extensions=["mdx_breakless_lists"]),
+        html_from_markdown=markdown.markdown(
+            document,
+            extensions=["toc", "mdx_breakless_lists"],
+            extension_configs={
+                "toc": {"anchorlink": True}
+            }
+        ),
     )
 
 

WebHostLib/requirements.txt

@@ -1,5 +1,5 @@
 flask>=3.0.3
-werkzeug>=3.0.4
+werkzeug>=3.0.6
 pony>=0.7.19
 waitress>=3.0.0
 Flask-Caching>=2.3.0

WebHostLib/static/styles/markdown.css

@@ -28,7 +28,7 @@
     font-weight: normal;
     font-family: LondrinaSolid-Regular, sans-serif;
     text-transform: uppercase;
-    cursor: pointer;
+    cursor: pointer; /* TODO: remove once we drop showdown.js */
     width: 100%;
     text-shadow: 1px 1px 4px #000000;
 }
@@ -37,7 +37,7 @@
     font-size: 38px;
     font-weight: normal;
     font-family: LondrinaSolid-Light, sans-serif;
-    cursor: pointer;
+    cursor: pointer; /* TODO: remove once we drop showdown.js */
     width: 100%;
     margin-top: 20px;
     margin-bottom: 0.5rem;
@@ -50,7 +50,7 @@
     font-family: LexendDeca-Regular, sans-serif;
     text-transform: none;
     text-align: left;
-    cursor: pointer;
+    cursor: pointer; /* TODO: remove once we drop showdown.js */
     width: 100%;
     margin-bottom: 0.5rem;
 }
@@ -59,7 +59,7 @@
     font-family: LexendDeca-Regular, sans-serif;
     text-transform: none;
     font-size: 24px;
-    cursor: pointer;
+    cursor: pointer; /* TODO: remove once we drop showdown.js */
     margin-bottom: 24px;
 }
 
@@ -67,20 +67,29 @@
     font-family: LexendDeca-Regular, sans-serif;
     text-transform: none;
     font-size: 22px;
-    cursor: pointer;
+    cursor: pointer; /* TODO: remove once we drop showdown.js */
 }
 
 .markdown h6, .markdown details summary.h6{
     font-family: LexendDeca-Regular, sans-serif;
     text-transform: none;
     font-size: 20px;
-    cursor: pointer;;
+    cursor: pointer; /* TODO: remove once we drop showdown.js */
 }
 
 .markdown h4, .markdown h5, .markdown h6{
     margin-bottom: 0.5rem;
 }
 
+.markdown h1 > a,
+.markdown h2 > a,
+.markdown h3 > a,
+.markdown h4 > a,
+.markdown h5 > a,
+.markdown h6 > a {
+    color: inherit;
+}
+
 .markdown ul{
     margin-top: 0.5rem;
     margin-bottom: 0.5rem;

WebHostLib/templates/gameInfo.html

@@ -11,7 +11,7 @@
 
 {% block body %}
     {% include 'header/'+theme+'Header.html' %}
-    <div id="game-info" class="markdown" data-lang="{{ lang }}" data-game="{{ game }}">
+    <div id="game-info" class="markdown" data-lang="{{ lang }}" data-game="{{ game | get_file_safe_name }}">
         <!-- Populated my JS / MD -->
     </div>
 {% endblock %}

WebHostLib/templates/playerOptions/macros.html

@@ -196,13 +196,14 @@
 {% macro OptionTitle(option_name, option) %}
     <label for="{{ option_name }}">
         {{ option.display_name|default(option_name) }}:
+        {% set rich_text = option.rich_text_doc or (option.rich_text_doc is none and world.web.rich_text_options_doc) %}
         <span
             class="interactive tooltip-container"
-            {% if not (option.rich_text_doc | default(world.web.rich_text_options_doc, true)) %}
+            {% if not rich_text %}
               data-tooltip="{{(option.__doc__ | default("Please document me!"))|replace('\n    ', '\n')|escape|trim}}"
             {% endif %}>
           (?)
-          {% if option.rich_text_doc | default(world.web.rich_text_options_doc, true) %}
+          {% if rich_text %}
             <div class="tooltip">
               {{ option.__doc__ | default("**Please document me!**") | rst_to_html | safe }}
             </div>

WebHostLib/templates/playerOptions/playerOptions.html

@@ -42,7 +42,7 @@
             A list of all games you have generated can be found on the <a href="/user-content">User Content Page</a>.
             <br />
             You may also download the
-            <a href="/static/generated/configs/{{ world_name }}.yaml">template file for this game</a>.
+            <a href="/static/generated/configs/{{ world_name | get_file_safe_name }}.yaml">template file for this game</a>.
         </p>
 
         <form id="options-form" method="post" enctype="application/x-www-form-urlencoded" action="generate-yaml">

WebHostLib/templates/tutorial.html

@@ -11,7 +11,7 @@
 {% endblock %}
 
 {% block body %}
-    <div id="tutorial-wrapper" class="markdown" data-game="{{ game }}" data-file="{{ file }}" data-lang="{{ lang }}">
+    <div id="tutorial-wrapper" class="markdown" data-game="{{ game | get_file_safe_name }}" data-file="{{ file | get_file_safe_name }}" data-lang="{{ lang }}">
         <!-- Content generated by JavaScript -->
     </div>
 {%  endblock %}

WebHostLib/tracker.py

@@ -5,7 +5,7 @@ from typing import Any, Callable, Dict, List, Optional, Set, Tuple, NamedTuple,
 from uuid import UUID
 from email.utils import parsedate_to_datetime
 
-from flask import render_template, make_response, Response, request
+from flask import make_response, render_template, request, Request, Response
 from werkzeug.exceptions import abort
 
 from MultiServer import Context, get_saving_second
@@ -298,17 +298,25 @@ class TrackerData:
         return self._multidata.get("spheres", [])
 
 
-def _process_if_request_valid(incoming_request, room: Optional[Room]) -> Optional[Response]:
+def _process_if_request_valid(incoming_request: Request, room: Optional[Room]) -> Optional[Response]:
     if not room:
         abort(404)
 
-    if_modified = incoming_request.headers.get("If-Modified-Since", None)
-    if if_modified:
-        if_modified = parsedate_to_datetime(if_modified)
+    if_modified_str: Optional[str] = incoming_request.headers.get("If-Modified-Since", None)
+    if if_modified_str:
+        if_modified = parsedate_to_datetime(if_modified_str)
+        if if_modified.tzinfo is None:
+            abort(400)  # standard requires "GMT" timezone
+        # database may use datetime.utcnow(), which is timezone-naive. convert to timezone-aware.
+        last_activity = room.last_activity
+        if last_activity.tzinfo is None:
+            last_activity = room.last_activity.replace(tzinfo=datetime.timezone.utc)
         # if_modified has less precision than last_activity, so we bring them to same precision
-        if if_modified >= room.last_activity.replace(microsecond=0):
+        if if_modified >= last_activity.replace(microsecond=0):
             return make_response("",  304)
 
+    return None
+
 
 @app.route("/tracker/<suuid:tracker>/<int:tracked_team>/<int:tracked_player>")
 def get_player_tracker(tracker: UUID, tracked_team: int, tracked_player: int, generic: bool = False) -> Response:

data/options.yaml

@@ -28,9 +28,9 @@
 name: Player{number}
 
 # Used to describe your yaml. Useful if you have multiple files.
-description: Default {{ game }} Template
+description: {{ yaml_dump("Default %s Template" % game) }}
 
-game: {{ game }}
+game: {{ yaml_dump(game) }}
 requires:
   version: {{ __version__ }} # Version of Archipelago required for this yaml to work as expected.
 
@@ -44,7 +44,7 @@ requires:
     {%- endfor -%}
 {% endmacro %}
 
-{{ game }}:
+{{ yaml_dump(game) }}:
   {%- for group_name, group_options in option_groups.items() %}
   # {{ group_name }}
 

docs/CODEOWNERS

@@ -143,7 +143,7 @@
 /worlds/shivers/ @GodlFire
 
 # A Short Hike
-/worlds/shorthike/ @chandler05
+/worlds/shorthike/ @chandler05 @BrandenEK
 
 # Sonic Adventure 2 Battle
 /worlds/sa2b/ @PoryGone @RaspberrySpace

docs/running from source.md

@@ -85,4 +85,4 @@ PyCharm has a built-in version control integration that supports Git.
 
 ## Running tests
 
-Run `pip install pytest pytest-subtests`, then use your IDE to run tests or run `pytest` from the source folder.
+Information about running tests can be found in [tests.md](https://github.com/ArchipelagoMW/Archipelago/blob/main/docs/tests.md#running-tests)

docs/tests.md

@@ -84,7 +84,19 @@ testing portions of your code that can be tested without relying on a multiworld
 
 ## Running Tests
 
-In PyCharm, running all tests can be done by right-clicking the root `test` directory and selecting `run Python tests`.
-If you do not have pytest installed, you may get import failures. To solve this, edit the run configuration, and set the
-working directory of the run to the Archipelago directory. If you only want to run your world's defined tests, repeat
-the steps for the test directory within your world.
+#### Using Pycharm
+
+In PyCharm, running all tests can be done by right-clicking the root test directory and selecting Run 'Archipelago Unittests'. 
+Unless you configured PyCharm to use pytest as a test runner, you may get import failures. To solve this, edit the run configuration, 
+and set the working directory to the Archipelago directory which contains all the project files. 
+
+If you only want to run your world's defined tests, repeat the steps for the test directory within your world.
+Your working directory should be the directory of your world in the worlds directory and the script should be the 
+tests folder within your world.
+
+You can also find the 'Archipelago Unittests' as an option in the dropdown at the top of the window
+next to the run and debug buttons.
+
+#### Running Tests without Pycharm
+
+Run `pip install pytest pytest-subtests`, then use your IDE to run tests or run `pytest` from the source folder.

requirements.txt

@@ -1,5 +1,5 @@
 colorama>=0.4.6
-websockets>=13.0.1
+websockets>=13.0.1,<14
 PyYAML>=6.0.2
 jellyfish>=1.1.0
 jinja2>=3.1.4

setup.py

@@ -5,7 +5,6 @@ import platform
 import shutil
 import sys
 import sysconfig
-import typing
 import warnings
 import zipfile
 import urllib.request
@@ -14,14 +13,14 @@ import json
 import threading
 import subprocess
 
-from collections.abc import Iterable
 from hashlib import sha3_512
 from pathlib import Path
+from typing import Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union
 
 
 # This is a bit jank. We need cx-Freeze to be able to run anything from this script, so install it
+requirement = 'cx-Freeze==7.2.0'
 try:
-    requirement = 'cx-Freeze==7.2.0'
     import pkg_resources
     try:
         pkg_resources.require(requirement)
@@ -30,7 +29,7 @@ try:
         install_cx_freeze = True
 except ImportError:
     install_cx_freeze = True
-    pkg_resources = None  # type: ignore [assignment]
+    pkg_resources = None  # type: ignore[assignment]
 
 if install_cx_freeze:
     # check if pip is available
@@ -61,7 +60,7 @@ from Cython.Build import cythonize
 
 
 # On  Python < 3.10 LogicMixin is not currently supported.
-non_apworlds: set = {
+non_apworlds: Set[str] = {
     "A Link to the Past",
     "Adventure",
     "ArchipIDLE",
@@ -84,7 +83,7 @@ non_apworlds: set = {
 if sys.version_info < (3,10):
     non_apworlds.add("Hollow Knight")
 
-def download_SNI():
+def download_SNI() -> None:
     print("Updating SNI")
     machine_to_go = {
         "x86_64": "amd64",
@@ -94,7 +93,7 @@ def download_SNI():
     platform_name = platform.system().lower()
     machine_name = platform.machine().lower()
     # force amd64 on macos until we have universal2 sni, otherwise resolve to GOARCH
-    machine_name = "amd64" if platform_name == "darwin" else machine_to_go.get(machine_name, machine_name)
+    machine_name = "universal" if platform_name == "darwin" else machine_to_go.get(machine_name, machine_name)
     with urllib.request.urlopen("https://api.github.com/repos/alttpo/sni/releases/latest") as request:
         data = json.load(request)
     files = data["assets"]
@@ -105,17 +104,19 @@ def download_SNI():
         download_url: str = file["browser_download_url"]
         machine_match = download_url.rsplit("-", 1)[1].split(".", 1)[0] == machine_name
         if platform_name in download_url and machine_match:
+            source_url = download_url
             # prefer "many" builds
             if "many" in download_url:
-                source_url = download_url
                 break
-            source_url = download_url
+            # prefer the correct windows or windows7 build
+            if platform_name == "windows" and ("windows7" in download_url) == (sys.version_info < (3, 9)):
+                break
 
     if source_url and source_url.endswith(".zip"):
         with urllib.request.urlopen(source_url) as download:
             with zipfile.ZipFile(io.BytesIO(download.read()), "r") as zf:
-                for member in zf.infolist():
-                    zf.extract(member, path="SNI")
+                for zf_member in zf.infolist():
+                    zf.extract(zf_member, path="SNI")
         print(f"Downloaded SNI from {source_url}")
 
     elif source_url and (source_url.endswith(".tar.xz") or source_url.endswith(".tar.gz")):
@@ -129,11 +130,13 @@ def download_SNI():
                         raise ValueError(f"Unexpected file '{member.name}' in {source_url}")
                     elif member.isdir() and not sni_dir:
                         sni_dir = member.name
-                    elif member.isfile() and not sni_dir or not member.name.startswith(sni_dir):
+                    elif member.isfile() and not sni_dir or sni_dir and not member.name.startswith(sni_dir):
                         raise ValueError(f"Expected folder before '{member.name}' in {source_url}")
                     elif member.isfile() and sni_dir:
                         tf.extract(member)
             # sadly SNI is in its own folder on non-windows, so we need to rename
+            if not sni_dir:
+                raise ValueError("Did not find SNI in archive")
             shutil.rmtree("SNI", True)
             os.rename(sni_dir, "SNI")
         print(f"Downloaded SNI from {source_url}")
@@ -145,7 +148,7 @@ def download_SNI():
         print(f"No SNI found for system spec {platform_name} {machine_name}")
 
 
-signtool: typing.Optional[str]
+signtool: Optional[str]
 if os.path.exists("X:/pw.txt"):
     print("Using signtool")
     with open("X:/pw.txt", encoding="utf-8-sig") as f:
@@ -197,13 +200,13 @@ extra_data = ["LICENSE", "data", "EnemizerCLI", "SNI"]
 extra_libs = ["libssl.so", "libcrypto.so"] if is_linux else []
 
 
-def remove_sprites_from_folder(folder):
+def remove_sprites_from_folder(folder: Path) -> None:
     for file in os.listdir(folder):
         if file != ".gitignore":
             os.remove(folder / file)
 
 
-def _threaded_hash(filepath):
+def _threaded_hash(filepath: Union[str, Path]) -> str:
     hasher = sha3_512()
     hasher.update(open(filepath, "rb").read())
     return base64.b85encode(hasher.digest()).decode()
@@ -217,11 +220,11 @@ class BuildCommand(setuptools.command.build.build):
     yes: bool
     last_yes: bool = False  # used by sub commands of build
 
-    def initialize_options(self):
+    def initialize_options(self) -> None:
         super().initialize_options()
         type(self).last_yes = self.yes = False
 
-    def finalize_options(self):
+    def finalize_options(self) -> None:
         super().finalize_options()
         type(self).last_yes = self.yes
 
@@ -233,27 +236,27 @@ class BuildExeCommand(cx_Freeze.command.build_exe.build_exe):
         ('extra-data=', None, 'Additional files to add.'),
     ]
     yes: bool
-    extra_data: Iterable  # [any] not available in 3.8
-    extra_libs: Iterable  # work around broken include_files
+    extra_data: Iterable[str]
+    extra_libs: Iterable[str]  # work around broken include_files
 
     buildfolder: Path
     libfolder: Path
     library: Path
     buildtime: datetime.datetime
 
-    def initialize_options(self):
+    def initialize_options(self) -> None:
         super().initialize_options()
         self.yes = BuildCommand.last_yes
         self.extra_data = []
         self.extra_libs = []
 
-    def finalize_options(self):
+    def finalize_options(self) -> None:
         super().finalize_options()
         self.buildfolder = self.build_exe
         self.libfolder = Path(self.buildfolder, "lib")
         self.library = Path(self.libfolder, "library.zip")
 
-    def installfile(self, path, subpath=None, keep_content: bool = False):
+    def installfile(self, path: Path, subpath: Optional[Union[str, Path]] = None, keep_content: bool = False) -> None:
         folder = self.buildfolder
         if subpath:
             folder /= subpath
@@ -268,7 +271,7 @@ class BuildExeCommand(cx_Freeze.command.build_exe.build_exe):
         else:
             print('Warning,', path, 'not found')
 
-    def create_manifest(self, create_hashes=False):
+    def create_manifest(self, create_hashes: bool = False) -> None:
         # Since the setup is now split into components and the manifest is not,
         # it makes most sense to just remove the hashes for now. Not aware of anyone using them.
         hashes = {}
@@ -290,7 +293,7 @@ class BuildExeCommand(cx_Freeze.command.build_exe.build_exe):
         json.dump(manifest, open(manifestpath, "wt"), indent=4)
         print("Created Manifest")
 
-    def run(self):
+    def run(self) -> None:
         # start downloading sni asap
         sni_thread = threading.Thread(target=download_SNI, name="SNI Downloader")
         sni_thread.start()
@@ -341,7 +344,7 @@ class BuildExeCommand(cx_Freeze.command.build_exe.build_exe):
 
         # post build steps
         if is_windows:  # kivy_deps is win32 only, linux picks them up automatically
-            from kivy_deps import sdl2, glew
+            from kivy_deps import sdl2, glew  # type: ignore
             for folder in sdl2.dep_bins + glew.dep_bins:
                 shutil.copytree(folder, self.libfolder, dirs_exist_ok=True)
                 print(f"copying {folder} -> {self.libfolder}")
@@ -362,7 +365,7 @@ class BuildExeCommand(cx_Freeze.command.build_exe.build_exe):
             self.installfile(Path(data))
 
         # kivi data files
-        import kivy
+        import kivy  # type: ignore[import-untyped]
         shutil.copytree(os.path.join(os.path.dirname(kivy.__file__), "data"),
                         self.buildfolder / "data",
                         dirs_exist_ok=True)
@@ -372,7 +375,7 @@ class BuildExeCommand(cx_Freeze.command.build_exe.build_exe):
         from worlds.AutoWorld import AutoWorldRegister
         assert not non_apworlds - set(AutoWorldRegister.world_types), \
             f"Unknown world {non_apworlds - set(AutoWorldRegister.world_types)} designated for .apworld"
-        folders_to_remove: typing.List[str] = []
+        folders_to_remove: List[str] = []
         disabled_worlds_folder = "worlds_disabled"
         for entry in os.listdir(disabled_worlds_folder):
             if os.path.isdir(os.path.join(disabled_worlds_folder, entry)):
@@ -393,7 +396,7 @@ class BuildExeCommand(cx_Freeze.command.build_exe.build_exe):
                 shutil.rmtree(world_directory)
         shutil.copyfile("meta.yaml", self.buildfolder / "Players" / "Templates" / "meta.yaml")
         try:
-            from maseya import z3pr
+            from maseya import z3pr  # type: ignore[import-untyped]
         except ImportError:
             print("Maseya Palette Shuffle not found, skipping data files.")
         else:
@@ -444,16 +447,16 @@ class AppImageCommand(setuptools.Command):
         ("app-exec=", None, "The application to run inside the image."),
         ("yes", "y", 'Answer "yes" to all questions.'),
     ]
-    build_folder: typing.Optional[Path]
-    dist_file: typing.Optional[Path]
-    app_dir: typing.Optional[Path]
+    build_folder: Optional[Path]
+    dist_file: Optional[Path]
+    app_dir: Optional[Path]
     app_name: str
-    app_exec: typing.Optional[Path]
-    app_icon: typing.Optional[Path]  # source file
+    app_exec: Optional[Path]
+    app_icon: Optional[Path]  # source file
     app_id: str  # lower case name, used for icon and .desktop
     yes: bool
 
-    def write_desktop(self):
+    def write_desktop(self) -> None:
         assert self.app_dir, "Invalid app_dir"
         desktop_filename = self.app_dir / f"{self.app_id}.desktop"
         with open(desktop_filename, 'w', encoding="utf-8") as f:
@@ -468,7 +471,7 @@ class AppImageCommand(setuptools.Command):
             )))
         desktop_filename.chmod(0o755)
 
-    def write_launcher(self, default_exe: Path):
+    def write_launcher(self, default_exe: Path) -> None:
         assert self.app_dir, "Invalid app_dir"
         launcher_filename = self.app_dir / "AppRun"
         with open(launcher_filename, 'w', encoding="utf-8") as f:
@@ -491,7 +494,7 @@ $APPDIR/$exe "$@"
 """)
         launcher_filename.chmod(0o755)
 
-    def install_icon(self, src: Path, name: typing.Optional[str] = None, symlink: typing.Optional[Path] = None):
+    def install_icon(self, src: Path, name: Optional[str] = None, symlink: Optional[Path] = None) -> None:
         assert self.app_dir, "Invalid app_dir"
         try:
             from PIL import Image
@@ -513,7 +516,8 @@ $APPDIR/$exe "$@"
         if symlink:
             symlink.symlink_to(dest_file.relative_to(symlink.parent))
 
-    def initialize_options(self):
+    def initialize_options(self) -> None:
+        assert self.distribution.metadata.name
         self.build_folder = None
         self.app_dir = None
         self.app_name = self.distribution.metadata.name
@@ -527,17 +531,22 @@ $APPDIR/$exe "$@"
         ))
         self.yes = False
 
-    def finalize_options(self):
+    def finalize_options(self) -> None:
+        assert self.build_folder
         if not self.app_dir:
             self.app_dir = self.build_folder.parent / "AppDir"
         self.app_id = self.app_name.lower()
 
-    def run(self):
+    def run(self) -> None:
+        assert self.build_folder and self.dist_file, "Command not properly set up"
+        assert (
+            self.app_icon and self.app_id and self.app_dir and self.app_exec and self.app_name
+        ), "AppImageCommand not properly set up"
         self.dist_file.parent.mkdir(parents=True, exist_ok=True)
         if self.app_dir.is_dir():
             shutil.rmtree(self.app_dir)
         self.app_dir.mkdir(parents=True)
-        opt_dir = self.app_dir / "opt" / self.distribution.metadata.name
+        opt_dir = self.app_dir / "opt" / self.app_name
         shutil.copytree(self.build_folder, opt_dir)
         root_icon = self.app_dir / f'{self.app_id}{self.app_icon.suffix}'
         self.install_icon(self.app_icon, self.app_id, symlink=root_icon)
@@ -548,7 +557,7 @@ $APPDIR/$exe "$@"
         subprocess.call(f'ARCH={build_arch} ./appimagetool -n "{self.app_dir}" "{self.dist_file}"', shell=True)
 
 
-def find_libs(*args: str) -> typing.Sequence[typing.Tuple[str, str]]:
+def find_libs(*args: str) -> Sequence[Tuple[str, str]]:
     """Try to find system libraries to be included."""
     if not args:
         return []
@@ -556,7 +565,7 @@ def find_libs(*args: str) -> typing.Sequence[typing.Tuple[str, str]]:
     arch = build_arch.replace('_', '-')
     libc = 'libc6'  # we currently don't support musl
 
-    def parse(line):
+    def parse(line: str) -> Tuple[Tuple[str, str, str], str]:
         lib, path = line.strip().split(' => ')
         lib, typ = lib.split(' ', 1)
         for test_arch in ('x86-64', 'i386', 'aarch64'):
@@ -577,26 +586,29 @@ def find_libs(*args: str) -> typing.Sequence[typing.Tuple[str, str]]:
         ldconfig = shutil.which("ldconfig")
         assert ldconfig, "Make sure ldconfig is in PATH"
         data = subprocess.run([ldconfig, "-p"], capture_output=True, text=True).stdout.split("\n")[1:]
-        find_libs.cache = {  # type: ignore [attr-defined]
+        find_libs.cache = {  # type: ignore[attr-defined]
             k: v for k, v in (parse(line) for line in data if "=>" in line)
         }
 
-    def find_lib(lib, arch, libc):
-        for k, v in find_libs.cache.items():
+    def find_lib(lib: str, arch: str, libc: str) -> Optional[str]:
+        cache: Dict[Tuple[str, str, str], str] = getattr(find_libs, "cache")
+        for k, v in cache.items():
             if k == (lib, arch, libc):
                 return v
-        for k, v, in find_libs.cache.items():
+        for k, v, in cache.items():
             if k[0].startswith(lib) and k[1] == arch and k[2] == libc:
                 return v
         return None
 
-    res = []
+    res: List[Tuple[str, str]] = []
     for arg in args:
         # try exact match, empty libc, empty arch, empty arch and libc
         file = find_lib(arg, arch, libc)
         file = file or find_lib(arg, arch, '')
         file = file or find_lib(arg, '', libc)
         file = file or find_lib(arg, '', '')
+        if not file:
+            raise ValueError(f"Could not find lib {arg}")
         # resolve symlinks
         for n in range(0, 5):
             res.append((file, os.path.join('lib', os.path.basename(file))))
@@ -620,7 +632,7 @@ cx_Freeze.setup(
             "packages": ["worlds", "kivy", "cymem", "websockets"],
             "includes": [],
             "excludes": ["numpy", "Cython", "PySide2", "PIL",
-                         "pandas"],
+                         "pandas", "zstandard"],
             "zip_include_packages": ["*"],
             "zip_exclude_packages": ["worlds", "sc2", "orjson"],  # TODO: remove orjson here once we drop py3.8 support
             "include_files": [],  # broken in cx 6.14.0, we use more special sauce now

test/__init__.py

@@ -4,6 +4,7 @@ import warnings
 import settings
 
 warnings.simplefilter("always")
+warnings.filterwarnings(action="ignore", category=DeprecationWarning, module="s2clientprotocol")
 settings.no_gui = True
 settings.skip_autosave = True
 

test/general/test_fill.py

@@ -688,8 +688,8 @@ class TestDistributeItemsRestrictive(unittest.TestCase):
         for item in multiworld.get_items():
             item.classification = ItemClassification.useful
 
-        multiworld.local_items[player1.id].value = set(names(player1.basic_items))
-        multiworld.local_items[player2.id].value = set(names(player2.basic_items))
+        multiworld.worlds[player1.id].options.local_items.value = set(names(player1.basic_items))
+        multiworld.worlds[player2.id].options.local_items.value = set(names(player2.basic_items))
         locality_rules(multiworld)
 
         distribute_items_restrictive(multiworld)
@@ -795,8 +795,8 @@ class TestBalanceMultiworldProgression(unittest.TestCase):
 
     def test_balances_progression(self) -> None:
         """Tests that progression balancing moves progression items earlier"""
-        self.multiworld.progression_balancing[self.player1.id].value = 50
-        self.multiworld.progression_balancing[self.player2.id].value = 50
+        self.multiworld.worlds[self.player1.id].options.progression_balancing.value = 50
+        self.multiworld.worlds[self.player2.id].options.progression_balancing.value = 50
 
         self.assertRegionContains(
             self.player1.regions[2], self.player2.prog_items[0])
@@ -808,8 +808,8 @@ class TestBalanceMultiworldProgression(unittest.TestCase):
 
     def test_balances_progression_light(self) -> None:
         """Test that progression balancing still moves items earlier on minimum value"""
-        self.multiworld.progression_balancing[self.player1.id].value = 1
-        self.multiworld.progression_balancing[self.player2.id].value = 1
+        self.multiworld.worlds[self.player1.id].options.progression_balancing.value = 1
+        self.multiworld.worlds[self.player2.id].options.progression_balancing.value = 1
 
         self.assertRegionContains(
             self.player1.regions[2], self.player2.prog_items[0])
@@ -822,8 +822,8 @@ class TestBalanceMultiworldProgression(unittest.TestCase):
 
     def test_balances_progression_heavy(self) -> None:
         """Test that progression balancing moves items earlier on maximum value"""
-        self.multiworld.progression_balancing[self.player1.id].value = 99
-        self.multiworld.progression_balancing[self.player2.id].value = 99
+        self.multiworld.worlds[self.player1.id].options.progression_balancing.value = 99
+        self.multiworld.worlds[self.player2.id].options.progression_balancing.value = 99
 
         self.assertRegionContains(
             self.player1.regions[2], self.player2.prog_items[0])
@@ -836,8 +836,8 @@ class TestBalanceMultiworldProgression(unittest.TestCase):
 
     def test_skips_balancing_progression(self) -> None:
         """Test that progression balancing is skipped when players have it disabled"""
-        self.multiworld.progression_balancing[self.player1.id].value = 0
-        self.multiworld.progression_balancing[self.player2.id].value = 0
+        self.multiworld.worlds[self.player1.id].options.progression_balancing.value = 0
+        self.multiworld.worlds[self.player2.id].options.progression_balancing.value = 0
 
         self.assertRegionContains(
             self.player1.regions[2], self.player2.prog_items[0])
@@ -849,8 +849,8 @@ class TestBalanceMultiworldProgression(unittest.TestCase):
 
     def test_ignores_priority_locations(self) -> None:
         """Test that progression items on priority locations don't get moved by balancing"""
-        self.multiworld.progression_balancing[self.player1.id].value = 50
-        self.multiworld.progression_balancing[self.player2.id].value = 50
+        self.multiworld.worlds[self.player1.id].options.progression_balancing.value = 50
+        self.multiworld.worlds[self.player2.id].options.progression_balancing.value = 50
 
         self.player2.prog_items[0].location.progress_type = LocationProgressType.PRIORITY
 

test/general/test_options.py

@@ -21,6 +21,17 @@ class TestOptions(unittest.TestCase):
                 self.assertFalse(hasattr(world_type, "options"),
                                  f"Unexpected assignment to {world_type.__name__}.options!")
 
+    def test_duplicate_options(self) -> None:
+        """Tests that a world doesn't reuse the same option class."""
+        for game_name, world_type in AutoWorldRegister.world_types.items():
+            with self.subTest(game=game_name):
+                seen_options = set()
+                for option in world_type.options_dataclass.type_hints.values():
+                    if not option.visibility:
+                        continue
+                    self.assertFalse(option in seen_options, f"{option} found in assigned options multiple times.")
+                    seen_options.add(option)
+
     def test_item_links_name_groups(self):
         """Tests that item links successfully unfold item_name_groups"""
         item_link_groups = [
@@ -67,4 +78,4 @@ class TestOptions(unittest.TestCase):
             if not world_type.hidden:
                 for option_key, option in world_type.options_dataclass.type_hints.items():
                     with self.subTest(game=gamename, option=option_key):
-                        pickle.dumps(option(option.default))
+                        pickle.dumps(option.from_any(option.default))

test/options/test_generate_templates.py

@@ -0,0 +1,55 @@
+import unittest
+
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import TYPE_CHECKING, Dict, Type
+from Utils import parse_yaml
+
+if TYPE_CHECKING:
+    from worlds.AutoWorld import World
+
+
+class TestGenerateYamlTemplates(unittest.TestCase):
+    old_world_types: Dict[str, Type["World"]]
+
+    def setUp(self) -> None:
+        import worlds.AutoWorld
+
+        self.old_world_types = worlds.AutoWorld.AutoWorldRegister.world_types
+
+    def tearDown(self) -> None:
+        import worlds.AutoWorld
+
+        worlds.AutoWorld.AutoWorldRegister.world_types = self.old_world_types
+
+        if "World: with colon" in worlds.AutoWorld.AutoWorldRegister.world_types:
+            del worlds.AutoWorld.AutoWorldRegister.world_types["World: with colon"]
+
+    def test_name_with_colon(self) -> None:
+        from Options import generate_yaml_templates
+        from worlds.AutoWorld import AutoWorldRegister
+        from worlds.AutoWorld import World
+
+        class WorldWithColon(World):
+            game = "World: with colon"
+            item_name_to_id = {}
+            location_name_to_id = {}
+
+        AutoWorldRegister.world_types = {WorldWithColon.game: WorldWithColon}
+        with TemporaryDirectory(f"archipelago_{__name__}") as temp_dir:
+            generate_yaml_templates(temp_dir)
+            path: Path
+            for path in Path(temp_dir).iterdir():
+                self.assertTrue(path.is_file())
+                self.assertTrue(path.suffix == ".yaml")
+                with path.open(encoding="utf-8") as f:
+                    try:
+                        data = parse_yaml(f)
+                    except:
+                        f.seek(0)
+                        print(f"Error in {path.name}:\n{f.read()}")
+                        raise
+                    self.assertIn("game", data)
+                    self.assertIn(":", data["game"])
+                    self.assertIn(data["game"], data)
+                    self.assertIsInstance(data[data["game"]], dict)

test/programs/data/weights/weights.yaml

@@ -0,0 +1,10 @@
+name: Player{number}
+game: Archipelago # we only need to test options work and this "supports" all the base options
+Archipelago:
+  progression_balancing:
+    0: 50
+    50: 50
+    99: 50
+  accessibility:
+    0: 50
+    2: 50

test/programs/test_generate.py

@@ -92,3 +92,48 @@ class TestGenerateMain(unittest.TestCase):
             user_path.cached_path = user_path_backup
 
         self.assertOutput(self.output_tempdir.name)
+
+
+class TestGenerateWeights(TestGenerateMain):
+    """Tests Generate.py using a weighted file to generate for multiple players."""
+
+    # this test will probably break if something in generation is changed that affects the seed before the weights get processed
+    # can be fixed by changing the expected_results dict
+    generate_dir = TestGenerateMain.generate_dir
+    run_dir = TestGenerateMain.run_dir
+    abs_input_dir = Path(__file__).parent / "data" / "weights"
+    rel_input_dir = abs_input_dir.relative_to(run_dir)  # directly supplied relative paths are relative to cwd
+    yaml_input_dir = abs_input_dir.relative_to(generate_dir)  # yaml paths are relative to user_path
+
+    # don't need to run these tests
+    test_generate_absolute = None
+    test_generate_relative = None
+
+    def test_generate_yaml(self):
+        from settings import get_settings
+        from Utils import user_path, local_path
+        settings = get_settings()
+        settings.generator.player_files_path = settings.generator.PlayerFilesPath(self.yaml_input_dir)
+        settings.generator.players = 5  # arbitrary number, should be enough
+        settings._filename = None
+        user_path_backup = user_path.cached_path
+        user_path.cached_path = local_path()
+        try:
+            sys.argv = [sys.argv[0], "--seed", "1"]
+            namespace, seed = Generate.main()
+        finally:
+            user_path.cached_path = user_path_backup
+
+        # there's likely a better way to do this, but hardcode the results from seed 1 to ensure they're always this
+        expected_results = {
+            "accessibility": [0, 2, 0, 2, 2],
+            "progression_balancing": [0, 50, 99, 0, 50],
+        }
+
+        self.assertEqual(seed, 1)
+        for option_name, results in expected_results.items():
+            for player, result in enumerate(results, 1):
+                self.assertEqual(
+                    result, getattr(namespace, option_name)[player].value,
+                    "Generated results from weights file did not match expected value."
+                )