diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 80aaf70c21..dd88d8d7d7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,6 +40,10 @@ jobs: run: | python -m pip install --upgrade pip python setup.py build_exe --yes + if ( $? -eq $false ) { + Write-Error "setup.py failed!" + exit 1 + } $NAME="$(ls build | Select-String -Pattern 'exe')".Split('.',2)[1] $ZIP_NAME="Archipelago_$NAME.7z" echo "$NAME -> $ZIP_NAME" @@ -49,12 +53,6 @@ jobs: Rename-Item "exe.$NAME" Archipelago 7z a -mx=9 -mhe=on -ms "../dist/$ZIP_NAME" Archipelago Rename-Item Archipelago "exe.$NAME" # inno_setup.iss expects the original name - - name: Store 7z - uses: actions/upload-artifact@v4 - with: - name: ${{ env.ZIP_NAME }} - path: dist/${{ env.ZIP_NAME }} - retention-days: 7 # keep for 7 days, should be enough - name: Build Setup run: | & "${env:ProgramFiles(x86)}\Inno Setup 6\iscc.exe" inno_setup.iss /DNO_SIGNTOOL @@ -65,11 +63,38 @@ jobs: $contents = Get-ChildItem -Path setups/*.exe -Force -Recurse $SETUP_NAME=$contents[0].Name echo "SETUP_NAME=$SETUP_NAME" >> $Env:GITHUB_ENV + - name: Check build loads expected worlds + shell: bash + run: | + cd build/exe* + mv Players/Templates/meta.yaml . + ls -1 Players/Templates | sort > setup-player-templates.txt + rm -R Players/Templates + timeout 30 ./ArchipelagoLauncher "Generate Template Options" || true + ls -1 Players/Templates | sort > generated-player-templates.txt + cmp setup-player-templates.txt generated-player-templates.txt \ + || diff setup-player-templates.txt generated-player-templates.txt + mv meta.yaml Players/Templates/ + - name: Test Generate + shell: bash + run: | + cd build/exe* + cp Players/Templates/Clique.yaml Players/ + timeout 30 ./ArchipelagoGenerate + - name: Store 7z + uses: actions/upload-artifact@v4 + with: + name: ${{ env.ZIP_NAME }} + path: dist/${{ env.ZIP_NAME }} + compression-level: 0 # .7z is incompressible by zip + if-no-files-found: error + retention-days: 7 # keep for 7 days, should be enough - name: Store Setup uses: actions/upload-artifact@v4 with: name: ${{ env.SETUP_NAME }} path: setups/${{ env.SETUP_NAME }} + if-no-files-found: error retention-days: 7 # keep for 7 days, should be enough build-ubuntu2004: @@ -110,7 +135,7 @@ jobs: echo -e "setup.py dist output:\n `ls dist`" cd dist && export APPIMAGE_NAME="`ls *.AppImage`" && cd .. export TAR_NAME="${APPIMAGE_NAME%.AppImage}.tar.gz" - (cd build && DIR_NAME="`ls | grep exe`" && mv "$DIR_NAME" Archipelago && tar -czvf ../dist/$TAR_NAME Archipelago && mv Archipelago "$DIR_NAME") + (cd build && DIR_NAME="`ls | grep exe`" && mv "$DIR_NAME" Archipelago && tar -cv Archipelago | gzip -8 > ../dist/$TAR_NAME && mv Archipelago "$DIR_NAME") echo "APPIMAGE_NAME=$APPIMAGE_NAME" >> $GITHUB_ENV echo "TAR_NAME=$TAR_NAME" >> $GITHUB_ENV # - copy code above to release.yml - @@ -118,15 +143,36 @@ jobs: run: | source venv/bin/activate python setup.py build_exe --yes + - name: Check build loads expected worlds + shell: bash + run: | + cd build/exe* + mv Players/Templates/meta.yaml . + ls -1 Players/Templates | sort > setup-player-templates.txt + rm -R Players/Templates + timeout 30 ./ArchipelagoLauncher "Generate Template Options" || true + ls -1 Players/Templates | sort > generated-player-templates.txt + cmp setup-player-templates.txt generated-player-templates.txt \ + || diff setup-player-templates.txt generated-player-templates.txt + mv meta.yaml Players/Templates/ + - name: Test Generate + shell: bash + run: | + cd build/exe* + cp Players/Templates/Clique.yaml Players/ + timeout 30 ./ArchipelagoGenerate - name: Store AppImage uses: actions/upload-artifact@v4 with: name: ${{ env.APPIMAGE_NAME }} path: dist/${{ env.APPIMAGE_NAME }} + if-no-files-found: error retention-days: 7 - name: Store .tar.gz uses: actions/upload-artifact@v4 with: name: ${{ env.TAR_NAME }} path: dist/${{ env.TAR_NAME }} + compression-level: 0 # .gz is incompressible by zip + if-no-files-found: error retention-days: 7 diff --git a/.github/workflows/ctest.yml b/.github/workflows/ctest.yml new file mode 100644 index 0000000000..9492c83c9e --- /dev/null +++ b/.github/workflows/ctest.yml @@ -0,0 +1,54 @@ +# Run CMake / CTest C++ unit tests + +name: ctest + +on: + push: + paths: + - '**.cc?' + - '**.cpp' + - '**.cxx' + - '**.hh?' + - '**.hpp' + - '**.hxx' + - '**.CMakeLists' + - '.github/workflows/ctest.yml' + pull_request: + paths: + - '**.cc?' + - '**.cpp' + - '**.cxx' + - '**.hh?' + - '**.hpp' + - '**.hxx' + - '**.CMakeLists' + - '.github/workflows/ctest.yml' + +jobs: + ctest: + runs-on: ${{ matrix.os }} + name: Test C++ ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + + steps: + - uses: actions/checkout@v4 + - uses: ilammy/msvc-dev-cmd@v1 + if: startsWith(matrix.os,'windows') + - uses: Bacondish2023/setup-googletest@v1 + with: + build-type: 'Release' + - name: Build tests + run: | + cd test/cpp + mkdir build + cmake -S . -B build/ -DCMAKE_BUILD_TYPE=Release + cmake --build build/ --config Release + ls + - name: Run tests + run: | + cd test/cpp + ctest --test-dir build/ -C Release --output-on-failure diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2d7f1253b7..3f8651d408 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -69,7 +69,7 @@ jobs: echo -e "setup.py dist output:\n `ls dist`" cd dist && export APPIMAGE_NAME="`ls *.AppImage`" && cd .. export TAR_NAME="${APPIMAGE_NAME%.AppImage}.tar.gz" - (cd build && DIR_NAME="`ls | grep exe`" && mv "$DIR_NAME" Archipelago && tar -czvf ../dist/$TAR_NAME Archipelago && mv Archipelago "$DIR_NAME") + (cd build && DIR_NAME="`ls | grep exe`" && mv "$DIR_NAME" Archipelago && tar -cv Archipelago | gzip -8 > ../dist/$TAR_NAME && mv Archipelago "$DIR_NAME") echo "APPIMAGE_NAME=$APPIMAGE_NAME" >> $GITHUB_ENV echo "TAR_NAME=$TAR_NAME" >> $GITHUB_ENV # - code above copied from build.yml - diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index b2530bd06c..3ad29b0077 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -24,7 +24,7 @@ on: - '.github/workflows/unittests.yml' jobs: - build: + unit: runs-on: ${{ matrix.os }} name: Test Python ${{ matrix.python.version }} ${{ matrix.os }} @@ -60,3 +60,32 @@ jobs: - name: Unittests run: | pytest -n auto + + hosting: + runs-on: ${{ matrix.os }} + name: Test hosting with ${{ matrix.python.version }} on ${{ matrix.os }} + + strategy: + matrix: + os: + - ubuntu-latest + python: + - {version: '3.11'} # current + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python.version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python.version }} + - name: Install dependencies + run: | + python -m venv venv + source venv/bin/activate + python -m pip install --upgrade pip + python ModuleUpdate.py --yes --force --append "WebHostLib/requirements.txt" + - name: Test hosting + run: | + source venv/bin/activate + export PYTHONPATH=$(pwd) + python test/hosting/__main__.py diff --git a/.gitignore b/.gitignore index 022abe38fe..5686f43de3 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,7 @@ Output Logs/ /installdelete.iss /data/user.kv /datapackage +/custom_worlds # Byte-compiled / optimized / DLL files __pycache__/ @@ -177,6 +178,7 @@ dmypy.json cython_debug/ # Cython intermediates +_speedups.c _speedups.cpp _speedups.html diff --git a/AdventureClient.py b/AdventureClient.py index 7bfbd5ef6b..206c55df9a 100644 --- a/AdventureClient.py +++ b/AdventureClient.py @@ -80,7 +80,7 @@ class AdventureContext(CommonContext): self.local_item_locations = {} self.dragon_speed_info = {} - options = Utils.get_options() + options = Utils.get_settings() self.display_msgs = options["adventure_options"]["display_msgs"] async def server_auth(self, password_requested: bool = False): @@ -102,7 +102,7 @@ class AdventureContext(CommonContext): def on_package(self, cmd: str, args: dict): if cmd == 'Connected': self.locations_array = None - if Utils.get_options()["adventure_options"].get("death_link", False): + if Utils.get_settings()["adventure_options"].get("death_link", False): self.set_deathlink = True async_start(self.get_freeincarnates_used()) elif cmd == "RoomInfo": @@ -415,8 +415,8 @@ async def atari_sync_task(ctx: AdventureContext): async def run_game(romfile): - auto_start = Utils.get_options()["adventure_options"].get("rom_start", True) - rom_args = Utils.get_options()["adventure_options"].get("rom_args") + auto_start = Utils.get_settings()["adventure_options"].get("rom_start", True) + rom_args = Utils.get_settings()["adventure_options"].get("rom_args") if auto_start is True: import webbrowser webbrowser.open(romfile) diff --git a/CommonClient.py b/CommonClient.py index 8af822cba5..8f1e64c059 100644 --- a/CommonClient.py +++ b/CommonClient.py @@ -493,6 +493,11 @@ class CommonContext: """Gets called before sending a Say to the server from the user. Returned text is sent, or sending is aborted if None is returned.""" return text + + def on_ui_command(self, text: str) -> None: + """Gets called by kivy when the user executes a command starting with `/` or `!`. + The command processor is still called; this is just intended for command echoing.""" + self.ui.print_json([{"text": text, "type": "color", "color": "orange"}]) def update_permissions(self, permissions: typing.Dict[str, int]): for permission_name, permission_flag in permissions.items(): diff --git a/Fill.py b/Fill.py index d8147b2eac..4967ff0736 100644 --- a/Fill.py +++ b/Fill.py @@ -483,15 +483,15 @@ def distribute_items_restrictive(multiworld: MultiWorld, if panic_method == "swap": fill_restrictive(multiworld, multiworld.state, defaultlocations, progitempool, swap=True, - on_place=mark_for_locking, name="Progression", single_player_placement=multiworld.players == 1) + name="Progression", single_player_placement=multiworld.players == 1) elif panic_method == "raise": fill_restrictive(multiworld, multiworld.state, defaultlocations, progitempool, swap=False, - on_place=mark_for_locking, name="Progression", single_player_placement=multiworld.players == 1) + name="Progression", single_player_placement=multiworld.players == 1) elif panic_method == "start_inventory": fill_restrictive(multiworld, multiworld.state, defaultlocations, progitempool, swap=False, allow_partial=True, - on_place=mark_for_locking, name="Progression", single_player_placement=multiworld.players == 1) + name="Progression", single_player_placement=multiworld.players == 1) if progitempool: for item in progitempool: logging.debug(f"Moved {item} to start_inventory to prevent fill failure.") diff --git a/Generate.py b/Generate.py index 67988bf8b3..1fbb9e76a4 100644 --- a/Generate.py +++ b/Generate.py @@ -1,10 +1,12 @@ from __future__ import annotations import argparse +import copy import logging import os import random import string +import sys import urllib.parse import urllib.request from collections import Counter @@ -15,21 +17,16 @@ import ModuleUpdate ModuleUpdate.update() -import copy import Utils import Options from BaseClasses import seeddigits, get_seed, PlandoOptions -from Main import main as ERmain -from settings import get_settings from Utils import parse_yamls, version_tuple, __version__, tuplize_version -from worlds.alttp.EntranceRandomizer import parse_arguments -from worlds.AutoWorld import AutoWorldRegister -from worlds import failed_world_loads def mystery_argparse(): - options = get_settings() - defaults = options.generator + from settings import get_settings + settings = get_settings() + defaults = settings.generator parser = argparse.ArgumentParser(description="CMD Generation Interface, defaults come from host.yaml.") parser.add_argument('--weights_file_path', default=defaults.weights_file_path, @@ -41,7 +38,7 @@ def mystery_argparse(): parser.add_argument('--seed', help='Define seed number to generate.', type=int) parser.add_argument('--multi', default=defaults.players, type=lambda value: max(int(value), 1)) parser.add_argument('--spoiler', type=int, default=defaults.spoiler) - parser.add_argument('--outputpath', default=options.general_options.output_path, + parser.add_argument('--outputpath', default=settings.general_options.output_path, help="Path to output folder. Absolute or relative to cwd.") # absolute or relative to cwd parser.add_argument('--race', action='store_true', default=defaults.race) parser.add_argument('--meta_file_path', default=defaults.meta_file_path) @@ -61,20 +58,23 @@ def mystery_argparse(): if not os.path.isabs(args.meta_file_path): args.meta_file_path = os.path.join(args.player_files_path, args.meta_file_path) args.plando: PlandoOptions = PlandoOptions.from_option_string(args.plando) - return args, options + return args def get_seed_name(random_source) -> str: return f"{random_source.randint(0, pow(10, seeddigits) - 1)}".zfill(seeddigits) -def main(args=None, callback=ERmain): +def main(args=None): + # __name__ == "__main__" check so unittests that already imported worlds don't trip this. + if __name__ == "__main__" and "worlds" in sys.modules: + raise Exception("Worlds system should not be loaded before logging init.") + if not args: - args, options = mystery_argparse() - else: - options = get_settings() + args = mystery_argparse() seed = get_seed(args.seed) + Utils.init_logging(f"Generate_{seed}", loglevel=args.log_level) random.seed(seed) seed_name = get_seed_name(random) @@ -143,6 +143,9 @@ def main(args=None, callback=ERmain): raise Exception(f"No weights found. " f"Provide a general weights file ({args.weights_file_path}) or individual player files. " f"A mix is also permitted.") + + from worlds.AutoWorld import AutoWorldRegister + from worlds.alttp.EntranceRandomizer import parse_arguments erargs = parse_arguments(['--multi', str(args.multi)]) erargs.seed = seed erargs.plando_options = args.plando @@ -234,7 +237,8 @@ def main(args=None, callback=ERmain): with open(os.path.join(args.outputpath if args.outputpath else ".", f"generate_{seed_name}.yaml"), "wt") as f: yaml.dump(important, f) - return callback(erargs, seed) + from Main import main as ERmain + return ERmain(erargs, seed) def read_weights_yamls(path) -> Tuple[Any, ...]: @@ -359,6 +363,8 @@ def update_weights(weights: dict, new_weights: dict, update_type: str, name: str def roll_meta_option(option_key, game: str, category_dict: Dict) -> Any: + from worlds import AutoWorldRegister + if not game: return get_choice(option_key, category_dict) if game in AutoWorldRegister.world_types: @@ -436,10 +442,13 @@ def handle_option(ret: argparse.Namespace, game_weights: dict, option_key: str, except Exception as e: raise Options.OptionError(f"Error generating option {option_key} in {ret.game}") from e else: + from worlds import AutoWorldRegister player_option.verify(AutoWorldRegister.world_types[ret.game], ret.name, plando_options) def roll_settings(weights: dict, plando_options: PlandoOptions = PlandoOptions.bosses): + from worlds import AutoWorldRegister + if "linked_options" in weights: weights = roll_linked_options(weights) @@ -466,6 +475,7 @@ def roll_settings(weights: dict, plando_options: PlandoOptions = PlandoOptions.b ret.game = get_choice("game", weights) if ret.game not in AutoWorldRegister.world_types: + from worlds import failed_world_loads picks = Utils.get_fuzzy_results(ret.game, list(AutoWorldRegister.world_types) + failed_world_loads, limit=1)[0] if picks[0] in failed_world_loads: raise Exception(f"No functional world found to handle game {ret.game}. " diff --git a/Launcher.py b/Launcher.py index 503ad5f8bd..e4b65be93a 100644 --- a/Launcher.py +++ b/Launcher.py @@ -19,7 +19,7 @@ import sys import webbrowser from os.path import isfile from shutil import which -from typing import Sequence, Union, Optional +from typing import Callable, Sequence, Union, Optional import Utils import settings @@ -160,8 +160,12 @@ def launch(exe, in_terminal=False): subprocess.Popen(exe) +refresh_components: Optional[Callable[[], None]] = None + + def run_gui(): from kvui import App, ContainerLayout, GridLayout, Button, Label, ScrollBox, Widget + from kivy.core.window import Window from kivy.uix.image import AsyncImage from kivy.uix.relativelayout import RelativeLayout @@ -169,11 +173,8 @@ def run_gui(): base_title: str = "Archipelago Launcher" container: ContainerLayout grid: GridLayout - - _tools = {c.display_name: c for c in components if c.type == Type.TOOL} - _clients = {c.display_name: c for c in components if c.type == Type.CLIENT} - _adjusters = {c.display_name: c for c in components if c.type == Type.ADJUSTER} - _miscs = {c.display_name: c for c in components if c.type == Type.MISC} + _tool_layout: Optional[ScrollBox] = None + _client_layout: Optional[ScrollBox] = None def __init__(self, ctx=None): self.title = self.base_title @@ -181,18 +182,7 @@ def run_gui(): self.icon = r"data/icon.png" super().__init__() - def build(self): - self.container = ContainerLayout() - self.grid = GridLayout(cols=2) - self.container.add_widget(self.grid) - self.grid.add_widget(Label(text="General", size_hint_y=None, height=40)) - self.grid.add_widget(Label(text="Clients", size_hint_y=None, height=40)) - tool_layout = ScrollBox() - tool_layout.layout.orientation = "vertical" - self.grid.add_widget(tool_layout) - client_layout = ScrollBox() - client_layout.layout.orientation = "vertical" - self.grid.add_widget(client_layout) + def _refresh_components(self) -> None: def build_button(component: Component) -> Widget: """ @@ -217,14 +207,49 @@ def run_gui(): return box_layout return button + # clear before repopulating + assert self._tool_layout and self._client_layout, "must call `build` first" + tool_children = reversed(self._tool_layout.layout.children) + for child in tool_children: + self._tool_layout.layout.remove_widget(child) + client_children = reversed(self._client_layout.layout.children) + for child in client_children: + self._client_layout.layout.remove_widget(child) + + _tools = {c.display_name: c for c in components if c.type == Type.TOOL} + _clients = {c.display_name: c for c in components if c.type == Type.CLIENT} + _adjusters = {c.display_name: c for c in components if c.type == Type.ADJUSTER} + _miscs = {c.display_name: c for c in components if c.type == Type.MISC} + for (tool, client) in itertools.zip_longest(itertools.chain( - self._tools.items(), self._miscs.items(), self._adjusters.items()), self._clients.items()): + _tools.items(), _miscs.items(), _adjusters.items() + ), _clients.items()): # column 1 if tool: - tool_layout.layout.add_widget(build_button(tool[1])) + self._tool_layout.layout.add_widget(build_button(tool[1])) # column 2 if client: - client_layout.layout.add_widget(build_button(client[1])) + self._client_layout.layout.add_widget(build_button(client[1])) + + def build(self): + self.container = ContainerLayout() + self.grid = GridLayout(cols=2) + self.container.add_widget(self.grid) + self.grid.add_widget(Label(text="General", size_hint_y=None, height=40)) + self.grid.add_widget(Label(text="Clients", size_hint_y=None, height=40)) + self._tool_layout = ScrollBox() + self._tool_layout.layout.orientation = "vertical" + self.grid.add_widget(self._tool_layout) + self._client_layout = ScrollBox() + self._client_layout.layout.orientation = "vertical" + self.grid.add_widget(self._client_layout) + + self._refresh_components() + + global refresh_components + refresh_components = self._refresh_components + + Window.bind(on_drop_file=self._on_drop_file) return self.container @@ -235,6 +260,14 @@ def run_gui(): else: launch(get_exe(button.component), button.component.cli) + def _on_drop_file(self, window: Window, filename: bytes, x: int, y: int) -> None: + """ When a patch file is dropped into the window, run the associated component. """ + file, component = identify(filename.decode()) + if file and component: + run_component(component, file) + else: + logging.warning(f"unable to identify component for {filename}") + def _stop(self, *largs): # ran into what appears to be https://groups.google.com/g/kivy-users/c/saWDLoYCSZ4 with PyCharm. # Closing the window explicitly cleans it up. @@ -243,10 +276,17 @@ def run_gui(): Launcher().run() + # avoiding Launcher reference leak + # and don't try to do something with widgets after window closed + global refresh_components + refresh_components = None + def run_component(component: Component, *args): if component.func: component.func(*args) + if refresh_components: + refresh_components() elif component.script_name: subprocess.run([*get_exe(component.script_name), *args]) else: diff --git a/MultiServer.py b/MultiServer.py index 22375da2b3..dc5e3d21ac 100644 --- a/MultiServer.py +++ b/MultiServer.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse import asyncio import collections +import contextlib import copy import datetime import functools @@ -176,7 +177,7 @@ class Context: location_name_groups: typing.Dict[str, typing.Dict[str, typing.Set[str]]] all_item_and_group_names: typing.Dict[str, typing.Set[str]] all_location_and_group_names: typing.Dict[str, typing.Set[str]] - non_hintable_names: typing.Dict[str, typing.Set[str]] + non_hintable_names: typing.Dict[str, typing.AbstractSet[str]] spheres: typing.List[typing.Dict[int, typing.Set[int]]] """ each sphere is { player: { location_id, ... } } """ logger: logging.Logger @@ -231,7 +232,7 @@ class Context: self.embedded_blacklist = {"host", "port"} self.client_ids: typing.Dict[typing.Tuple[int, int], datetime.datetime] = {} self.auto_save_interval = 60 # in seconds - self.auto_saver_thread = None + self.auto_saver_thread: typing.Optional[threading.Thread] = None self.save_dirty = False self.tags = ['AP'] self.games: typing.Dict[int, str] = {} @@ -268,6 +269,11 @@ class Context: for world_name, world in worlds.AutoWorldRegister.world_types.items(): self.non_hintable_names[world_name] = world.hint_blacklist + for game_package in self.gamespackage.values(): + # remove groups from data sent to clients + del game_package["item_name_groups"] + del game_package["location_name_groups"] + def _init_game_data(self): for game_name, game_package in self.gamespackage.items(): if "checksum" in game_package: @@ -1926,8 +1932,6 @@ class ServerCommandProcessor(CommonCommandProcessor): def _cmd_exit(self) -> bool: """Shutdown the server""" self.ctx.server.ws_server.close() - if self.ctx.shutdown_task: - self.ctx.shutdown_task.cancel() self.ctx.exit_event.set() return True @@ -2285,7 +2289,8 @@ def parse_args() -> argparse.Namespace: async def auto_shutdown(ctx, to_cancel=None): - await asyncio.sleep(ctx.auto_shutdown) + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(ctx.exit_event.wait(), ctx.auto_shutdown) def inactivity_shutdown(): ctx.server.ws_server.close() @@ -2305,7 +2310,8 @@ async def auto_shutdown(ctx, to_cancel=None): if seconds < 0: inactivity_shutdown() else: - await asyncio.sleep(seconds) + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(ctx.exit_event.wait(), seconds) def load_server_cert(path: str, cert_key: typing.Optional[str]) -> "ssl.SSLContext": diff --git a/NetUtils.py b/NetUtils.py index 076fdc3ba4..f8d698c74f 100644 --- a/NetUtils.py +++ b/NetUtils.py @@ -198,7 +198,8 @@ class JSONtoTextParser(metaclass=HandlerMeta): "slateblue": "6D8BE8", "plum": "AF99EF", "salmon": "FA8072", - "white": "FFFFFF" + "white": "FFFFFF", + "orange": "FF7700", } def __init__(self, ctx): diff --git a/Utils.py b/Utils.py index a7fd7f4f33..f89330cf7c 100644 --- a/Utils.py +++ b/Utils.py @@ -553,6 +553,7 @@ def init_logging(name: str, loglevel: typing.Union[str, int] = logging.INFO, wri f"Archipelago ({__version__}) logging initialized" f" on {platform.platform()}" f" running Python {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" + f"{' (frozen)' if is_frozen() else ''}" ) diff --git a/WebHost.py b/WebHost.py index 9b5edd322f..08ef3c4307 100644 --- a/WebHost.py +++ b/WebHost.py @@ -12,6 +12,9 @@ ModuleUpdate.update() import Utils import settings +if typing.TYPE_CHECKING: + from flask import Flask + Utils.local_path.cached_path = os.path.dirname(__file__) or "." # py3.8 is not abs. remove "." when dropping 3.8 settings.no_gui = True configpath = os.path.abspath("config.yaml") @@ -19,7 +22,7 @@ if not os.path.exists(configpath): # fall back to config.yaml in home configpath = os.path.abspath(Utils.user_path('config.yaml')) -def get_app(): +def get_app() -> "Flask": from WebHostLib import register, cache, app as raw_app from WebHostLib.models import db @@ -55,6 +58,7 @@ def create_ordered_tutorials_file() -> typing.List[typing.Dict[str, typing.Any]] worlds[game] = world base_target_path = Utils.local_path("WebHostLib", "static", "generated", "docs") + shutil.rmtree(base_target_path, ignore_errors=True) for game, world in worlds.items(): # copy files from world's docs folder to the generated folder target_path = os.path.join(base_target_path, game) diff --git a/WebHostLib/api/__init__.py b/WebHostLib/api/__init__.py index 22d1f19f6b..4003243a28 100644 --- a/WebHostLib/api/__init__.py +++ b/WebHostLib/api/__init__.py @@ -5,7 +5,6 @@ from uuid import UUID from flask import Blueprint, abort, url_for import worlds.Files -from .. import cache from ..models import Room, Seed api_endpoints = Blueprint('api', __name__, url_prefix="/api") @@ -49,21 +48,4 @@ def room_info(room: UUID): } -@api_endpoints.route('/datapackage') -@cache.cached() -def get_datapackage(): - from worlds import network_data_package - return network_data_package - - -@api_endpoints.route('/datapackage_checksum') -@cache.cached() -def get_datapackage_checksums(): - from worlds import network_data_package - version_package = { - game: game_data["checksum"] for game, game_data in network_data_package["games"].items() - } - return version_package - - -from . import generate, user # trigger registration +from . import generate, user, datapackage # trigger registration diff --git a/WebHostLib/api/datapackage.py b/WebHostLib/api/datapackage.py new file mode 100644 index 0000000000..3fb472d95d --- /dev/null +++ b/WebHostLib/api/datapackage.py @@ -0,0 +1,32 @@ +from flask import abort + +from Utils import restricted_loads +from WebHostLib import cache +from WebHostLib.models import GameDataPackage +from . import api_endpoints + + +@api_endpoints.route('/datapackage') +@cache.cached() +def get_datapackage(): + from worlds import network_data_package + return network_data_package + + +@api_endpoints.route('/datapackage/') +@cache.memoize(timeout=3600) +def get_datapackage_by_checksum(checksum: str): + package = GameDataPackage.get(checksum=checksum) + if package: + return restricted_loads(package.data) + return abort(404) + + +@api_endpoints.route('/datapackage_checksum') +@cache.cached() +def get_datapackage_checksums(): + from worlds import network_data_package + version_package = { + game: game_data["checksum"] for game, game_data in network_data_package["games"].items() + } + return version_package diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 3a86cb551d..9f70165b61 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -168,17 +168,28 @@ def get_random_port(): def get_static_server_data() -> dict: import worlds data = { - "non_hintable_names": {}, - "gamespackage": worlds.network_data_package["games"], - "item_name_groups": {world_name: world.item_name_groups for world_name, world in - worlds.AutoWorldRegister.world_types.items()}, - "location_name_groups": {world_name: world.location_name_groups for world_name, world in - worlds.AutoWorldRegister.world_types.items()}, + "non_hintable_names": { + world_name: world.hint_blacklist + for world_name, world in worlds.AutoWorldRegister.world_types.items() + }, + "gamespackage": { + world_name: { + key: value + for key, value in game_package.items() + if key not in ("item_name_groups", "location_name_groups") + } + for world_name, game_package in worlds.network_data_package["games"].items() + }, + "item_name_groups": { + world_name: world.item_name_groups + for world_name, world in worlds.AutoWorldRegister.world_types.items() + }, + "location_name_groups": { + world_name: world.location_name_groups + for world_name, world in worlds.AutoWorldRegister.world_types.items() + }, } - for world_name, world in worlds.AutoWorldRegister.world_types.items(): - data["non_hintable_names"][world_name] = world.hint_blacklist - return data @@ -266,12 +277,15 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, ctx.logger.exception("Could not determine port. Likely hosting failure.") with db_session: ctx.auto_shutdown = Room.get(id=room_id).timeout + if ctx.saving: + setattr(asyncio.current_task(), "save", lambda: ctx._save(True)) ctx.shutdown_task = asyncio.create_task(auto_shutdown(ctx, [])) await ctx.shutdown_task except (KeyboardInterrupt, SystemExit): if ctx.saving: ctx._save() + setattr(asyncio.current_task(), "save", None) except Exception as e: with db_session: room = Room.get(id=room_id) @@ -281,8 +295,12 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, else: if ctx.saving: ctx._save() + setattr(asyncio.current_task(), "save", None) finally: try: + ctx.save_dirty = False # make sure the saving thread does not write to DB after final wakeup + ctx.exit_event.set() # make sure the saving thread stops at some point + # NOTE: async saving should probably be an async task and could be merged with shutdown_task with (db_session): # ensure the Room does not spin up again on its own, minute of safety buffer room = Room.get(id=room_id) @@ -294,13 +312,32 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, rooms_shutting_down.put(room_id) class Starter(threading.Thread): + _tasks: typing.List[asyncio.Future] + + def __init__(self): + super().__init__() + self._tasks = [] + + def _done(self, task: asyncio.Future): + self._tasks.remove(task) + task.result() + def run(self): while 1: next_room = rooms_to_run.get(block=True, timeout=None) - asyncio.run_coroutine_threadsafe(start_room(next_room), loop) + task = asyncio.run_coroutine_threadsafe(start_room(next_room), loop) + self._tasks.append(task) + task.add_done_callback(self._done) logging.info(f"Starting room {next_room} on {name}.") starter = Starter() starter.daemon = True starter.start() - loop.run_forever() + try: + loop.run_forever() + finally: + # save all tasks that want to be saved during shutdown + for task in asyncio.all_tasks(loop): + save: typing.Optional[typing.Callable[[], typing.Any]] = getattr(task, "save", None) + if save: + save() diff --git a/WebHostLib/requirements.txt b/WebHostLib/requirements.txt index 62707d78cf..3452c9d416 100644 --- a/WebHostLib/requirements.txt +++ b/WebHostLib/requirements.txt @@ -1,9 +1,10 @@ -flask>=3.0.0 +flask>=3.0.3 +werkzeug>=3.0.3 pony>=0.7.17 -waitress>=2.1.2 -Flask-Caching>=2.1.0 -Flask-Compress>=1.14 -Flask-Limiter>=3.5.0 +waitress>=3.0.0 +Flask-Caching>=2.3.0 +Flask-Compress>=1.15 +Flask-Limiter>=3.7.0 bokeh>=3.1.1; python_version <= '3.8' -bokeh>=3.3.2; python_version >= '3.9' -markupsafe>=2.1.3 +bokeh>=3.4.1; python_version >= '3.9' +markupsafe>=2.1.5 diff --git a/WebHostLib/static/styles/playerOptions/playerOptions.css b/WebHostLib/static/styles/playerOptions/playerOptions.css index 6165e3a0f6..56c9263d33 100644 --- a/WebHostLib/static/styles/playerOptions/playerOptions.css +++ b/WebHostLib/static/styles/playerOptions/playerOptions.css @@ -15,7 +15,7 @@ html { border-radius: 8px; padding: 1rem; color: #eeffeb; - word-break: break-all; + word-break: break-word; } #player-options #player-options-header h1 { margin-bottom: 0; diff --git a/WebHostLib/static/styles/playerOptions/playerOptions.scss b/WebHostLib/static/styles/playerOptions/playerOptions.scss index 525b8ef154..06bde759d2 100644 --- a/WebHostLib/static/styles/playerOptions/playerOptions.scss +++ b/WebHostLib/static/styles/playerOptions/playerOptions.scss @@ -16,7 +16,7 @@ html{ border-radius: 8px; padding: 1rem; color: #eeffeb; - word-break: break-all; + word-break: break-word; #player-options-header{ h1{ diff --git a/WebHostLib/templates/weightedOptions/macros.html b/WebHostLib/templates/weightedOptions/macros.html index 5b8944a438..55a56e3285 100644 --- a/WebHostLib/templates/weightedOptions/macros.html +++ b/WebHostLib/templates/weightedOptions/macros.html @@ -1,9 +1,9 @@ {% macro Toggle(option_name, option) %} - {{ RangeRow(option_name, option, "No", "false") }} - {{ RangeRow(option_name, option, "Yes", "true") }} - {{ RandomRows(option_name, option) }} + {{ RangeRow(option_name, option, "No", "false", False, "true" if option.default else "false") }} + {{ RangeRow(option_name, option, "Yes", "true", False, "true" if option.default else "false") }} + {{ RandomRow(option_name, option) }}
{% endmacro %} @@ -18,10 +18,14 @@ {% for id, name in option.name_lookup.items() %} {% if name != 'random' %} - {{ RangeRow(option_name, option, option.get_option_name(id), name) }} + {% if option.default != 'random' %} + {{ RangeRow(option_name, option, option.get_option_name(id), name, False, name if option.get_option_name(option.default)|lower == name else None) }} + {% else %} + {{ RangeRow(option_name, option, option.get_option_name(id), name) }} + {% endif %} {% endif %} {% endfor %} - {{ RandomRows(option_name, option) }} + {{ RandomRow(option_name, option) }} {% endmacro %} @@ -34,7 +38,7 @@ Normal range: {{ option.range_start }} - {{ option.range_end }} {% if option.special_range_names %}

- The following values has special meaning, and may fall outside the normal range. + The following values have special meanings, and may fall outside the normal range.