diff --git a/AHITClient.py b/AHITClient.py new file mode 100644 index 0000000000..6ed7d7b49d --- /dev/null +++ b/AHITClient.py @@ -0,0 +1,8 @@ +from worlds.ahit.Client import launch +import Utils +import ModuleUpdate +ModuleUpdate.update() + +if __name__ == "__main__": + Utils.init_logging("AHITClient", exception_logger="Client") + launch() diff --git a/BaseClasses.py b/BaseClasses.py index 94eb10de43..ada18f1e1d 100644 --- a/BaseClasses.py +++ b/BaseClasses.py @@ -1453,14 +1453,6 @@ class Tutorial(NamedTuple): authors: List[str] -class OptionGroup(NamedTuple): - """Define a grouping of options""" - name: str - """Name of the group to categorize this option in for display on the WebHost and in generated YAMLS.""" - options: List[Type[Options.Option]] - """Options to be in the defined group. """ - - class PlandoOptions(IntFlag): none = 0b0000 items = 0b0001 diff --git a/MultiServer.py b/MultiServer.py index 194f0a67fd..e95e44dd7d 100644 --- a/MultiServer.py +++ b/MultiServer.py @@ -508,7 +508,7 @@ class Context: self.logger.exception(e) self._start_async_saving() - def _start_async_saving(self): + def _start_async_saving(self, atexit_save: bool = True): if not self.auto_saver_thread: def save_regularly(): # time.time() is platform dependent, so using the expensive datetime method instead @@ -532,8 +532,9 @@ class Context: self.auto_saver_thread = threading.Thread(target=save_regularly, daemon=True) self.auto_saver_thread.start() - import atexit - atexit.register(self._save, True) # make sure we save on exit too + if atexit_save: + import atexit + atexit.register(self._save, True) # make sure we save on exit too def get_save(self) -> dict: self.recheck_hints() diff --git a/Options.py b/Options.py index 6b4db10ac4..39fd567656 100644 --- a/Options.py +++ b/Options.py @@ -746,6 +746,7 @@ class NamedRange(Range): class FreezeValidKeys(AssembleOptions): def __new__(mcs, name, bases, attrs): + assert not "_valid_keys" in attrs, "'_valid_keys' gets set by FreezeValidKeys, define 'valid_keys' instead." if "valid_keys" in attrs: attrs["_valid_keys"] = frozenset(attrs["valid_keys"]) return super(FreezeValidKeys, mcs).__new__(mcs, name, bases, attrs) @@ -1123,6 +1124,14 @@ class DeathLinkMixin: death_link: DeathLink +class OptionGroup(typing.NamedTuple): + """Define a grouping of options.""" + name: str + """Name of the group to categorize these options in for display on the WebHost and in generated YAMLS.""" + options: typing.List[typing.Type[Option[typing.Any]]] + """Options to be in the defined group.""" + + def generate_yaml_templates(target_folder: typing.Union[str, "pathlib.Path"], generate_hidden: bool = True): import os diff --git a/README.md b/README.md index c009d54fbe..4633c99c66 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,9 @@ Currently, the following games are supported: * Yoshi's Island * Mario & Luigi: Superstar Saga * Bomb Rush Cyberfunk +* Aquaria * Yu-Gi-Oh! Ultimate Masters: World Championship Tournament 2006 +* A Hat in Time For setup and instructions check out our [tutorials page](https://archipelago.gg/tutorial/). Downloads can be found at [Releases](https://github.com/ArchipelagoMW/Archipelago/releases), including compiled diff --git a/WebHost.py b/WebHost.py index 8ccf6b68c2..9b5edd322f 100644 --- a/WebHost.py +++ b/WebHost.py @@ -117,7 +117,7 @@ if __name__ == "__main__": logging.basicConfig(format='[%(asctime)s] %(message)s', level=logging.INFO) from WebHostLib.lttpsprites import update_sprites_lttp - from WebHostLib.autolauncher import autohost, autogen + from WebHostLib.autolauncher import autohost, autogen, stop from WebHostLib.options import create as create_options_files try: @@ -138,3 +138,11 @@ if __name__ == "__main__": else: from waitress import serve serve(app, port=app.config["PORT"], threads=app.config["WAITRESS_THREADS"]) + else: + from time import sleep + try: + while True: + sleep(1) # wait for process to be killed + except (SystemExit, KeyboardInterrupt): + pass + stop() # stop worker threads diff --git a/WebHostLib/autolauncher.py b/WebHostLib/autolauncher.py index 78fff6c509..08a1309ebc 100644 --- a/WebHostLib/autolauncher.py +++ b/WebHostLib/autolauncher.py @@ -3,16 +3,26 @@ from __future__ import annotations import json import logging import multiprocessing -import time import typing -from uuid import UUID from datetime import timedelta, datetime +from threading import Event, Thread +from uuid import UUID from pony.orm import db_session, select, commit from Utils import restricted_loads from .locker import Locker, AlreadyRunningException +_stop_event = Event() + + +def stop(): + """Stops previously launched threads""" + global _stop_event + stop_event = _stop_event + _stop_event = Event() # new event for new threads + stop_event.set() + def handle_generation_success(seed_id): logging.info(f"Generation finished for seed {seed_id}") @@ -63,6 +73,7 @@ def cleanup(): def autohost(config: dict): def keep_running(): + stop_event = _stop_event try: with Locker("autohost"): cleanup() @@ -72,26 +83,25 @@ def autohost(config: dict): hosters.append(hoster) hoster.start() - while 1: - time.sleep(0.1) + while not stop_event.wait(0.1): with db_session: rooms = select( room for room in Room if room.last_activity >= datetime.utcnow() - timedelta(days=3)) for room in rooms: # we have to filter twice, as the per-room timeout can't currently be PonyORM transpiled. - if room.last_activity >= datetime.utcnow() - timedelta(seconds=room.timeout): + if room.last_activity >= datetime.utcnow() - timedelta(seconds=room.timeout + 5): hosters[room.id.int % len(hosters)].start_room(room.id) except AlreadyRunningException: logging.info("Autohost reports as already running, not starting another.") - import threading - threading.Thread(target=keep_running, name="AP_Autohost").start() + Thread(target=keep_running, name="AP_Autohost").start() def autogen(config: dict): def keep_running(): + stop_event = _stop_event try: with Locker("autogen"): @@ -112,8 +122,7 @@ def autogen(config: dict): commit() select(generation for generation in Generation if generation.state == STATE_ERROR).delete() - while 1: - time.sleep(0.1) + while not stop_event.wait(0.1): with db_session: # for update locks the database row(s) during transaction, preventing writes from elsewhere to_start = select( @@ -124,8 +133,7 @@ def autogen(config: dict): except AlreadyRunningException: logging.info("Autogen reports as already running, not starting another.") - import threading - threading.Thread(target=keep_running, name="AP_Autogen").start() + Thread(target=keep_running, name="AP_Autogen").start() multiworlds: typing.Dict[type(Room.id), MultiworldInstance] = {} diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 04b4b6a0a0..16769b7a76 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -74,6 +74,7 @@ class WebHostContext(Context): def _load_game_data(self): for key, value in self.static_server_data.items(): + # NOTE: attributes are mutable and shared, so they will have to be copied before being modified setattr(self, key, value) self.non_hintable_names = collections.defaultdict(frozenset, self.non_hintable_names) @@ -101,18 +102,37 @@ class WebHostContext(Context): multidata = self.decompress(room.seed.multidata) game_data_packages = {} + + static_gamespackage = self.gamespackage # this is shared across all rooms + static_item_name_groups = self.item_name_groups + static_location_name_groups = self.location_name_groups + self.gamespackage = {"Archipelago": static_gamespackage["Archipelago"]} # this may be modified by _load + self.item_name_groups = {} + self.location_name_groups = {} + for game in list(multidata.get("datapackage", {})): game_data = multidata["datapackage"][game] if "checksum" in game_data: - if self.gamespackage.get(game, {}).get("checksum") == game_data["checksum"]: - # non-custom. remove from multidata + if static_gamespackage.get(game, {}).get("checksum") == game_data["checksum"]: + # non-custom. remove from multidata and use static data # games package could be dropped from static data once all rooms embed data package del multidata["datapackage"][game] else: row = GameDataPackage.get(checksum=game_data["checksum"]) if row: # None if rolled on >= 0.3.9 but uploaded to <= 0.3.8. multidata should be complete game_data_packages[game] = Utils.restricted_loads(row.data) + continue + else: + self.logger.warning(f"Did not find game_data_package for {game}: {game_data['checksum']}") + self.gamespackage[game] = static_gamespackage.get(game, {}) + self.item_name_groups[game] = static_item_name_groups.get(game, {}) + self.location_name_groups[game] = static_location_name_groups.get(game, {}) + if not game_data_packages: + # all static -> use the static dicts directly + self.gamespackage = static_gamespackage + self.item_name_groups = static_item_name_groups + self.location_name_groups = static_location_name_groups return self._load(multidata, game_data_packages, True) @db_session @@ -122,7 +142,7 @@ class WebHostContext(Context): savegame_data = Room.get(id=self.room_id).multisave if savegame_data: self.set_save(restricted_loads(Room.get(id=self.room_id).multisave)) - self._start_async_saving() + self._start_async_saving(atexit_save=False) threading.Thread(target=self.listen_to_db_commands, daemon=True).start() @db_session @@ -212,59 +232,66 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, loop = asyncio.get_event_loop() async def start_room(room_id): - try: - logger = set_up_logging(room_id) - ctx = WebHostContext(static_server_data, logger) - ctx.load(room_id) - ctx.init_save() + with Locker(f"RoomLocker {room_id}"): try: - ctx.server = websockets.serve(functools.partial(server, ctx=ctx), ctx.host, ctx.port, ssl=ssl_context) + logger = set_up_logging(room_id) + ctx = WebHostContext(static_server_data, logger) + ctx.load(room_id) + ctx.init_save() + try: + ctx.server = websockets.serve( + functools.partial(server, ctx=ctx), ctx.host, ctx.port, ssl=ssl_context) - await ctx.server - except OSError: # likely port in use - ctx.server = websockets.serve(functools.partial(server, ctx=ctx), ctx.host, 0, ssl=ssl_context) + await ctx.server + except OSError: # likely port in use + ctx.server = websockets.serve( + functools.partial(server, ctx=ctx), ctx.host, 0, ssl=ssl_context) - await ctx.server - port = 0 - for wssocket in ctx.server.ws_server.sockets: - socketname = wssocket.getsockname() - if wssocket.family == socket.AF_INET6: - # Prefer IPv4, as most users seem to not have working ipv6 support - if not port: + await ctx.server + port = 0 + for wssocket in ctx.server.ws_server.sockets: + socketname = wssocket.getsockname() + if wssocket.family == socket.AF_INET6: + # Prefer IPv4, as most users seem to not have working ipv6 support + if not port: + port = socketname[1] + elif wssocket.family == socket.AF_INET: port = socketname[1] - elif wssocket.family == socket.AF_INET: - port = socketname[1] - if port: - ctx.logger.info(f'Hosting game at {host}:{port}') + if port: + ctx.logger.info(f'Hosting game at {host}:{port}') + with db_session: + room = Room.get(id=ctx.room_id) + room.last_port = port + else: + ctx.logger.exception("Could not determine port. Likely hosting failure.") with db_session: - room = Room.get(id=ctx.room_id) - room.last_port = port + ctx.auto_shutdown = Room.get(id=room_id).timeout + ctx.shutdown_task = asyncio.create_task(auto_shutdown(ctx, [])) + await ctx.shutdown_task + + except (KeyboardInterrupt, SystemExit): + if ctx.saving: + ctx._save() + except Exception as e: + with db_session: + room = Room.get(id=room_id) + room.last_port = -1 + logger.exception(e) + raise else: - ctx.logger.exception("Could not determine port. Likely hosting failure.") - with db_session: - ctx.auto_shutdown = Room.get(id=room_id).timeout - ctx.shutdown_task = asyncio.create_task(auto_shutdown(ctx, [])) - await ctx.shutdown_task - - # ensure auto launch is on the same page in regard to room activity. - with db_session: - room: Room = Room.get(id=ctx.room_id) - room.last_activity = datetime.datetime.utcnow() - datetime.timedelta(seconds=room.timeout + 60) - - except (KeyboardInterrupt, SystemExit): - with db_session: - room = Room.get(id=room_id) - # ensure the Room does not spin up again on its own, minute of safety buffer - room.last_activity = datetime.datetime.utcnow() - datetime.timedelta(minutes=1, seconds=room.timeout) - except Exception: - with db_session: - room = Room.get(id=room_id) - room.last_port = -1 - # ensure the Room does not spin up again on its own, minute of safety buffer - room.last_activity = datetime.datetime.utcnow() - datetime.timedelta(minutes=1, seconds=room.timeout) - raise - finally: - rooms_shutting_down.put(room_id) + if ctx.saving: + ctx._save() + finally: + try: + with (db_session): + # ensure the Room does not spin up again on its own, minute of safety buffer + room = Room.get(id=room_id) + room.last_activity = datetime.datetime.utcnow() - \ + datetime.timedelta(minutes=1, seconds=room.timeout) + logging.info(f"Shutting down room {room_id} on {name}.") + finally: + await asyncio.sleep(5) + rooms_shutting_down.put(room_id) class Starter(threading.Thread): def run(self): diff --git a/WebHostLib/generate.py b/WebHostLib/generate.py index ee1ce591ee..a78560cb0b 100644 --- a/WebHostLib/generate.py +++ b/WebHostLib/generate.py @@ -70,37 +70,41 @@ def generate(race=False): flash(options) else: meta = get_meta(request.form, race) - results, gen_options = roll_options(options, set(meta["plando_options"])) - - if any(type(result) == str for result in results.values()): - return render_template("checkResult.html", results=results) - elif len(gen_options) > app.config["MAX_ROLL"]: - flash(f"Sorry, generating of multiworlds is limited to {app.config['MAX_ROLL']} players. " - f"If you have a larger group, please generate it yourself and upload it.") - elif len(gen_options) >= app.config["JOB_THRESHOLD"]: - gen = Generation( - options=pickle.dumps({name: vars(options) for name, options in gen_options.items()}), - # convert to json compatible - meta=json.dumps(meta), - state=STATE_QUEUED, - owner=session["_id"]) - commit() - - return redirect(url_for("wait_seed", seed=gen.id)) - else: - try: - seed_id = gen_game({name: vars(options) for name, options in gen_options.items()}, - meta=meta, owner=session["_id"].int) - except BaseException as e: - from .autolauncher import handle_generation_failure - handle_generation_failure(e) - return render_template("seedError.html", seed_error=(e.__class__.__name__ + ": " + str(e))) - - return redirect(url_for("view_seed", seed=seed_id)) + return start_generation(options, meta) return render_template("generate.html", race=race, version=__version__) +def start_generation(options: Dict[str, Union[dict, str]], meta: Dict[str, Any]): + results, gen_options = roll_options(options, set(meta["plando_options"])) + + if any(type(result) == str for result in results.values()): + return render_template("checkResult.html", results=results) + elif len(gen_options) > app.config["MAX_ROLL"]: + flash(f"Sorry, generating of multiworlds is limited to {app.config['MAX_ROLL']} players. " + f"If you have a larger group, please generate it yourself and upload it.") + elif len(gen_options) >= app.config["JOB_THRESHOLD"]: + gen = Generation( + options=pickle.dumps({name: vars(options) for name, options in gen_options.items()}), + # convert to json compatible + meta=json.dumps(meta), + state=STATE_QUEUED, + owner=session["_id"]) + commit() + + return redirect(url_for("wait_seed", seed=gen.id)) + else: + try: + seed_id = gen_game({name: vars(options) for name, options in gen_options.items()}, + meta=meta, owner=session["_id"].int) + except BaseException as e: + from .autolauncher import handle_generation_failure + handle_generation_failure(e) + return render_template("seedError.html", seed_error=(e.__class__.__name__ + ": " + str(e))) + + return redirect(url_for("view_seed", seed=seed_id)) + + def gen_game(gen_options: dict, meta: Optional[Dict[str, Any]] = None, owner=None, sid=None): if not meta: meta: Dict[str, Any] = {} diff --git a/WebHostLib/options.py b/WebHostLib/options.py index e631d31b03..94f173df70 100644 --- a/WebHostLib/options.py +++ b/WebHostLib/options.py @@ -1,33 +1,33 @@ import collections.abc -import os -import yaml -import requests import json -import flask +import os +from textwrap import dedent +from typing import Dict, Union + +import yaml +from flask import redirect, render_template, request, Response import Options -from Options import Visibility -from flask import redirect, render_template, request, Response -from worlds.AutoWorld import AutoWorldRegister from Utils import local_path -from textwrap import dedent +from worlds.AutoWorld import AutoWorldRegister from . import app, cache -def create(): +def create() -> None: target_folder = local_path("WebHostLib", "static", "generated") yaml_folder = os.path.join(target_folder, "configs") Options.generate_yaml_templates(yaml_folder) -def get_world_theme(game_name: str): +def get_world_theme(game_name: str) -> str: if game_name in AutoWorldRegister.world_types: return AutoWorldRegister.world_types[game_name].web.theme return 'grass' -def render_options_page(template: str, world_name: str, is_complex: bool = False): +def render_options_page(template: str, world_name: str, is_complex: bool = False) -> Union[Response, str]: + visibility_flag = Options.Visibility.complex_ui if is_complex else Options.Visibility.simple_ui world = AutoWorldRegister.world_types[world_name] if world.hidden or world.web.options_page is False: return redirect("games") @@ -39,13 +39,8 @@ def render_options_page(template: str, world_name: str, is_complex: bool = False grouped_options = {group: {} for group in ordered_groups} for option_name, option in world.options_dataclass.type_hints.items(): # Exclude settings from options pages if their visibility is disabled - if not is_complex and option.visibility < Visibility.simple_ui: - continue - - if is_complex and option.visibility < Visibility.complex_ui: - continue - - grouped_options[option_groups.get(option, "Game Options")][option_name] = option + if visibility_flag in option.visibility: + grouped_options[option_groups.get(option, "Game Options")][option_name] = option return render_template( template, @@ -58,26 +53,12 @@ def render_options_page(template: str, world_name: str, is_complex: bool = False ) -def generate_game(player_name: str, formatted_options: dict): - payload = { - "race": 0, - "hint_cost": 10, - "forfeit_mode": "auto", - "remaining_mode": "disabled", - "collect_mode": "goal", - "weights": { - player_name: formatted_options, - }, - } - r = requests.post("https://archipelago.gg/api/generate", json=payload) - if 200 <= r.status_code <= 299: - response_data = r.json() - return redirect(response_data["url"]) - else: - return r.text +def generate_game(options: Dict[str, Union[dict, str]]) -> Union[Response, str]: + from .generate import start_generation + return start_generation(options, {"plando_options": ["items", "connections", "texts", "bosses"]}) -def send_yaml(player_name: str, formatted_options: dict): +def send_yaml(player_name: str, formatted_options: dict) -> Response: response = Response(yaml.dump(formatted_options, sort_keys=False)) response.headers["Content-Type"] = "text/yaml" response.headers["Content-Disposition"] = f"attachment; filename={player_name}.yaml" @@ -85,7 +66,7 @@ def send_yaml(player_name: str, formatted_options: dict): @app.template_filter("dedent") -def filter_dedent(text: str): +def filter_dedent(text: str) -> str: return dedent(text).strip("\n ") @@ -98,10 +79,6 @@ def test_ordered(obj): @cache.cached() def option_presets(game: str) -> Response: world = AutoWorldRegister.world_types[game] - presets = {} - - if world.web.options_presets: - presets = presets | world.web.options_presets class SetEncoder(json.JSONEncoder): def default(self, obj): @@ -110,8 +87,8 @@ def option_presets(game: str) -> Response: return list(obj) return json.JSONEncoder.default(self, obj) - json_data = json.dumps(presets, cls=SetEncoder) - response = flask.Response(json_data) + json_data = json.dumps(world.web.options_presets, cls=SetEncoder) + response = Response(json_data) response.headers["Content-Type"] = "application/json" return response @@ -169,7 +146,7 @@ def generate_weighted_yaml(game: str): } if intent_generate: - return generate_game(player_name, formatted_options) + return generate_game({player_name: formatted_options}) else: return send_yaml(player_name, formatted_options) @@ -243,7 +220,7 @@ def generate_yaml(game: str): } if intent_generate: - return generate_game(player_name, formatted_options) + return generate_game({player_name: formatted_options}) else: return send_yaml(player_name, formatted_options) diff --git a/WebHostLib/templates/playerOptions/macros.html b/WebHostLib/templates/playerOptions/macros.html index 64964682fe..c4d97255d8 100644 --- a/WebHostLib/templates/playerOptions/macros.html +++ b/WebHostLib/templates/playerOptions/macros.html @@ -114,7 +114,7 @@ {% macro ItemDict(option_name, option, world) %} {{ OptionTitle(option_name, option) }}