MultiServer: fix Windows compatibility (#6010)

Co-authored-by: Ishigh1 <bonjour940@yahoo.fr>
Co-authored-by: Duck <31627079+duckboycool@users.noreply.github.com>
Co-authored-by: black-sliver <59490463+black-sliver@users.noreply.github.com>

Main.py

@@ -207,6 +207,9 @@ def main(args, seed=None, baked_server_options: dict[str, object] | None = None)
     else:
         logger.info("Progression balancing skipped.")
 
+    AutoWorld.call_all(multiworld, "finalize_multiworld")
+    AutoWorld.call_all(multiworld, "pre_output")
+
     # we're about to output using multithreading, so we're removing the global random state to prevent accidental use
     multiworld.random.passthrough = False
 

MultiServer.py

@@ -21,6 +21,7 @@ import time
 import typing
 import weakref
 import zlib
+from signal import SIGINT, SIGTERM, signal
 
 import ModuleUpdate
 
@@ -1302,6 +1303,13 @@ class CommandMeta(type):
             commands.update(base.commands)
         commands.update({command_name[5:]: method for command_name, method in attrs.items() if
                          command_name.startswith("_cmd_")})
+        for command_name, method in commands.items():
+            # wrap async def functions so they run on default asyncio loop
+            if inspect.iscoroutinefunction(method):
+                def _wrapper(self, *args, _method=method, **kwargs):
+                    return async_start(_method(self, *args, **kwargs))
+                functools.update_wrapper(_wrapper, method)
+                commands[command_name] = _wrapper
         return super(CommandMeta, cls).__new__(cls, name, bases, attrs)
 
 
@@ -2564,6 +2572,8 @@ async def console(ctx: Context):
             input_text = await queue.get()
             queue.task_done()
             ctx.commandprocessor(input_text)
+        except asyncio.exceptions.CancelledError:
+            ctx.logger.info("ConsoleTask cancelled")
         except:
             import traceback
             traceback.print_exc()
@@ -2730,6 +2740,26 @@ async def main(args: argparse.Namespace):
     console_task = asyncio.create_task(console(ctx))
     if ctx.auto_shutdown:
         ctx.shutdown_task = asyncio.create_task(auto_shutdown(ctx, [console_task]))
+
+    def stop():
+        try:
+            for remove_signal in [SIGINT, SIGTERM]:
+                asyncio.get_event_loop().remove_signal_handler(remove_signal)
+        except NotImplementedError:
+            pass
+        ctx.commandprocessor._cmd_exit()
+
+    def shutdown(signum, frame):
+        stop()
+
+    try:
+        for sig in [SIGINT, SIGTERM]:
+            asyncio.get_event_loop().add_signal_handler(sig, stop)
+    except NotImplementedError:
+        # add_signal_handler is only implemented for UNIX platforms
+        for sig in [SIGINT, SIGTERM]:
+            signal(sig, shutdown)
+
     await ctx.exit_event.wait()
     console_task.cancel()
     if ctx.shutdown_task:

OptionsCreator.py

@@ -29,7 +29,7 @@ import webbrowser
 import re
 from urllib.parse import urlparse
 from worlds.AutoWorld import AutoWorldRegister, World
-from Options import (Option, Toggle, TextChoice, Choice, FreeText, NamedRange, Range, OptionSet, OptionList, Removed,
+from Options import (Option, Toggle, TextChoice, Choice, FreeText, NamedRange, Range, OptionSet, OptionList,
                      OptionCounter, Visibility)
 
 
@@ -318,26 +318,28 @@ class OptionsCreator(ThemedApp):
         else:
             self.show_result_snack("Name cannot be longer than 16 characters.")
 
-    def create_range(self, option: typing.Type[Range], name: str):
+    def create_range(self, option: typing.Type[Range], name: str, bind=True):
         def update_text(range_box: VisualRange):
             self.options[name] = int(range_box.slider.value)
             range_box.tag.text = str(int(range_box.slider.value))
             return
 
         box = VisualRange(option=option, name=name)
-        box.slider.bind(on_touch_move=lambda _, _1: update_text(box))
+        if bind:
+            box.slider.bind(value=lambda _, _1: update_text(box))
         self.options[name] = option.default
         return box
 
     def create_named_range(self, option: typing.Type[NamedRange], name: str):
         def set_to_custom(range_box: VisualNamedRange):
-            if (not self.options[name] == range_box.range.slider.value) \
-                    and (not self.options[name] in option.special_range_names or
-                         range_box.range.slider.value != option.special_range_names[self.options[name]]):
-                # we should validate the touch here,
-                # but this is much cheaper
+            range_box.range.tag.text = str(int(range_box.range.slider.value))
+            if range_box.range.slider.value in option.special_range_names.values():
+                value = next(key for key, val in option.special_range_names.items()
+                             if val == range_box.range.slider.value)
+                self.options[name] = value
+                set_button_text(box.choice, value.title())
+            else:
                 self.options[name] = int(range_box.range.slider.value)
-                range_box.range.tag.text = str(int(range_box.range.slider.value))
                 set_button_text(range_box.choice, "Custom")
 
         def set_button_text(button: MDButton, text: str):
@@ -346,7 +348,7 @@ class OptionsCreator(ThemedApp):
         def set_value(text: str, range_box: VisualNamedRange):
             range_box.range.slider.value = min(max(option.special_range_names[text.lower()], option.range_start),
                                                option.range_end)
-            range_box.range.tag.text = str(int(range_box.range.slider.value))
+            range_box.range.tag.text = str(option.special_range_names[text.lower()])
             set_button_text(range_box.choice, text)
             self.options[name] = text.lower()
             range_box.range.slider.dropdown.dismiss()
@@ -355,13 +357,18 @@ class OptionsCreator(ThemedApp):
             # for some reason this fixes an issue causing some to not open
             box.range.slider.dropdown.open()
 
-        box = VisualNamedRange(option=option, name=name, range_widget=self.create_range(option, name))
-        if option.default in option.special_range_names:
+        box = VisualNamedRange(option=option, name=name, range_widget=self.create_range(option, name, bind=False))
+        default: int | str = option.default
+        if default in option.special_range_names:
             # value can get mismatched in this case
-            box.range.slider.value = min(max(option.special_range_names[option.default], option.range_start),
+            box.range.slider.value = min(max(option.special_range_names[default], option.range_start),
                                                option.range_end)
             box.range.tag.text = str(int(box.range.slider.value))
-        box.range.slider.bind(on_touch_move=lambda _, _2: set_to_custom(box))
+        elif default in option.special_range_names.values():
+            # better visual
+            default = next(key for key, val in option.special_range_names.items() if val == option.default)
+            set_button_text(box.choice, default.title())
+        box.range.slider.bind(value=lambda _, _2: set_to_custom(box))
         items = [
             {
                 "text": choice.title(),
@@ -371,7 +378,7 @@ class OptionsCreator(ThemedApp):
         ]
         box.range.slider.dropdown = MDDropdownMenu(caller=box.choice, items=items)
         box.choice.bind(on_release=open_dropdown)
-        self.options[name] = option.default
+        self.options[name] = default
         return box
 
     def create_free_text(self, option: typing.Type[FreeText] | typing.Type[TextChoice], name: str):
@@ -447,8 +454,12 @@ class OptionsCreator(ThemedApp):
         valid_keys = sorted(option.valid_keys)
         if option.verify_item_name:
             valid_keys += list(world.item_name_to_id.keys())
+            if option.convert_name_groups:
+                valid_keys += list(world.item_name_groups.keys())
         if option.verify_location_name:
             valid_keys += list(world.location_name_to_id.keys())
+            if option.convert_name_groups:
+                valid_keys += list(world.location_name_groups.keys())
 
         if not issubclass(option, OptionCounter):
             def apply_changes(button):
@@ -470,14 +481,6 @@ class OptionsCreator(ThemedApp):
         dialog.scrollbox.layout.spacing = dp(5)
         dialog.scrollbox.layout.padding = [0, dp(5), 0, 0]
 
-        if name not in self.options:
-            # convert from non-mutable to mutable
-            # We use list syntax even for sets, set behavior is enforced through GUI
-            if issubclass(option, OptionCounter):
-                self.options[name] = deepcopy(option.default)
-            else:
-                self.options[name] = sorted(option.default)
-
         if issubclass(option, OptionCounter):
             for value in sorted(self.options[name]):
                 dialog.add_set_item(value, self.options[name].get(value, None))
@@ -491,6 +494,15 @@ class OptionsCreator(ThemedApp):
     def create_option_set_list_counter(self, option: typing.Type[OptionList] | typing.Type[OptionSet] |
                                        typing.Type[OptionCounter], name: str, world: typing.Type[World]):
         main_button = MDButton(MDButtonText(text="Edit"), on_release=lambda x: self.create_popup(option, name, world))
+
+        if name not in self.options:
+            # convert from non-mutable to mutable
+            # We use list syntax even for sets, set behavior is enforced through GUI
+            if issubclass(option, OptionCounter):
+                self.options[name] = deepcopy(option.default)
+            else:
+                self.options[name] = sorted(option.default)
+
         return main_button
 
     def create_option(self, option: typing.Type[Option], name: str, world: typing.Type[World]) -> Widget:

UndertaleClient.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 import os
 import sys
+import time
 import asyncio
 import typing
 import bsdiff4
@@ -15,6 +16,9 @@ from CommonClient import CommonContext, server_loop, \
     gui_enabled, ClientCommandProcessor, logger, get_base_parser
 from Utils import async_start
 
+# Heartbeat for position sharing via bounces, in seconds
+UNDERTALE_STATUS_INTERVAL = 30.0
+UNDERTALE_ONLINE_TIMEOUT  = 60.0
 
 class UndertaleCommandProcessor(ClientCommandProcessor):
     def __init__(self, ctx):
@@ -109,6 +113,11 @@ class UndertaleContext(CommonContext):
         self.completed_routes = {"pacifist": 0, "genocide": 0, "neutral": 0}
         # self.save_game_folder: files go in this path to pass data between us and the actual game
         self.save_game_folder = os.path.expandvars(r"%localappdata%/UNDERTALE")
+        self.last_sent_position: typing.Optional[tuple] = None
+        self.last_room: typing.Optional[str] = None
+        self.last_status_write: float = 0.0
+        self.other_undertale_status: dict[int, dict] = {}
+
 
     def patch_game(self):
         with open(Utils.user_path("Undertale", "data.win"), "rb") as f:
@@ -219,6 +228,9 @@ async def process_undertale_cmd(ctx: UndertaleContext, cmd: str, args: dict):
         await ctx.send_msgs([{"cmd": "SetNotify", "keys": [str(ctx.slot)+" RoutesDone neutral",
                                                            str(ctx.slot)+" RoutesDone pacifist",
                                                            str(ctx.slot)+" RoutesDone genocide"]}])
+        if any(info.game == "Undertale" and slot != ctx.slot 
+               for slot, info in ctx.slot_info.items()):
+            ctx.set_notify("undertale_room_status")
         if args["slot_data"]["only_flakes"]:
             with open(os.path.join(ctx.save_game_folder, "GenoNoChest.flag"), "w") as f:
                 f.close()
@@ -263,6 +275,12 @@ async def process_undertale_cmd(ctx: UndertaleContext, cmd: str, args: dict):
         if str(ctx.slot)+" RoutesDone pacifist" in args["keys"]:
             if args["keys"][str(ctx.slot) + " RoutesDone pacifist"] is not None:
                 ctx.completed_routes["pacifist"] = args["keys"][str(ctx.slot)+" RoutesDone pacifist"]
+        if "undertale_room_status" in args["keys"] and args["keys"]["undertale_room_status"]:
+            status = args["keys"]["undertale_room_status"]
+            ctx.other_undertale_status = {
+                int(key): val for key, val in status.items()
+                if int(key) != ctx.slot
+            }
     elif cmd == "SetReply":
         if args["value"] is not None:
             if str(ctx.slot)+" RoutesDone pacifist" == args["key"]:
@@ -271,17 +289,19 @@ async def process_undertale_cmd(ctx: UndertaleContext, cmd: str, args: dict):
                 ctx.completed_routes["genocide"] = args["value"]
             elif str(ctx.slot)+" RoutesDone neutral" == args["key"]:
                 ctx.completed_routes["neutral"] = args["value"]
+        if args.get("key") == "undertale_room_status" and args.get("value"):
+            ctx.other_undertale_status = {
+                int(key): val for key, val in args["value"].items()
+                if int(key) != ctx.slot
+            }
     elif cmd == "ReceivedItems":
         start_index = args["index"]
 
         if start_index == 0:
             ctx.items_received = []
         elif start_index != len(ctx.items_received):
-            sync_msg = [{"cmd": "Sync"}]
-            if ctx.locations_checked:
-                sync_msg.append({"cmd": "LocationChecks",
-                                 "locations": list(ctx.locations_checked)})
-            await ctx.send_msgs(sync_msg)
+            await ctx.check_locations(ctx.locations_checked)
+            await ctx.send_msgs([{"cmd": "Sync"}])
         if start_index == len(ctx.items_received):
             counter = -1
             placedWeapon = 0
@@ -368,9 +388,8 @@ async def process_undertale_cmd(ctx: UndertaleContext, cmd: str, args: dict):
                 f.close()
 
     elif cmd == "Bounced":
-        tags = args.get("tags", [])
-        if "Online" in tags:
-            data = args.get("data", {})
+        data = args.get("data", {})
+        if "x" in data and "room" in data:
             if data["player"] != ctx.slot and data["player"] is not None:
                 filename = f"FRISK" + str(data["player"]) + ".playerspot"
                 with open(os.path.join(ctx.save_game_folder, filename), "w") as f:
@@ -381,21 +400,63 @@ async def process_undertale_cmd(ctx: UndertaleContext, cmd: str, args: dict):
 
 async def multi_watcher(ctx: UndertaleContext):
     while not ctx.exit_event.is_set():
-        path = ctx.save_game_folder
-        for root, dirs, files in os.walk(path):
-            for file in files:
-                if "spots.mine" in file and "Online" in ctx.tags:
-                    with open(os.path.join(root, file), "r") as mine:
-                        this_x = mine.readline()
-                        this_y = mine.readline()
-                        this_room = mine.readline()
-                        this_sprite = mine.readline()
-                        this_frame = mine.readline()
-                        mine.close()
-                    message = [{"cmd": "Bounce", "tags": ["Online"],
-                                "data": {"player": ctx.slot, "x": this_x, "y": this_y, "room": this_room,
-                                         "spr": this_sprite, "frm": this_frame}}]
-                    await ctx.send_msgs(message)
+        if "Online" in ctx.tags and any(
+                info.game == "Undertale" and slot != ctx.slot
+                for slot, info in ctx.slot_info.items()):
+            now = time.time()
+            path = ctx.save_game_folder
+            for root, dirs, files in os.walk(path):
+                for file in files:
+                    if "spots.mine" in file:
+                        with open(os.path.join(root, file), "r") as mine:
+                            this_x = mine.readline()
+                            this_y = mine.readline()
+                            this_room = mine.readline()
+                            this_sprite = mine.readline()
+                            this_frame = mine.readline()
+
+                        if this_room != ctx.last_room or \
+                                now - ctx.last_status_write >= UNDERTALE_STATUS_INTERVAL:
+                            ctx.last_room = this_room
+                            ctx.last_status_write = now
+                            await ctx.send_msgs([{
+                                "cmd": "Set",
+                                "key": "undertale_room_status",
+                                "default": {},
+                                "want_reply": False,
+                                "operations": [{"operation": "update",
+                                                "value": {str(ctx.slot): {"room": this_room,
+                                                                           "time": now}}}]
+                            }])
+
+                        # If player was visible but timed out (heartbeat) or left the room, remove them.
+                        for slot, entry in ctx.other_undertale_status.items():
+                            if entry.get("room") != this_room or \
+                                    now - entry.get("time", now) > UNDERTALE_ONLINE_TIMEOUT:
+                                playerspot = os.path.join(ctx.save_game_folder,
+                                                          f"FRISK{slot}.playerspot")
+                                if os.path.exists(playerspot):
+                                    os.remove(playerspot)
+
+                        current_position = (this_x, this_y, this_room, this_sprite, this_frame)
+                        if current_position == ctx.last_sent_position:
+                            continue
+
+                        # Empty status dict = no data yet → send to bootstrap.
+                        online_in_room = any(
+                            entry.get("room") == this_room and
+                            now - entry.get("time", now) <= UNDERTALE_ONLINE_TIMEOUT
+                            for entry in ctx.other_undertale_status.values()
+                        )
+                        if ctx.other_undertale_status and not online_in_room:
+                            continue
+
+                        message = [{"cmd": "Bounce", "games": ["Undertale"],
+                                    "data": {"player": ctx.slot, "x": this_x, "y": this_y,
+                                             "room": this_room, "spr": this_sprite,
+                                             "frm": this_frame}}]
+                        await ctx.send_msgs(message)
+                        ctx.last_sent_position = current_position
 
         await asyncio.sleep(0.1)
 
@@ -409,10 +470,9 @@ async def game_watcher(ctx: UndertaleContext):
                 for file in files:
                     if ".item" in file:
                         os.remove(os.path.join(root, file))
-            sync_msg = [{"cmd": "Sync"}]
-            if ctx.locations_checked:
-                sync_msg.append({"cmd": "LocationChecks", "locations": list(ctx.locations_checked)})
-            await ctx.send_msgs(sync_msg)
+            await ctx.check_locations(ctx.locations_checked)
+            await ctx.send_msgs([{"cmd": "Sync"}])
+
             ctx.syncing = False
         if ctx.got_deathlink:
             ctx.got_deathlink = False
@@ -447,7 +507,7 @@ async def game_watcher(ctx: UndertaleContext):
                         for l in lines:
                             sending = sending+[(int(l.rstrip('\n')))+12000]
                     finally:
-                        await ctx.send_msgs([{"cmd": "LocationChecks", "locations": sending}])
+                        await ctx.check_locations(sending)
                 if "victory" in file and str(ctx.route) in file:
                     victory = True
                 if ".playerspot" in file and "Online" not in ctx.tags:

Utils.py

@@ -23,6 +23,7 @@ from time import sleep
 from typing import BinaryIO, Coroutine, Optional, Set, Dict, Any, Union, TypeGuard
 from yaml import load, load_all, dump
 from pathspec import PathSpec, GitIgnoreSpec
+from typing_extensions import deprecated
 
 try:
     from yaml import CLoader as UnsafeLoader, CSafeLoader as SafeLoader, CDumper as Dumper
@@ -315,6 +316,7 @@ def get_public_ipv6() -> str:
     return ip
 
 
+@deprecated("Utils.get_options() is deprecated. Use the settings API instead.")
 def get_options() -> Settings:
     deprecate("Utils.get_options() is deprecated. Use the settings API instead.")
     return get_settings()
@@ -1003,6 +1005,7 @@ def async_start(co: Coroutine[None, None, typing.Any], name: Optional[str] = Non
 
 
 def deprecate(message: str, add_stacklevels: int = 0):
+    """also use typing_extensions.deprecated wherever you use this"""
     if __debug__:
         raise Exception(message)
     warnings.warn(message, stacklevel=2 + add_stacklevels)
@@ -1067,6 +1070,7 @@ def _extend_freeze_support() -> None:
     multiprocessing.freeze_support = multiprocessing.spawn.freeze_support = _freeze_support if is_frozen() else _noop
 
 
+@deprecated("Use multiprocessing.freeze_support() instead")
 def freeze_support() -> None:
     """This now only calls multiprocessing.freeze_support since we are patching freeze_support on module load."""
     import multiprocessing

test/bases.py

@@ -248,6 +248,7 @@ class WorldTestBase(unittest.TestCase):
         with self.subTest("Game", game=self.game, seed=self.multiworld.seed):
             distribute_items_restrictive(self.multiworld)
             call_all(self.multiworld, "post_fill")
+            call_all(self.multiworld, "finalize_multiworld")
             self.assertTrue(fulfills_accessibility(), "Collected all locations, but can't beat the game.")
             placed_items = [loc.item for loc in self.multiworld.get_locations() if loc.item and loc.item.code]
             self.assertLessEqual(len(self.multiworld.itempool), len(placed_items),

test/general/test_ids.py

@@ -88,6 +88,7 @@ class TestIDs(unittest.TestCase):
                 multiworld = setup_solo_multiworld(world_type)
                 distribute_items_restrictive(multiworld)
                 call_all(multiworld, "post_fill")
+                call_all(multiworld, "finalize_multiworld")
                 datapackage = world_type.get_data_package_data()
                 for item_group, item_names in datapackage["item_name_groups"].items():
                     self.assertIsInstance(item_group, str,

test/general/test_implemented.py

@@ -46,6 +46,8 @@ class TestImplemented(unittest.TestCase):
             with self.subTest(game=game_name, seed=multiworld.seed):
                 distribute_items_restrictive(multiworld)
                 call_all(multiworld, "post_fill")
+                call_all(multiworld, "finalize_multiworld")
+                call_all(multiworld, "pre_output")
                 for key, data in multiworld.worlds[1].fill_slot_data().items():
                     self.assertIsInstance(key, str, "keys in slot data must be a string")
                     convert_to_base_types(data)  # only put base data types into slot data
@@ -93,6 +95,7 @@ class TestImplemented(unittest.TestCase):
             with self.subTest(game=game_name, seed=multiworld.seed):
                 distribute_items_restrictive(multiworld)
                 call_all(multiworld, "post_fill")
+                call_all(multiworld, "finalize_multiworld")
 
                 # Note: `multiworld.get_spheres()` iterates a set of locations, so the order that locations are checked
                 # is nondeterministic and may vary between runs with the same seed.

test/general/test_items.py

@@ -123,6 +123,7 @@ class TestBase(unittest.TestCase):
             call_all(multiworld, "pre_fill")
             distribute_items_restrictive(multiworld)
             call_all(multiworld, "post_fill")
+            call_all(multiworld, "finalize_multiworld")
             self.assertTrue(multiworld.can_beat_game(CollectionState(multiworld)), f"seed = {multiworld.seed}")
 
         for game_name, world_type in AutoWorldRegister.world_types.items():

test/multiworld/test_multiworlds.py

@@ -61,6 +61,7 @@ class TestAllGamesMultiworld(MultiworldTestBase):
         with self.subTest("filling multiworld", seed=self.multiworld.seed):
             distribute_items_restrictive(self.multiworld)
             call_all(self.multiworld, "post_fill")
+            call_all(self.multiworld, "finalize_multiworld")
             self.assertTrue(self.fulfills_accessibility(), "Collected all locations, but can't beat the game")
 
 
@@ -78,4 +79,5 @@ class TestTwoPlayerMulti(MultiworldTestBase):
         with self.subTest("filling multiworld", games=world_type.game, seed=self.multiworld.seed):
             distribute_items_restrictive(self.multiworld)
             call_all(self.multiworld, "post_fill")
+            call_all(self.multiworld, "finalize_multiworld")
             self.assertTrue(self.fulfills_accessibility(), "Collected all locations, but can't beat the game")

worlds/AutoWorld.py

@@ -430,6 +430,23 @@ class World(metaclass=AutoWorldRegister):
         This happens before progression balancing, so the items may not be in their final locations yet.
         """
 
+    def finalize_multiworld(self) -> None:
+        """
+        Optional Method that is called after fill and progression balancing.
+        This is the last stage of generation where worlds may change logically relevant data,
+        such as item placements and connections. To not break assumptions,
+        only ever increase accessibility, never decrease it.
+        """
+        pass
+
+    def pre_output(self):
+        """
+        Optional method that is called before output generation.
+        Items and connections are not meant to be moved anymore,
+        anything that would affect logical spheres is forbidden at this point.
+        """
+        pass
+
     def generate_output(self, output_directory: str) -> None:
         """
         This method gets called from a threadpool, do not use multiworld.random here.