diff --git a/.gitattributes b/.gitattributes index 537a05f68b..5ab5379334 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ worlds/blasphemous/region_data.py linguist-generated=true +worlds/yachtdice/YachtWeights.py linguist-generated=true diff --git a/.github/pyright-config.json b/.github/pyright-config.json index 6ad7fa5f19..b6561afa46 100644 --- a/.github/pyright-config.json +++ b/.github/pyright-config.json @@ -1,8 +1,21 @@ { "include": [ - "type_check.py", + "../BizHawkClient.py", + "../Patch.py", + "../test/param.py", + "../test/general/test_groups.py", + "../test/general/test_helpers.py", + "../test/general/test_memory.py", + "../test/general/test_names.py", + "../test/multiworld/__init__.py", + "../test/multiworld/test_multiworlds.py", + "../test/netutils/__init__.py", + "../test/programs/__init__.py", + "../test/programs/test_multi_server.py", + "../test/utils/__init__.py", + "../test/webhost/test_descriptions.py", "../worlds/AutoSNIClient.py", - "../Patch.py" + "type_check.py" ], "exclude": [ @@ -16,7 +29,7 @@ "reportMissingImports": true, "reportMissingTypeStubs": true, - "pythonVersion": "3.8", + "pythonVersion": "3.10", "pythonPlatform": "Windows", "executionEnvironments": [ diff --git a/.github/workflows/analyze-modified-files.yml b/.github/workflows/analyze-modified-files.yml index c9995fa2d0..b59336fafe 100644 --- a/.github/workflows/analyze-modified-files.yml +++ b/.github/workflows/analyze-modified-files.yml @@ -53,7 +53,7 @@ jobs: - uses: actions/setup-python@v5 if: env.diff != '' with: - python-version: 3.8 + python-version: '3.10' - name: "Install dependencies" if: env.diff != '' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 23c463fb94..2b450fe46e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,14 +24,15 @@ env: jobs: # build-release-macos: # LF volunteer - build-win-py38: # RCs will still be built and signed by hand + build-win: # RCs will still be built and signed by hand runs-on: windows-latest steps: - uses: actions/checkout@v4 - name: Install python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '~3.12.7' + check-latest: true - name: Download run-time dependencies run: | Invoke-WebRequest -Uri https://github.com/Ijwu/Enemizer/releases/download/${Env:ENEMIZER_VERSION}/win-x64.zip -OutFile enemizer.zip @@ -111,10 +112,11 @@ jobs: - name: Get a recent python uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '~3.12.7' + check-latest: true - name: Install build-time dependencies run: | - echo "PYTHON=python3.11" >> $GITHUB_ENV + echo "PYTHON=python3.12" >> $GITHUB_ENV wget -nv https://github.com/AppImage/AppImageKit/releases/download/$APPIMAGETOOL_VERSION/appimagetool-x86_64.AppImage chmod a+rx appimagetool-x86_64.AppImage ./appimagetool-x86_64.AppImage --appimage-extract @@ -130,7 +132,7 @@ jobs: # charset-normalizer was somehow incomplete in the github runner "${{ env.PYTHON }}" -m venv venv source venv/bin/activate - "${{ env.PYTHON }}" -m pip install --upgrade pip PyGObject charset-normalizer + "${{ env.PYTHON }}" -m pip install --upgrade pip "PyGObject<3.51.0" charset-normalizer python setup.py build_exe --yes bdist_appimage --yes echo -e "setup.py build output:\n `ls build`" echo -e "setup.py dist output:\n `ls dist`" diff --git a/.github/workflows/ctest.yml b/.github/workflows/ctest.yml index 9492c83c9e..610f6d7477 100644 --- a/.github/workflows/ctest.yml +++ b/.github/workflows/ctest.yml @@ -11,7 +11,7 @@ on: - '**.hh?' - '**.hpp' - '**.hxx' - - '**.CMakeLists' + - '**/CMakeLists.txt' - '.github/workflows/ctest.yml' pull_request: paths: @@ -21,7 +21,7 @@ on: - '**.hh?' - '**.hpp' - '**.hxx' - - '**.CMakeLists' + - '**/CMakeLists.txt' - '.github/workflows/ctest.yml' jobs: @@ -36,9 +36,9 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: ilammy/msvc-dev-cmd@v1 + - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 if: startsWith(matrix.os,'windows') - - uses: Bacondish2023/setup-googletest@v1 + - uses: Bacondish2023/setup-googletest@49065d1f7a6d21f6134864dd65980fe5dbe06c73 with: build-type: 'Release' - name: Build tests diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3f8651d408..f12e8fb80c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,10 +44,11 @@ jobs: - name: Get a recent python uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '~3.12.7' + check-latest: true - name: Install build-time dependencies run: | - echo "PYTHON=python3.11" >> $GITHUB_ENV + echo "PYTHON=python3.12" >> $GITHUB_ENV wget -nv https://github.com/AppImage/AppImageKit/releases/download/$APPIMAGETOOL_VERSION/appimagetool-x86_64.AppImage chmod a+rx appimagetool-x86_64.AppImage ./appimagetool-x86_64.AppImage --appimage-extract @@ -63,7 +64,7 @@ jobs: # charset-normalizer was somehow incomplete in the github runner "${{ env.PYTHON }}" -m venv venv source venv/bin/activate - "${{ env.PYTHON }}" -m pip install --upgrade pip PyGObject charset-normalizer + "${{ env.PYTHON }}" -m pip install --upgrade pip "PyGObject<3.51.0" charset-normalizer python setup.py build_exe --yes bdist_appimage --yes echo -e "setup.py build output:\n `ls build`" echo -e "setup.py dist output:\n `ls dist`" diff --git a/.github/workflows/scan-build.yml b/.github/workflows/scan-build.yml index 5234d862b4..ac84207062 100644 --- a/.github/workflows/scan-build.yml +++ b/.github/workflows/scan-build.yml @@ -40,10 +40,10 @@ jobs: run: | wget https://apt.llvm.org/llvm.sh chmod +x ./llvm.sh - sudo ./llvm.sh 17 + sudo ./llvm.sh 19 - name: Install scan-build command run: | - sudo apt install clang-tools-17 + sudo apt install clang-tools-19 - name: Get a recent python uses: actions/setup-python@v5 with: @@ -56,7 +56,7 @@ jobs: - name: scan-build run: | source venv/bin/activate - scan-build-17 --status-bugs -o scan-build-reports -disable-checker deadcode.DeadStores python setup.py build -y + scan-build-19 --status-bugs -o scan-build-reports -disable-checker deadcode.DeadStores python setup.py build -y - name: Store report if: failure() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/strict-type-check.yml b/.github/workflows/strict-type-check.yml index bafd572a26..2ccdad8d11 100644 --- a/.github/workflows/strict-type-check.yml +++ b/.github/workflows/strict-type-check.yml @@ -26,7 +26,7 @@ jobs: - name: "Install dependencies" run: | - python -m pip install --upgrade pip pyright==1.1.358 + python -m pip install --upgrade pip pyright==1.1.392.post0 python ModuleUpdate.py --append "WebHostLib/requirements.txt" --force --yes - name: "pyright: strict check on specific files" diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index a38fef8fda..88b5d12987 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -33,13 +33,11 @@ jobs: matrix: os: [ubuntu-latest] python: - - {version: '3.8'} - - {version: '3.9'} - {version: '3.10'} - {version: '3.11'} - {version: '3.12'} include: - - python: {version: '3.8'} # win7 compat + - python: {version: '3.10'} # old compat os: windows-latest - python: {version: '3.12'} # current os: windows-latest diff --git a/.gitignore b/.gitignore index 791f7b1bb7..5da42dc1e0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ *_Spoiler.txt *.bmbp *.apbp +*.apcivvi *.apl2ac *.apm3 *.apmc diff --git a/AdventureClient.py b/AdventureClient.py index 24c6a4c4fc..91567fc0a0 100644 --- a/AdventureClient.py +++ b/AdventureClient.py @@ -511,7 +511,7 @@ if __name__ == '__main__': import colorama - colorama.init() + colorama.just_fix_windows_console() asyncio.run(main()) colorama.deinit() diff --git a/BaseClasses.py b/BaseClasses.py index 46edeb5ea0..3d0004806c 100644 --- a/BaseClasses.py +++ b/BaseClasses.py @@ -1,18 +1,16 @@ from __future__ import annotations import collections -import itertools import functools import logging import random import secrets -import typing # this can go away when Python 3.8 support is dropped from argparse import Namespace from collections import Counter, deque from collections.abc import Collection, MutableSequence from enum import IntEnum, IntFlag from typing import (AbstractSet, Any, Callable, ClassVar, Dict, Iterable, Iterator, List, Mapping, NamedTuple, - Optional, Protocol, Set, Tuple, Union, Type) + Optional, Protocol, Set, Tuple, Union, TYPE_CHECKING) from typing_extensions import NotRequired, TypedDict @@ -20,7 +18,8 @@ import NetUtils import Options import Utils -if typing.TYPE_CHECKING: +if TYPE_CHECKING: + from entrance_rando import ERPlacementState from worlds import AutoWorld @@ -231,7 +230,7 @@ class MultiWorld(): for player in self.player_ids: world_type = AutoWorld.AutoWorldRegister.world_types[self.game[player]] self.worlds[player] = world_type(self, player) - options_dataclass: typing.Type[Options.PerGameCommonOptions] = world_type.options_dataclass + options_dataclass: type[Options.PerGameCommonOptions] = world_type.options_dataclass self.worlds[player].options = options_dataclass(**{option_key: getattr(args, option_key)[player] for option_key in options_dataclass.type_hints}) @@ -428,12 +427,12 @@ class MultiWorld(): def get_location(self, location_name: str, player: int) -> Location: return self.regions.location_cache[player][location_name] - def get_all_state(self, use_cache: bool) -> CollectionState: + def get_all_state(self, use_cache: bool, allow_partial_entrances: bool = False) -> CollectionState: cached = getattr(self, "_all_state", None) if use_cache and cached: return cached.copy() - ret = CollectionState(self) + ret = CollectionState(self, allow_partial_entrances) for item in self.itempool: self.worlds[item.player].collect(ret, item) @@ -606,6 +605,49 @@ class MultiWorld(): state.collect(location.item, True, location) locations -= sphere + def get_sendable_spheres(self) -> Iterator[Set[Location]]: + """ + yields a set of multiserver sendable locations (location.item.code: int) for each logical sphere + + If there are unreachable locations, the last sphere of reachable locations is followed by an empty set, + and then a set of all of the unreachable locations. + """ + state = CollectionState(self) + locations: Set[Location] = set() + events: Set[Location] = set() + for location in self.get_filled_locations(): + if type(location.item.code) is int: + locations.add(location) + else: + events.add(location) + + while locations: + sphere: Set[Location] = set() + + # cull events out + done_events: Set[Union[Location, None]] = {None} + while done_events: + done_events = set() + for event in events: + if event.can_reach(state): + state.collect(event.item, True, event) + done_events.add(event) + events -= done_events + + for location in locations: + if location.can_reach(state): + sphere.add(location) + + yield sphere + if not sphere: + if locations: + yield locations # unreachable locations + break + + for location in sphere: + state.collect(location.item, True, location) + locations -= sphere + def fulfills_accessibility(self, state: Optional[CollectionState] = None): """Check if accessibility rules are fulfilled with current or supplied state.""" if not state: @@ -676,10 +718,11 @@ class CollectionState(): path: Dict[Union[Region, Entrance], PathValue] locations_checked: Set[Location] stale: Dict[int, bool] + allow_partial_entrances: bool additional_init_functions: List[Callable[[CollectionState, MultiWorld], None]] = [] additional_copy_functions: List[Callable[[CollectionState, CollectionState], CollectionState]] = [] - def __init__(self, parent: MultiWorld): + def __init__(self, parent: MultiWorld, allow_partial_entrances: bool = False): self.prog_items = {player: Counter() for player in parent.get_all_ids()} self.multiworld = parent self.reachable_regions = {player: set() for player in parent.get_all_ids()} @@ -688,6 +731,7 @@ class CollectionState(): self.path = {} self.locations_checked = set() self.stale = {player: True for player in parent.get_all_ids()} + self.allow_partial_entrances = allow_partial_entrances for function in self.additional_init_functions: function(self, parent) for items in parent.precollected_items.values(): @@ -722,6 +766,8 @@ class CollectionState(): if new_region in reachable_regions: blocked_connections.remove(connection) elif connection.can_reach(self): + if self.allow_partial_entrances and not new_region: + continue assert new_region, f"tried to search through an Entrance \"{connection}\" with no connected Region" reachable_regions.add(new_region) blocked_connections.remove(connection) @@ -747,7 +793,9 @@ class CollectionState(): if new_region in reachable_regions: blocked_connections.remove(connection) elif connection.can_reach(self): - assert new_region, f"tried to search through an Entrance \"{connection}\" with no Region" + if self.allow_partial_entrances and not new_region: + continue + assert new_region, f"tried to search through an Entrance \"{connection}\" with no connected Region" reachable_regions.add(new_region) blocked_connections.remove(connection) blocked_connections.update(new_region.exits) @@ -767,6 +815,7 @@ class CollectionState(): ret.advancements = self.advancements.copy() ret.path = self.path.copy() ret.locations_checked = self.locations_checked.copy() + ret.allow_partial_entrances = self.allow_partial_entrances for function in self.additional_copy_functions: ret = function(self, ret) return ret @@ -820,21 +869,40 @@ class CollectionState(): def has(self, item: str, player: int, count: int = 1) -> bool: return self.prog_items[player][item] >= count + # for loops are specifically used in all/any/count methods, instead of all()/any()/sum(), to avoid the overhead of + # creating and iterating generator instances. In `return all(player_prog_items[item] for item in items)`, the + # argument to all() would be a new generator instance, for example. def has_all(self, items: Iterable[str], player: int) -> bool: """Returns True if each item name of items is in state at least once.""" - return all(self.prog_items[player][item] for item in items) + player_prog_items = self.prog_items[player] + for item in items: + if not player_prog_items[item]: + return False + return True def has_any(self, items: Iterable[str], player: int) -> bool: """Returns True if at least one item name of items is in state at least once.""" - return any(self.prog_items[player][item] for item in items) + player_prog_items = self.prog_items[player] + for item in items: + if player_prog_items[item]: + return True + return False def has_all_counts(self, item_counts: Mapping[str, int], player: int) -> bool: """Returns True if each item name is in the state at least as many times as specified.""" - return all(self.prog_items[player][item] >= count for item, count in item_counts.items()) + player_prog_items = self.prog_items[player] + for item, count in item_counts.items(): + if player_prog_items[item] < count: + return False + return True def has_any_count(self, item_counts: Mapping[str, int], player: int) -> bool: """Returns True if at least one item name is in the state at least as many times as specified.""" - return any(self.prog_items[player][item] >= count for item, count in item_counts.items()) + player_prog_items = self.prog_items[player] + for item, count in item_counts.items(): + if player_prog_items[item] >= count: + return True + return False def count(self, item: str, player: int) -> int: return self.prog_items[player][item] @@ -862,11 +930,20 @@ class CollectionState(): def count_from_list(self, items: Iterable[str], player: int) -> int: """Returns the cumulative count of items from a list present in state.""" - return sum(self.prog_items[player][item_name] for item_name in items) + player_prog_items = self.prog_items[player] + total = 0 + for item_name in items: + total += player_prog_items[item_name] + return total def count_from_list_unique(self, items: Iterable[str], player: int) -> int: """Returns the cumulative count of items from a list present in state. Ignores duplicates of the same item.""" - return sum(self.prog_items[player][item_name] > 0 for item_name in items) + player_prog_items = self.prog_items[player] + total = 0 + for item_name in items: + if player_prog_items[item_name] > 0: + total += 1 + return total # item name group related def has_group(self, item_name_group: str, player: int, count: int = 1) -> bool: @@ -931,6 +1008,11 @@ class CollectionState(): self.stale[item.player] = True +class EntranceType(IntEnum): + ONE_WAY = 1 + TWO_WAY = 2 + + class Entrance: access_rule: Callable[[CollectionState], bool] = staticmethod(lambda state: True) hide_path: bool = False @@ -938,19 +1020,24 @@ class Entrance: name: str parent_region: Optional[Region] connected_region: Optional[Region] = None + randomization_group: int + randomization_type: EntranceType # LttP specific, TODO: should make a LttPEntrance addresses = None target = None - def __init__(self, player: int, name: str = "", parent: Optional[Region] = None) -> None: + def __init__(self, player: int, name: str = "", parent: Optional[Region] = None, + randomization_group: int = 0, randomization_type: EntranceType = EntranceType.ONE_WAY) -> None: self.name = name self.parent_region = parent self.player = player + self.randomization_group = randomization_group + self.randomization_type = randomization_type def can_reach(self, state: CollectionState) -> bool: assert self.parent_region, f"called can_reach on an Entrance \"{self}\" with no parent_region" if self.parent_region.can_reach(state) and self.access_rule(state): - if not self.hide_path and not self in state.path: + if not self.hide_path and self not in state.path: state.path[self] = (self.name, state.path.get(self.parent_region, (self.parent_region.name, None))) return True @@ -962,6 +1049,32 @@ class Entrance: self.addresses = addresses region.entrances.append(self) + def is_valid_source_transition(self, er_state: "ERPlacementState") -> bool: + """ + Determines whether this is a valid source transition, that is, whether the entrance + randomizer is allowed to pair it to place any other regions. By default, this is the + same as a reachability check, but can be modified by Entrance implementations to add + other restrictions based on the placement state. + + :param er_state: The current (partial) state of the ongoing entrance randomization + """ + return self.can_reach(er_state.collection_state) + + def can_connect_to(self, other: Entrance, dead_end: bool, er_state: "ERPlacementState") -> bool: + """ + Determines whether a given Entrance is a valid target transition, that is, whether + the entrance randomizer is allowed to pair this Entrance to that Entrance. By default, + only allows connection between entrances of the same type (one ways only go to one ways, + two ways always go to two ways) and prevents connecting an exit to itself in coupled mode. + + :param other: The proposed Entrance to connect to + :param dead_end: Whether the other entrance considered a dead end by Entrance randomization + :param er_state: The current (partial) state of the ongoing entrance randomization + """ + # the implementation of coupled causes issues for self-loops since the reverse entrance will be the + # same as the forward entrance. In uncoupled they are ok. + return self.randomization_type == other.randomization_type and (not er_state.coupled or self.name != other.name) + def __repr__(self): multiworld = self.parent_region.multiworld if self.parent_region else None return multiworld.get_name_string_for_object(self) if multiworld else f'{self.name} (Player {self.player})' @@ -975,7 +1088,7 @@ class Region: entrances: List[Entrance] exits: List[Entrance] locations: List[Location] - entrance_type: ClassVar[Type[Entrance]] = Entrance + entrance_type: ClassVar[type[Entrance]] = Entrance class Register(MutableSequence): region_manager: MultiWorld.RegionManager @@ -1075,7 +1188,7 @@ class Region: return entrance.parent_region.get_connecting_entrance(is_main_entrance) def add_locations(self, locations: Dict[str, Optional[int]], - location_type: Optional[Type[Location]] = None) -> None: + location_type: Optional[type[Location]] = None) -> None: """ Adds locations to the Region object, where location_type is your Location class and locations is a dict of location names to address. @@ -1111,8 +1224,18 @@ class Region: self.exits.append(exit_) return exit_ + def create_er_target(self, name: str) -> Entrance: + """ + Creates and returns an Entrance object as an entrance to this region + + :param name: name of the Entrance being created + """ + entrance = self.entrance_type(self.player, name) + entrance.connect(self) + return entrance + def add_exits(self, exits: Union[Iterable[str], Dict[str, Optional[str]]], - rules: Dict[str, Callable[[CollectionState], bool]] = None) -> None: + rules: Dict[str, Callable[[CollectionState], bool]] = None) -> List[Entrance]: """ Connects current region to regions in exit dictionary. Passed region names must exist first. @@ -1122,10 +1245,14 @@ class Region: """ if not isinstance(exits, Dict): exits = dict.fromkeys(exits) - for connecting_region, name in exits.items(): - self.connect(self.multiworld.get_region(connecting_region, self.player), - name, - rules[connecting_region] if rules and connecting_region in rules else None) + return [ + self.connect( + self.multiworld.get_region(connecting_region, self.player), + name, + rules[connecting_region] if rules and connecting_region in rules else None, + ) + for connecting_region, name in exits.items() + ] def __repr__(self): return self.multiworld.get_name_string_for_object(self) if self.multiworld else f'{self.name} (Player {self.player})' @@ -1209,13 +1336,26 @@ class Location: class ItemClassification(IntFlag): - filler = 0b0000 # aka trash, as in filler items like ammo, currency etc, - progression = 0b0001 # Item that is logically relevant - useful = 0b0010 # Item that is generally quite useful, but not required for anything logical - trap = 0b0100 # detrimental item - skip_balancing = 0b1000 # should technically never occur on its own - # Item that is logically relevant, but progression balancing should not touch. - # Typically currency or other counted items. + filler = 0b0000 + """ aka trash, as in filler items like ammo, currency etc """ + + progression = 0b0001 + """ Item that is logically relevant. + Protects this item from being placed on excluded or unreachable locations. """ + + useful = 0b0010 + """ Item that is especially useful. + Protects this item from being placed on excluded or unreachable locations. + When combined with another flag like "progression", it means "an especially useful progression item". """ + + trap = 0b0100 + """ Item that is detrimental in some way. """ + + skip_balancing = 0b1000 + """ should technically never occur on its own + Item that is logically relevant, but progression balancing should not touch. + Typically currency or other counted items. """ + progression_skip_balancing = 0b1001 # only progression gets balanced def as_flag(self) -> int: @@ -1264,6 +1404,10 @@ class Item: def trap(self) -> bool: return ItemClassification.trap in self.classification + @property + def filler(self) -> bool: + return not (self.advancement or self.useful or self.trap) + @property def excludable(self) -> bool: return not (self.advancement or self.useful) @@ -1386,14 +1530,21 @@ class Spoiler: # second phase, sphere 0 removed_precollected: List[Item] = [] - for item in (i for i in chain.from_iterable(multiworld.precollected_items.values()) if i.advancement): - logging.debug('Checking if %s (Player %d) is required to beat the game.', item.name, item.player) - multiworld.precollected_items[item.player].remove(item) - multiworld.state.remove(item) - if not multiworld.can_beat_game(): - multiworld.push_precollected(item) - else: - removed_precollected.append(item) + + for precollected_items in multiworld.precollected_items.values(): + # The list of items is mutated by removing one item at a time to determine if each item is required to beat + # the game, and re-adding that item if it was required, so a copy needs to be made before iterating. + for item in precollected_items.copy(): + if not item.advancement: + continue + logging.debug('Checking if %s (Player %d) is required to beat the game.', item.name, item.player) + precollected_items.remove(item) + multiworld.state.remove(item) + if not multiworld.can_beat_game(): + # Add the item back into `precollected_items` and collect it into `multiworld.state`. + multiworld.push_precollected(item) + else: + removed_precollected.append(item) # we are now down to just the required progress items in collection_spheres. Unfortunately # the previous pruning stage could potentially have made certain items dependant on others @@ -1532,7 +1683,7 @@ class Spoiler: [f" {location}: {item}" for (location, item) in sphere.items()] if isinstance(sphere, dict) else [f" {item}" for item in sphere])) for (sphere_nr, sphere) in self.playthrough.items()])) if self.unreachables: - outfile.write('\n\nUnreachable Items:\n\n') + outfile.write('\n\nUnreachable Progression Items:\n\n') outfile.write( '\n'.join(['%s: %s' % (unreachable.item, unreachable) for unreachable in self.unreachables])) diff --git a/CommonClient.py b/CommonClient.py index 47100a7383..6bd1bb4ce6 100644 --- a/CommonClient.py +++ b/CommonClient.py @@ -23,7 +23,7 @@ if __name__ == "__main__": from MultiServer import CommandProcessor from NetUtils import (Endpoint, decode, NetworkItem, encode, JSONtoTextParser, ClientStatus, Permission, NetworkSlot, - RawJSONtoTextParser, add_json_text, add_json_location, add_json_item, JSONTypes, SlotType) + RawJSONtoTextParser, add_json_text, add_json_location, add_json_item, JSONTypes, HintStatus, SlotType) from Utils import Version, stream_input, async_start from worlds import network_data_package, AutoWorldRegister import os @@ -31,6 +31,7 @@ import ssl if typing.TYPE_CHECKING: import kvui + import argparse logger = logging.getLogger("Client") @@ -412,10 +413,11 @@ class CommonContext: await self.server.socket.close() if self.server_task is not None: await self.server_task + self.ui.update_hints() async def send_msgs(self, msgs: typing.List[typing.Any]) -> None: """ `msgs` JSON serializable """ - if not self.server or not self.server.socket.open or self.server.socket.closed: + if not self.server or self.server.socket.state != websockets.protocol.State.OPEN: return await self.server.socket.send(encode(msgs)) @@ -458,6 +460,13 @@ class CommonContext: await self.send_msgs([payload]) await self.send_msgs([{"cmd": "Get", "keys": ["_read_race_mode"]}]) + async def check_locations(self, locations: typing.Collection[int]) -> set[int]: + """Send new location checks to the server. Returns the set of actually new locations that were sent.""" + locations = set(locations) & self.missing_locations + if locations: + await self.send_msgs([{"cmd": 'LocationChecks', "locations": tuple(locations)}]) + return locations + async def console_input(self) -> str: if self.ui: self.ui.focus_textinput() @@ -551,7 +560,14 @@ class CommonContext: await self.ui_task if self.input_task: self.input_task.cancel() - + + # Hints + def update_hint(self, location: int, finding_player: int, status: typing.Optional[HintStatus]) -> None: + msg = {"cmd": "UpdateHint", "location": location, "player": finding_player} + if status is not None: + msg["status"] = status + async_start(self.send_msgs([msg]), name="update_hint") + # DataPackage async def prepare_data_package(self, relevant_games: typing.Set[str], remote_date_package_versions: typing.Dict[str, int], @@ -693,8 +709,16 @@ class CommonContext: logger.exception(msg, exc_info=exc_info, extra={'compact_gui': True}) self._messagebox_connection_loss = self.gui_error(msg, exc_info[1]) - def make_gui(self) -> typing.Type["kvui.GameManager"]: - """To return the Kivy App class needed for run_gui so it can be overridden before being built""" + def make_gui(self) -> "type[kvui.GameManager]": + """ + To return the Kivy `App` class needed for `run_gui` so it can be overridden before being built + + Common changes are changing `base_title` to update the window title of the client and + updating `logging_pairs` to automatically make new tabs that can be filled with their respective logger. + + ex. `logging_pairs.append(("Foo", "Bar"))` + will add a "Bar" tab which follows the logger returned from `logging.getLogger("Foo")` + """ from kvui import GameManager class TextManager(GameManager): @@ -883,6 +907,7 @@ async def process_server_cmd(ctx: CommonContext, args: dict): ctx.disconnected_intentionally = True ctx.event_invalid_game() elif 'IncompatibleVersion' in errors: + ctx.disconnected_intentionally = True raise Exception('Server reported your client version as incompatible. ' 'This probably means you have to update.') elif 'InvalidItemsHandling' in errors: @@ -1033,6 +1058,32 @@ def get_base_parser(description: typing.Optional[str] = None): return parser +def handle_url_arg(args: "argparse.Namespace", + parser: "typing.Optional[argparse.ArgumentParser]" = None) -> "argparse.Namespace": + """ + Parse the url arg "archipelago://name:pass@host:port" from launcher into correct launch args for CommonClient + If alternate data is required the urlparse response is saved back to args.url if valid + """ + if not args.url: + return args + + url = urllib.parse.urlparse(args.url) + if url.scheme != "archipelago": + if not parser: + parser = get_base_parser() + parser.error(f"bad url, found {args.url}, expected url in form of archipelago://archipelago.gg:38281") + return args + + args.url = url + args.connect = url.netloc + if url.username: + args.name = urllib.parse.unquote(url.username) + if url.password: + args.password = urllib.parse.unquote(url.password) + + return args + + def run_as_textclient(*args): class TextContext(CommonContext): # Text Mode to use !hint and such with games that have no text entry @@ -1045,7 +1096,7 @@ def run_as_textclient(*args): if password_requested and not self.password: await super(TextContext, self).server_auth(password_requested) await self.get_username() - await self.send_connect() + await self.send_connect(game="") def on_package(self, cmd: str, args: dict): if cmd == "Connected": @@ -1074,20 +1125,10 @@ def run_as_textclient(*args): parser.add_argument("url", nargs="?", help="Archipelago connection url") args = parser.parse_args(args) - # handle if text client is launched using the "archipelago://name:pass@host:port" url from webhost - if args.url: - url = urllib.parse.urlparse(args.url) - if url.scheme == "archipelago": - args.connect = url.netloc - if url.username: - args.name = urllib.parse.unquote(url.username) - if url.password: - args.password = urllib.parse.unquote(url.password) - else: - parser.error(f"bad url, found {args.url}, expected url in form of archipelago://archipelago.gg:38281") + args = handle_url_arg(args, parser=parser) # use colorama to display colored text highlighting on windows - colorama.init() + colorama.just_fix_windows_console() asyncio.run(main(args)) colorama.deinit() diff --git a/FF1Client.py b/FF1Client.py index b7c58e2061..748a95b72c 100644 --- a/FF1Client.py +++ b/FF1Client.py @@ -261,7 +261,7 @@ if __name__ == '__main__': parser = get_base_parser() args = parser.parse_args() - colorama.init() + colorama.just_fix_windows_console() asyncio.run(main(args)) colorama.deinit() diff --git a/Fill.py b/Fill.py index 706cca6574..d1773c8213 100644 --- a/Fill.py +++ b/Fill.py @@ -36,7 +36,8 @@ def sweep_from_pool(base_state: CollectionState, itempool: typing.Sequence[Item] def fill_restrictive(multiworld: MultiWorld, base_state: CollectionState, locations: typing.List[Location], item_pool: typing.List[Item], single_player_placement: bool = False, lock: bool = False, swap: bool = True, on_place: typing.Optional[typing.Callable[[Location], None]] = None, - allow_partial: bool = False, allow_excluded: bool = False, name: str = "Unknown") -> None: + allow_partial: bool = False, allow_excluded: bool = False, one_item_per_player: bool = True, + name: str = "Unknown") -> None: """ :param multiworld: Multiworld to be filled. :param base_state: State assumed before fill. @@ -63,14 +64,22 @@ def fill_restrictive(multiworld: MultiWorld, base_state: CollectionState, locati placed = 0 while any(reachable_items.values()) and locations: - # grab one item per player - items_to_place = [items.pop() - for items in reachable_items.values() if items] + if one_item_per_player: + # grab one item per player + items_to_place = [items.pop() + for items in reachable_items.values() if items] + else: + next_player = multiworld.random.choice([player for player, items in reachable_items.items() if items]) + items_to_place = [] + if item_pool: + items_to_place.append(reachable_items[next_player].pop()) + for item in items_to_place: for p, pool_item in enumerate(item_pool): if pool_item is item: item_pool.pop(p) break + maximum_exploration_state = sweep_from_pool( base_state, item_pool + unplaced_items, multiworld.get_filled_locations(item.player) if single_player_placement else None) @@ -226,18 +235,30 @@ def remaining_fill(multiworld: MultiWorld, locations: typing.List[Location], itempool: typing.List[Item], name: str = "Remaining", - move_unplaceable_to_start_inventory: bool = False) -> None: + move_unplaceable_to_start_inventory: bool = False, + check_location_can_fill: bool = False) -> None: unplaced_items: typing.List[Item] = [] placements: typing.List[Location] = [] swapped_items: typing.Counter[typing.Tuple[int, str]] = Counter() total = min(len(itempool), len(locations)) placed = 0 + + # Optimisation: Decide whether to do full location.can_fill check (respect excluded), or only check the item rule + if check_location_can_fill: + state = CollectionState(multiworld) + + def location_can_fill_item(location_to_fill: Location, item_to_fill: Item): + return location_to_fill.can_fill(state, item_to_fill, check_access=False) + else: + def location_can_fill_item(location_to_fill: Location, item_to_fill: Item): + return location_to_fill.item_rule(item_to_fill) + while locations and itempool: item_to_place = itempool.pop() spot_to_fill: typing.Optional[Location] = None for i, location in enumerate(locations): - if location.item_rule(item_to_place): + if location_can_fill_item(location, item_to_place): # popping by index is faster than removing by content, spot_to_fill = locations.pop(i) # skipping a scan for the element @@ -258,7 +279,7 @@ def remaining_fill(multiworld: MultiWorld, location.item = None placed_item.location = None - if location.item_rule(item_to_place): + if location_can_fill_item(location, item_to_place): # Add this item to the existing placement, and # add the old item to the back of the queue spot_to_fill = placements.pop(i) @@ -480,7 +501,14 @@ def distribute_items_restrictive(multiworld: MultiWorld, if prioritylocations: # "priority fill" fill_restrictive(multiworld, multiworld.state, prioritylocations, progitempool, - single_player_placement=single_player, swap=False, on_place=mark_for_locking, name="Priority") + single_player_placement=single_player, swap=False, on_place=mark_for_locking, + name="Priority", one_item_per_player=True, allow_partial=True) + + if prioritylocations: + # retry with one_item_per_player off because some priority fills can fail to fill with that optimization + fill_restrictive(multiworld, multiworld.state, prioritylocations, progitempool, + single_player_placement=single_player, swap=False, on_place=mark_for_locking, + name="Priority Retry", one_item_per_player=False) accessibility_corrections(multiworld, multiworld.state, prioritylocations, progitempool) defaultlocations = prioritylocations + defaultlocations @@ -509,7 +537,8 @@ def distribute_items_restrictive(multiworld: MultiWorld, if progitempool: raise FillError( f"Not enough locations for progression items. " - f"There are {len(progitempool)} more progression items than there are available locations.", + f"There are {len(progitempool)} more progression items than there are available locations.\n" + f"Unfilled locations:\n{multiworld.get_unfilled_locations()}.", multiworld=multiworld, ) accessibility_corrections(multiworld, multiworld.state, defaultlocations) @@ -527,7 +556,7 @@ def distribute_items_restrictive(multiworld: MultiWorld, if excludedlocations: raise FillError( f"Not enough filler items for excluded locations. " - f"There are {len(excludedlocations)} more excluded locations than filler or trap items.", + f"There are {len(excludedlocations)} more excluded locations than excludable items.", multiworld=multiworld, ) @@ -548,6 +577,26 @@ def distribute_items_restrictive(multiworld: MultiWorld, print_data = {"items": items_counter, "locations": locations_counter} logging.info(f"Per-Player counts: {print_data})") + more_locations = locations_counter - items_counter + more_items = items_counter - locations_counter + for player in multiworld.player_ids: + if more_locations[player]: + logging.error( + f"Player {multiworld.get_player_name(player)} had {more_locations[player]} more locations than items.") + elif more_items[player]: + logging.warning( + f"Player {multiworld.get_player_name(player)} had {more_items[player]} more items than locations.") + if unfilled: + raise FillError( + f"Unable to fill all locations.\n" + + f"Unfilled locations({len(unfilled)}): {unfilled}" + ) + else: + logging.warning( + f"Unable to place all items.\n" + + f"Unplaced items({len(unplaced)}): {unplaced}" + ) + def flood_items(multiworld: MultiWorld) -> None: # get items to distribute @@ -978,15 +1027,32 @@ def distribute_planned(multiworld: MultiWorld) -> None: multiworld.random.shuffle(items) count = 0 err: typing.List[str] = [] - successful_pairs: typing.List[typing.Tuple[Item, Location]] = [] + successful_pairs: typing.List[typing.Tuple[int, Item, Location]] = [] + claimed_indices: typing.Set[typing.Optional[int]] = set() for item_name in items: - item = multiworld.worlds[player].create_item(item_name) + index_to_delete: typing.Optional[int] = None + if from_pool: + try: + # If from_pool, try to find an existing item with this name & player in the itempool and use it + index_to_delete, item = next( + (i, item) for i, item in enumerate(multiworld.itempool) + if item.player == player and item.name == item_name and i not in claimed_indices + ) + except StopIteration: + warn( + f"Could not remove {item_name} from pool for {multiworld.player_name[player]} as it's already missing from it.", + placement['force']) + item = multiworld.worlds[player].create_item(item_name) + else: + item = multiworld.worlds[player].create_item(item_name) + for location in reversed(candidates): if (location.address is None) == (item.code is None): # either both None or both not None if not location.item: if location.item_rule(item): if location.can_fill(multiworld.state, item, False): - successful_pairs.append((item, location)) + successful_pairs.append((index_to_delete, item, location)) + claimed_indices.add(index_to_delete) candidates.remove(location) count = count + 1 break @@ -998,6 +1064,7 @@ def distribute_planned(multiworld: MultiWorld) -> None: err.append(f"Cannot place {item_name} into already filled location {location}.") else: err.append(f"Mismatch between {item_name} and {location}, only one is an event.") + if count == maxcount: break if count < placement['count']['min']: @@ -1005,17 +1072,16 @@ def distribute_planned(multiworld: MultiWorld) -> None: failed( f"Plando block failed to place {m - count} of {m} item(s) for {multiworld.player_name[player]}, error(s): {' '.join(err)}", placement['force']) - for (item, location) in successful_pairs: + + # Sort indices in reverse so we can remove them one by one + successful_pairs = sorted(successful_pairs, key=lambda successful_pair: successful_pair[0] or 0, reverse=True) + + for (index, item, location) in successful_pairs: multiworld.push_item(location, item, collect=False) location.locked = True logging.debug(f"Plando placed {item} at {location}") - if from_pool: - try: - multiworld.itempool.remove(item) - except ValueError: - warn( - f"Could not remove {item} from pool for {multiworld.player_name[player]} as it's already missing from it.", - placement['force']) + if index is not None: # If this item is from_pool and was found in the pool, remove it. + multiworld.itempool.pop(index) except Exception as e: raise Exception( diff --git a/Generate.py b/Generate.py index 8aba72abaf..b057db25a3 100644 --- a/Generate.py +++ b/Generate.py @@ -42,7 +42,9 @@ def mystery_argparse(): help="Path to output folder. Absolute or relative to cwd.") # absolute or relative to cwd parser.add_argument('--race', action='store_true', default=defaults.race) parser.add_argument('--meta_file_path', default=defaults.meta_file_path) - parser.add_argument('--log_level', default='info', help='Sets log level') + parser.add_argument('--log_level', default=defaults.loglevel, help='Sets log level') + parser.add_argument('--log_time', help="Add timestamps to STDOUT", + default=defaults.logtime, action='store_true') parser.add_argument("--csv_output", action="store_true", help="Output rolled player options to csv (made for async multiworld).") parser.add_argument("--plando", default=defaults.plando_options, @@ -75,7 +77,7 @@ def main(args=None) -> Tuple[argparse.Namespace, int]: seed = get_seed(args.seed) - Utils.init_logging(f"Generate_{seed}", loglevel=args.log_level) + Utils.init_logging(f"Generate_{seed}", loglevel=args.log_level, add_timestamp=args.log_time) random.seed(seed) seed_name = get_seed_name(random) @@ -114,7 +116,14 @@ def main(args=None) -> Tuple[argparse.Namespace, int]: os.path.join(args.player_files_path, fname) not in {args.meta_file_path, args.weights_file_path}: path = os.path.join(args.player_files_path, fname) try: - weights_cache[fname] = read_weights_yamls(path) + weights_for_file = [] + for doc_idx, yaml in enumerate(read_weights_yamls(path)): + if yaml is None: + logging.warning(f"Ignoring empty yaml document #{doc_idx + 1} in {fname}") + else: + weights_for_file.append(yaml) + weights_cache[fname] = tuple(weights_for_file) + except Exception as e: raise ValueError(f"File {fname} is invalid. Please fix your yaml.") from e @@ -431,7 +440,7 @@ def roll_settings(weights: dict, plando_options: PlandoOptions = PlandoOptions.b if "linked_options" in weights: weights = roll_linked_options(weights) - valid_keys = set() + valid_keys = {"triggers"} if "triggers" in weights: weights = roll_triggers(weights, weights["triggers"], valid_keys) @@ -490,15 +499,23 @@ def roll_settings(weights: dict, plando_options: PlandoOptions = PlandoOptions.b for option_key, option in world_type.options_dataclass.type_hints.items(): handle_option(ret, game_weights, option_key, option, plando_options) valid_keys.add(option_key) - for option_key in game_weights: - if option_key in {"triggers", *valid_keys}: - continue - logging.warning(f"{option_key} is not a valid option name for {ret.game} and is not present in triggers.") + + # TODO remove plando_items after moving it to the options system + valid_keys.add("plando_items") if PlandoOptions.items in plando_options: ret.plando_items = copy.deepcopy(game_weights.get("plando_items", [])) if ret.game == "A Link to the Past": + # TODO there are still more LTTP options not on the options system + valid_keys |= {"sprite_pool", "sprite", "random_sprite_on_event"} roll_alttp_settings(ret, game_weights) + # log a warning for options within a game section that aren't determined as valid + for option_key in game_weights: + if option_key in valid_keys: + continue + logging.warning(f"{option_key} is not a valid option name for {ret.game} and is not present in triggers " + f"for player {ret.name}.") + return ret diff --git a/LICENSE b/LICENSE index 40716cff42..60d31b7b7d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,7 @@ MIT License Copyright (c) 2017 LLCoolDave -Copyright (c) 2022 Berserker66 +Copyright (c) 2025 Berserker66 Copyright (c) 2022 CaitSith2 Copyright (c) 2021 LegendaryLinux diff --git a/Launcher.py b/Launcher.py index f04d67a5aa..22c0944ab1 100644 --- a/Launcher.py +++ b/Launcher.py @@ -126,12 +126,13 @@ def handle_uri(path: str, launch_args: Tuple[str, ...]) -> None: elif component.display_name == "Text Client": text_client_component = component - from kvui import App, Button, BoxLayout, Label, Clock, Window + if client_component is None: + run_component(text_client_component, *launch_args) + return + + from kvui import App, Button, BoxLayout, Label, Window class Popup(App): - timer_label: Label - remaining_time: Optional[int] - def __init__(self): self.title = "Connect to Multiworld" self.icon = r"data/icon.png" @@ -139,48 +140,25 @@ def handle_uri(path: str, launch_args: Tuple[str, ...]) -> None: def build(self): layout = BoxLayout(orientation="vertical") + layout.add_widget(Label(text="Select client to open and connect with.")) + button_row = BoxLayout(orientation="horizontal", size_hint=(1, 0.4)) - if client_component is None: - self.remaining_time = 7 - label_text = (f"A game client able to parse URIs was not detected for {game}.\n" - f"Launching Text Client in 7 seconds...") - self.timer_label = Label(text=label_text) - layout.add_widget(self.timer_label) - Clock.schedule_interval(self.update_label, 1) - else: - layout.add_widget(Label(text="Select client to open and connect with.")) - button_row = BoxLayout(orientation="horizontal", size_hint=(1, 0.4)) + text_client_button = Button( + text=text_client_component.display_name, + on_release=lambda *args: run_component(text_client_component, *launch_args) + ) + button_row.add_widget(text_client_button) - text_client_button = Button( - text=text_client_component.display_name, - on_release=lambda *args: run_component(text_client_component, *launch_args) - ) - button_row.add_widget(text_client_button) + game_client_button = Button( + text=client_component.display_name, + on_release=lambda *args: run_component(client_component, *launch_args) + ) + button_row.add_widget(game_client_button) - game_client_button = Button( - text=client_component.display_name, - on_release=lambda *args: run_component(client_component, *launch_args) - ) - button_row.add_widget(game_client_button) - - layout.add_widget(button_row) + layout.add_widget(button_row) return layout - def update_label(self, dt): - if self.remaining_time > 1: - # countdown the timer and string replace the number - self.remaining_time -= 1 - self.timer_label.text = self.timer_label.text.replace( - str(self.remaining_time + 1), str(self.remaining_time) - ) - else: - # our timer is finished so launch text client and close down - run_component(text_client_component, *launch_args) - Clock.unschedule(self.update_label) - App.get_running_app().stop() - Window.close() - def _stop(self, *largs): # see run_gui Launcher _stop comment for details self.root_window.close() @@ -246,9 +224,8 @@ refresh_components: Optional[Callable[[], None]] = None def run_gui(): - from kvui import App, ContainerLayout, GridLayout, Button, Label, ScrollBox, Widget + from kvui import App, ContainerLayout, GridLayout, Button, Label, ScrollBox, Widget, ApAsyncImage from kivy.core.window import Window - from kivy.uix.image import AsyncImage from kivy.uix.relativelayout import RelativeLayout class Launcher(App): @@ -281,8 +258,8 @@ def run_gui(): button.component = component button.bind(on_release=self.component_action) if component.icon != "icon": - image = AsyncImage(source=icon_paths[component.icon], - size=(38, 38), size_hint=(None, 1), pos=(5, 0)) + image = ApAsyncImage(source=icon_paths[component.icon], + size=(38, 38), size_hint=(None, 1), pos=(5, 0)) box_layout = RelativeLayout(size_hint_y=None, height=40) box_layout.add_widget(button) box_layout.add_widget(image) diff --git a/LinksAwakeningClient.py b/LinksAwakeningClient.py index 298788098d..aac6c2f214 100644 --- a/LinksAwakeningClient.py +++ b/LinksAwakeningClient.py @@ -28,6 +28,7 @@ from CommonClient import (CommonContext, get_base_parser, gui_enabled, logger, from NetUtils import ClientStatus from worlds.ladx.Common import BASE_ID as LABaseID from worlds.ladx.GpsTracker import GpsTracker +from worlds.ladx.TrackerConsts import storage_key from worlds.ladx.ItemTracker import ItemTracker from worlds.ladx.LADXR.checkMetadata import checkMetadataTable from worlds.ladx.Locations import get_locations_to_id, meta_to_name @@ -100,19 +101,23 @@ class LAClientConstants: WRamCheckSize = 0x4 WRamSafetyValue = bytearray([0]*WRamCheckSize) + wRamStart = 0xC000 + hRamStart = 0xFF80 + hRamSize = 0x80 + MinGameplayValue = 0x06 MaxGameplayValue = 0x1A VictoryGameplayAndSub = 0x0102 - class RAGameboy(): cache = [] - cache_start = 0 - cache_size = 0 last_cache_read = None socket = None def __init__(self, address, port) -> None: + self.cache_start = LAClientConstants.wRamStart + self.cache_size = LAClientConstants.hRamStart + LAClientConstants.hRamSize - LAClientConstants.wRamStart + self.address = address self.port = port self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) @@ -131,9 +136,14 @@ class RAGameboy(): async def get_retroarch_status(self): return await self.send_command("GET_STATUS") - def set_cache_limits(self, cache_start, cache_size): - self.cache_start = cache_start - self.cache_size = cache_size + def set_checks_range(self, checks_start, checks_size): + self.checks_start = checks_start + self.checks_size = checks_size + + def set_location_range(self, location_start, location_size, critical_addresses): + self.location_start = location_start + self.location_size = location_size + self.critical_location_addresses = critical_addresses def send(self, b): if type(b) is str: @@ -188,21 +198,57 @@ class RAGameboy(): if not await self.check_safe_gameplay(): return - cache = [] - remaining_size = self.cache_size - while remaining_size: - block = await self.async_read_memory(self.cache_start + len(cache), remaining_size) - remaining_size -= len(block) - cache += block + attempts = 0 + while True: + # RA doesn't let us do an atomic read of a large enough block of RAM + # Some bytes can't change in between reading location_block and hram_block + location_block = await self.read_memory_block(self.location_start, self.location_size) + hram_block = await self.read_memory_block(LAClientConstants.hRamStart, LAClientConstants.hRamSize) + verification_block = await self.read_memory_block(self.location_start, self.location_size) + + valid = True + for address in self.critical_location_addresses: + if location_block[address - self.location_start] != verification_block[address - self.location_start]: + valid = False + + if valid: + break + + attempts += 1 + + # Shouldn't really happen, but keep it from choking + if attempts > 5: + return + + checks_block = await self.read_memory_block(self.checks_start, self.checks_size) if not await self.check_safe_gameplay(): return - self.cache = cache + self.cache = bytearray(self.cache_size) + + start = self.checks_start - self.cache_start + self.cache[start:start + len(checks_block)] = checks_block + + start = self.location_start - self.cache_start + self.cache[start:start + len(location_block)] = location_block + + start = LAClientConstants.hRamStart - self.cache_start + self.cache[start:start + len(hram_block)] = hram_block + self.last_cache_read = time.time() + + async def read_memory_block(self, address: int, size: int): + block = bytearray() + remaining_size = size + while remaining_size: + chunk = await self.async_read_memory(address + len(block), remaining_size) + remaining_size -= len(chunk) + block += chunk + + return block async def read_memory_cache(self, addresses): - # TODO: can we just update once per frame? if not self.last_cache_read or self.last_cache_read + 0.1 < time.time(): await self.update_cache() if not self.cache: @@ -235,7 +281,7 @@ class RAGameboy(): def check_command_response(self, command: str, response: bytes): if command == "VERSION": - ok = re.match("\d+\.\d+\.\d+", response.decode('ascii')) is not None + ok = re.match(r"\d+\.\d+\.\d+", response.decode('ascii')) is not None else: ok = response.startswith(command.encode()) if not ok: @@ -359,11 +405,12 @@ class LinksAwakeningClient(): auth = binascii.hexlify(await self.gameboy.async_read_memory(0x0134, 12)).decode() self.auth = auth - async def wait_and_init_tracker(self): + async def wait_and_init_tracker(self, magpie: MagpieBridge): await self.wait_for_game_ready() self.tracker = LocationTracker(self.gameboy) self.item_tracker = ItemTracker(self.gameboy) self.gps_tracker = GpsTracker(self.gameboy) + magpie.gps_tracker = self.gps_tracker async def recved_item_from_ap(self, item_id, from_player, next_index): # Don't allow getting an item until you've got your first check @@ -405,9 +452,11 @@ class LinksAwakeningClient(): return (await self.gameboy.read_memory_cache([LAClientConstants.wGameplayType]))[LAClientConstants.wGameplayType] == 1 async def main_tick(self, item_get_cb, win_cb, deathlink_cb): + await self.gameboy.update_cache() await self.tracker.readChecks(item_get_cb) await self.item_tracker.readItems() await self.gps_tracker.read_location() + await self.gps_tracker.read_entrances() current_health = (await self.gameboy.read_memory_cache([LAClientConstants.wLinkHealth]))[LAClientConstants.wLinkHealth] if self.deathlink_debounce and current_health != 0: @@ -465,6 +514,10 @@ class LinksAwakeningContext(CommonContext): magpie_task = None won = False + @property + def slot_storage_key(self): + return f"{self.slot_info[self.slot].name}_{storage_key}" + def __init__(self, server_address: typing.Optional[str], password: typing.Optional[str], magpie: typing.Optional[bool]) -> None: self.client = LinksAwakeningClient() self.slot_data = {} @@ -507,7 +560,19 @@ class LinksAwakeningContext(CommonContext): self.ui_task = asyncio.create_task(self.ui.async_run(), name="UI") async def send_checks(self): - message = [{"cmd": 'LocationChecks', "locations": self.found_checks}] + message = [{"cmd": "LocationChecks", "locations": self.found_checks}] + await self.send_msgs(message) + + async def send_new_entrances(self, entrances: typing.Dict[str, str]): + # Store the entrances we find on the server for future sessions + message = [{ + "cmd": "Set", + "key": self.slot_storage_key, + "default": {}, + "want_reply": False, + "operations": [{"operation": "update", "value": entrances}], + }] + await self.send_msgs(message) had_invalid_slot_data = None @@ -536,6 +601,12 @@ class LinksAwakeningContext(CommonContext): logger.info("victory!") await self.send_msgs(message) self.won = True + + async def request_found_entrances(self): + await self.send_msgs([{"cmd": "Get", "keys": [self.slot_storage_key]}]) + + # Ask for updates so that players can co-op entrances in a seed + await self.send_msgs([{"cmd": "SetNotify", "keys": [self.slot_storage_key]}]) async def on_deathlink(self, data: typing.Dict[str, typing.Any]) -> None: if self.ENABLE_DEATHLINK: @@ -560,6 +631,10 @@ class LinksAwakeningContext(CommonContext): while self.client.auth == None: await asyncio.sleep(0.1) + + # Just return if we're closing + if self.exit_event.is_set(): + return self.auth = self.client.auth await self.send_connect() @@ -572,6 +647,12 @@ class LinksAwakeningContext(CommonContext): if cmd == "ReceivedItems": for index, item in enumerate(args["items"], start=args["index"]): self.client.recvd_checks[index] = item + + if cmd == "Retrieved" and self.magpie_enabled and self.slot_storage_key in args["keys"]: + self.client.gps_tracker.receive_found_entrances(args["keys"][self.slot_storage_key]) + + if cmd == "SetReply" and self.magpie_enabled and args["key"] == self.slot_storage_key: + self.client.gps_tracker.receive_found_entrances(args["value"]) async def sync(self): sync_msg = [{'cmd': 'Sync'}] @@ -585,6 +666,12 @@ class LinksAwakeningContext(CommonContext): checkMetadataTable[check.id])] for check in ladxr_checks] self.new_checks(checks, [check.id for check in ladxr_checks]) + for check in ladxr_checks: + if check.value and check.linkedItem: + linkedItem = check.linkedItem + if 'condition' not in linkedItem or linkedItem['condition'](self.slot_data): + self.client.item_tracker.setExtraItem(check.linkedItem['item'], check.linkedItem['qty']) + async def victory(): await self.send_victory() @@ -618,12 +705,20 @@ class LinksAwakeningContext(CommonContext): if not self.client.recvd_checks: await self.sync() - await self.client.wait_and_init_tracker() + await self.client.wait_and_init_tracker(self.magpie) + min_tick_duration = 0.1 + last_tick = time.time() while True: await self.client.main_tick(on_item_get, victory, deathlink) - await asyncio.sleep(0.1) + now = time.time() + tick_duration = now - last_tick + sleep_duration = max(min_tick_duration - tick_duration, 0) + await asyncio.sleep(sleep_duration) + + last_tick = now + if self.last_resend + 5.0 < now: self.last_resend = now await self.send_checks() @@ -631,8 +726,15 @@ class LinksAwakeningContext(CommonContext): try: self.magpie.set_checks(self.client.tracker.all_checks) await self.magpie.set_item_tracker(self.client.item_tracker) - await self.magpie.send_gps(self.client.gps_tracker) self.magpie.slot_data = self.slot_data + + if self.client.gps_tracker.needs_found_entrances: + await self.request_found_entrances() + self.client.gps_tracker.needs_found_entrances = False + + new_entrances = await self.magpie.send_gps(self.client.gps_tracker) + if new_entrances: + await self.send_new_entrances(new_entrances) except Exception: # Don't let magpie errors take out the client pass @@ -701,6 +803,6 @@ async def main(): await ctx.shutdown() if __name__ == '__main__': - colorama.init() + colorama.just_fix_windows_console() asyncio.run(main()) colorama.deinit() diff --git a/LttPAdjuster.py b/LttPAdjuster.py index 7e33a3d5ef..963557e8da 100644 --- a/LttPAdjuster.py +++ b/LttPAdjuster.py @@ -33,10 +33,15 @@ WINDOW_MIN_HEIGHT = 525 WINDOW_MIN_WIDTH = 425 class AdjusterWorld(object): + class AdjusterSubWorld(object): + def __init__(self, random): + self.random = random + def __init__(self, sprite_pool): import random self.sprite_pool = {1: sprite_pool} self.per_slot_randoms = {1: random} + self.worlds = {1: self.AdjusterSubWorld(random)} class ArgumentDefaultsHelpFormatter(argparse.RawTextHelpFormatter): diff --git a/MMBN3Client.py b/MMBN3Client.py index 140a98745c..4945d49221 100644 --- a/MMBN3Client.py +++ b/MMBN3Client.py @@ -370,7 +370,7 @@ if __name__ == "__main__": import colorama - colorama.init() + colorama.just_fix_windows_console() asyncio.run(main()) colorama.deinit() diff --git a/Main.py b/Main.py index 4008ca5e90..d0e7a7f879 100644 --- a/Main.py +++ b/Main.py @@ -148,50 +148,44 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No else: multiworld.worlds[1].options.non_local_items.value = set() multiworld.worlds[1].options.local_items.value = set() - + + AutoWorld.call_all(multiworld, "connect_entrances") AutoWorld.call_all(multiworld, "generate_basic") # remove starting inventory from pool items. # Because some worlds don't actually create items during create_items this has to be as late as possible. - if any(getattr(multiworld.worlds[player].options, "start_inventory_from_pool", None) for player in multiworld.player_ids): - new_items: List[Item] = [] - old_items: List[Item] = [] - depletion_pool: Dict[int, Dict[str, int]] = { - player: getattr(multiworld.worlds[player].options, - "start_inventory_from_pool", - StartInventoryPool({})).value.copy() - for player in multiworld.player_ids - } - for player, items in depletion_pool.items(): - player_world: AutoWorld.World = multiworld.worlds[player] - for count in items.values(): - for _ in range(count): - new_items.append(player_world.create_filler()) - target: int = sum(sum(items.values()) for items in depletion_pool.values()) - for i, item in enumerate(multiworld.itempool): - if depletion_pool[item.player].get(item.name, 0): - target -= 1 - depletion_pool[item.player][item.name] -= 1 - # quick abort if we have found all items - if not target: - old_items.extend(multiworld.itempool[i+1:]) - break - else: - old_items.append(item) + fallback_inventory = StartInventoryPool({}) + depletion_pool: Dict[int, Dict[str, int]] = { + player: getattr(multiworld.worlds[player].options, "start_inventory_from_pool", fallback_inventory).value.copy() + for player in multiworld.player_ids + } + target_per_player = { + player: sum(target_items.values()) for player, target_items in depletion_pool.items() if target_items + } - # leftovers? - if target: - for player, remaining_items in depletion_pool.items(): - remaining_items = {name: count for name, count in remaining_items.items() if count} - if remaining_items: - logger.warning(f"{multiworld.get_player_name(player)}" - f" is trying to remove items from their pool that don't exist: {remaining_items}") - # find all filler we generated for the current player and remove until it matches - removables = [item for item in new_items if item.player == player] - for _ in range(sum(remaining_items.values())): - new_items.remove(removables.pop()) - assert len(multiworld.itempool) == len(new_items + old_items), "Item Pool amounts should not change." - multiworld.itempool[:] = new_items + old_items + if target_per_player: + new_itempool: List[Item] = [] + + # Make new itempool with start_inventory_from_pool items removed + for item in multiworld.itempool: + if depletion_pool[item.player].get(item.name, 0): + depletion_pool[item.player][item.name] -= 1 + else: + new_itempool.append(item) + + # Create filler in place of the removed items, warn if any items couldn't be found in the multiworld itempool + for player, target in target_per_player.items(): + unfound_items = {item: count for item, count in depletion_pool[player].items() if count} + + if unfound_items: + player_name = multiworld.get_player_name(player) + logger.warning(f"{player_name} tried to remove items from their pool that don't exist: {unfound_items}") + + needed_items = target_per_player[player] - sum(unfound_items.values()) + new_itempool += [multiworld.worlds[player].create_filler() for _ in range(needed_items)] + + assert len(multiworld.itempool) == len(new_itempool), "Item Pool amounts should not change." + multiworld.itempool[:] = new_itempool multiworld.link_items() @@ -249,6 +243,7 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No def write_multidata(): import NetUtils + from NetUtils import HintStatus slot_data = {} client_versions = {} games = {} @@ -273,10 +268,10 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No for slot in multiworld.player_ids: slot_data[slot] = multiworld.worlds[slot].fill_slot_data() - def precollect_hint(location): + def precollect_hint(location: Location, auto_status: HintStatus): entrance = er_hint_data.get(location.player, {}).get(location.address, "") hint = NetUtils.Hint(location.item.player, location.player, location.address, - location.item.code, False, entrance, location.item.flags) + location.item.code, False, entrance, location.item.flags, auto_status) precollected_hints[location.player].add(hint) if location.item.player not in multiworld.groups: precollected_hints[location.item.player].add(hint) @@ -289,19 +284,22 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No if type(location.address) == int: assert location.item.code is not None, "item code None should be event, " \ "location.address should then also be None. Location: " \ - f" {location}" + f" {location}, Item: {location.item}" assert location.address not in locations_data[location.player], ( f"Locations with duplicate address. {location} and " f"{locations_data[location.player][location.address]}") locations_data[location.player][location.address] = \ location.item.code, location.item.player, location.item.flags + auto_status = HintStatus.HINT_AVOID if location.item.trap else HintStatus.HINT_PRIORITY if location.name in multiworld.worlds[location.player].options.start_location_hints: - precollect_hint(location) + if not location.item.trap: # Unspecified status for location hints, except traps + auto_status = HintStatus.HINT_UNSPECIFIED + precollect_hint(location, auto_status) elif location.item.name in multiworld.worlds[location.item.player].options.start_hints: - precollect_hint(location) + precollect_hint(location, auto_status) elif any([location.item.name in multiworld.worlds[player].options.start_hints for player in multiworld.groups.get(location.item.player, {}).get("players", [])]): - precollect_hint(location) + precollect_hint(location, auto_status) # embedded data package data_package = { @@ -313,11 +311,10 @@ def main(args, seed=None, baked_server_options: Optional[Dict[str, object]] = No # get spheres -> filter address==None -> skip empty spheres: List[Dict[int, Set[int]]] = [] - for sphere in multiworld.get_spheres(): + for sphere in multiworld.get_sendable_spheres(): current_sphere: Dict[int, Set[int]] = collections.defaultdict(set) for sphere_location in sphere: - if type(sphere_location.address) is int: - current_sphere[sphere_location.player].add(sphere_location.address) + current_sphere[sphere_location.player].add(sphere_location.address) if current_sphere: spheres.append(dict(current_sphere)) diff --git a/ModuleUpdate.py b/ModuleUpdate.py index f49182bb78..04cf25ea55 100644 --- a/ModuleUpdate.py +++ b/ModuleUpdate.py @@ -5,8 +5,15 @@ import multiprocessing import warnings -if sys.version_info < (3, 8, 6): - raise RuntimeError("Incompatible Python Version. 3.8.7+ is supported.") +if sys.platform in ("win32", "darwin") and sys.version_info < (3, 10, 11): + # Official micro version updates. This should match the number in docs/running from source.md. + raise RuntimeError(f"Incompatible Python Version found: {sys.version_info}. Official 3.10.15+ is supported.") +elif sys.platform in ("win32", "darwin") and sys.version_info < (3, 10, 15): + # There are known security issues, but no easy way to install fixed versions on Windows for testing. + warnings.warn(f"Python Version {sys.version_info} has security issues. Don't use in production.") +elif sys.version_info < (3, 10, 1): + # Other platforms may get security backports instead of micro updates, so the number is unreliable. + raise RuntimeError(f"Incompatible Python Version found: {sys.version_info}. 3.10.1+ is supported.") # don't run update if environment is frozen/compiled or if not the parent process (skip in subprocess) _skip_update = bool(getattr(sys, "frozen", False) or multiprocessing.parent_process()) diff --git a/MultiServer.py b/MultiServer.py index 847a0b281c..a860342f06 100644 --- a/MultiServer.py +++ b/MultiServer.py @@ -28,9 +28,11 @@ ModuleUpdate.update() if typing.TYPE_CHECKING: import ssl + from NetUtils import ServerConnection -import websockets import colorama +import websockets +from websockets.extensions.permessage_deflate import PerMessageDeflate try: # ponyorm is a requirement for webhost, not default server, so may not be importable from pony.orm.dbapiprovider import OperationalError @@ -41,10 +43,11 @@ import NetUtils import Utils from Utils import version_tuple, restricted_loads, Version, async_start, get_intended_text from NetUtils import Endpoint, ClientStatus, NetworkItem, decode, encode, NetworkPlayer, Permission, NetworkSlot, \ - SlotType, LocationStore + SlotType, LocationStore, Hint, HintStatus +from BaseClasses import ItemClassification min_client_version = Version(0, 1, 6) -colorama.init() +colorama.just_fix_windows_console() def remove_from_list(container, value): @@ -118,13 +121,14 @@ def get_saving_second(seed_name: str, interval: int = 60) -> int: class Client(Endpoint): version = Version(0, 0, 0) - tags: typing.List[str] = [] + tags: typing.List[str] remote_items: bool remote_start_inventory: bool no_items: bool no_locations: bool + no_text: bool - def __init__(self, socket: websockets.WebSocketServerProtocol, ctx: Context): + def __init__(self, socket: "ServerConnection", ctx: Context) -> None: super().__init__(socket) self.auth = False self.team = None @@ -174,6 +178,7 @@ class Context: "compatibility": int} # team -> slot id -> list of clients authenticated to slot. clients: typing.Dict[int, typing.Dict[int, typing.List[Client]]] + endpoints: list[Client] locations: LocationStore # typing.Dict[int, typing.Dict[int, typing.Tuple[int, int, int]]] location_checks: typing.Dict[typing.Tuple[int, int], typing.Set[int]] hints_used: typing.Dict[typing.Tuple[int, int], int] @@ -228,7 +233,7 @@ class Context: self.hint_cost = hint_cost self.location_check_points = location_check_points self.hints_used = collections.defaultdict(int) - self.hints: typing.Dict[team_slot, typing.Set[NetUtils.Hint]] = collections.defaultdict(set) + self.hints: typing.Dict[team_slot, typing.Set[Hint]] = collections.defaultdict(set) self.release_mode: str = release_mode self.remaining_mode: str = remaining_mode self.collect_mode: str = collect_mode @@ -319,7 +324,7 @@ class Context: # General networking async def send_msgs(self, endpoint: Endpoint, msgs: typing.Iterable[dict]) -> bool: - if not endpoint.socket or not endpoint.socket.open: + if not endpoint.socket or endpoint.socket.state != websockets.protocol.State.OPEN: return False msg = self.dumper(msgs) try: @@ -334,7 +339,7 @@ class Context: return True async def send_encoded_msgs(self, endpoint: Endpoint, msg: str) -> bool: - if not endpoint.socket or not endpoint.socket.open: + if not endpoint.socket or endpoint.socket.state != websockets.protocol.State.OPEN: return False try: await endpoint.socket.send(msg) @@ -350,7 +355,7 @@ class Context: async def broadcast_send_encoded_msgs(self, endpoints: typing.Iterable[Endpoint], msg: str) -> bool: sockets = [] for endpoint in endpoints: - if endpoint.socket and endpoint.socket.open: + if endpoint.socket and endpoint.socket.state == websockets.protocol.State.OPEN: sockets.append(endpoint.socket) try: websockets.broadcast(sockets, msg) @@ -363,18 +368,28 @@ class Context: return True def broadcast_all(self, msgs: typing.List[dict]): - msgs = self.dumper(msgs) - endpoints = (endpoint for endpoint in self.endpoints if endpoint.auth) - async_start(self.broadcast_send_encoded_msgs(endpoints, msgs)) + msg_is_text = all(msg["cmd"] == "PrintJSON" for msg in msgs) + data = self.dumper(msgs) + endpoints = ( + endpoint + for endpoint in self.endpoints + if endpoint.auth and not (msg_is_text and endpoint.no_text) + ) + async_start(self.broadcast_send_encoded_msgs(endpoints, data)) def broadcast_text_all(self, text: str, additional_arguments: dict = {}): self.logger.info("Notice (all): %s" % text) self.broadcast_all([{**{"cmd": "PrintJSON", "data": [{ "text": text }]}, **additional_arguments}]) def broadcast_team(self, team: int, msgs: typing.List[dict]): - msgs = self.dumper(msgs) - endpoints = (endpoint for endpoint in itertools.chain.from_iterable(self.clients[team].values())) - async_start(self.broadcast_send_encoded_msgs(endpoints, msgs)) + msg_is_text = all(msg["cmd"] == "PrintJSON" for msg in msgs) + data = self.dumper(msgs) + endpoints = ( + endpoint + for endpoint in itertools.chain.from_iterable(self.clients[team].values()) + if not (msg_is_text and endpoint.no_text) + ) + async_start(self.broadcast_send_encoded_msgs(endpoints, data)) def broadcast(self, endpoints: typing.Iterable[Client], msgs: typing.List[dict]): msgs = self.dumper(msgs) @@ -388,13 +403,13 @@ class Context: await on_client_disconnected(self, endpoint) def notify_client(self, client: Client, text: str, additional_arguments: dict = {}): - if not client.auth: + if not client.auth or client.no_text: return self.logger.info("Notice (Player %s in team %d): %s" % (client.name, client.team + 1, text)) async_start(self.send_msgs(client, [{"cmd": "PrintJSON", "data": [{ "text": text }], **additional_arguments}])) def notify_client_multiple(self, client: Client, texts: typing.List[str], additional_arguments: dict = {}): - if not client.auth: + if not client.auth or client.no_text: return async_start(self.send_msgs(client, [{"cmd": "PrintJSON", "data": [{ "text": text }], **additional_arguments} @@ -443,7 +458,7 @@ class Context: self.slot_info = decoded_obj["slot_info"] self.games = {slot: slot_info.game for slot, slot_info in self.slot_info.items()} - self.groups = {slot: slot_info.group_members for slot, slot_info in self.slot_info.items() + self.groups = {slot: set(slot_info.group_members) for slot, slot_info in self.slot_info.items() if slot_info.type == SlotType.group} self.clients = {0: {}} @@ -656,13 +671,29 @@ class Context: return max(1, int(self.hint_cost * 0.01 * len(self.locations[slot]))) return 0 - def recheck_hints(self, team: typing.Optional[int] = None, slot: typing.Optional[int] = None): + def recheck_hints(self, team: typing.Optional[int] = None, slot: typing.Optional[int] = None, + changed: typing.Optional[typing.Set[team_slot]] = None) -> None: + """Refreshes the hints for the specified team/slot. Providing 'None' for either team or slot + will refresh all teams or all slots respectively. If a set is passed for 'changed', each (team,slot) + pair that has at least one hint modified will be added to the set. + """ for hint_team, hint_slot in self.hints: - if (team is None or team == hint_team) and (slot is None or slot == hint_slot): - self.hints[hint_team, hint_slot] = { - hint.re_check(self, hint_team) for hint in - self.hints[hint_team, hint_slot] - } + if team != hint_team and team is not None: + continue # Check specified team only, all if team is None + if slot != hint_slot and slot is not None: + continue # Check specified slot only, all if slot is None + new_hints: typing.Set[Hint] = set() + for hint in self.hints[hint_team, hint_slot]: + new_hint = hint.re_check(self, hint_team) + new_hints.add(new_hint) + if hint == new_hint: + continue + for player in self.slot_set(hint.receiving_player) | {hint.finding_player}: + if changed is not None: + changed.add((hint_team,player)) + if slot is not None and slot != player: + self.replace_hint(hint_team, player, hint, new_hint) + self.hints[hint_team, hint_slot] = new_hints def get_rechecked_hints(self, team: int, slot: int): self.recheck_hints(team, slot) @@ -711,7 +742,7 @@ class Context: else: return self.player_names[team, slot] - def notify_hints(self, team: int, hints: typing.List[NetUtils.Hint], only_new: bool = False, + def notify_hints(self, team: int, hints: typing.List[Hint], only_new: bool = False, recipients: typing.Sequence[int] = None): """Send and remember hints.""" if only_new: @@ -726,29 +757,41 @@ class Context: concerns[player].append(data) if not hint.local and data not in concerns[hint.finding_player]: concerns[hint.finding_player].append(data) - # remember hints in all cases - # since hints are bidirectional, finding player and receiving player, - # we can check once if hint already exists - if hint not in self.hints[team, hint.finding_player]: - self.hints[team, hint.finding_player].add(hint) - new_hint_events.add(hint.finding_player) - for player in self.slot_set(hint.receiving_player): - self.hints[team, player].add(hint) - new_hint_events.add(player) + # only remember hints that were not already found at the time of creation + if not hint.found: + # since hints are bidirectional, finding player and receiving player, + # we can check once if hint already exists + if hint not in self.hints[team, hint.finding_player]: + self.hints[team, hint.finding_player].add(hint) + new_hint_events.add(hint.finding_player) + for player in self.slot_set(hint.receiving_player): + self.hints[team, player].add(hint) + new_hint_events.add(player) self.logger.info("Notice (Team #%d): %s" % (team + 1, format_hint(self, team, hint))) for slot in new_hint_events: self.on_new_hint(team, slot) for slot, hint_data in concerns.items(): if recipients is None or slot in recipients: - clients = self.clients[team].get(slot) + clients = filter(lambda c: not c.no_text, self.clients[team].get(slot, [])) if not clients: continue client_hints = [datum[1] for datum in sorted(hint_data, key=lambda x: x[0].finding_player != slot)] for client in clients: async_start(self.send_msgs(client, client_hints)) + def get_hint(self, team: int, finding_player: int, seeked_location: int) -> typing.Optional[Hint]: + for hint in self.hints[team, finding_player]: + if hint.location == seeked_location and hint.finding_player == finding_player: + return hint + return None + + def replace_hint(self, team: int, slot: int, old_hint: Hint, new_hint: Hint) -> None: + if old_hint in self.hints[team, slot]: + self.hints[team, slot].remove(old_hint) + self.hints[team, slot].add(new_hint) + # "events" def on_goal_achieved(self, client: Client): @@ -790,7 +833,7 @@ def update_aliases(ctx: Context, team: int): async_start(ctx.send_encoded_msgs(client, cmd)) -async def server(websocket, path: str = "/", ctx: Context = None): +async def server(websocket: "ServerConnection", path: str = "/", ctx: Context = None) -> None: client = Client(websocket, ctx) ctx.endpoints.append(client) @@ -881,6 +924,10 @@ async def on_client_joined(ctx: Context, client: Client): "If your client supports it, " "you may have additional local commands you can list with /help.", {"type": "Tutorial"}) + if not any(isinstance(extension, PerMessageDeflate) for extension in client.socket.protocol.extensions): + ctx.notify_client(client, "Warning: your client does not support compressed websocket connections! " + "It may stop working in the future. If you are a player, please report this to the " + "client's developer.") ctx.client_connection_timers[client.team, client.slot] = datetime.datetime.now(datetime.timezone.utc) @@ -947,9 +994,13 @@ def get_status_string(ctx: Context, team: int, tag: str): tagged = len([client for client in ctx.clients[team][slot] if tag in client.tags]) completion_text = f"({len(ctx.location_checks[team, slot])}/{len(ctx.locations[slot])})" tag_text = f" {tagged} of which are tagged {tag}" if connected and tag else "" - goal_text = " and has finished." if ctx.client_game_state[team, slot] == ClientStatus.CLIENT_GOAL else "." + status_text = ( + " and has finished." if ctx.client_game_state[team, slot] == ClientStatus.CLIENT_GOAL else + " and is ready." if ctx.client_game_state[team, slot] == ClientStatus.CLIENT_READY else + "." + ) text += f"\n{ctx.get_aliased_name(team, slot)} has {connected} connection{'' if connected == 1 else 's'}" \ - f"{tag_text}{goal_text} {completion_text}" + f"{tag_text}{status_text} {completion_text}" return text @@ -1027,21 +1078,37 @@ def send_items_to(ctx: Context, team: int, target_slot: int, *items: NetworkItem def register_location_checks(ctx: Context, team: int, slot: int, locations: typing.Iterable[int], count_activity: bool = True): + slot_locations = ctx.locations[slot] new_locations = set(locations) - ctx.location_checks[team, slot] - new_locations.intersection_update(ctx.locations[slot]) # ignore location IDs unknown to this multidata + new_locations.intersection_update(slot_locations) # ignore location IDs unknown to this multidata if new_locations: if count_activity: ctx.client_activity_timers[team, slot] = datetime.datetime.now(datetime.timezone.utc) + + sortable: list[tuple[int, int, int, int]] = [] for location in new_locations: - item_id, target_player, flags = ctx.locations[slot][location] + # extract all fields to avoid runtime overhead in LocationStore + item_id, target_player, flags = slot_locations[location] + # sort/group by receiver and item + sortable.append((target_player, item_id, location, flags)) + + info_texts: list[dict[str, typing.Any]] = [] + for target_player, item_id, location, flags in sorted(sortable): new_item = NetworkItem(item_id, location, slot, flags) send_items_to(ctx, team, target_player, new_item) ctx.logger.info('(Team #%d) %s sent %s to %s (%s)' % ( team + 1, ctx.player_names[(team, slot)], ctx.item_names[ctx.slot_info[target_player].game][item_id], ctx.player_names[(team, target_player)], ctx.location_names[ctx.slot_info[slot].game][location])) - info_text = json_format_send_event(new_item, target_player) - ctx.broadcast_team(team, [info_text]) + if len(info_texts) >= 140: + # split into chunks that are close to compression window of 64K but not too big on the wire + # (roughly 1300-2600 bytes after compression depending on repetitiveness) + ctx.broadcast_team(team, info_texts) + info_texts.clear() + info_texts.append(json_format_send_event(new_item, target_player)) + ctx.broadcast_team(team, info_texts) + del info_texts + del sortable ctx.location_checks[team, slot] |= new_locations send_new_items(ctx) @@ -1050,14 +1117,15 @@ def register_location_checks(ctx: Context, team: int, slot: int, locations: typi "hint_points": get_slot_points(ctx, team, slot), "checked_locations": new_locations, # send back new checks only }]) - old_hints = ctx.hints[team, slot].copy() - ctx.recheck_hints(team, slot) - if old_hints != ctx.hints[team, slot]: - ctx.on_changed_hints(team, slot) + updated_slots: typing.Set[tuple[int, int]] = set() + ctx.recheck_hints(team, slot, updated_slots) + for hint_team, hint_slot in updated_slots: + ctx.on_changed_hints(hint_team, hint_slot) ctx.save() -def collect_hints(ctx: Context, team: int, slot: int, item: typing.Union[int, str]) -> typing.List[NetUtils.Hint]: +def collect_hints(ctx: Context, team: int, slot: int, item: typing.Union[int, str], auto_status: HintStatus) \ + -> typing.List[Hint]: hints = [] slots: typing.Set[int] = {slot} for group_id, group in ctx.groups.items(): @@ -1067,31 +1135,58 @@ def collect_hints(ctx: Context, team: int, slot: int, item: typing.Union[int, st seeked_item_id = item if isinstance(item, int) else ctx.item_names_for_game(ctx.games[slot])[item] for finding_player, location_id, item_id, receiving_player, item_flags \ in ctx.locations.find_item(slots, seeked_item_id): - found = location_id in ctx.location_checks[team, finding_player] - entrance = ctx.er_hint_data.get(finding_player, {}).get(location_id, "") - hints.append(NetUtils.Hint(receiving_player, finding_player, location_id, item_id, found, entrance, - item_flags)) + prev_hint = ctx.get_hint(team, finding_player, location_id) + if prev_hint: + hints.append(prev_hint) + else: + found = location_id in ctx.location_checks[team, finding_player] + entrance = ctx.er_hint_data.get(finding_player, {}).get(location_id, "") + new_status = auto_status + if found: + new_status = HintStatus.HINT_FOUND + elif item_flags & ItemClassification.trap: + new_status = HintStatus.HINT_AVOID + hints.append(Hint(receiving_player, finding_player, location_id, item_id, found, entrance, + item_flags, new_status)) return hints -def collect_hint_location_name(ctx: Context, team: int, slot: int, location: str) -> typing.List[NetUtils.Hint]: +def collect_hint_location_name(ctx: Context, team: int, slot: int, location: str, auto_status: HintStatus) \ + -> typing.List[Hint]: seeked_location: int = ctx.location_names_for_game(ctx.games[slot])[location] - return collect_hint_location_id(ctx, team, slot, seeked_location) + return collect_hint_location_id(ctx, team, slot, seeked_location, auto_status) -def collect_hint_location_id(ctx: Context, team: int, slot: int, seeked_location: int) -> typing.List[NetUtils.Hint]: +def collect_hint_location_id(ctx: Context, team: int, slot: int, seeked_location: int, auto_status: HintStatus) \ + -> typing.List[Hint]: + prev_hint = ctx.get_hint(team, slot, seeked_location) + if prev_hint: + return [prev_hint] result = ctx.locations[slot].get(seeked_location, (None, None, None)) if any(result): item_id, receiving_player, item_flags = result found = seeked_location in ctx.location_checks[team, slot] entrance = ctx.er_hint_data.get(slot, {}).get(seeked_location, "") - return [NetUtils.Hint(receiving_player, slot, seeked_location, item_id, found, entrance, item_flags)] + new_status = auto_status + if found: + new_status = HintStatus.HINT_FOUND + elif item_flags & ItemClassification.trap: + new_status = HintStatus.HINT_AVOID + return [Hint(receiving_player, slot, seeked_location, item_id, found, entrance, item_flags, + new_status)] return [] -def format_hint(ctx: Context, team: int, hint: NetUtils.Hint) -> str: +status_names: typing.Dict[HintStatus, str] = { + HintStatus.HINT_FOUND: "(found)", + HintStatus.HINT_UNSPECIFIED: "(unspecified)", + HintStatus.HINT_NO_PRIORITY: "(no priority)", + HintStatus.HINT_AVOID: "(avoid)", + HintStatus.HINT_PRIORITY: "(priority)", +} +def format_hint(ctx: Context, team: int, hint: Hint) -> str: text = f"[Hint]: {ctx.player_names[team, hint.receiving_player]}'s " \ f"{ctx.item_names[ctx.slot_info[hint.receiving_player].game][hint.item]} is " \ f"at {ctx.location_names[ctx.slot_info[hint.finding_player].game][hint.location]} " \ @@ -1099,7 +1194,8 @@ def format_hint(ctx: Context, team: int, hint: NetUtils.Hint) -> str: if hint.entrance: text += f" at {hint.entrance}" - return text + (". (found)" if hint.found else ".") + + return text + ". " + status_names.get(hint.status, "(unknown)") def json_format_send_event(net_item: NetworkItem, receiving_player: int): @@ -1503,7 +1599,7 @@ class ClientMessageProcessor(CommonCommandProcessor): def get_hints(self, input_text: str, for_location: bool = False) -> bool: points_available = get_client_points(self.ctx, self.client) cost = self.ctx.get_hint_cost(self.client.slot) - + auto_status = HintStatus.HINT_UNSPECIFIED if for_location else HintStatus.HINT_PRIORITY if not input_text: hints = {hint.re_check(self.ctx, self.client.team) for hint in self.ctx.hints[self.client.team, self.client.slot]} @@ -1529,9 +1625,9 @@ class ClientMessageProcessor(CommonCommandProcessor): self.output(f"Sorry, \"{hint_name}\" is marked as non-hintable.") hints = [] elif not for_location: - hints = collect_hints(self.ctx, self.client.team, self.client.slot, hint_id) + hints = collect_hints(self.ctx, self.client.team, self.client.slot, hint_id, auto_status) else: - hints = collect_hint_location_id(self.ctx, self.client.team, self.client.slot, hint_id) + hints = collect_hint_location_id(self.ctx, self.client.team, self.client.slot, hint_id, auto_status) else: game = self.ctx.games[self.client.slot] @@ -1551,16 +1647,16 @@ class ClientMessageProcessor(CommonCommandProcessor): hints = [] for item_name in self.ctx.item_name_groups[game][hint_name]: if item_name in self.ctx.item_names_for_game(game): # ensure item has an ID - hints.extend(collect_hints(self.ctx, self.client.team, self.client.slot, item_name)) + hints.extend(collect_hints(self.ctx, self.client.team, self.client.slot, item_name, auto_status)) elif not for_location and hint_name in self.ctx.item_names_for_game(game): # item name - hints = collect_hints(self.ctx, self.client.team, self.client.slot, hint_name) + hints = collect_hints(self.ctx, self.client.team, self.client.slot, hint_name, auto_status) elif hint_name in self.ctx.location_name_groups[game]: # location group name hints = [] for loc_name in self.ctx.location_name_groups[game][hint_name]: if loc_name in self.ctx.location_names_for_game(game): - hints.extend(collect_hint_location_name(self.ctx, self.client.team, self.client.slot, loc_name)) + hints.extend(collect_hint_location_name(self.ctx, self.client.team, self.client.slot, loc_name, auto_status)) else: # location name - hints = collect_hint_location_name(self.ctx, self.client.team, self.client.slot, hint_name) + hints = collect_hint_location_name(self.ctx, self.client.team, self.client.slot, hint_name, auto_status) else: self.output(response) @@ -1725,7 +1821,9 @@ async def process_client_cmd(ctx: Context, client: Client, args: dict): ctx.clients[team][slot].append(client) client.version = args['version'] client.tags = args['tags'] - client.no_locations = 'TextOnly' in client.tags or 'Tracker' in client.tags + client.no_locations = "TextOnly" in client.tags or "Tracker" in client.tags + # set NoText for old PopTracker clients that predate the tag to save traffic + client.no_text = "NoText" in client.tags or ("PopTracker" in client.tags and client.version < (0, 5, 1)) connected_packet = { "cmd": "Connected", "team": client.team, "slot": client.slot, @@ -1798,6 +1896,9 @@ async def process_client_cmd(ctx: Context, client: Client, args: dict): client.tags = args["tags"] if set(old_tags) != set(client.tags): client.no_locations = 'TextOnly' in client.tags or 'Tracker' in client.tags + client.no_text = "NoText" in client.tags or ( + "PopTracker" in client.tags and client.version < (0, 5, 1) + ) ctx.broadcast_text_all( f"{ctx.get_aliased_name(client.team, client.slot)} (Team #{client.team + 1}) has changed tags " f"from {old_tags} to {client.tags}.", @@ -1826,19 +1927,63 @@ async def process_client_cmd(ctx: Context, client: Client, args: dict): for location in args["locations"]: if type(location) is not int: await ctx.send_msgs(client, - [{'cmd': 'InvalidPacket', "type": "arguments", "text": 'LocationScouts', + [{'cmd': 'InvalidPacket', "type": "arguments", + "text": 'Locations has to be a list of integers', "original_cmd": cmd}]) return target_item, target_player, flags = ctx.locations[client.slot][location] if create_as_hint: - hints.extend(collect_hint_location_id(ctx, client.team, client.slot, location)) + hints.extend(collect_hint_location_id(ctx, client.team, client.slot, location, + HintStatus.HINT_UNSPECIFIED)) locs.append(NetworkItem(target_item, location, target_player, flags)) ctx.notify_hints(client.team, hints, only_new=create_as_hint == 2) if locs and create_as_hint: ctx.save() await ctx.send_msgs(client, [{'cmd': 'LocationInfo', 'locations': locs}]) - + + elif cmd == 'UpdateHint': + location = args["location"] + player = args["player"] + status = args["status"] + if not isinstance(player, int) or not isinstance(location, int) \ + or (status is not None and not isinstance(status, int)): + await ctx.send_msgs(client, + [{'cmd': 'InvalidPacket', "type": "arguments", "text": 'UpdateHint', + "original_cmd": cmd}]) + return + hint = ctx.get_hint(client.team, player, location) + if not hint: + return # Ignored safely + if client.slot not in ctx.slot_set(hint.receiving_player): + await ctx.send_msgs(client, + [{'cmd': 'InvalidPacket', "type": "arguments", "text": 'UpdateHint: No Permission', + "original_cmd": cmd}]) + return + new_hint = hint + if status is None: + return + try: + status = HintStatus(status) + except ValueError: + await ctx.send_msgs(client, + [{'cmd': 'InvalidPacket', "type": "arguments", + "text": 'UpdateHint: Invalid Status', "original_cmd": cmd}]) + return + if status == HintStatus.HINT_FOUND: + await ctx.send_msgs(client, + [{'cmd': 'InvalidPacket', "type": "arguments", + "text": 'UpdateHint: Cannot manually update status to "HINT_FOUND"', "original_cmd": cmd}]) + return + new_hint = new_hint.re_prioritize(ctx, status) + if hint == new_hint: + return + ctx.replace_hint(client.team, hint.finding_player, hint, new_hint) + ctx.replace_hint(client.team, hint.receiving_player, hint, new_hint) + ctx.save() + ctx.on_changed_hints(client.team, hint.finding_player) + ctx.on_changed_hints(client.team, hint.receiving_player) + elif cmd == 'StatusUpdate': update_client_status(ctx, client, args["status"]) @@ -1886,6 +2031,7 @@ async def process_client_cmd(ctx: Context, client: Client, args: dict): args["cmd"] = "SetReply" value = ctx.stored_data.get(args["key"], args.get("default", 0)) args["original_value"] = copy.copy(value) + args["slot"] = client.slot for operation in args["operations"]: func = modify_functions[operation["operation"]] value = func(value, operation["value"]) @@ -1961,7 +2107,7 @@ class ServerCommandProcessor(CommonCommandProcessor): def _cmd_exit(self) -> bool: """Shutdown the server""" try: - self.ctx.server.ws_server.close() + self.ctx.server.server.close() finally: self.ctx.exit_event.set() return True @@ -2143,9 +2289,9 @@ class ServerCommandProcessor(CommonCommandProcessor): hints = [] for item_name_from_group in self.ctx.item_name_groups[game][item]: if item_name_from_group in self.ctx.item_names_for_game(game): # ensure item has an ID - hints.extend(collect_hints(self.ctx, team, slot, item_name_from_group)) + hints.extend(collect_hints(self.ctx, team, slot, item_name_from_group, HintStatus.HINT_PRIORITY)) else: # item name or id - hints = collect_hints(self.ctx, team, slot, item) + hints = collect_hints(self.ctx, team, slot, item, HintStatus.HINT_PRIORITY) if hints: self.ctx.notify_hints(team, hints) @@ -2179,14 +2325,17 @@ class ServerCommandProcessor(CommonCommandProcessor): if usable: if isinstance(location, int): - hints = collect_hint_location_id(self.ctx, team, slot, location) + hints = collect_hint_location_id(self.ctx, team, slot, location, + HintStatus.HINT_UNSPECIFIED) elif game in self.ctx.location_name_groups and location in self.ctx.location_name_groups[game]: hints = [] for loc_name_from_group in self.ctx.location_name_groups[game][location]: if loc_name_from_group in self.ctx.location_names_for_game(game): - hints.extend(collect_hint_location_name(self.ctx, team, slot, loc_name_from_group)) + hints.extend(collect_hint_location_name(self.ctx, team, slot, loc_name_from_group, + HintStatus.HINT_UNSPECIFIED)) else: - hints = collect_hint_location_name(self.ctx, team, slot, location) + hints = collect_hint_location_name(self.ctx, team, slot, location, + HintStatus.HINT_UNSPECIFIED) if hints: self.ctx.notify_hints(team, hints) else: @@ -2276,6 +2425,8 @@ def parse_args() -> argparse.Namespace: parser.add_argument('--cert_key', help="Path to SSL Certificate Key file") parser.add_argument('--loglevel', default=defaults["loglevel"], choices=['debug', 'info', 'warning', 'error', 'critical']) + parser.add_argument('--logtime', help="Add timestamps to STDOUT", + default=defaults["logtime"], action='store_true') parser.add_argument('--location_check_points', default=defaults["location_check_points"], type=int) parser.add_argument('--hint_cost', default=defaults["hint_cost"], type=int) parser.add_argument('--disable_item_cheat', default=defaults["disable_item_cheat"], action='store_true') @@ -2326,7 +2477,7 @@ async def auto_shutdown(ctx, to_cancel=None): await asyncio.wait_for(ctx.exit_event.wait(), ctx.auto_shutdown) def inactivity_shutdown(): - ctx.server.ws_server.close() + ctx.server.server.close() ctx.exit_event.set() if to_cancel: for task in to_cancel: @@ -2356,7 +2507,9 @@ def load_server_cert(path: str, cert_key: typing.Optional[str]) -> "ssl.SSLConte async def main(args: argparse.Namespace): - Utils.init_logging("Server", loglevel=args.loglevel.lower()) + Utils.init_logging(name="Server", + loglevel=args.loglevel.lower(), + add_timestamp=args.logtime) ctx = Context(args.host, args.port, args.server_password, args.password, args.location_check_points, args.hint_cost, not args.disable_item_cheat, args.release_mode, args.collect_mode, diff --git a/NetUtils.py b/NetUtils.py index 4776b228db..f2ae2a63a0 100644 --- a/NetUtils.py +++ b/NetUtils.py @@ -5,11 +5,20 @@ import enum import warnings from json import JSONEncoder, JSONDecoder -import websockets +if typing.TYPE_CHECKING: + from websockets import WebSocketServerProtocol as ServerConnection from Utils import ByValue, Version +class HintStatus(ByValue, enum.IntEnum): + HINT_UNSPECIFIED = 0 + HINT_NO_PRIORITY = 10 + HINT_AVOID = 20 + HINT_PRIORITY = 30 + HINT_FOUND = 40 + + class JSONMessagePart(typing.TypedDict, total=False): text: str # optional @@ -19,6 +28,8 @@ class JSONMessagePart(typing.TypedDict, total=False): player: int # if type == item indicates item flags flags: int + # if type == hint_status + hint_status: HintStatus class ClientStatus(ByValue, enum.IntEnum): @@ -141,7 +152,7 @@ decode = JSONDecoder(object_hook=_object_hook).decode class Endpoint: - socket: websockets.WebSocketServerProtocol + socket: "ServerConnection" def __init__(self, socket): self.socket = socket @@ -184,6 +195,7 @@ class JSONTypes(str, enum.Enum): location_name = "location_name" location_id = "location_id" entrance_name = "entrance_name" + hint_status = "hint_status" class JSONtoTextParser(metaclass=HandlerMeta): @@ -224,7 +236,7 @@ class JSONtoTextParser(metaclass=HandlerMeta): def _handle_player_id(self, node: JSONMessagePart): player = int(node["text"]) - node["color"] = 'magenta' if player == self.ctx.slot else 'yellow' + node["color"] = 'magenta' if self.ctx.slot_concerns_self(player) else 'yellow' node["text"] = self.ctx.player_names[player] return self._handle_color(node) @@ -265,6 +277,10 @@ class JSONtoTextParser(metaclass=HandlerMeta): node["color"] = 'blue' return self._handle_color(node) + def _handle_hint_status(self, node: JSONMessagePart): + node["color"] = status_colors.get(node["hint_status"], "red") + return self._handle_color(node) + class RawJSONtoTextParser(JSONtoTextParser): def _handle_color(self, node: JSONMessagePart): @@ -297,6 +313,27 @@ def add_json_location(parts: list, location_id: int, player: int = 0, **kwargs) parts.append({"text": str(location_id), "player": player, "type": JSONTypes.location_id, **kwargs}) +status_names: typing.Dict[HintStatus, str] = { + HintStatus.HINT_FOUND: "(found)", + HintStatus.HINT_UNSPECIFIED: "(unspecified)", + HintStatus.HINT_NO_PRIORITY: "(no priority)", + HintStatus.HINT_AVOID: "(avoid)", + HintStatus.HINT_PRIORITY: "(priority)", +} +status_colors: typing.Dict[HintStatus, str] = { + HintStatus.HINT_FOUND: "green", + HintStatus.HINT_UNSPECIFIED: "white", + HintStatus.HINT_NO_PRIORITY: "slateblue", + HintStatus.HINT_AVOID: "salmon", + HintStatus.HINT_PRIORITY: "plum", +} + + +def add_json_hint_status(parts: list, hint_status: HintStatus, text: typing.Optional[str] = None, **kwargs): + parts.append({"text": text if text != None else status_names.get(hint_status, "(unknown)"), + "hint_status": hint_status, "type": JSONTypes.hint_status, **kwargs}) + + class Hint(typing.NamedTuple): receiving_player: int finding_player: int @@ -305,14 +342,21 @@ class Hint(typing.NamedTuple): found: bool entrance: str = "" item_flags: int = 0 + status: HintStatus = HintStatus.HINT_UNSPECIFIED def re_check(self, ctx, team) -> Hint: - if self.found: + if self.found and self.status == HintStatus.HINT_FOUND: return self found = self.location in ctx.location_checks[team, self.finding_player] if found: - return Hint(self.receiving_player, self.finding_player, self.location, self.item, found, self.entrance, - self.item_flags) + return self._replace(found=found, status=HintStatus.HINT_FOUND) + return self + + def re_prioritize(self, ctx, status: HintStatus) -> Hint: + if self.found and status != HintStatus.HINT_FOUND: + status = HintStatus.HINT_FOUND + if status != self.status: + return self._replace(status=status) return self def __hash__(self): @@ -334,10 +378,7 @@ class Hint(typing.NamedTuple): else: add_json_text(parts, "'s World") add_json_text(parts, ". ") - if self.found: - add_json_text(parts, "(found)", type="color", color="green") - else: - add_json_text(parts, "(not found)", type="color", color="red") + add_json_hint_status(parts, self.status) return {"cmd": "PrintJSON", "data": parts, "type": "Hint", "receiving": self.receiving_player, @@ -383,6 +424,8 @@ class _LocationStore(dict, typing.MutableMapping[int, typing.Dict[int, typing.Tu checked = state[team, slot] if not checked: # This optimizes the case where everyone connects to a fresh game at the same time. + if slot not in self: + raise KeyError(slot) return [] return [location_id for location_id in self[slot] if diff --git a/OoTAdjuster.py b/OoTAdjuster.py index 9519b191e7..1581d65398 100644 --- a/OoTAdjuster.py +++ b/OoTAdjuster.py @@ -1,7 +1,6 @@ import tkinter as tk import argparse import logging -import random import os import zipfile from itertools import chain @@ -197,7 +196,6 @@ def set_icon(window): def adjust(args): # Create a fake multiworld and OOTWorld to use as a base multiworld = MultiWorld(1) - multiworld.per_slot_randoms = {1: random} ootworld = OOTWorld(multiworld, 1) # Set options in the fake OOTWorld for name, option in chain(cosmetic_options.items(), sfx_options.items()): diff --git a/OoTClient.py b/OoTClient.py index 1154904173..6a87b9e722 100644 --- a/OoTClient.py +++ b/OoTClient.py @@ -346,7 +346,7 @@ if __name__ == '__main__': import colorama - colorama.init() + colorama.just_fix_windows_console() asyncio.run(main()) colorama.deinit() diff --git a/Options.py b/Options.py index 992348cb54..95b9b468c6 100644 --- a/Options.py +++ b/Options.py @@ -137,7 +137,7 @@ class Option(typing.Generic[T], metaclass=AssembleOptions): If this is False, the docstring is instead interpreted as plain text, and displayed as-is on the WebHost with whitespace preserved. - If this is None, it inherits the value of `World.rich_text_options_doc`. For + If this is None, it inherits the value of `WebWorld.rich_text_options_doc`. For backwards compatibility, this defaults to False, but worlds are encouraged to set it to True and use reStructuredText for their Option documentation. @@ -496,7 +496,7 @@ class TextChoice(Choice): def __init__(self, value: typing.Union[str, int]): assert isinstance(value, str) or isinstance(value, int), \ - f"{value} is not a valid option for {self.__class__.__name__}" + f"'{value}' is not a valid option for '{self.__class__.__name__}'" self.value = value @property @@ -617,17 +617,17 @@ class PlandoBosses(TextChoice, metaclass=BossMeta): used_locations.append(location) used_bosses.append(boss) if not cls.valid_boss_name(boss): - raise ValueError(f"{boss.title()} is not a valid boss name.") + raise ValueError(f"'{boss.title()}' is not a valid boss name.") if not cls.valid_location_name(location): - raise ValueError(f"{location.title()} is not a valid boss location name.") + raise ValueError(f"'{location.title()}' is not a valid boss location name.") if not cls.can_place_boss(boss, location): - raise ValueError(f"{location.title()} is not a valid location for {boss.title()} to be placed.") + raise ValueError(f"'{location.title()}' is not a valid location for {boss.title()} to be placed.") else: if cls.duplicate_bosses: if not cls.valid_boss_name(option): - raise ValueError(f"{option} is not a valid boss name.") + raise ValueError(f"'{option}' is not a valid boss name.") else: - raise ValueError(f"{option.title()} is not formatted correctly.") + raise ValueError(f"'{option.title()}' is not formatted correctly.") @classmethod def can_place_boss(cls, boss: str, location: str) -> bool: @@ -689,9 +689,9 @@ class Range(NumericOption): @classmethod def weighted_range(cls, text) -> Range: if text == "random-low": - return cls(cls.triangular(cls.range_start, cls.range_end, cls.range_start)) + return cls(cls.triangular(cls.range_start, cls.range_end, 0.0)) elif text == "random-high": - return cls(cls.triangular(cls.range_start, cls.range_end, cls.range_end)) + return cls(cls.triangular(cls.range_start, cls.range_end, 1.0)) elif text == "random-middle": return cls(cls.triangular(cls.range_start, cls.range_end)) elif text.startswith("random-range-"): @@ -717,11 +717,11 @@ class Range(NumericOption): f"{random_range[0]}-{random_range[1]} is outside allowed range " f"{cls.range_start}-{cls.range_end} for option {cls.__name__}") if text.startswith("random-range-low"): - return cls(cls.triangular(random_range[0], random_range[1], random_range[0])) + return cls(cls.triangular(random_range[0], random_range[1], 0.0)) elif text.startswith("random-range-middle"): return cls(cls.triangular(random_range[0], random_range[1])) elif text.startswith("random-range-high"): - return cls(cls.triangular(random_range[0], random_range[1], random_range[1])) + return cls(cls.triangular(random_range[0], random_range[1], 1.0)) else: return cls(random.randint(random_range[0], random_range[1])) @@ -739,8 +739,16 @@ class Range(NumericOption): return str(self.value) @staticmethod - def triangular(lower: int, end: int, tri: typing.Optional[int] = None) -> int: - return int(round(random.triangular(lower, end, tri), 0)) + def triangular(lower: int, end: int, tri: float = 0.5) -> int: + """ + Integer triangular distribution for `lower` inclusive to `end` inclusive. + + Expects `lower <= end` and `0.0 <= tri <= 1.0`. The result of other inputs is undefined. + """ + # Use the continuous range [lower, end + 1) to produce an integer result in [lower, end]. + # random.triangular is actually [a, b] and not [a, b), so there is a very small chance of getting exactly b even + # when a != b, so ensure the result is never more than `end`. + return min(end, math.floor(random.triangular(0.0, 1.0, tri) * (end - lower + 1) + lower)) class NamedRange(Range): @@ -754,7 +762,7 @@ class NamedRange(Range): elif value > self.range_end and value not in self.special_range_names.values(): raise Exception(f"{value} is higher than maximum {self.range_end} for option {self.__class__.__name__} " + f"and is also not one of the supported named special values: {self.special_range_names}") - + # See docstring for key in self.special_range_names: if key != key.lower(): @@ -817,18 +825,21 @@ class VerifyKeys(metaclass=FreezeValidKeys): for item_name in self.value: if item_name not in world.item_names: picks = get_fuzzy_results(item_name, world.item_names, limit=1) - raise Exception(f"Item {item_name} from option {self} " - f"is not a valid item name from {world.game}. " + raise Exception(f"Item '{item_name}' from option '{self}' " + f"is not a valid item name from '{world.game}'. " f"Did you mean '{picks[0][0]}' ({picks[0][1]}% sure)") elif self.verify_location_name: for location_name in self.value: if location_name not in world.location_names: picks = get_fuzzy_results(location_name, world.location_names, limit=1) - raise Exception(f"Location {location_name} from option {self} " - f"is not a valid location name from {world.game}. " + raise Exception(f"Location '{location_name}' from option '{self}' " + f"is not a valid location name from '{world.game}'. " f"Did you mean '{picks[0][0]}' ({picks[0][1]}% sure)") + def __iter__(self) -> typing.Iterator[typing.Any]: + return self.value.__iter__() + class OptionDict(Option[typing.Dict[str, typing.Any]], VerifyKeys, typing.Mapping[str, typing.Any]): default = {} supports_weighting = False @@ -860,6 +871,8 @@ class ItemDict(OptionDict): verify_item_name = True def __init__(self, value: typing.Dict[str, int]): + if any(item_count is None for item_count in value.values()): + raise Exception("Items must have counts associated with them. Please provide positive integer values in the format \"item\": count .") if any(item_count < 1 for item_count in value.values()): raise Exception("Cannot have non-positive item counts.") super(ItemDict, self).__init__(value) @@ -1106,11 +1119,11 @@ class PlandoConnections(Option[typing.List[PlandoConnection]], metaclass=Connect used_entrances.append(entrance) used_exits.append(exit) if not cls.validate_entrance_name(entrance): - raise ValueError(f"{entrance.title()} is not a valid entrance.") + raise ValueError(f"'{entrance.title()}' is not a valid entrance.") if not cls.validate_exit_name(exit): - raise ValueError(f"{exit.title()} is not a valid exit.") + raise ValueError(f"'{exit.title()}' is not a valid exit.") if not cls.can_connect(entrance, exit): - raise ValueError(f"Connection between {entrance.title()} and {exit.title()} is invalid.") + raise ValueError(f"Connection between '{entrance.title()}' and '{exit.title()}' is invalid.") @classmethod def from_any(cls, data: PlandoConFromAnyType) -> Self: @@ -1175,7 +1188,7 @@ class PlandoConnections(Option[typing.List[PlandoConnection]], metaclass=Connect class Accessibility(Choice): """ Set rules for reachability of your items/locations. - + **Full:** ensure everything can be reached and acquired. **Minimal:** ensure what is needed to reach your goal can be acquired. @@ -1193,7 +1206,7 @@ class Accessibility(Choice): class ItemsAccessibility(Accessibility): """ Set rules for reachability of your items/locations. - + **Full:** ensure everything can be reached and acquired. **Minimal:** ensure what is needed to reach your goal can be acquired. @@ -1244,12 +1257,16 @@ class CommonOptions(metaclass=OptionsMetaProperty): progression_balancing: ProgressionBalancing accessibility: Accessibility - def as_dict(self, *option_names: str, casing: str = "snake") -> typing.Dict[str, typing.Any]: + def as_dict(self, + *option_names: str, + casing: typing.Literal["snake", "camel", "pascal", "kebab"] = "snake", + toggles_as_bools: bool = False) -> typing.Dict[str, typing.Any]: """ Returns a dictionary of [str, Option.value] :param option_names: names of the options to return :param casing: case of the keys to return. Supports `snake`, `camel`, `pascal`, `kebab` + :param toggles_as_bools: whether toggle options should be output as bools instead of strings """ assert option_names, "options.as_dict() was used without any option names." option_results = {} @@ -1271,6 +1288,8 @@ class CommonOptions(metaclass=OptionsMetaProperty): value = getattr(self, option_name).value if isinstance(value, set): value = sorted(value) + elif toggles_as_bools and issubclass(type(self).type_hints[option_name], Toggle): + value = bool(value) option_results[display_name] = value else: raise ValueError(f"{option_name} not found in {tuple(type(self).type_hints)}") @@ -1368,8 +1387,8 @@ class ItemLinks(OptionList): picks_group = get_fuzzy_results(item_name, world.item_name_groups.keys(), limit=1) picks_group = f" or '{picks_group[0][0]}' ({picks_group[0][1]}% sure)" if allow_item_groups else "" - raise Exception(f"Item {item_name} from item link {item_link} " - f"is not a valid item from {world.game} for {pool_name}. " + raise Exception(f"Item '{item_name}' from item link '{item_link}' " + f"is not a valid item from '{world.game}' for '{pool_name}'. " f"Did you mean '{picks[0][0]}' ({picks[0][1]}% sure){picks_group}") if allow_item_groups: pool |= world.item_name_groups.get(item_name, {item_name}) @@ -1460,22 +1479,26 @@ it. def get_option_groups(world: typing.Type[World], visibility_level: Visibility = Visibility.template) -> typing.Dict[ str, typing.Dict[str, typing.Type[Option[typing.Any]]]]: """Generates and returns a dictionary for the option groups of a specified world.""" - option_groups = {option: option_group.name - for option_group in world.web.option_groups - for option in option_group.options} + option_to_name = {option: option_name for option_name, option in world.options_dataclass.type_hints.items()} + + ordered_groups = {group.name: group.options for group in world.web.option_groups} + # add a default option group for uncategorized options to get thrown into - ordered_groups = ["Game Options"] - [ordered_groups.append(group) for group in option_groups.values() if group not in ordered_groups] - grouped_options = {group: {} for group in ordered_groups} - for option_name, option in world.options_dataclass.type_hints.items(): - if visibility_level & option.visibility: - grouped_options[option_groups.get(option, "Game Options")][option_name] = option + if "Game Options" not in ordered_groups: + grouped_options = set(option for group in ordered_groups.values() for option in group) + ungrouped_options = [option for option in option_to_name if option not in grouped_options] + # only add the game options group if we have ungrouped options + if ungrouped_options: + ordered_groups = {**{"Game Options": ungrouped_options}, **ordered_groups} - # if the world doesn't have any ungrouped options, this group will be empty so just remove it - if not grouped_options["Game Options"]: - del grouped_options["Game Options"] - - return grouped_options + return { + group: { + option_to_name[option]: option + for option in group_options + if (visibility_level in option.visibility and option in option_to_name) + } + for group, group_options in ordered_groups.items() + } def generate_yaml_templates(target_folder: typing.Union[str, "pathlib.Path"], generate_hidden: bool = True) -> None: @@ -1556,10 +1579,11 @@ def dump_player_options(multiworld: MultiWorld) -> None: player_output = { "Game": multiworld.game[player], "Name": multiworld.get_player_name(player), + "ID": player, } output.append(player_output) for option_key, option in world.options_dataclass.type_hints.items(): - if issubclass(Removed, option): + if option.visibility == Visibility.none: continue display_name = getattr(option, "display_name", option_key) player_output[display_name] = getattr(world.options, option_key).current_option_name @@ -1568,7 +1592,7 @@ def dump_player_options(multiworld: MultiWorld) -> None: game_option_names.append(display_name) with open(output_path(f"generate_{multiworld.seed_name}.csv"), mode="w", newline="") as file: - fields = ["Game", "Name", *all_option_names] + fields = ["ID", "Game", "Name", *all_option_names] writer = DictWriter(file, fields) writer.writeheader() writer.writerows(output) diff --git a/README.md b/README.md index 0e57bce53b..d119d560a7 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,11 @@ Currently, the following games are supported: * Kingdom Hearts 1 * Mega Man 2 * Yacht Dice +* Faxanadu +* Saving Princess +* Castlevania: Circle of the Moon +* Inscryption +* Civilization VI For setup and instructions check out our [tutorials page](https://archipelago.gg/tutorial/). Downloads can be found at [Releases](https://github.com/ArchipelagoMW/Archipelago/releases), including compiled diff --git a/SNIClient.py b/SNIClient.py index 19440e1dc5..1156bf6040 100644 --- a/SNIClient.py +++ b/SNIClient.py @@ -243,6 +243,9 @@ class SNIContext(CommonContext): # Once the games handled by SNIClient gets made to be remote items, # this will no longer be needed. async_start(self.send_msgs([{"cmd": "LocationScouts", "locations": list(new_locations)}])) + + if self.client_handler is not None: + self.client_handler.on_package(self, cmd, args) def run_gui(self) -> None: from kvui import GameManager @@ -732,6 +735,6 @@ async def main() -> None: if __name__ == '__main__': - colorama.init() + colorama.just_fix_windows_console() asyncio.run(main()) colorama.deinit() diff --git a/UndertaleClient.py b/UndertaleClient.py index dfacee148a..1c522fac92 100644 --- a/UndertaleClient.py +++ b/UndertaleClient.py @@ -500,7 +500,7 @@ def main(): import colorama - colorama.init() + colorama.just_fix_windows_console() asyncio.run(_main()) colorama.deinit() diff --git a/Utils.py b/Utils.py index 2dfcd9d3e1..c7f13f144d 100644 --- a/Utils.py +++ b/Utils.py @@ -19,8 +19,7 @@ import warnings from argparse import Namespace from settings import Settings, get_settings from time import sleep -from typing import BinaryIO, Coroutine, Optional, Set, Dict, Any, Union -from typing_extensions import TypeGuard +from typing import BinaryIO, Coroutine, Optional, Set, Dict, Any, Union, TypeGuard from yaml import load, load_all, dump try: @@ -48,7 +47,7 @@ class Version(typing.NamedTuple): return ".".join(str(item) for item in self) -__version__ = "0.5.1" +__version__ = "0.6.0" version_tuple = tuplize_version(__version__) is_linux = sys.platform.startswith("linux") @@ -153,8 +152,15 @@ def home_path(*path: str) -> str: if hasattr(home_path, 'cached_path'): pass elif sys.platform.startswith('linux'): - home_path.cached_path = os.path.expanduser('~/Archipelago') - os.makedirs(home_path.cached_path, 0o700, exist_ok=True) + xdg_data_home = os.getenv('XDG_DATA_HOME', os.path.expanduser('~/.local/share')) + home_path.cached_path = xdg_data_home + '/Archipelago' + if not os.path.isdir(home_path.cached_path): + legacy_home_path = os.path.expanduser('~/Archipelago') + if os.path.isdir(legacy_home_path): + os.renames(legacy_home_path, home_path.cached_path) + os.symlink(home_path.cached_path, legacy_home_path) + else: + os.makedirs(home_path.cached_path, 0o700, exist_ok=True) else: # not implemented home_path.cached_path = local_path() # this will generate the same exceptions we got previously @@ -422,7 +428,8 @@ class RestrictedUnpickler(pickle.Unpickler): if module == "builtins" and name in safe_builtins: return getattr(builtins, name) # used by MultiServer -> savegame/multidata - if module == "NetUtils" and name in {"NetworkItem", "ClientStatus", "Hint", "SlotType", "NetworkSlot"}: + if module == "NetUtils" and name in {"NetworkItem", "ClientStatus", "Hint", + "SlotType", "NetworkSlot", "HintStatus"}: return getattr(self.net_utils_module, name) # Options and Plando are unpickled by WebHost -> Generate if module == "worlds.generic" and name == "PlandoItem": @@ -436,7 +443,8 @@ class RestrictedUnpickler(pickle.Unpickler): else: mod = importlib.import_module(module) obj = getattr(mod, name) - if issubclass(obj, (self.options_module.Option, self.options_module.PlandoConnection)): + if issubclass(obj, (self.options_module.Option, self.options_module.PlandoConnection, + self.options_module.PlandoText)): return obj # Forbid everything else. raise pickle.UnpicklingError(f"global '{module}.{name}' is forbidden") @@ -485,9 +493,9 @@ def get_text_after(text: str, start: str) -> str: loglevel_mapping = {'error': logging.ERROR, 'info': logging.INFO, 'warning': logging.WARNING, 'debug': logging.DEBUG} -def init_logging(name: str, loglevel: typing.Union[str, int] = logging.INFO, write_mode: str = "w", - log_format: str = "[%(name)s at %(asctime)s]: %(message)s", - exception_logger: typing.Optional[str] = None): +def init_logging(name: str, loglevel: typing.Union[str, int] = logging.INFO, + write_mode: str = "w", log_format: str = "[%(name)s at %(asctime)s]: %(message)s", + add_timestamp: bool = False, exception_logger: typing.Optional[str] = None): import datetime loglevel: int = loglevel_mapping.get(loglevel, loglevel) log_folder = user_path("logs") @@ -514,11 +522,15 @@ def init_logging(name: str, loglevel: typing.Union[str, int] = logging.INFO, wri def filter(self, record: logging.LogRecord) -> bool: return self.condition(record) - file_handler.addFilter(Filter("NoStream", lambda record: not getattr(record, "NoFile", False))) + file_handler.addFilter(Filter("NoStream", lambda record: not getattr(record, "NoFile", False))) + file_handler.addFilter(Filter("NoCarriageReturn", lambda record: '\r' not in record.getMessage())) root_logger.addHandler(file_handler) if sys.stdout: + formatter = logging.Formatter(fmt='[%(asctime)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S') stream_handler = logging.StreamHandler(sys.stdout) stream_handler.addFilter(Filter("NoFile", lambda record: not getattr(record, "NoStream", False))) + if add_timestamp: + stream_handler.setFormatter(formatter) root_logger.addHandler(stream_handler) # Relay unhandled exceptions to logger. @@ -530,7 +542,8 @@ def init_logging(name: str, loglevel: typing.Union[str, int] = logging.INFO, wri sys.__excepthook__(exc_type, exc_value, exc_traceback) return logging.getLogger(exception_logger).exception("Uncaught exception", - exc_info=(exc_type, exc_value, exc_traceback)) + exc_info=(exc_type, exc_value, exc_traceback), + extra={"NoStream": exception_logger is None}) return orig_hook(exc_type, exc_value, exc_traceback) handle_exception._wrapped = True @@ -553,7 +566,7 @@ def init_logging(name: str, loglevel: typing.Union[str, int] = logging.INFO, wri import platform logging.info( f"Archipelago ({__version__}) logging initialized" - f" on {platform.platform()}" + f" on {platform.platform()} process {os.getpid()}" f" running Python {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" f"{' (frozen)' if is_frozen() else ''}" ) @@ -855,11 +868,10 @@ def async_start(co: Coroutine[None, None, typing.Any], name: Optional[str] = Non task.add_done_callback(_faf_tasks.discard) -def deprecate(message: str): +def deprecate(message: str, add_stacklevels: int = 0): if __debug__: raise Exception(message) - import warnings - warnings.warn(message) + warnings.warn(message, stacklevel=2 + add_stacklevels) class DeprecateDict(dict): @@ -873,10 +885,9 @@ class DeprecateDict(dict): def __getitem__(self, item: Any) -> Any: if self.should_error: - deprecate(self.log_message) + deprecate(self.log_message, add_stacklevels=1) elif __debug__: - import warnings - warnings.warn(self.log_message) + warnings.warn(self.log_message, stacklevel=2) return super().__getitem__(item) @@ -930,7 +941,7 @@ def freeze_support() -> None: def visualize_regions(root_region: Region, file_name: str, *, show_entrance_names: bool = False, show_locations: bool = True, show_other_regions: bool = True, - linetype_ortho: bool = True) -> None: + linetype_ortho: bool = True, regions_to_highlight: set[Region] | None = None) -> None: """Visualize the layout of a world as a PlantUML diagram. :param root_region: The region from which to start the diagram from. (Usually the "Menu" region of your world.) @@ -946,16 +957,22 @@ def visualize_regions(root_region: Region, file_name: str, *, Items without ID will be shown in italics. :param show_other_regions: (default True) If enabled, regions that can't be reached by traversing exits are shown. :param linetype_ortho: (default True) If enabled, orthogonal straight line parts will be used; otherwise polylines. + :param regions_to_highlight: Regions that will be highlighted in green if they are reachable. Example usage in World code: from Utils import visualize_regions - visualize_regions(self.multiworld.get_region("Menu", self.player), "my_world.puml") + state = self.multiworld.get_all_state(False) + state.update_reachable_regions(self.player) + visualize_regions(self.get_region("Menu"), "my_world.puml", show_entrance_names=True, + regions_to_highlight=state.reachable_regions[self.player]) Example usage in Main code: from Utils import visualize_regions for player in multiworld.player_ids: visualize_regions(multiworld.get_region("Menu", player), f"{multiworld.get_out_file_name_base(player)}.puml") """ + if regions_to_highlight is None: + regions_to_highlight = set() assert root_region.multiworld, "The multiworld attribute of root_region has to be filled" from BaseClasses import Entrance, Item, Location, LocationProgressType, MultiWorld, Region from collections import deque @@ -1008,7 +1025,7 @@ def visualize_regions(root_region: Region, file_name: str, *, uml.append(f"\"{fmt(region)}\" : {{field}} {lock}{fmt(location)}") def visualize_region(region: Region) -> None: - uml.append(f"class \"{fmt(region)}\"") + uml.append(f"class \"{fmt(region)}\" {'#00FF00' if region in regions_to_highlight else ''}") if show_locations: visualize_locations(region) visualize_exits(region) diff --git a/WargrooveClient.py b/WargrooveClient.py index f9971f7a6c..f900e05e3f 100644 --- a/WargrooveClient.py +++ b/WargrooveClient.py @@ -446,6 +446,6 @@ if __name__ == '__main__': parser = get_base_parser(description="Wargroove Client, for text interfacing.") args, rest = parser.parse_known_args() - colorama.init() + colorama.just_fix_windows_console() asyncio.run(main(args)) colorama.deinit() diff --git a/WebHost.py b/WebHost.py index 3bf75eb35a..768eeb5122 100644 --- a/WebHost.py +++ b/WebHost.py @@ -17,7 +17,7 @@ from Utils import get_file_safe_name if typing.TYPE_CHECKING: from flask import Flask -Utils.local_path.cached_path = os.path.dirname(__file__) or "." # py3.8 is not abs. remove "." when dropping 3.8 +Utils.local_path.cached_path = os.path.dirname(__file__) settings.no_gui = True configpath = os.path.abspath("config.yaml") if not os.path.exists(configpath): # fall back to config.yaml in home @@ -34,7 +34,7 @@ def get_app() -> "Flask": app.config.from_file(configpath, yaml.safe_load) logging.info(f"Updated config from {configpath}") # inside get_app() so it's usable in systems like gunicorn, which do not run WebHost.py, but import it. - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(allow_abbrev=False) parser.add_argument('--config_override', default=None, help="Path to yaml config file that overrules config.yaml.") args = parser.parse_known_args()[0] diff --git a/WebHostLib/__init__.py b/WebHostLib/__init__.py index e19492377d..789cc44c44 100644 --- a/WebHostLib/__init__.py +++ b/WebHostLib/__init__.py @@ -40,6 +40,8 @@ app.config["SECRET_KEY"] = bytes(socket.gethostname(), encoding="utf-8") app.config["JOB_THRESHOLD"] = 1 # after what time in seconds should generation be aborted, freeing the queue slot. Can be set to None to disable. app.config["JOB_TIME"] = 600 +# memory limit for generator processes in bytes +app.config["GENERATOR_MEMORY_LIMIT"] = 4294967296 app.config['SESSION_PERMANENT'] = True # waitress uses one thread for I/O, these are for processing of views that then get sent @@ -86,6 +88,6 @@ def register(): from WebHostLib.customserver import run_server_process # to trigger app routing picking up on it - from . import tracker, upload, landing, check, generate, downloads, api, stats, misc, robots, options + from . import tracker, upload, landing, check, generate, downloads, api, stats, misc, robots, options, session app.register_blueprint(api.api_endpoints) diff --git a/WebHostLib/api/__init__.py b/WebHostLib/api/__init__.py index cf05e87374..d0b9d05c16 100644 --- a/WebHostLib/api/__init__.py +++ b/WebHostLib/api/__init__.py @@ -3,13 +3,13 @@ from typing import List, Tuple from flask import Blueprint -from ..models import Seed +from ..models import Seed, Slot api_endpoints = Blueprint('api', __name__, url_prefix="/api") def get_players(seed: Seed) -> List[Tuple[str, str]]: - return [(slot.player_name, slot.game) for slot in seed.slots] + return [(slot.player_name, slot.game) for slot in seed.slots.order_by(Slot.player_id)] from . import datapackage, generate, room, user # trigger registration diff --git a/WebHostLib/api/user.py b/WebHostLib/api/user.py index 116d3afa22..0ddb6fe83e 100644 --- a/WebHostLib/api/user.py +++ b/WebHostLib/api/user.py @@ -30,4 +30,4 @@ def get_seeds(): "creation_time": seed.creation_time, "players": get_players(seed.slots), }) - return jsonify(response) \ No newline at end of file + return jsonify(response) diff --git a/WebHostLib/autolauncher.py b/WebHostLib/autolauncher.py index 1aaab6730e..60c613fb85 100644 --- a/WebHostLib/autolauncher.py +++ b/WebHostLib/autolauncher.py @@ -6,6 +6,7 @@ import multiprocessing import typing from datetime import timedelta, datetime from threading import Event, Thread +from typing import Any from uuid import UUID from pony.orm import db_session, select, commit @@ -53,7 +54,21 @@ def launch_generator(pool: multiprocessing.pool.Pool, generation: Generation): generation.state = STATE_STARTED -def init_db(pony_config: dict): +def init_generator(config: dict[str, Any]) -> None: + try: + import resource + except ModuleNotFoundError: + pass # unix only module + else: + # set soft limit for memory to from config (default 4GiB) + soft_limit = config["GENERATOR_MEMORY_LIMIT"] + old_limit, hard_limit = resource.getrlimit(resource.RLIMIT_AS) + if soft_limit != old_limit: + resource.setrlimit(resource.RLIMIT_AS, (soft_limit, hard_limit)) + logging.debug(f"Changed AS mem limit {old_limit} -> {soft_limit}") + del resource, soft_limit, hard_limit + + pony_config = config["PONY"] db.bind(**pony_config) db.generate_mapping() @@ -105,8 +120,8 @@ def autogen(config: dict): try: with Locker("autogen"): - with multiprocessing.Pool(config["GENERATORS"], initializer=init_db, - initargs=(config["PONY"],), maxtasksperchild=10) as generator_pool: + with multiprocessing.Pool(config["GENERATORS"], initializer=init_generator, + initargs=(config,), maxtasksperchild=10) as generator_pool: with db_session: to_start = select(generation for generation in Generation if generation.state == STATE_STARTED) diff --git a/WebHostLib/check.py b/WebHostLib/check.py index 97cb797f7a..4e0cf1178f 100644 --- a/WebHostLib/check.py +++ b/WebHostLib/check.py @@ -105,8 +105,9 @@ def roll_options(options: Dict[str, Union[dict, str]], plando_options=plando_options) else: for i, yaml_data in enumerate(yaml_datas): - rolled_results[f"{filename}/{i + 1}"] = roll_settings(yaml_data, - plando_options=plando_options) + if yaml_data is not None: + rolled_results[f"{filename}/{i + 1}"] = roll_settings(yaml_data, + plando_options=plando_options) except Exception as e: if e.__cause__: results[filename] = f"Failed to generate options in {filename}: {e} - {e.__cause__}" diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 12a28acc99..b7e6128213 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -118,6 +118,7 @@ class WebHostContext(Context): self.gamespackage = {"Archipelago": static_gamespackage.get("Archipelago", {})} # this may be modified by _load self.item_name_groups = {"Archipelago": static_item_name_groups.get("Archipelago", {})} self.location_name_groups = {"Archipelago": static_location_name_groups.get("Archipelago", {})} + missing_checksum = False for game in list(multidata.get("datapackage", {})): game_data = multidata["datapackage"][game] @@ -133,11 +134,13 @@ class WebHostContext(Context): continue else: self.logger.warning(f"Did not find game_data_package for {game}: {game_data['checksum']}") + else: + missing_checksum = True # Game rolled on old AP and will load data package from multidata self.gamespackage[game] = static_gamespackage.get(game, {}) self.item_name_groups[game] = static_item_name_groups.get(game, {}) self.location_name_groups[game] = static_location_name_groups.get(game, {}) - if not game_data_packages: + if not game_data_packages and not missing_checksum: # all static -> use the static dicts directly self.gamespackage = static_gamespackage self.item_name_groups = static_item_name_groups @@ -307,7 +310,7 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, await ctx.server port = 0 - for wssocket in ctx.server.ws_server.sockets: + for wssocket in ctx.server.server.sockets: socketname = wssocket.getsockname() if wssocket.family == socket.AF_INET6: # Prefer IPv4, as most users seem to not have working ipv6 support diff --git a/WebHostLib/generate.py b/WebHostLib/generate.py index b19f3d4835..0bd9f7e5e0 100644 --- a/WebHostLib/generate.py +++ b/WebHostLib/generate.py @@ -31,11 +31,11 @@ def get_meta(options_source: dict, race: bool = False) -> Dict[str, Union[List[s server_options = { "hint_cost": int(options_source.get("hint_cost", ServerOptions.hint_cost)), - "release_mode": options_source.get("release_mode", ServerOptions.release_mode), - "remaining_mode": options_source.get("remaining_mode", ServerOptions.remaining_mode), - "collect_mode": options_source.get("collect_mode", ServerOptions.collect_mode), + "release_mode": str(options_source.get("release_mode", ServerOptions.release_mode)), + "remaining_mode": str(options_source.get("remaining_mode", ServerOptions.remaining_mode)), + "collect_mode": str(options_source.get("collect_mode", ServerOptions.collect_mode)), "item_cheat": bool(int(options_source.get("item_cheat", not ServerOptions.disable_item_cheat))), - "server_password": options_source.get("server_password", None), + "server_password": str(options_source.get("server_password", None)), } generator_options = { "spoiler": int(options_source.get("spoiler", GeneratorOptions.spoiler)), diff --git a/WebHostLib/misc.py b/WebHostLib/misc.py index c49b1ae178..6be0e470b3 100644 --- a/WebHostLib/misc.py +++ b/WebHostLib/misc.py @@ -18,13 +18,6 @@ def get_world_theme(game_name: str): return 'grass' -@app.before_request -def register_session(): - session.permanent = True # technically 31 days after the last visit - if not session.get("_id", None): - session["_id"] = uuid4() # uniquely identify each session without needing a login - - @app.errorhandler(404) @app.errorhandler(jinja2.exceptions.TemplateNotFound) def page_not_found(err): diff --git a/WebHostLib/requirements.txt b/WebHostLib/requirements.txt index 5c79415312..b7b14dea1e 100644 --- a/WebHostLib/requirements.txt +++ b/WebHostLib/requirements.txt @@ -5,9 +5,7 @@ waitress>=3.0.0 Flask-Caching>=2.3.0 Flask-Compress>=1.15 Flask-Limiter>=3.8.0 -bokeh>=3.1.1; python_version <= '3.8' -bokeh>=3.4.3; python_version == '3.9' -bokeh>=3.5.2; python_version >= '3.10' +bokeh>=3.5.2 markupsafe>=2.1.5 Markdown>=3.7 mdx-breakless-lists>=1.0.1 diff --git a/WebHostLib/session.py b/WebHostLib/session.py new file mode 100644 index 0000000000..d5dab7d6e6 --- /dev/null +++ b/WebHostLib/session.py @@ -0,0 +1,31 @@ +from uuid import uuid4, UUID + +from flask import session, render_template + +from WebHostLib import app + + +@app.before_request +def register_session(): + session.permanent = True # technically 31 days after the last visit + if not session.get("_id", None): + session["_id"] = uuid4() # uniquely identify each session without needing a login + + +@app.route('/session') +def show_session(): + return render_template( + "session.html", + ) + + +@app.route('/session/') +def set_session(_id: str): + new_id: UUID = UUID(_id, version=4) + old_id: UUID = session["_id"] + if old_id != new_id: + session["_id"] = new_id + return render_template( + "session.html", + old_id=old_id, + ) diff --git a/WebHostLib/static/assets/faq/en.md b/WebHostLib/static/assets/faq/en.md index e64535b42d..96e526612b 100644 --- a/WebHostLib/static/assets/faq/en.md +++ b/WebHostLib/static/assets/faq/en.md @@ -22,7 +22,7 @@ players to rely upon each other to complete their game. While a multiworld game traditionally requires all players to be playing the same game, a multi-game multiworld allows players to randomize any of the supported games, and send items between them. This allows players of different -games to interact with one another in a single multiplayer environment. Archipelago supports multi-game multiworld. +games to interact with one another in a single multiplayer environment. Archipelago supports multi-game multiworlds. Here is a list of our [Supported Games](https://archipelago.gg/games). ## Can I generate a single-player game with Archipelago? diff --git a/WebHostLib/static/styles/timespinnerTracker.css b/WebHostLib/static/styles/timespinnerTracker.css index 007c6a19ba..640b584684 100644 --- a/WebHostLib/static/styles/timespinnerTracker.css +++ b/WebHostLib/static/styles/timespinnerTracker.css @@ -75,6 +75,27 @@ #inventory-table img.acquired.green{ /*32CD32*/ filter: hue-rotate(84deg) saturate(10) brightness(0.7); } +#inventory-table img.acquired.hotpink{ /*FF69B4*/ + filter: sepia(100%) hue-rotate(300deg) saturate(10); +} +#inventory-table img.acquired.lightsalmon{ /*FFA07A*/ + filter: sepia(100%) hue-rotate(347deg) saturate(10); +} +#inventory-table img.acquired.crimson{ /*DB143B*/ + filter: sepia(100%) hue-rotate(318deg) saturate(10) brightness(0.86); +} + +#inventory-table span{ + color: #B4B4A0; + font-size: 40px; + max-width: 40px; + max-height: 40px; + filter: grayscale(100%) contrast(75%) brightness(30%); +} + +#inventory-table span.acquired{ + filter: none; +} #inventory-table div.image-stack{ display: grid; diff --git a/WebHostLib/templates/hostRoom.html b/WebHostLib/templates/hostRoom.html index 8e76dafc12..c5996d181e 100644 --- a/WebHostLib/templates/hostRoom.html +++ b/WebHostLib/templates/hostRoom.html @@ -178,8 +178,15 @@ }) .then(text => new DOMParser().parseFromString(text, 'text/html')) .then(newDocument => { - let el = newDocument.getElementById("host-room-info"); - document.getElementById("host-room-info").innerHTML = el.innerHTML; + ["host-room-info", "slots-table"].forEach(function(id) { + const newEl = newDocument.getElementById(id); + const oldEl = document.getElementById(id); + if (oldEl && newEl) { + oldEl.innerHTML = newEl.innerHTML; + } else if (newEl) { + console.warn(`Did not find element to replace for ${id}`) + } + }); }); } diff --git a/WebHostLib/templates/islandFooter.html b/WebHostLib/templates/islandFooter.html index 08cf227990..7de14f0d82 100644 --- a/WebHostLib/templates/islandFooter.html +++ b/WebHostLib/templates/islandFooter.html @@ -1,6 +1,6 @@ {% block footer %}