From 354c9aea4c89fcd4aae604601220ab95fb31c467 Mon Sep 17 00:00:00 2001 From: Lexipherous Date: Wed, 30 Aug 2023 22:54:37 +0100 Subject: [PATCH 01/46] Added ability to define custom port ranges the WebHost will use for game servers, instead of pure random. --- WebHostLib/__init__.py | 1 + WebHostLib/autolauncher.py | 3 +- WebHostLib/customserver.py | 42 ++++++++++++++++++++++---- docs/webhost configuration sample.yaml | 4 +++ 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/WebHostLib/__init__.py b/WebHostLib/__init__.py index a59e3aa553..16f62f69ae 100644 --- a/WebHostLib/__init__.py +++ b/WebHostLib/__init__.py @@ -29,6 +29,7 @@ app.config["SELFLAUNCHKEY"] = None # can point to a SSL Certificate Key to encr app.config["SELFGEN"] = True # application process is in charge of scheduling Generations. app.config["DEBUG"] = False app.config["PORT"] = 80 +app.config["GAME_PORTS"] = "49152-65535" app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER app.config['MAX_CONTENT_LENGTH'] = 64 * 1024 * 1024 # 64 megabyte limit # if you want to deploy, make sure you have a non-guessable secret key diff --git a/WebHostLib/autolauncher.py b/WebHostLib/autolauncher.py index 0475a63297..8821272db5 100644 --- a/WebHostLib/autolauncher.py +++ b/WebHostLib/autolauncher.py @@ -180,6 +180,7 @@ class MultiworldInstance(): self.cert = config["SELFLAUNCHCERT"] self.key = config["SELFLAUNCHKEY"] self.host = config["HOST_ADDRESS"] + self.game_ports = config["GAME_PORTS"] def start(self): if self.process and self.process.is_alive(): @@ -188,7 +189,7 @@ class MultiworldInstance(): logging.info(f"Spinning up {self.room_id}") process = multiprocessing.Process(group=None, target=run_server_process, args=(self.room_id, self.ponyconfig, get_static_server_data(), - self.cert, self.key, self.host), + self.cert, self.key, self.host, self.game_ports), name="MultiHost") process.start() # bind after start to prevent thread sync issues with guardian. diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 8fbf692dec..cf153069cc 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -19,6 +19,8 @@ import Utils from MultiServer import Context, server, auto_shutdown, ServerCommandProcessor, ClientMessageProcessor, load_server_cert from Utils import restricted_loads, cache_argsless + +from . import app from .models import Command, GameDataPackage, Room, db @@ -84,13 +86,13 @@ class WebHostContext(Context): time.sleep(5) @db_session - def load(self, room_id: int): + def load(self, room_id: int, game_ports: str): self.room_id = room_id room = Room.get(id=room_id) if room.last_port: self.port = room.last_port else: - self.port = get_random_port() + self.port = get_random_port(game_ports) multidata = self.decompress(room.seed.multidata) game_data_packages = {} @@ -133,8 +135,36 @@ class WebHostContext(Context): return d -def get_random_port(): - return random.randint(49152, 65535) +def get_random_port(game_ports): + config_range_list = game_ports.split(",") + available_ports = [] + for item in config_range_list: + if '-' in item: + start, end = map(int, item.split('-')) + available_ports.extend(range(start, end+1)) + else: + available_ports.append(int(item)) + + port = get_port_from_list(available_ports) + + return port + + +def get_port_from_list(available_ports: list) -> int: + while available_ports: + port = random.choice(available_ports) + available_ports.remove(port) + + if not is_port_in_use(port): + break + else: + port = 0 + return port + + +def is_port_in_use(port: int) -> bool: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + return s.connect_ex(('localhost', port)) == 0 @cache_argsless @@ -157,7 +187,7 @@ def get_static_server_data() -> dict: def run_server_process(room_id, ponyconfig: dict, static_server_data: dict, cert_file: typing.Optional[str], cert_key_file: typing.Optional[str], - host: str): + host: str, game_ports: str): # establish DB connection for multidata and multisave db.bind(**ponyconfig) db.generate_mapping(check_tables=False) @@ -165,7 +195,7 @@ def run_server_process(room_id, ponyconfig: dict, static_server_data: dict, async def main(): Utils.init_logging(str(room_id), write_mode="a") ctx = WebHostContext(static_server_data) - ctx.load(room_id) + ctx.load(room_id, game_ports) ctx.init_save() ssl_context = load_server_cert(cert_file, cert_key_file) if cert_file else None try: diff --git a/docs/webhost configuration sample.yaml b/docs/webhost configuration sample.yaml index 70050b0590..70e03ec112 100644 --- a/docs/webhost configuration sample.yaml +++ b/docs/webhost configuration sample.yaml @@ -17,6 +17,10 @@ # Web hosting port #PORT: 80 +# Ports used for game hosting. Values can be specific ports, port ranges or both. Default is: "49152-65535" +# Examples of valid values: "40000-41000,49152-65535" +#GAME_PORTS: "49152-65535" + # Place where uploads go. #UPLOAD_FOLDER: uploads From 392a45ec89c8cfef0e4b263203f49f8bb21a318c Mon Sep 17 00:00:00 2001 From: Lexipherous Date: Sat, 9 Sep 2023 14:40:52 +0100 Subject: [PATCH 02/46] - Added better fallback to default port range when a custom range fails - Updated config to be clearer --- WebHostLib/customserver.py | 9 +++++++++ docs/webhost configuration sample.yaml | 1 + 2 files changed, 10 insertions(+) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index cf153069cc..1fe0fc4bf6 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -146,6 +146,15 @@ def get_random_port(game_ports): available_ports.append(int(item)) port = get_port_from_list(available_ports) + if port == 0: + logging.info("Unable to find port. Expanding search to the default ports. (49152-65535)") + checked_ports = [] + while len(set(checked_ports)) < (65535-49152)+1: + port = random.randint(49152, 65535) + if not is_port_in_use(port): + break + else: + checked_ports.append(port) return port diff --git a/docs/webhost configuration sample.yaml b/docs/webhost configuration sample.yaml index 70e03ec112..2a78e18997 100644 --- a/docs/webhost configuration sample.yaml +++ b/docs/webhost configuration sample.yaml @@ -19,6 +19,7 @@ # Ports used for game hosting. Values can be specific ports, port ranges or both. Default is: "49152-65535" # Examples of valid values: "40000-41000,49152-65535" +# If ports within the range(s) are already in use, the WebHost will fallback to the default "49152-65535" range. #GAME_PORTS: "49152-65535" # Place where uploads go. From b326045cb72b1b0ccd6fb199ea66761dc99bacf4 Mon Sep 17 00:00:00 2001 From: Lexipherous Date: Wed, 30 Aug 2023 22:54:37 +0100 Subject: [PATCH 03/46] Added ability to define custom port ranges the WebHost will use for game servers, instead of pure random. --- WebHostLib/__init__.py | 1 + WebHostLib/autolauncher.py | 3 +- WebHostLib/customserver.py | 42 ++++++++++++++++++++++---- docs/webhost configuration sample.yaml | 4 +++ 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/WebHostLib/__init__.py b/WebHostLib/__init__.py index 441f3272fd..49d57c9d9d 100644 --- a/WebHostLib/__init__.py +++ b/WebHostLib/__init__.py @@ -29,6 +29,7 @@ app.config["SELFLAUNCHKEY"] = None # can point to a SSL Certificate Key to encr app.config["SELFGEN"] = True # application process is in charge of scheduling Generations. app.config["DEBUG"] = False app.config["PORT"] = 80 +app.config["GAME_PORTS"] = "49152-65535" app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER app.config['MAX_CONTENT_LENGTH'] = 64 * 1024 * 1024 # 64 megabyte limit # if you want to deploy, make sure you have a non-guessable secret key diff --git a/WebHostLib/autolauncher.py b/WebHostLib/autolauncher.py index 0475a63297..8821272db5 100644 --- a/WebHostLib/autolauncher.py +++ b/WebHostLib/autolauncher.py @@ -180,6 +180,7 @@ class MultiworldInstance(): self.cert = config["SELFLAUNCHCERT"] self.key = config["SELFLAUNCHKEY"] self.host = config["HOST_ADDRESS"] + self.game_ports = config["GAME_PORTS"] def start(self): if self.process and self.process.is_alive(): @@ -188,7 +189,7 @@ class MultiworldInstance(): logging.info(f"Spinning up {self.room_id}") process = multiprocessing.Process(group=None, target=run_server_process, args=(self.room_id, self.ponyconfig, get_static_server_data(), - self.cert, self.key, self.host), + self.cert, self.key, self.host, self.game_ports), name="MultiHost") process.start() # bind after start to prevent thread sync issues with guardian. diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 8fbf692dec..cf153069cc 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -19,6 +19,8 @@ import Utils from MultiServer import Context, server, auto_shutdown, ServerCommandProcessor, ClientMessageProcessor, load_server_cert from Utils import restricted_loads, cache_argsless + +from . import app from .models import Command, GameDataPackage, Room, db @@ -84,13 +86,13 @@ class WebHostContext(Context): time.sleep(5) @db_session - def load(self, room_id: int): + def load(self, room_id: int, game_ports: str): self.room_id = room_id room = Room.get(id=room_id) if room.last_port: self.port = room.last_port else: - self.port = get_random_port() + self.port = get_random_port(game_ports) multidata = self.decompress(room.seed.multidata) game_data_packages = {} @@ -133,8 +135,36 @@ class WebHostContext(Context): return d -def get_random_port(): - return random.randint(49152, 65535) +def get_random_port(game_ports): + config_range_list = game_ports.split(",") + available_ports = [] + for item in config_range_list: + if '-' in item: + start, end = map(int, item.split('-')) + available_ports.extend(range(start, end+1)) + else: + available_ports.append(int(item)) + + port = get_port_from_list(available_ports) + + return port + + +def get_port_from_list(available_ports: list) -> int: + while available_ports: + port = random.choice(available_ports) + available_ports.remove(port) + + if not is_port_in_use(port): + break + else: + port = 0 + return port + + +def is_port_in_use(port: int) -> bool: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + return s.connect_ex(('localhost', port)) == 0 @cache_argsless @@ -157,7 +187,7 @@ def get_static_server_data() -> dict: def run_server_process(room_id, ponyconfig: dict, static_server_data: dict, cert_file: typing.Optional[str], cert_key_file: typing.Optional[str], - host: str): + host: str, game_ports: str): # establish DB connection for multidata and multisave db.bind(**ponyconfig) db.generate_mapping(check_tables=False) @@ -165,7 +195,7 @@ def run_server_process(room_id, ponyconfig: dict, static_server_data: dict, async def main(): Utils.init_logging(str(room_id), write_mode="a") ctx = WebHostContext(static_server_data) - ctx.load(room_id) + ctx.load(room_id, game_ports) ctx.init_save() ssl_context = load_server_cert(cert_file, cert_key_file) if cert_file else None try: diff --git a/docs/webhost configuration sample.yaml b/docs/webhost configuration sample.yaml index 70050b0590..70e03ec112 100644 --- a/docs/webhost configuration sample.yaml +++ b/docs/webhost configuration sample.yaml @@ -17,6 +17,10 @@ # Web hosting port #PORT: 80 +# Ports used for game hosting. Values can be specific ports, port ranges or both. Default is: "49152-65535" +# Examples of valid values: "40000-41000,49152-65535" +#GAME_PORTS: "49152-65535" + # Place where uploads go. #UPLOAD_FOLDER: uploads From 1815645994ca353234a2c86655f8a8cbd7a8f8a5 Mon Sep 17 00:00:00 2001 From: Lexipherous Date: Sat, 9 Sep 2023 14:40:52 +0100 Subject: [PATCH 04/46] - Added better fallback to default port range when a custom range fails - Updated config to be clearer --- WebHostLib/customserver.py | 9 +++++++++ docs/webhost configuration sample.yaml | 1 + 2 files changed, 10 insertions(+) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index cf153069cc..1fe0fc4bf6 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -146,6 +146,15 @@ def get_random_port(game_ports): available_ports.append(int(item)) port = get_port_from_list(available_ports) + if port == 0: + logging.info("Unable to find port. Expanding search to the default ports. (49152-65535)") + checked_ports = [] + while len(set(checked_ports)) < (65535-49152)+1: + port = random.randint(49152, 65535) + if not is_port_in_use(port): + break + else: + checked_ports.append(port) return port diff --git a/docs/webhost configuration sample.yaml b/docs/webhost configuration sample.yaml index 70e03ec112..2a78e18997 100644 --- a/docs/webhost configuration sample.yaml +++ b/docs/webhost configuration sample.yaml @@ -19,6 +19,7 @@ # Ports used for game hosting. Values can be specific ports, port ranges or both. Default is: "49152-65535" # Examples of valid values: "40000-41000,49152-65535" +# If ports within the range(s) are already in use, the WebHost will fallback to the default "49152-65535" range. #GAME_PORTS: "49152-65535" # Place where uploads go. From 8f4e4cf6b2cc672beae7321d1c202365d33f6e61 Mon Sep 17 00:00:00 2001 From: Lexipherous Date: Sun, 4 Feb 2024 18:47:08 +0000 Subject: [PATCH 05/46] Updated soft-fail message --- WebHostLib/customserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 9048d51ff7..ba971b1683 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -151,7 +151,6 @@ def get_random_port(game_ports): port = get_port_from_list(available_ports) if port == 0: - logging.info("Unable to find port. Expanding search to the default ports. (49152-65535)") checked_ports = [] while len(set(checked_ports)) < (65535-49152)+1: port = random.randint(49152, 65535) @@ -159,6 +158,7 @@ def get_random_port(game_ports): break else: checked_ports.append(port) + logging.info(f"Unable to find an available port in custom range. Expanded search to the default ports. Hosting on port {port}.") return port From 41b0c7edc6440d5d84c75f911e3bc855472f46a4 Mon Sep 17 00:00:00 2001 From: lexipherous Date: Tue, 18 Mar 2025 16:06:48 +0000 Subject: [PATCH 06/46] Removed dead import from customserver.py --- WebHostLib/customserver.py | 1 - requirements.txt | 39 ++++++++++++++++++++++++++------------ 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 1b888d19d6..12a28acc99 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -22,7 +22,6 @@ import Utils from MultiServer import Context, server, auto_shutdown, ServerCommandProcessor, ClientMessageProcessor, load_server_cert from Utils import restricted_loads, cache_argsless -from . import app from .locker import Locker from .models import Command, GameDataPackage, Room, db diff --git a/requirements.txt b/requirements.txt index 946546cb69..fab6cfc06c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,29 @@ -colorama>=0.4.6 -websockets>=13.0.1,<14 -PyYAML>=6.0.2 -jellyfish>=1.1.0 -jinja2>=3.1.4 -schema>=0.7.7 -kivy>=2.3.0 bsdiff4>=1.2.4 -platformdirs>=4.2.2 -certifi>=2024.8.30 -cython>=3.0.11 +bokeh==3.5.2 +certifi>=2023.11.17 +colorama==0.4.5 cymem>=2.0.8 -orjson>=3.10.7 -typing_extensions>=4.12.2 +cython>=3.0.5 +factorio-rcon-py==2.0.1 +Flask==3.0.3 +Flask-Caching==2.3.0 +Flask-Compress==1.15 +Flask-Limiter==3.8.0 +gitpython==3.1.24 +jellyfish>=1.0.3 +jinja2>=3.1.2 +kivy>=2.2.0 +maseya-z3pr==1.0.0rc1 +orjson>=3.9.10 +platformdirs>=4.0.0 +pyevermizer>=0.44.0 +Pymem==1.10.0 +pony==0.7.19 +PyYAML>=6.0.1 +schema>=0.7.5 +tqdm==4.66.1 +waitress==3.0.0 +websockets>=11.0.3 +Werkzeug==3.0.6 +Markdown==3.7 +mdx-breakless-lists==1.0.1 \ No newline at end of file From 7c1726bcc7a1f465e71d0e7d79b855b3a2fe0e1b Mon Sep 17 00:00:00 2001 From: Lexipherous Date: Fri, 21 Mar 2025 09:57:17 +0000 Subject: [PATCH 07/46] Update requirements.txt Settings requirements to main core branch --- requirements.txt | 39 ++++++++++++--------------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/requirements.txt b/requirements.txt index fab6cfc06c..cd045b874b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,29 +1,14 @@ +colorama>=0.4.6 +websockets>=13.0.1,<14 +PyYAML>=6.0.2 +jellyfish>=1.1.0 +jinja2>=3.1.4 +schema>=0.7.7 +kivy>=2.3.0 bsdiff4>=1.2.4 -bokeh==3.5.2 -certifi>=2023.11.17 -colorama==0.4.5 +platformdirs>=4.2.2 +certifi>=2024.12.14 +cython>=3.0.11 cymem>=2.0.8 -cython>=3.0.5 -factorio-rcon-py==2.0.1 -Flask==3.0.3 -Flask-Caching==2.3.0 -Flask-Compress==1.15 -Flask-Limiter==3.8.0 -gitpython==3.1.24 -jellyfish>=1.0.3 -jinja2>=3.1.2 -kivy>=2.2.0 -maseya-z3pr==1.0.0rc1 -orjson>=3.9.10 -platformdirs>=4.0.0 -pyevermizer>=0.44.0 -Pymem==1.10.0 -pony==0.7.19 -PyYAML>=6.0.1 -schema>=0.7.5 -tqdm==4.66.1 -waitress==3.0.0 -websockets>=11.0.3 -Werkzeug==3.0.6 -Markdown==3.7 -mdx-breakless-lists==1.0.1 \ No newline at end of file +orjson>=3.10.7 +typing_extensions>=4.12.2 From ed77f58f13e886726f026d97570ceb33fab5f542 Mon Sep 17 00:00:00 2001 From: Uriel Date: Thu, 5 Mar 2026 07:42:33 -0300 Subject: [PATCH 08/46] fix what reviewers said and add some improvements --- WebHostLib/__init__.py | 2 +- WebHostLib/customserver.py | 54 +++++++++++++------------- WebHostLib/requirements.txt | 1 + docs/webhost configuration sample.yaml | 7 ++-- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/WebHostLib/__init__.py b/WebHostLib/__init__.py index 3afacefc19..eaf605063c 100644 --- a/WebHostLib/__init__.py +++ b/WebHostLib/__init__.py @@ -43,7 +43,7 @@ app.config["SELFLAUNCHKEY"] = None # can point to a SSL Certificate Key to encr app.config["SELFGEN"] = True # application process is in charge of scheduling Generations. app.config["DEBUG"] = False app.config["PORT"] = 80 -app.config["GAME_PORTS"] = "49152-65535" +app.config["GAME_PORTS"] = "49152-65535,0" app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER app.config['MAX_CONTENT_LENGTH'] = 64 * 1024 * 1024 # 64 megabyte limit # if you want to deploy, make sure you have a non-guessable secret key diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 68b43bcfdb..4d3648d752 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -4,16 +4,17 @@ import asyncio import collections import datetime import functools +import itertools import logging import multiprocessing import pickle -import random import socket import threading import time import typing import sys +import more_itertools import websockets from pony.orm import commit, db_session, select @@ -116,7 +117,7 @@ class WebHostContext(Context): if room.last_port: self.port = room.last_port else: - self.port = get_random_port(game_ports) + self.port = 0 multidata = self.decompress(room.seed.multidata) game_data_packages = {} @@ -185,36 +186,27 @@ class WebHostContext(Context): def get_random_port(game_ports): config_range_list = game_ports.split(",") available_ports = [] + ephemeral_allowed = False for item in config_range_list: if '-' in item: start, end = map(int, item.split('-')) - available_ports.extend(range(start, end+1)) + available_ports.append(range(start, end+1)) + elif item == "0": + ephemeral_allowed = True else: - available_ports.append(int(item)) + available_ports.append([int(item)]) - port = get_port_from_list(available_ports) - if port == 0: - checked_ports = [] - while len(set(checked_ports)) < (65535-49152)+1: - port = random.randint(49152, 65535) - if not is_port_in_use(port): - break - else: - checked_ports.append(port) - logging.info(f"Unable to find an available port in custom range. Expanded search to the default ports. Hosting on port {port}.") - - return port + return get_port_from_list(more_itertools.interleave_randomly(*available_ports), ephemeral_allowed) -def get_port_from_list(available_ports: list) -> int: - while available_ports: - port = random.choice(available_ports) - available_ports.remove(port) - +def get_port_from_list(available_ports: typing.Iterable[int], ephemeral_allowed: bool) -> int: + # limit amount of checked ports to 1024 + for port in itertools.islice(available_ports, 1024): if not is_port_in_use(port): break else: - port = 0 + if ephemeral_allowed: return 0 + raise OSError(98, "No available ports") return port @@ -339,7 +331,18 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, ctx.load(room_id, game_ports) ctx.init_save() assert ctx.server is None - try: + if ctx.port == 0: + ctx.server = websockets.serve( + functools.partial(server, ctx=ctx), + ctx.host, + get_random_port(game_ports), + ssl=get_ssl_context(), + # In original code, this extension wasn't included when port was 0, should I leave that behavior? + # Or was it a bug? + extensions=[server_per_message_deflate_factory], + ) + await ctx.server + else: ctx.server = websockets.serve( functools.partial(server, ctx=ctx), ctx.host, @@ -348,11 +351,6 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, extensions=[server_per_message_deflate_factory], ) await ctx.server - except OSError: # likely port in use - ctx.server = websockets.serve( - functools.partial(server, ctx=ctx), ctx.host, 0, ssl=get_ssl_context()) - - await ctx.server port = 0 for wssocket in ctx.server.ws_server.sockets: socketname = wssocket.getsockname() diff --git a/WebHostLib/requirements.txt b/WebHostLib/requirements.txt index c9a923680a..20a5756822 100644 --- a/WebHostLib/requirements.txt +++ b/WebHostLib/requirements.txt @@ -12,3 +12,4 @@ markupsafe>=3.0.2 setproctitle>=1.3.5 mistune>=3.1.3 docutils>=0.22.2 +more-itertools>=10.8.0 diff --git a/docs/webhost configuration sample.yaml b/docs/webhost configuration sample.yaml index b0cda327ec..42dcada132 100644 --- a/docs/webhost configuration sample.yaml +++ b/docs/webhost configuration sample.yaml @@ -17,10 +17,11 @@ # Web hosting port #PORT: 80 -# Ports used for game hosting. Values can be specific ports, port ranges or both. Default is: "49152-65535" +# Ports used for game hosting. Values can be specific ports, port ranges or both. Default is: "49152-65535,0" +# Zero means it will use a random free port if there is none in the range available # Examples of valid values: "40000-41000,49152-65535" -# If ports within the range(s) are already in use, the WebHost will fallback to the default "49152-65535" range. -#GAME_PORTS: "49152-65535" +# If ports within the range(s) are already in use, the WebHost will fallback to the default "49152-65535,0" range. +#GAME_PORTS: "49152-65535,0" # Place where uploads go. #UPLOAD_FOLDER: uploads From 88dc83e55780b9b32ac997f1b86ffbbca2fbfd8b Mon Sep 17 00:00:00 2001 From: Uriel Date: Thu, 5 Mar 2026 07:47:32 -0300 Subject: [PATCH 09/46] remove unused argument --- WebHostLib/customserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 4d3648d752..36ded46b2e 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -111,7 +111,7 @@ class WebHostContext(Context): commit() @db_session - def load(self, room_id: int, game_ports: str): + def load(self, room_id: int): self.room_id = room_id room = Room.get(id=room_id) if room.last_port: @@ -328,7 +328,7 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, try: logger = set_up_logging(room_id) ctx = WebHostContext(static_server_data, logger) - ctx.load(room_id, game_ports) + ctx.load(room_id) ctx.init_save() assert ctx.server is None if ctx.port == 0: From 8800124c4eeb3d155481b306cadab6c1b1fd269c Mon Sep 17 00:00:00 2001 From: Uriel Date: Thu, 5 Mar 2026 08:08:45 -0300 Subject: [PATCH 10/46] try fixing test with try --- WebHostLib/customserver.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 36ded46b2e..eebcd446d4 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -331,6 +331,18 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, ctx.load(room_id) ctx.init_save() assert ctx.server is None + if ctx.port != 0: + try: + ctx.server = websockets.serve( + functools.partial(server, ctx=ctx), + ctx.host, + ctx.port, + ssl=get_ssl_context(), + extensions=[server_per_message_deflate_factory], + ) + await ctx.server + except OSError: + ctx.port = 0 if ctx.port == 0: ctx.server = websockets.serve( functools.partial(server, ctx=ctx), @@ -342,15 +354,6 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, extensions=[server_per_message_deflate_factory], ) await ctx.server - else: - ctx.server = websockets.serve( - functools.partial(server, ctx=ctx), - ctx.host, - ctx.port, - ssl=get_ssl_context(), - extensions=[server_per_message_deflate_factory], - ) - await ctx.server port = 0 for wssocket in ctx.server.ws_server.sockets: socketname = wssocket.getsockname() From f8b730308dcb9abee18107bd23071a561aa54bfa Mon Sep 17 00:00:00 2001 From: Uriel Date: Thu, 5 Mar 2026 08:35:51 -0300 Subject: [PATCH 11/46] use yaml lists instead of string for config --- WebHostLib/__init__.py | 2 +- WebHostLib/customserver.py | 11 +++++------ docs/webhost configuration sample.yaml | 10 +++++----- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/WebHostLib/__init__.py b/WebHostLib/__init__.py index eaf605063c..11a9619a7e 100644 --- a/WebHostLib/__init__.py +++ b/WebHostLib/__init__.py @@ -43,7 +43,7 @@ app.config["SELFLAUNCHKEY"] = None # can point to a SSL Certificate Key to encr app.config["SELFGEN"] = True # application process is in charge of scheduling Generations. app.config["DEBUG"] = False app.config["PORT"] = 80 -app.config["GAME_PORTS"] = "49152-65535,0" +app.config["GAME_PORTS"] = ["49152-65535", 0] app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER app.config['MAX_CONTENT_LENGTH'] = 64 * 1024 * 1024 # 64 megabyte limit # if you want to deploy, make sure you have a non-guessable secret key diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index eebcd446d4..275e281350 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -183,15 +183,14 @@ class WebHostContext(Context): return d -def get_random_port(game_ports): - config_range_list = game_ports.split(",") +def get_random_port(game_ports: list): available_ports = [] ephemeral_allowed = False - for item in config_range_list: - if '-' in item: + for item in game_ports: + if type(item) is str and '-' in item: start, end = map(int, item.split('-')) available_ports.append(range(start, end+1)) - elif item == "0": + elif int(item) == "0": ephemeral_allowed = True else: available_ports.append([int(item)]) @@ -277,7 +276,7 @@ def tear_down_logging(room_id): def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, cert_file: typing.Optional[str], cert_key_file: typing.Optional[str], - host: str, game_ports: str, rooms_to_run: multiprocessing.Queue, + host: str, game_ports: list, rooms_to_run: multiprocessing.Queue, rooms_shutting_down: multiprocessing.Queue): from setproctitle import setproctitle diff --git a/docs/webhost configuration sample.yaml b/docs/webhost configuration sample.yaml index 42dcada132..059faeeef9 100644 --- a/docs/webhost configuration sample.yaml +++ b/docs/webhost configuration sample.yaml @@ -17,11 +17,11 @@ # Web hosting port #PORT: 80 -# Ports used for game hosting. Values can be specific ports, port ranges or both. Default is: "49152-65535,0" -# Zero means it will use a random free port if there is none in the range available -# Examples of valid values: "40000-41000,49152-65535" -# If ports within the range(s) are already in use, the WebHost will fallback to the default "49152-65535,0" range. -#GAME_PORTS: "49152-65535,0" +# Ports used for game hosting. Values can be specific ports, port ranges or both. Default is: [49152-65535, 0] +# Zero means it will use a random free port if there is no port in the next 1024 randomly chosen ports from the range +# Examples of valid values: [40000-41000, 49152-65535] +# If ports within the range(s) are already in use, the WebHost will fallback to the default [49152-65535, 0] range. +#GAME_PORTS: [49152-65535, 0] # Place where uploads go. #UPLOAD_FOLDER: uploads From d6473fa0ed635e6dd7c7329e6b028c4b4fe5fd5d Mon Sep 17 00:00:00 2001 From: Uriel Date: Thu, 5 Mar 2026 08:42:03 -0300 Subject: [PATCH 12/46] fix value type bug on ephemeral type --- WebHostLib/customserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 275e281350..6ee01fabff 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -190,7 +190,7 @@ def get_random_port(game_ports: list): if type(item) is str and '-' in item: start, end = map(int, item.split('-')) available_ports.append(range(start, end+1)) - elif int(item) == "0": + elif int(item) == 0: ephemeral_allowed = True else: available_ports.append([int(item)]) From 0a0faefab269eefd1036e3cb782160b20c466f8c Mon Sep 17 00:00:00 2001 From: Uriel Date: Thu, 5 Mar 2026 09:53:23 -0300 Subject: [PATCH 13/46] reuse sockets with websockets api instead of opening and closing them --- WebHostLib/customserver.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 6ee01fabff..099f64843d 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -183,7 +183,7 @@ class WebHostContext(Context): return d -def get_random_port(game_ports: list): +def get_random_port(game_ports: list, host): available_ports = [] ephemeral_allowed = False for item in game_ports: @@ -195,23 +195,23 @@ def get_random_port(game_ports: list): else: available_ports.append([int(item)]) - return get_port_from_list(more_itertools.interleave_randomly(*available_ports), ephemeral_allowed) + return get_port_from_list(more_itertools.interleave_randomly(*available_ports), ephemeral_allowed, host) -def get_port_from_list(available_ports: typing.Iterable[int], ephemeral_allowed: bool) -> int: +def get_port_from_list(available_ports: typing.Iterable[int], ephemeral_allowed: bool, host) -> socket.socket: # limit amount of checked ports to 1024 for port in itertools.islice(available_ports, 1024): - if not is_port_in_use(port): - break + sock = get_socket_if_free(host, port) + if sock is not None: return sock else: - if ephemeral_allowed: return 0 + if ephemeral_allowed: return socket.create_server((host, 0)) raise OSError(98, "No available ports") - return port - -def is_port_in_use(port: int) -> bool: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - return s.connect_ex(('localhost', port)) == 0 +def get_socket_if_free(host, port: int) -> socket.socket | None: + try: + return socket.create_server((host, port)) + except OSError: + return None @cache_argsless @@ -345,8 +345,7 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, if ctx.port == 0: ctx.server = websockets.serve( functools.partial(server, ctx=ctx), - ctx.host, - get_random_port(game_ports), + sock=get_random_port(game_ports, ctx.host), ssl=get_ssl_context(), # In original code, this extension wasn't included when port was 0, should I leave that behavior? # Or was it a bug? From b0615590fc72d7c37d65b77ce90b5fe2c1ce0b35 Mon Sep 17 00:00:00 2001 From: Uriel Date: Thu, 5 Mar 2026 10:34:33 -0300 Subject: [PATCH 14/46] add used ports cache and filter used ports when looking for ports --- WebHostLib/customserver.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 099f64843d..0133059141 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -15,6 +15,7 @@ import typing import sys import more_itertools +import psutil import websockets from pony.orm import commit, db_session, select @@ -197,10 +198,16 @@ def get_random_port(game_ports: list, host): return get_port_from_list(more_itertools.interleave_randomly(*available_ports), ephemeral_allowed, host) +def get_ttl_hash(seconds = 1800): + return round(time.time() / seconds) + +@functools.lru_cache() +def get_used_ports(ttl = get_ttl_hash()): + return frozenset(map(lambda c: c.laddr.port, psutil.net_connections("tcp4"))) def get_port_from_list(available_ports: typing.Iterable[int], ephemeral_allowed: bool, host) -> socket.socket: # limit amount of checked ports to 1024 - for port in itertools.islice(available_ports, 1024): + for port in itertools.islice(filter(lambda p: p not in get_used_ports(), available_ports), 1024): sock = get_socket_if_free(host, port) if sock is not None: return sock else: From 08a6ee2b3a48e425aa3ce61cb052e99c592fe43e Mon Sep 17 00:00:00 2001 From: Uriel Date: Thu, 5 Mar 2026 11:15:33 -0300 Subject: [PATCH 15/46] fix port randomizer --- WebHostLib/customserver.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 0133059141..45f6ac4ba9 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -14,7 +14,7 @@ import time import typing import sys -import more_itertools +from more_itertools import value_chain, random_permutation import psutil import websockets from pony.orm import commit, db_session, select @@ -194,9 +194,13 @@ def get_random_port(game_ports: list, host): elif int(item) == 0: ephemeral_allowed = True else: - available_ports.append([int(item)]) + available_ports.append(int(item)) - return get_port_from_list(more_itertools.interleave_randomly(*available_ports), ephemeral_allowed, host) + return get_port_from_list( + random_permutation( + filter(lambda p: p not in get_used_ports(), value_chain(*available_ports)), + 1024), + ephemeral_allowed, host) def get_ttl_hash(seconds = 1800): return round(time.time() / seconds) @@ -207,7 +211,7 @@ def get_used_ports(ttl = get_ttl_hash()): def get_port_from_list(available_ports: typing.Iterable[int], ephemeral_allowed: bool, host) -> socket.socket: # limit amount of checked ports to 1024 - for port in itertools.islice(filter(lambda p: p not in get_used_ports(), available_ports), 1024): + for port in available_ports: sock = get_socket_if_free(host, port) if sock is not None: return sock else: From 61f893437a11259157b209248239f0c833d8e16c Mon Sep 17 00:00:00 2001 From: Uriel Date: Thu, 5 Mar 2026 15:51:09 -0300 Subject: [PATCH 16/46] Apply suggestions from code review Co-authored-by: Duck <31627079+duckboycool@users.noreply.github.com> --- WebHostLib/__init__.py | 6 ------ WebHostLib/customserver.py | 6 +++--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/WebHostLib/__init__.py b/WebHostLib/__init__.py index 11a9619a7e..9ef642c4d6 100644 --- a/WebHostLib/__init__.py +++ b/WebHostLib/__init__.py @@ -41,13 +41,7 @@ app.config["SELFLAUNCH"] = True # application process is in charge of launching app.config["SELFLAUNCHCERT"] = None # can point to a SSL Certificate to encrypt Room websocket connections app.config["SELFLAUNCHKEY"] = None # can point to a SSL Certificate Key to encrypt Room websocket connections app.config["SELFGEN"] = True # application process is in charge of scheduling Generations. -app.config["DEBUG"] = False -app.config["PORT"] = 80 app.config["GAME_PORTS"] = ["49152-65535", 0] -app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER -app.config['MAX_CONTENT_LENGTH'] = 64 * 1024 * 1024 # 64 megabyte limit -# if you want to deploy, make sure you have a non-guessable secret key -app.config["SECRET_KEY"] = bytes(socket.gethostname(), encoding="utf-8") # at what amount of worlds should scheduling be used, instead of rolling in the web-thread app.config["JOB_THRESHOLD"] = 1 # after what time in seconds should generation be aborted, freeing the queue slot. Can be set to None to disable. diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 45f6ac4ba9..99d86da3a4 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -184,12 +184,12 @@ class WebHostContext(Context): return d -def get_random_port(game_ports: list, host): +def get_random_port(game_ports: list[str | int], host: str): available_ports = [] ephemeral_allowed = False for item in game_ports: - if type(item) is str and '-' in item: - start, end = map(int, item.split('-')) + if isinstance(item, str) and "-" in item: + start, end = map(int, item.split("-")) available_ports.append(range(start, end+1)) elif int(item) == 0: ephemeral_allowed = True From 551dbf44f643b6754460031d0990375f9ee3f584 Mon Sep 17 00:00:00 2001 From: Uriel Date: Thu, 5 Mar 2026 16:03:29 -0300 Subject: [PATCH 17/46] fix some reviews --- WebHostLib/customserver.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 99d86da3a4..67e2d2954f 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -197,20 +197,24 @@ def get_random_port(game_ports: list[str | int], host: str): available_ports.append(int(item)) return get_port_from_list( + # limit amount of checked ports to 1024 random_permutation( filter(lambda p: p not in get_used_ports(), value_chain(*available_ports)), 1024), ephemeral_allowed, host) -def get_ttl_hash(seconds = 1800): - return round(time.time() / seconds) -@functools.lru_cache() -def get_used_ports(ttl = get_ttl_hash()): - return frozenset(map(lambda c: c.laddr.port, psutil.net_connections("tcp4"))) +_last_used_ports = (frozenset(map(lambda c: c.laddr.port, psutil.net_connections("tcp4"))), round(time.time() / 900)) +def get_used_ports(): + global _last_used_ports + t_hash = round(time.time() / 900) + if _last_used_ports[1] != t_hash: + _last_used_ports = (frozenset(map(lambda c: c.laddr.port, psutil.net_connections("tcp4"))), t_hash) + + return _last_used_ports[0] + def get_port_from_list(available_ports: typing.Iterable[int], ephemeral_allowed: bool, host) -> socket.socket: - # limit amount of checked ports to 1024 for port in available_ports: sock = get_socket_if_free(host, port) if sock is not None: return sock @@ -218,6 +222,7 @@ def get_port_from_list(available_ports: typing.Iterable[int], ephemeral_allowed: if ephemeral_allowed: return socket.create_server((host, 0)) raise OSError(98, "No available ports") + def get_socket_if_free(host, port: int) -> socket.socket | None: try: return socket.create_server((host, port)) @@ -358,8 +363,6 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, functools.partial(server, ctx=ctx), sock=get_random_port(game_ports, ctx.host), ssl=get_ssl_context(), - # In original code, this extension wasn't included when port was 0, should I leave that behavior? - # Or was it a bug? extensions=[server_per_message_deflate_factory], ) await ctx.server From f03d1cad3e31688794c22d181eaadc4341423f3d Mon Sep 17 00:00:00 2001 From: Uriel Date: Thu, 5 Mar 2026 17:01:08 -0300 Subject: [PATCH 18/46] use weights for random port and remove more-itertools --- WebHostLib/customserver.py | 72 +++++++++++++++++++++++++------------ WebHostLib/requirements.txt | 1 - 2 files changed, 49 insertions(+), 24 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 67e2d2954f..1dd55eed54 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -8,13 +8,13 @@ import itertools import logging import multiprocessing import pickle +import random import socket import threading import time import typing import sys -from more_itertools import value_chain, random_permutation import psutil import websockets from pony.orm import commit, db_session, select @@ -184,24 +184,56 @@ class WebHostContext(Context): return d -def get_random_port(game_ports: list[str | int], host: str): - available_ports = [] +@functools.cache +def parse_game_ports(game_ports: tuple[str | int]): + available_ports: list[range | list[int]] = [] + weights = [] ephemeral_allowed = False + total_length = 0 + for item in game_ports: if isinstance(item, str) and "-" in item: start, end = map(int, item.split("-")) - available_ports.append(range(start, end+1)) + x = range(start, end + 1) + total_length += len(x) + weights.append(total_length) + available_ports.append(x) elif int(item) == 0: ephemeral_allowed = True else: - available_ports.append(int(item)) + total_length += 1 + weights.append(total_length) + available_ports.append([int(item)]) - return get_port_from_list( - # limit amount of checked ports to 1024 - random_permutation( - filter(lambda p: p not in get_used_ports(), value_chain(*available_ports)), - 1024), - ephemeral_allowed, host) + return available_ports, weights, total_length, ephemeral_allowed + + +def get_random_port(game_ports: list[str | int], host: str) -> socket.socket: + # convert to tuple because its hashable + available_ports, weights, length, ephemeral_allowed = parse_game_ports(tuple(game_ports)) + ports = random.choices(available_ports, cum_weights=weights, k=len(available_ports)) + remaining = 1024 + for r in ports: + r_length = len(r) + if isinstance(r, range): + random_range = itertools.islice( + filter( + lambda p: p not in get_used_ports(), + map(lambda _: random.randint(r.start, r.stop), range(r_length)) + ), + remaining) + port = get_port_from_list(random_range, host) + else: + port = get_port_from_list(filter(lambda p: p not in get_used_ports(), r), host) + remaining -= r_length + + if port is not None: return port + if remaining <= 0: break + + if ephemeral_allowed: + return socket.create_server((host, 0)) + + raise OSError(98, "No available ports") _last_used_ports = (frozenset(map(lambda c: c.laddr.port, psutil.net_connections("tcp4"))), round(time.time() / 900)) @@ -214,20 +246,14 @@ def get_used_ports(): return _last_used_ports[0] -def get_port_from_list(available_ports: typing.Iterable[int], ephemeral_allowed: bool, host) -> socket.socket: +def get_port_from_list(available_ports: typing.Iterable[int], host: str) -> socket.socket | None: for port in available_ports: - sock = get_socket_if_free(host, port) - if sock is not None: return sock - else: - if ephemeral_allowed: return socket.create_server((host, 0)) - raise OSError(98, "No available ports") + try: + return socket.create_server((host, port)) + except OSError: + _ = None - -def get_socket_if_free(host, port: int) -> socket.socket | None: - try: - return socket.create_server((host, port)) - except OSError: - return None + return None @cache_argsless diff --git a/WebHostLib/requirements.txt b/WebHostLib/requirements.txt index 20a5756822..c9a923680a 100644 --- a/WebHostLib/requirements.txt +++ b/WebHostLib/requirements.txt @@ -12,4 +12,3 @@ markupsafe>=3.0.2 setproctitle>=1.3.5 mistune>=3.1.3 docutils>=0.22.2 -more-itertools>=10.8.0 From 980a229aaacdc8f7c90ada7f03574fa61f1285ec Mon Sep 17 00:00:00 2001 From: Uriel Date: Thu, 5 Mar 2026 18:36:15 -0300 Subject: [PATCH 19/46] fix net_connections not working on macOS --- WebHostLib/customserver.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 1dd55eed54..8dfe73f44c 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -235,13 +235,36 @@ def get_random_port(game_ports: list[str | int], host: str) -> socket.socket: raise OSError(98, "No available ports") +def try_processes(p): + try: + return map(lambda c: c.laddr.port, p.net_connections("tcp4")) + except psutil.AccessDenied: + return [] -_last_used_ports = (frozenset(map(lambda c: c.laddr.port, psutil.net_connections("tcp4"))), round(time.time() / 900)) +def net_connections() -> typing.Iterable[int]: + # Don't even try to check if system using AIX + if psutil._common.AIX: + return [] + + try: + return map(lambda c: c.laddr.port, psutil.net_connections("tcp4")) + # raises AccessDenied when done on macOS + except psutil.AccessDenied: + # flatten the list of iterables + return itertools.chain.from_iterable(map( + # get the net connections of the process and then map its ports + try_processes, + # this method has caching handled by psutil + psutil.process_iter(["net_connections"]) + )) + + +_last_used_ports = (frozenset(net_connections()), round(time.time() / 900)) def get_used_ports(): global _last_used_ports t_hash = round(time.time() / 900) if _last_used_ports[1] != t_hash: - _last_used_ports = (frozenset(map(lambda c: c.laddr.port, psutil.net_connections("tcp4"))), t_hash) + _last_used_ports = (frozenset(net_connections()), t_hash) return _last_used_ports[0] From 60773ddf8337546927a338043d8626a0b0ee4b42 Mon Sep 17 00:00:00 2001 From: Uriel Date: Sat, 7 Mar 2026 14:43:06 -0300 Subject: [PATCH 20/46] rename variables and functions --- WebHostLib/customserver.py | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 8dfe73f44c..44271d7069 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -186,7 +186,7 @@ class WebHostContext(Context): @functools.cache def parse_game_ports(game_ports: tuple[str | int]): - available_ports: list[range | list[int]] = [] + parsed_ports: list[range | list[int]] = [] weights = [] ephemeral_allowed = False total_length = 0 @@ -197,23 +197,23 @@ def parse_game_ports(game_ports: tuple[str | int]): x = range(start, end + 1) total_length += len(x) weights.append(total_length) - available_ports.append(x) + parsed_ports.append(x) elif int(item) == 0: ephemeral_allowed = True else: total_length += 1 weights.append(total_length) - available_ports.append([int(item)]) + parsed_ports.append([int(item)]) - return available_ports, weights, total_length, ephemeral_allowed + return parsed_ports, weights, total_length, ephemeral_allowed -def get_random_port(game_ports: list[str | int], host: str) -> socket.socket: +def create_random_port_socket(game_ports: list[str | int], host: str) -> socket.socket: # convert to tuple because its hashable - available_ports, weights, length, ephemeral_allowed = parse_game_ports(tuple(game_ports)) - ports = random.choices(available_ports, cum_weights=weights, k=len(available_ports)) + parsed_ports, weights, length, ephemeral_allowed = parse_game_ports(tuple(game_ports)) + port_ranges = random.choices(parsed_ports, cum_weights=weights, k=len(parsed_ports)) remaining = 1024 - for r in ports: + for r in port_ranges: r_length = len(r) if isinstance(r, range): random_range = itertools.islice( @@ -222,9 +222,9 @@ def get_random_port(game_ports: list[str | int], host: str) -> socket.socket: map(lambda _: random.randint(r.start, r.stop), range(r_length)) ), remaining) - port = get_port_from_list(random_range, host) + port = create_socket_from_port_list(random_range, host) else: - port = get_port_from_list(filter(lambda p: p not in get_used_ports(), r), host) + port = create_socket_from_port_list(filter(lambda p: p not in get_used_ports(), r), host) remaining -= r_length if port is not None: return port @@ -235,13 +235,15 @@ def get_random_port(game_ports: list[str | int], host: str) -> socket.socket: raise OSError(98, "No available ports") -def try_processes(p): + +def try_processes(p: psutil.Process) -> typing.Iterable[int]: try: - return map(lambda c: c.laddr.port, p.net_connections("tcp4")) + return map(lambda c: c.laddr.port, p.get_active_net_connections("tcp4")) except psutil.AccessDenied: return [] -def net_connections() -> typing.Iterable[int]: + +def get_active_net_connections() -> typing.Iterable[int]: # Don't even try to check if system using AIX if psutil._common.AIX: return [] @@ -259,17 +261,17 @@ def net_connections() -> typing.Iterable[int]: )) -_last_used_ports = (frozenset(net_connections()), round(time.time() / 900)) +_last_used_ports = (frozenset(get_active_net_connections()), round(time.time() / 900)) def get_used_ports(): global _last_used_ports t_hash = round(time.time() / 900) if _last_used_ports[1] != t_hash: - _last_used_ports = (frozenset(net_connections()), t_hash) + _last_used_ports = (frozenset(get_active_net_connections()), t_hash) return _last_used_ports[0] -def get_port_from_list(available_ports: typing.Iterable[int], host: str) -> socket.socket | None: +def create_socket_from_port_list(available_ports: typing.Iterable[int], host: str) -> socket.socket | None: for port in available_ports: try: return socket.create_server((host, port)) @@ -410,7 +412,7 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, if ctx.port == 0: ctx.server = websockets.serve( functools.partial(server, ctx=ctx), - sock=get_random_port(game_ports, ctx.host), + sock=create_random_port_socket(game_ports, ctx.host), ssl=get_ssl_context(), extensions=[server_per_message_deflate_factory], ) From c290386950f32195ad7e79e15ce128bb1893bc67 Mon Sep 17 00:00:00 2001 From: Uriel Date: Sat, 7 Mar 2026 14:59:04 -0300 Subject: [PATCH 21/46] lazy init `get_used_ports` --- WebHostLib/customserver.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 44271d7069..3f31d92116 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -261,14 +261,13 @@ def get_active_net_connections() -> typing.Iterable[int]: )) -_last_used_ports = (frozenset(get_active_net_connections()), round(time.time() / 900)) def get_used_ports(): - global _last_used_ports - t_hash = round(time.time() / 900) - if _last_used_ports[1] != t_hash: - _last_used_ports = (frozenset(get_active_net_connections()), t_hash) + last_used_ports: tuple[frozenset[int], int] = getattr(get_used_ports, "last", None) + t_hash = time.time() // 900 + if last_used_ports is None or last_used_ports[1] != t_hash: + setattr(get_used_ports, "last", (frozenset(get_active_net_connections()), t_hash)) - return _last_used_ports[0] + return last_used_ports[0] def create_socket_from_port_list(available_ports: typing.Iterable[int], host: str) -> socket.socket | None: From 6a94a9e6ca97c120903c29829b71d7d6b5b56a8b Mon Sep 17 00:00:00 2001 From: Uriel Date: Sat, 7 Mar 2026 15:02:08 -0300 Subject: [PATCH 22/46] change `game_ports` to be `tuple` --- WebHostLib/customserver.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 3f31d92116..afdc88e699 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -208,9 +208,8 @@ def parse_game_ports(game_ports: tuple[str | int]): return parsed_ports, weights, total_length, ephemeral_allowed -def create_random_port_socket(game_ports: list[str | int], host: str) -> socket.socket: - # convert to tuple because its hashable - parsed_ports, weights, length, ephemeral_allowed = parse_game_ports(tuple(game_ports)) +def create_random_port_socket(game_ports: tuple[str | int], host: str) -> socket.socket: + parsed_ports, weights, length, ephemeral_allowed = parse_game_ports(game_ports) port_ranges = random.choices(parsed_ports, cum_weights=weights, k=len(parsed_ports)) remaining = 1024 for r in port_ranges: @@ -236,7 +235,7 @@ def create_random_port_socket(game_ports: list[str | int], host: str) -> socket. raise OSError(98, "No available ports") -def try_processes(p: psutil.Process) -> typing.Iterable[int]: +def try_conns_per_process(p: psutil.Process) -> typing.Iterable[int]: try: return map(lambda c: c.laddr.port, p.get_active_net_connections("tcp4")) except psutil.AccessDenied: @@ -255,7 +254,7 @@ def get_active_net_connections() -> typing.Iterable[int]: # flatten the list of iterables return itertools.chain.from_iterable(map( # get the net connections of the process and then map its ports - try_processes, + try_conns_per_process, # this method has caching handled by psutil psutil.process_iter(["net_connections"]) )) @@ -411,7 +410,8 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, if ctx.port == 0: ctx.server = websockets.serve( functools.partial(server, ctx=ctx), - sock=create_random_port_socket(game_ports, ctx.host), + # convert to tuple because its hashable + sock=create_random_port_socket(tuple(game_ports), ctx.host), ssl=get_ssl_context(), extensions=[server_per_message_deflate_factory], ) From 6779b4fcf37c8f966121eca4fe554496a719f906 Mon Sep 17 00:00:00 2001 From: Uriel Date: Sat, 7 Mar 2026 15:07:31 -0300 Subject: [PATCH 23/46] fix last_used_ports not being updated locally --- WebHostLib/customserver.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index afdc88e699..29f9f0402d 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -261,10 +261,11 @@ def get_active_net_connections() -> typing.Iterable[int]: def get_used_ports(): - last_used_ports: tuple[frozenset[int], int] = getattr(get_used_ports, "last", None) + last_used_ports: tuple[frozenset[int], float] = getattr(get_used_ports, "last", None) t_hash = time.time() // 900 if last_used_ports is None or last_used_ports[1] != t_hash: - setattr(get_used_ports, "last", (frozenset(get_active_net_connections()), t_hash)) + last_used_ports = (frozenset(get_active_net_connections()), t_hash) + setattr(get_used_ports, "last", last_used_ports) return last_used_ports[0] From 368eafae860535a2f7e47ec2671391b120bc74f8 Mon Sep 17 00:00:00 2001 From: Uriel Date: Sat, 7 Mar 2026 16:21:27 -0300 Subject: [PATCH 24/46] fix random choices and move game_port conversion into tuple --- WebHostLib/customserver.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 29f9f0402d..c2d9690a12 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -210,7 +210,8 @@ def parse_game_ports(game_ports: tuple[str | int]): def create_random_port_socket(game_ports: tuple[str | int], host: str) -> socket.socket: parsed_ports, weights, length, ephemeral_allowed = parse_game_ports(game_ports) - port_ranges = random.choices(parsed_ports, cum_weights=weights, k=len(parsed_ports)) + # try to randomize the order of parsed ports with weights, but don't have duplicates of them + port_ranges = list(dict.fromkeys(random.choices(parsed_ports, weights=weights, k=len(parsed_ports)) + parsed_ports)) remaining = 1024 for r in port_ranges: r_length = len(r) @@ -342,8 +343,8 @@ def tear_down_logging(room_id): def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, cert_file: typing.Optional[str], cert_key_file: typing.Optional[str], - host: str, game_ports: list, rooms_to_run: multiprocessing.Queue, - rooms_shutting_down: multiprocessing.Queue): + host: str, game_ports: typing.Iterable[str | int], + rooms_to_run: multiprocessing.Queue, rooms_shutting_down: multiprocessing.Queue): from setproctitle import setproctitle setproctitle(name) @@ -359,6 +360,10 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, resource.setrlimit(resource.RLIMIT_NOFILE, (file_limit, file_limit)) del resource, file_limit + # convert to tuple because its hashable + if not isinstance(game_ports, tuple): + game_ports = tuple(game_ports) + # establish DB connection for multidata and multisave db.bind(**ponyconfig) db.generate_mapping(check_tables=False) @@ -411,8 +416,7 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, if ctx.port == 0: ctx.server = websockets.serve( functools.partial(server, ctx=ctx), - # convert to tuple because its hashable - sock=create_random_port_socket(tuple(game_ports), ctx.host), + sock=create_random_port_socket(game_ports, ctx.host), ssl=get_ssl_context(), extensions=[server_per_message_deflate_factory], ) From e2823aa044ee2bfa53ac047ebc20de707c805f10 Mon Sep 17 00:00:00 2001 From: Uriel Date: Sat, 7 Mar 2026 16:52:57 -0300 Subject: [PATCH 25/46] Apply suggestions from code review Co-authored-by: Duck <31627079+duckboycool@users.noreply.github.com> --- WebHostLib/customserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index c2d9690a12..ac2e404531 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -219,7 +219,7 @@ def create_random_port_socket(game_ports: tuple[str | int], host: str) -> socket random_range = itertools.islice( filter( lambda p: p not in get_used_ports(), - map(lambda _: random.randint(r.start, r.stop), range(r_length)) + map(lambda _: random.randrange(r.start, r.stop, r.step), range(r_length)) ), remaining) port = create_socket_from_port_list(random_range, host) @@ -276,7 +276,7 @@ def create_socket_from_port_list(available_ports: typing.Iterable[int], host: st try: return socket.create_server((host, port)) except OSError: - _ = None + pass return None From 9ab7c56791218f59d1d6b9d597def14016d6966b Mon Sep 17 00:00:00 2001 From: Uriel Date: Sat, 7 Mar 2026 17:13:50 -0300 Subject: [PATCH 26/46] use a named tuple on parse_game_ports --- WebHostLib/customserver.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index ac2e404531..c3965a9282 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -184,8 +184,14 @@ class WebHostContext(Context): return d +class GameRangePorts(typing.NamedTuple): + parsed_ports: list[range | list[int]] + weights: list[int] + ephemeral_allowed: bool + + @functools.cache -def parse_game_ports(game_ports: tuple[str | int]): +def parse_game_ports(game_ports: tuple[str | int]) -> GameRangePorts: parsed_ports: list[range | list[int]] = [] weights = [] ephemeral_allowed = False @@ -205,13 +211,15 @@ def parse_game_ports(game_ports: tuple[str | int]): weights.append(total_length) parsed_ports.append([int(item)]) - return parsed_ports, weights, total_length, ephemeral_allowed + return GameRangePorts(parsed_ports, weights, ephemeral_allowed) def create_random_port_socket(game_ports: tuple[str | int], host: str) -> socket.socket: - parsed_ports, weights, length, ephemeral_allowed = parse_game_ports(game_ports) + parsed_ports, weights, ephemeral_allowed = parse_game_ports(game_ports) # try to randomize the order of parsed ports with weights, but don't have duplicates of them - port_ranges = list(dict.fromkeys(random.choices(parsed_ports, weights=weights, k=len(parsed_ports)) + parsed_ports)) + port_ranges = list( + dict.fromkeys(random.choices(parsed_ports, cum_weights=weights, k=len(parsed_ports)) + parsed_ports) + ) remaining = 1024 for r in port_ranges: r_length = len(r) From 4ea7fbbcbac451848e8acf7c3a13e17343c52973 Mon Sep 17 00:00:00 2001 From: Uriel Date: Sat, 7 Mar 2026 17:18:03 -0300 Subject: [PATCH 27/46] only use ranges --- WebHostLib/customserver.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index c3965a9282..d66d2ca1fe 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -185,14 +185,14 @@ class WebHostContext(Context): class GameRangePorts(typing.NamedTuple): - parsed_ports: list[range | list[int]] + parsed_ports: list[range] weights: list[int] ephemeral_allowed: bool @functools.cache def parse_game_ports(game_ports: tuple[str | int]) -> GameRangePorts: - parsed_ports: list[range | list[int]] = [] + parsed_ports: list[range] = [] weights = [] ephemeral_allowed = False total_length = 0 @@ -209,7 +209,8 @@ def parse_game_ports(game_ports: tuple[str | int]) -> GameRangePorts: else: total_length += 1 weights.append(total_length) - parsed_ports.append([int(item)]) + num = int(item) + parsed_ports.append(range(num, num+1)) return GameRangePorts(parsed_ports, weights, ephemeral_allowed) @@ -222,18 +223,14 @@ def create_random_port_socket(game_ports: tuple[str | int], host: str) -> socket ) remaining = 1024 for r in port_ranges: - r_length = len(r) - if isinstance(r, range): - random_range = itertools.islice( - filter( - lambda p: p not in get_used_ports(), - map(lambda _: random.randrange(r.start, r.stop, r.step), range(r_length)) - ), - remaining) - port = create_socket_from_port_list(random_range, host) - else: - port = create_socket_from_port_list(filter(lambda p: p not in get_used_ports(), r), host) - remaining -= r_length + random_range = itertools.islice( + filter( + lambda p: p not in get_used_ports(), + map(lambda _: random.randrange(r.start, r.stop, r.step), range(len(r))) + ), + remaining) + port = create_socket_from_port_list(random_range, host) + remaining -= len(r) if port is not None: return port if remaining <= 0: break From 779dd466589fb72ec10227ad68cf91fb2e193156 Mon Sep 17 00:00:00 2001 From: Uriel Date: Sat, 7 Mar 2026 17:26:54 -0300 Subject: [PATCH 28/46] do it the duck way --- WebHostLib/customserver.py | 38 +++++++++++--------------------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index d66d2ca1fe..376903fcf5 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -215,25 +215,19 @@ def parse_game_ports(game_ports: tuple[str | int]) -> GameRangePorts: return GameRangePorts(parsed_ports, weights, ephemeral_allowed) +def weighted_random(ranges: list[range], cum_weights: list[int]): + [picked] = random.choices(ranges, cum_weights=cum_weights) + return random.randrange(picked.start, picked.stop, picked.step) + + def create_random_port_socket(game_ports: tuple[str | int], host: str) -> socket.socket: parsed_ports, weights, ephemeral_allowed = parse_game_ports(game_ports) - # try to randomize the order of parsed ports with weights, but don't have duplicates of them - port_ranges = list( - dict.fromkeys(random.choices(parsed_ports, cum_weights=weights, k=len(parsed_ports)) + parsed_ports) - ) - remaining = 1024 - for r in port_ranges: - random_range = itertools.islice( - filter( - lambda p: p not in get_used_ports(), - map(lambda _: random.randrange(r.start, r.stop, r.step), range(len(r))) - ), - remaining) - port = create_socket_from_port_list(random_range, host) - remaining -= len(r) - - if port is not None: return port - if remaining <= 0: break + for _ in range(1024): + port_num = weighted_random(parsed_ports, weights) + try: + return socket.create_server((host, port_num)) + except OSError: + pass if ephemeral_allowed: return socket.create_server((host, 0)) @@ -276,16 +270,6 @@ def get_used_ports(): return last_used_ports[0] -def create_socket_from_port_list(available_ports: typing.Iterable[int], host: str) -> socket.socket | None: - for port in available_ports: - try: - return socket.create_server((host, port)) - except OSError: - pass - - return None - - @cache_argsless def get_static_server_data() -> dict: import worlds From 33f03387c4a013aaa9957579e7067768f385ce59 Mon Sep 17 00:00:00 2001 From: Uriel Date: Sat, 7 Mar 2026 17:37:50 -0300 Subject: [PATCH 29/46] this should check all usable ports before failing --- WebHostLib/customserver.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 376903fcf5..8647715b9a 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -222,8 +222,17 @@ def weighted_random(ranges: list[range], cum_weights: list[int]): def create_random_port_socket(game_ports: tuple[str | int], host: str) -> socket.socket: parsed_ports, weights, ephemeral_allowed = parse_game_ports(game_ports) - for _ in range(1024): + used_ports = get_used_ports() + i = 1024 + while True: port_num = weighted_random(parsed_ports, weights) + if port_num in used_ports: + continue + + i -=1 + if i == 0: + break + try: return socket.create_server((host, port_num)) except OSError: @@ -237,7 +246,7 @@ def create_random_port_socket(game_ports: tuple[str | int], host: str) -> socket def try_conns_per_process(p: psutil.Process) -> typing.Iterable[int]: try: - return map(lambda c: c.laddr.port, p.get_active_net_connections("tcp4")) + return map(lambda c: c.laddr.port, p.net_connections("tcp4")) except psutil.AccessDenied: return [] From eebd83df7619c457dc17c7dc643f6da0d74c959c Mon Sep 17 00:00:00 2001 From: Uriel Date: Sat, 7 Mar 2026 17:48:21 -0300 Subject: [PATCH 30/46] fix while loop --- WebHostLib/customserver.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 8647715b9a..4922995241 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -224,14 +224,12 @@ def create_random_port_socket(game_ports: tuple[str | int], host: str) -> socket parsed_ports, weights, ephemeral_allowed = parse_game_ports(game_ports) used_ports = get_used_ports() i = 1024 - while True: + while i > 0: port_num = weighted_random(parsed_ports, weights) if port_num in used_ports: continue - i -=1 - if i == 0: - break + i -= 0 try: return socket.create_server((host, port_num)) From 62f56e165a0947f3561dc06c424356113bdf898f Mon Sep 17 00:00:00 2001 From: Uriel Date: Sat, 7 Mar 2026 18:10:44 -0300 Subject: [PATCH 31/46] add return type to weighted random --- WebHostLib/customserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 4922995241..8a13556019 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -215,7 +215,7 @@ def parse_game_ports(game_ports: tuple[str | int]) -> GameRangePorts: return GameRangePorts(parsed_ports, weights, ephemeral_allowed) -def weighted_random(ranges: list[range], cum_weights: list[int]): +def weighted_random(ranges: list[range], cum_weights: list[int]) -> int: [picked] = random.choices(ranges, cum_weights=cum_weights) return random.randrange(picked.start, picked.stop, picked.step) From 62afec9733a749cd6014bba6d49f46e30e561d07 Mon Sep 17 00:00:00 2001 From: Uriel Date: Sat, 7 Mar 2026 22:24:13 -0300 Subject: [PATCH 32/46] Update WebHostLib/customserver.py Co-authored-by: black-sliver <59490463+black-sliver@users.noreply.github.com> --- WebHostLib/customserver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 8a13556019..a3bdba86de 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -227,6 +227,7 @@ def create_random_port_socket(game_ports: tuple[str | int], host: str) -> socket while i > 0: port_num = weighted_random(parsed_ports, weights) if port_num in used_ports: + used_ports = get_used_ports() continue i -= 0 From 9653c8d29c42e507adcf01e8221584643e3b9e46 Mon Sep 17 00:00:00 2001 From: Uriel Date: Sat, 7 Mar 2026 22:50:04 -0300 Subject: [PATCH 33/46] simplify tuple conversion check --- WebHostLib/customserver.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index a3bdba86de..f968d412ed 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -358,8 +358,7 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, del resource, file_limit # convert to tuple because its hashable - if not isinstance(game_ports, tuple): - game_ports = tuple(game_ports) + game_ports = tuple(game_ports) # establish DB connection for multidata and multisave db.bind(**ponyconfig) From 10d290833916132df3ccd2520760355672af76df Mon Sep 17 00:00:00 2001 From: Uriel Date: Sun, 8 Mar 2026 06:32:22 -0300 Subject: [PATCH 34/46] add tests --- WebHostLib/customserver.py | 6 +- test/webhost/test_port_allocation.py | 100 +++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 test/webhost/test_port_allocation.py diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index f968d412ed..dddad61d05 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -191,7 +191,7 @@ class GameRangePorts(typing.NamedTuple): @functools.cache -def parse_game_ports(game_ports: tuple[str | int]) -> GameRangePorts: +def parse_game_ports(game_ports: tuple[str | int, ...]) -> GameRangePorts: parsed_ports: list[range] = [] weights = [] ephemeral_allowed = False @@ -220,10 +220,10 @@ def weighted_random(ranges: list[range], cum_weights: list[int]) -> int: return random.randrange(picked.start, picked.stop, picked.step) -def create_random_port_socket(game_ports: tuple[str | int], host: str) -> socket.socket: +def create_random_port_socket(game_ports: tuple[str | int, ...], host: str) -> socket.socket: parsed_ports, weights, ephemeral_allowed = parse_game_ports(game_ports) used_ports = get_used_ports() - i = 1024 + i = 1024 if len(parsed_ports) > 0 else 0 while i > 0: port_num = weighted_random(parsed_ports, weights) if port_num in used_ports: diff --git a/test/webhost/test_port_allocation.py b/test/webhost/test_port_allocation.py new file mode 100644 index 0000000000..de22abbe7a --- /dev/null +++ b/test/webhost/test_port_allocation.py @@ -0,0 +1,100 @@ +import os +import statistics +import timeit +import unittest + +from WebHostLib.customserver import parse_game_ports, create_random_port_socket, get_used_ports + +ci = bool(os.environ.get("CI")) + + +class TestWebDescriptions(unittest.TestCase): + def test_parse_game_ports(self) -> None: + """Ensure that game ports with ranges are parsed correctly""" + val = parse_game_ports(("1000-2000", "2000-5000", "1000-2000", 20, 40, "20", "0")) + self.assertEqual(len(val.parsed_ports), 6, "Parsed port ranges is not the expected length") + self.assertEqual(len(val.weights), 6, "Parsed weights are not the expected length") + + self.assertEqual(val.parsed_ports[0], range(1000, 2001), "The first range wasn't parsed correctly") + self.assertEqual(val.parsed_ports[1], range(2000, 5001), "The second range wasn't parsed correctly") + self.assertEqual(val.parsed_ports[0], val.parsed_ports[2], + "The first and third range are not the same when they should be") + self.assertEqual(val.parsed_ports[3], range(20, 21), "The fourth range wasn't parsed correctly") + self.assertEqual(val.parsed_ports[4], range(40, 41), "The fifth range was not parsed correctly") + self.assertEqual(val.parsed_ports[3], val.parsed_ports[5], + "The fourth and last range are not the same when they should be") + + self.assertTrue(val.ephemeral_allowed, "The ephemeral allowed flag is not set even though it was passed") + + self.assertListEqual(val.weights, [1001, 4002, 5003, 5004, 5005, 5006], + "Cumulative weights are not the expected value") + + def test_parse_game_port_errors(self) -> None: + """Ensure that game ports with incorrect values raise the expected error""" + with self.assertRaises(ValueError, msg="Negative numbers didn't get interpreted as an invalid range"): + parse_game_ports(tuple("-50215")) + with self.assertRaises(ValueError, msg="Text got interpreted as a valid number"): + parse_game_ports(tuple("dwafawg")) + with self.assertRaises( + ValueError, + msg="A range with an extra dash at the end didn't get interpreted as an invalid number because of it's end dash" + ): + parse_game_ports(tuple("20-21215-")) + with self.assertRaises(ValueError, msg="Text got interpreted as a valid number for the start of a range"): + parse_game_ports(tuple("f-21215")) + + def test_random_port_socket_edge_cases(self) -> None: + # Try giving an empty tuple and fail over it + with self.assertRaises(OSError) as err: + create_random_port_socket(tuple(), "127.0.0.1") + self.assertEqual(err.exception.errno, 98, "Raised an unexpected error code") + self.assertEqual(err.exception.strerror, "No available ports", "Raised an unexpected error string") + + # Try only having ephemeral ports enabled + try: + create_random_port_socket(("0",), "127.0.0.1").close() + except OSError as err: + self.assertEqual(err.errno, 98, "Raised an unexpected error code") + # If it returns our error string that means something is wrong with our code + self.assertNotEqual(err.strerror, "No available ports", + "Raised an unexpected error string") + + # @unittest.skipUnless(ci, "can't guarantee free ports outside of CI") + def test_random_port_socket(self) -> None: + sockets = [] + for _ in range(6): + socket = create_random_port_socket(("8080-8085",), "127.0.0.1") + sockets.append(socket) + _, port = socket.getsockname() + self.assertIn(port, range(8080,8086), "Port of socket was not inside the expected range") + for s in sockets: + s.close() + + # Compared averages were calculated with a range of 100 in a Linux machine and then rounded up + sockets.clear() + time = [] + size = 65535 - (len(get_used_ports()) + 1024 + 4000) + for _ in range(10): + time.append(timeit.timeit(lambda: sockets.append( + create_random_port_socket(("1024-30000", "30001-65535"), "127.0.0.1") + ), number=size)) + + for s in sockets: + s.close() + + self.assertLess(statistics.fmean(time), 1.2, + f"Time took to allocate {size} ports consecutively is higher than expected") + + sockets.clear() + time.clear() + size = 65535 - (len(get_used_ports()) + 1024 + 5) + for _ in range(10): + time.append(timeit.timeit(lambda: sockets.append( + create_random_port_socket(("1024-30000", "30001-65535"), "127.0.0.1") + ), number=size)) + + for s in sockets: + s.close() + + self.assertLess(statistics.fmean(time), 5, + f"Time took to allocate {size} ports consecutively is higher than expected") From 27257204064d729bcaf6f0a2cefb6638c19eab92 Mon Sep 17 00:00:00 2001 From: Uriel Date: Sun, 8 Mar 2026 06:38:47 -0300 Subject: [PATCH 35/46] reformat file and change `create_random_port_socket` test --- WebHostLib/customserver.py | 4 +-- test/webhost/test_port_allocation.py | 41 ++++++++-------------------- 2 files changed, 13 insertions(+), 32 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index dddad61d05..f9f45eebd3 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -210,7 +210,7 @@ def parse_game_ports(game_ports: tuple[str | int, ...]) -> GameRangePorts: total_length += 1 weights.append(total_length) num = int(item) - parsed_ports.append(range(num, num+1)) + parsed_ports.append(range(num, num + 1)) return GameRangePorts(parsed_ports, weights, ephemeral_allowed) @@ -492,7 +492,7 @@ def run_server_process(name: str, ponyconfig: dict, static_server_data: dict, def run(self): while 1: - next_room = rooms_to_run.get(block=True, timeout=None) + next_room = rooms_to_run.get(block=True, timeout=None) gc.collect() task = asyncio.run_coroutine_threadsafe(start_room(next_room), loop) self._tasks.append(task) diff --git a/test/webhost/test_port_allocation.py b/test/webhost/test_port_allocation.py index de22abbe7a..4fbf32e6ed 100644 --- a/test/webhost/test_port_allocation.py +++ b/test/webhost/test_port_allocation.py @@ -1,6 +1,4 @@ import os -import statistics -import timeit import unittest from WebHostLib.customserver import parse_game_ports, create_random_port_socket, get_used_ports @@ -44,6 +42,7 @@ class TestWebDescriptions(unittest.TestCase): parse_game_ports(tuple("f-21215")) def test_random_port_socket_edge_cases(self) -> None: + """Verify if edge cases on creation of random port socket is working fine""" # Try giving an empty tuple and fail over it with self.assertRaises(OSError) as err: create_random_port_socket(tuple(), "127.0.0.1") @@ -59,42 +58,24 @@ class TestWebDescriptions(unittest.TestCase): self.assertNotEqual(err.strerror, "No available ports", "Raised an unexpected error string") - # @unittest.skipUnless(ci, "can't guarantee free ports outside of CI") + @unittest.skipUnless(ci, "can't guarantee free ports outside of CI") def test_random_port_socket(self) -> None: + """Verify if returned sockets use the correct port ranges""" sockets = [] for _ in range(6): socket = create_random_port_socket(("8080-8085",), "127.0.0.1") sockets.append(socket) _, port = socket.getsockname() - self.assertIn(port, range(8080,8086), "Port of socket was not inside the expected range") + self.assertIn(port, range(8080, 8086), "Port of socket was not inside the expected range") for s in sockets: s.close() - # Compared averages were calculated with a range of 100 in a Linux machine and then rounded up sockets.clear() - time = [] - size = 65535 - (len(get_used_ports()) + 1024 + 4000) - for _ in range(10): - time.append(timeit.timeit(lambda: sockets.append( - create_random_port_socket(("1024-30000", "30001-65535"), "127.0.0.1") - ), number=size)) + for _ in range(30_000): + socket = create_random_port_socket(("30000-65535",), "127.0.0.1") + sockets.append(socket) + _, port = socket.getsockname() + self.assertIn(port, range(30_000, 65536), "Port of socket was not inside the expected range") - for s in sockets: - s.close() - - self.assertLess(statistics.fmean(time), 1.2, - f"Time took to allocate {size} ports consecutively is higher than expected") - - sockets.clear() - time.clear() - size = 65535 - (len(get_used_ports()) + 1024 + 5) - for _ in range(10): - time.append(timeit.timeit(lambda: sockets.append( - create_random_port_socket(("1024-30000", "30001-65535"), "127.0.0.1") - ), number=size)) - - for s in sockets: - s.close() - - self.assertLess(statistics.fmean(time), 5, - f"Time took to allocate {size} ports consecutively is higher than expected") + for s in sockets: + s.close() From 7f2be5f0f538a0b50affeb057acae5d6f841ce29 Mon Sep 17 00:00:00 2001 From: Uriel Date: Sun, 8 Mar 2026 14:49:06 -0300 Subject: [PATCH 36/46] add more test cases for parse_game_ports --- test/webhost/test_port_allocation.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/test/webhost/test_port_allocation.py b/test/webhost/test_port_allocation.py index 4fbf32e6ed..76727ae5cd 100644 --- a/test/webhost/test_port_allocation.py +++ b/test/webhost/test_port_allocation.py @@ -6,7 +6,7 @@ from WebHostLib.customserver import parse_game_ports, create_random_port_socket, ci = bool(os.environ.get("CI")) -class TestWebDescriptions(unittest.TestCase): +class TestPortAllocating(unittest.TestCase): def test_parse_game_ports(self) -> None: """Ensure that game ports with ranges are parsed correctly""" val = parse_game_ports(("1000-2000", "2000-5000", "1000-2000", 20, 40, "20", "0")) @@ -27,6 +27,19 @@ class TestWebDescriptions(unittest.TestCase): self.assertListEqual(val.weights, [1001, 4002, 5003, 5004, 5005, 5006], "Cumulative weights are not the expected value") + val = parse_game_ports(()) + self.assertListEqual(val.parsed_ports, [], "Empty list of game port returned something") + self.assertFalse(val.ephemeral_allowed, "Empty list returned that ephemeral is allowed") + + val = parse_game_ports((0,)) + self.assertListEqual(val.parsed_ports, [], "Empty list of ranges returned something") + self.assertTrue(val.ephemeral_allowed, "List with just 0 is not allowing ephemeral ports") + + val = parse_game_ports((1,)) + self.assertEqual(val.parsed_ports, [range(1,2)], "Parsed ports doesn't contain the expected values") + self.assertFalse(val.ephemeral_allowed, "List with just single port returned that ephemeral is allowed") + + def test_parse_game_port_errors(self) -> None: """Ensure that game ports with incorrect values raise the expected error""" with self.assertRaises(ValueError, msg="Negative numbers didn't get interpreted as an invalid range"): From 07e2381cbb20517247168661d9076b2d49c647e4 Mon Sep 17 00:00:00 2001 From: Uriel Date: Sun, 8 Mar 2026 21:01:19 -0300 Subject: [PATCH 37/46] try to prevent busy-looping on create random port socket when doing test --- test/webhost/test_port_allocation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/webhost/test_port_allocation.py b/test/webhost/test_port_allocation.py index 76727ae5cd..b9fed9fbd6 100644 --- a/test/webhost/test_port_allocation.py +++ b/test/webhost/test_port_allocation.py @@ -84,7 +84,7 @@ class TestPortAllocating(unittest.TestCase): s.close() sockets.clear() - for _ in range(30_000): + for _ in range(30_000 - (len(get_used_ports()) + 100)): socket = create_random_port_socket(("30000-65535",), "127.0.0.1") sockets.append(socket) _, port = socket.getsockname() From f81e2fdf73600d7d26bee05a1fe4a63187c5ed0e Mon Sep 17 00:00:00 2001 From: Uriel Date: Mon, 9 Mar 2026 00:32:25 -0300 Subject: [PATCH 38/46] simplify parse game port tests to one assertListEqual --- test/webhost/test_port_allocation.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/test/webhost/test_port_allocation.py b/test/webhost/test_port_allocation.py index b9fed9fbd6..9f0e579365 100644 --- a/test/webhost/test_port_allocation.py +++ b/test/webhost/test_port_allocation.py @@ -10,20 +10,11 @@ class TestPortAllocating(unittest.TestCase): def test_parse_game_ports(self) -> None: """Ensure that game ports with ranges are parsed correctly""" val = parse_game_ports(("1000-2000", "2000-5000", "1000-2000", 20, 40, "20", "0")) - self.assertEqual(len(val.parsed_ports), 6, "Parsed port ranges is not the expected length") - self.assertEqual(len(val.weights), 6, "Parsed weights are not the expected length") - - self.assertEqual(val.parsed_ports[0], range(1000, 2001), "The first range wasn't parsed correctly") - self.assertEqual(val.parsed_ports[1], range(2000, 5001), "The second range wasn't parsed correctly") - self.assertEqual(val.parsed_ports[0], val.parsed_ports[2], - "The first and third range are not the same when they should be") - self.assertEqual(val.parsed_ports[3], range(20, 21), "The fourth range wasn't parsed correctly") - self.assertEqual(val.parsed_ports[4], range(40, 41), "The fifth range was not parsed correctly") - self.assertEqual(val.parsed_ports[3], val.parsed_ports[5], - "The fourth and last range are not the same when they should be") + self.assertListEqual(val.parsed_ports, + [range(1000, 2001), range(2000, 5001), range(1000, 2001), range(20, 21), range(40, 41), + range(20, 21)], "The parsed game ports are not the expected values") self.assertTrue(val.ephemeral_allowed, "The ephemeral allowed flag is not set even though it was passed") - self.assertListEqual(val.weights, [1001, 4002, 5003, 5004, 5005, 5006], "Cumulative weights are not the expected value") @@ -36,10 +27,9 @@ class TestPortAllocating(unittest.TestCase): self.assertTrue(val.ephemeral_allowed, "List with just 0 is not allowing ephemeral ports") val = parse_game_ports((1,)) - self.assertEqual(val.parsed_ports, [range(1,2)], "Parsed ports doesn't contain the expected values") + self.assertEqual(val.parsed_ports, [range(1, 2)], "Parsed ports doesn't contain the expected values") self.assertFalse(val.ephemeral_allowed, "List with just single port returned that ephemeral is allowed") - def test_parse_game_port_errors(self) -> None: """Ensure that game ports with incorrect values raise the expected error""" with self.assertRaises(ValueError, msg="Negative numbers didn't get interpreted as an invalid range"): From f76ea191c1b755364d99ce1a07005735cd82ca08 Mon Sep 17 00:00:00 2001 From: Uriel Date: Mon, 9 Mar 2026 03:09:51 -0300 Subject: [PATCH 39/46] make the range lesser for port test --- test/webhost/test_port_allocation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/webhost/test_port_allocation.py b/test/webhost/test_port_allocation.py index 9f0e579365..673565e6c7 100644 --- a/test/webhost/test_port_allocation.py +++ b/test/webhost/test_port_allocation.py @@ -74,7 +74,7 @@ class TestPortAllocating(unittest.TestCase): s.close() sockets.clear() - for _ in range(30_000 - (len(get_used_ports()) + 100)): + for _ in range(20_000 - len(get_used_ports())): socket = create_random_port_socket(("30000-65535",), "127.0.0.1") sockets.append(socket) _, port = socket.getsockname() From 8421ccce128549c41f413839388061128fbd0769 Mon Sep 17 00:00:00 2001 From: Uriel Date: Mon, 9 Mar 2026 03:44:32 -0300 Subject: [PATCH 40/46] reduce range on macOS --- test/webhost/test_port_allocation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/webhost/test_port_allocation.py b/test/webhost/test_port_allocation.py index 673565e6c7..4368097fce 100644 --- a/test/webhost/test_port_allocation.py +++ b/test/webhost/test_port_allocation.py @@ -1,4 +1,5 @@ import os +import platform import unittest from WebHostLib.customserver import parse_game_ports, create_random_port_socket, get_used_ports @@ -74,7 +75,8 @@ class TestPortAllocating(unittest.TestCase): s.close() sockets.clear() - for _ in range(20_000 - len(get_used_ports())): + length = 5_000 if platform.system() == "Darwin" else (30_000 - len(get_used_ports())) + for _ in range(length): socket = create_random_port_socket(("30000-65535",), "127.0.0.1") sockets.append(socket) _, port = socket.getsockname() From 1748048b440916832e8a7b9f4bc9bb0450b90d76 Mon Sep 17 00:00:00 2001 From: Uriel Date: Mon, 9 Mar 2026 05:09:05 -0300 Subject: [PATCH 41/46] Apply suggestions from code review Co-authored-by: black-sliver <59490463+black-sliver@users.noreply.github.com> --- test/webhost/test_port_allocation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/webhost/test_port_allocation.py b/test/webhost/test_port_allocation.py index 4368097fce..664e997c03 100644 --- a/test/webhost/test_port_allocation.py +++ b/test/webhost/test_port_allocation.py @@ -2,6 +2,7 @@ import os import platform import unittest +from Utils import is_macos from WebHostLib.customserver import parse_game_ports, create_random_port_socket, get_used_ports ci = bool(os.environ.get("CI")) @@ -75,7 +76,7 @@ class TestPortAllocating(unittest.TestCase): s.close() sockets.clear() - length = 5_000 if platform.system() == "Darwin" else (30_000 - len(get_used_ports())) + length = 5_000 if is_macos else (30_000 - len(get_used_ports())) for _ in range(length): socket = create_random_port_socket(("30000-65535",), "127.0.0.1") sockets.append(socket) From aff006a85f55058e25f47081ee463092e5eb2444 Mon Sep 17 00:00:00 2001 From: Uriel Date: Mon, 9 Mar 2026 07:00:44 -0300 Subject: [PATCH 42/46] Update WebHostLib/customserver.py Co-authored-by: black-sliver <59490463+black-sliver@users.noreply.github.com> --- WebHostLib/customserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index f9f45eebd3..a378959e74 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -270,7 +270,7 @@ def get_active_net_connections() -> typing.Iterable[int]: def get_used_ports(): last_used_ports: tuple[frozenset[int], float] = getattr(get_used_ports, "last", None) - t_hash = time.time() // 900 + t_hash = round(time.time() / 90) # cache for 90 seconds if last_used_ports is None or last_used_ports[1] != t_hash: last_used_ports = (frozenset(get_active_net_connections()), t_hash) setattr(get_used_ports, "last", last_used_ports) From 805b978403a70898caca26cc2f7b0332970ee9c9 Mon Sep 17 00:00:00 2001 From: Uriel Date: Mon, 9 Mar 2026 13:55:34 -0300 Subject: [PATCH 43/46] Apply suggestions from code review Co-authored-by: Doug Hoskisson --- WebHostLib/customserver.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index a378959e74..037b768a34 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -193,7 +193,7 @@ class GameRangePorts(typing.NamedTuple): @functools.cache def parse_game_ports(game_ports: tuple[str | int, ...]) -> GameRangePorts: parsed_ports: list[range] = [] - weights = [] + weights: list[int] = [] ephemeral_allowed = False total_length = 0 @@ -247,13 +247,13 @@ def try_conns_per_process(p: psutil.Process) -> typing.Iterable[int]: try: return map(lambda c: c.laddr.port, p.net_connections("tcp4")) except psutil.AccessDenied: - return [] + return () def get_active_net_connections() -> typing.Iterable[int]: # Don't even try to check if system using AIX - if psutil._common.AIX: - return [] + if psutil.AIX: + return () try: return map(lambda c: c.laddr.port, psutil.net_connections("tcp4")) From bd3686597f0cebe815981cbbb1fdc7f4fed84bb3 Mon Sep 17 00:00:00 2001 From: Uriel Date: Mon, 9 Mar 2026 13:56:14 -0300 Subject: [PATCH 44/46] remove unused import --- test/webhost/test_port_allocation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/webhost/test_port_allocation.py b/test/webhost/test_port_allocation.py index 664e997c03..d20e82295e 100644 --- a/test/webhost/test_port_allocation.py +++ b/test/webhost/test_port_allocation.py @@ -1,5 +1,4 @@ import os -import platform import unittest from Utils import is_macos From baad3ceede0d4aacbe004b9737d9b7e9cee59b33 Mon Sep 17 00:00:00 2001 From: Uriel Date: Mon, 9 Mar 2026 16:33:54 -0300 Subject: [PATCH 45/46] Update WebHostLib/customserver.py Co-authored-by: Doug Hoskisson --- WebHostLib/customserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 037b768a34..2a640b4b5b 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -269,7 +269,7 @@ def get_active_net_connections() -> typing.Iterable[int]: def get_used_ports(): - last_used_ports: tuple[frozenset[int], float] = getattr(get_used_ports, "last", None) + last_used_ports: tuple[frozenset[int], float] | None = getattr(get_used_ports, "last", None) t_hash = round(time.time() / 90) # cache for 90 seconds if last_used_ports is None or last_used_ports[1] != t_hash: last_used_ports = (frozenset(get_active_net_connections()), t_hash) From d57b3078b541c15bfc9c3137e337a5e08bc00944 Mon Sep 17 00:00:00 2001 From: Uriel Date: Mon, 9 Mar 2026 16:41:17 -0300 Subject: [PATCH 46/46] use generator expressions --- WebHostLib/customserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WebHostLib/customserver.py b/WebHostLib/customserver.py index 2a640b4b5b..ae812fdf2f 100644 --- a/WebHostLib/customserver.py +++ b/WebHostLib/customserver.py @@ -245,7 +245,7 @@ def create_random_port_socket(game_ports: tuple[str | int, ...], host: str) -> s def try_conns_per_process(p: psutil.Process) -> typing.Iterable[int]: try: - return map(lambda c: c.laddr.port, p.net_connections("tcp4")) + return (c.laddr.port for c in p.net_connections("tcp4")) except psutil.AccessDenied: return () @@ -256,7 +256,7 @@ def get_active_net_connections() -> typing.Iterable[int]: return () try: - return map(lambda c: c.laddr.port, psutil.net_connections("tcp4")) + return (c.laddr.port for c in psutil.net_connections("tcp4")) # raises AccessDenied when done on macOS except psutil.AccessDenied: # flatten the list of iterables