forked from mirror/Archipelago
36bab6f52a
- The Shane and Sebastian Portrait filler items werent tagged properly. - The Beach Farm adds one secretsanity check, so it needs to be in the allsanity preset - The Allsanity preset is renamed to "Maxsanity" to encourage better defined terminology and Allsanity means something else - The "All Random" preset has been removed entirely. It has been the cause of too many beginner footguns over the years. People can still achieve the effect manually, but at least they'll have to try a little bit to ruin their own experience.
289 lines
9.9 KiB
Python
289 lines
9.9 KiB
Python
import os
|
|
import time
|
|
import unittest
|
|
from dataclasses import dataclass
|
|
from statistics import mean, median, variance, stdev
|
|
from typing import List
|
|
|
|
from BaseClasses import get_seed
|
|
from Fill import distribute_items_restrictive, balance_multiworld_progression
|
|
from worlds import AutoWorld
|
|
from ..bases import SVTestCase, setup_multiworld
|
|
from ..options.presets import default_6_x_x, maxsanity_no_mods_6_x_x, minimal_locations_maximal_items, maxsanity_mods_7_x_x, maxsanity_no_mods_7_x_x, \
|
|
default_7_x_x
|
|
|
|
default_number_generations = 25
|
|
acceptable_deviation = 4
|
|
|
|
|
|
@dataclass
|
|
class PerformanceResults:
|
|
case: SVTestCase
|
|
|
|
amount_of_players: int
|
|
results: List[float]
|
|
acceptable_mean: float
|
|
|
|
def __repr__(self):
|
|
size = size_name(self.amount_of_players)
|
|
|
|
total_time = sum(self.results)
|
|
mean_time = mean(self.results)
|
|
median_time = median(self.results)
|
|
stdev_time = stdev(self.results, mean_time)
|
|
variance_time = variance(self.results, mean_time)
|
|
|
|
return f"""Generated {len(self.results)} {size} multiworlds in {total_time:.2f} seconds. Average {mean_time:.2f} seconds (Acceptable: {self.acceptable_mean:.2f})
|
|
Mean: {mean_time:.2f} Median: {median_time:.2f} Stdeviation: {stdev_time:.2f} Variance: {variance_time:.4f} Deviation percent: {stdev_time / mean_time:.2%}"""
|
|
|
|
|
|
class SVPerformanceTestCase(SVTestCase):
|
|
acceptable_time_per_player: float
|
|
results: List[PerformanceResults]
|
|
|
|
# Set False to not call the fill in the tests"""
|
|
skip_fill: bool = True
|
|
# Set True to print results as CSV"""
|
|
csv: bool = False
|
|
|
|
@classmethod
|
|
def setUpClass(cls) -> None:
|
|
performance_tests_key = "performance"
|
|
if performance_tests_key not in os.environ or os.environ[performance_tests_key] != "True":
|
|
raise unittest.SkipTest("Performance tests disabled")
|
|
|
|
super().setUpClass()
|
|
|
|
fill_tests_key = "fill"
|
|
if fill_tests_key in os.environ:
|
|
cls.skip_fill = os.environ[fill_tests_key] != "True"
|
|
|
|
fixed_seed_key = "fixed_seed"
|
|
if fixed_seed_key in os.environ:
|
|
cls.fixed_seed = bool(os.environ[fixed_seed_key])
|
|
else:
|
|
cls.fixed_seed = False
|
|
|
|
number_generations_key = "number_gen"
|
|
if number_generations_key in os.environ:
|
|
cls.number_generations = int(os.environ[number_generations_key])
|
|
else:
|
|
cls.number_generations = default_number_generations
|
|
|
|
csv_key = "csv"
|
|
if csv_key in os.environ:
|
|
cls.csv = bool(os.environ[csv_key])
|
|
|
|
@classmethod
|
|
def tearDownClass(cls) -> None:
|
|
if cls.csv:
|
|
csved_results = (f"{type(result.case).__name__},{result.amount_of_players},{val:.6f}"
|
|
for result in cls.results for val in result.results)
|
|
for r in csved_results:
|
|
print(r)
|
|
else:
|
|
case = None
|
|
for result in cls.results:
|
|
if type(result.case) is not case:
|
|
case = type(result.case)
|
|
print(case.__name__)
|
|
print(result)
|
|
print()
|
|
|
|
super().tearDownClass()
|
|
|
|
def performance_test_multiworld(self, options):
|
|
amount_of_players = len(options)
|
|
acceptable_average_time = self.acceptable_time_per_player * amount_of_players
|
|
total_time = 0
|
|
all_times = []
|
|
seeds = [get_seed() for _ in range(self.number_generations)] if not self.fixed_seed else [74898321050311118687] * self.number_generations
|
|
|
|
for i, seed in enumerate(seeds):
|
|
with self.subTest(f"Seed: {seed}"):
|
|
time_before = time.time()
|
|
|
|
print("Starting world setup")
|
|
multiworld = setup_multiworld(options, seed)
|
|
if not self.skip_fill:
|
|
distribute_items_restrictive(multiworld)
|
|
AutoWorld.call_all(multiworld, 'post_fill')
|
|
if multiworld.players > 1:
|
|
balance_multiworld_progression(multiworld)
|
|
|
|
time_after = time.time()
|
|
elapsed_time = time_after - time_before
|
|
total_time += elapsed_time
|
|
all_times.append(elapsed_time)
|
|
print(f"Multiworld {i + 1}/{self.number_generations} [{seed}] generated in {elapsed_time:.4f} seconds")
|
|
# tester.assertLessEqual(elapsed_time, acceptable_average_time * acceptable_deviation)
|
|
|
|
self.results.append(PerformanceResults(self, amount_of_players, all_times, acceptable_average_time))
|
|
self.assertLessEqual(mean(all_times), acceptable_average_time)
|
|
|
|
|
|
def size_name(number_players):
|
|
if number_players == 1:
|
|
return "solo"
|
|
elif number_players == 2:
|
|
return "duo"
|
|
elif number_players == 3:
|
|
return "trio"
|
|
return f"{number_players}-player"
|
|
|
|
|
|
class TestDefaultOptions6xx(SVPerformanceTestCase):
|
|
acceptable_time_per_player = 2
|
|
options = default_6_x_x()
|
|
results = []
|
|
|
|
def test_solo(self):
|
|
number_players = 1
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
def test_duo(self):
|
|
number_players = 2
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
@unittest.skip
|
|
def test_5_player(self):
|
|
number_players = 5
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
@unittest.skip
|
|
def test_10_player(self):
|
|
number_players = 10
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
|
|
class TestDefaultOptions7xx(SVPerformanceTestCase):
|
|
acceptable_time_per_player = 2
|
|
options = default_7_x_x()
|
|
results = []
|
|
|
|
def test_solo(self):
|
|
number_players = 1
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
def test_duo(self):
|
|
number_players = 2
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
def test_5_player(self):
|
|
number_players = 5
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
@unittest.skip
|
|
def test_10_player(self):
|
|
number_players = 10
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
|
|
class TestMinLocationMaxItems(SVPerformanceTestCase):
|
|
acceptable_time_per_player = 0.3
|
|
options = minimal_locations_maximal_items()
|
|
results = []
|
|
|
|
def test_solo(self):
|
|
number_players = 1
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
def test_duo(self):
|
|
number_players = 2
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
def test_5_player(self):
|
|
number_players = 5
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
def test_10_player(self):
|
|
number_players = 10
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
|
|
class TestMaxsanityWithoutMods6xx(SVPerformanceTestCase):
|
|
acceptable_time_per_player = 10
|
|
options = maxsanity_no_mods_6_x_x()
|
|
results = []
|
|
|
|
def test_solo(self):
|
|
number_players = 1
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
@unittest.skip
|
|
def test_duo(self):
|
|
number_players = 2
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
@unittest.skip
|
|
def test_5_player(self):
|
|
number_players = 5
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
@unittest.skip
|
|
def test_10_player(self):
|
|
number_players = 10
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
|
|
class TestMaxsanityWithoutMods7xx(SVPerformanceTestCase):
|
|
acceptable_time_per_player = 10
|
|
options = maxsanity_no_mods_7_x_x()
|
|
results = []
|
|
|
|
def test_solo(self):
|
|
number_players = 1
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
def test_duo(self):
|
|
number_players = 2
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
@unittest.skip
|
|
def test_5_player(self):
|
|
number_players = 5
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
@unittest.skip
|
|
def test_10_player(self):
|
|
number_players = 10
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
|
|
class TestMaxsanityWithMods7xx(SVPerformanceTestCase):
|
|
acceptable_time_per_player = 25
|
|
options = maxsanity_mods_7_x_x()
|
|
results = []
|
|
|
|
@unittest.skip
|
|
def test_solo(self):
|
|
number_players = 1
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|
|
|
|
@unittest.skip
|
|
def test_duo(self):
|
|
number_players = 2
|
|
multiworld_options = [self.options] * number_players
|
|
self.performance_test_multiworld(multiworld_options)
|