Now possible to run generated modded seeds even without the json files.

* Process burnt results.
* Add fluidbox info to machine definition
* Recipe base cost now uses cheapest product source.
* free sample exclusions no longer hard-coded
* science pack pool exclusions no longer hard-coded
* burnt result is forced to have cost of machine used to obtain the results factored into the cost of crafting.
* name of machine is no longer assumed to be name of item.  machine definition now list item sources specifically.
* science pack difficulty is now the minimum of average estimated difficulty of unlocked recipes and 8.
* Actually exclude archipelago-extractor from mod list.

FactorioClient.py

@@ -65,6 +65,7 @@ class FactorioContext(CommonContext):
         self.factorio_json_text_parser = FactorioJSONtoTextParser(self)
         self.energy_link_increment = 0
         self.last_deplete = 0
+        self.custom_data_package = 0
 
     async def server_auth(self, password_requested: bool = False):
         if password_requested and not self.password:
@@ -170,7 +171,7 @@ async def game_watcher(ctx: FactorioContext):
                     if ctx.locations_checked != research_data:
                         bridge_logger.debug(
                             f"New researches done: "
-                            f"{[lookup_id_to_name[rid] for rid in research_data - ctx.locations_checked]}")
+                            f"{[lookup_id_to_name.get(rid, f'Unknown Research (ID: {rid})') for rid in research_data - ctx.locations_checked]}")
                         ctx.locations_checked = research_data
                         await ctx.send_msgs([{"cmd": 'LocationChecks', "locations": tuple(research_data)}])
                     death_link_tick = data.get("death_link_tick", 0)
@@ -268,7 +269,11 @@ async def factorio_server_watcher(ctx: FactorioContext):
                     transfer_item: NetworkItem = ctx.items_received[ctx.send_index]
                     item_id = transfer_item.item
                     player_name = ctx.player_names[transfer_item.player]
-                    if item_id not in Factorio.item_id_to_name:
+                    if ctx.custom_data_package:
+                        item_name = Factorio.item_id_to_name.get(item_id, f"Unknown Item (ID: {item_id})")
+                        factorio_server_logger.info(f"Sending {item_name} to Nauvis from {player_name}.{(' (Item name might not match the seed.)' if Factorio.data_version else '')}")
+                        commands[ctx.send_index] = f'/ap-get-technology {item_id}\t{ctx.send_index}\t{player_name}'
+                    elif item_id not in Factorio.item_id_to_name:
                         factorio_server_logger.error(f"Cannot send unknown item ID: {item_id}")
                     else:
                         item_name = Factorio.item_id_to_name[item_id]
@@ -297,6 +302,7 @@ async def get_info(ctx: FactorioContext, rcon_client: factorio_rcon.RCONClient):
     # 0.2.0 addition, not present earlier
     death_link = bool(info.get("death_link", False))
     ctx.energy_link_increment = info.get("energy_link", 0)
+    ctx.custom_data_package = info.get("custom_data_package", 0)
     logger.debug(f"Energy Link Increment: {ctx.energy_link_increment}")
     if ctx.energy_link_increment and ctx.ui:
         ctx.ui.enable_energy_link()

worlds/factorio/Mod.py

@@ -14,7 +14,7 @@ import Patch
 from . import Options
 
 from .Technologies import tech_table, recipes, free_sample_exclusions, progressive_technology_table, \
-    base_tech_table, tech_to_progressive_lookup, fluids
+    base_tech_table, tech_to_progressive_lookup, fluids, mods, tech_table, factorio_base_id
 
 template_env: Optional[jinja2.Environment] = None
 
@@ -35,7 +35,8 @@ base_info = {
     "dependencies": [
         "base >= 1.1.0",
         "? science-not-invited",
-        "? factory-levels"
+        "? factory-levels",
+        "! archipelago-extractor"
     ]
 }
 
@@ -117,6 +118,10 @@ def generate_mod(world, output_directory: str):
                 return base - (base - low) * distance
         return random.uniform(low, high)
 
+    all_items = tech_table.copy()
+    all_items["Attack Trap"] = factorio_base_id - 1
+    all_items["Evolution Trap"] = factorio_base_id - 2
+
     template_data = {
         "locations": locations, "player_names": multiworld.player_name, "tech_table": tech_table,
         "base_tech_table": base_tech_table, "tech_to_progressive_lookup": tech_to_progressive_lookup,
@@ -135,11 +140,13 @@ def generate_mod(world, output_directory: str):
         "free_sample_blacklist": {item: 1 for item in free_sample_exclusions},
         "progressive_technology_table": {tech.name: tech.progressive for tech in
                                          progressive_technology_table.values()},
+        "item_id_to_name": {f"{item_id}": item_name for item_name, item_id in all_items.items()},
         "custom_recipes": world.custom_recipes,
         "max_science_pack": multiworld.max_science_pack[player].value,
         "liquids": fluids,
         "goal": multiworld.goal[player].value,
-        "energy_link": multiworld.energy_link[player].value
+        "energy_link": multiworld.energy_link[player].value,
+        "custom_data_package": 1 if mods else 0
     }
 
     for factorio_option in Options.factorio_options:
@@ -192,6 +199,8 @@ def generate_mod(world, output_directory: str):
         f.write(locale_content)
     info = base_info.copy()
     info["name"] = mod_name
+    for mod in mods.values():
+        info["dependencies"].append(f"{mod.name} >= {mod.version}")
     with open(os.path.join(mod_dir, "info.json"), "wt") as f:
         json.dump(info, f, indent=4)
 

worlds/factorio/Technologies.py

@@ -4,6 +4,7 @@ import json
 import logging
 import os
 import string
+from sys import getrecursionlimit
 from collections import Counter
 from concurrent.futures import ThreadPoolExecutor
 from typing import Dict, Set, FrozenSet, Tuple, Union, List, Any
@@ -22,13 +23,14 @@ def load_json_data(data_name: str) -> Union[List[str], Dict[str, Any]]:
     import pkgutil
     return json.loads(pkgutil.get_data(__name__, "data/" + data_name + ".json").decode())
 
-
+# TODO: Make use of the lab information. (it has info on the science packs)
 techs_future = pool.submit(load_json_data, "techs")
 recipes_future = pool.submit(load_json_data, "recipes")
 resources_future = pool.submit(load_json_data, "resources")
 machines_future = pool.submit(load_json_data, "machines")
 fluids_future = pool.submit(load_json_data, "fluids")
 items_future = pool.submit(load_json_data, "items")
+mods_future = pool.submit(load_json_data, "mods")
 
 tech_table: Dict[str, int] = {}
 technology_table: Dict[str, Technology] = {}
@@ -94,6 +96,8 @@ class CustomTechnology(Technology):
 
     def __init__(self, origin: Technology, world, allowed_packs: Set[str], player: int):
         ingredients = origin.ingredients & allowed_packs
+        if origin.ingredients and not ingredients:
+            logging.warning(f"Technology {origin.name} has no vanilla science packs. Custom science packs are not supported.")
         military_allowed = "military-science-pack" in allowed_packs \
                            and ((ingredients & {"chemical-science-pack", "production-science-pack", "utility-science-pack"})
                                 or origin.name == "rocket-silo")
@@ -103,7 +107,8 @@ class CustomTechnology(Technology):
                 ingredients.add("military-science-pack")
             ingredients = list(ingredients)
             ingredients.sort()  # deterministic sample
-            ingredients = world.random.sample(ingredients, world.random.randint(1, len(ingredients)))
+            if ingredients:
+                ingredients = world.random.sample(ingredients, world.random.randint(1, len(ingredients)))
         elif origin.name == "rocket-silo" and military_allowed:
             ingredients.add("military-science-pack")
         super(CustomTechnology, self).__init__(origin.name, ingredients, origin.factorio_id)
@@ -115,13 +120,19 @@ class Recipe(FactorioElement):
     ingredients: Dict[str, int]
     products: Dict[str, int]
     energy: float
+    mining: bool
+    burning: bool
+    unlocked_at_start: bool
 
-    def __init__(self, name: str, category: str, ingredients: Dict[str, int], products: Dict[str, int], energy: float):
+    def __init__(self, name: str, category: str, ingredients: Dict[str, int], products: Dict[str, int], energy: float, mining: bool = False, unlocked_at_start: bool = False, burning: bool = False):
         self.name = name
         self.category = category
         self.ingredients = ingredients
         self.products = products
         self.energy = energy
+        self.mining = mining
+        self.burning = burning
+        self.unlocked_at_start = unlocked_at_start
 
     def __repr__(self):
         return f"{self.__class__.__name__}({self.name})"
@@ -147,28 +158,56 @@ class Recipe(FactorioElement):
     @property
     def rel_cost(self) -> float:
         ingredients = sum(self.ingredients.values())
-        return min(ingredients / amount for product, amount in self.products.items())
+        if all(amount == 0 for amount in self.products.values()):
+            return float('inf')
+        return min(ingredients / amount for product, amount in self.products.items() if amount > 0)
 
     @property
     def base_cost(self) -> Dict[str, int]:
         ingredients = Counter()
         for ingredient, cost in self.ingredients.items():
             if ingredient in all_product_sources:
+                recipe_counters: Dict[str, (Recipe, Counter)] = {}
                 for recipe in all_product_sources[ingredient]:
+                    recipe_ingredients = Counter()
                     if recipe.ingredients:
-                        ingredients.update({name: amount * cost / recipe.products[ingredient] for name, amount in
+                        recipe_ingredients.update({name: amount * cost / recipe.products[ingredient] for name, amount in
                                             recipe.base_cost.items()})
                     else:
-                        ingredients[ingredient] += recipe.energy * cost / recipe.products[ingredient]
+                        recipe_ingredients[ingredient] += recipe.energy * cost / recipe.products[ingredient]
+                    recipe_counters[recipe.name] = (recipe, recipe_ingredients)
+                selected_recipe_ingredients = None
+                for recipe_name, (recipe, recipe_ingredients) in recipe_counters.items():
+                    if not selected_recipe_ingredients or (
+                            sum([rel_cost.get(name, high_cost_item) * value for name, value in recipe_ingredients.items()])
+                            < sum([rel_cost.get(name, high_cost_item) * value for name, value in selected_recipe_ingredients.items()])):
+                        selected_recipe_ingredients = recipe_ingredients
+                ingredients.update(selected_recipe_ingredients)
             else:
                 ingredients[ingredient] += cost
         return ingredients
 
+    recursion_loop = 0
+    max_recursion_loop = 0.85 * getrecursionlimit()
+
+    def detect_recursive_loop(self) -> bool:
+        Recipe.recursion_loop += 1
+        if Recipe.max_recursion_loop < Recipe.recursion_loop:
+            Recipe.recursion_loop = 0
+            return True
+        for ingredient in self.ingredients.keys():
+            if ingredient in all_product_sources:
+                for ingredient_recipe in all_product_sources[ingredient]:
+                    if ingredient_recipe.ingredients:
+                        if ingredient_recipe.detect_recursive_loop():
+                            return True
+        Recipe.recursion_loop -= 1
+        return False
+
     @property
     def total_energy(self) -> float:
         """Total required energy (crafting time) for single craft"""
-        # TODO: multiply mining energy by 2 since drill has 0.5 speed
-        total_energy = self.energy
+        total_energy = (self.energy / machines[self.crafting_machine].speed)
         for ingredient, cost in self.ingredients.items():
             if ingredient in all_product_sources:
                 selected_recipe_energy = float('inf')
@@ -182,10 +221,71 @@ class Recipe(FactorioElement):
 
 
 class Machine(FactorioElement):
-    def __init__(self, name, categories):
+    def __init__(self, name, categories, machine_type, speed, item_sources, input_fluids, output_fluids):
         self.name: str = name
         self.categories: set = categories
+        self.machine_type: str = machine_type
+        self.speed: float = speed
+        self.item_sources: set = item_sources
+        self.input_fluids: int = int(input_fluids)
+        self.output_fluids: int = int(output_fluids)
 
+class Lab(FactorioElement):
+    def __init__(self, name, inputs):
+        self.name: str = name
+        self.inputs: set = inputs
+
+
+class Mod(FactorioElement):
+    def __init__(self, name, version):
+        self.name: str = name
+        self.version: str = version
+
+
+class Item(FactorioElement):
+    def __init__(self, name, stack_size, stackable, place_result, burnt_result, fuel_value, fuel_category, rocket_launch_products):
+        self.name: str = name
+        self.stack_size: int = stack_size
+        self.stackable: bool = stackable
+        self.place_result: str = place_result
+        self.burnt_result: str = burnt_result
+        self.fuel_value: int = fuel_value
+        self.fuel_category: str = fuel_category
+        self.rocket_launch_products: Dict[str, int] = rocket_launch_products
+        self.product_sources: Set[Recipe] = set()
+
+class Fluid(FactorioElement):
+    def __init__(self, name, default_temperature, max_temperature, heat_capacity):
+        self.name: str = name
+        if max_temperature == "inf":
+            max_temperature = 2**64
+        self.default_temperature: int = default_temperature
+        self.max_temperature: int = max_temperature
+        self.heat_capacity = heat_capacity
+        self.product_sources: Set[Recipe] = set()
+
+
+items: Dict[str, Item] = {}
+for name, item_data in items_future.result().items():
+    item = Item(name,
+                item_data.get("stack_size"),
+                item_data.get("stackable"),
+                item_data.get("place_result", None),
+                item_data.get("burnt_result", None),
+                item_data.get("fuel_value", 0),
+                item_data.get("fuel_category", None),
+                item_data.get("rocket_launch_products", {}))
+    items[name] = item
+del items_future
+
+fluids: Dict[str, Fluid] = {}
+for name, fluid_data in fluids_future.result().items():
+    fluid = Fluid(name,
+                  fluid_data.get("default_temperature", 0),
+                  fluid_data.get("max_temperature", 0),
+                  fluid_data.get("heat_capacity", 1000))
+    fluids[name] = fluid
+del fluids_future
 
 recipe_sources: Dict[str, Set[str]] = {}  # recipe_name -> technology source
 
@@ -213,38 +313,149 @@ for resource_name, resource_data in resources_future.result().items():
         if "required_fluid" in resource_data else {},
         "products": {data["name"]: data["amount"] for data in resource_data["products"].values()},
         "energy": resource_data["mining_time"],
-        "category": resource_data["category"]
+        "category": resource_data["category"],
+        "mining": True,
+        "unlocked_at_start": True
     }
 del resources_future
 
+machines: Dict[str, Machine] = {}
+labs: Dict[str, Lab] = {}
+rel_cost = {}
+high_cost_item = 10000
+
+for name, prototype in machines_future.result().items():
+    machine_prototype = prototype.get("common", {}).get("type", None)
+    machine_item_sources = set(prototype.get("common", {}).get("placeable_by", {}))
+    for machine_type, machine_data in prototype.items():
+        if machine_type == "lab":
+            lab = Lab(name, machine_data.get("inputs", set()))
+            labs[name] = lab
+        if machine_type == "offshore-pump":
+            fluid = machine_data.get("fluid", None)
+            speed = machine_data.get("speed", None)
+            if not fluid or not speed:
+                continue
+            category = f"offshore-pumping-{fluid}-{speed}"
+            raw_recipes[category] = {
+                "ingredients": {},
+                "products": {fluid: (speed*60)},
+                "energy": 1,
+                "category": category,
+                "mining": True,
+                "unlocked_at_start": True
+            }
+            machine = Machine(name, {category}, machine_prototype, 1, machine_item_sources, 0, 1)
+            machines[name] = machine
+        if machine_type == "crafting":
+            categories = machine_data.get("categories", set())
+            if not categories:
+                continue
+            # TODO: Use speed / fluid_box info
+            speed = machine_data.get("speed", 1)
+            input_fluid_box = machine_data.get("input_fluid_box", 0)
+            output_fluid_box = machine_data.get("output_fluid_box", 0)
+            machine = Machine(name, set(categories), machine_prototype, speed, machine_item_sources, input_fluid_box, output_fluid_box)
+            machines[name] = machine
+        if machine_type == "mining":
+            categories = machine_data.get("categories", set())
+            if not categories:
+                continue
+            speed = machine_data.get("speed", 1)
+            input_fluid_box = machine_data.get("input_fluid_box", False)  # Can this machine mine resources with required fluids?
+            output_fluid_box = machine_data.get("output_fluid_box", False)  # Can this machine mine fluid resources?
+            machine = machines.setdefault(name, Machine(name, set(categories), machine_prototype, speed, machine_item_sources, input_fluid_box, output_fluid_box))
+            machine.categories |= set(categories)  # character has both crafting and basic-solid
+            machine.speed = (machine.speed + speed) / 2
+            machines[name] = machine
+        if machine_type == "boiler":
+            input_fluid = machine_data.get("input_fluid")
+            output_fluid = machine_data.get("output_fluid")
+            target_temperature = machine_data.get("target_temperature")
+            energy_usage = machine_data.get("energy_usage")
+            amount = energy_usage / (target_temperature - fluids[input_fluid].default_temperature) / fluids[input_fluid].heat_capacity
+            amount *= 60
+            amount = int(amount)
+            category = f"boiling-{amount}-{input_fluid}-to-{output_fluid}-at-{target_temperature}-degrees-centigrade"
+            raw_recipes[category] = {
+                "ingredients": {input_fluid: amount},
+                "products": {output_fluid: amount},
+                "energy": 1,
+                "category": category,
+                "mining": True,
+                "unlocked_at_start": True
+            }
+            machine = Machine(name, {category}, machine_prototype, 1, machine_item_sources, 1, 1)
+            machines[name] = machine
+        if machine_type == "fuel_burner":
+            categories = set(machine_data.get("categories"))
+            fuel_items = {name: item for name, item in items.items() if item.burnt_result and item.fuel_category in categories}
+            if not fuel_items:
+                continue
+            energy_usage = machine_data.get("energy_usage")
+            for item_name, item in fuel_items.items():
+                recipe_name = f"burning-{item_name}-for-{item.burnt_result}"
+                raw_recipes[recipe_name] = {
+                    "ingredients": {item_name: 1},
+                    "products": {item.burnt_result: 1},
+                    "energy": item.fuel_value,
+                    "category": item.fuel_category,
+                    "burning": True,
+                    "unlocked_at_start": True
+                }
+            machine = machines.get(name, Machine(name, categories, machine_prototype, energy_usage * 60, machine_item_sources, 0, 0))
+            machines[name] = machine
+
+
+        # TODO: set up machine/recipe pairs for burners in order to retrieve the burnt_result from items.
+        # TODO: set up machine/recipe pairs for retrieving rocket_launch_products from items.
+
+del machines_future
+
 for recipe_name, recipe_data in raw_recipes.items():
     # example:
     # "accumulator":{"ingredients":{"iron-plate":2,"battery":5},"products":{"accumulator":1},"category":"crafting"}
     # FIXME: add mining?
-    recipe = Recipe(recipe_name, recipe_data["category"], recipe_data["ingredients"],
-                    recipe_data["products"], recipe_data["energy"] if "energy" in recipe_data else 0)
+    recipe = Recipe(recipe_name,
+                    recipe_data["category"],
+                    recipe_data["ingredients"],
+                    recipe_data["products"],
+                    recipe_data.get("energy", 0),
+                    recipe_data.get("mining", False),
+                    recipe_data.get("unlocked_at_start", False),
+                    recipe_data.get("burning", False))
     recipes[recipe_name] = recipe
-    if set(recipe.products).isdisjoint(
-            # prevents loop recipes like uranium centrifuging
-            set(recipe.ingredients)) and ("empty-barrel" not in recipe.products or recipe.name == "empty-barrel") and \
-            not recipe_name.endswith("-reprocessing"):
-        for product_name in recipe.products:
+    if set(recipe.products).isdisjoint(set(recipe.ingredients)):
+        for product_name in [product_name for product_name, amount in recipe.products.items() if amount > 0]:
             all_product_sources.setdefault(product_name, set()).add(recipe)
+            if recipe.detect_recursive_loop():
+                # prevents loop recipes like uranium centrifuging and fluid unbarreling
+                all_product_sources.setdefault(product_name, set()).remove(recipe)
+                if not all_product_sources[product_name]:
+                    del (all_product_sources[product_name])
+            if product_name in items:
+                items[product_name].product_sources.add(recipe)
+            if product_name in fluids:
+                fluids[product_name].product_sources.add(recipe)
 
 
-machines: Dict[str, Machine] = {}
+machines["assembling-machine-1"].categories |= machines["assembling-machine-3"].categories  # mod enables this
+machines["assembling-machine-2"].categories |= machines["assembling-machine-3"].categories
+machines["rocket-silo"].input_fluids = 3
+# machines["character"].categories.add("basic-crafting")
+# charter only knows the categories of "crafting" and "basic-solid" by default.
 
-for name, categories in machines_future.result().items():
-    machine = Machine(name, set(categories))
-    machines[name] = machine
 
-# add electric mining drill as a crafting machine to resolve basic-solid (mining)
-machines["electric-mining-drill"] = Machine("electric-mining-drill", {"basic-solid"})
-machines["pumpjack"] = Machine("pumpjack", {"basic-fluid"})
-machines["assembling-machine-1"].categories.add("crafting-with-fluid")  # mod enables this
-machines["character"].categories.add("basic-crafting")  # somehow this is implied and not exported
+mods: Dict[str, Mod] = {}
+
+for name, version in mods_future.result().items():
+    if name in ["base", "archipelago-extractor"]:
+        continue
+    mod = Mod(name, version)
+    mods[name] = mod
+
+del mods_future
 
-del machines_future
 
 # build requirements graph for all technology ingredients
 
@@ -254,7 +465,10 @@ for technology in technology_table.values():
 
 
 def unlock_just_tech(recipe: Recipe, _done) -> Set[Technology]:
-    current_technologies = recipe.unlocking_technologies
+    if recipe.unlocked_at_start:
+        current_technologies = set()
+    else:
+        current_technologies = recipe.unlocking_technologies
     for ingredient_name in recipe.ingredients:
         current_technologies |= recursively_get_unlocking_technologies(ingredient_name, _done,
                                                                        unlock_func=unlock_just_tech)
@@ -262,7 +476,10 @@ def unlock_just_tech(recipe: Recipe, _done) -> Set[Technology]:
 
 
 def unlock(recipe: Recipe, _done) -> Set[Technology]:
-    current_technologies = recipe.unlocking_technologies
+    if recipe.unlocked_at_start:
+        current_technologies = set()
+    else:
+        current_technologies = recipe.unlocking_technologies
     for ingredient_name in recipe.ingredients:
         current_technologies |= recursively_get_unlocking_technologies(ingredient_name, _done, unlock_func=unlock)
     current_technologies |= required_category_technologies[recipe.category]
@@ -289,21 +506,46 @@ def recursively_get_unlocking_technologies(ingredient_name, _done=None, unlock_f
     return current_technologies
 
 
+for item_name, item in items.items():
+    if item.place_result and machines.get(item.place_result, None):
+        machines[item.place_result].item_sources |= {item_name}
+
+
 required_machine_technologies: Dict[str, FrozenSet[Technology]] = {}
-for ingredient_name in machines:
+for ingredient_name, machine in machines.items():
+    if ingredient_name == "character":
+        required_machine_technologies[ingredient_name] = frozenset()
+        continue
+    techs = recursively_get_unlocking_technologies(ingredient_name)
+    for item_name in machine.item_sources:
+        techs |= recursively_get_unlocking_technologies(item_name)
+    required_machine_technologies[ingredient_name] = frozenset(techs)
+
+for ingredient_name in labs:
     required_machine_technologies[ingredient_name] = frozenset(recursively_get_unlocking_technologies(ingredient_name))
 
 logical_machines = {}
 machine_tech_cost = {}
+
+for category in machines["character"].categories:
+    machine_tech_cost[category] = (10000, "character", machines["character"].speed)
+
 for machine in machines.values():
+    if machine.name == "character":
+        continue
     for category in machine.categories:
-        current_cost, current_machine = machine_tech_cost.get(category, (10000, "character"))
         machine_cost = len(required_machine_technologies[machine.name])
-        if machine_cost < current_cost:
-            machine_tech_cost[category] = machine_cost, machine.name
+        if machine.machine_type == "character" and not machine_cost:
+            machine_cost = 10000
+        if category in machine_tech_cost:
+            current_cost, current_machine, current_speed = machine_tech_cost.get(category)
+            if machine_cost < current_cost or (machine_cost == current_cost and machine.speed > current_speed):
+                machine_tech_cost[category] = machine_cost, machine.name, machine.speed
+        else:
+            machine_tech_cost[category] = machine_cost, machine.name, machine.speed
 
 machine_per_category: Dict[str: str] = {}
-for category, (cost, machine_name) in machine_tech_cost.items():
+for category, (cost, machine_name, speed) in machine_tech_cost.items():
     machine_per_category[category] = machine_name
 
 del machine_tech_cost
@@ -313,6 +555,10 @@ required_category_technologies: Dict[str, FrozenSet[FrozenSet[Technology]]] = {}
 for category_name, machine_name in machine_per_category.items():
     techs = set()
     techs |= recursively_get_unlocking_technologies(machine_name)
+    if category_name in machines["character"].categories and techs:
+        # Character crafting/mining categories always have no tech assigned.
+        techs = set()
+        machine_per_category[category_name] = "character"
     required_category_technologies[category_name] = frozenset(techs)
 
 required_technologies: Dict[str, FrozenSet[Technology]] = Utils.KeyedDefaultDict(lambda ingredient_name: frozenset(
@@ -333,7 +579,10 @@ def get_rocket_requirements(silo_recipe: Recipe, part_recipe: Recipe, satellite_
     return {tech.name for tech in techs}
 
 
-free_sample_exclusions: Set[str] = all_ingredient_names | {"rocket-part"}
+free_sample_exclusions: Set[str] = all_ingredient_names.copy()  # Rocket-silo crafting recipe results to be added here.
+for name, recipe in recipes.items():
+    if machines[recipe.crafting_machine].machine_type == "rocket-silo":
+        free_sample_exclusions |= set(recipe.products)
 
 # progressive technologies
 # auto-progressive
@@ -447,37 +696,49 @@ useless_technologies: Set[str] = {tech_name for tech_name in common_tech_table
 
 lookup_id_to_name: Dict[int, str] = {item_id: item_name for item_name, item_id in tech_table.items()}
 
-rel_cost = {
-    "wood": 10000,
-    "iron-ore": 1,
-    "copper-ore": 1,
-    "stone": 1,
-    "crude-oil": 0.5,
-    "water": 0.001,
-    "coal": 1,
-    "raw-fish": 1000,
-    "steam": 0.01,
-    "used-up-uranium-fuel-cell": 1000
-}
+for name, recipe in {name: recipe for name, recipe in recipes.items() if recipe.mining and not recipe.ingredients}.items():
+    machine = machines[machine_per_category[recipe.category]]
+    cost = recipe.energy / machine.speed
+    for product_name, amount in recipe.products.items():
+        rel_cost[product_name] = cost / amount
 
-exclusion_list: Set[str] = all_ingredient_names | {"rocket-part", "used-up-uranium-fuel-cell"}
-fluids: Set[str] = set(fluids_future.result())
-del fluids_future
+
+def get_estimated_difficulty(recipe: Recipe):
+    base_ingredients = recipe.base_cost
+    cost = 0
+
+    for ingredient_name, amount in base_ingredients.items():
+        cost += rel_cost.get(ingredient_name, high_cost_item) * amount
+    if recipe.burning:
+        for item in machines[recipe.crafting_machine].item_sources:
+            if items[item].product_sources:
+                cost += min([get_estimated_difficulty(recipe) for recipe in items[item].product_sources])
+            else:
+                cost += high_cost_item
+
+    # print(f"{recipe.name}: {cost} ({({ingredient_name: rel_cost.get(ingredient_name, high_cost_item) * amount for ingredient_name, amount in base_ingredients.items()})})")
+    return cost
+
+
+for name, recipe in {name: recipe for name, recipe in recipes.items() if recipe.mining and recipe.ingredients}.items():
+    machine = machines[machine_per_category[recipe.category]]
+    cost = (recipe.energy / machine.speed) + get_estimated_difficulty(recipe)
+    for product_name, amount in recipe.products.items():
+        rel_cost[product_name] = cost / amount
+
+exclusion_list: Set[str] = free_sample_exclusions.copy()  # Also exclude the burnt results.
 
 
 @Utils.cache_argsless
 def get_science_pack_pools() -> Dict[str, Set[str]]:
-    def get_estimated_difficulty(recipe: Recipe):
-        base_ingredients = recipe.base_cost
-        cost = 0
-
-        for ingredient_name, amount in base_ingredients.items():
-            cost += rel_cost.get(ingredient_name, 1) * amount
-        return cost
-
     science_pack_pools: Dict[str, Set[str]] = {}
     already_taken = exclusion_list.copy()
-    current_difficulty = 5
+    unlocked_recipes = {name: recipe for name, recipe in recipes.items()
+                        if recipe.unlocked_at_start
+                        and not recipe.recursive_unlocking_technologies
+                        and get_estimated_difficulty(recipe) < high_cost_item}  # wood in recipe is expensive.
+    average_starting_difficulty = sum([get_estimated_difficulty(recipe) for name, recipe in unlocked_recipes.items()]) / len(unlocked_recipes)
+    current_difficulty = min(average_starting_difficulty, 8)
     for science_pack in Options.MaxSciencePack.get_ordered_science_packs():
         current = science_pack_pools[science_pack] = set()
         for name, recipe in recipes.items():
@@ -487,9 +748,7 @@ def get_science_pack_pools() -> Dict[str, Set[str]]:
 
         if science_pack == "automation-science-pack":
             # Can't handcraft automation science if fluids end up in its recipe, making the seed impossible.
-            current -= fluids
-        elif science_pack == "logistic-science-pack":
-            current |= {"steam"}
+            current -= set(fluids)
 
         current -= already_taken
         already_taken |= current
@@ -498,10 +757,9 @@ def get_science_pack_pools() -> Dict[str, Set[str]]:
     return science_pack_pools
 
 
-item_stack_sizes: Dict[str, int] = items_future.result()
-non_stacking_items: Set[str] = {item for item, stack in item_stack_sizes.items() if stack == 1}
-stacking_items: Set[str] = set(item_stack_sizes) - non_stacking_items
-valid_ingredients: Set[str] = stacking_items | fluids
+non_stacking_items: Set[str] = {name for name, item in items.items() if not item.stackable}
+stacking_items: Set[str] = set(items) - non_stacking_items
+valid_ingredients: Set[str] = stacking_items | set(fluids)
 
 # cleanup async helpers
 pool.shutdown()

worlds/factorio/__init__.py

@@ -8,7 +8,7 @@ from .Technologies import base_tech_table, recipe_sources, base_technology_table
     all_ingredient_names, all_product_sources, required_technologies, get_rocket_requirements, \
     progressive_technology_table, common_tech_table, tech_to_progressive_lookup, progressive_tech_table, \
     get_science_pack_pools, Recipe, recipes, technology_table, tech_table, factorio_base_id, useless_technologies, \
-    fluids, stacking_items, valid_ingredients
+    fluids, stacking_items, valid_ingredients, mods
 from .Shapes import get_shapes
 from .Mod import generate_mod
 from .Options import factorio_options, MaxSciencePack, Silo, Satellite, TechTreeInformation, Goal
@@ -54,8 +54,8 @@ class Factorio(World):
     item_name_groups = {
         "Progressive": set(progressive_tech_table.keys()),
     }
-    data_version = 5
-    required_client_version = (0, 3, 0)
+    data_version = 0 if mods else 5
+    required_client_version = (0, 3, 5) if mods else (0, 3, 0)   # TODO: Update required_client_version to (0, 3, 6) when that version releases.
 
     def __init__(self, world, player: int):
         super(Factorio, self).__init__(world, player)
@@ -224,7 +224,7 @@ class Factorio(World):
                 liquids_used += 1 if new_ingredient in fluids else 0
             new_ingredients[new_ingredient] = 1
         return Recipe(original.name, self.get_category(original.category, liquids_used), new_ingredients,
-                      original.products, original.energy)
+                      original.products, original.energy, original.mining, original.unlocked_at_start)
 
     def make_balanced_recipe(self, original: Recipe, pool: typing.Set[str], factor: float = 1,
                              allow_liquids: int = 2) -> Recipe:
@@ -258,6 +258,8 @@ class Factorio(World):
                 ingredient_energy = 2
             if not ingredient_raw:
                 ingredient_raw = 1
+            if not ingredient_energy:
+                ingredient_energy = 1/60
             if remaining_num_ingredients == 1:
                 max_raw = 1.1 * remaining_raw
                 min_raw = 0.9 * remaining_raw
@@ -293,13 +295,18 @@ class Factorio(World):
                 continue  # can't use this ingredient as we already have maximum liquid in our recipe.
 
             ingredient_recipe = recipes.get(ingredient, None)
-            if not ingredient_recipe and ingredient.endswith("-barrel"):
-                ingredient_recipe = recipes.get(f"fill-{ingredient}", None)
+            if not ingredient_recipe and ingredient in all_product_sources:
+                ingredient_recipe = min(all_product_sources[ingredient], key=lambda recipe: recipe.rel_cost)
             if not ingredient_recipe:
                 logging.warning(f"missing recipe for {ingredient}")
                 continue
             ingredient_raw = sum((count for ingredient, count in ingredient_recipe.base_cost.items()))
             ingredient_energy = ingredient_recipe.total_energy
+            if not ingredient_raw:
+                ingredient_raw = 1
+            if not ingredient_energy:
+                ingredient_energy = 1/60
+
             num_raw = remaining_raw / ingredient_raw / remaining_num_ingredients
             num_energy = remaining_energy / ingredient_energy / remaining_num_ingredients
             num = int(min(num_raw, num_energy))
@@ -317,7 +324,7 @@ class Factorio(World):
             logging.warning("could not randomize recipe")
 
         return Recipe(original.name, self.get_category(original.category, liquids_used), new_ingredients,
-                      original.products, original.energy)
+                      original.products, original.energy, original.mining, original.unlocked_at_start)
 
     def set_custom_technologies(self):
         custom_technologies = {}
@@ -330,11 +337,20 @@ class Factorio(World):
         original_rocket_part = recipes["rocket-part"]
         science_pack_pools = get_science_pack_pools()
         valid_pool = sorted(science_pack_pools[self.world.max_science_pack[self.player].get_max_pack()] & valid_ingredients)
+        if len(valid_pool) < 3:
+            # Not enough items in max pool. Extend to entire pool.
+            valid_pool = []
+            for pack in self.world.max_science_pack[self.player].get_ordered_science_packs():
+                valid_pool += sorted(science_pack_pools[pack] & valid_ingredients)
+            if len(valid_pool) < 3:
+                raise Exception("Not enough ingredients available for generation.")
         self.world.random.shuffle(valid_pool)
         self.custom_recipes = {"rocket-part": Recipe("rocket-part", original_rocket_part.category,
                                                      {valid_pool[x]: 10 for x in range(3)},
                                                      original_rocket_part.products,
-                                                     original_rocket_part.energy)}
+                                                     original_rocket_part.energy,
+                                                     original_rocket_part.mining,
+                                                     original_rocket_part.unlocked_at_start)}
 
         if self.world.recipe_ingredients[self.player]:
             valid_pool = []
@@ -363,7 +379,7 @@ class Factorio(World):
         bridge = "ap-energy-bridge"
         new_recipe = self.make_quick_recipe(
             Recipe(bridge, "crafting", {"replace_1": 1, "replace_2": 1, "replace_3": 1},
-                   {bridge: 1}, 10),
+                   {bridge: 1}, 10, False, self.world.energy_link[self.player].value),
             sorted(science_pack_pools[self.world.max_science_pack[self.player].get_ordered_science_packs()[0]]))
         for ingredient_name in new_recipe.ingredients:
             new_recipe.ingredients[ingredient_name] = self.world.random.randint(10, 100)

worlds/factorio/data/fluids.json

@@ -1 +1 @@
-["fluid-unknown","water","crude-oil","steam","heavy-oil","light-oil","petroleum-gas","sulfuric-acid","lubricant"]
+{"fluid-unknown":{"default_temperature":0,"max_temperature":0,"heat_capacity":1000},"water":{"default_temperature":15,"max_temperature":100,"heat_capacity":200},"crude-oil":{"default_temperature":25,"max_temperature":25,"heat_capacity":100},"steam":{"default_temperature":15,"max_temperature":1000,"heat_capacity":200},"heavy-oil":{"default_temperature":25,"max_temperature":25,"heat_capacity":100},"light-oil":{"default_temperature":25,"max_temperature":25,"heat_capacity":100},"petroleum-gas":{"default_temperature":25,"max_temperature":25,"heat_capacity":100},"sulfuric-acid":{"default_temperature":25,"max_temperature":25,"heat_capacity":100},"lubricant":{"default_temperature":25,"max_temperature":25,"heat_capacity":100}}

worlds/factorio/data/machines.json

@@ -1 +1 @@
-{"stone-furnace":{"smelting":true},"steel-furnace":{"smelting":true},"electric-furnace":{"smelting":true},"assembling-machine-1":{"crafting":true,"basic-crafting":true,"advanced-crafting":true},"assembling-machine-2":{"basic-crafting":true,"crafting":true,"advanced-crafting":true,"crafting-with-fluid":true},"assembling-machine-3":{"basic-crafting":true,"crafting":true,"advanced-crafting":true,"crafting-with-fluid":true},"oil-refinery":{"oil-processing":true},"chemical-plant":{"chemistry":true},"centrifuge":{"centrifuging":true},"rocket-silo":{"rocket-building":true},"character":{"crafting":true}}
+{"boiler":{"boiler":{"input_fluid":"water","output_fluid":"steam","target_temperature":165,"energy_usage":30000},"common":{"type":"boiler","placeable_by":{"boiler":true}}},"nuclear-reactor":{"fuel_burner":{"categories":{"nuclear":true},"energy_usage":666666.666666666627861559391021728515625},"common":{"type":"reactor","placeable_by":{"nuclear-reactor":true}}},"heat-exchanger":{"boiler":{"input_fluid":"water","output_fluid":"steam","target_temperature":500,"energy_usage":166666.66666666665696538984775543212890625},"common":{"type":"boiler","placeable_by":{"heat-exchanger":true}}},"burner-mining-drill":{"mining":{"categories":{"basic-solid":true},"speed":0.25,"input_fluid_box":false,"output_fluid_box":false},"common":{"type":"mining-drill","placeable_by":{"burner-mining-drill":true}}},"electric-mining-drill":{"mining":{"categories":{"basic-solid":true},"speed":0.5,"input_fluid_box":true,"output_fluid_box":false},"common":{"type":"mining-drill","placeable_by":{"electric-mining-drill":true}}},"offshore-pump":{"offshore-pump":{"fluid":"water","speed":20},"common":{"type":"offshore-pump","placeable_by":{"offshore-pump":true}}},"pumpjack":{"mining":{"categories":{"basic-fluid":true},"speed":1,"input_fluid_box":false,"output_fluid_box":true},"common":{"type":"mining-drill","placeable_by":{"pumpjack":true}}},"stone-furnace":{"crafting":{"speed":1,"categories":{"smelting":true},"input_fluid_box":0,"output_fluid_box":0},"common":{"type":"furnace","placeable_by":{"stone-furnace":true}}},"steel-furnace":{"crafting":{"speed":2,"categories":{"smelting":true},"input_fluid_box":0,"output_fluid_box":0},"common":{"type":"furnace","placeable_by":{"steel-furnace":true}}},"electric-furnace":{"crafting":{"speed":2,"categories":{"smelting":true},"input_fluid_box":0,"output_fluid_box":0},"common":{"type":"furnace","placeable_by":{"electric-furnace":true}}},"assembling-machine-1":{"crafting":{"speed":0.5,"categories":{"crafting":true,"basic-crafting":true,"advanced-crafting":true},"input_fluid_box":0,"output_fluid_box":0},"common":{"type":"assembling-machine","placeable_by":{"assembling-machine-1":true}}},"assembling-machine-2":{"crafting":{"speed":0.75,"categories":{"basic-crafting":true,"crafting":true,"advanced-crafting":true,"crafting-with-fluid":true},"input_fluid_box":1,"output_fluid_box":1},"common":{"type":"assembling-machine","placeable_by":{"assembling-machine-2":true}}},"assembling-machine-3":{"crafting":{"speed":1.25,"categories":{"basic-crafting":true,"crafting":true,"advanced-crafting":true,"crafting-with-fluid":true},"input_fluid_box":1,"output_fluid_box":1},"common":{"type":"assembling-machine","placeable_by":{"assembling-machine-3":true}}},"oil-refinery":{"crafting":{"speed":1,"categories":{"oil-processing":true},"input_fluid_box":2,"output_fluid_box":3},"common":{"type":"assembling-machine","placeable_by":{"oil-refinery":true}}},"chemical-plant":{"crafting":{"speed":1,"categories":{"chemistry":true},"input_fluid_box":2,"output_fluid_box":2},"common":{"type":"assembling-machine","placeable_by":{"chemical-plant":true}}},"centrifuge":{"crafting":{"speed":1,"categories":{"centrifuging":true},"input_fluid_box":0,"output_fluid_box":0},"common":{"type":"assembling-machine","placeable_by":{"centrifuge":true}}},"lab":{"lab":{"inputs":["automation-science-pack","logistic-science-pack","military-science-pack","chemical-science-pack","production-science-pack","utility-science-pack","space-science-pack"]},"common":{"type":"lab","placeable_by":{"lab":true}}},"rocket-silo":{"crafting":{"speed":1,"categories":{"rocket-building":true},"fixed_recipe":"rocket-part","input_fluid_box":0,"output_fluid_box":0},"common":{"type":"rocket-silo","placeable_by":{"rocket-silo":true}}},"character":{"crafting":{"speed":1,"categories":{"crafting":true},"input_fluid_box":0,"output_fluid_box":0},"mining":{"categories":{"basic-solid":true},"speed":0.5,"input_fluid_box":false,"output_fluid_box":false},"common":{"type":"character"}}}

worlds/factorio/data/mod/info.json

@@ -9,6 +9,7 @@
     "dependencies": [
       "base >= 1.1.0",
       "? science-not-invited",
-      "? factory-levels"
+      "? factory-levels",
+      "! archipelago-extractor"
     ]
 }

worlds/factorio/data/mod_template/control.lua

@@ -13,6 +13,7 @@ GOAL = {{ goal }}
 ARCHIPELAGO_DEATH_LINK_SETTING = "archipelago-death-link-{{ slot_player }}-{{ seed_name }}"
 ENERGY_INCREMENT = {{ energy_link * 1000000 }}
 ENERGY_LINK_EFFICIENCY = 0.75
+CUSTOM_DATA_PACKAGE = {{ custom_data_package }}
 
 if settings.global[ARCHIPELAGO_DEATH_LINK_SETTING].value then
     DEATH_LINK = 1
@@ -515,6 +516,10 @@ commands.add_command("ap-get-technology", "Grant a technology, used by the Archi
     local item_name = chunks[1]
     local index = chunks[2]
     local source = chunks[3] or "Archipelago"
+
+    if CUSTOM_DATA_PACKAGE == 1 then
+        item_name = item_id_to_name[item_name] or item_name
+    end
     if index == -1 then -- for coop sync and restoring from an older savegame
         tech = force.technologies[item_name]
         if tech.researched ~= true then
@@ -571,7 +576,8 @@ commands.add_command("ap-rcon-info", "Used by the Archipelago client to get info
         ["slot_name"] = SLOT_NAME,
         ["seed_name"] = SEED_NAME,
         ["death_link"] = DEATH_LINK,
-        ["energy_link"] = ENERGY_INCREMENT
+        ["energy_link"] = ENERGY_INCREMENT,
+        ["custom_data_package"] = CUSTOM_DATA_PACKAGE
     }))
 end)
 
@@ -598,3 +604,4 @@ end)
 
 -- data
 progressive_technologies = {{ dict_to_lua(progressive_technology_table) }}
+item_id_to_name = {{ dict_to_lua(item_id_to_name) }}

worlds/factorio/data/mod_template/data-final-fixes.lua

@@ -141,6 +141,9 @@ end
 {# This got complex, but seems to be required to hit all corner cases #}
 function adjust_energy(recipe_name, factor)
     local recipe = data.raw.recipe[recipe_name]
+    if recipe == nil then
+        error("Some mod that is installed has removed recipe \"" .. recipe_name .. "\"")
+    end
     local energy = recipe.energy_required
 
     if (recipe.normal ~= nil) then
@@ -168,6 +171,9 @@ end
 
 function set_energy(recipe_name, energy)
     local recipe = data.raw.recipe[recipe_name]
+    if recipe == nil then
+        error("Some mod that is installed has removed recipe \"" .. recipe_name .. "\"")
+    end
 
     if (recipe.normal ~= nil) then
         recipe.normal.energy_required = energy
@@ -200,6 +206,9 @@ data.raw["ammo"]["artillery-shell"].stack_size = 10
 {# each randomized tech gets set to be invisible, with new nodes added that trigger those #}
 {%- for original_tech_name, item_name, receiving_player, advancement in locations %}
 original_tech = technologies["{{original_tech_name}}"]
+if original_tech == nil then
+    error("Some mod that is installed has removed tech \"{{original_tech_name}}\"")
+end
 {#- the tech researched by the local player #}
 new_tree_copy = table.deepcopy(template_tech)
 new_tree_copy.name = "ap-{{ tech_table[original_tech_name] }}-"{# use AP ID #}
@@ -232,13 +241,13 @@ data:extend{new_tree_copy}
 {% endfor %}
 {% if recipe_time_scale %}
 {%- for recipe_name, recipe in recipes.items() %}
-{%- if recipe.category not in ("basic-solid", "basic-fluid") %}
+{%- if not recipe.mining and not recipe.burning %}
 adjust_energy("{{ recipe_name }}", {{ flop_random(*recipe_time_scale) }})
 {%- endif %}
 {%- endfor -%}
 {% elif recipe_time_range %}
 {%- for recipe_name, recipe in recipes.items() %}
-{%- if recipe.category not in ("basic-solid", "basic-fluid") %}
+{%- if not recipe.mining and not recipe.burning %}
 set_energy("{{ recipe_name }}", {{ flop_random(*recipe_time_range) }})
 {%- endif %}
 {%- endfor -%}

worlds/factorio/data/mods.json

@@ -0,0 +1 @@
+{"base":"1.1.69","archipelago-extractor":"1.1.1"}