Core: Add .apignore format to not include files in APWorld Builder (#5779)

Utils.py

@@ -22,6 +22,7 @@ from settings import Settings, get_settings
 from time import sleep
 from typing import BinaryIO, Coroutine, Optional, Set, Dict, Any, Union, TypeGuard
 from yaml import load, load_all, dump
+from pathspec import PathSpec, GitIgnoreSpec
 
 try:
     from yaml import CLoader as UnsafeLoader, CSafeLoader as SafeLoader, CDumper as Dumper
@@ -387,6 +388,14 @@ def store_data_package_for_checksum(game: str, data: typing.Dict[str, Any]) -> N
             logging.debug(f"Could not store data package: {e}")
 
 
+def read_apignore(filename: str | pathlib.Path) -> PathSpec | None:
+    try:
+        with open(filename) as ignore_file:
+            return GitIgnoreSpec.from_lines(ignore_file)
+    except FileNotFoundError:
+        return None
+
+
 def get_default_adjuster_settings(game_name: str) -> Namespace:
     import LttPAdjuster
     adjuster_settings = Namespace()

data/GLOBAL.apignore

@@ -0,0 +1,13 @@
+# This file specifies patterns that are ignored by default for any world built with the "Build APWorlds" component.
+# These patterns can be overriden by a world-specific .apignore using !-prefixed patterns for negation.
+
+# Auto-created folders
+__MACOSX
+.DS_Store
+__pycache__
+
+# Unneeded files
+/archipelago.json
+/.apignore
+/.git
+/.gitignore

docs/apworld specification.md

@@ -79,10 +79,26 @@ will be packaged into an `.apworld` with a manifest file inside of it that looks
 
 This is the recommended workflow for packaging your world to an `.apworld`.
 
-## Extra Data
+### .apignore Exclusions
 
-The zip can contain arbitrary files in addition what was specified above.
+By default, any additional files inside of the world folder will be packaged into the resulting `.apworld` archive and
+can then be read by the world. However, if there are any other files that aren't needed in the resulting `.apworld`, you
+can automatically prevent the build component from including them by specifying them in a file called `.apignore` inside
+the root of the world folder.
 
+The `.apignore` file selects files in the same way as the `.gitignore` format with patterns separated by line describing
+which files to ignore. For example, an `.apignore` like this:
+
+```gitignore
+*.iso
+scripts/
+!scripts/needed.py
+```
+
+would ignore any `.iso` files and anything in the scripts folder except for `scripts/needed.py`.
+
+Some exclusions are made by default for all worlds such as `__pycache__` folders. These are listed in the
+`GLOBAL.apignore` file inside of the `data` directory.
 
 ## Caveats
 

requirements.txt

@@ -13,5 +13,6 @@ cymem>=2.0.13
 orjson>=3.11.4
 typing_extensions>=4.15.0
 pyshortcuts>=1.9.6
+pathspec>=0.12.1
 kivymd @ git+https://github.com/kivymd/KivyMD@5ff9d0d
 kivymd>=2.0.1.dev0

worlds/LauncherComponents.py

@@ -5,7 +5,7 @@ import weakref
 from enum import Enum, auto
 from typing import Optional, Callable, List, Iterable, Tuple
 
-from Utils import local_path, open_filename, is_frozen, is_kivy_running, open_file, user_path
+from Utils import local_path, open_filename, is_frozen, is_kivy_running, open_file, user_path, read_apignore
 
 
 class Type(Enum):
@@ -279,6 +279,10 @@ if not is_frozen():
             games = [(worldname, worldtype) for worldname, worldtype in AutoWorldRegister.world_types.items()
                      if not worldtype.zip_path]
 
+        global_apignores = read_apignore(local_path("data", "GLOBAL.apignore"))
+        if not global_apignores:
+            raise RuntimeError("Could not read global apignore file for build component")
+
         apworlds_folder = os.path.join("build", "apworlds")
         os.makedirs(apworlds_folder, exist_ok=True)
         for worldname, worldtype in games:
@@ -306,18 +310,17 @@ if not is_frozen():
             apworld = APWorldContainer(str(zip_path))
             apworld.game = worldtype.game
             manifest.update(apworld.get_manifest())
-            apworld.manifest_path = f"{file_name}/archipelago.json"
-            with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED,
-                                 compresslevel=9) as zf:
-                for path in pathlib.Path(world_directory).rglob("*"):
-                    relative_path = os.path.join(*path.parts[path.parts.index("worlds") + 1:])
-                    if "__MACOSX" in relative_path or ".DS_STORE" in relative_path or "__pycache__" in relative_path:
-                        continue
-                    if not relative_path.endswith("archipelago.json"):
-                        zf.write(path, relative_path)
+            apworld.manifest_path = os.path.join(file_name, "archipelago.json")
+
+            local_ignores = read_apignore(pathlib.Path(world_directory, ".apignore"))
+            apignores = global_apignores + local_ignores if local_ignores else global_apignores
+
+            with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED, compresslevel=9) as zf:
+                for file in apignores.match_tree_files(world_directory, negate=True):
+                    zf.write(pathlib.Path(world_directory, file), pathlib.Path(file_name, file))
+
                 zf.writestr(apworld.manifest_path, json.dumps(manifest))
         open_folder(apworlds_folder)
 
-
     components.append(Component("Build APWorlds", func=_build_apworlds, cli=True,
                                 description="Build APWorlds from loose-file world folders."))