From 98af2ec35aba6004f3372af7a69a9bb5a13186bb Mon Sep 17 00:00:00 2001 From: embeddedt <42941056+embeddedt@users.noreply.github.com> Date: Sun, 27 Apr 2025 18:35:28 -0400 Subject: [PATCH] Rewrite resource pack caching to use a tree --- .../ReloadableResourceManagerMixin.java | 18 -- .../resources/PackResourcesCacheEngine.java | 210 ++++++++++-------- .../ForgePathPackResourcesMixin.java | 107 +++++++++ 3 files changed, 223 insertions(+), 112 deletions(-) delete mode 100644 common/src/main/java/org/embeddedt/modernfix/common/mixin/perf/resourcepacks/ReloadableResourceManagerMixin.java create mode 100644 forge/src/main/java/org/embeddedt/modernfix/forge/mixin/perf/resourcepacks/ForgePathPackResourcesMixin.java diff --git a/common/src/main/java/org/embeddedt/modernfix/common/mixin/perf/resourcepacks/ReloadableResourceManagerMixin.java b/common/src/main/java/org/embeddedt/modernfix/common/mixin/perf/resourcepacks/ReloadableResourceManagerMixin.java deleted file mode 100644 index 3fd155cf..00000000 --- a/common/src/main/java/org/embeddedt/modernfix/common/mixin/perf/resourcepacks/ReloadableResourceManagerMixin.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.embeddedt.modernfix.common.mixin.perf.resourcepacks; - -import net.minecraft.server.packs.resources.ReloadableResourceManager; -import org.embeddedt.modernfix.ModernFix; -import org.embeddedt.modernfix.resources.PackResourcesCacheEngine; -import org.spongepowered.asm.mixin.Mixin; -import org.spongepowered.asm.mixin.injection.At; -import org.spongepowered.asm.mixin.injection.Inject; -import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable; - -@Mixin(ReloadableResourceManager.class) -public class ReloadableResourceManagerMixin { - @Inject(method = "createReload", at = @At("HEAD")) - private void invalidateResourceCaches(CallbackInfoReturnable cir) { - ModernFix.LOGGER.info("Invalidating pack caches"); - PackResourcesCacheEngine.invalidate(); - } -} diff --git a/common/src/main/java/org/embeddedt/modernfix/resources/PackResourcesCacheEngine.java b/common/src/main/java/org/embeddedt/modernfix/resources/PackResourcesCacheEngine.java index 2c347e36..479c8398 100644 --- a/common/src/main/java/org/embeddedt/modernfix/resources/PackResourcesCacheEngine.java +++ b/common/src/main/java/org/embeddedt/modernfix/resources/PackResourcesCacheEngine.java @@ -3,88 +3,131 @@ package org.embeddedt.modernfix.resources; import com.google.common.base.Joiner; import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.mojang.datafixers.util.Pair; +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet; import net.minecraft.resources.ResourceLocation; +import net.minecraft.server.packs.PackResources; import net.minecraft.server.packs.PackType; import org.embeddedt.modernfix.ModernFix; -import org.embeddedt.modernfix.platform.ModernFixPlatformHooks; import org.embeddedt.modernfix.util.PackTypeHelper; +import org.jetbrains.annotations.Nullable; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; -import java.util.function.BiFunction; import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; import java.util.stream.Stream; /** * The core of the resource pack cache system. - * - * Using a dedicated set and also separate lists is important; testing without this showed a huge performance - * drop. */ public class PackResourcesCacheEngine { private static final Joiner SLASH_JOINER = Joiner.on('/'); - private final Map> namespacesByType; - private final Set containedPaths; - private final EnumMap>> resourceListings; + static class Node { + Map children; + + void optimize() { + if (children != null) { + for (var entry : children.entrySet()) { + var oldNode = entry.getValue(); + oldNode.optimize(); + if (oldNode.children == null) { + entry.setValue(EMPTY); + } + } + children = Map.copyOf(children); + } else { + children = Map.of(); + } + } + + void collectResources(String namespace, Path baseNioPath, String[] pathComponents, int curIndex, int maxDepth, PackResources.ResourceOutput output) { + if (curIndex > maxDepth) { + return; + } + if (curIndex < pathComponents.length) { + String component; + do { + component = pathComponents[curIndex]; + if (!component.isEmpty()) { + break; + } + curIndex++; + maxDepth++; + } while(true); + + Node n = getChild(component); + if (n != null) { + n.collectResources(namespace, baseNioPath, pathComponents, curIndex + 1, maxDepth, output); + } + } else { + // We reached the desired path. Collect all resources + this.outputResources(namespace, baseNioPath, String.join("/", pathComponents), output); + } + + } + + void outputResources(String namespace, Path baseNioPath, String path, PackResources.ResourceOutput output) { + if (children.isEmpty()) { + // This is a terminal node. + ResourceLocation location = new ResourceLocation(namespace, path); + output.accept(location, () -> Files.newInputStream(baseNioPath.resolve(path))); + } else { + for (var entry : children.entrySet()) { + entry.getValue().outputResources(namespace, baseNioPath, path + "/" + entry.getKey(), output); + } + } + } + + @Nullable + Node getChild(String name) { + return children.get(name); + } + } + + private static final Node EMPTY = new Node(); + + private final Node root = new Node(); + private final Map rootPathsByType = new Object2ObjectOpenHashMap<>(); + private volatile boolean cacheGenerationFlag = false; private List cacheGenerationTasks = new ArrayList<>(); private Path debugPath; - public PackResourcesCacheEngine(Function> namespacesRetriever, BiFunction basePathRetriever) { - this.namespacesByType = new EnumMap<>(PackType.class); - for(PackType type : PackType.values()) { - if(!PackTypeHelper.isVanillaPackType(type)) - continue; - this.namespacesByType.put(type, namespacesRetriever.apply(type)); - } - this.containedPaths = new ObjectOpenHashSet<>(); - this.resourceListings = new EnumMap<>(PackType.class); + public PackResourcesCacheEngine(Function basePathRetriever) { // used for log message - this.debugPath = basePathRetriever.apply(PackType.CLIENT_RESOURCES, "minecraft").toAbsolutePath(); + this.debugPath = basePathRetriever.apply(PackType.CLIENT_RESOURCES).toAbsolutePath(); + this.root.children = new Object2ObjectOpenHashMap<>(); + ObjectOpenHashSet pathKeys = new ObjectOpenHashSet<>(); for(PackType type : PackType.values()) { - Collection namespaces = PackTypeHelper.isVanillaPackType(type) ? this.namespacesByType.get(type) : namespacesRetriever.apply(type); - Collection> namespacedRoots = namespaces.stream().map(s -> Pair.of(s, basePathRetriever.apply(type, s).toAbsolutePath())).collect(Collectors.toList()); + var typeRoot = new Node(); + this.root.children.put(type.getDirectory(), typeRoot); + Path root = basePathRetriever.apply(type); + this.rootPathsByType.put(type, root); cacheGenerationTasks.add(() -> { - ImmutableMap.Builder> packTypedMap = ImmutableMap.builder(); - for(Pair pair : namespacedRoots) { - try { - ImmutableList.Builder namespacedList = ImmutableList.builder(); - String namespace = pair.getFirst(); - Path root = pair.getSecond(); - String[] prefix = new String[] { type.getDirectory(), namespace }; - try (Stream stream = Files.find(root, Integer.MAX_VALUE, (p, a) -> a.isRegularFile())) { - stream - .map(path -> root.relativize(path.toAbsolutePath())) - .filter(PackResourcesCacheEngine::isValidCachedResourcePath) - .forEach(path -> { - CachedResourcePath cachedPath = new CachedResourcePath(prefix, path); - synchronized (this.containedPaths) { - this.containedPaths.add(cachedPath); + try { + try (Stream stream = Files.find(root, Integer.MAX_VALUE, (p, a) -> a.isRegularFile())) { + stream + .map(path -> root.relativize(path.toAbsolutePath())) + .filter(PackResourcesCacheEngine::isValidCachedResourcePath) + .forEach(path -> { + var node = typeRoot; + for (Path component : path) { + String key = pathKeys.addOrGet(component.toString()); + if (node.children == null) { + node.children = new Object2ObjectOpenHashMap<>(); } - //if(!cachedPath.getFileName().endsWith(".mcmeta")) - namespacedList.add(cachedPath); - }); - } - packTypedMap.put(namespace, namespacedList.build()); - } catch(IOException ignored) { + node = node.children.computeIfAbsent(key, $ -> new Node()); + } + }); } - } - synchronized (this.resourceListings) { - this.resourceListings.put(type, packTypedMap.build()); + } catch(IOException ignored) { } }); } - cacheGenerationTasks.add(() -> { - ((ObjectOpenHashSet)this.containedPaths).trim(); - }); + cacheGenerationTasks.add(this.root::optimize); } private static boolean isValidCachedResourcePath(Path path) { @@ -103,8 +146,9 @@ public class PackResourcesCacheEngine { } public Set getNamespaces(PackType type) { + awaitLoad(); if(PackTypeHelper.isVanillaPackType(type)) - return this.namespacesByType.get(type); + return this.root.getChild(type.getDirectory()).children.keySet(); else return null; } @@ -131,56 +175,34 @@ public class PackResourcesCacheEngine { } } - public boolean hasResource(String path) { - awaitLoad(); - return this.containedPaths.contains(new CachedResourcePath(path)); - } - public boolean hasResource(String[] paths) { awaitLoad(); - return this.containedPaths.contains(new CachedResourcePath(paths)); + var node = this.root; + for (String path : paths) { + if (path.isEmpty()) { + continue; + } + node = node.children.get(path); + if (node == null) { + //ModernFix.LOGGER.info("Does not have " + String.join("/", paths)); + return false; + } + } + return true; } - public Collection getResources(PackType type, String resourceNamespace, String pathIn, int maxDepth, Predicate filter) { + public void collectResources(PackType type, String resourceNamespace, String[] components, int maxDepth, PackResources.ResourceOutput output) { if(!PackTypeHelper.isVanillaPackType(type)) throw new IllegalArgumentException("Only vanilla PackTypes are supported"); awaitLoad(); - List paths = resourceListings.get(type).getOrDefault(resourceNamespace, Collections.emptyList()); - if(paths.isEmpty()) - return Collections.emptyList(); - String testPath = pathIn.endsWith("/") ? pathIn : (pathIn + "/"); - ArrayList resources = new ArrayList<>(); - for(CachedResourcePath cachePath : paths) { - if((cachePath.getNameCount() - 2) > maxDepth) - continue; - String fullPath = cachePath.getFullPath(2); - String fullTestPath = fullPath.endsWith("/") ? fullPath : (fullPath + "/"); - if(!fullTestPath.startsWith(testPath)) { - continue; - } - ResourceLocation foundResource = ResourceLocation.fromNamespaceAndPath(resourceNamespace, fullPath); - if(!filter.test(foundResource)) - continue; - resources.add(foundResource); - } - return resources; - } - - private static final WeakHashMap cachingPacks = new WeakHashMap<>(); - public static void track(ICachingResourcePack pack) { - synchronized (cachingPacks) { - cachingPacks.put(pack, Boolean.TRUE); - } - } - - public static void invalidate() { - if(!ModernFixPlatformHooks.INSTANCE.isDevEnv()) + var node = this.root.getChild(type.getDirectory()); + if (node == null) { return; - synchronized (cachingPacks) { - cachingPacks.keySet().forEach(pack -> { - if(pack != null) - pack.invalidateCache(); - }); } + node = node.getChild(resourceNamespace); + if (node == null) { + return; + } + node.collectResources(resourceNamespace, this.rootPathsByType.get(type).resolve(resourceNamespace), components, 0, maxDepth, output); } } diff --git a/forge/src/main/java/org/embeddedt/modernfix/forge/mixin/perf/resourcepacks/ForgePathPackResourcesMixin.java b/forge/src/main/java/org/embeddedt/modernfix/forge/mixin/perf/resourcepacks/ForgePathPackResourcesMixin.java new file mode 100644 index 00000000..9abf6a84 --- /dev/null +++ b/forge/src/main/java/org/embeddedt/modernfix/forge/mixin/perf/resourcepacks/ForgePathPackResourcesMixin.java @@ -0,0 +1,107 @@ +package org.embeddedt.modernfix.forge.mixin.perf.resourcepacks; + +import net.minecraft.resources.ResourceLocation; +import net.minecraft.server.packs.PackResources; +import net.minecraft.server.packs.PackType; +import net.minecraftforge.forgespi.locating.IModFile; +import net.minecraftforge.resource.PathPackResources; +import org.embeddedt.modernfix.ModernFix; +import org.embeddedt.modernfix.forge.load.ModResourcePackPathFixer; +import org.embeddedt.modernfix.resources.ICachingResourcePack; +import org.embeddedt.modernfix.resources.PackResourcesCacheEngine; +import org.embeddedt.modernfix.util.PackTypeHelper; +import org.jetbrains.annotations.NotNull; +import org.spongepowered.asm.mixin.Mixin; +import org.spongepowered.asm.mixin.Shadow; +import org.spongepowered.asm.mixin.injection.At; +import org.spongepowered.asm.mixin.injection.Inject; +import org.spongepowered.asm.mixin.injection.Redirect; +import org.spongepowered.asm.mixin.injection.callback.CallbackInfo; +import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable; + +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.util.Objects; +import java.util.Set; + +@Mixin(value = PathPackResources.class, priority = 1100) +public abstract class ForgePathPackResourcesMixin implements ICachingResourcePack { + @Shadow(remap = false) protected abstract Path resolve(String... paths); + + @Shadow(remap = false) @NotNull + protected abstract Set getNamespacesFromDisk(PackType type); + + @Shadow(remap = false) private static String[] getPathFromLocation(PackType type, ResourceLocation location) { + throw new AssertionError(); + } + + private PackResourcesCacheEngine cacheEngine; + + private IModFile mfix$resolveFileOverride; + + @Inject(method = "", at = @At("TAIL")) + private void cacheResources(String packId, boolean isBuiltin, final Path source, CallbackInfo ci) { + // handle buggy mods instantiating at the root path, but only if they didn't override at all + // (otherwise they may have handled resolve() already) + if(((Object)this).getClass() == PathPackResources.class) + this.mfix$resolveFileOverride = ModResourcePackPathFixer.getModFileByRootPath(source); + if(this.mfix$resolveFileOverride != null) + ModernFix.LOGGER.warn("PathResourcePack base class instantiated with root path of mod file {}. This probably means a mod should be calling ResourcePackLoader.createPackForMod instead. Applying workaround.", mfix$resolveFileOverride.getFileName()); + invalidateCache(); + } + + @Inject(method = "resolve", at = @At("HEAD"), cancellable = true, remap = false) + private void resolveViaModFile(String[] paths, CallbackInfoReturnable cir) { + if(this.mfix$resolveFileOverride != null) + cir.setReturnValue(this.mfix$resolveFileOverride.findResource(paths)); + } + + private PackResourcesCacheEngine generateResourceCache() { + synchronized (this) { + PackResourcesCacheEngine engine = this.cacheEngine; + if(engine != null) + return engine; + this.cacheEngine = engine = new PackResourcesCacheEngine((type) -> this.resolve(type.getDirectory())); + return engine; + } + } + + @Override + public void invalidateCache() { + this.cacheEngine = null; + } + + @Redirect(method = "getNamespaces", at = @At(value = "INVOKE", target = "Lnet/minecraftforge/resource/PathPackResources;getNamespacesFromDisk(Lnet/minecraft/server/packs/PackType;)Ljava/util/Set;")) + private Set useCacheForNamespaces(PathPackResources instance, PackType type) { + PackResourcesCacheEngine engine = cacheEngine; + if(engine != null) { + Set namespaces = engine.getNamespaces(type); + if(namespaces != null) + return namespaces; + } + return this.getNamespacesFromDisk(type); + } + + @Redirect(method = "getRootResource", at = @At(value = "INVOKE", target = "Ljava/nio/file/Files;exists(Ljava/nio/file/Path;[Ljava/nio/file/LinkOption;)Z")) + private boolean useCacheForExistence(Path path, LinkOption[] options, String[] originalPaths) { + // the cache only stores things with a namespace and pack type + if(originalPaths.length < 3 || (!Objects.equals(originalPaths[0], "assets") && !Objects.equals(originalPaths[0], "data"))) + return Files.exists(path, options); + else + return this.generateResourceCache().hasResource(originalPaths); + } + + /** + * @author embeddedt + * @reason Use cached listing of mod resources + */ + @Inject(method = "listResources", at = @At("HEAD"), cancellable = true) + private void fastGetResources(PackType type, String namespace, String path, PackResources.ResourceOutput resourceOutput, CallbackInfo ci) + { + if(!PackTypeHelper.isVanillaPackType(type)) + return; + ci.cancel(); + this.generateResourceCache().collectResources(type, namespace, path.split("/"), Integer.MAX_VALUE, resourceOutput); + } +}