From 4cde23f4fe2c3c8422ea21195a6f6f9125ef7dc8 Mon Sep 17 00:00:00 2001 From: embeddedt <42941056+embeddedt@users.noreply.github.com> Date: Sun, 27 Apr 2025 18:35:28 -0400 Subject: [PATCH] Rewrite resource pack caching to use a tree --- .../ReloadableResourceManagerMixin.java | 18 -- .../resources/PackResourcesCacheEngine.java | 210 ++++++++++-------- .../ForgePathPackResourcesMixin.java | 11 +- 3 files changed, 118 insertions(+), 121 deletions(-) delete mode 100644 common/src/main/java/org/embeddedt/modernfix/common/mixin/perf/resourcepacks/ReloadableResourceManagerMixin.java diff --git a/common/src/main/java/org/embeddedt/modernfix/common/mixin/perf/resourcepacks/ReloadableResourceManagerMixin.java b/common/src/main/java/org/embeddedt/modernfix/common/mixin/perf/resourcepacks/ReloadableResourceManagerMixin.java deleted file mode 100644 index 3fd155cf..00000000 --- a/common/src/main/java/org/embeddedt/modernfix/common/mixin/perf/resourcepacks/ReloadableResourceManagerMixin.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.embeddedt.modernfix.common.mixin.perf.resourcepacks; - -import net.minecraft.server.packs.resources.ReloadableResourceManager; -import org.embeddedt.modernfix.ModernFix; -import org.embeddedt.modernfix.resources.PackResourcesCacheEngine; -import org.spongepowered.asm.mixin.Mixin; -import org.spongepowered.asm.mixin.injection.At; -import org.spongepowered.asm.mixin.injection.Inject; -import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable; - -@Mixin(ReloadableResourceManager.class) -public class ReloadableResourceManagerMixin { - @Inject(method = "createReload", at = @At("HEAD")) - private void invalidateResourceCaches(CallbackInfoReturnable cir) { - ModernFix.LOGGER.info("Invalidating pack caches"); - PackResourcesCacheEngine.invalidate(); - } -} diff --git a/common/src/main/java/org/embeddedt/modernfix/resources/PackResourcesCacheEngine.java b/common/src/main/java/org/embeddedt/modernfix/resources/PackResourcesCacheEngine.java index e99b6628..479c8398 100644 --- a/common/src/main/java/org/embeddedt/modernfix/resources/PackResourcesCacheEngine.java +++ b/common/src/main/java/org/embeddedt/modernfix/resources/PackResourcesCacheEngine.java @@ -3,88 +3,131 @@ package org.embeddedt.modernfix.resources; import com.google.common.base.Joiner; import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.mojang.datafixers.util.Pair; +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet; import net.minecraft.resources.ResourceLocation; +import net.minecraft.server.packs.PackResources; import net.minecraft.server.packs.PackType; import org.embeddedt.modernfix.ModernFix; -import org.embeddedt.modernfix.platform.ModernFixPlatformHooks; import org.embeddedt.modernfix.util.PackTypeHelper; +import org.jetbrains.annotations.Nullable; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; -import java.util.function.BiFunction; import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; import java.util.stream.Stream; /** * The core of the resource pack cache system. - * - * Using a dedicated set and also separate lists is important; testing without this showed a huge performance - * drop. */ public class PackResourcesCacheEngine { private static final Joiner SLASH_JOINER = Joiner.on('/'); - private final Map> namespacesByType; - private final Set containedPaths; - private final EnumMap>> resourceListings; + static class Node { + Map children; + + void optimize() { + if (children != null) { + for (var entry : children.entrySet()) { + var oldNode = entry.getValue(); + oldNode.optimize(); + if (oldNode.children == null) { + entry.setValue(EMPTY); + } + } + children = Map.copyOf(children); + } else { + children = Map.of(); + } + } + + void collectResources(String namespace, Path baseNioPath, String[] pathComponents, int curIndex, int maxDepth, PackResources.ResourceOutput output) { + if (curIndex > maxDepth) { + return; + } + if (curIndex < pathComponents.length) { + String component; + do { + component = pathComponents[curIndex]; + if (!component.isEmpty()) { + break; + } + curIndex++; + maxDepth++; + } while(true); + + Node n = getChild(component); + if (n != null) { + n.collectResources(namespace, baseNioPath, pathComponents, curIndex + 1, maxDepth, output); + } + } else { + // We reached the desired path. Collect all resources + this.outputResources(namespace, baseNioPath, String.join("/", pathComponents), output); + } + + } + + void outputResources(String namespace, Path baseNioPath, String path, PackResources.ResourceOutput output) { + if (children.isEmpty()) { + // This is a terminal node. + ResourceLocation location = new ResourceLocation(namespace, path); + output.accept(location, () -> Files.newInputStream(baseNioPath.resolve(path))); + } else { + for (var entry : children.entrySet()) { + entry.getValue().outputResources(namespace, baseNioPath, path + "/" + entry.getKey(), output); + } + } + } + + @Nullable + Node getChild(String name) { + return children.get(name); + } + } + + private static final Node EMPTY = new Node(); + + private final Node root = new Node(); + private final Map rootPathsByType = new Object2ObjectOpenHashMap<>(); + private volatile boolean cacheGenerationFlag = false; private List cacheGenerationTasks = new ArrayList<>(); private Path debugPath; - public PackResourcesCacheEngine(Function> namespacesRetriever, BiFunction basePathRetriever) { - this.namespacesByType = new EnumMap<>(PackType.class); - for(PackType type : PackType.values()) { - if(!PackTypeHelper.isVanillaPackType(type)) - continue; - this.namespacesByType.put(type, namespacesRetriever.apply(type)); - } - this.containedPaths = new ObjectOpenHashSet<>(); - this.resourceListings = new EnumMap<>(PackType.class); + public PackResourcesCacheEngine(Function basePathRetriever) { // used for log message - this.debugPath = basePathRetriever.apply(PackType.CLIENT_RESOURCES, "minecraft").toAbsolutePath(); + this.debugPath = basePathRetriever.apply(PackType.CLIENT_RESOURCES).toAbsolutePath(); + this.root.children = new Object2ObjectOpenHashMap<>(); + ObjectOpenHashSet pathKeys = new ObjectOpenHashSet<>(); for(PackType type : PackType.values()) { - Collection namespaces = PackTypeHelper.isVanillaPackType(type) ? this.namespacesByType.get(type) : namespacesRetriever.apply(type); - Collection> namespacedRoots = namespaces.stream().map(s -> Pair.of(s, basePathRetriever.apply(type, s).toAbsolutePath())).collect(Collectors.toList()); + var typeRoot = new Node(); + this.root.children.put(type.getDirectory(), typeRoot); + Path root = basePathRetriever.apply(type); + this.rootPathsByType.put(type, root); cacheGenerationTasks.add(() -> { - ImmutableMap.Builder> packTypedMap = ImmutableMap.builder(); - for(Pair pair : namespacedRoots) { - try { - ImmutableList.Builder namespacedList = ImmutableList.builder(); - String namespace = pair.getFirst(); - Path root = pair.getSecond(); - String[] prefix = new String[] { type.getDirectory(), namespace }; - try (Stream stream = Files.find(root, Integer.MAX_VALUE, (p, a) -> a.isRegularFile())) { - stream - .map(path -> root.relativize(path.toAbsolutePath())) - .filter(PackResourcesCacheEngine::isValidCachedResourcePath) - .forEach(path -> { - CachedResourcePath cachedPath = new CachedResourcePath(prefix, path); - synchronized (this.containedPaths) { - this.containedPaths.add(cachedPath); + try { + try (Stream stream = Files.find(root, Integer.MAX_VALUE, (p, a) -> a.isRegularFile())) { + stream + .map(path -> root.relativize(path.toAbsolutePath())) + .filter(PackResourcesCacheEngine::isValidCachedResourcePath) + .forEach(path -> { + var node = typeRoot; + for (Path component : path) { + String key = pathKeys.addOrGet(component.toString()); + if (node.children == null) { + node.children = new Object2ObjectOpenHashMap<>(); } - //if(!cachedPath.getFileName().endsWith(".mcmeta")) - namespacedList.add(cachedPath); - }); - } - packTypedMap.put(namespace, namespacedList.build()); - } catch(IOException ignored) { + node = node.children.computeIfAbsent(key, $ -> new Node()); + } + }); } - } - synchronized (this.resourceListings) { - this.resourceListings.put(type, packTypedMap.build()); + } catch(IOException ignored) { } }); } - cacheGenerationTasks.add(() -> { - ((ObjectOpenHashSet)this.containedPaths).trim(); - }); + cacheGenerationTasks.add(this.root::optimize); } private static boolean isValidCachedResourcePath(Path path) { @@ -103,8 +146,9 @@ public class PackResourcesCacheEngine { } public Set getNamespaces(PackType type) { + awaitLoad(); if(PackTypeHelper.isVanillaPackType(type)) - return this.namespacesByType.get(type); + return this.root.getChild(type.getDirectory()).children.keySet(); else return null; } @@ -131,56 +175,34 @@ public class PackResourcesCacheEngine { } } - public boolean hasResource(String path) { - awaitLoad(); - return this.containedPaths.contains(new CachedResourcePath(path)); - } - public boolean hasResource(String[] paths) { awaitLoad(); - return this.containedPaths.contains(new CachedResourcePath(paths)); + var node = this.root; + for (String path : paths) { + if (path.isEmpty()) { + continue; + } + node = node.children.get(path); + if (node == null) { + //ModernFix.LOGGER.info("Does not have " + String.join("/", paths)); + return false; + } + } + return true; } - public Collection getResources(PackType type, String resourceNamespace, String pathIn, int maxDepth, Predicate filter) { + public void collectResources(PackType type, String resourceNamespace, String[] components, int maxDepth, PackResources.ResourceOutput output) { if(!PackTypeHelper.isVanillaPackType(type)) throw new IllegalArgumentException("Only vanilla PackTypes are supported"); awaitLoad(); - List paths = resourceListings.get(type).getOrDefault(resourceNamespace, Collections.emptyList()); - if(paths.isEmpty()) - return Collections.emptyList(); - String testPath = pathIn.endsWith("/") ? pathIn : (pathIn + "/"); - ArrayList resources = new ArrayList<>(); - for(CachedResourcePath cachePath : paths) { - if((cachePath.getNameCount() - 2) > maxDepth) - continue; - String fullPath = cachePath.getFullPath(2); - String fullTestPath = fullPath.endsWith("/") ? fullPath : (fullPath + "/"); - if(!fullTestPath.startsWith(testPath)) { - continue; - } - ResourceLocation foundResource = new ResourceLocation(resourceNamespace, fullPath); - if(!filter.test(foundResource)) - continue; - resources.add(foundResource); - } - return resources; - } - - private static final WeakHashMap cachingPacks = new WeakHashMap<>(); - public static void track(ICachingResourcePack pack) { - synchronized (cachingPacks) { - cachingPacks.put(pack, Boolean.TRUE); - } - } - - public static void invalidate() { - if(!ModernFixPlatformHooks.INSTANCE.isDevEnv()) + var node = this.root.getChild(type.getDirectory()); + if (node == null) { return; - synchronized (cachingPacks) { - cachingPacks.keySet().forEach(pack -> { - if(pack != null) - pack.invalidateCache(); - }); } + node = node.getChild(resourceNamespace); + if (node == null) { + return; + } + node.collectResources(resourceNamespace, this.rootPathsByType.get(type).resolve(resourceNamespace), components, 0, maxDepth, output); } } diff --git a/forge/src/main/java/org/embeddedt/modernfix/forge/mixin/perf/resourcepacks/ForgePathPackResourcesMixin.java b/forge/src/main/java/org/embeddedt/modernfix/forge/mixin/perf/resourcepacks/ForgePathPackResourcesMixin.java index 89655e5b..9abf6a84 100644 --- a/forge/src/main/java/org/embeddedt/modernfix/forge/mixin/perf/resourcepacks/ForgePathPackResourcesMixin.java +++ b/forge/src/main/java/org/embeddedt/modernfix/forge/mixin/perf/resourcepacks/ForgePathPackResourcesMixin.java @@ -8,7 +8,6 @@ import net.minecraftforge.resource.PathPackResources; import org.embeddedt.modernfix.ModernFix; import org.embeddedt.modernfix.forge.load.ModResourcePackPathFixer; import org.embeddedt.modernfix.resources.ICachingResourcePack; -import org.embeddedt.modernfix.resources.NewResourcePackAdapter; import org.embeddedt.modernfix.resources.PackResourcesCacheEngine; import org.embeddedt.modernfix.util.PackTypeHelper; import org.jetbrains.annotations.NotNull; @@ -23,7 +22,6 @@ import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; -import java.util.Collection; import java.util.Objects; import java.util.Set; @@ -51,7 +49,6 @@ public abstract class ForgePathPackResourcesMixin implements ICachingResourcePac if(this.mfix$resolveFileOverride != null) ModernFix.LOGGER.warn("PathResourcePack base class instantiated with root path of mod file {}. This probably means a mod should be calling ResourcePackLoader.createPackForMod instead. Applying workaround.", mfix$resolveFileOverride.getFileName()); invalidateCache(); - PackResourcesCacheEngine.track(this); } @Inject(method = "resolve", at = @At("HEAD"), cancellable = true, remap = false) @@ -65,7 +62,7 @@ public abstract class ForgePathPackResourcesMixin implements ICachingResourcePac PackResourcesCacheEngine engine = this.cacheEngine; if(engine != null) return engine; - this.cacheEngine = engine = new PackResourcesCacheEngine(this::getNamespacesFromDisk, (type, namespace) -> this.resolve(type.getDirectory(), namespace)); + this.cacheEngine = engine = new PackResourcesCacheEngine((type) -> this.resolve(type.getDirectory())); return engine; } } @@ -105,10 +102,6 @@ public abstract class ForgePathPackResourcesMixin implements ICachingResourcePac if(!PackTypeHelper.isVanillaPackType(type)) return; ci.cancel(); - Collection allPossibleResources = this.generateResourceCache().getResources(type, namespace, path, Integer.MAX_VALUE, p -> true); - NewResourcePackAdapter.sendToOutput(location -> { - Path target = resolve(getPathFromLocation(location.getPath().startsWith("lang/") ? PackType.CLIENT_RESOURCES : type, location)); - return () -> Files.newInputStream(target); - }, resourceOutput, allPossibleResources); + this.generateResourceCache().collectResources(type, namespace, path.split("/"), Integer.MAX_VALUE, resourceOutput); } }