Rewrite resource pack caching to use a tree
This commit is contained in:
embeddedt
DerCommander323
parent
10b65219bc
commit
98af2ec35a
|
|
@ -1,18 +0,0 @@
|
||||||
package org.embeddedt.modernfix.common.mixin.perf.resourcepacks;
|
|
||||||
|
|
||||||
import net.minecraft.server.packs.resources.ReloadableResourceManager;
|
|
||||||
import org.embeddedt.modernfix.ModernFix;
|
|
||||||
import org.embeddedt.modernfix.resources.PackResourcesCacheEngine;
|
|
||||||
import org.spongepowered.asm.mixin.Mixin;
|
|
||||||
import org.spongepowered.asm.mixin.injection.At;
|
|
||||||
import org.spongepowered.asm.mixin.injection.Inject;
|
|
||||||
import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable;
|
|
||||||
|
|
||||||
@Mixin(ReloadableResourceManager.class)
|
|
||||||
public class ReloadableResourceManagerMixin {
|
|
||||||
@Inject(method = "createReload", at = @At("HEAD"))
|
|
||||||
private void invalidateResourceCaches(CallbackInfoReturnable<?> cir) {
|
|
||||||
ModernFix.LOGGER.info("Invalidating pack caches");
|
|
||||||
PackResourcesCacheEngine.invalidate();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -3,88 +3,131 @@ package org.embeddedt.modernfix.resources;
|
||||||
import com.google.common.base.Joiner;
|
import com.google.common.base.Joiner;
|
||||||
import com.google.common.base.Stopwatch;
|
import com.google.common.base.Stopwatch;
|
||||||
import com.google.common.collect.ImmutableList;
|
import com.google.common.collect.ImmutableList;
|
||||||
import com.google.common.collect.ImmutableMap;
|
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
|
||||||
import com.mojang.datafixers.util.Pair;
|
|
||||||
import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet;
|
import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet;
|
||||||
import net.minecraft.resources.ResourceLocation;
|
import net.minecraft.resources.ResourceLocation;
|
||||||
|
import net.minecraft.server.packs.PackResources;
|
||||||
import net.minecraft.server.packs.PackType;
|
import net.minecraft.server.packs.PackType;
|
||||||
import org.embeddedt.modernfix.ModernFix;
|
import org.embeddedt.modernfix.ModernFix;
|
||||||
import org.embeddedt.modernfix.platform.ModernFixPlatformHooks;
|
|
||||||
import org.embeddedt.modernfix.util.PackTypeHelper;
|
import org.embeddedt.modernfix.util.PackTypeHelper;
|
||||||
|
import org.jetbrains.annotations.Nullable;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.function.BiFunction;
|
|
||||||
import java.util.function.Function;
|
import java.util.function.Function;
|
||||||
import java.util.function.Predicate;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
import java.util.stream.Stream;
|
import java.util.stream.Stream;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The core of the resource pack cache system.
|
* The core of the resource pack cache system.
|
||||||
*
|
|
||||||
* Using a dedicated set and also separate lists is important; testing without this showed a huge performance
|
|
||||||
* drop.
|
|
||||||
*/
|
*/
|
||||||
public class PackResourcesCacheEngine {
|
public class PackResourcesCacheEngine {
|
||||||
private static final Joiner SLASH_JOINER = Joiner.on('/');
|
private static final Joiner SLASH_JOINER = Joiner.on('/');
|
||||||
|
|
||||||
private final Map<PackType, Set<String>> namespacesByType;
|
static class Node {
|
||||||
private final Set<CachedResourcePath> containedPaths;
|
Map<String, Node> children;
|
||||||
private final EnumMap<PackType, Map<String, List<CachedResourcePath>>> resourceListings;
|
|
||||||
|
void optimize() {
|
||||||
|
if (children != null) {
|
||||||
|
for (var entry : children.entrySet()) {
|
||||||
|
var oldNode = entry.getValue();
|
||||||
|
oldNode.optimize();
|
||||||
|
if (oldNode.children == null) {
|
||||||
|
entry.setValue(EMPTY);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
children = Map.copyOf(children);
|
||||||
|
} else {
|
||||||
|
children = Map.of();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void collectResources(String namespace, Path baseNioPath, String[] pathComponents, int curIndex, int maxDepth, PackResources.ResourceOutput output) {
|
||||||
|
if (curIndex > maxDepth) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (curIndex < pathComponents.length) {
|
||||||
|
String component;
|
||||||
|
do {
|
||||||
|
component = pathComponents[curIndex];
|
||||||
|
if (!component.isEmpty()) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
curIndex++;
|
||||||
|
maxDepth++;
|
||||||
|
} while(true);
|
||||||
|
|
||||||
|
Node n = getChild(component);
|
||||||
|
if (n != null) {
|
||||||
|
n.collectResources(namespace, baseNioPath, pathComponents, curIndex + 1, maxDepth, output);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// We reached the desired path. Collect all resources
|
||||||
|
this.outputResources(namespace, baseNioPath, String.join("/", pathComponents), output);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
void outputResources(String namespace, Path baseNioPath, String path, PackResources.ResourceOutput output) {
|
||||||
|
if (children.isEmpty()) {
|
||||||
|
// This is a terminal node.
|
||||||
|
ResourceLocation location = new ResourceLocation(namespace, path);
|
||||||
|
output.accept(location, () -> Files.newInputStream(baseNioPath.resolve(path)));
|
||||||
|
} else {
|
||||||
|
for (var entry : children.entrySet()) {
|
||||||
|
entry.getValue().outputResources(namespace, baseNioPath, path + "/" + entry.getKey(), output);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Nullable
|
||||||
|
Node getChild(String name) {
|
||||||
|
return children.get(name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final Node EMPTY = new Node();
|
||||||
|
|
||||||
|
private final Node root = new Node();
|
||||||
|
private final Map<PackType, Path> rootPathsByType = new Object2ObjectOpenHashMap<>();
|
||||||
|
|
||||||
private volatile boolean cacheGenerationFlag = false;
|
private volatile boolean cacheGenerationFlag = false;
|
||||||
private List<Runnable> cacheGenerationTasks = new ArrayList<>();
|
private List<Runnable> cacheGenerationTasks = new ArrayList<>();
|
||||||
private Path debugPath;
|
private Path debugPath;
|
||||||
|
|
||||||
public PackResourcesCacheEngine(Function<PackType, Set<String>> namespacesRetriever, BiFunction<PackType, String, Path> basePathRetriever) {
|
public PackResourcesCacheEngine(Function<PackType, Path> basePathRetriever) {
|
||||||
this.namespacesByType = new EnumMap<>(PackType.class);
|
|
||||||
for(PackType type : PackType.values()) {
|
|
||||||
if(!PackTypeHelper.isVanillaPackType(type))
|
|
||||||
continue;
|
|
||||||
this.namespacesByType.put(type, namespacesRetriever.apply(type));
|
|
||||||
}
|
|
||||||
this.containedPaths = new ObjectOpenHashSet<>();
|
|
||||||
this.resourceListings = new EnumMap<>(PackType.class);
|
|
||||||
// used for log message
|
// used for log message
|
||||||
this.debugPath = basePathRetriever.apply(PackType.CLIENT_RESOURCES, "minecraft").toAbsolutePath();
|
this.debugPath = basePathRetriever.apply(PackType.CLIENT_RESOURCES).toAbsolutePath();
|
||||||
|
this.root.children = new Object2ObjectOpenHashMap<>();
|
||||||
|
ObjectOpenHashSet<String> pathKeys = new ObjectOpenHashSet<>();
|
||||||
for(PackType type : PackType.values()) {
|
for(PackType type : PackType.values()) {
|
||||||
Collection<String> namespaces = PackTypeHelper.isVanillaPackType(type) ? this.namespacesByType.get(type) : namespacesRetriever.apply(type);
|
var typeRoot = new Node();
|
||||||
Collection<Pair<String, Path>> namespacedRoots = namespaces.stream().map(s -> Pair.of(s, basePathRetriever.apply(type, s).toAbsolutePath())).collect(Collectors.toList());
|
this.root.children.put(type.getDirectory(), typeRoot);
|
||||||
|
Path root = basePathRetriever.apply(type);
|
||||||
|
this.rootPathsByType.put(type, root);
|
||||||
cacheGenerationTasks.add(() -> {
|
cacheGenerationTasks.add(() -> {
|
||||||
ImmutableMap.Builder<String, List<CachedResourcePath>> packTypedMap = ImmutableMap.builder();
|
try {
|
||||||
for(Pair<String, Path> pair : namespacedRoots) {
|
try (Stream<Path> stream = Files.find(root, Integer.MAX_VALUE, (p, a) -> a.isRegularFile())) {
|
||||||
try {
|
stream
|
||||||
ImmutableList.Builder<CachedResourcePath> namespacedList = ImmutableList.builder();
|
.map(path -> root.relativize(path.toAbsolutePath()))
|
||||||
String namespace = pair.getFirst();
|
.filter(PackResourcesCacheEngine::isValidCachedResourcePath)
|
||||||
Path root = pair.getSecond();
|
.forEach(path -> {
|
||||||
String[] prefix = new String[] { type.getDirectory(), namespace };
|
var node = typeRoot;
|
||||||
try (Stream<Path> stream = Files.find(root, Integer.MAX_VALUE, (p, a) -> a.isRegularFile())) {
|
for (Path component : path) {
|
||||||
stream
|
String key = pathKeys.addOrGet(component.toString());
|
||||||
.map(path -> root.relativize(path.toAbsolutePath()))
|
if (node.children == null) {
|
||||||
.filter(PackResourcesCacheEngine::isValidCachedResourcePath)
|
node.children = new Object2ObjectOpenHashMap<>();
|
||||||
.forEach(path -> {
|
|
||||||
CachedResourcePath cachedPath = new CachedResourcePath(prefix, path);
|
|
||||||
synchronized (this.containedPaths) {
|
|
||||||
this.containedPaths.add(cachedPath);
|
|
||||||
}
|
}
|
||||||
//if(!cachedPath.getFileName().endsWith(".mcmeta"))
|
node = node.children.computeIfAbsent(key, $ -> new Node());
|
||||||
namespacedList.add(cachedPath);
|
}
|
||||||
});
|
});
|
||||||
}
|
|
||||||
packTypedMap.put(namespace, namespacedList.build());
|
|
||||||
} catch(IOException ignored) {
|
|
||||||
}
|
}
|
||||||
}
|
} catch(IOException ignored) {
|
||||||
synchronized (this.resourceListings) {
|
|
||||||
this.resourceListings.put(type, packTypedMap.build());
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
cacheGenerationTasks.add(() -> {
|
cacheGenerationTasks.add(this.root::optimize);
|
||||||
((ObjectOpenHashSet<CachedResourcePath>)this.containedPaths).trim();
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static boolean isValidCachedResourcePath(Path path) {
|
private static boolean isValidCachedResourcePath(Path path) {
|
||||||
|
|
@ -103,8 +146,9 @@ public class PackResourcesCacheEngine {
|
||||||
}
|
}
|
||||||
|
|
||||||
public Set<String> getNamespaces(PackType type) {
|
public Set<String> getNamespaces(PackType type) {
|
||||||
|
awaitLoad();
|
||||||
if(PackTypeHelper.isVanillaPackType(type))
|
if(PackTypeHelper.isVanillaPackType(type))
|
||||||
return this.namespacesByType.get(type);
|
return this.root.getChild(type.getDirectory()).children.keySet();
|
||||||
else
|
else
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
@ -131,56 +175,34 @@ public class PackResourcesCacheEngine {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean hasResource(String path) {
|
|
||||||
awaitLoad();
|
|
||||||
return this.containedPaths.contains(new CachedResourcePath(path));
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean hasResource(String[] paths) {
|
public boolean hasResource(String[] paths) {
|
||||||
awaitLoad();
|
awaitLoad();
|
||||||
return this.containedPaths.contains(new CachedResourcePath(paths));
|
var node = this.root;
|
||||||
|
for (String path : paths) {
|
||||||
|
if (path.isEmpty()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
node = node.children.get(path);
|
||||||
|
if (node == null) {
|
||||||
|
//ModernFix.LOGGER.info("Does not have " + String.join("/", paths));
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Collection<ResourceLocation> getResources(PackType type, String resourceNamespace, String pathIn, int maxDepth, Predicate<ResourceLocation> filter) {
|
public void collectResources(PackType type, String resourceNamespace, String[] components, int maxDepth, PackResources.ResourceOutput output) {
|
||||||
if(!PackTypeHelper.isVanillaPackType(type))
|
if(!PackTypeHelper.isVanillaPackType(type))
|
||||||
throw new IllegalArgumentException("Only vanilla PackTypes are supported");
|
throw new IllegalArgumentException("Only vanilla PackTypes are supported");
|
||||||
awaitLoad();
|
awaitLoad();
|
||||||
List<CachedResourcePath> paths = resourceListings.get(type).getOrDefault(resourceNamespace, Collections.emptyList());
|
var node = this.root.getChild(type.getDirectory());
|
||||||
if(paths.isEmpty())
|
if (node == null) {
|
||||||
return Collections.emptyList();
|
|
||||||
String testPath = pathIn.endsWith("/") ? pathIn : (pathIn + "/");
|
|
||||||
ArrayList<ResourceLocation> resources = new ArrayList<>();
|
|
||||||
for(CachedResourcePath cachePath : paths) {
|
|
||||||
if((cachePath.getNameCount() - 2) > maxDepth)
|
|
||||||
continue;
|
|
||||||
String fullPath = cachePath.getFullPath(2);
|
|
||||||
String fullTestPath = fullPath.endsWith("/") ? fullPath : (fullPath + "/");
|
|
||||||
if(!fullTestPath.startsWith(testPath)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
ResourceLocation foundResource = ResourceLocation.fromNamespaceAndPath(resourceNamespace, fullPath);
|
|
||||||
if(!filter.test(foundResource))
|
|
||||||
continue;
|
|
||||||
resources.add(foundResource);
|
|
||||||
}
|
|
||||||
return resources;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static final WeakHashMap<ICachingResourcePack, Boolean> cachingPacks = new WeakHashMap<>();
|
|
||||||
public static void track(ICachingResourcePack pack) {
|
|
||||||
synchronized (cachingPacks) {
|
|
||||||
cachingPacks.put(pack, Boolean.TRUE);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void invalidate() {
|
|
||||||
if(!ModernFixPlatformHooks.INSTANCE.isDevEnv())
|
|
||||||
return;
|
return;
|
||||||
synchronized (cachingPacks) {
|
|
||||||
cachingPacks.keySet().forEach(pack -> {
|
|
||||||
if(pack != null)
|
|
||||||
pack.invalidateCache();
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
node = node.getChild(resourceNamespace);
|
||||||
|
if (node == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
node.collectResources(resourceNamespace, this.rootPathsByType.get(type).resolve(resourceNamespace), components, 0, maxDepth, output);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,107 @@
|
||||||
|
package org.embeddedt.modernfix.forge.mixin.perf.resourcepacks;
|
||||||
|
|
||||||
|
import net.minecraft.resources.ResourceLocation;
|
||||||
|
import net.minecraft.server.packs.PackResources;
|
||||||
|
import net.minecraft.server.packs.PackType;
|
||||||
|
import net.minecraftforge.forgespi.locating.IModFile;
|
||||||
|
import net.minecraftforge.resource.PathPackResources;
|
||||||
|
import org.embeddedt.modernfix.ModernFix;
|
||||||
|
import org.embeddedt.modernfix.forge.load.ModResourcePackPathFixer;
|
||||||
|
import org.embeddedt.modernfix.resources.ICachingResourcePack;
|
||||||
|
import org.embeddedt.modernfix.resources.PackResourcesCacheEngine;
|
||||||
|
import org.embeddedt.modernfix.util.PackTypeHelper;
|
||||||
|
import org.jetbrains.annotations.NotNull;
|
||||||
|
import org.spongepowered.asm.mixin.Mixin;
|
||||||
|
import org.spongepowered.asm.mixin.Shadow;
|
||||||
|
import org.spongepowered.asm.mixin.injection.At;
|
||||||
|
import org.spongepowered.asm.mixin.injection.Inject;
|
||||||
|
import org.spongepowered.asm.mixin.injection.Redirect;
|
||||||
|
import org.spongepowered.asm.mixin.injection.callback.CallbackInfo;
|
||||||
|
import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable;
|
||||||
|
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.LinkOption;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
@Mixin(value = PathPackResources.class, priority = 1100)
|
||||||
|
public abstract class ForgePathPackResourcesMixin implements ICachingResourcePack {
|
||||||
|
@Shadow(remap = false) protected abstract Path resolve(String... paths);
|
||||||
|
|
||||||
|
@Shadow(remap = false) @NotNull
|
||||||
|
protected abstract Set<String> getNamespacesFromDisk(PackType type);
|
||||||
|
|
||||||
|
@Shadow(remap = false) private static String[] getPathFromLocation(PackType type, ResourceLocation location) {
|
||||||
|
throw new AssertionError();
|
||||||
|
}
|
||||||
|
|
||||||
|
private PackResourcesCacheEngine cacheEngine;
|
||||||
|
|
||||||
|
private IModFile mfix$resolveFileOverride;
|
||||||
|
|
||||||
|
@Inject(method = "<init>", at = @At("TAIL"))
|
||||||
|
private void cacheResources(String packId, boolean isBuiltin, final Path source, CallbackInfo ci) {
|
||||||
|
// handle buggy mods instantiating at the root path, but only if they didn't override at all
|
||||||
|
// (otherwise they may have handled resolve() already)
|
||||||
|
if(((Object)this).getClass() == PathPackResources.class)
|
||||||
|
this.mfix$resolveFileOverride = ModResourcePackPathFixer.getModFileByRootPath(source);
|
||||||
|
if(this.mfix$resolveFileOverride != null)
|
||||||
|
ModernFix.LOGGER.warn("PathResourcePack base class instantiated with root path of mod file {}. This probably means a mod should be calling ResourcePackLoader.createPackForMod instead. Applying workaround.", mfix$resolveFileOverride.getFileName());
|
||||||
|
invalidateCache();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Inject(method = "resolve", at = @At("HEAD"), cancellable = true, remap = false)
|
||||||
|
private void resolveViaModFile(String[] paths, CallbackInfoReturnable<Path> cir) {
|
||||||
|
if(this.mfix$resolveFileOverride != null)
|
||||||
|
cir.setReturnValue(this.mfix$resolveFileOverride.findResource(paths));
|
||||||
|
}
|
||||||
|
|
||||||
|
private PackResourcesCacheEngine generateResourceCache() {
|
||||||
|
synchronized (this) {
|
||||||
|
PackResourcesCacheEngine engine = this.cacheEngine;
|
||||||
|
if(engine != null)
|
||||||
|
return engine;
|
||||||
|
this.cacheEngine = engine = new PackResourcesCacheEngine((type) -> this.resolve(type.getDirectory()));
|
||||||
|
return engine;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void invalidateCache() {
|
||||||
|
this.cacheEngine = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Redirect(method = "getNamespaces", at = @At(value = "INVOKE", target = "Lnet/minecraftforge/resource/PathPackResources;getNamespacesFromDisk(Lnet/minecraft/server/packs/PackType;)Ljava/util/Set;"))
|
||||||
|
private Set<String> useCacheForNamespaces(PathPackResources instance, PackType type) {
|
||||||
|
PackResourcesCacheEngine engine = cacheEngine;
|
||||||
|
if(engine != null) {
|
||||||
|
Set<String> namespaces = engine.getNamespaces(type);
|
||||||
|
if(namespaces != null)
|
||||||
|
return namespaces;
|
||||||
|
}
|
||||||
|
return this.getNamespacesFromDisk(type);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Redirect(method = "getRootResource", at = @At(value = "INVOKE", target = "Ljava/nio/file/Files;exists(Ljava/nio/file/Path;[Ljava/nio/file/LinkOption;)Z"))
|
||||||
|
private boolean useCacheForExistence(Path path, LinkOption[] options, String[] originalPaths) {
|
||||||
|
// the cache only stores things with a namespace and pack type
|
||||||
|
if(originalPaths.length < 3 || (!Objects.equals(originalPaths[0], "assets") && !Objects.equals(originalPaths[0], "data")))
|
||||||
|
return Files.exists(path, options);
|
||||||
|
else
|
||||||
|
return this.generateResourceCache().hasResource(originalPaths);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author embeddedt
|
||||||
|
* @reason Use cached listing of mod resources
|
||||||
|
*/
|
||||||
|
@Inject(method = "listResources", at = @At("HEAD"), cancellable = true)
|
||||||
|
private void fastGetResources(PackType type, String namespace, String path, PackResources.ResourceOutput resourceOutput, CallbackInfo ci)
|
||||||
|
{
|
||||||
|
if(!PackTypeHelper.isVanillaPackType(type))
|
||||||
|
return;
|
||||||
|
ci.cancel();
|
||||||
|
this.generateResourceCache().collectResources(type, namespace, path.split("/"), Integer.MAX_VALUE, resourceOutput);
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user