From 3d07a51d6d2cd3c5f1d00076f95a5a880f9dc475 Mon Sep 17 00:00:00 2001 From: Marlon Date: Thu, 16 Jul 2026 12:26:59 +0200 Subject: [PATCH] Fix live loading and organize cached output --- FModel/App.xaml.cs | 3 +- FModel/Framework/CacheManager.cs | 106 ++++++++++++++++++ .../ApiEndpoints/ValorantApiEndpoint.cs | 4 +- FModel/ViewModels/AssetsFolderViewModel.cs | 5 +- FModel/ViewModels/CUE4ParseViewModel.cs | 54 ++++++--- FModel/ViewModels/Commands/LoadCommand.cs | 79 ++++++------- FModel/Views/SettingsView.xaml.cs | 2 +- 7 files changed, 193 insertions(+), 60 deletions(-) create mode 100644 FModel/Framework/CacheManager.cs diff --git a/FModel/App.xaml.cs b/FModel/App.xaml.cs index 31da19b9..36a1b8c1 100644 --- a/FModel/App.xaml.cs +++ b/FModel/App.xaml.cs @@ -108,7 +108,7 @@ public partial class App Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, "Backups")); if (createMe) Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, "Exports")); Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, "Logs")); - Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, ".data")); + CacheManager.EnsureDirectories(); const string template = "{Timestamp:yyyy-MM-dd HH:mm:ss} [{Level:u3}] {Enriched}: {Message:lj}{NewLine}{Exception}"; Log.Logger = new LoggerConfiguration() @@ -125,6 +125,7 @@ public partial class App #endif .CreateLogger(); + CacheManager.MigrateLegacyFiles(); Log.Information("Version {Version} ({CommitId})", Constants.APP_VERSION, Constants.APP_COMMIT_ID); Log.Information("{OS}", GetOperatingSystemProductName()); Log.Information("{RuntimeVer}", RuntimeInformation.FrameworkDescription); diff --git a/FModel/Framework/CacheManager.cs b/FModel/Framework/CacheManager.cs new file mode 100644 index 00000000..55b55478 --- /dev/null +++ b/FModel/Framework/CacheManager.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections.Generic; +using System.IO; +using FModel.Settings; +using Serilog; + +namespace FModel.Framework; + +public static class CacheManager +{ + public static string DataDirectory => + Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, ".data")).FullName; + + public static string ChunksDirectory => + Directory.CreateDirectory(Path.Combine(DataDirectory, "chunks")).FullName; + + public static string ManifestsDirectory => + Directory.CreateDirectory(Path.Combine(DataDirectory, "manifests")).FullName; + + public static string MappingsDirectory => + Directory.CreateDirectory(Path.Combine(DataDirectory, "mappings")).FullName; + + public static void EnsureDirectories() + { + _ = ChunksDirectory; + _ = ManifestsDirectory; + _ = MappingsDirectory; + } + + public static void MigrateLegacyFiles() + { + EnsureDirectories(); + + var movedFiles = 0; + var skippedFiles = 0; + var sources = new HashSet(StringComparer.OrdinalIgnoreCase) + { + Path.GetFullPath(UserSettings.Default.OutputDirectory), + Path.GetFullPath(DataDirectory) + }; + + foreach (var sourceDirectory in sources) + { + foreach (var file in new DirectoryInfo(sourceDirectory).EnumerateFiles("*", SearchOption.TopDirectoryOnly)) + { + var destinationDirectory = GetDestinationDirectory(file.Name); + if (destinationDirectory is null) continue; + + var destinationPath = Path.Combine(destinationDirectory, file.Name); + if (File.Exists(destinationPath)) + { + skippedFiles++; + continue; + } + + try + { + File.Move(file.FullName, destinationPath); + UpdateMappingEndpointPath(file.FullName, destinationPath); + movedFiles++; + } + catch (Exception e) when (e is IOException or UnauthorizedAccessException) + { + Log.Warning(e, "Could not migrate cache file '{CacheFile}'", file.FullName); + } + } + } + + if (movedFiles > 0) + Log.Information("Migrated {CacheFileCount} cached file(s) into dedicated cache directories", movedFiles); + if (skippedFiles > 0) + Log.Warning("Skipped {CacheFileCount} cache migration(s) because the destination file already exists", skippedFiles); + } + + private static string GetDestinationDirectory(string fileName) + { + if (fileName.EndsWith(".jmap.gz", StringComparison.OrdinalIgnoreCase)) + return MappingsDirectory; + + return Path.GetExtension(fileName).ToLowerInvariant() switch + { + ".chunk" or ".iochunk" or ".iopart" or ".utoc" => ChunksDirectory, + ".manifest" => ManifestsDirectory, + ".usmap" or ".jmap" => MappingsDirectory, + _ => null + }; + } + + private static void UpdateMappingEndpointPath(string oldPath, string newPath) + { + if (!newPath.StartsWith(MappingsDirectory, StringComparison.OrdinalIgnoreCase)) return; + + if (UserSettings.Default.PerDirectory is null) return; + + foreach (var directorySettings in UserSettings.Default.PerDirectory.Values) + { + if (directorySettings.Endpoints is null) continue; + + foreach (var endpoint in directorySettings.Endpoints) + { + if (endpoint is not null && string.Equals(endpoint.FilePath, oldPath, StringComparison.OrdinalIgnoreCase)) + endpoint.FilePath = newPath; + } + } + } +} diff --git a/FModel/ViewModels/ApiEndpoints/ValorantApiEndpoint.cs b/FModel/ViewModels/ApiEndpoints/ValorantApiEndpoint.cs index 2a450517..7cc15660 100644 --- a/FModel/ViewModels/ApiEndpoints/ValorantApiEndpoint.cs +++ b/FModel/ViewModels/ApiEndpoints/ValorantApiEndpoint.cs @@ -78,7 +78,7 @@ public class VManifest public async ValueTask PrefetchChunk(VChunk chunk, CancellationToken cancellationToken) { - var chunkPath = Path.Combine(UserSettings.Default.OutputDirectory, ".data", $"{chunk.Id}.chunk"); + var chunkPath = Path.Combine(CacheManager.ChunksDirectory, $"{chunk.Id}.chunk"); if (File.Exists(chunkPath)) return; using var response = await _client.GetAsync(chunk.GetUrl(), cancellationToken).ConfigureAwait(false); @@ -91,7 +91,7 @@ public class VManifest public async Task GetChunkBytes(VChunk chunk, CancellationToken cancellationToken) { - var chunkPath = Path.Combine(UserSettings.Default.OutputDirectory, ".data", $"{chunk.Id}.chunk"); + var chunkPath = Path.Combine(CacheManager.ChunksDirectory, $"{chunk.Id}.chunk"); byte[] chunkBytes; if (File.Exists(chunkPath)) diff --git a/FModel/ViewModels/AssetsFolderViewModel.cs b/FModel/ViewModels/AssetsFolderViewModel.cs index c76b0ad3..70f6eeaf 100644 --- a/FModel/ViewModels/AssetsFolderViewModel.cs +++ b/FModel/ViewModels/AssetsFolderViewModel.cs @@ -288,13 +288,16 @@ public class AssetsFolderViewModel lastNode?.AssetsList.Add(entry); } + Folders.AddRange(treeItems); + if (treeItems.Count > 0) { + // Select after publishing the collection. Selecting a detached TreeItem lets WPF + // auto-select the first root (usually the synthetic "Content" bucket) instead. var projectName = ApplicationService.ApplicationView.CUE4Parse.Provider.ProjectName; (treeItems.FirstOrDefault(x => x.Header.Equals(projectName, StringComparison.OrdinalIgnoreCase)) ?? treeItems[0]).IsSelected = true; } - Folders.AddRange(treeItems); ApplicationService.ApplicationView.CUE4Parse.SearchVm.ChangeCollection(entries); foreach (var folder in Folders) diff --git a/FModel/ViewModels/CUE4ParseViewModel.cs b/FModel/ViewModels/CUE4ParseViewModel.cs index 1c05422e..364033c0 100644 --- a/FModel/ViewModels/CUE4ParseViewModel.cs +++ b/FModel/ViewModels/CUE4ParseViewModel.cs @@ -236,7 +236,7 @@ public class CUE4ParseViewModel : ViewModel Provider.OnDemandOptions = new IoStoreOnDemandOptions { ChunkHostUri = new Uri("https://egdownload.fastly-edge.com/", UriKind.Absolute), - ChunkCacheDirectory = Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, ".data")), + ChunkCacheDirectory = new DirectoryInfo(CacheManager.ChunksDirectory), DownloaderClient = _chunkClient }; @@ -253,11 +253,10 @@ public class CUE4ParseViewModel : ViewModel throw new FileLoadException("Could not load latest Fortnite manifest, you may have to switch to your local installation."); } - var cacheDir = Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, ".data")).FullName; var manifestOptions = new ManifestParseOptions { - ChunkCacheDirectory = cacheDir, - ManifestCacheDirectory = cacheDir, + ChunkCacheDirectory = CacheManager.ChunksDirectory, + ManifestCacheDirectory = CacheManager.ManifestsDirectory, ChunkBaseUrl = "https://egdownload.fastly-edge.com/Builds/Fortnite/CloudDir/", Decompressor = Compression.Decompressor, Client = _chunkClient, @@ -285,25 +284,16 @@ public class CUE4ParseViewModel : ViewModel IoStoreOnDemand.Read(new StreamReader(ioStoreOnDemandFile.GetStream())); } - Parallel.ForEach(manifest.Files.Where(x => _fnLiveRegex.IsMatch(x.FileName)), fileManifest => - { - p.RegisterVfs(fileManifest.FileName, [fileManifest.GetStream()], - it => new FRandomAccessStreamArchive(it, manifest.FindFile(it)!.GetStream(), p.Versions)); - }); + RegisterFortniteLiveArchives(p, manifest, cancellationToken); var manifests = _apiEndpointView.DillyApi.GetManifests(cancellationToken); var downloadUrl = manifests.First(x => x.AppName == "Fortnite_Studio").DownloadUrl; - using var client = new HttpClient(); - var manifestBytes = client.GetByteArrayAsync(downloadUrl).GetAwaiter().GetResult(); + var manifestBytes = _chunkClient.GetByteArrayAsync(downloadUrl, cancellationToken).GetAwaiter().GetResult(); var uefnManifest = FBuildPatchAppManifest.Deserialize(manifestBytes, manifestOptions); - Parallel.ForEach(uefnManifest.Files.Where(x => _fnLiveRegex.IsMatch(x.FileName)), fileManifest => - { - p.RegisterVfs(fileManifest.FileName, [fileManifest.GetStream()], - it => new FRandomAccessStreamArchive(it, uefnManifest.FindFile(it)!.GetStream(), p.Versions)); - }); + RegisterFortniteLiveArchives(p, uefnManifest, cancellationToken); var elapsedTime = Stopwatch.GetElapsedTime(startTs); FLogger.Append(ELog.Information, () => @@ -351,6 +341,36 @@ public class CUE4ParseViewModel : ViewModel }); } + private void RegisterFortniteLiveArchives(StreamedFileProvider provider, FBuildPatchAppManifest manifest, + CancellationToken cancellationToken) + { + var archiveFiles = manifest.Files.Where(x => + _fnLiveRegex.IsMatch(x.FileName) && + (x.FileName.EndsWith(".pak", StringComparison.OrdinalIgnoreCase) || + x.FileName.EndsWith(".utoc", StringComparison.OrdinalIgnoreCase) || + x.FileName.EndsWith(".uondemandtoc", StringComparison.OrdinalIgnoreCase))).ToList(); + var parallelOptions = new ParallelOptions { CancellationToken = cancellationToken }; + + Parallel.ForEach(archiveFiles.Where(x => !x.FileName.EndsWith(".uondemandtoc", StringComparison.OrdinalIgnoreCase)), + parallelOptions, fileManifest => + { + provider.RegisterVfs(fileManifest.FileName, [fileManifest.GetStream()], + it => new FRandomAccessStreamArchive(it, manifest.FindFile(it)!.GetStream(), provider.Versions)); + }); + + // V2 on-demand TOCs are large and span many BuildPatch chunks. Reading them through CUE4Parse's synchronous + // archive interface downloads those chunks one at a time. Materialize each TOC through EpicManifestParser's + // parallel path first, then register it normally so the on-demand containers remain available. + foreach (var fileManifest in archiveFiles.Where(x => x.FileName.EndsWith(".uondemandtoc", StringComparison.OrdinalIgnoreCase))) + { + cancellationToken.ThrowIfCancellationRequested(); + using var stream = fileManifest.GetStream(); + var data = stream.SaveBytesAsync(8, cancellationToken).GetAwaiter().GetResult(); + using var archive = new FByteArchive(fileManifest.FileName, data, provider.Versions); + provider.RegisterVfs(new IoChunkToc(archive)); + } + } + /// /// load virtual files system from GameDirectory /// @@ -447,7 +467,7 @@ public class CUE4ParseViewModel : ViewModel endpoint.Path = "$.mappings.ZStandard"; } - var mappingsFolder = Path.Combine(UserSettings.Default.OutputDirectory, ".data"); + var mappingsFolder = CacheManager.MappingsDirectory; var mappings = _apiEndpointView.DynamicApi.GetMappings(CancellationToken.None, endpoint.Url, endpoint.Path); if (mappings is { Length: > 0 }) { diff --git a/FModel/ViewModels/Commands/LoadCommand.cs b/FModel/ViewModels/Commands/LoadCommand.cs index f476ea24..c29a695e 100644 --- a/FModel/ViewModels/Commands/LoadCommand.cs +++ b/FModel/ViewModels/Commands/LoadCommand.cs @@ -59,50 +59,53 @@ public class LoadCommand : ViewModelCommand _applicationView.IsAssetsExplorerVisible = true; Helper.CloseWindow("Search For Packages"); // close search window if opened - await Task.WhenAll( - _applicationView.CUE4Parse.LoadLocalizedResources(), // load locres if not already loaded, - _applicationView.CUE4Parse.LoadVirtualPaths(), // load virtual paths if not already loaded - _threadWorkerView.Begin(cancellationToken => + // Populate the package tree before doing supplemental reads. For streamed providers those + // reads may download BuildPatch chunks and must not delay or hide an otherwise valid tree. + await _threadWorkerView.Begin(cancellationToken => + { + // filter what to show + _applicationView.Status.UpdateStatusLabel("Packages", "Filtering"); + switch (UserSettings.Default.LoadingMode) { - // filter what to show - _applicationView.Status.UpdateStatusLabel("Packages", "Filtering"); - switch (UserSettings.Default.LoadingMode) + case ELoadingMode.Multiple: { - case ELoadingMode.Multiple: + var l = (IList) parameter; + if (l.Count == 0) { - var l = (IList) parameter; - if (l.Count == 0) - { - UserSettings.Default.LoadingMode = ELoadingMode.All; - goto case ELoadingMode.All; - } + UserSettings.Default.LoadingMode = ELoadingMode.All; + goto case ELoadingMode.All; + } - var directoryFilesToShow = l.Cast(); - FilterDirectoryFilesToDisplay(cancellationToken, directoryFilesToShow); - break; - } - case ELoadingMode.All: - { - FilterDirectoryFilesToDisplay(cancellationToken, null); - break; - } - case ELoadingMode.AllButNew: - case ELoadingMode.AllButModified: - { - FilterNewOrModifiedFilesToDisplay(cancellationToken); - break; - } - case ELoadingMode.AllButPatched: - { - FilterPacthedFilesToDisplay(cancellationToken); - break; - } - default: throw new ArgumentOutOfRangeException(); + var directoryFilesToShow = l.Cast(); + FilterDirectoryFilesToDisplay(cancellationToken, directoryFilesToShow); + break; } + case ELoadingMode.All: + { + FilterDirectoryFilesToDisplay(cancellationToken, null); + break; + } + case ELoadingMode.AllButNew: + case ELoadingMode.AllButModified: + { + FilterNewOrModifiedFilesToDisplay(cancellationToken); + break; + } + case ELoadingMode.AllButPatched: + { + FilterPacthedFilesToDisplay(cancellationToken); + break; + } + default: throw new ArgumentOutOfRangeException(); + } - _discordHandler.UpdatePresence(_applicationView.CUE4Parse); - }) - ).ConfigureAwait(false); + _discordHandler.UpdatePresence(_applicationView.CUE4Parse); + }).ConfigureAwait(false); + + // These enrich later package reads but are not required to display the archive contents. + // Run them sequentially to avoid competing BuildPatch download bursts on Fortnite Live. + await _applicationView.CUE4Parse.LoadVirtualPaths().ConfigureAwait(false); + await _applicationView.CUE4Parse.LoadLocalizedResources().ConfigureAwait(false); #if DEBUG loadingTime.Stop(); FLogger.Append(ELog.Debug, () => diff --git a/FModel/Views/SettingsView.xaml.cs b/FModel/Views/SettingsView.xaml.cs index a076f3be..015729b5 100644 --- a/FModel/Views/SettingsView.xaml.cs +++ b/FModel/Views/SettingsView.xaml.cs @@ -120,7 +120,7 @@ public partial class SettingsView var openFileDialog = new OpenFileDialog { Title = "Select a mapping file", - InitialDirectory = Path.Combine(UserSettings.Default.OutputDirectory, ".data"), + InitialDirectory = CacheManager.MappingsDirectory, Filter = "USMAP Files (*.usmap, *.jmap, *.jmap.gz)|*.usmap;*.jmap;*.jmap.gz|All Files (*.*)|*.*" };