Fix live loading and organize cached output

FModel/App.xaml.cs

@@ -108,7 +108,7 @@ public partial class App
         Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, "Backups"));
         if (createMe) Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, "Exports"));
         Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, "Logs"));
-        Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, ".data"));
+        CacheManager.EnsureDirectories();
 
         const string template = "{Timestamp:yyyy-MM-dd HH:mm:ss} [{Level:u3}] {Enriched}: {Message:lj}{NewLine}{Exception}";
         Log.Logger = new LoggerConfiguration()
@@ -125,6 +125,7 @@ public partial class App
 #endif
             .CreateLogger();
 
+        CacheManager.MigrateLegacyFiles();
         Log.Information("Version {Version} ({CommitId})", Constants.APP_VERSION, Constants.APP_COMMIT_ID);
         Log.Information("{OS}", GetOperatingSystemProductName());
         Log.Information("{RuntimeVer}", RuntimeInformation.FrameworkDescription);

FModel/Framework/CacheManager.cs

@@ -0,0 +1,106 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using FModel.Settings;
+using Serilog;
+
+namespace FModel.Framework;
+
+public static class CacheManager
+{
+    public static string DataDirectory =>
+        Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, ".data")).FullName;
+
+    public static string ChunksDirectory =>
+        Directory.CreateDirectory(Path.Combine(DataDirectory, "chunks")).FullName;
+
+    public static string ManifestsDirectory =>
+        Directory.CreateDirectory(Path.Combine(DataDirectory, "manifests")).FullName;
+
+    public static string MappingsDirectory =>
+        Directory.CreateDirectory(Path.Combine(DataDirectory, "mappings")).FullName;
+
+    public static void EnsureDirectories()
+    {
+        _ = ChunksDirectory;
+        _ = ManifestsDirectory;
+        _ = MappingsDirectory;
+    }
+
+    public static void MigrateLegacyFiles()
+    {
+        EnsureDirectories();
+
+        var movedFiles = 0;
+        var skippedFiles = 0;
+        var sources = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
+        {
+            Path.GetFullPath(UserSettings.Default.OutputDirectory),
+            Path.GetFullPath(DataDirectory)
+        };
+
+        foreach (var sourceDirectory in sources)
+        {
+            foreach (var file in new DirectoryInfo(sourceDirectory).EnumerateFiles("*", SearchOption.TopDirectoryOnly))
+            {
+                var destinationDirectory = GetDestinationDirectory(file.Name);
+                if (destinationDirectory is null) continue;
+
+                var destinationPath = Path.Combine(destinationDirectory, file.Name);
+                if (File.Exists(destinationPath))
+                {
+                    skippedFiles++;
+                    continue;
+                }
+
+                try
+                {
+                    File.Move(file.FullName, destinationPath);
+                    UpdateMappingEndpointPath(file.FullName, destinationPath);
+                    movedFiles++;
+                }
+                catch (Exception e) when (e is IOException or UnauthorizedAccessException)
+                {
+                    Log.Warning(e, "Could not migrate cache file '{CacheFile}'", file.FullName);
+                }
+            }
+        }
+
+        if (movedFiles > 0)
+            Log.Information("Migrated {CacheFileCount} cached file(s) into dedicated cache directories", movedFiles);
+        if (skippedFiles > 0)
+            Log.Warning("Skipped {CacheFileCount} cache migration(s) because the destination file already exists", skippedFiles);
+    }
+
+    private static string GetDestinationDirectory(string fileName)
+    {
+        if (fileName.EndsWith(".jmap.gz", StringComparison.OrdinalIgnoreCase))
+            return MappingsDirectory;
+
+        return Path.GetExtension(fileName).ToLowerInvariant() switch
+        {
+            ".chunk" or ".iochunk" or ".iopart" or ".utoc" => ChunksDirectory,
+            ".manifest" => ManifestsDirectory,
+            ".usmap" or ".jmap" => MappingsDirectory,
+            _ => null
+        };
+    }
+
+    private static void UpdateMappingEndpointPath(string oldPath, string newPath)
+    {
+        if (!newPath.StartsWith(MappingsDirectory, StringComparison.OrdinalIgnoreCase)) return;
+
+        if (UserSettings.Default.PerDirectory is null) return;
+
+        foreach (var directorySettings in UserSettings.Default.PerDirectory.Values)
+        {
+            if (directorySettings.Endpoints is null) continue;
+
+            foreach (var endpoint in directorySettings.Endpoints)
+            {
+                if (endpoint is not null && string.Equals(endpoint.FilePath, oldPath, StringComparison.OrdinalIgnoreCase))
+                    endpoint.FilePath = newPath;
+            }
+        }
+    }
+}

FModel/ViewModels/ApiEndpoints/ValorantApiEndpoint.cs

@@ -78,7 +78,7 @@ public class VManifest
 
     public async ValueTask PrefetchChunk(VChunk chunk, CancellationToken cancellationToken)
     {
-        var chunkPath = Path.Combine(UserSettings.Default.OutputDirectory, ".data", $"{chunk.Id}.chunk");
+        var chunkPath = Path.Combine(CacheManager.ChunksDirectory, $"{chunk.Id}.chunk");
         if (File.Exists(chunkPath)) return;
         using var response = await _client.GetAsync(chunk.GetUrl(), cancellationToken).ConfigureAwait(false);
 
@@ -91,7 +91,7 @@ public class VManifest
 
     public async Task<byte[]> GetChunkBytes(VChunk chunk, CancellationToken cancellationToken)
     {
-        var chunkPath = Path.Combine(UserSettings.Default.OutputDirectory, ".data", $"{chunk.Id}.chunk");
+        var chunkPath = Path.Combine(CacheManager.ChunksDirectory, $"{chunk.Id}.chunk");
         byte[] chunkBytes;
 
         if (File.Exists(chunkPath))

FModel/ViewModels/AssetsFolderViewModel.cs

@@ -288,13 +288,16 @@ public class AssetsFolderViewModel
                 lastNode?.AssetsList.Add(entry);
             }
 
+            Folders.AddRange(treeItems);
+
             if (treeItems.Count > 0)
             {
+                // Select after publishing the collection. Selecting a detached TreeItem lets WPF
+                // auto-select the first root (usually the synthetic "Content" bucket) instead.
                 var projectName = ApplicationService.ApplicationView.CUE4Parse.Provider.ProjectName;
                 (treeItems.FirstOrDefault(x => x.Header.Equals(projectName, StringComparison.OrdinalIgnoreCase)) ?? treeItems[0]).IsSelected = true;
             }
 
-            Folders.AddRange(treeItems);
             ApplicationService.ApplicationView.CUE4Parse.SearchVm.ChangeCollection(entries);
 
             foreach (var folder in Folders)

FModel/ViewModels/CUE4ParseViewModel.cs

@@ -236,7 +236,7 @@ public class CUE4ParseViewModel : ViewModel
             Provider.OnDemandOptions = new IoStoreOnDemandOptions
             {
                 ChunkHostUri = new Uri("https://egdownload.fastly-edge.com/", UriKind.Absolute),
-                ChunkCacheDirectory = Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, ".data")),
+                ChunkCacheDirectory = new DirectoryInfo(CacheManager.ChunksDirectory),
                 DownloaderClient = _chunkClient
             };
 
@@ -253,11 +253,10 @@ public class CUE4ParseViewModel : ViewModel
                                 throw new FileLoadException("Could not load latest Fortnite manifest, you may have to switch to your local installation.");
                             }
 
-                            var cacheDir = Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, ".data")).FullName;
                             var manifestOptions = new ManifestParseOptions
                             {
-                                ChunkCacheDirectory = cacheDir,
-                                ManifestCacheDirectory = cacheDir,
+                                ChunkCacheDirectory = CacheManager.ChunksDirectory,
+                                ManifestCacheDirectory = CacheManager.ManifestsDirectory,
                                 ChunkBaseUrl = "https://egdownload.fastly-edge.com/Builds/Fortnite/CloudDir/",
                                 Decompressor = Compression.Decompressor,
                                 Client = _chunkClient,
@@ -285,25 +284,16 @@ public class CUE4ParseViewModel : ViewModel
                                 IoStoreOnDemand.Read(new StreamReader(ioStoreOnDemandFile.GetStream()));
                             }
 
-                            Parallel.ForEach(manifest.Files.Where(x => _fnLiveRegex.IsMatch(x.FileName)), fileManifest =>
-                            {
-                                p.RegisterVfs(fileManifest.FileName, [fileManifest.GetStream()],
-                                    it => new FRandomAccessStreamArchive(it, manifest.FindFile(it)!.GetStream(), p.Versions));
-                            });
+                            RegisterFortniteLiveArchives(p, manifest, cancellationToken);
 
                             var manifests = _apiEndpointView.DillyApi.GetManifests(cancellationToken);
                             var downloadUrl = manifests.First(x => x.AppName == "Fortnite_Studio").DownloadUrl;
 
-                            using var client = new HttpClient();
-                            var manifestBytes = client.GetByteArrayAsync(downloadUrl).GetAwaiter().GetResult();
+                            var manifestBytes = _chunkClient.GetByteArrayAsync(downloadUrl, cancellationToken).GetAwaiter().GetResult();
 
                             var uefnManifest = FBuildPatchAppManifest.Deserialize(manifestBytes, manifestOptions);
 
-                            Parallel.ForEach(uefnManifest.Files.Where(x => _fnLiveRegex.IsMatch(x.FileName)), fileManifest =>
-                            {
-                                p.RegisterVfs(fileManifest.FileName, [fileManifest.GetStream()],
-                                    it => new FRandomAccessStreamArchive(it, uefnManifest.FindFile(it)!.GetStream(), p.Versions));
-                            });
+                            RegisterFortniteLiveArchives(p, uefnManifest, cancellationToken);
 
                             var elapsedTime = Stopwatch.GetElapsedTime(startTs);
                             FLogger.Append(ELog.Information, () =>
@@ -351,6 +341,36 @@ public class CUE4ParseViewModel : ViewModel
         });
     }
 
+    private void RegisterFortniteLiveArchives(StreamedFileProvider provider, FBuildPatchAppManifest manifest,
+        CancellationToken cancellationToken)
+    {
+        var archiveFiles = manifest.Files.Where(x =>
+            _fnLiveRegex.IsMatch(x.FileName) &&
+            (x.FileName.EndsWith(".pak", StringComparison.OrdinalIgnoreCase) ||
+             x.FileName.EndsWith(".utoc", StringComparison.OrdinalIgnoreCase) ||
+             x.FileName.EndsWith(".uondemandtoc", StringComparison.OrdinalIgnoreCase))).ToList();
+        var parallelOptions = new ParallelOptions { CancellationToken = cancellationToken };
+
+        Parallel.ForEach(archiveFiles.Where(x => !x.FileName.EndsWith(".uondemandtoc", StringComparison.OrdinalIgnoreCase)),
+            parallelOptions, fileManifest =>
+            {
+                provider.RegisterVfs(fileManifest.FileName, [fileManifest.GetStream()],
+                    it => new FRandomAccessStreamArchive(it, manifest.FindFile(it)!.GetStream(), provider.Versions));
+            });
+
+        // V2 on-demand TOCs are large and span many BuildPatch chunks. Reading them through CUE4Parse's synchronous
+        // archive interface downloads those chunks one at a time. Materialize each TOC through EpicManifestParser's
+        // parallel path first, then register it normally so the on-demand containers remain available.
+        foreach (var fileManifest in archiveFiles.Where(x => x.FileName.EndsWith(".uondemandtoc", StringComparison.OrdinalIgnoreCase)))
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            using var stream = fileManifest.GetStream();
+            var data = stream.SaveBytesAsync(8, cancellationToken).GetAwaiter().GetResult();
+            using var archive = new FByteArchive(fileManifest.FileName, data, provider.Versions);
+            provider.RegisterVfs(new IoChunkToc(archive));
+        }
+    }
+
     /// <summary>
     /// load virtual files system from GameDirectory
     /// </summary>
@@ -447,7 +467,7 @@ public class CUE4ParseViewModel : ViewModel
                     endpoint.Path = "$.mappings.ZStandard";
                 }
 
-                var mappingsFolder = Path.Combine(UserSettings.Default.OutputDirectory, ".data");
+                var mappingsFolder = CacheManager.MappingsDirectory;
                 var mappings = _apiEndpointView.DynamicApi.GetMappings(CancellationToken.None, endpoint.Url, endpoint.Path);
                 if (mappings is { Length: > 0 })
                 {

FModel/ViewModels/Commands/LoadCommand.cs

@@ -59,50 +59,53 @@ public class LoadCommand : ViewModelCommand<LoadingModesViewModel>
         _applicationView.IsAssetsExplorerVisible = true;
         Helper.CloseWindow<AdonisWindow>("Search For Packages"); // close search window if opened
 
-        await Task.WhenAll(
-            _applicationView.CUE4Parse.LoadLocalizedResources(), // load locres if not already loaded,
-            _applicationView.CUE4Parse.LoadVirtualPaths(), // load virtual paths if not already loaded
-            _threadWorkerView.Begin(cancellationToken =>
+        // Populate the package tree before doing supplemental reads. For streamed providers those
+        // reads may download BuildPatch chunks and must not delay or hide an otherwise valid tree.
+        await _threadWorkerView.Begin(cancellationToken =>
+        {
+            // filter what to show
+            _applicationView.Status.UpdateStatusLabel("Packages", "Filtering");
+            switch (UserSettings.Default.LoadingMode)
             {
-                // filter what to show
-                _applicationView.Status.UpdateStatusLabel("Packages", "Filtering");
-                switch (UserSettings.Default.LoadingMode)
+                case ELoadingMode.Multiple:
                 {
-                    case ELoadingMode.Multiple:
+                    var l = (IList) parameter;
+                    if (l.Count == 0)
                     {
-                        var l = (IList) parameter;
-                        if (l.Count == 0)
-                        {
-                            UserSettings.Default.LoadingMode = ELoadingMode.All;
-                            goto case ELoadingMode.All;
-                        }
+                        UserSettings.Default.LoadingMode = ELoadingMode.All;
+                        goto case ELoadingMode.All;
+                    }
 
-                        var directoryFilesToShow = l.Cast<FileItem>();
-                        FilterDirectoryFilesToDisplay(cancellationToken, directoryFilesToShow);
-                        break;
-                    }
-                    case ELoadingMode.All:
-                    {
-                        FilterDirectoryFilesToDisplay(cancellationToken, null);
-                        break;
-                    }
-                    case ELoadingMode.AllButNew:
-                    case ELoadingMode.AllButModified:
-                    {
-                        FilterNewOrModifiedFilesToDisplay(cancellationToken);
-                        break;
-                    }
-                    case ELoadingMode.AllButPatched:
-                    {
-                        FilterPacthedFilesToDisplay(cancellationToken);
-                        break;
-                    }
-                    default: throw new ArgumentOutOfRangeException();
+                    var directoryFilesToShow = l.Cast<FileItem>();
+                    FilterDirectoryFilesToDisplay(cancellationToken, directoryFilesToShow);
+                    break;
                 }
+                case ELoadingMode.All:
+                {
+                    FilterDirectoryFilesToDisplay(cancellationToken, null);
+                    break;
+                }
+                case ELoadingMode.AllButNew:
+                case ELoadingMode.AllButModified:
+                {
+                    FilterNewOrModifiedFilesToDisplay(cancellationToken);
+                    break;
+                }
+                case ELoadingMode.AllButPatched:
+                {
+                    FilterPacthedFilesToDisplay(cancellationToken);
+                    break;
+                }
+                default: throw new ArgumentOutOfRangeException();
+            }
 
-                _discordHandler.UpdatePresence(_applicationView.CUE4Parse);
-            })
-        ).ConfigureAwait(false);
+            _discordHandler.UpdatePresence(_applicationView.CUE4Parse);
+        }).ConfigureAwait(false);
+
+        // These enrich later package reads but are not required to display the archive contents.
+        // Run them sequentially to avoid competing BuildPatch download bursts on Fortnite Live.
+        await _applicationView.CUE4Parse.LoadVirtualPaths().ConfigureAwait(false);
+        await _applicationView.CUE4Parse.LoadLocalizedResources().ConfigureAwait(false);
 #if DEBUG
         loadingTime.Stop();
         FLogger.Append(ELog.Debug, () =>

FModel/Views/SettingsView.xaml.cs

@@ -120,7 +120,7 @@ public partial class SettingsView
         var openFileDialog = new OpenFileDialog
         {
             Title = "Select a mapping file",
-            InitialDirectory = Path.Combine(UserSettings.Default.OutputDirectory, ".data"),
+            InitialDirectory = CacheManager.MappingsDirectory,
             Filter = "USMAP Files (*.usmap, *.jmap, *.jmap.gz)|*.usmap;*.jmap;*.jmap.gz|All Files (*.*)|*.*"
         };