diff --git a/CUE4Parse b/CUE4Parse index 76a4c53a..fe7db2d0 160000 --- a/CUE4Parse +++ b/CUE4Parse @@ -1 +1 @@ -Subproject commit 76a4c53afe1f26c24a84973f5228a6ef475de8b7 +Subproject commit fe7db2d0f06ee33df69d49b5566a1bc00ce33db2 diff --git a/FModel/App.xaml.cs b/FModel/App.xaml.cs index 2dd04be6..fb282ea6 100644 --- a/FModel/App.xaml.cs +++ b/FModel/App.xaml.cs @@ -110,7 +110,7 @@ public partial class App Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, "Backups")); if (createMe) Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, "Exports")); Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, "Logs")); - Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, ".data")); + CacheManager.EnsureDirectories(); #if DEBUG var filePath = Path.Combine(UserSettings.Default.OutputDirectory, "Logs", $"FModel-Debug-Log-{DateTime.Now:yyyy-MM-dd}.log"); @@ -137,6 +137,7 @@ public partial class App .MinimumLevel.Override("CUE4Parse_Conversion", LogEventLevel.Verbose).WriteTo.Sink(ImGuiSink.Instance) .CreateLogger(); + CacheManager.MigrateLegacyFiles(); Log.Information("Version {Version} ({CommitId})", Constants.APP_VERSION, Constants.APP_COMMIT_ID); Log.Information("{OS}", GetOperatingSystemProductName()); Log.Information("{RuntimeVer}", RuntimeInformation.FrameworkDescription); diff --git a/FModel/Framework/CacheManager.cs b/FModel/Framework/CacheManager.cs new file mode 100644 index 00000000..55b55478 --- /dev/null +++ b/FModel/Framework/CacheManager.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections.Generic; +using System.IO; +using FModel.Settings; +using Serilog; + +namespace FModel.Framework; + +public static class CacheManager +{ + public static string DataDirectory => + Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, ".data")).FullName; + + public static string ChunksDirectory => + Directory.CreateDirectory(Path.Combine(DataDirectory, "chunks")).FullName; + + public static string ManifestsDirectory => + Directory.CreateDirectory(Path.Combine(DataDirectory, "manifests")).FullName; + + public static string MappingsDirectory => + Directory.CreateDirectory(Path.Combine(DataDirectory, "mappings")).FullName; + + public static void EnsureDirectories() + { + _ = ChunksDirectory; + _ = ManifestsDirectory; + _ = MappingsDirectory; + } + + public static void MigrateLegacyFiles() + { + EnsureDirectories(); + + var movedFiles = 0; + var skippedFiles = 0; + var sources = new HashSet(StringComparer.OrdinalIgnoreCase) + { + Path.GetFullPath(UserSettings.Default.OutputDirectory), + Path.GetFullPath(DataDirectory) + }; + + foreach (var sourceDirectory in sources) + { + foreach (var file in new DirectoryInfo(sourceDirectory).EnumerateFiles("*", SearchOption.TopDirectoryOnly)) + { + var destinationDirectory = GetDestinationDirectory(file.Name); + if (destinationDirectory is null) continue; + + var destinationPath = Path.Combine(destinationDirectory, file.Name); + if (File.Exists(destinationPath)) + { + skippedFiles++; + continue; + } + + try + { + File.Move(file.FullName, destinationPath); + UpdateMappingEndpointPath(file.FullName, destinationPath); + movedFiles++; + } + catch (Exception e) when (e is IOException or UnauthorizedAccessException) + { + Log.Warning(e, "Could not migrate cache file '{CacheFile}'", file.FullName); + } + } + } + + if (movedFiles > 0) + Log.Information("Migrated {CacheFileCount} cached file(s) into dedicated cache directories", movedFiles); + if (skippedFiles > 0) + Log.Warning("Skipped {CacheFileCount} cache migration(s) because the destination file already exists", skippedFiles); + } + + private static string GetDestinationDirectory(string fileName) + { + if (fileName.EndsWith(".jmap.gz", StringComparison.OrdinalIgnoreCase)) + return MappingsDirectory; + + return Path.GetExtension(fileName).ToLowerInvariant() switch + { + ".chunk" or ".iochunk" or ".iopart" or ".utoc" => ChunksDirectory, + ".manifest" => ManifestsDirectory, + ".usmap" or ".jmap" => MappingsDirectory, + _ => null + }; + } + + private static void UpdateMappingEndpointPath(string oldPath, string newPath) + { + if (!newPath.StartsWith(MappingsDirectory, StringComparison.OrdinalIgnoreCase)) return; + + if (UserSettings.Default.PerDirectory is null) return; + + foreach (var directorySettings in UserSettings.Default.PerDirectory.Values) + { + if (directorySettings.Endpoints is null) continue; + + foreach (var endpoint in directorySettings.Endpoints) + { + if (endpoint is not null && string.Equals(endpoint.FilePath, oldPath, StringComparison.OrdinalIgnoreCase)) + endpoint.FilePath = newPath; + } + } + } +} diff --git a/FModel/ViewModels/ApiEndpoints/ValorantApiEndpoint.cs b/FModel/ViewModels/ApiEndpoints/ValorantApiEndpoint.cs index 2a450517..7cc15660 100644 --- a/FModel/ViewModels/ApiEndpoints/ValorantApiEndpoint.cs +++ b/FModel/ViewModels/ApiEndpoints/ValorantApiEndpoint.cs @@ -78,7 +78,7 @@ public class VManifest public async ValueTask PrefetchChunk(VChunk chunk, CancellationToken cancellationToken) { - var chunkPath = Path.Combine(UserSettings.Default.OutputDirectory, ".data", $"{chunk.Id}.chunk"); + var chunkPath = Path.Combine(CacheManager.ChunksDirectory, $"{chunk.Id}.chunk"); if (File.Exists(chunkPath)) return; using var response = await _client.GetAsync(chunk.GetUrl(), cancellationToken).ConfigureAwait(false); @@ -91,7 +91,7 @@ public class VManifest public async Task GetChunkBytes(VChunk chunk, CancellationToken cancellationToken) { - var chunkPath = Path.Combine(UserSettings.Default.OutputDirectory, ".data", $"{chunk.Id}.chunk"); + var chunkPath = Path.Combine(CacheManager.ChunksDirectory, $"{chunk.Id}.chunk"); byte[] chunkBytes; if (File.Exists(chunkPath)) diff --git a/FModel/ViewModels/AssetsFolderViewModel.cs b/FModel/ViewModels/AssetsFolderViewModel.cs index c76b0ad3..70f6eeaf 100644 --- a/FModel/ViewModels/AssetsFolderViewModel.cs +++ b/FModel/ViewModels/AssetsFolderViewModel.cs @@ -288,13 +288,16 @@ public class AssetsFolderViewModel lastNode?.AssetsList.Add(entry); } + Folders.AddRange(treeItems); + if (treeItems.Count > 0) { + // Select after publishing the collection. Selecting a detached TreeItem lets WPF + // auto-select the first root (usually the synthetic "Content" bucket) instead. var projectName = ApplicationService.ApplicationView.CUE4Parse.Provider.ProjectName; (treeItems.FirstOrDefault(x => x.Header.Equals(projectName, StringComparison.OrdinalIgnoreCase)) ?? treeItems[0]).IsSelected = true; } - Folders.AddRange(treeItems); ApplicationService.ApplicationView.CUE4Parse.SearchVm.ChangeCollection(entries); foreach (var folder in Folders) diff --git a/FModel/ViewModels/CUE4ParseViewModel.cs b/FModel/ViewModels/CUE4ParseViewModel.cs index 2d4923d7..4a1bd597 100644 --- a/FModel/ViewModels/CUE4ParseViewModel.cs +++ b/FModel/ViewModels/CUE4ParseViewModel.cs @@ -64,6 +64,7 @@ using CUE4Parse.UE4.Wwise; using CUE4Parse.Utils; using CUE4Parse_Conversion.Exporters; using CUE4Parse_Conversion.Sounds; +using CUE4Parse.GameTypes.LordOfMysteries.FileProvider; using CUE4Parse.MappingsProvider.Jmap; using CUE4Parse.MappingsProvider.Usmap; using EpicManifestParser; @@ -207,6 +208,7 @@ public class CUE4ParseViewModel : ViewModel _ when versionContainer.Game is EGame.GAME_AshEchoes => new AEDefaultFileProvider(gameDirectory, SearchOption.AllDirectories, versionContainer, pathComparer), _ when versionContainer.Game is EGame.GAME_BlackStigma => new DefaultFileProvider(gameDirectory, SearchOption.AllDirectories, versionContainer, StringComparer.Ordinal), _ when versionContainer.Game is EGame.GAME_HonorofKingsWorld => new HoKWDefaultFileProvider(gameDirectory, SearchOption.AllDirectories, versionContainer, pathComparer), + _ when versionContainer.Game is EGame.GAME_LordOfMysteries => new LoMDefaultFileProvider(gameDirectory, SearchOption.AllDirectories, versionContainer, pathComparer), _ => new DefaultFileProvider(gameDirectory, SearchOption.AllDirectories, versionContainer, pathComparer) }; @@ -234,7 +236,7 @@ public class CUE4ParseViewModel : ViewModel Provider.OnDemandOptions = new IoStoreOnDemandOptions { ChunkHostUri = new Uri("https://egdownload.fastly-edge.com/", UriKind.Absolute), - ChunkCacheDirectory = Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, ".data")), + ChunkCacheDirectory = new DirectoryInfo(CacheManager.ChunksDirectory), DownloaderClient = _chunkClient }; @@ -251,11 +253,10 @@ public class CUE4ParseViewModel : ViewModel throw new FileLoadException("Could not load latest Fortnite manifest, you may have to switch to your local installation."); } - var cacheDir = Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, ".data")).FullName; var manifestOptions = new ManifestParseOptions { - ChunkCacheDirectory = cacheDir, - ManifestCacheDirectory = cacheDir, + ChunkCacheDirectory = CacheManager.ChunksDirectory, + ManifestCacheDirectory = CacheManager.ManifestsDirectory, ChunkBaseUrl = "https://egdownload.fastly-edge.com/Builds/Fortnite/CloudDir/", Decompressor = Compression.Decompressor, Client = _chunkClient, @@ -283,25 +284,16 @@ public class CUE4ParseViewModel : ViewModel IoStoreOnDemand.Read(new StreamReader(ioStoreOnDemandFile.GetStream())); } - Parallel.ForEach(manifest.Files.Where(x => _fnLiveRegex.IsMatch(x.FileName)), fileManifest => - { - p.RegisterVfs(fileManifest.FileName, [fileManifest.GetStream()], - it => new FRandomAccessStreamArchive(it, manifest.FindFile(it)!.GetStream(), p.Versions)); - }); + RegisterFortniteLiveArchives(p, manifest, cancellationToken); var manifests = _apiEndpointView.DillyApi.GetManifests(cancellationToken); var downloadUrl = manifests.First(x => x.AppName == "Fortnite_Studio").DownloadUrl; - using var client = new HttpClient(); - var manifestBytes = client.GetByteArrayAsync(downloadUrl).GetAwaiter().GetResult(); + var manifestBytes = _chunkClient.GetByteArrayAsync(downloadUrl, cancellationToken).GetAwaiter().GetResult(); var uefnManifest = FBuildPatchAppManifest.Deserialize(manifestBytes, manifestOptions); - Parallel.ForEach(uefnManifest.Files.Where(x => _fnLiveRegex.IsMatch(x.FileName)), fileManifest => - { - p.RegisterVfs(fileManifest.FileName, [fileManifest.GetStream()], - it => new FRandomAccessStreamArchive(it, uefnManifest.FindFile(it)!.GetStream(), p.Versions)); - }); + RegisterFortniteLiveArchives(p, uefnManifest, cancellationToken); var elapsedTime = Stopwatch.GetElapsedTime(startTs); FLogger.Append(ELog.Information, () => @@ -349,6 +341,36 @@ public class CUE4ParseViewModel : ViewModel }); } + private void RegisterFortniteLiveArchives(StreamedFileProvider provider, FBuildPatchAppManifest manifest, + CancellationToken cancellationToken) + { + var archiveFiles = manifest.Files.Where(x => + _fnLiveRegex.IsMatch(x.FileName) && + (x.FileName.EndsWith(".pak", StringComparison.OrdinalIgnoreCase) || + x.FileName.EndsWith(".utoc", StringComparison.OrdinalIgnoreCase) || + x.FileName.EndsWith(".uondemandtoc", StringComparison.OrdinalIgnoreCase))).ToList(); + var parallelOptions = new ParallelOptions { CancellationToken = cancellationToken }; + + Parallel.ForEach(archiveFiles.Where(x => !x.FileName.EndsWith(".uondemandtoc", StringComparison.OrdinalIgnoreCase)), + parallelOptions, fileManifest => + { + provider.RegisterVfs(fileManifest.FileName, [fileManifest.GetStream()], + it => new FRandomAccessStreamArchive(it, manifest.FindFile(it)!.GetStream(), provider.Versions)); + }); + + // V2 on-demand TOCs are large and span many BuildPatch chunks. Reading them through CUE4Parse's synchronous + // archive interface downloads those chunks one at a time. Materialize each TOC through EpicManifestParser's + // parallel path first, then register it normally so the on-demand containers remain available. + foreach (var fileManifest in archiveFiles.Where(x => x.FileName.EndsWith(".uondemandtoc", StringComparison.OrdinalIgnoreCase))) + { + cancellationToken.ThrowIfCancellationRequested(); + using var stream = fileManifest.GetStream(); + var data = stream.SaveBytesAsync(8, cancellationToken).GetAwaiter().GetResult(); + using var archive = new FByteArchive(fileManifest.FileName, data, provider.Versions); + provider.RegisterVfs(new IoChunkToc(archive)); + } + } + /// /// load virtual files system from GameDirectory /// @@ -445,7 +467,7 @@ public class CUE4ParseViewModel : ViewModel endpoint.Path = "$.mappings.ZStandard"; } - var mappingsFolder = Path.Combine(UserSettings.Default.OutputDirectory, ".data"); + var mappingsFolder = CacheManager.MappingsDirectory; var mappings = _apiEndpointView.DynamicApi.GetMappings(CancellationToken.None, endpoint.Url, endpoint.Path); if (mappings is { Length: > 0 }) { @@ -1244,7 +1266,7 @@ public class CUE4ParseViewModel : ViewModel { if (!TabControl.CanAddTabs) return false; - TabControl.AddTab($"{verseDigest.ProjectName}.verse"); + TabControl.AddTab($"{verseDigest.Name}.verse"); TabControl.SelectedTab.Highlighter = AvalonExtensions.HighlighterSelector("verse"); TabControl.SelectedTab.SetDocumentText(verseDigest.ReadableCode, false, false); return true; @@ -1629,7 +1651,7 @@ public class CUE4ParseViewModel : ViewModel if (dummy is not UClass || pointer.Object.Value is not UClass blueprint) continue; - cppList.Add(blueprint.DecompileBlueprintToPseudo(pkg.Mappings, cookedMetaData)); + cppList.Add(blueprint.DecompileBlueprintToPseudo(cookedMetaData)); } if (cppList.Count == 0) return false; diff --git a/FModel/ViewModels/Commands/LoadCommand.cs b/FModel/ViewModels/Commands/LoadCommand.cs index f476ea24..c29a695e 100644 --- a/FModel/ViewModels/Commands/LoadCommand.cs +++ b/FModel/ViewModels/Commands/LoadCommand.cs @@ -59,50 +59,53 @@ public class LoadCommand : ViewModelCommand _applicationView.IsAssetsExplorerVisible = true; Helper.CloseWindow("Search For Packages"); // close search window if opened - await Task.WhenAll( - _applicationView.CUE4Parse.LoadLocalizedResources(), // load locres if not already loaded, - _applicationView.CUE4Parse.LoadVirtualPaths(), // load virtual paths if not already loaded - _threadWorkerView.Begin(cancellationToken => + // Populate the package tree before doing supplemental reads. For streamed providers those + // reads may download BuildPatch chunks and must not delay or hide an otherwise valid tree. + await _threadWorkerView.Begin(cancellationToken => + { + // filter what to show + _applicationView.Status.UpdateStatusLabel("Packages", "Filtering"); + switch (UserSettings.Default.LoadingMode) { - // filter what to show - _applicationView.Status.UpdateStatusLabel("Packages", "Filtering"); - switch (UserSettings.Default.LoadingMode) + case ELoadingMode.Multiple: { - case ELoadingMode.Multiple: + var l = (IList) parameter; + if (l.Count == 0) { - var l = (IList) parameter; - if (l.Count == 0) - { - UserSettings.Default.LoadingMode = ELoadingMode.All; - goto case ELoadingMode.All; - } + UserSettings.Default.LoadingMode = ELoadingMode.All; + goto case ELoadingMode.All; + } - var directoryFilesToShow = l.Cast(); - FilterDirectoryFilesToDisplay(cancellationToken, directoryFilesToShow); - break; - } - case ELoadingMode.All: - { - FilterDirectoryFilesToDisplay(cancellationToken, null); - break; - } - case ELoadingMode.AllButNew: - case ELoadingMode.AllButModified: - { - FilterNewOrModifiedFilesToDisplay(cancellationToken); - break; - } - case ELoadingMode.AllButPatched: - { - FilterPacthedFilesToDisplay(cancellationToken); - break; - } - default: throw new ArgumentOutOfRangeException(); + var directoryFilesToShow = l.Cast(); + FilterDirectoryFilesToDisplay(cancellationToken, directoryFilesToShow); + break; } + case ELoadingMode.All: + { + FilterDirectoryFilesToDisplay(cancellationToken, null); + break; + } + case ELoadingMode.AllButNew: + case ELoadingMode.AllButModified: + { + FilterNewOrModifiedFilesToDisplay(cancellationToken); + break; + } + case ELoadingMode.AllButPatched: + { + FilterPacthedFilesToDisplay(cancellationToken); + break; + } + default: throw new ArgumentOutOfRangeException(); + } - _discordHandler.UpdatePresence(_applicationView.CUE4Parse); - }) - ).ConfigureAwait(false); + _discordHandler.UpdatePresence(_applicationView.CUE4Parse); + }).ConfigureAwait(false); + + // These enrich later package reads but are not required to display the archive contents. + // Run them sequentially to avoid competing BuildPatch download bursts on Fortnite Live. + await _applicationView.CUE4Parse.LoadVirtualPaths().ConfigureAwait(false); + await _applicationView.CUE4Parse.LoadLocalizedResources().ConfigureAwait(false); #if DEBUG loadingTime.Stop(); FLogger.Append(ELog.Debug, () => diff --git a/FModel/ViewModels/GameFileViewModel.cs b/FModel/ViewModels/GameFileViewModel.cs index f8841805..f01ac7bf 100644 --- a/FModel/ViewModels/GameFileViewModel.cs +++ b/FModel/ViewModels/GameFileViewModel.cs @@ -366,6 +366,7 @@ public class GameFileViewModel(GameFile asset) : ViewModel case "stinfo": case "ushaderbytecode": case "upipelinecache": + case "dxbc": AssetCategory = EAssetCategory.ByteCode; break; case "wav": diff --git a/FModel/Views/SettingsView.xaml.cs b/FModel/Views/SettingsView.xaml.cs index ede4f0a6..7702ab0c 100644 --- a/FModel/Views/SettingsView.xaml.cs +++ b/FModel/Views/SettingsView.xaml.cs @@ -122,8 +122,8 @@ public partial class SettingsView var openFileDialog = new OpenFileDialog { Title = "Select a mapping file", - InitialDirectory = Path.Combine(UserSettings.Default.OutputDirectory, ".data"), - Filter = "USMAP Files (*.usmap)|*.usmap|All Files (*.*)|*.*" + InitialDirectory = CacheManager.MappingsDirectory, + Filter = "USMAP Files (*.usmap, *.jmap, *.jmap.gz)|*.usmap;*.jmap;*.jmap.gz|All Files (*.*)|*.*" }; if (!openFileDialog.ShowDialog().GetValueOrDefault())