// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.

using System;
using System.Collections.Generic;
using System.Linq;
using Duplicati.Library.Interface;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Main.Database;
using Duplicati.Library.Main.Volumes;
using Duplicati.Library.Utility;
using CoCoL;
using System.Threading.Tasks;
using System.Threading;

namespace Duplicati.Library.Main.Operation
{
    internal class RestoreHandler
    {
        /// <summary>
        /// The tag used for logging
        /// </summary>
        private static readonly string LOGTAG = Logging.Log.LogTagFromType<RestoreHandler>();

        private readonly Options m_options;
        private byte[] m_blockbuffer;
        private readonly RestoreResults m_result;
        private static readonly string DIRSEP = Util.DirectorySeparatorString;

        public RestoreHandler(Options options, RestoreResults result)
        {
            m_options = options;
            m_result = result;
        }

        /// <summary>
        /// Checks if the tempdir has enough free space relative to the volume size.
        /// Logs a warning if free space is less than 4 times the volume size.
        /// </summary>
        private void CheckTempDirFreeSpace()
        {
            try
            {
                // Get the tempdir path
                var tempDir = TempFolder.SystemTempPath;

                // Get free space in tempdir using the Utility method
                var spaceInfo = Library.Utility.Utility.GetFreeSpaceForPath(tempDir);
                if (spaceInfo == null)
                {
                    // Could not determine free space, skip the check
                    return;
                }

                // Get the volume size from options
                var volumeSize = m_options.VolumeSize;

                // Check if free space is less than 4 times the volume size
                if (spaceInfo.Value.FreeSpace < volumeSize * Library.Utility.Utility.VOLUME_SIZE_FREE_SPACE_MULTIPLIER)
                {
                    Logging.Log.WriteWarningMessage(LOGTAG, "TempDirLowSpace", null,
                        $"The temporary folder '{tempDir}' has limited free space ({Library.Utility.Utility.FormatSizeString(spaceInfo.Value.FreeSpace)}). " +
                        $"It is recommended to have at least {Library.Utility.Utility.VOLUME_SIZE_FREE_SPACE_MULTIPLIER} times the volume size ({Library.Utility.Utility.FormatSizeString(volumeSize * Library.Utility.Utility.VOLUME_SIZE_FREE_SPACE_MULTIPLIER)}) available for optimal restore operation.");
                }
            }
            catch
            {
                // Ignore errors during free space check
            }
        }

        /// <summary>
        /// Gets the compression module by parsing the filename
        /// </summary>
        /// <param name="filename">The filename to parse</param>
        /// <returns>The compression module</returns>
        public static string GetCompressionModule(string filename)
        {
            var tmp = VolumeBase.ParseFilename(filename);
            if (tmp == null)
                throw new UserInformationException(string.Format("Unable to parse filename to valid entry: {0}", filename), "FailedToParseRemoteName");

            return tmp.CompressionModule;
        }

        public static RecreateDatabaseHandler.NumberedFilterFilelistDelegate GetNumberedFilelistFilterDelegate(DateTime time, long[] versions, bool singleTimeMatch = false)
        {
            versions ??= [];
            if (versions.Length == 0 && time.Ticks == 0)
                return null;

            return (filelist) => FilterNumberedFilelist(filelist, time, versions, singleTimeMatch);
        }

        public static IEnumerable<KeyValuePair<long, IParsedVolume>> FilterNumberedFilelist(IEnumerable<IParsedVolume> filelist, DateTime time, long[] versions, bool singleTimeMatch = false)
        {
            if (time.Kind == DateTimeKind.Unspecified)
                throw new Exception("Unspecified datetime instance, must be either local or UTC");

            // Make sure the resolution is the same (i.e. no milliseconds)
            if (time.Ticks > 0)
                time = Library.Utility.Utility.DeserializeDateTime(Library.Utility.Utility.SerializeDateTime(time)).ToUniversalTime();

            versions ??= [];
            // Unwrap, so we do not query the remote storage twice
            var lst = (from n in filelist
                       where n.FileType == RemoteVolumeType.Files
                       orderby n.Time descending
                       select n).ToArray();

            var numbers = lst.Zip(Enumerable.Range(0, lst.Length), (a, b) => new KeyValuePair<long, IParsedVolume>(b, a)).ToList();

            if (time.Ticks > 0 && versions.Length > 0)
                return from n in numbers
                       where (singleTimeMatch ? n.Value.Time == time : n.Value.Time <= time) && versions.Contains(n.Key)
                       select n;
            else if (time.Ticks > 0)
                return from n in numbers
                       where (singleTimeMatch ? n.Value.Time == time : n.Value.Time <= time)
                       select n;
            else if (versions.Length > 0)
                return from n in numbers
                       where versions.Contains(n.Key)
                       select n;
            else
                return numbers;
        }

        public async Task RunAsync(string[] paths, IBackendManager backendManager, Library.Utility.IFilter filter, IRestoreDestinationProvider restoreDestination)
        {
            m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Begin);

            // Check tempdir free space before starting restore
            CheckTempDirFreeSpace();

            // Set the restore path in the results for logging purposes
            var restorePath = restoreDestination.TargetDestination;
            m_result.RestorePath = string.IsNullOrEmpty(restorePath)
                ? null
                : restorePath;

            // If we have both target paths and a filter, combine them into a single filter
            filter = JoinedFilterExpression.Join(new FilterExpression(paths), filter);

            LocalRestoreDatabase db = null;
            TempFile tmpdb = null;
            try
            {
                if (!m_options.NoLocalDb && SystemIO.IO_OS.FileExists(m_options.Dbpath))
                {
                    db = await LocalRestoreDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken)
                        .ConfigureAwait(false);
                }
                else
                {
                    Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, building a temporary database");
                    tmpdb = new TempFile();
                    RecreateDatabaseHandler.NumberedFilterFilelistDelegate filelistfilter = GetNumberedFilelistFilterDelegate(m_options.Time, m_options.Version);
                    db = await LocalRestoreDatabase.CreateAsync(tmpdb, null, m_result.TaskControl.ProgressToken)
                        .ConfigureAwait(false);
                    m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result);
                    using (new Logging.Timer(LOGTAG, "RecreateTempDbForRestore", "Recreate temporary database for restore"))
                        await new RecreateDatabaseHandler(m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults)
                            .DoRunAsync(backendManager, db, false, filter, filelistfilter, null)
                            .ConfigureAwait(false);

                    if (!m_options.SkipMetadata)
                        await ApplyStoredMetadata(m_options, new RestoreHandlerMetadataStorage(), restoreDestination, m_result.TaskControl.ProgressToken);

                    //If we have --version set, we need to adjust, as the db has only the required versions
                    //TODO: Bit of a hack to set options that way
                    if (m_options.Version != null && m_options.Version.Length > 0)
                        m_options.RawOptions["version"] = string.Join(",", Enumerable.Range(0, m_options.Version.Length).Select(x => x.ToString()));
                }

                if (m_options.RestoreLegacy)
                    await DoRunAsync(backendManager, db, filter, restoreDestination, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
                else
                    await DoRunNewAsync(backendManager, db, filter, restoreDestination, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
            }
            finally
            {
                if (db != null)
                    await db.DisposeAsync().ConfigureAwait(false);
                tmpdb?.Dispose();
            }
        }

        private static async Task PatchWithBlocklist(LocalRestoreDatabase database, BlockVolumeReader blocks, Options options, RestoreResults result, byte[] blockbuffer, RestoreHandlerMetadataStorage metadatastorage, IRestoreDestinationProvider restoreDestination, CancellationToken cancellationToken)
        {
            var blocksize = options.Blocksize;
            var updateCounter = 0L;
            var fullblockverification = options.FullBlockVerification;

            using var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm);
            await using var blockmarker = await database.CreateBlockMarkerAsync(cancellationToken).ConfigureAwait(false);
            await using var volumekeeper = await database.GetMissingBlockData(blocks, options.Blocksize, cancellationToken).ConfigureAwait(false);
            await foreach (var restorelist in volumekeeper.FilesWithMissingBlocks(cancellationToken).ConfigureAwait(false))
            {
                var targetpath = restorelist.Path;

                if (options.Dryrun)
                {
                    Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchFile", "Would patch file with remote data: {0}", targetpath);
                }
                else
                {
                    Logging.Log.WriteVerboseMessage(LOGTAG, "PatchingFile", "Patching file with remote data: {0}", targetpath);

                    try
                    {
                        if (!options.Dryrun)
                        {
                            var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath);
                            if (!string.IsNullOrWhiteSpace(folderpath))
                                if (await restoreDestination.CreateFolderIfNotExists(folderpath, cancellationToken).ConfigureAwait(false))
                                    Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for  file {1}", folderpath, targetpath);
                        }

                        // TODO: Much faster if we iterate the volume and checks what blocks are used,
                        // because the compressors usually like sequential reading
                        using (var file = await restoreDestination.OpenWrite(targetpath, cancellationToken).ConfigureAwait(false))
                            await foreach (var targetblock in restorelist.Blocks(cancellationToken).ConfigureAwait(false))
                            {
                                file.Position = targetblock.Offset;
                                var size = blocks.ReadBlock(targetblock.Key, blockbuffer);
                                if (targetblock.Size == size)
                                {
                                    var valid = !fullblockverification;
                                    if (!valid)
                                    {
                                        var key = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size));
                                        if (targetblock.Key == key)
                                            valid = true;
                                        else
                                            Logging.Log.WriteWarningMessage(LOGTAG, "InvalidBlock", null, "Invalid block detected for {0}, expected hash: {1}, actual hash: {2}", targetpath, targetblock.Key, key);
                                    }

                                    if (valid)
                                    {
                                        file.Write(blockbuffer, 0, size);
                                        await blockmarker
                                            .SetBlockRestored(restorelist.FileID, targetblock.Offset / blocksize, targetblock.Key, size, false, cancellationToken)
                                            .ConfigureAwait(false);
                                    }
                                }
                                else
                                {
                                    Logging.Log.WriteWarningMessage(LOGTAG, "WrongBlockSize", null, "Block with hash {0} should have size {1} but has size {2}", targetblock.Key, targetblock.Size, size);
                                }
                            }

                        if ((++updateCounter) % 20 == 0)
                            await blockmarker
                                .UpdateProcessed(result.OperationProgressUpdater, cancellationToken)
                                .ConfigureAwait(false);
                    }
                    catch (Exception ex)
                    {
                        Logging.Log.WriteWarningMessage(LOGTAG, "PatchFailed", ex, "Failed to patch file: \"{0}\", message: {1}, message: {1}", targetpath, ex.Message);
                        if (options.UnittestMode)
                            throw;
                    }
                }
            }

            if (!options.SkipMetadata)
            {
                await foreach (var restoremetadata in volumekeeper.MetadataWithMissingBlocks(cancellationToken).ConfigureAwait(false))
                {
                    var targetpath = restoremetadata.Path;
                    Logging.Log.WriteVerboseMessage(LOGTAG, "RecordingMetadata", "Recording metadata from remote data: {0}", targetpath);

                    try
                    {
                        // TODO: When we support multi-block metadata this needs to deal with it
                        using (var ms = new System.IO.MemoryStream())
                        {
                            await foreach (var targetblock in restoremetadata.Blocks(cancellationToken).ConfigureAwait(false))
                            {
                                ms.Position = targetblock.Offset;
                                var size = blocks.ReadBlock(targetblock.Key, blockbuffer);
                                if (targetblock.Size == size)
                                {
                                    ms.Write(blockbuffer, 0, size);
                                    await blockmarker
                                        .SetBlockRestored(restoremetadata.FileID, targetblock.Offset / blocksize, targetblock.Key, size, true, cancellationToken)
                                        .ConfigureAwait(false);
                                }
                            }

                            ms.Position = 0;
                            metadatastorage.Add(targetpath, ms);
                            //blockmarker.RecordMetadata(restoremetadata.FileID, ms);
                        }
                    }
                    catch (Exception ex)
                    {
                        Logging.Log.WriteWarningMessage(LOGTAG, "MetatdataRecordFailed", ex, "Failed to record metadata for file: \"{0}\", message: {1}", targetpath, ex.Message);
                        if (options.UnittestMode)
                            throw;
                    }
                }
            }
            await blockmarker
                .UpdateProcessed(result.OperationProgressUpdater, cancellationToken)
                .ConfigureAwait(false);
            await blockmarker.CommitAsync(cancellationToken).ConfigureAwait(false);
        }

        private static async Task ApplyStoredMetadata(Options options, RestoreHandlerMetadataStorage metadatastorage, IRestoreDestinationProvider restoreDestination, CancellationToken cancellationToken)
        {
            foreach (var metainfo in metadatastorage.Records)
            {
                var targetpath = metainfo.Key;

                if (options.Dryrun)
                {
                    Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchMetadata", "Would patch metadata with remote data: {0}", targetpath);
                }
                else
                {
                    Logging.Log.WriteVerboseMessage(LOGTAG, "PatchingMetadata", "Patching metadata with remote data: {0}", targetpath);
                    try
                    {
                        if (!options.Dryrun)
                        {
                            var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath);
                            // Only create the folder if the folder path is different from the target path
                            // (i.e., targetpath is not a root-level folder)
                            // Also skip if the folder is outside the target destination
                            if (!string.IsNullOrEmpty(folderpath) && folderpath != targetpath && (string.IsNullOrWhiteSpace(restoreDestination.TargetDestination) || Util.IsPathInsideTarget(folderpath, restoreDestination.TargetDestination)))
                            {
                                if (await restoreDestination.CreateFolderIfNotExists(folderpath, cancellationToken).ConfigureAwait(false))
                                    Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for target {1}", folderpath, targetpath);
                            }
                        }

                        await ApplyMetadata(targetpath, metainfo.Value, options, restoreDestination, cancellationToken).ConfigureAwait(false);
                    }
                    catch (Exception ex)
                    {
                        Logging.Log.WriteWarningMessage(LOGTAG, "MetadataWriteFailed", ex, "Failed to apply metadata to file: \"{0}\", message: {1}", targetpath, ex.Message);
                        if (options.UnittestMode)
                            throw;
                    }
                }
            }
        }

        /// <summary>
        /// Perform the restore operation.
        /// This is the new implementation, which utilizes a CSP network of processes to perform the restore.
        /// </summary>
        /// <param name="database">The database containing information about the restore.</param>
        /// <param name="filter">The filter of which files to restore.</param>
        /// <param name="restoreDestination">The destination to restore to.</param>
        /// <param name="cancellationToken">The cancellation token to monitor for cancellation requests.</param>
        private async Task DoRunNewAsync(IBackendManager backendManager, LocalRestoreDatabase database, IFilter filter, IRestoreDestinationProvider restoreDestination, CancellationToken cancellationToken)
        {
            // Perform initial setup
            await Utility.UpdateOptionsFromDb(database, m_options, cancellationToken)
                .ConfigureAwait(false);
            await Utility.VerifyOptionsAndUpdateDatabase(database, m_options, cancellationToken)
                .ConfigureAwait(false);

            if (m_options.RestoreChannelBufferSize < 0)
                throw new UserInformationException("Restore channel buffer size must be greater than or equal to 0", "RestoreChannelBufferSizeTooSmall");
            if (m_options.RestoreVolumeDecryptors <= 0)
                throw new UserInformationException("Restore volume decryptors must be greater than 0", "RestoreVolumeDecryptorsTooSmall");
            if (m_options.RestoreVolumeDecompressors <= 0)
                throw new UserInformationException("Restore volume decompressors must be greater than 0", "RestoreVolumeDecompressorsTooSmall");
            if (m_options.RestoreVolumeDownloaders <= 0)
                throw new UserInformationException("Restore volume downloaders must be greater than 0", "RestoreVolumeDownloadersTooSmall");

            // Verify the backend if necessary
            if (!m_options.NoBackendverification)
            {
                m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify);
                await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, cancellationToken).ConfigureAwait(false);
            }

            // Prepare the block and file list and create the directory structure
            m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList);
            using (new Logging.Timer(LOGTAG, "PrepareBlockList", "PrepareBlockList"))
                await PrepareBlockAndFileList(database, m_options, filter, restoreDestination, m_result)
                    .ConfigureAwait(false);
            m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders);
            using (new Logging.Timer(LOGTAG, "CreateDirectory", "CreateDirectory"))
                await CreateDirectoryStructure(database, restoreDestination, string.IsNullOrEmpty(restoreDestination.TargetDestination), m_options, m_result).ConfigureAwait(false);

            // At this point, there should be no more writes to the database, so we have to unlock the database:
            await database.Transaction
                .CommitAsync("CommitBeforeRestore", token: cancellationToken)
                .ConfigureAwait(false);

            using var setup_log_timer = new Logging.Timer(LOGTAG, "RestoreNetworkSetup", "RestoreNetworkSetup");
            var fileprocessor_requests = Enumerable.Range(0, m_options.RestoreFileProcessors).Select(_ => ChannelManager.CreateChannel<Restore.BlockRequest>(buffersize: Math.Max(m_options.RestoreChannelBufferSize, 1))).ToArray();
            var fileprocessor_responses = Enumerable.Range(0, m_options.RestoreFileProcessors).Select(_ => ChannelManager.CreateChannel<Task<Restore.DataBlock>>(buffersize: m_options.RestoreChannelBufferSize)).ToArray();

            // Configure channels and process parameters
            Restore.Channels channels = new(m_options);
            // Set the deadlock timer threshold to 1 minute per 10 MB of volume size
            var volsize = await database.GetLargestVolumeAsync(cancellationToken).ConfigureAwait(false);
            volsize = volsize > 0 ? volsize : m_options.VolumeSize;
            Restore.DeadlockTimer.initial_threshold = (int)TimeSpan.FromMinutes(1).TotalMilliseconds * Math.Max(1, (int)(volsize / (10L * 1024L * 1024L)));
            Restore.FileProcessor.file_processors_restoring_files = m_options.RestoreFileProcessors;

            // Initialize priority files synchronization
            var priorityFiles = restoreDestination.GetPriorityFiles();
            Restore.FileProcessor.priority_files_remaining = priorityFiles.Count;
            Restore.FileProcessor.priority_files_completed = new TaskCompletionSource();

            // Create the process network
            var filelister = Restore.FileLister.Run(channels, database, m_options, m_result, priorityFiles);
            var fileprocessors = Enumerable.Range(0, m_options.RestoreFileProcessors).Select(i => Restore.FileProcessor.Run(channels, database, fileprocessor_requests[i], fileprocessor_responses[i], restoreDestination, m_options, m_result)).ToArray();
            var blockmanager = Restore.BlockManager.Run(channels, database, fileprocessor_requests, fileprocessor_responses, m_options, m_result);
            var volumecache = Restore.VolumeManager.Run(channels, m_options, m_result);
            var volumedownloaders = Enumerable.Range(0, m_options.RestoreVolumeDownloaders).Select(i => Restore.VolumeDownloader.Run(channels, database, backendManager, m_options, m_result)).ToArray();
            var volumedecryptors = Enumerable.Range(0, m_options.RestoreVolumeDecryptors).Select(i => Restore.VolumeDecryptor.Run(channels, backendManager, m_options)).ToArray();
            var volumedecompressors = Enumerable.Range(0, m_options.RestoreVolumeDecompressors).Select(i => Restore.VolumeDecompressor.Run(channels, m_options)).ToArray();

            setup_log_timer.Dispose();

            // Wait for the network to complete
            Task[] all =
                [
                    filelister,
                    ..fileprocessors,
                    blockmanager,
                    volumecache,
                    ..volumedownloaders,
                    ..volumedecryptors,
                    ..volumedecompressors
                ];

            // Start the progress updater
            using (new Logging.Timer(LOGTAG, "RestoreNetworkWait", "RestoreNetworkWait"))
            using (var kill_updater = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
            {
                var updater = Task.Run(async () =>
                {
                    while (!kill_updater.Token.IsCancellationRequested)
                    {
                        m_result.OperationProgressUpdater.UpdatefilesProcessed(m_result.RestoredFiles, m_result.SizeOfRestoredFiles);
                        await Task.Delay(1000, kill_updater.Token).ConfigureAwait(false);
                    }
                }, kill_updater.Token);

                await Task.WhenAll(all).ConfigureAwait(false);
                kill_updater.Cancel();
            }

            await database.Transaction
                .CommitAsync("CommitAfterRestore", token: cancellationToken)
                .ConfigureAwait(false);

            await database.DisposePoolAsync().ConfigureAwait(false);

            if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                return;

            m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PostRestoreVerify);

            // If any errors occurred, log them
            if (m_result.BrokenRemoteFiles.Count > 0 || m_result.BrokenLocalFiles.Count > 0)
            {
                var nl = Environment.NewLine;
                int maxN = 10;
                long remoteFirstN = Math.Min(maxN, m_result.BrokenRemoteFiles.Count);
                string remoteFirst = remoteFirstN < m_result.BrokenRemoteFiles.Count ? $"first {maxN} " : string.Empty;
                long localFirstN = Math.Min(maxN, m_result.BrokenLocalFiles.Count);
                string localFirst = localFirstN < m_result.BrokenLocalFiles.Count ? $"first {maxN} " : string.Empty;

                string remoteMessage = m_result.BrokenRemoteFiles.Count > 0 ? $"Failed to download {m_result.BrokenRemoteFiles.Count} remote files." : string.Empty;
                string remoteList = m_result.BrokenRemoteFiles.Count > 0 ? $"The following {remoteFirst}remote files failed to download, which may be the cause:{nl}{string.Join(nl, m_result.BrokenRemoteFiles.Take(maxN))}{nl}" : string.Empty;
                string localMessage = m_result.BrokenLocalFiles.Count > 0 ? $"Failed to restore {m_result.BrokenLocalFiles.Count} local files." : string.Empty;
                string localList = m_result.BrokenLocalFiles.Count > 0 ? $"The following {localFirst}local files failed to restore:{nl}{string.Join(nl, m_result.BrokenLocalFiles.Take(maxN))}{nl}" : string.Empty;

                Logging.Log.WriteErrorMessage(LOGTAG, "RestoreFailures", null, $"{remoteMessage}{nl}{localMessage}{nl}{remoteList}{nl}{localList}");
            }
            else if (m_result.RestoredFiles == 0)
                Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored");

            // Drop the temp tables
            await database.DropRestoreTable(cancellationToken).ConfigureAwait(false);
            await backendManager.WaitForEmptyAsync(database, m_result.TaskControl.ProgressToken).ConfigureAwait(false);

            // Report that the restore is complete
            m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Finalize);
            m_result.EndTime = DateTime.UtcNow;
        }

        /// <summary>
        /// Creates a filter that matches only the given priority files.
        /// Priority file names are matched as suffix patterns (e.g., "geometry.json" becomes "*geometry.json").
        /// </summary>
        /// <param name="priorityFiles">The list of priority file names to include.</param>
        /// <returns>A filter that includes only files matching the priority file patterns.</returns>
        private static Library.Utility.IFilter BuildPriorityFilter(IList<string> priorityFiles)
        {
            var priorityPatterns = priorityFiles.Select(pf => "*" + pf);
            return new FilterExpression(priorityPatterns, true);
        }

        /// <summary>
        /// Creates a filter that excludes the given priority files from the original filter.
        /// The exclude filter is placed first so it takes precedence in the joined expression.
        /// </summary>
        /// <param name="priorityFiles">The list of priority file names to exclude.</param>
        /// <param name="originalFilter">The original filter to combine with.</param>
        /// <returns>A filter that matches the original filter but excludes priority files.</returns>
        private static Library.Utility.IFilter BuildExcludePriorityFilter(IList<string> priorityFiles, Library.Utility.IFilter originalFilter)
        {
            var excludePatterns = priorityFiles.Select(pf => "*" + pf);
            var excludeFilter = new FilterExpression(excludePatterns, false);
            // Put exclude first so it takes precedence in JoinedFilterExpression
            return JoinedFilterExpression.Join(excludeFilter, originalFilter);
        }

        /// <summary>
        /// Perform the restore operation.
        /// This is the legacy implementation, which performs the restore in a single thread. Kept as in case the new implementation fails.
        /// </summary>
        /// <param name="backendManager">The backend manager for downloading volumes.</param>
        /// <param name="database">The database containing information about the restore.</param>
        /// <param name="filter">The filter of which files to restore.</param>
        /// <param name="restoreDestination">The destination to restore to.</param>
        /// <param name="cancellationToken">The cancellation token to monitor for cancellation requests.</param>
        private async Task DoRunAsync(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, IRestoreDestinationProvider restoreDestination, CancellationToken cancellationToken)
        {
            using (var metadatastorage = new RestoreHandlerMetadataStorage())
            {
                // One-time setup
                await Utility.UpdateOptionsFromDb(database, m_options, cancellationToken)
                    .ConfigureAwait(false);
                await Utility.VerifyOptionsAndUpdateDatabase(database, m_options, cancellationToken)
                    .ConfigureAwait(false);
                m_blockbuffer = new byte[m_options.Blocksize];

                if (!m_options.NoBackendverification)
                {
                    m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify);
                    await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, cancellationToken).ConfigureAwait(false);
                }

                // Get priority files from restore destination
                var priorityFiles = restoreDestination.GetPriorityFiles();

                if (priorityFiles.Count > 0)
                {
                    // Phase 1: Restore priority files only
                    Logging.Log.WriteInformationMessage(LOGTAG, "PriorityFilesRestore", "Restoring {0} priority file(s) first: {1}", priorityFiles.Count, string.Join(", ", priorityFiles));
                    var priorityFilter = BuildPriorityFilter(priorityFiles);
                    var restoredBefore = m_result.RestoredFiles;

                    await RestoreCoreAsync(backendManager, database, priorityFilter, restoreDestination, metadatastorage, cancellationToken)
                        .ConfigureAwait(false);

                    // Warn about any priority files not found in backup
                    var restoredInPhase1 = m_result.RestoredFiles - restoredBefore;
                    if (restoredInPhase1 == 0)
                    {
                        foreach (var pf in priorityFiles)
                            Logging.Log.WriteWarningMessage(LOGTAG, "PriorityFileNotFound", null,
                                "Priority file '{0}' was not found in the backup. This may cause subsequent restore steps to fail.", pf);
                    }

                    // Phase 2: Restore remaining files (excluding priority files)
                    Logging.Log.WriteInformationMessage(LOGTAG, "RemainingFilesRestore", "Restoring remaining files (excluding priority files)");
                    var phase2Filter = BuildExcludePriorityFilter(priorityFiles, filter);
                    await RestoreCoreAsync(backendManager, database, phase2Filter, restoreDestination, metadatastorage, cancellationToken)
                        .ConfigureAwait(false);
                }
                else
                {
                    // No priority files — single phase
                    await RestoreCoreAsync(backendManager, database, filter, restoreDestination, metadatastorage, cancellationToken)
                        .ConfigureAwait(false);
                }
            }

            m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Finalize);
            m_result.EndTime = DateTime.UtcNow;
        }

        /// <summary>
        /// Core restore logic extracted from the legacy restore flow.
        /// Performs the full restore pipeline: prepare file/block lists, scan for existing blocks,
        /// download and patch from remote volumes, restore empty files, apply metadata, and verify.
        /// </summary>
        /// <param name="backendManager">The backend manager for downloading volumes.</param>
        /// <param name="database">The database containing information about the restore.</param>
        /// <param name="filter">The filter of which files to restore in this phase.</param>
        /// <param name="restoreDestination">The destination to restore to.</param>
        /// <param name="metadatastorage">Shared metadata storage across phases.</param>
        /// <param name="cancellationToken">The cancellation token to monitor for cancellation requests.</param>
        private async Task RestoreCoreAsync(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, IRestoreDestinationProvider restoreDestination, RestoreHandlerMetadataStorage metadatastorage, CancellationToken cancellationToken)
        {
            //Figure out what files are to be patched, and what blocks are needed
            m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList);
            using (new Logging.Timer(LOGTAG, "PrepareBlockList", "PrepareBlockList"))
                await PrepareBlockAndFileList(database, m_options, filter, restoreDestination, m_result)
                    .ConfigureAwait(false);

            //Make the entire output setup
            m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders);
            using (new Logging.Timer(LOGTAG, "CreateDirectory", "CreateDirectory"))
                await CreateDirectoryStructure(database, restoreDestination, string.IsNullOrEmpty(restoreDestination.TargetDestination), m_options, m_result)
                    .ConfigureAwait(false);

            //If we are patching an existing target folder, do not touch stuff that is already updated
            m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_ScanForExistingFiles);
            using (var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm))
            using (var filehasher = HashFactory.CreateHasher(m_options.FileHashAlgorithm))
            using (new Logging.Timer(LOGTAG, "ScanForExistingTargetBlocks", "ScanForExistingTargetBlocks"))
                await ScanForExistingTargetBlocks(database, m_blockbuffer, blockhasher, filehasher, m_options, restoreDestination, m_result).ConfigureAwait(false);

            //Look for existing blocks in the original source files only
            if (m_options.UseLocalBlocks && !string.IsNullOrEmpty(restoreDestination.TargetDestination))
            {
                using (var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm))
                using (new Logging.Timer(LOGTAG, "ScanForExistingSourceBlocksFast", "ScanForExistingSourceBlocksFast"))
                {
                    m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_ScanForLocalBlocks);
                    await ScanForExistingSourceBlocksFast(database, m_options, m_blockbuffer, blockhasher, restoreDestination, m_result).ConfigureAwait(false);
                }
            }

            if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
            {
                await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false);
                return;
            }

            // If other local files already have the blocks we want, we use them instead of downloading
            if (m_options.UseLocalBlocks)
            {
                m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PatchWithLocalBlocks);
                using (var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm))
                using (new Logging.Timer(LOGTAG, "PatchWithLocalBlocks", "PatchWithLocalBlocks"))
                    await ScanForExistingSourceBlocks(database, m_options, m_blockbuffer, blockhasher, m_result, metadatastorage, restoreDestination).ConfigureAwait(false);
            }

            if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
            {
                await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false);
                return;
            }

            // Fill BLOCKS with remote sources
            List<IRemoteVolume> volumes;
            using (new Logging.Timer(LOGTAG, "GetMissingVolumes", "GetMissingVolumes"))
                volumes = await database
                    .GetMissingVolumes(cancellationToken)
                    .ToListAsync(cancellationToken: cancellationToken)
                    .ConfigureAwait(false);

            if (volumes.Count > 0)
            {
                Logging.Log.WriteInformationMessage(LOGTAG, "RemoteFileCount", "{0} remote files are required to restore", volumes.Count);
                m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_DownloadingRemoteFiles);
            }

            var brokenFiles = new List<string>();

            using (new Logging.Timer(LOGTAG, "PatchWithBlocklist", "PatchWithBlocklist"))
                await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(volumes, cancellationToken).ConfigureAwait(false))
                {
                    try
                    {
                        if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                        {
                            await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false);
                            return;
                        }

                        using (tmpfile)
                        using (var blocks = new BlockVolumeReader(GetCompressionModule(name), tmpfile, m_options))
                            await PatchWithBlocklist(database, blocks, m_options, m_result, m_blockbuffer, metadatastorage, restoreDestination, cancellationToken)
                                .ConfigureAwait(false);
                    }
                    catch (Exception ex)
                    {
                        brokenFiles.Add(name);
                        Logging.Log.WriteErrorMessage(LOGTAG, "PatchingFailed", ex, "Failed to patch with remote file: \"{0}\", message: {1}", name, ex.Message);
                        if (ex.IsAbortException())
                            throw;
                    }
                }

            var fileErrors = 0L;

            // Restore empty files. They might not have any blocks so don't appear in any volume.
            await foreach (var file in database.GetFilesToRestore(true, cancellationToken).Where(item => item.Length == 0).ConfigureAwait(false))
            {
                Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreEmptyFile", "Restoring empty file \"{0}\"", file.Path);

                try
                {
                    var folderpath = SystemIO.IO_OS.PathGetDirectoryName(file.Path);
                    if (!string.IsNullOrEmpty(folderpath))
                    {
                        await restoreDestination.CreateFolderIfNotExists(folderpath, cancellationToken).ConfigureAwait(false);
                    }
                    // Just create the file and close it right away, empty statement is intentional.
                    using (var stream = await restoreDestination.OpenWrite(file.Path, cancellationToken).ConfigureAwait(false))
                    {
                        try
                        {
                            stream.SetLength(0);
                        }
                        catch (NotSupportedException)
                        {
                            // Some streams do not support setting length, ignore
                        }
                    }
                }
                catch (Exception ex)
                {
                    fileErrors++;
                    Logging.Log.WriteErrorMessage(LOGTAG, "RestoreFileFailed", ex, "Failed to restore empty file: \"{0}\". Error message was: {1}", file.Path, ex.Message);
                    if (ex.IsAbortException())
                        throw;
                }
            }

            // Enforcing the length of files is now already done during ScanForExistingTargetBlocks
            // and thus not necessary anymore.

            // Apply metadata
            if (!m_options.SkipMetadata)
                await ApplyStoredMetadata(m_options, metadatastorage, restoreDestination, m_result.TaskControl.ProgressToken).ConfigureAwait(false);

            if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                return;

            m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PostRestoreVerify);

            if (m_options.PerformRestoredFileVerification)
            {
                // After all blocks in the files are restored, verify the file hash
                using (var filehasher = HashFactory.CreateHasher(m_options.FileHashAlgorithm))
                using (new Logging.Timer(LOGTAG, "RestoreVerification", "RestoreVerification"))
                    await foreach (var file in database.GetFilesToRestore(true, cancellationToken).ConfigureAwait(false))
                    {
                        try
                        {
                            if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                            {
                                await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false);
                                return;
                            }

                            Logging.Log.WriteVerboseMessage(LOGTAG, "TestFileIntegrity", "Testing restored file integrity: {0}", file.Path);

                            string key;
                            long size;
                            using (var fs = await restoreDestination.OpenRead(file.Path, cancellationToken).ConfigureAwait(false))
                            {
                                size = fs.Length;
                                key = Convert.ToBase64String(filehasher.ComputeHash(fs));
                            }

                            if (key != file.Hash)
                                throw new Exception(string.Format("Failed to restore file: \"{0}\". File hash is {1}, expected hash is {2}", file.Path, key, file.Hash));
                            m_result.RestoredFiles++;
                            m_result.SizeOfRestoredFiles += size;
                        }
                        catch (Exception ex)
                        {
                            fileErrors++;
                            Logging.Log.WriteErrorMessage(LOGTAG, "RestoreFileFailed", ex, "Failed to restore file: \"{0}\". Error message was: {1}", file.Path, ex.Message);
                            if (ex.IsAbortException())
                                throw;
                        }
                    }
            }

            if (fileErrors > 0 && brokenFiles.Count > 0)
                Logging.Log.WriteInformationMessage(LOGTAG, "RestoreFailures", "Failed to restore {0} files, additionally the following files failed to download, which may be the cause:{1}{2}", fileErrors, Environment.NewLine, string.Join(Environment.NewLine, brokenFiles));
            else if (fileErrors > 0)
                Logging.Log.WriteInformationMessage(LOGTAG, "RestoreFailures", "Failed to restore {0} files", fileErrors);

            // Drop the temp tables
            await database.DropRestoreTable(cancellationToken).ConfigureAwait(false);
            await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false);
        }

        public static async Task<bool> ApplyMetadata(string path, System.IO.Stream stream, Options options, IRestoreDestinationProvider restoreDestination, CancellationToken cancellationToken)
        {
            // TODO This has been modified to return a bool indicating if anything was written to properly report the number of files restored. The legacy restore doesn't check this, which produces an error in the CI where it reports that no files have been restored, even though one have. It's in Duplicati/UnitTests/SymLinkTests.cs the test SymLinkTests.SymLinkExists() that fails on the very last assert that there are 0 warnings.
            using (var tr = new System.IO.StreamReader(stream))
            using (var jr = new Newtonsoft.Json.JsonTextReader(tr))
            {
                var metadata = new Newtonsoft.Json.JsonSerializer().Deserialize<Dictionary<string, string>>(jr);
                // If this is dry-run, we stop after having deserialized the metadata
                if (options.Dryrun)
                    return false;

                return await restoreDestination.WriteMetadata(path, metadata, options.RestoreSymlinkMetadata, options.RestorePermissions, cancellationToken).ConfigureAwait(false);
            }
        }

        private static async Task ScanForExistingSourceBlocksFast(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, IRestoreDestinationProvider restoreDestination, RestoreResults result)
        {
            // Fill BLOCKS with data from known local source files
            await using var blockmarker = await database.CreateBlockMarkerAsync(result.TaskControl.ProgressToken).ConfigureAwait(false);
            var updateCount = 0L;
            await foreach (var entry in database.GetFilesAndSourceBlocksFast(options.Blocksize, result.TaskControl.ProgressToken).ConfigureAwait(false))
            {
                var targetpath = entry.TargetPath;
                var targetfileid = entry.TargetFileID;
                var sourcepath = entry.SourcePath;
                var patched = false;

                try
                {
                    if (SystemIO.IO_OS.FileExists(sourcepath))
                    {
                        if (!options.Dryrun)
                        {
                            var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath);
                            if (!string.IsNullOrEmpty(folderpath))
                            {
                                if (await restoreDestination.CreateFolderIfNotExists(folderpath, result.TaskControl.ProgressToken).ConfigureAwait(false))
                                    Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for  file {1}", folderpath, targetpath);
                            }
                        }

                        using (var targetstream = options.Dryrun ? null : await restoreDestination.OpenWrite(targetpath, result.TaskControl.ProgressToken).ConfigureAwait(false))
                        {
                            try
                            {
                                using var sourcestream = SystemIO.IO_OS.FileOpenRead(sourcepath);
                                await foreach (var block in entry.Blocks(result.TaskControl.ProgressToken).ConfigureAwait(false))
                                {
                                    if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                                        return;

                                    //TODO: Handle metadata

                                    if (sourcestream.Length > block.Offset)
                                    {
                                        sourcestream.Position = block.Offset;

                                        int size = Library.Utility.Utility.ForceStreamRead(sourcestream, blockbuffer, blockbuffer.Length);
                                        if (size == block.Size)
                                        {
                                            var key = Convert.ToBase64String(hasher.ComputeHash(blockbuffer, 0, size));
                                            if (key == block.Hash)
                                            {
                                                patched = true;
                                                if (!options.Dryrun)
                                                {
                                                    targetstream.Position = block.Offset;
                                                    targetstream.Write(blockbuffer, 0, size);
                                                }

                                                await blockmarker.SetBlockRestored(targetfileid, block.Index, key, block.Size, false, result.TaskControl.ProgressToken)
                                                    .ConfigureAwait(false);
                                            }
                                        }
                                    }
                                }
                            }
                            catch (Exception ex)
                            {
                                Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with data from local file \"{1}\", message: {2}", targetpath, sourcepath, ex.Message);
                                if (ex.IsAbortException())
                                    throw;
                            }
                        }

                        if ((++updateCount) % 20 == 0)
                        {
                            await blockmarker
                                .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken)
                                .ConfigureAwait(false);
                            if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                                return;
                        }

                    }
                    else
                    {
                        Logging.Log.WriteVerboseMessage(LOGTAG, "LocalSourceMissing", "Local source file not found: {0}", sourcepath);
                    }
                }
                catch (Exception ex)
                {
                    Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with local data, message: {1}", targetpath, ex.Message);
                    if (ex.IsAbortException())
                        throw;
                    if (options.UnittestMode)
                        throw;
                }

                if (patched)
                    Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is patched with some local data: {0}", targetpath);
                else
                    Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is not patched any local data: {0}", targetpath);

                if (patched && options.Dryrun)
                    Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchWithLocal", "Would patch file with local data: {0}", targetpath);
            }

            await blockmarker
                .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken)
                .ConfigureAwait(false);
            await blockmarker.CommitAsync(result.TaskControl.ProgressToken).ConfigureAwait(false);
        }

        private static async Task ScanForExistingSourceBlocks(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result, RestoreHandlerMetadataStorage metadatastorage, IRestoreDestinationProvider restoreDestination)
        {
            // Fill BLOCKS with data from known local source files
            await using var blockmarker = await database.CreateBlockMarkerAsync(result.TaskControl.ProgressToken).ConfigureAwait(false);
            var updateCount = 0L;
            await foreach (var restorelist in database.GetFilesAndSourceBlocks(options.SkipMetadata, options.Blocksize, result.TaskControl.ProgressToken).ConfigureAwait(false))
            {
                var targetpath = restorelist.TargetPath;
                var targetfileid = restorelist.TargetFileID;
                var patched = false;
                try
                {
                    if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                        return;

                    if (!options.Dryrun)
                    {
                        var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath);
                        if (!string.IsNullOrEmpty(folderpath))
                        {
                            if (await restoreDestination.CreateFolderIfNotExists(folderpath, result.TaskControl.ProgressToken).ConfigureAwait(false))
                                Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for file {1}", folderpath, targetpath);
                        }
                    }

                    using (var file = options.Dryrun ? null : await restoreDestination.OpenWrite(targetpath, result.TaskControl.ProgressToken).ConfigureAwait(false))
                        await foreach (var targetblock in restorelist.Blocks(result.TaskControl.ProgressToken).ConfigureAwait(false))
                        {
                            await foreach (var source in targetblock.BlockSources(result.TaskControl.ProgressToken).ConfigureAwait(false))
                            {
                                try
                                {
                                    if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                                        return;

                                    if (SystemIO.IO_OS.FileExists(source.Path))
                                    {
                                        if (source.IsMetadata)
                                        {
                                            // TODO: Handle this by reconstructing
                                            // metadata from file and checking the hash

                                            continue;
                                        }
                                        else
                                        {
                                            using var sourcefile = SystemIO.IO_OS.FileOpenRead(source.Path);
                                            sourcefile.Position = source.Offset;
                                            int size = Library.Utility.Utility.ForceStreamRead(sourcefile, blockbuffer, blockbuffer.Length);
                                            if (size == targetblock.Size)
                                            {
                                                var key = Convert.ToBase64String(hasher.ComputeHash(blockbuffer, 0, size));
                                                if (key == targetblock.Hash)
                                                {
                                                    if (!options.Dryrun)
                                                    {
                                                        if (targetblock.IsMetadata)
                                                            metadatastorage.Add(targetpath, new System.IO.MemoryStream(blockbuffer, 0, size));
                                                        else
                                                        {
                                                            file.Position = targetblock.Offset;
                                                            file.Write(blockbuffer, 0, size);
                                                        }
                                                    }

                                                    await blockmarker
                                                        .SetBlockRestored(targetfileid, targetblock.Index, key, targetblock.Size, false, result.TaskControl.ProgressToken)
                                                        .ConfigureAwait(false);
                                                    patched = true;
                                                    break;
                                                }
                                            }
                                        }
                                    }
                                }
                                catch (Exception ex)
                                {
                                    Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with data from local file \"{1}\", message: {2}", targetpath, source.Path, ex.Message);
                                    if (ex.IsAbortException())
                                        throw;
                                }
                            }
                        }

                    if ((++updateCount) % 20 == 0)
                        await blockmarker
                            .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken)
                            .ConfigureAwait(false);
                }
                catch (Exception ex)
                {
                    Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with local data, message: {1}", targetpath, ex.Message);
                    if (options.UnittestMode)
                        throw;
                }

                if (patched)
                    Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is patched with some local data: {0}", targetpath);
                else
                    Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is not patched any local data: {0}", targetpath);

                if (patched && options.Dryrun)
                    Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchWithLocal", string.Format("Would patch file with local data: {0}", targetpath));
            }

            await blockmarker
                .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken)
                .ConfigureAwait(false);
            await blockmarker.CommitAsync(result.TaskControl.ProgressToken).ConfigureAwait(false);
        }

        private static async Task PrepareBlockAndFileList(LocalRestoreDatabase database, Options options, Library.Utility.IFilter filter, IRestoreDestinationProvider restoreDestination, RestoreResults result)
        {
            // Create a temporary table FILES by selecting the files from fileset that matches a specific operation id
            // Delete all entries from the temp table that are excluded by the filter(s)
            using (new Logging.Timer(LOGTAG, "PrepareRestoreFileList", "PrepareRestoreFileList"))
            {
                var c = await database
                    .PrepareRestoreFilelist(options.Time, options.Version, filter, result.TaskControl.ProgressToken)
                    .ConfigureAwait(false);
                result.OperationProgressUpdater.UpdatefileCount(c.Item1, c.Item2, true);
            }

            using (new Logging.Timer(LOGTAG, "SetTargetPaths", "SetTargetPaths"))
                if (!string.IsNullOrEmpty(restoreDestination.TargetDestination))
                {
                    // Find the largest common prefix
                    var largest_prefix = options.DontCompressRestorePaths
                        ? "" :
                        await database.GetLargestPrefix(result.TaskControl.ProgressToken).ConfigureAwait(false);

                    Logging.Log.WriteVerboseMessage(LOGTAG, "MappingRestorePath", "Mapping restore path prefix to \"{0}\" to \"{1}\"", largest_prefix, Util.AppendDirSeparator(restoreDestination.TargetDestination));

                    // Set the target paths, special care with C:\ and /
                    await database
                        .SetTargetPaths(largest_prefix, Util.AppendDirSeparator(restoreDestination.TargetDestination), result.TaskControl.ProgressToken)
                        .ConfigureAwait(false);
                }
                else
                {
                    await database.SetTargetPaths("", "", result.TaskControl.ProgressToken).ConfigureAwait(false);
                }

            // Create a temporary table BLOCKS that lists all blocks that needs to be recovered
            using (new Logging.Timer(LOGTAG, "FindMissingBlocks", "FindMissingBlocks"))
                await database
                    .FindMissingBlocks(options.SkipMetadata, result.TaskControl.ProgressToken)
                    .ConfigureAwait(false);

            // Create temporary tables and triggers that automatically track progress
            using (new Logging.Timer(LOGTAG, "CreateProgressTracker", "CreateProgressTracker"))
                await database
                    .CreateProgressTracker(false, result.TaskControl.ProgressToken)
                    .ConfigureAwait(false);

        }

        private static async Task CreateDirectoryStructure(LocalRestoreDatabase database, IRestoreDestinationProvider restoreDestination, bool restoreToOriginalLocation, Options options, RestoreResults result)
        {
            // This part is not protected by try/catch as we need the target folder to exist
            if (!string.IsNullOrEmpty(restoreDestination.TargetDestination))
            {
                if (options.Dryrun)
                {
                    Logging.Log.WriteDryrunMessage(LOGTAG, "WouldCreateFolder", "Would create folder: {0}", restoreDestination.TargetDestination);
                }
                else if (await restoreDestination.CreateFolderIfNotExists(restoreDestination.TargetDestination, result.TaskControl.ProgressToken).ConfigureAwait(false))
                {
                    Logging.Log.WriteVerboseMessage(LOGTAG, "CreateFolder", "Created root restore folder: {0}", restoreDestination.TargetDestination);
                    result.RestoredFolders++;
                }
            }

            await foreach (var folder in database.GetTargetFolders(result.TaskControl.ProgressToken).ConfigureAwait(false))
            {
                try
                {
                    if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                        return;

                    if (options.Dryrun)
                        Logging.Log.WriteDryrunMessage(LOGTAG, "WouldCreateFolder", "Would create folder: {0}", folder);
                    else if (await restoreDestination.CreateFolderIfNotExists(folder, result.TaskControl.ProgressToken).ConfigureAwait(false))
                    {
                        Logging.Log.WriteVerboseMessage(LOGTAG, "CreateFolder", "Created folder: {0}", folder);
                        result.RestoredFolders++;
                    }
                }
                catch (Exception ex)
                {
                    Logging.Log.WriteWarningMessage(LOGTAG, "FolderCreateFailed", ex, "Failed to create folder: \"{0}\", message: {1}", folder, ex.Message);
                    if (options.UnittestMode)
                        throw;
                }
            }
        }

        private static async Task ResetReadOnlyAttributeIfNeeded(string path, Options options, IRestoreDestinationProvider restoreDestination, CancellationToken cancellationToken)
        {
            if (await restoreDestination.HasReadOnlyAttribute(path, cancellationToken).ConfigureAwait(false))
            {
                if (options.Dryrun)
                    Logging.Log.WriteDryrunMessage(LOGTAG, "WouldResetReadOnlyAttribute", "Would reset read-only attribute on file: {0}", path);
                else
                    await restoreDestination.ClearReadOnlyAttribute(path, cancellationToken).ConfigureAwait(false);
            }
        }

        private static async Task ScanForExistingTargetBlocks(LocalRestoreDatabase database, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm blockhasher, System.Security.Cryptography.HashAlgorithm filehasher, Options options, IRestoreDestinationProvider restoreDestination, RestoreResults result)
        {
            // Scan existing files for existing BLOCKS
            await using var blockmarker = await database.CreateBlockMarkerAsync(result.TaskControl.ProgressToken).ConfigureAwait(false);
            var updateCount = 0L;
            await foreach (var restorelist in database.GetExistingFilesWithBlocks(result.TaskControl.ProgressToken).ConfigureAwait(false))
            {
                var rename = !options.Overwrite;
                var targetpath = restorelist.TargetPath;
                var targetfileid = restorelist.TargetFileID;
                var targetfilehash = restorelist.TargetHash;
                var targetfilelength = restorelist.Length;
                if (await restoreDestination.FileExists(targetpath, result.TaskControl.ProgressToken).ConfigureAwait(false))
                {
                    try
                    {
                        if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                            return;

                        var currentfilelength = await restoreDestination.GetFileLength(targetpath, result.TaskControl.ProgressToken).ConfigureAwait(false);
                        var wasTruncated = false;

                        // Adjust file length in overwrite mode if necessary (smaller is ok, will be extended during restore)
                        // We do it before scanning for blocks. This allows full verification on files that only needs to
                        // be truncated (i.e. forthwritten log files).
                        if (!rename && currentfilelength > targetfilelength)
                        {
                            await ResetReadOnlyAttributeIfNeeded(targetpath, options, restoreDestination, result.TaskControl.ProgressToken).ConfigureAwait(false);
                            if (options.Dryrun)
                                Logging.Log.WriteDryrunMessage(LOGTAG, "WouldTruncateFile", "Would truncate file '{0}' to length of {1:N0} bytes", targetpath, targetfilelength);
                            else
                            {
                                using (var file = await restoreDestination.OpenWrite(targetpath, result.TaskControl.ProgressToken).ConfigureAwait(false))
                                    file.SetLength(targetfilelength);
                                currentfilelength = targetfilelength;
                            }
                            wasTruncated = true;
                        }

                        // If file size does not match and we have to rename on conflict,
                        // the whole scan can be skipped here because all blocks have to be restored anyway.
                        // For the other cases, we will check block and and file hashes and look for blocks
                        // to be restored and files that can already be verified.
                        if (!rename || currentfilelength == targetfilelength)
                        {
                            // a file hash for verification will only be necessary if the file has exactly
                            // the wanted size so we have a chance to already mark the file as data-verified.
                            bool calcFileHash = (currentfilelength == targetfilelength);
                            if (calcFileHash) filehasher.Initialize();

                            using (var file = await restoreDestination.OpenRead(targetpath, result.TaskControl.ProgressToken).ConfigureAwait(false))
                            using (var block = new Blockprocessor(file, blockbuffer))
                                await foreach (var targetblock in restorelist.Blocks(result.TaskControl.ProgressToken).ConfigureAwait(false))
                                {
                                    var size = block.Readblock();
                                    if (size <= 0)
                                        break;

                                    //TODO: Handle Metadata

                                    bool blockhashmatch = false;
                                    if (size == targetblock.Size)
                                    {
                                        // Parallelize file hash calculation on rename. Running read-only on same array should not cause conflicts or races.
                                        // Actually, in future always calculate the file hash and mark the file data as already verified.

                                        System.Threading.Tasks.Task calcFileHashTask = null;
                                        if (calcFileHash)
                                            calcFileHashTask = System.Threading.Tasks.Task.Run(
                                                () => filehasher.TransformBlock(blockbuffer, 0, size, blockbuffer, 0));

                                        var key = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size));

                                        if (calcFileHashTask != null) await calcFileHashTask.ConfigureAwait(false); // wait because blockbuffer will be overwritten.

                                        if (key == targetblock.Hash)
                                        {
                                            await blockmarker
                                                .SetBlockRestored(targetfileid, targetblock.Index, key, size, false, result.TaskControl.ProgressToken)
                                                .ConfigureAwait(false);
                                            blockhashmatch = true;
                                        }
                                    }
                                    if (calcFileHash && !blockhashmatch) // will not be necessary anymore
                                    {
                                        filehasher.TransformFinalBlock(blockbuffer, 0, 0); // So a new initialize will not throw
                                        calcFileHash = false;
                                        if (rename) // file does not match. So break.
                                            break;
                                    }
                                }

                            bool fullfilehashmatch = false;
                            if (calcFileHash) // now check if files are identical
                            {
                                filehasher.TransformFinalBlock(blockbuffer, 0, 0);
                                var filekey = Convert.ToBase64String(filehasher.Hash);
                                fullfilehashmatch = (filekey == targetfilehash);
                            }

                            if (!rename && !fullfilehashmatch && !wasTruncated) // Reset read-only attribute (if set) to overwrite
                                await ResetReadOnlyAttributeIfNeeded(targetpath, options, restoreDestination, result.TaskControl.ProgressToken).ConfigureAwait(false);

                            if (fullfilehashmatch)
                            {
                                //TODO: Check metadata to trigger rename? If metadata changed, it will still be restored for the file in-place.
                                await blockmarker
                                    .SetFileDataVerified(targetfileid, result.TaskControl.ProgressToken)
                                    .ConfigureAwait(false);
                                Logging.Log.WriteVerboseMessage(LOGTAG, "TargetExistsInCorrectVersion", "Target file exists{1} and is correct version: {0}", targetpath, wasTruncated ? " (but was truncated)" : "");
                                rename = false;
                            }
                            else if (rename)
                            {
                                // The new file will have none of the correct blocks,
                                // even if the scanned file had some
                                await blockmarker
                                    .SetAllBlocksMissing(targetfileid, result.TaskControl.ProgressToken)
                                    .ConfigureAwait(false);
                            }
                        }

                        if ((++updateCount) % 20 == 0)
                        {
                            await blockmarker
                                .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken)
                                .ConfigureAwait(false);
                            if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
                                return;
                        }
                    }
                    catch (Exception ex)
                    {
                        Logging.Log.WriteWarningMessage(LOGTAG, "TargetFileReadError", ex, "Failed to read target file: \"{0}\", message: {1}", targetpath, ex.Message);
                        if (ex.IsAbortException())
                            throw;
                        if (options.UnittestMode)
                            throw;
                    }
                }
                else
                {
                    Logging.Log.WriteVerboseMessage(LOGTAG, "MissingTargetFile", "Target file does not exist: {0}", targetpath);
                    rename = false;
                }

                if (rename)
                {
                    //Select a new filename
                    var ext = SystemIO.IO_OS.PathGetExtension(targetpath) ?? "";
                    if (!string.IsNullOrEmpty(ext) && !ext.StartsWith(".", StringComparison.Ordinal))
                        ext = "." + ext;

                    // First we try with a simple date append, assuming that there are not many conflicts there
                    var newname = SystemIO.IO_OS.PathChangeExtension(targetpath, null) + "." + database.RestoreTime.ToLocalTime().ToString("yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture);
                    var tr = newname + ext;
                    var c = 0;
                    while (await restoreDestination.FileExists(tr, result.TaskControl.ProgressToken).ConfigureAwait(false) && c < 1000)
                    {
                        try
                        {
                            // If we have a file with the correct name,
                            // it is most likely the file we want
                            filehasher.Initialize();

                            string key;
                            using (var file = await restoreDestination.OpenRead(tr, result.TaskControl.ProgressToken).ConfigureAwait(false))
                                key = Convert.ToBase64String(filehasher.ComputeHash(file));

                            if (key == targetfilehash)
                            {
                                //TODO: Also needs metadata check to make correct decision.
                                //      We stick to the policy to restore metadata in place, if data ok. So, metadata block may be restored.
                                await blockmarker
                                    .SetAllBlocksRestored(targetfileid, false, result.TaskControl.ProgressToken)
                                    .ConfigureAwait(false);
                                await blockmarker
                                    .SetFileDataVerified(targetfileid, result.TaskControl.ProgressToken)
                                    .ConfigureAwait(false);
                                break;
                            }
                        }
                        catch (Exception ex)
                        {
                            Logging.Log.WriteWarningMessage(LOGTAG, "FailedToReadRestoreTarget", ex, "Failed to read candidate restore target {0}", tr);
                            if (options.UnittestMode)
                                throw;
                        }
                        tr = newname + " (" + (c++).ToString() + ")" + ext;
                    }

                    newname = tr;

                    Logging.Log.WriteVerboseMessage(LOGTAG, "TargetFileRetargeted", "Target file exists and will be restored to: {0}", newname);
                    await database
                        .UpdateTargetPath(targetfileid, newname, result.TaskControl.ProgressToken)
                        .ConfigureAwait(false);
                }

            }

            await blockmarker
                .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken)
                .ConfigureAwait(false);
            await blockmarker
                .CommitAsync(result.TaskControl.ProgressToken)
                .ConfigureAwait(false);
        }
    }
}
