PublicArchive/duplicati
2
0
Fork 0
mirror of https://github.com/duplicati/duplicati.git synced 2026-05-07 23:59:36 -04:00
Code Issues Packages Projects Releases Wiki Activity

Re-applied the path-storage fix as the merge failed somehow

Browse Source
This commit is contained in:
Kenneth Skovhede
2018-06-14 10:12:24 +02:00
parent 729baf76e6
commit cbc18970d5
19 changed files with 362 additions and 301 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/Database/Database schema/9. Refactor Paths.sql
+63
View File
@@ -0,0 +1,63 @@
/*
The PathPrefix contains a set
of path prefixes, used to minimize
the space required to store paths
*/
CREATE TABLE "PathPrefix" (
"ID" INTEGER PRIMARY KEY,
"Prefix" TEXT NOT NULL
);
CREATE UNIQUE INDEX "PathPrefixPrefix" ON "PathPrefix" ("Prefix");
/*
The FileLookup table contains an ID
for each path and each version
of the data and metadata
*/
CREATE TABLE "FileLookup" (
"ID" INTEGER PRIMARY KEY,
"PrefixID" INTEGER NOT NULL,
"Path" TEXT NOT NULL,
"BlocksetID" INTEGER NOT NULL,
"MetadataID" INTEGER NOT NULL
);
/* Fast path based lookup, single properties are auto-indexed */
CREATE UNIQUE INDEX "FileLookupPath" ON "FileLookup" ("PrefixID", "Path", "BlocksetID", "MetadataID");
/* Build the prefix table */
INSERT INTO "PathPrefix" ("Prefix")
SELECT DISTINCT
CASE SUBSTR("Path", LENGTH("Path")) WHEN '/' THEN
rtrim(SUBSTR("Path", 1, LENGTH("Path")-1), replace(replace(SUBSTR("Path", 1, LENGTH("Path")-1), "\", "/"), '/', ''))
ELSE
rtrim("Path", replace(replace("Path", "\", "/"), '/', ''))
END AS "Prefix"
FROM "File";
/* Build the path lookup table */
INSERT INTO "FileLookup" ("Path", "PrefixID", "BlocksetID", "MetadataID")
SELECT
SUBSTR("Path", LENGTH("ParentFolder") + 1) AS "Path",
"ID" AS "PrefixID",
"BlocksetID",
"MetadataID"
FROM
(SELECT "Path", "BlocksetID", "MetadataID",
CASE SUBSTR("Path", LENGTH("Path")) WHEN '/' THEN
rtrim(SUBSTR("Path", 1, LENGTH("Path")-1), replace(replace(SUBSTR("Path", 1, LENGTH("Path")-1), "\", "/"), '/', ''))
ELSE
rtrim("Path", replace(replace("Path", "\", "/"), '/', ''))
END AS "ParentFolder"
FROM "File") "A" INNER JOIN "PathPrefix" "B" ON "A"."ParentFolder" = "B"."Prefix";
DROP TABLE "File";
CREATE VIEW "File" AS SELECT "A"."ID" AS "ID", "B"."Prefix" || "A"."Path" AS "Path", "A"."BlocksetID" AS "BlocksetID", "A"."MetadataID" AS "MetadataID" FROM "FileLookup" "A", "PathPrefix" "B" WHERE "A"."PrefixID" = "B"."ID";
UPDATE "Version" SET "Version" = 9;
Duplicati/Library/Main/Database/Database schema/Schema.sql
+27 -9
View File
@@ -82,21 +82,39 @@ CREATE TABLE "FilesetEntry" (
CREATE INDEX "FilesetentryFileIdIndex" on "FilesetEntry" ("FileID");
/*
The PathPrefix contains a set
of path prefixes, used to minimize
the space required to store paths
*/
CREATE TABLE "PathPrefix" (
"ID" INTEGER PRIMARY KEY,
"Prefix" TEXT NOT NULL
);
CREATE UNIQUE INDEX "PathPrefixPrefix" ON "PathPrefix" ("Prefix");
/*
The FileEntry contains an ID
The FileLookup table contains an ID
for each path and each version
of the data and metadata
*/
CREATE TABLE "File" (
"ID" INTEGER PRIMARY KEY,
"Path" TEXT NOT NULL,
"BlocksetID" INTEGER NOT NULL,
"MetadataID" INTEGER NOT NULL
CREATE TABLE "FileLookup" (
"ID" INTEGER PRIMARY KEY,
"PrefixID" INTEGER NOT NULL,
"Path" TEXT NOT NULL,
"BlocksetID" INTEGER NOT NULL,
"MetadataID" INTEGER NOT NULL
);
/* Fast path based lookup */
CREATE UNIQUE INDEX "FilePath" ON "File" ("Path", "BlocksetID", "MetadataID");
/* Fast path based lookup, single properties are auto-indexed */
CREATE UNIQUE INDEX "FileLookupPath" ON "FileLookup" ("PrefixID", "Path", "BlocksetID", "MetadataID");
/*
The File view contains an ID
for each path and each version
of the data and metadata
*/
CREATE VIEW "File" AS SELECT "A"."ID" AS "ID", "B"."Prefix" || "A"."Path" AS "Path", "A"."BlocksetID" AS "BlocksetID", "A"."MetadataID" AS "MetadataID" FROM "FileLookup" "A", "PathPrefix" "B" WHERE "A"."PrefixID" = "B"."ID";
/*
The blocklist hashes are hashes of
@@ -244,4 +262,4 @@ CREATE TABLE "Configuration" (
"Value" TEXT NOT NULL
);
INSERT INTO "Version" ("Version") VALUES (7);
INSERT INTO "Version" ("Version") VALUES (9);
Duplicati/Library/Main/Database/LocalBackupDatabase.cs
+85 -183
View File
@@ -91,7 +91,6 @@ namespace Duplicati.Library.Main.Database
private readonly System.Data.IDbCommand m_insertfileOperationCommand;
private PathLookupHelper<PathEntryKeeper> m_pathLookup;
private Dictionary<string, long> m_blockCache;
private long m_filesetId;
@@ -131,8 +130,8 @@ namespace Duplicati.Library.Main.Database
m_findmetadatasetCommand.CommandText = @"SELECT ""A"".""ID"" FROM ""Metadataset"" A, ""BlocksetEntry"" B, ""Block"" C WHERE ""A"".""BlocksetID"" = ""B"".""BlocksetID"" AND ""B"".""BlockID"" = ""C"".""ID"" AND ""C"".""Hash"" = ? AND ""C"".""Size"" = ?";
m_findmetadatasetCommand.AddParameters(2);
m_findfilesetCommand.CommandText = @"SELECT ""ID"" FROM ""File"" WHERE ""BlocksetID"" = ? AND ""MetadataID"" = ? AND ""Path"" = ?";
m_findfilesetCommand.AddParameters(3);
m_findfilesetCommand.CommandText = @"SELECT ""ID"" FROM ""FileLookup"" WHERE ""BlocksetID"" = ? AND ""MetadataID"" = ? AND ""Path"" = ? AND ""PrefixID"" = ?";
m_findfilesetCommand.AddParameters(4);
m_insertblockCommand.CommandText = @"INSERT INTO ""Block"" (""Hash"", ""VolumeID"", ""Size"") VALUES (?, ?, ?); SELECT last_insert_rowid();";
m_insertblockCommand.AddParameters(3);
@@ -140,8 +139,8 @@ namespace Duplicati.Library.Main.Database
m_insertfileOperationCommand.CommandText = @"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (?, ?, ?)";
m_insertfileOperationCommand.AddParameters(3);
m_insertfileCommand.CommandText = @"INSERT INTO ""File"" (""Path"",""BlocksetID"", ""MetadataID"") VALUES (?, ? ,?); SELECT last_insert_rowid();";
m_insertfileCommand.AddParameters(3);
m_insertfileCommand.CommandText = @"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"",""BlocksetID"", ""MetadataID"") VALUES (?, ?, ? ,?); SELECT last_insert_rowid();";
m_insertfileCommand.AddParameters(4);
m_insertblocksetCommand.CommandText = @"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (?, ?); SELECT last_insert_rowid();";
m_insertblocksetCommand.AddParameters(2);
@@ -158,22 +157,22 @@ namespace Duplicati.Library.Main.Database
m_insertmetadatasetCommand.CommandText = @"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (?); SELECT last_insert_rowid();";
m_insertmetadatasetCommand.AddParameter();
m_selectfilelastmodifiedCommand.CommandText = @"SELECT ""A"".""ID"", ""B"".""LastModified"" FROM (SELECT ""ID"" FROM ""File"" WHERE ""Path"" = ?) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = ?";
m_selectfilelastmodifiedCommand.AddParameters(2);
m_selectfilelastmodifiedCommand.CommandText = @"SELECT ""A"".""ID"", ""B"".""LastModified"" FROM (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ?) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = ?";
m_selectfilelastmodifiedCommand.AddParameters(3);
//Need a temporary table with path/lastmodified lookups
m_findfileCommand.CommandText =
@" SELECT ""File"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " +
@" FROM ""File"", ""FilesetEntry"", ""Fileset"", ""Blockset"" ""FileBlockset"", ""Metadataset"", ""Blockset"" ""MetaBlockset"" " +
@" WHERE ""File"".""Path"" = ? " +
@" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" " +
@" AND ""FileBlockset"".""ID"" = ""File"".""BlocksetID"" " +
@" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" " +
m_findfileCommand.CommandText =
@" SELECT ""FileLookup"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " +
@" FROM ""FileLookup"", ""FilesetEntry"", ""Fileset"", ""Blockset"" ""FileBlockset"", ""Metadataset"", ""Blockset"" ""MetaBlockset"" " +
@" WHERE ""FileLookup"".""PrefixID"" = ? AND ""FileLookup"".""Path"" = ? " +
@" AND ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" " +
@" AND ""FileBlockset"".""ID"" = ""FileLookup"".""BlocksetID"" " +
@" AND ""Metadataset"".""ID"" = ""FileLookup"".""MetadataID"" AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" " +
@" ORDER BY ""Fileset"".""Timestamp"" DESC " +
@" LIMIT 1 ";
m_findfileCommand.AddParameters(1);
m_findfileCommand.AddParameters(2);
m_selectfileHashCommand.CommandText = @"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"", ""File"" WHERE ""Blockset"".""ID"" = ""File"".""BlocksetID"" AND ""File"".""ID"" = ? ";
m_selectfileHashCommand.CommandText = @"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"", ""FileLookup"" WHERE ""Blockset"".""ID"" = ""FileLookup"".""BlocksetID"" AND ""FileLookup"".""ID"" = ? ";
m_selectfileHashCommand.AddParameters(1);
m_selectblocklistHashesCommand.CommandText = @"SELECT ""Hash"" FROM ""BlocklistHash"" WHERE ""BlocksetID"" = ? ORDER BY ""Index"" ASC ";
@@ -186,75 +185,6 @@ namespace Duplicati.Library.Main.Database
/// <param name="options">The option settings</param>
public void BuildLookupTable(Options options)
{
if (options.UseFilepathCache)
m_pathLookup = new PathLookupHelper<PathEntryKeeper>(true);
//Populate the lookup tables
using (var cmd = m_connection.CreateCommand())
{
//Need a temporary table with path/lastmodified lookups
var scantableDefinition =
@"SELECT ""A1"".""ID"" AS ""FileID"", ""A1"".""Lastmodified"" AS ""Lastmodified"", ""A1"".""Path"" AS ""Path"", ""C"".""Length"" AS ""Length"", ""F"".""Fullhash"" AS ""Metahash"", ""F"".""Length"" AS ""Metasize"", ""A1"".""BlocksetID"" " +
@" FROM (SELECT ""File"".""ID"", ""File"".""BlocksetID"", ""File"".""MetadataID"", ""FilesetEntry"".""Lastmodified"", ""File"".""Path"", ""Fileset"".""Timestamp"" " +
@" FROM ""FilesetEntry"", ""Fileset"", ""File"" WHERE ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" " +
@" ) ""A1"" LEFT JOIN " +
@" (SELECT ""File"".""Path"", ""Fileset"".""Timestamp"" " +
@" FROM ""FilesetEntry"", ""Fileset"", ""File"" WHERE ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" " +
@" ) ""A2"" ON ""A1"".""Path"" = ""A2"".""Path"" AND ""A1"".""Timestamp"" < ""A2"".""Timestamp"" " +
@" , ""Blockset"" ""C"", ""Metadataset"" ""E"", ""Blockset"" ""F"" " +
@" WHERE ""A2"".""Path"" IS NULL " +
@" AND ""C"".""ID"" = ""A1"".""BlocksetID"" " +
@" AND ""A1"".""MetadataID"" = ""E"".""ID"" " +
@" AND ""F"".""ID"" = ""E"".""BlocksetID"" ";
if (m_pathLookup != null)
using(new Logging.Timer(LOGTAG, "BuildLastModified", "Build path lastmodified lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FileID"", ""Lastmodified"", ""Length"", ""Path"", ""Metahash"", ""Metasize"" FROM ({0}) WHERE ""BlocksetID"" >= 0 ", scantableDefinition)))
while (rd.Read())
{
var id = rd.GetInt64(0);
var lastmodified = new DateTime(rd.GetInt64(1), DateTimeKind.Utc);
var filesize = rd.GetInt64(2);
var path = rd.GetString(3);
var metahash = rd.GetString(4);
var metasize = rd.GetInt64(5);
m_pathLookup.Insert(path, new PathEntryKeeper(id, lastmodified, filesize, metahash, metasize));
}
if (m_pathLookup != null)
try
{
using(new Logging.Timer(LOGTAG, "BuildPathTable", "Build path lookup table"))
using (var rd = cmd.ExecuteReader(@" SELECT ""Path"", ""BlocksetID"", ""MetadataID"", ""ID"" FROM ""File"" "))
while (rd.Read())
{
var path = rd.GetValue(0).ToString();
var blocksetid = rd.GetInt64(1);
var metadataid = rd.GetInt64(2);
var filesetid = rd.GetInt64(3);
PathEntryKeeper r;
if (!m_pathLookup.TryFind(path, out r))
{
r = new PathEntryKeeper(-1, new DateTime(0, DateTimeKind.Utc), -1, null, -1);
r.AddFilesetID(blocksetid, metadataid, filesetid);
m_pathLookup.Insert(path, r);
}
else
r.AddFilesetID(blocksetid, metadataid, filesetid);
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate file entries detected, run repair to fix it", ex);
}
var tc = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Remotevolume"" WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"") AND ""State"" NOT IN (?, ?, ?, ?);", 0, RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString());
if (tc > 0)
throw new InvalidDataException("Detected blocks that are not reachable in the block table");
}
if (options.UseBlockCache)
{
string failedhash = null;
@@ -435,7 +365,10 @@ namespace Duplicati.Library.Main.Database
/// <summary>
/// Adds a metadata set to the database, and returns a value indicating if the record was new
/// </summary>
/// <param name="hash">The metadata hash</param>
/// <param name="filehash">The hash of the metadata</param>
/// <param name="size">The size of the metadata</param>
/// <param name="blocksetid">The ID of the blockset with the data</param>
/// <param name="transaction">The transaction to use</param>
/// <param name="metadataid">The id of the metadata set</param>
/// <returns>True if the set was added to the database, false otherwise</returns>
public bool AddMetadataset(string filehash, long size, long blocksetid, out long metadataid, System.Data.IDbTransaction transaction = null)
@@ -453,6 +386,48 @@ namespace Duplicati.Library.Main.Database
}
}
/// <summary>
/// Adds a file record to the database
/// </summary>
/// <param name="pathprefixid">The path prefix ID</param>
/// <param name="filename">The path to the file</param>
/// <param name="lastmodified">The time the file was modified</param>
/// <param name="blocksetID">The ID of the hashkey for the file</param>
/// <param name="metadataID">The ID for the metadata</param>
/// <param name="transaction">The transaction to use for insertion, or null for no transaction</param>
public void AddFile(long pathprefixid, string filename, DateTime lastmodified, long blocksetID, long metadataID, System.Data.IDbTransaction transaction)
{
var fileidobj = -1L;
m_findfilesetCommand.Transaction = transaction;
m_findfilesetCommand.SetParameterValue(0, blocksetID);
m_findfilesetCommand.SetParameterValue(1, metadataID);
m_findfilesetCommand.SetParameterValue(2, filename);
m_findfilesetCommand.SetParameterValue(3, pathprefixid);
fileidobj = m_findfilesetCommand.ExecuteScalarInt64();
if (fileidobj == -1)
{
using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
{
m_insertfileCommand.Transaction = tr.Parent;
m_insertfileCommand.SetParameterValue(0, pathprefixid);
m_insertfileCommand.SetParameterValue(1, filename);
m_insertfileCommand.SetParameterValue(2, blocksetID);
m_insertfileCommand.SetParameterValue(3, metadataID);
fileidobj = m_insertfileCommand.ExecuteScalarInt64();
tr.Commit();
}
}
m_insertfileOperationCommand.Transaction = transaction;
m_insertfileOperationCommand.SetParameterValue(0, m_filesetId);
m_insertfileOperationCommand.SetParameterValue(1, fileidobj);
m_insertfileOperationCommand.SetParameterValue(2, lastmodified.ToUniversalTime().Ticks);
m_insertfileOperationCommand.ExecuteNonQuery();
}
/// <summary>
/// Adds a file record to the database
/// </summary>
@@ -462,61 +437,9 @@ namespace Duplicati.Library.Main.Database
/// <param name="metadataID">The ID for the metadata</param>
/// <param name="transaction">The transaction to use for insertion, or null for no transaction</param>
public void AddFile(string filename, DateTime lastmodified, long blocksetID, long metadataID, System.Data.IDbTransaction transaction)
{
var fileidobj = -1L;
PathEntryKeeper entry = null;
var entryFound = false;
if (m_pathLookup != null)
{
if (entryFound = (m_pathLookup.TryFind(filename, out entry) && entry != null))
{
var fid = entry.GetFilesetID(blocksetID, metadataID);
if (fid >= 0)
fileidobj = fid;
}
}
else
{
m_findfilesetCommand.Transaction = transaction;
m_findfilesetCommand.SetParameterValue(0, blocksetID);
m_findfilesetCommand.SetParameterValue(1, metadataID);
m_findfilesetCommand.SetParameterValue(2, filename);
fileidobj = m_findfilesetCommand.ExecuteScalarInt64();
}
if (fileidobj == -1)
{
using(var tr = new TemporaryTransactionWrapper(m_connection, transaction))
{
m_insertfileCommand.Transaction = tr.Parent;
m_insertfileCommand.SetParameterValue(0, filename);
m_insertfileCommand.SetParameterValue(1, blocksetID);
m_insertfileCommand.SetParameterValue(2, metadataID);
fileidobj = m_insertfileCommand.ExecuteScalarInt64();
tr.Commit();
// We do not need to update this, because we will not ask for the same file twice
if (m_pathLookup != null)
{
if (!entryFound)
{
entry = new PathEntryKeeper(-1, new DateTime(0, DateTimeKind.Utc), -1, null, -1);
entry.AddFilesetID(blocksetID, metadataID, fileidobj);
m_pathLookup.Insert(filename, entry);
}
else
entry.AddFilesetID(blocksetID, metadataID, fileidobj);
}
}
}
m_insertfileOperationCommand.Transaction = transaction;
m_insertfileOperationCommand.SetParameterValue(0, m_filesetId);
m_insertfileOperationCommand.SetParameterValue(1, fileidobj);
m_insertfileOperationCommand.SetParameterValue(2, lastmodified.ToUniversalTime().Ticks);
m_insertfileOperationCommand.ExecuteNonQuery();
{
var split = SplitIntoPrefixAndName(filename);
AddFile(GetOrCreatePathPrefix(split.Key, transaction), split.Value, lastmodified, blocksetID, metadataID, transaction);
}
public void AddUnmodifiedFile(long fileid, DateTime lastmodified, System.Data.IDbTransaction transaction = null)
@@ -538,11 +461,12 @@ namespace Duplicati.Library.Main.Database
AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID, transaction);
}
public long GetFileLastModified(string path, long filesetid, out DateTime oldModified, System.Data.IDbTransaction transaction = null)
public long GetFileLastModified(long prefixid, string path, long filesetid, out DateTime oldModified, System.Data.IDbTransaction transaction = null)
{
m_selectfileHashCommand.Transaction = transaction;
m_selectfilelastmodifiedCommand.SetParameterValue(0, path);
m_selectfilelastmodifiedCommand.SetParameterValue(1, filesetid);
m_selectfilelastmodifiedCommand.Transaction = transaction;
m_selectfilelastmodifiedCommand.SetParameterValue(0, prefixid);
m_selectfilelastmodifiedCommand.SetParameterValue(1, path);
m_selectfilelastmodifiedCommand.SetParameterValue(2, filesetid);
using (var rd = m_selectfilelastmodifiedCommand.ExecuteReader())
if (rd.Read())
{
@@ -554,18 +478,19 @@ namespace Duplicati.Library.Main.Database
return -1;
}
public long GetFileEntry(string path, long filesetid, out DateTime oldModified, out long lastFileSize, out string oldMetahash, out long oldMetasize)
public long GetFileEntry(long prefixid, string path, long lastfilesetid, out DateTime oldModified, out long lastFileSize, out string oldMetahash, out long oldMetasize)
{
if (m_pathLookup != null)
{
PathEntryKeeper tmp;
if (m_pathLookup.TryFind(path, out tmp) && tmp != null && tmp.FileID >= 0)
m_findfileCommand.SetParameterValue(0, prefixid);
m_findfileCommand.SetParameterValue(1, path);
using (var rd = m_findfileCommand.ExecuteReader())
if (rd.Read())
{
oldModified = tmp.Lastmodified;
lastFileSize = tmp.Filesize;
oldMetahash = tmp.Metahash;
oldMetasize = tmp.Metasize;
return tmp.FileID;
oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc);
lastFileSize = rd.GetInt64(2);
oldMetahash = rd.GetString(3);
oldMetasize = rd.GetInt64(4);
return rd.ConvertValueToInt64(0);
}
else
{
@@ -575,31 +500,9 @@ namespace Duplicati.Library.Main.Database
oldMetasize = -1;
return -1;
}
}
else
{
m_findfileCommand.SetParameterValue(0, path);
using(var rd = m_findfileCommand.ExecuteReader())
if (rd.Read())
{
oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc);
lastFileSize = rd.GetInt64(2);
oldMetahash = rd.GetString(3);
oldMetasize = rd.GetInt64(4);
return rd.ConvertValueToInt64(0);
}
else
{
oldModified = new DateTime(0, DateTimeKind.Utc);
lastFileSize = -1;
oldMetahash = null;
oldMetasize = -1;
return -1;
}
}
}
public string GetFileHash(long fileid)
{
m_selectfileHashCommand.SetParameterValue(0, fileid);
@@ -612,8 +515,6 @@ namespace Duplicati.Library.Main.Database
public override void Dispose ()
{
m_pathLookup = null;
base.Dispose();
}
@@ -633,8 +534,8 @@ namespace Duplicati.Library.Main.Database
using (var cmd = m_connection.CreateCommand())
{
var lastFilesetId = cmd.ExecuteScalarInt64(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC LIMIT 1");
var count = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" NOT IN (?, ?)", -1, lastFilesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);
var size = cmd.ExecuteScalarInt64(@"SELECT SUM(""Blockset"".""Length"") FROM ""File"", ""FilesetEntry"", ""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" NOT IN (?, ?)", -1, lastFilesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);
var count = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FileLookup"" INNER JOIN ""FilesetEntry"" ON ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""FileLookup"".""BlocksetID"" NOT IN (?, ?)", -1, lastFilesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);
var size = cmd.ExecuteScalarInt64(@"SELECT SUM(""Blockset"".""Length"") FROM ""FileLookup"", ""FilesetEntry"", ""Blockset"" WHERE ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""FileLookup"".""BlocksetID"" NOT IN (?, ?)", -1, lastFilesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);
return new Tuple<long, long>(count, size);
}
@@ -644,6 +545,7 @@ namespace Duplicati.Library.Main.Database
{
using(var cmd = m_connection.CreateCommand(transaction))
{
// TODO: Optimize these queries to not use the "File" view
var lastFilesetId = GetPreviousFilesetID(cmd);
results.AddedFolders = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" = ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", 0, m_filesetId, FOLDER_BLOCKSET_ID, lastFilesetId);
results.AddedSymlinks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" = ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", 0, m_filesetId, SYMLINK_BLOCKSET_ID, lastFilesetId);
@@ -659,7 +561,7 @@ namespace Duplicati.Library.Main.Database
var tmpName2 = "TmpFileList-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
try
{
var subqueryFiles = @"SELECT ""File"".""Path"", ""A"".""Fullhash"" AS ""Filehash"", ""B"".""Fullhash"" AS ""Metahash"" FROM ""File"", ""FilesetEntry"", ""Blockset"" A, ""Blockset"" B, ""Metadataset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""A"".""ID"" = ""File"".""BlocksetID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""B"".""ID"" ";
var subqueryFiles = @"SELECT ""File"".""Path"" AS ""Path"", ""A"".""Fullhash"" AS ""Filehash"", ""B"".""Fullhash"" AS ""Metahash"" FROM ""File"", ""FilesetEntry"", ""Blockset"" A, ""Blockset"" B, ""Metadataset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""A"".""ID"" = ""File"".""BlocksetID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""B"".""ID"" ";
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS " + subqueryFiles, tmpName1), lastFilesetId);
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS " + subqueryFiles, tmpName2), m_filesetId);
Duplicati/Library/Main/Database/LocalBugReportDatabase.cs
+8 -3
View File
@@ -39,7 +39,9 @@ namespace Duplicati.Library.Main.Database
{
cmd.Transaction = tr;
var tablename = "PathMap-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
// TODO: Rewrite this to use PathPrefix
// TODO: Needs to be much faster
using(var upcmd = m_connection.CreateCommand())
{
@@ -60,8 +62,11 @@ namespace Duplicati.Library.Main.Database
cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Message"" = ""ERASED!"" WHERE ""Message"" LIKE ""%/%"" OR ""Message"" LIKE ""%:\%"" ");
cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Exception"" = ""ERASED!"" WHERE ""Exception"" LIKE ""%/%"" OR ""Exception"" LIKE ""%:\%"" ");
cmd.ExecuteNonQuery(string.Format(@"UPDATE ""File"" SET ""Path"" = (SELECT ""Obfuscated"" FROM ""{0}"" WHERE ""Path"" = ""RealPath"") ", tablename));
cmd.ExecuteNonQuery(string.Format(@"CREATE TABLE ""FixedFile"" AS SELECT ""B"".""ID"" AS ""ID"", ""A"".""Obfuscated"" AS ""Path"", ""B"".""BlocksetID"" AS ""BlocksetID"", ""B"".""MetadataID"" AS ""MetadataID"" FROM ""{0}"" ""A"", ""File"" ""B"" WHERE ""A"".""RealPath"" = ""B"".""Path"") ", tablename));
cmd.ExecuteNonQuery(@"DROP TABLE ""File"" ");
cmd.ExecuteNonQuery(@"DROP TABLE ""PathPrefix"" ");
cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", tablename));
using(new Logging.Timer(LOGTAG, "CommitUpdateBugReport", "CommitUpdateBugReport"))
Duplicati/Library/Main/Database/LocalDatabase.cs
+83 -4
View File
@@ -28,6 +28,9 @@ namespace Duplicati.Library.Main.Database
private readonly System.Data.IDbCommand m_insertremotelogCommand;
private readonly System.Data.IDbCommand m_insertIndexBlockLink;
private readonly System.Data.IDbCommand m_findpathprefixCommand;
private readonly System.Data.IDbCommand m_insertpathprefixCommand;
protected BasicResults m_result;
public const long FOLDER_BLOCKSET_ID = -100;
@@ -106,6 +109,8 @@ namespace Duplicati.Library.Main.Database
m_selectremotevolumeIdCommand = connection.CreateCommand();
m_createremotevolumeCommand = connection.CreateCommand();
m_insertIndexBlockLink = connection.CreateCommand();
m_findpathprefixCommand = connection.CreateCommand();
m_insertpathprefixCommand = connection.CreateCommand();
m_insertlogCommand.CommandText = @"INSERT INTO ""LogData"" (""OperationID"", ""Timestamp"", ""Type"", ""Message"", ""Exception"") VALUES (?, ?, ?, ?, ?)";
m_insertlogCommand.AddParameters(5);
@@ -133,6 +138,12 @@ namespace Duplicati.Library.Main.Database
m_insertIndexBlockLink.CommandText = @"INSERT INTO ""IndexBlockLink"" (""IndexVolumeID"", ""BlockVolumeID"") VALUES (?, ?)";
m_insertIndexBlockLink.AddParameters(2);
m_findpathprefixCommand.CommandText = @"SELECT ""ID"" FROM ""PathPrefix"" WHERE ""Prefix"" = ?";
m_findpathprefixCommand.AddParameter();
m_insertpathprefixCommand.CommandText = @"INSERT INTO ""PathPrefix"" (""Prefix"") VALUES (?); SELECT last_insert_rowid(); ";
m_insertpathprefixCommand.AddParameter();
}
internal void SetResult(BasicResults result)
@@ -418,7 +429,7 @@ namespace Duplicati.Library.Main.Database
bsIdsSubQuery = string.Format(@"SELECT ""ID"" FROM ""{0}"" ", blocksetidstable);
deletecmd.Parameters.Clear();
deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""File"" WHERE ""BlocksetID"" IN ({0}) OR ""MetadataID"" IN ({0})", bsIdsSubQuery));
deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""FileLookup"" WHERE ""BlocksetID"" IN ({0}) OR ""MetadataID"" IN ({0})", bsIdsSubQuery));
deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" IN ({0})", bsIdsSubQuery));
deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""Blockset"" WHERE ""ID"" IN ({0})", bsIdsSubQuery));
deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ({0})", bsIdsSubQuery));
@@ -799,7 +810,7 @@ ON
throw new Exception("Detected non-empty blocksets with no associated blocks!");
}
if (cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" WHERE ""BlocksetID"" != ? AND ""BlocksetID"" != ? AND NOT ""BlocksetID"" IN (SELECT ""ID"" FROM ""Blockset"")", 0, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID) != 0)
if (cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FileLookup"" WHERE ""BlocksetID"" != ? AND ""BlocksetID"" != ? AND NOT ""BlocksetID"" IN (SELECT ""ID"" FROM ""Blockset"")", 0, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID) != 0)
throw new Exception("Detected files associated with non-existing blocksets!");
if (verifyfilelists)
@@ -807,9 +818,9 @@ ON
using(var cmd2 = m_connection.CreateCommand(transaction))
foreach(var filesetid in cmd.ExecuteReaderEnumerable(@"SELECT ""ID"" FROM ""Fileset"" ").Select(x => x.ConvertValueToInt64(0, -1)))
{
var expandedCmd = string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS);
var expandedCmd = string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS);
var expandedlist = cmd2.ExecuteScalarInt64(expandedCmd, 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, filesetid);
//var storedfilelist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""File"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" != ? AND ""File"".""BlocksetID"" != ?"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);
//var storedfilelist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""FileLookup"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" != ? AND ""FileLookup"".""BlocksetID"" != ?"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);
var storedlist = cmd2.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FilesetID"" = ?", 0, filesetid);
if (expandedlist != storedlist)
@@ -1160,6 +1171,8 @@ ORDER BY
{
using(var cmd = m_connection.CreateCommand())
{
// TODO: Optimize this to not rely on the "File" view, and not instantiate the paths in full
cmd.Transaction = transaction;
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL)", Tablename));
using(var tr = new TemporaryTransactionWrapper(m_connection, transaction))
@@ -1417,5 +1430,71 @@ ORDER BY
);
}
}
/// <summary>
/// The current index into the path prefix buffer
/// </summary>
private int m_pathPrefixIndex = 0;
/// <summary>
/// The path prefix lookup list
/// </summary>
private readonly KeyValuePair<string, long>[] m_pathPrefixLookup = new KeyValuePair<string, long>[5];
/// <summary>
/// Gets the path prefix ID, optionally creating it in the process.
/// </summary>
/// <returns>The path prefix ID.</returns>
/// <param name="prefix">The path to get the prefix for.</param>
/// <param name="transaction">The transaction to use for insertion, or null for no transaction</param>
public long GetOrCreatePathPrefix(string prefix, System.Data.IDbTransaction transaction)
{
// Ring-buffer style lookup
for (var i = 0; i < m_pathPrefixLookup.Length; i++)
{
var ix = (i + m_pathPrefixIndex) % m_pathPrefixLookup.Length;
if (string.Equals(m_pathPrefixLookup[ix].Key, prefix, StringComparison.Ordinal))
return m_pathPrefixLookup[ix].Value;
}
m_findpathprefixCommand.Transaction = transaction;
m_findpathprefixCommand.SetParameterValue(0, prefix);
var id = m_findpathprefixCommand.ExecuteScalarInt64();
if (id < 0)
{
m_insertpathprefixCommand.Transaction = transaction;
m_insertpathprefixCommand.SetParameterValue(0, prefix);
id = m_insertpathprefixCommand.ExecuteScalarInt64();
}
m_pathPrefixIndex = (m_pathPrefixIndex + 1) % m_pathPrefixLookup.Length;
m_pathPrefixLookup[m_pathPrefixIndex] = new KeyValuePair<string, long>(prefix, id);
return id;
}
/// <summary>
/// The path separators on this system
/// </summary>
private static readonly char[] _pathseparators = new char[] {
System.IO.Path.DirectorySeparatorChar,
System.IO.Path.AltDirectorySeparatorChar,
};
/// <summary>
/// Helper method that splits a path on the last path separator
/// </summary>
/// <returns>The prefix and name.</returns>
/// <param name="path">The path to split.</param>
public static KeyValuePair<string, string> SplitIntoPrefixAndName(string path)
{
if (path == null || path.Length == 0)
throw new ArgumentException($"Invalid path: {path}", nameof(path));
int nLast = path.TrimEnd(_pathseparators).LastIndexOfAny(_pathseparators);
if (nLast >= 0)
return new KeyValuePair<string, string>(path.Substring(0, nLast + 1), path.Substring(nLast + 1));
throw new ArgumentException($"Invalid path: {path}", nameof(path));
}
}
}
Duplicati/Library/Main/Database/LocalDeleteDatabase.cs
+5 -5
View File
@@ -80,9 +80,9 @@ namespace Duplicati.Library.Main.Database
//Then we delete anything that is no longer being referenced
cmd.ExecuteNonQuery(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Fileset"")");
cmd.ExecuteNonQuery(@"DELETE FROM ""ChangeJournalData"" WHERE ""FilesetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Fileset"")");
cmd.ExecuteNonQuery(@"DELETE FROM ""File"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"") ");
cmd.ExecuteNonQuery(@"DELETE FROM ""Metadataset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""MetadataID"" FROM ""File"") ");
cmd.ExecuteNonQuery(@"DELETE FROM ""Blockset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlocksetID"" FROM ""File"" UNION SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"") ");
cmd.ExecuteNonQuery(@"DELETE FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"") ");
cmd.ExecuteNonQuery(@"DELETE FROM ""Metadataset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""MetadataID"" FROM ""FileLookup"") ");
cmd.ExecuteNonQuery(@"DELETE FROM ""Blockset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlocksetID"" FROM ""FileLookup"" UNION SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"") ");
cmd.ExecuteNonQuery(@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Blockset"") ");
cmd.ExecuteNonQuery(@"DELETE FROM ""BlocklistHash"" WHERE ""BlocksetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Blockset"") ");
@@ -129,8 +129,8 @@ namespace Duplicati.Library.Main.Database
var tmptablename = "UsageReport-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
var usedBlocks = @"SELECT SUM(""Block"".""Size"") AS ""ActiveSize"", ""Block"".""VolumeID"" AS ""VolumeID"" FROM ""Block"", ""Remotevolume"" WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Block"".""ID"" NOT IN (SELECT ""Block"".""ID"" FROM ""Block"",""DeletedBlock"" WHERE ""Block"".""Hash"" = ""DeletedBlock"".""Hash"" AND ""Block"".""Size"" = ""DeletedBlock"".""Size"") GROUP BY ""Block"".""VolumeID"" ";
var lastmodifiedFile = @"SELECT ""Block"".""VolumeID"" AS ""VolumeID"", ""Fileset"".""Timestamp"" AS ""Sorttime"" FROM ""Fileset"", ""FilesetEntry"", ""File"", ""BlocksetEntry"", ""Block"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" ";
var lastmodifiedMetadata = @"SELECT ""Block"".""VolumeID"" AS ""VolumeID"", ""Fileset"".""Timestamp"" AS ""Sorttime"" FROM ""Fileset"", ""FilesetEntry"", ""File"", ""BlocksetEntry"", ""Block"", ""Metadataset"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""File"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" ";
var lastmodifiedFile = @"SELECT ""Block"".""VolumeID"" AS ""VolumeID"", ""Fileset"".""Timestamp"" AS ""Sorttime"" FROM ""Fileset"", ""FilesetEntry"", ""FileLookup"", ""BlocksetEntry"", ""Block"" WHERE ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" ";
var lastmodifiedMetadata = @"SELECT ""Block"".""VolumeID"" AS ""VolumeID"", ""Fileset"".""Timestamp"" AS ""Sorttime"" FROM ""Fileset"", ""FilesetEntry"", ""FileLookup"", ""BlocksetEntry"", ""Block"", ""Metadataset"" WHERE ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" AND ""FileLookup"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" ";
var scantime = @"SELECT ""VolumeID"" AS ""VolumeID"", MIN(""Sorttime"") AS ""Sorttime"" FROM (" + lastmodifiedFile + @" UNION " + lastmodifiedMetadata + @") GROUP BY ""VolumeID"" ";
var active = @"SELECT ""A"".""ActiveSize"" AS ""ActiveSize"", 0 AS ""InactiveSize"", ""A"".""VolumeID"" AS ""VolumeID"", CASE WHEN ""B"".""Sorttime"" IS NULL THEN 0 ELSE ""B"".""Sorttime"" END AS ""Sorttime"" FROM (" + usedBlocks + @") A LEFT OUTER JOIN (" + scantime + @") B ON ""B"".""VolumeID"" = ""A"".""VolumeID"" ";
Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs
+3 -3
View File
@@ -63,7 +63,7 @@ namespace Duplicati.Library.Main.Database
var sql = string.Format(
@"SELECT DISTINCT ""FilesetID"" FROM (" +
@"SELECT ""FilesetID"" FROM ""FilesetEntry"" WHERE ""FileID"" IN ( SELECT ""ID"" FROM ""File"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN ( SELECT ""ID"" From ""Block"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0})))))" +
@"SELECT ""FilesetID"" FROM ""FilesetEntry"" WHERE ""FileID"" IN ( SELECT ""ID"" FROM ""FileLookup"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN ( SELECT ""ID"" From ""Block"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0})))))" +
" UNION " +
@"SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0}))" +
")",
@@ -137,9 +137,9 @@ namespace Duplicati.Library.Main.Database
{
var sql = string.Format(
@"SELECT DISTINCT ""Name"" FROM ( " +
@" SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""File"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0}))))))) " +
@" SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0}))))))) " +
@" UNION " +
@" SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""Metadataset"" WHERE ""ID"" IN ( SELECT ""MetadataID"" FROM ""File"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0}))))))))" +
@" SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""Metadataset"" WHERE ""ID"" IN ( SELECT ""MetadataID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0}))))))))" +
@")",
string.Join(",", items.Select(x => "?"))
);
Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs
+2 -2
View File
@@ -24,9 +24,9 @@ namespace Duplicati.Library.Main.Database
{
private const string BROKEN_FILE_IDS = @"
SELECT DISTINCT ""ID"" FROM (
SELECT ""ID"" AS ""ID"", ""BlocksetID"" AS ""BlocksetID"" FROM ""File"" WHERE ""BlocksetID"" != {0} AND ""BlocksetID"" != {1}
SELECT ""ID"" AS ""ID"", ""BlocksetID"" AS ""BlocksetID"" FROM ""FileLookup"" WHERE ""BlocksetID"" != {0} AND ""BlocksetID"" != {1}
UNION
SELECT ""A"".""ID"" AS ""ID"", ""B"".""BlocksetID"" AS ""BlocksetID"" FROM ""File"" A LEFT JOIN ""Metadataset"" B ON ""A"".""MetadataID"" = ""B"".""ID""
SELECT ""A"".""ID"" AS ""ID"", ""B"".""BlocksetID"" AS ""BlocksetID"" FROM ""FileLookup"" A LEFT JOIN ""Metadataset"" B ON ""A"".""MetadataID"" = ""B"".""ID""
)
WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN
(
Duplicati/Library/Main/Database/LocalPurgeDatabase.cs
+2 -2
View File
@@ -50,7 +50,7 @@ namespace Duplicati.Library.Main.Database
internal long CountOrphanFiles(System.Data.IDbTransaction transaction)
{
using (var cmd = m_connection.CreateCommand(transaction))
using (var rd = cmd.ExecuteReader(@"SELECT COUNT(*) FROM ""File"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"")"))
using (var rd = cmd.ExecuteReader(@"SELECT COUNT(*) FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"")"))
if (rd.Read())
return rd.ConvertValueToInt64(0, 0);
else
@@ -156,7 +156,7 @@ namespace Duplicati.Library.Main.Database
using (var cmd = m_connection.CreateCommand(m_transaction))
{
RemovedFileCount = cmd.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""{0}""", m_tablename), 0);
RemovedFileSize = cmd.ExecuteScalarInt64(string.Format(@"SELECT SUM(""C"".""Length"") FROM ""{0}"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" ", m_tablename), 0);
RemovedFileSize = cmd.ExecuteScalarInt64(string.Format(@"SELECT SUM(""C"".""Length"") FROM ""{0}"" A, ""FileLookup"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" ", m_tablename), 0);
var filesetcount = cmd.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = " + ParentID), 0);
if (filesetcount == RemovedFileCount)
throw new Duplicati.Library.Interface.UserInformationException(string.Format("Refusing to purge {0} files from fileset with ID {1}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", RemovedFileCount, ParentID), "PurgeWouldRemoveEntireFileset");
Duplicati/Library/Main/Database/LocalRecreateDatabase.cs
+23 -47
View File
@@ -65,8 +65,6 @@ namespace Duplicati.Library.Main.Database
private readonly System.Data.IDbCommand m_insertBlockCommand;
private readonly System.Data.IDbCommand m_insertDuplicateBlockCommand;
private readonly PathLookupHelper<PathEntryKeeper> m_filesetLookup;
private string m_tempblocklist;
private string m_tempsmalllist;
@@ -152,9 +150,9 @@ namespace Duplicati.Library.Main.Database
m_findHashBlockCommand = m_connection.CreateCommand();
m_insertBlockCommand = m_connection.CreateCommand();
m_insertDuplicateBlockCommand = m_connection.CreateCommand();
m_insertFileCommand.CommandText = @"INSERT INTO ""File"" (""Path"", ""BlocksetID"", ""MetadataID"") VALUES (?,?,?); SELECT last_insert_rowid();";
m_insertFileCommand.AddParameters(3);
m_insertFileCommand.CommandText = @"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") VALUES (?,?,?,?); SELECT last_insert_rowid();";
m_insertFileCommand.AddParameters(4);
m_insertFilesetEntryCommand.CommandText = @"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (?,?,?)";
m_insertFilesetEntryCommand.AddParameters(3);
@@ -183,8 +181,8 @@ namespace Duplicati.Library.Main.Database
m_findMetadatasetCommand.CommandText = @"SELECT ""Metadataset"".""ID"" FROM ""Metadataset"",""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Blockset"".""FullHash"" = ? AND ""Blockset"".""Length"" = ? ";
m_findMetadatasetCommand.AddParameters(2);
m_findFilesetCommand.CommandText = @"SELECT ""ID"" FROM ""File"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? ";
m_findFilesetCommand.AddParameters(3);
m_findFilesetCommand.CommandText = @"SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? ";
m_findFilesetCommand.AddParameters(4);
m_findblocklisthashCommand.CommandText = string.Format(@"SELECT DISTINCT ""BlockListHash"" FROM ""{0}"" WHERE ""BlockListHash"" = ? ", m_tempblocklist);
m_findblocklisthashCommand.AddParameters(1);
@@ -197,9 +195,6 @@ namespace Duplicati.Library.Main.Database
m_insertDuplicateBlockCommand.CommandText = @"INSERT INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") VALUES ((SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?), ?)";
m_insertDuplicateBlockCommand.AddParameters(3);
if (options.UseFilepathCache)
m_filesetLookup = new PathLookupHelper<PathEntryKeeper>();
}
public void FindMissingBlocklistHashes(long hashsize, long blocksize, System.Data.IDbTransaction transaction)
@@ -292,59 +287,40 @@ namespace Duplicati.Library.Main.Database
}
}
public void AddDirectoryEntry(long filesetid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction)
public void AddDirectoryEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction)
{
AddEntry(FilelistEntryType.Folder, filesetid, path, time, FOLDER_BLOCKSET_ID, metadataid, transaction);
AddEntry(FilelistEntryType.Folder, filesetid, pathprefixid, path, time, FOLDER_BLOCKSET_ID, metadataid, transaction);
}
public void AddSymlinkEntry(long filesetid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction)
public void AddSymlinkEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction)
{
AddEntry(FilelistEntryType.Symlink, filesetid, path, time, SYMLINK_BLOCKSET_ID, metadataid, transaction);
AddEntry(FilelistEntryType.Symlink, filesetid, pathprefixid, path, time, SYMLINK_BLOCKSET_ID, metadataid, transaction);
}
public void AddFileEntry(long filesetid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction)
public void AddFileEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction)
{
AddEntry(FilelistEntryType.File , filesetid, path, time, blocksetid, metadataid, transaction);
AddEntry(FilelistEntryType.File, filesetid, pathprefixid, path, time, blocksetid, metadataid, transaction);
}
private void AddEntry(FilelistEntryType type, long filesetid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction)
private void AddEntry(FilelistEntryType type, long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction)
{
var fileid = -1L;
if (m_filesetLookup != null)
{
PathEntryKeeper e;
if (m_filesetLookup.TryFind(path, out e))
fileid = e.GetFilesetID(blocksetid, metadataid);
}
else
{
m_findFilesetCommand.Transaction = transaction;
m_findFilesetCommand.SetParameterValue(0, path);
m_findFilesetCommand.SetParameterValue(1, blocksetid);
m_findFilesetCommand.SetParameterValue(2, metadataid);
fileid = m_findFilesetCommand.ExecuteScalarInt64(-1);
}
m_findFilesetCommand.Transaction = transaction;
m_findFilesetCommand.SetParameterValue(0, pathprefixid);
m_findFilesetCommand.SetParameterValue(1, path);
m_findFilesetCommand.SetParameterValue(2, blocksetid);
m_findFilesetCommand.SetParameterValue(3, metadataid);
fileid = m_findFilesetCommand.ExecuteScalarInt64(-1);
if (fileid < 0)
{
m_insertFileCommand.Transaction = transaction;
m_insertFileCommand.SetParameterValue(0, path);
m_insertFileCommand.SetParameterValue(1, blocksetid);
m_insertFileCommand.SetParameterValue(2, metadataid);
m_insertFileCommand.SetParameterValue(0, pathprefixid);
m_insertFileCommand.SetParameterValue(1, path);
m_insertFileCommand.SetParameterValue(2, blocksetid);
m_insertFileCommand.SetParameterValue(3, metadataid);
fileid = m_insertFileCommand.ExecuteScalarInt64(-1);
if (m_filesetLookup != null)
{
PathEntryKeeper e;
if (m_filesetLookup.TryFind(path, out e))
e.AddFilesetID(blocksetid, metadataid, fileid);
else
{
e = new PathEntryKeeper();
e.AddFilesetID(blocksetid, metadataid, fileid);
m_filesetLookup.Insert(path, e);
}
}
}
m_insertFilesetEntryCommand.Transaction = transaction;
Duplicati/Library/Main/Database/LocalRepairDatabase.cs
+10 -10
View File
@@ -196,8 +196,8 @@ namespace Duplicati.Library.Main.Database
public IEnumerable<IRemoteVolume> GetFilesetsUsingMissingBlocks()
{
var blocks = @"SELECT DISTINCT ""File"".""ID"" AS ID FROM ""{0}"", ""Block"", ""Blockset"", ""BlocksetEntry"", ""File"" WHERE ""Block"".""Hash"" = ""{0}"".""Hash"" AND ""Block"".""Size"" = ""{0}"".""Size"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""File"".""BlocksetID"" = ""Blockset"".""ID"" ";
var blocklists = @"SELECT DISTINCT ""File"".""ID"" AS ID FROM ""{0}"", ""Block"", ""Blockset"", ""BlocklistHash"", ""File"" WHERE ""Block"".""Hash"" = ""{0}"".""Hash"" AND ""Block"".""Size"" = ""{0}"".""Size"" AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" AND ""File"".""BlocksetID"" = ""Blockset"".""ID"" ";
var blocks = @"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{0}"", ""Block"", ""Blockset"", ""BlocksetEntry"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{0}"".""Hash"" AND ""Block"".""Size"" = ""{0}"".""Size"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" ";
var blocklists = @"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{0}"", ""Block"", ""Blockset"", ""BlocklistHash"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{0}"".""Hash"" AND ""Block"".""Size"" = ""{0}"".""Size"" AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" ";
var cmdtxt = @"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""FilesetEntry"", ""Fileset"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""RemoteVolume"".""Type"" = ? AND ""FilesetEntry"".""FileID"" IN (SELECT DISTINCT ""ID"" FROM ( " + blocks + " UNION " + blocklists + " ))";
@@ -285,9 +285,9 @@ namespace Duplicati.Library.Main.Database
c2.ExecuteNonQuery(null, rd.GetValue(0), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(0));
}
cmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""File"" WHERE ""ID"" NOT IN (SELECT ""ID"" FROM ""{0}"") ", tablename));
cmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT ""ID"" FROM ""{0}"") ", tablename));
cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}-Ix"" ON ""{0}"" (""ID"", ""MetadataID"")", tablename));
cmd.ExecuteNonQuery(string.Format(@"UPDATE ""File"" SET ""MetadataID"" = (SELECT ""MetadataID"" FROM ""{0}"" A WHERE ""A"".""ID"" = ""File"".""ID"") ", tablename));
cmd.ExecuteNonQuery(string.Format(@"UPDATE ""FileLookup"" SET ""MetadataID"" = (SELECT ""MetadataID"" FROM ""{0}"" A WHERE ""A"".""ID"" = ""FileLookup"".""ID"") ", tablename));
cmd.ExecuteNonQuery(string.Format(@"DROP TABLE ""{0}"" ", tablename));
cmd.CommandText = sql_count;
@@ -306,23 +306,23 @@ namespace Duplicati.Library.Main.Database
using(var tr = m_connection.BeginTransaction())
using(var cmd = m_connection.CreateCommand(tr))
{
var sql_count = @"SELECT COUNT(*) FROM (SELECT ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Duplicates"" FROM ""File"" GROUP BY ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Duplicates"" > 1";
var sql_count = @"SELECT COUNT(*) FROM (SELECT ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Duplicates"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Duplicates"" > 1";
var x = cmd.ExecuteScalarInt64(sql_count, 0);
if (x > 0)
{
Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntries", "Found duplicate file entries, repairing");
var sql = @"SELECT ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM (
SELECT MIN(""ID"") AS ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""File"" GROUP BY ""Path"", ""BlocksetID"", ""MetadataID"")
var sql = @"SELECT ""ID"", ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM (
SELECT MIN(""ID"") AS ""ID"", ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"")
WHERE ""Entries"" > 1 ORDER BY ""ID""";
using(var c2 = m_connection.CreateCommand(tr))
{
c2.CommandText = @"UPDATE ""FilesetEntry"" SET ""FileID"" = ? WHERE ""FileID"" IN (SELECT ""ID"" FROM ""File"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ?)";
c2.CommandText += @"; DELETE FROM ""File"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? AND ""ID"" != ?";
c2.CommandText = @"UPDATE ""FilesetEntry"" SET ""FileID"" = ? WHERE ""FileID"" IN (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ?)";
c2.CommandText += @"; DELETE FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? AND ""ID"" != ?";
foreach(var rd in cmd.ExecuteReaderEnumerable(sql))
c2.ExecuteNonQuery(null, rd.GetValue(0), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(0));
c2.ExecuteNonQuery(null, rd.GetValue(0), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(4), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(4), rd.GetValue(0));
}
cmd.CommandText = sql_count;
Duplicati/Library/Main/Database/LocalRestoreDatabase.cs
+2
View File
@@ -213,6 +213,8 @@ namespace Duplicati.Library.Main.Database
// better suited to speed up commit on UpdateBlocks
cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}_FileIdIndexIndex"" ON ""{0}"" (""FileId"", ""Index"")", m_tempblocktable));
// TODO: Optimize to use the path prefix
if (filter == null || filter.Empty)
{
// Simple case, restore everything
Duplicati/Library/Main/Duplicati.Library.Main.csproj
+8 -4
View File
@@ -197,6 +197,14 @@
<EmbeddedResource Include="Database\Database schema\2. Use Lastmodified.sql" />
<EmbeddedResource Include="Database\Database schema\3. Add grace delete period.sql" />
<EmbeddedResource Include="Database\Database schema\4. Add index.sql" />
<EmbeddedResource Include="Database\Database schema\5. Optimize BlockSet-Tables.sql" />
<EmbeddedResource Include="Database\Database schema\6. Optimize FileSetEntry-Table.sql" />
<EmbeddedResource Include="Database\Database schema\7. Add index.sql" />
<EmbeddedResource Include="Database\Database schema\5. Optimize BlockSet-Tables.sql" />
<EmbeddedResource Include="Database\Database schema\6. Optimize FileSetEntry-Table.sql" />
<EmbeddedResource Include="Database\Database schema\7. Add index.sql" />
<EmbeddedResource Include="Database\Database schema\8. Add volume USN.sql" />
<EmbeddedResource Include="Database\Database schema\9. Refactor Paths.sql" />
</ItemGroup>
<Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
@@ -207,10 +215,6 @@
</Target>
-->
<ItemGroup>
<EmbeddedResource Include="Database\Database schema\5. Optimize BlockSet-Tables.sql" />
<EmbeddedResource Include="Database\Database schema\6. Optimize FileSetEntry-Table.sql" />
<EmbeddedResource Include="Database\Database schema\7. Add index.sql" />
<EmbeddedResource Include="Database\Database schema\8. Add volume USN.sql" />
<Content Include="default_compressed_extensions.txt">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs
+11 -6
View File
@@ -95,18 +95,18 @@ namespace Duplicati.Library.Main.Operation.Backup
return RunOnMain(() => m_database.AddSymlinkEntry(filename, metadataid, lastModified, m_transaction));
}
public Task<KeyValuePair<long, DateTime>> GetFileLastModifiedAsync(string path, long lastfilesetid)
public Task<KeyValuePair<long, DateTime>> GetFileLastModifiedAsync(long prefixid, string path, long lastfilesetid)
{
return RunOnMain(() =>
{
DateTime lastModified;
var id = m_database.GetFileLastModified(path, lastfilesetid, out lastModified, m_transaction);
var id = m_database.GetFileLastModified(prefixid, path, lastfilesetid, out lastModified, m_transaction);
return new KeyValuePair<long, DateTime>(id, lastModified);
});
}
public Task<FileEntryData> GetFileEntryAsync(string path, long lastfilesetid)
public Task<FileEntryData> GetFileEntryAsync(long prefixid, string path, long lastfilesetid)
{
return RunOnMain(() => {
DateTime oldModified;
@@ -114,7 +114,7 @@ namespace Duplicati.Library.Main.Operation.Backup
string oldMetahash;
long oldMetasize;
var id = m_database.GetFileEntry(path, lastfilesetid, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize);
var id = m_database.GetFileEntry(prefixid, path, lastfilesetid, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize);
return
id < 0 ?
null :
@@ -132,9 +132,14 @@ namespace Duplicati.Library.Main.Operation.Backup
});
}
public Task AddFileAsync(string filename, DateTime lastmodified, long blocksetid, long metadataid)
public Task<long> GetOrCreatePathPrefix(string prefix)
{
return RunOnMain(() => m_database.AddFile(filename, lastmodified, blocksetid, metadataid, m_transaction));
return RunOnMain(() => m_database.GetOrCreatePathPrefix(prefix, m_transaction));
}
public Task AddFileAsync(long prefixid, string filename, DateTime lastmodified, long blocksetid, long metadataid)
{
return RunOnMain(() => m_database.AddFile(prefixid, filename, lastmodified, blocksetid, metadataid, m_transaction));
}
public Task AddUnmodifiedAsync(long fileid, DateTime lastModified)
Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs
+2 -2
View File
@@ -109,12 +109,12 @@ namespace Duplicati.Library.Main.Operation.Backup
Logging.Log.WriteVerboseMessage(FILELOGTAG, "WoudlAddChangedFile", "Would add changed file {0}, size {1}", e.Path, Library.Utility.Utility.FormatSizeString(filesize));
}
await database.AddFileAsync(e.Path, e.LastWrite, filestreamdata.Blocksetid, metadataid);
await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid);
}
else if (e.MetadataChanged)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileMetadataChanged", "File has only metadata changes {0}", e.Path);
await database.AddFileAsync(e.Path, e.LastWrite, filestreamdata.Blocksetid, metadataid);
await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid);
}
else /*if (e.OldId >= 0)*/
{
Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs
+22 -2
View File
@@ -38,6 +38,10 @@ namespace Duplicati.Library.Main.Operation.Backup
// From input
public string Path;
// Split
public long PathPrefixID;
public string Filename;
// From database
public long OldId;
public DateTime OldModified;
@@ -66,6 +70,7 @@ namespace Duplicati.Library.Main.Operation.Backup
async self =>
{
var emptymetadata = Utility.WrapMetadata(new Dictionary<string, string>(), options);
var prevprefix = new KeyValuePair<string, long>(null, -1);
while (true)
{
@@ -96,12 +101,25 @@ namespace Duplicati.Library.Main.Operation.Backup
{
try
{
var split = Database.LocalDatabase.SplitIntoPrefixAndName(path);
long prefixid;
if (string.Equals(prevprefix.Key, split.Key, StringComparison.Ordinal))
prefixid = prevprefix.Value;
else
{
prefixid = await database.GetOrCreatePathPrefix(split.Key);
prevprefix = new KeyValuePair<string, long>(split.Key, prefixid);
}
if (options.CheckFiletimeOnly || options.DisableFiletimeCheck)
{
var tmp = await database.GetFileLastModifiedAsync(path, lastfilesetid);
var tmp = await database.GetFileLastModifiedAsync(prefixid, split.Value, lastfilesetid);
await self.Output.WriteAsync(new FileEntry() {
OldId = tmp.Key < 0 ? -1 : tmp.Key,
Path = path,
PathPrefixID = prefixid,
Filename = split.Value,
Attributes = attributes,
LastWrite = lastwrite,
OldModified = tmp.Key < 0 ? new DateTime(0) : tmp.Value,
@@ -112,10 +130,12 @@ namespace Duplicati.Library.Main.Operation.Backup
}
else
{
var res = await database.GetFileEntryAsync(path, lastfilesetid);
var res = await database.GetFileEntryAsync(prefixid, split.Value, lastfilesetid);
await self.Output.WriteAsync(new FileEntry() {
OldId = res == null ? -1 : res.id,
Path = path,
PathPrefixID = prefixid,
Filename = split.Value,
Attributes = attributes,
LastWrite = lastwrite,
OldModified = res == null ? new DateTime(0) : res.modified,
Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs
+6 -3
View File
@@ -227,11 +227,14 @@ namespace Duplicati.Library.Main.Operation
if (expectedmetablocks <= 1) expectedmetablocklisthashes = 0;
var metadataid = long.MinValue;
var split = Database.LocalDatabase.SplitIntoPrefixAndName(fe.Path);
var prefixid = restoredb.GetOrCreatePathPrefix(split.Key, tr);
switch (fe.Type)
{
case FilelistEntryType.Folder:
metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr);
restoredb.AddDirectoryEntry(filesetid, fe.Path, fe.Time, metadataid, tr);
restoredb.AddDirectoryEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, tr);
break;
case FilelistEntryType.File:
var expectedblocks = (fe.Size + blocksize - 1) / blocksize;
@@ -240,7 +243,7 @@ namespace Duplicati.Library.Main.Operation
var blocksetid = restoredb.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes, tr);
metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr);
restoredb.AddFileEntry(filesetid, fe.Path, fe.Time, blocksetid, metadataid, tr);
restoredb.AddFileEntry(filesetid, prefixid, split.Value, fe.Time, blocksetid, metadataid, tr);
if (fe.Size <= blocksize)
{
@@ -255,7 +258,7 @@ namespace Duplicati.Library.Main.Operation
break;
case FilelistEntryType.Symlink:
metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr);
restoredb.AddSymlinkEntry(filesetid, fe.Path, fe.Time, metadataid, tr);
restoredb.AddSymlinkEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, tr);
break;
default:
Logging.Log.WriteWarningMessage(LOGTAG, "SkippingUnknownFileEntry", null, "Skipping file-entry with unknown type {0}: {1} ", fe.Type, fe.Path);
Duplicati/Library/Main/Options.cs
-14
View File
@@ -512,7 +512,6 @@ namespace Duplicati.Library.Main
new CommandLineArgument("skip-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.SkipmetadataShort, Strings.Options.SkipmetadataLong, "false"),
new CommandLineArgument("restore-permissions", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RestorepermissionsShort, Strings.Options.RestorepermissionsLong, "false"),
new CommandLineArgument("skip-restore-verification", CommandLineArgument.ArgumentType.Boolean, Strings.Options.SkiprestoreverificationShort, Strings.Options.SkiprestoreverificationLong, "false"),
new CommandLineArgument("disable-filepath-cache", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablefilepathcacheShort, Strings.Options.DisablefilepathcacheLong, "true"),
new CommandLineArgument("use-block-cache", CommandLineArgument.ArgumentType.Boolean, Strings.Options.UseblockcacheShort, Strings.Options.UseblockcacheLong, "false"),
new CommandLineArgument("changed-files", CommandLineArgument.ArgumentType.Path, Strings.Options.ChangedfilesShort, Strings.Options.ChangedfilesLong),
new CommandLineArgument("deleted-files", CommandLineArgument.ArgumentType.Path, Strings.Options.DeletedfilesShort, Strings.Options.DeletedfilesLong("changed-files")),
@@ -1578,19 +1577,6 @@ namespace Duplicati.Library.Main
get { return Library.Utility.Utility.ParseBoolOption(m_options, "disable-synthetic-filelist"); }
}
/// <summary>
/// Flag indicating if the filepath cache is disabled
/// </summary>
public bool UseFilepathCache
{
get
{
string s;
m_options.TryGetValue("disable-filepath-cache", out s);
return !Library.Utility.Utility.ParseBool(s, true);
}
}
/// <summary>
/// Flag indicating if the in-memory block cache is used
/// </summary>
Duplicati/Library/Main/Strings.cs
-2
View File
@@ -147,8 +147,6 @@ namespace Duplicati.Library.Main.Strings
public static string DeletedfilesShort { get { return LC.L(@"List of deleted files"); } }
public static string FilehashlookupsizeLong { get { return LC.L(@"A fragment of memory is used to reduce database lookups. You should not change this value unless you get warnings in the log."); } }
public static string FilehashlookupsizeShort { get { return LC.L(@"Memory used by the file hash"); } }
public static string DisablefilepathcacheLong { get { return LC.L(@"This option can be used to reduce the memory footprint by not keeping paths and modification timestamps in memory"); } }
public static string DisablefilepathcacheShort { get { return LC.L(@"Reduce memory footprint by disabling in-memory lookups"); } }
public static string UseblockcacheShort { get { return LC.L(@"This option can be used to increase speed in exchange for extra memory use."); } }
public static string UseblockcacheLong { get { return LC.L(@"Store an in-memory block cache"); } }
public static string StoremetadataLong { get { return LC.L(@"Stores metadata, such as file timestamps and attributes. This increases the required storage space as well as the processing time."); } }