Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Offer an option to upgrade streams. #1969

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions LiteDB.Tests/Database/Upgrade_Tests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -88,5 +88,38 @@ public void Migrage_From_V4_No_FileExtension()
}
}
}

[Fact]
public void Migrage_From_V4_Stream()
{
// v5 upgrades only from v4!

var original = "../../../Utils/Legacy/v4.db";

using (FileStream fs = new FileStream(original, FileMode.Open, FileAccess.Read))
using (MemoryStream ms = new MemoryStream())
{
fs.CopyTo(ms);
ms.Flush();
ms.Seek(0, SeekOrigin.Begin);

using (var db = new LiteDatabase(ms, upgrade:true))
{
// convert and open database
var col1 = db.GetCollection("col1");

col1.Count().Should().Be(3);
}

ms.Seek(0, SeekOrigin.Begin);
using (var db = new LiteDatabase(ms, upgrade: true))
{
// database already converted
var col1 = db.GetCollection("col1");

col1.Count().Should().Be(3);
}
}
}
}
}
8 changes: 7 additions & 1 deletion LiteDB/Client/Database/LiteDatabase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,14 @@ public LiteDatabase(ConnectionString connectionString, BsonMapper mapper = null)
/// <param name="stream">DataStream reference </param>
/// <param name="mapper">BsonMapper mapper reference</param>
/// <param name="logStream">LogStream reference </param>
public LiteDatabase(Stream stream, BsonMapper mapper = null, Stream logStream = null)
/// <param name="upgrade">Try to upgrade the stream. If specifying true and the stream needs to be upgraded, the stream must be writeable</param>
public LiteDatabase(Stream stream, BsonMapper mapper = null, Stream logStream = null, bool upgrade = false)
{
if (upgrade && !LiteEngine.Upgrade(stream))
{//Assume it's already upgraded and rewind the stream
stream.Seek(0, SeekOrigin.Begin);
}

var settings = new EngineSettings
{
DataStream = stream ?? throw new ArgumentNullException(nameof(stream)),
Expand Down
115 changes: 77 additions & 38 deletions LiteDB/Engine/Engine/Upgrade.cs
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using static LiteDB.Constants;

Expand Down Expand Up @@ -29,56 +27,97 @@ public static bool Upgrade(string filename, string password = null, Collation co

settings.Filename = FileHelper.GetSufixFile(filename, "-temp", true);

var buffer = new byte[PAGE_SIZE * 2];
IFileReader reader;

using (var stream = new FileStream(filename, FileMode.Open, FileAccess.Read))
{
// read first 16k
stream.Read(buffer, 0, buffer.Length);

// checks if v8 plain data or encrypted (first byte = 1)
if ((Encoding.UTF8.GetString(buffer, HeaderPage.P_HEADER_INFO, HeaderPage.HEADER_INFO.Length) == HeaderPage.HEADER_INFO &&
buffer[HeaderPage.P_FILE_VERSION] == HeaderPage.FILE_VERSION) ||
buffer[0] == 1)
{
if (!TryUpgradeStreamInternal(password, stream, settings))
return false;
}
}

// checks if v7 (plain or encrypted)
if (Encoding.UTF8.GetString(buffer, 25, HeaderPage.HEADER_INFO.Length) == HeaderPage.HEADER_INFO &&
buffer[52] == 7)
{
reader = new FileReaderV7(stream, password);
}
else
{
throw new LiteException(0, "Invalid data file format to upgrade");
}
// rename source filename to backup name
File.Move(filename, backup);

// rename temp file into filename
File.Move(settings.Filename, filename);

return true;
}

using (var engine = new LiteEngine(settings))
/// <summary>
/// Upgrade old version of LiteDB into new LiteDB file structure. Returns true if database was completed converted
/// If database already in current version just return false
/// </summary>
public static bool Upgrade(Stream stream, string password = null, Collation collation = null)
{
using (MemoryStream ms = new MemoryStream())
{
var settings = new EngineSettings
{
// copy all database to new Log file with NO checkpoint during all rebuild
engine.Pragma(Pragmas.CHECKPOINT, 0);
DataStream = ms,
Password = password,
Collation = collation
};


engine.RebuildContent(reader);

// after rebuild, copy log bytes into data file
engine.Checkpoint();
if (!TryUpgradeStreamInternal(password, stream, settings))
return false;
ms.Flush();
ms.Seek(0, SeekOrigin.Begin);

stream.Seek(0, SeekOrigin.Begin);
stream.SetLength(0);
ms.CopyTo(stream);

stream.Flush();
stream.Seek(0, SeekOrigin.Begin);
return true;
}

// re-enable auto-checkpoint pragma
engine.Pragma(Pragmas.CHECKPOINT, 1000);
}

// copy userVersion from old datafile
engine.Pragma("USER_VERSION", (reader as FileReaderV7).UserVersion);
}
private static bool TryUpgradeStreamInternal(string password, Stream stream, EngineSettings settings)
{
var buffer = new byte[PAGE_SIZE * 2];
IFileReader reader;
// read first 16k
stream.Read(buffer, 0, buffer.Length);

// checks if v8 plain data or encrypted (first byte = 1)
if ((Encoding.UTF8.GetString(buffer, HeaderPage.P_HEADER_INFO, HeaderPage.HEADER_INFO.Length) == HeaderPage.HEADER_INFO &&
buffer[HeaderPage.P_FILE_VERSION] == HeaderPage.FILE_VERSION) ||
buffer[0] == 1)
{
return false;
}

// rename source filename to backup name
File.Move(filename, backup);
// checks if v7 (plain or encrypted)
if (Encoding.UTF8.GetString(buffer, 25, HeaderPage.HEADER_INFO.Length) == HeaderPage.HEADER_INFO &&
buffer[52] == 7)
{
reader = new FileReaderV7(stream, password);
}
else
{
throw new LiteException(0, "Invalid data file format to upgrade");
}

// rename temp file into filename
File.Move(settings.Filename, filename);
using (var engine = new LiteEngine(settings))
{
// copy all database to new Log file with NO checkpoint during all rebuild
engine.Pragma(Pragmas.CHECKPOINT, 0);

engine.RebuildContent(reader);

// after rebuild, copy log bytes into data file
engine.Checkpoint();

// re-enable auto-checkpoint pragma
engine.Pragma(Pragmas.CHECKPOINT, 1000);

// copy userVersion from old datafile
engine.Pragma("USER_VERSION", (reader as FileReaderV7).UserVersion);
}

return true;
}
Expand Down