flashii/sharp-chat
3
0
Fork 0
Code Issues Pull requests Releases Activity

Added MariaDB -> SQLite conversion utility.

Browse source
This commit is contained in:
flash 2025-04-28 19:22:31 +00:00
parent c537df792e
commit eae379e933
Signed by: flash
GPG key ID: 2C9C2C574D47FE3E
6 changed files with 82 additions and 18 deletions

10
SharpChat.MariaDB/MariaDBConnection.cs
View file

@ -4,8 +4,10 @@ using System.Data;
namespace SharpChat.MariaDB;
public class MariaDBConnection(MySqlConnection conn) : IDisposable {
public MySqlConnection Connection { get; } = conn;
public async Task<int> RunCommand(string command, params MySqlParameter[] parameters) {
using MySqlCommand cmd = conn.CreateCommand();
using MySqlCommand cmd = Connection.CreateCommand();
if(parameters?.Length > 0)
cmd.Parameters.AddRange(parameters);
cmd.CommandText = command;
@ -13,7 +15,7 @@ public class MariaDBConnection(MySqlConnection conn) : IDisposable {
}
public async Task<MySqlDataReader?> RunQuery(string command, params MySqlParameter[] parameters) {
using MySqlCommand cmd = conn.CreateCommand();
using MySqlCommand cmd = Connection.CreateCommand();
if(parameters?.Length > 0)
cmd.Parameters.AddRange(parameters);
cmd.CommandText = command;
@ -22,7 +24,7 @@ public class MariaDBConnection(MySqlConnection conn) : IDisposable {
public async Task<T> RunQueryValue<T>(string command, params MySqlParameter[] parameters)
where T : struct {
using MySqlCommand cmd = conn.CreateCommand();
using MySqlCommand cmd = Connection.CreateCommand();
if(parameters?.Length > 0)
cmd.Parameters.AddRange(parameters);
cmd.CommandText = command;
@ -47,6 +49,6 @@ public class MariaDBConnection(MySqlConnection conn) : IDisposable {
if(disposed)
return;
disposed = true;
conn.Dispose();
Connection.Dispose();
}
}

10
SharpChat.SQLite/SQLiteConnection.cs
View file

@ -5,8 +5,10 @@ using NativeSQLiteConnection = System.Data.SQLite.SQLiteConnection;
namespace SharpChat.SQLite;
public class SQLiteConnection(NativeSQLiteConnection conn) : IDisposable {
public NativeSQLiteConnection Connection { get; } = conn;
public async Task<int> RunCommand(string command, params SQLiteParameter[] parameters) {
using SQLiteCommand cmd = conn.CreateCommand();
using SQLiteCommand cmd = Connection.CreateCommand();
if(parameters?.Length > 0)
cmd.Parameters.AddRange(parameters);
cmd.CommandText = command;
@ -14,7 +16,7 @@ public class SQLiteConnection(NativeSQLiteConnection conn) : IDisposable {
}
public async Task<DbDataReader?> RunQuery(string command, params SQLiteParameter[] parameters) {
using SQLiteCommand cmd = conn.CreateCommand();
using SQLiteCommand cmd = Connection.CreateCommand();
if(parameters?.Length > 0)
cmd.Parameters.AddRange(parameters);
cmd.CommandText = command;
@ -23,7 +25,7 @@ public class SQLiteConnection(NativeSQLiteConnection conn) : IDisposable {
public async Task<T> RunQueryValue<T>(string command, params SQLiteParameter[] parameters)
where T : struct {
using SQLiteCommand cmd = conn.CreateCommand();
using SQLiteCommand cmd = Connection.CreateCommand();
if(parameters?.Length > 0)
cmd.Parameters.AddRange(parameters);
cmd.CommandText = command;
@ -50,6 +52,6 @@ public class SQLiteConnection(NativeSQLiteConnection conn) : IDisposable {
disposed = true;
RunCommand("VACUUM").Wait();
conn.Dispose();
Connection.Dispose();
}
}

6
SharpChat.SQLite/SQLiteMessageStorage.cs
View file

@ -60,9 +60,9 @@ public class SQLiteMessageStorage(ILogger logger, SQLiteConnection conn) : Messa
reader.GetString("msg_type"),
reader.IsDBNull(reader.GetOrdinal("msg_sender")) ? null : reader.GetString("msg_sender"),
reader.IsDBNull(reader.GetOrdinal("msg_sender_name")) ? string.Empty : reader.GetString("msg_sender_name"),
ColourInheritable.FromMisuzu(reader.GetInt32("msg_sender_colour")),
reader.GetInt32("msg_sender_rank"),
SQLiteUserPermissionsConverter.From((SQLiteUserPermissions)reader.GetInt32("msg_sender_perms")),
ColourInheritable.FromMisuzu((int)reader.GetInt64("msg_sender_colour")),
(int)reader.GetInt64("msg_sender_rank"),
SQLiteUserPermissionsConverter.From((SQLiteUserPermissions)reader.GetInt64("msg_sender_perms")),
reader.IsDBNull(reader.GetOrdinal("msg_sender_nick")) ? string.Empty : reader.GetString("msg_sender_nick"),
DateTimeOffset.Parse(reader.GetString("msg_created")),
reader.IsDBNull(reader.GetOrdinal("msg_deleted")) ? null : DateTimeOffset.Parse(reader.GetString("msg_deleted")),

2
SharpChat.SQLite/SQLiteMigrations.cs
View file

@ -10,7 +10,7 @@ public class SQLiteMigrations(ILogger logger, SQLiteConnection conn) {
async Task doMigration(int expect, Func<Task> action) {
if(version < expect) {
logger.ZLogInformation($"Upgrading to version {version}...");
logger.ZLogInformation($"Upgrading to version {version + 1}...");
await action();
++version;
}

9
SharpChat.SQLite/SQLiteStorage.cs
View file

@ -22,21 +22,22 @@ public class SQLiteStorage(ILogger logger, string connString) : Storage, IDispos
await new SQLiteMigrations(logger, Connection).RunMigrations();
}
public static string BuildConnectionString(Config config) {
public static string BuildConnectionString(Config config, bool journalling = true) {
return BuildConnectionString(
config.ReadValue("path", DEFAULT)!,
config.ReadValue("pass")
config.ReadValue("pass"),
config.ReadValue("journal", journalling)
);
}
public static string BuildConnectionString(string path, string? password) {
public static string BuildConnectionString(string path, string? password, bool journalling = true) {
return new SQLiteConnectionStringBuilder {
DataSource = string.IsNullOrWhiteSpace(path) ? MEMORY : path,
DateTimeFormat = SQLiteDateFormats.ISO8601,
DateTimeKind = DateTimeKind.Utc,
FailIfMissing = false,
ForeignKeys = true,
JournalMode = SQLiteJournalModeEnum.Wal,
JournalMode = journalling ? SQLiteJournalModeEnum.Wal : SQLiteJournalModeEnum.Off,
LegacyFormat = false,
Password = string.IsNullOrWhiteSpace(password) ? null : password,
ReadOnly = false,

63
SharpChat/Program.cs
View file

@ -1,9 +1,12 @@
using Microsoft.Extensions.Logging;
using MySqlConnector;
using SharpChat;
using SharpChat.Configuration;
using SharpChat.Flashii;
using SharpChat.MariaDB;
using SharpChat.SQLite;
using System.Data.SQLite;
using System.Reflection.PortableExecutable;
using System.Text;
using ZLogger;
using ZLogger.Providers;
@ -58,8 +61,10 @@ void exitHandler() {
if(cts.IsCancellationRequested)
return;
cts.Cancel();
logger.ZLogInformation($"Shutdown requested through console...");
try {
cts.Cancel();
logger.ZLogInformation($"Shutdown requested through console...");
} catch(ObjectDisposedException) { }
}
AppDomain.CurrentDomain.ProcessExit += (sender, ev) => { exitHandler(); };
@ -149,6 +154,60 @@ using StreamConfig config = StreamConfig.FromPath(configFile);
if(cts.IsCancellationRequested) return;
if(args.Contains("--convert-db")) {
logger.ZLogInformation($"Converting MariaDB storage to SQLite");
MariaDBStorage mariadbStorage = new(logFactory.CreateLogger("mariadb"), MariaDBStorage.BuildConnectionString(config.ScopeTo("mariadb")));
await mariadbStorage.UpgradeStorage();
using SQLiteStorage sqlite = new(logFactory.CreateLogger("sqlite"), SQLiteStorage.BuildConnectionString(config.ScopeTo("sqlite"), false));
await sqlite.UpgradeStorage();
using MariaDBConnection mariadb = await mariadbStorage.CreateConnection();
long rows = await mariadb.RunQueryValue<long>("SELECT COUNT(*) FROM sqc_events");
using MySqlCommand export = mariadb.Connection.CreateCommand();
export.CommandText = "SELECT event_id, event_type, UNIX_TIMESTAMP(event_created) AS event_created, UNIX_TIMESTAMP(event_deleted) AS event_deleted, event_channel, event_sender, event_sender_name, event_sender_colour, event_sender_rank, event_sender_nick, event_sender_perms, event_data FROM sqc_events";
export.CommandTimeout = int.MaxValue;
using MySqlDataReader reader = await export.ExecuteReaderAsync();
using SQLiteCommand import = sqlite.Connection.Connection.CreateCommand();
import.CommandText = "INSERT OR IGNORE INTO messages (msg_id, msg_type, msg_created, msg_deleted, msg_channel, msg_sender, msg_sender_name, msg_sender_colour, msg_sender_rank, msg_sender_nick, msg_sender_perms, msg_data) VALUES (@id, @type, @created, @deleted, @channel, @sender, @sender_name, @sender_colour, @sender_rank, @sender_nick, @sender_perms, @data)";
long completed = 0;
DateTimeOffset lastReport = DateTimeOffset.UtcNow;
logger.ZLogInformation($"Beginning conversion of {rows} rows...");
while(reader.Read()) {
if(cts.IsCancellationRequested) return;
import.Parameters.Clear();
import.Parameters.Add(new SQLiteParameter("id", reader.GetInt64("event_id").ToString()));
import.Parameters.Add(new SQLiteParameter("type", reader.GetString("event_type")));
import.Parameters.Add(new SQLiteParameter("created", DateTimeOffset.FromUnixTimeSeconds(reader.GetInt32("event_created")).ToString("s") + "Z"));
import.Parameters.Add(new SQLiteParameter("deleted", reader.IsDBNull(reader.GetOrdinal("event_deleted")) ? null : DateTimeOffset.FromUnixTimeSeconds(reader.GetInt32("event_deleted")).ToString("s") + "Z"));
import.Parameters.Add(new SQLiteParameter("channel", reader.IsDBNull(reader.GetOrdinal("event_channel")) ? null : reader.GetString("event_channel")));
import.Parameters.Add(new SQLiteParameter("sender", reader.IsDBNull(reader.GetOrdinal("event_sender")) ? null : reader.GetString("event_sender")));
import.Parameters.Add(new SQLiteParameter("sender_name", reader.IsDBNull(reader.GetOrdinal("event_sender_name")) ? string.Empty : reader.GetString("event_sender_name")));
ColourInheritable colour = ColourInheritable.FromMisuzu(reader.GetInt32("event_sender_colour"));
import.Parameters.Add(new SQLiteParameter("sender_colour", colour.Rgb.HasValue ? colour.Rgb.Value : null));
import.Parameters.Add(new SQLiteParameter("sender_rank", reader.IsDBNull(reader.GetOrdinal("event_sender_rank")) ? null : reader.GetInt32("event_sender_rank")));
import.Parameters.Add(new SQLiteParameter("sender_nick", reader.IsDBNull(reader.GetOrdinal("event_sender_nick")) ? string.Empty : reader.GetString("event_sender_nick")));
import.Parameters.Add(new SQLiteParameter("sender_perms", SQLiteUserPermissionsConverter.To(MariaDBUserPermissionsConverter.From((MariaDBUserPermissions)reader.GetInt32("event_sender_perms")))));
import.Parameters.Add(new SQLiteParameter("data", reader.GetString("event_data")));
await import.PrepareAsync();
await import.ExecuteNonQueryAsync();
++completed;
if(DateTimeOffset.UtcNow - lastReport > TimeSpan.FromMinutes(1)) {
lastReport = DateTimeOffset.UtcNow;
double completion = (double)completed / rows;
logger.ZLogInformation($"{completed} of {rows} converted ({completion:P2})...");
}
}
logger.ZLogInformation($"Converted all {completed} rows!");
return;
}
logger.ZLogInformation($"Initialising HTTP client...");
using HttpClient httpClient = new(new HttpClientHandler() {
UseProxy = false,