flashii/backup-manager
Archived
1
0
Fork 0
Code Pull requests Releases Activity

Store each database individually.

Browse source
This commit is contained in:
flash 2019-11-06 18:04:53 +01:00
parent 33b50e9323
commit 1e8e180a78
1 changed files with 83 additions and 142 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

223
BackupManager/Program.cs
View file

@ -11,10 +11,8 @@ using System.Text;
using System.Xml; using System.Xml;
using System.Xml.Serialization; using System.Xml.Serialization;
namespace BackupManager namespace BackupManager {
{ public static class Program {
public static class Program
{
public readonly static Stopwatch sw = new Stopwatch(); public readonly static Stopwatch sw = new Stopwatch();
private const string CONFIG_NAME = @"FlashiiBackupManager.v1.xml"; private const string CONFIG_NAME = @"FlashiiBackupManager.v1.xml";
@ -26,9 +24,7 @@ namespace BackupManager
public static string Basename public static string Basename
=> $@"{Environment.MachineName} {Startup.Year:0000}-{Startup.Month:00}-{Startup.Day:00} {Startup.Hour:00}{Startup.Minute:00}{Startup.Second:00}"; => $@"{Environment.MachineName} {Startup.Year:0000}-{Startup.Month:00}-{Startup.Day:00} {Startup.Hour:00}{Startup.Minute:00}{Startup.Second:00}";
public static string DatabaseDumpName public static string BackupName
=> $@"{Basename}.sql.gz";
public static string UserDataName
=> $@"{Basename}.zip"; => $@"{Basename}.zip";
private static Config Config; private static Config Config;
@ -37,19 +33,17 @@ namespace BackupManager
CONFIG_NAME CONFIG_NAME
); );
public static string FSPath => string.IsNullOrWhiteSpace(Config.FileSystemPathV2) public static string BackupStore => string.IsNullOrWhiteSpace(Config.FileSystemPathV2)
? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Personal), @"Backups") ? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Personal), @"Backups")
: Config.FileSystemPathV2; : Config.FileSystemPathV2;
public static bool Headless; public static bool Headless;
public static string WindowsToUnixPath(this string path) public static string WindowsToUnixPath(this string path) {
{
return IsWindows ? path.Replace('\\', '/') : path; return IsWindows ? path.Replace('\\', '/') : path;
} }
public static Stream ToXml(this object obj, bool pretty = false) public static Stream ToXml(this object obj, bool pretty = false) {
{
MemoryStream ms = new MemoryStream(); MemoryStream ms = new MemoryStream();
XmlSerializer xs = new XmlSerializer(obj.GetType()); XmlSerializer xs = new XmlSerializer(obj.GetType());
@ -60,8 +54,7 @@ namespace BackupManager
return ms; return ms;
} }
public static T FromXml<T>(Stream xml) public static T FromXml<T>(Stream xml) {
{
if (xml.CanSeek) if (xml.CanSeek)
xml.Seek(0, SeekOrigin.Begin); xml.Seek(0, SeekOrigin.Begin);
@ -69,30 +62,26 @@ namespace BackupManager
return (T)xs.Deserialize(xml); return (T)xs.Deserialize(xml);
} }
public static void SaveConfig() public static void SaveConfig() {
{
Log(@"Saving configuration..."); Log(@"Saving configuration...");
using (FileStream fs = new FileStream(ConfigPath, FileMode.Create, FileAccess.Write)) using (FileStream fs = new FileStream(ConfigPath, FileMode.Create, FileAccess.Write))
using (Stream cs = Config.ToXml(true)) using (Stream cs = Config.ToXml(true))
cs.CopyTo(fs); cs.CopyTo(fs);
} }
public static void LoadConfig() public static void LoadConfig() {
{
Log(@"Loading configuration..."); Log(@"Loading configuration...");
using (FileStream fs = File.OpenRead(ConfigPath)) using (FileStream fs = File.OpenRead(ConfigPath))
Config = FromXml<Config>(fs); Config = FromXml<Config>(fs);
} }
public static void Main(string[] args) public static void Main(string[] args) {
{
Headless = args.Contains(@"-cron") || args.Contains(@"-headless"); Headless = args.Contains(@"-cron") || args.Contains(@"-headless");
Log(@"Flashii Backup Manager"); Log(@"Flashii Backup Manager");
sw.Start(); sw.Start();
if (!File.Exists(ConfigPath)) if (!File.Exists(ConfigPath)) {
{
Config = new Config(); Config = new Config();
SaveConfig(); SaveConfig();
Error(@"No configuration file exists, created a blank one. Be sure to fill it out properly."); Error(@"No configuration file exists, created a blank one. Be sure to fill it out properly.");
@ -100,46 +89,55 @@ namespace BackupManager
LoadConfig(); LoadConfig();
if (!Directory.Exists(FSPath)) if (!Directory.Exists(BackupStore))
Directory.CreateDirectory(FSPath); Directory.CreateDirectory(BackupStore);
Log(@"Database backup..."); Log(@"Creating backup archive...");
string sqldump = CreateDbDump(); string archivePath = Path.GetTempFileName();
using (Stream s = File.OpenRead(sqldump)) using (FileStream fs = File.OpenWrite(archivePath))
using (Stream g = GZipEncodeStream(s)) using (ZipArchive archive = new ZipArchive(fs, ZipArchiveMode.Create)) {
{ Log(@"Database backup...");
Upload(DatabaseDumpName, g);
Log($@"MariaDB dump saved.");
}
File.Delete(sqldump); string sqldefaults = Path.GetTempFileName();
if (Directory.Exists(Config.MisuzuPath)) using (FileStream sqlConfFs = File.Open(sqldefaults, FileMode.Open, FileAccess.ReadWrite))
{ using (StreamWriter sw = new StreamWriter(sqlConfFs)) {
Log(@"Filesystem backup..."); sw.WriteLine(@"[client]");
string mszConfig = Path.Combine(Config.MisuzuPath, @"config/config.ini"); sw.WriteLine($@"user={Config.MySqlUser}");
sw.WriteLine($@"password={Config.MySqlPass}");
if (!File.Exists(mszConfig)) sw.WriteLine(@"default-character-set=utf8mb4");
Error(@"Could not find Misuzu config.");
string mszStore = Path.Combine(Config.MisuzuPath, @"store");
if (!Directory.Exists(mszStore))
Error(@"Could not find Misuzu storage directory.");
string archivePath = CreateMisuzuDataBackup(mszConfig, mszStore);
using (FileStream fs = File.OpenRead(archivePath))
{
Upload(UserDataName, fs);
Log($@"Misuzu data saved.");
} }
File.Delete(archivePath); string[] databases = Config.MySqlDatabases.Split(' ');
foreach (string database in databases)
CreateDbDump(archive, sqldefaults, database);
Log($@"MariaDB dump done.");
File.Delete(sqldefaults);
if (Directory.Exists(Config.MisuzuPath)) {
Log(@"Filesystem backup...");
string mszConfig = Path.Combine(Config.MisuzuPath, @"config/config.ini");
if (!File.Exists(mszConfig))
Error(@"Could not find Misuzu config.");
string mszStore = Path.Combine(Config.MisuzuPath, @"store");
if (!Directory.Exists(mszStore))
Error(@"Could not find Misuzu storage directory.");
CreateMisuzuDataBackup(archive, mszConfig, mszStore);
}
} }
string targetPath = Path.Combine(BackupStore, BackupName);
File.Move(archivePath, targetPath);
Log($@"Moved backup archive to {targetPath}");
SaveConfig(); SaveConfig();
sw.Stop(); sw.Stop();
Log($@"Done! Took {sw.Elapsed}.", true); Log($@"Done! Took {sw.Elapsed}.", true);
@ -149,12 +147,9 @@ namespace BackupManager
#endif #endif
} }
public static void Log(object line, bool forceSatori = false) public static void Log(object line, bool forceSatori = false) {
{ if (!Headless) {
if (!Headless) if (sw?.IsRunning == true) {
{
if (sw?.IsRunning == true)
{
ConsoleColor fg = Console.ForegroundColor; ConsoleColor fg = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Yellow; Console.ForegroundColor = ConsoleColor.Yellow;
Console.Write(sw.ElapsedMilliseconds.ToString().PadRight(10)); Console.Write(sw.ElapsedMilliseconds.ToString().PadRight(10));
@ -168,10 +163,8 @@ namespace BackupManager
SatoriBroadcast(line.ToString()); SatoriBroadcast(line.ToString());
} }
public static void Error(object line, int exit = 0x00DEAD00) public static void Error(object line, int exit = 0x00DEAD00) {
{ if (!Headless) {
if (!Headless)
{
Console.ForegroundColor = ConsoleColor.Red; Console.ForegroundColor = ConsoleColor.Red;
Log(line); Log(line);
Console.ResetColor(); Console.ResetColor();
@ -186,72 +179,41 @@ namespace BackupManager
Environment.Exit(exit); Environment.Exit(exit);
} }
public static void Upload(string name, Stream stream) public static void CreateMisuzuDataBackup(ZipArchive archive, string configPath, string storePath) {
{ Log(@"Storing non-volatile Misuzu data...");
Log($@"Saving '{name}'...");
string filename = Path.Combine(FSPath, name); archive.CreateEntryFromFile(configPath, @"misuzu/config/config.ini", CompressionLevel.Optimal);
using (FileStream fs = File.OpenWrite(filename)) string[] storeFiles = Directory.GetFiles(storePath, @"*", SearchOption.AllDirectories);
stream.CopyTo(fs);
foreach (string file in storeFiles)
archive.CreateEntryFromFile(
file,
@"misuzu/store/" + file.Replace(storePath, string.Empty).WindowsToUnixPath().Trim('/'),
CompressionLevel.Optimal
);
} }
public static string CreateMisuzuDataBackup(string configPath, string storePath) public static void CreateDbDump(ZipArchive archive, string defaults, string database) {
{ Log($@"Dumping {database}...");
Log(@"Creating Zip archive containing non-volatile Misuzu data...");
string tmpName = Path.GetTempFileName();
using (FileStream fs = File.OpenWrite(tmpName))
using (ZipArchive za = new ZipArchive(fs, ZipArchiveMode.Create))
{
za.CreateEntryFromFile(configPath, @"config/config.ini", CompressionLevel.Optimal);
string[] storeFiles = Directory.GetFiles(storePath, @"*", SearchOption.AllDirectories);
foreach (string file in storeFiles)
za.CreateEntryFromFile(
file,
@"store/" + file.Replace(storePath, string.Empty).WindowsToUnixPath().Trim('/'),
CompressionLevel.Optimal
);
}
return tmpName;
}
public static string CreateDbDump()
{
Log(@"Dumping MariaDB Databases...");
string sqldefaults = Path.GetTempFileName();
using (FileStream fs = File.Open(sqldefaults, FileMode.Open, FileAccess.ReadWrite))
using (StreamWriter sw = new StreamWriter(fs))
{
sw.WriteLine(@"[client]");
sw.WriteLine($@"user={Config.MySqlUser}");
sw.WriteLine($@"password={Config.MySqlPass}");
sw.WriteLine(@"default-character-set=utf8mb4");
}
string sqldump = Path.GetTempFileName(); string sqldump = Path.GetTempFileName();
StringBuilder mysqldumpArgs = new StringBuilder(); StringBuilder mysqldumpArgs = new StringBuilder();
mysqldumpArgs.AppendFormat(@"--defaults-file={0} ", sqldefaults); mysqldumpArgs.AppendFormat(@"--defaults-file={0} ", defaults);
mysqldumpArgs.Append(@"--single-transaction "); mysqldumpArgs.Append(@"--single-transaction ");
mysqldumpArgs.Append(@"--tz-utc --triggers "); mysqldumpArgs.Append(@"--tz-utc --triggers ");
mysqldumpArgs.Append(@"--routines --hex-blob "); mysqldumpArgs.Append(@"--routines --hex-blob ");
mysqldumpArgs.Append(@"--add-locks --order-by-primary "); mysqldumpArgs.Append(@"--add-locks --order-by-primary ");
mysqldumpArgs.AppendFormat(@"--result-file={0} ", sqldump); mysqldumpArgs.AppendFormat(@"--result-file={0} ", sqldump);
mysqldumpArgs.Append(@"-l -Q -q -B "); // lock, quote names, quick, databases list mysqldumpArgs.Append(@"-l -Q -q -B "); // lock, quote names, quick, database list
mysqldumpArgs.Append(Config.MySqlDatabases); mysqldumpArgs.Append(database);
#if DEBUG #if DEBUG
Log($@"mysqldump args: {mysqldumpArgs}"); Log($@"mysqldump args: {mysqldumpArgs}");
#endif #endif
Process p = Process.Start(new ProcessStartInfo Process p = Process.Start(new ProcessStartInfo {
{
FileName = IsWindows ? Config.MySqlDumpPathWindows : Config.MySqlDumpPath, FileName = IsWindows ? Config.MySqlDumpPathWindows : Config.MySqlDumpPath,
Arguments = mysqldumpArgs.ToString(), Arguments = mysqldumpArgs.ToString(),
UseShellExecute = false, UseShellExecute = false,
@ -260,25 +222,12 @@ namespace BackupManager
p.WaitForExit(); p.WaitForExit();
File.Delete(sqldefaults); archive.CreateEntryFromFile(sqldump, $@"mariadb/{database}.sql", CompressionLevel.Optimal);
return sqldump; File.Delete(sqldump);
} }
public static Stream GZipEncodeStream(Stream input) public static void SatoriBroadcast(string text, bool error = false) {
{
Log(@"Compressing stream...");
MemoryStream output = new MemoryStream();
using (GZipStream gz = new GZipStream(output, CompressionLevel.Optimal, true))
input.CopyTo(gz);
output.Seek(0, SeekOrigin.Begin);
return output;
}
public static void SatoriBroadcast(string text, bool error = false)
{
if (string.IsNullOrEmpty(text) if (string.IsNullOrEmpty(text)
|| Config == null || Config == null
|| string.IsNullOrWhiteSpace(Config.SatoriHost) || string.IsNullOrWhiteSpace(Config.SatoriHost)
@ -288,19 +237,13 @@ namespace BackupManager
IPAddress ip = null; IPAddress ip = null;
try try {
{
ip = IPAddress.Parse(Config.SatoriHost); ip = IPAddress.Parse(Config.SatoriHost);
} } catch {
catch try {
{
try
{
// forcing IPv4 here, it seems to explode with IPv6 and i don't really want to figure out why // forcing IPv4 here, it seems to explode with IPv6 and i don't really want to figure out why
ip = Dns.GetHostAddresses(Config.SatoriHost).FirstOrDefault(x => x.AddressFamily == AddressFamily.InterNetwork); ip = Dns.GetHostAddresses(Config.SatoriHost).FirstOrDefault(x => x.AddressFamily == AddressFamily.InterNetwork);
} } catch {
catch
{
ip = null; ip = null;
} }
} }
@ -325,8 +268,7 @@ namespace BackupManager
StringBuilder messageBuilder = new StringBuilder(); StringBuilder messageBuilder = new StringBuilder();
using (HMACSHA256 hmac = new HMACSHA256(Encoding.UTF8.GetBytes(Config.SatoriSecret))) using (HMACSHA256 hmac = new HMACSHA256(Encoding.UTF8.GetBytes(Config.SatoriSecret))) {
{
byte[] hash = hmac.ComputeHash(Encoding.UTF8.GetBytes(text)); byte[] hash = hmac.ComputeHash(Encoding.UTF8.GetBytes(text));
foreach (byte b in hash) foreach (byte b in hash)
@ -339,8 +281,7 @@ namespace BackupManager
messageBytes[0] = messageBytes[messageBytes.Length - 1] = 0x0F; messageBytes[0] = messageBytes[messageBytes.Length - 1] = 0x0F;
Encoding.UTF8.GetBytes(message).CopyTo(messageBytes, 1); Encoding.UTF8.GetBytes(message).CopyTo(messageBytes, 1);
using (Socket sock = new Socket(endPoint.AddressFamily, SocketType.Stream, ProtocolType.Tcp)) using (Socket sock = new Socket(endPoint.AddressFamily, SocketType.Stream, ProtocolType.Tcp)) {
{
sock.NoDelay = sock.Blocking = true; sock.NoDelay = sock.Blocking = true;
sock.Connect(endPoint); sock.Connect(endPoint);
sock.Send(messageBytes); sock.Send(messageBytes);