diff --git a/lib/index b/lib/index index 939dcd1..ccf75ed 160000 --- a/lib/index +++ b/lib/index @@ -1 +1 @@ -Subproject commit 939dcd10fe7f33d567e4bf327b284dcb8a9bdd63 +Subproject commit ccf75ede4ae02aa184a51f565ee0b4f0ace29c6f diff --git a/public/settings/data.php b/public/settings/data.php index 0e6b406..4e7203e 100644 --- a/public/settings/data.php +++ b/public/settings/data.php @@ -2,6 +2,8 @@ namespace Misuzu; use ZipArchive; +use Index\XString; +use Index\IO\FileStream; use Misuzu\Users\User; use Misuzu\Users\UserSession; @@ -12,18 +14,91 @@ if(!UserSession::hasCurrent()) { return; } -function db_to_zip(ZipArchive $archive, int $userId, string $filename, string $query, int $params = 1): void { - $prepare = DB::prepare($query); +$dbConn = $msz->getDbConn(); - if($params < 2) { - $prepare->bind('user_id', $userId); - } else { - for($i = 1; $i <= $params; $i++) { - $prepare->bind('user_id_' . $i, $userId); +function db_to_zip(ZipArchive $archive, int $userId, string $baseName, array $fieldInfos, string $userIdField = 'user_id'): string { + global $dbConn; + + $fields = []; + + foreach($fieldInfos as $key => $fieldInfo) { + $fieldInfo = explode(':', $fieldInfo); + $fieldInfo = [ + 'name' => $fieldInfo[0], + 'type' => $fieldInfo[1] ?? 'n', + 'nullable' => ($fieldInfo[2] ?? '') === 'n', + ]; + $fieldInfo['is_null'] = $fieldInfo['type'] === 'n'; + + if(!$fieldInfo['is_null']) { + $selectField = $fieldInfo['name']; + + if($fieldInfo['type'] === 'a') + $selectField = sprintf('INET6_NTOA(%s)', $selectField); + elseif($fieldInfo['type'] === 't') + $selectField = sprintf('UNIX_TIMESTAMP(%s)', $selectField); + + $fields[] = $selectField; } + + $fieldInfos[$key] = $fieldInfo; } - $archive->addFromString($filename, json_encode($prepare->fetchAll(), JSON_PRETTY_PRINT)); + $tmpName = sys_get_temp_dir() . DIRECTORY_SEPARATOR . sprintf('msz-user-data-%d-%s-%s.tmp', $userId, $baseName, XString::random(8)); + $tmpStream = FileStream::newWrite($tmpName); + + try { + $stmt = $dbConn->prepare(sprintf('SELECT %s FROM msz_%s WHERE %s = ?', implode(', ', $fields), $baseName, $userIdField)); + $stmt->addParameter(1, $userId); + $stmt->execute(); + $result = $stmt->getResult(); + + while($result->next()) { + $args = -1; + $row = []; + + foreach($fieldInfos as $fieldInfo) { + $fieldValue = null; + + if(!$fieldInfo['is_null']) { + ++$args; + + if(!$fieldInfo['nullable'] || !$result->isNull($args)) { + if($fieldInfo['type'] === 's' || $fieldInfo['type'] === 'a' || $fieldInfo['type'] === 'j') { + $fieldValue = $result->getString($args); + + if($fieldInfo['type'] === 'j') + $fieldValue = json_decode($fieldValue); + } elseif($fieldInfo['type'] === 't' || $fieldInfo['type'] === 'b' || $fieldInfo['type'] === 'i') { + $fieldValue = $result->getInteger($args); + + if($fieldInfo['type'] === 'b') + $fieldValue = $fieldValue !== 0; + elseif($fieldInfo['type'] === 't') { + $fieldValue = date(DATE_ATOM, $fieldValue); + if(str_ends_with($fieldValue, '+00:00')) + $fieldValue = substr($fieldValue, 0, -6) . 'Z'; + } + } else { + $fieldValue = '?'; + } + } + } + + $row[$fieldInfo['name']] = $fieldValue; + } + + $tmpStream->write(json_encode($row, JSON_INVALID_UTF8_SUBSTITUTE)); + $tmpStream->write("\n"); + } + } finally { + $tmpStream->flush(); + $tmpStream->close(); + } + + $archive->addFile($tmpName, $baseName . '.jsonl'); + + return $tmpName; } $errors = []; @@ -32,7 +107,7 @@ $currentUserId = $currentUser->getId(); if(isset($_POST['action']) && is_string($_POST['action'])) { if(isset($_POST['password']) && is_string($_POST['password']) - && $currentUser->checkPassword($_POST['password'] ?? '')) { + && ($currentUser->checkPassword($_POST['password'] ?? ''))) { switch($_POST['action']) { case 'data': $msz->createAuditLog('PERSONAL_DATA_DOWNLOAD'); @@ -44,26 +119,36 @@ if(isset($_POST['action']) && is_string($_POST['action'])) { if(!is_file($filePath)) { if($archive->open($filePath, ZipArchive::CREATE | ZIPARCHIVE::OVERWRITE) === true) { - db_to_zip($archive, $currentUserId, 'audit_log.json', 'SELECT *, INET6_NTOA(`log_ip`) AS `log_ip` FROM `msz_audit_log` WHERE `user_id` = :user_id'); - db_to_zip($archive, $currentUserId, 'auth_tfa.json', 'SELECT * FROM `msz_auth_tfa` WHERE `user_id` = :user_id'); - db_to_zip($archive, $currentUserId, 'changelog_changes.json', 'SELECT * FROM `msz_changelog_changes` WHERE `user_id` = :user_id'); - db_to_zip($archive, $currentUserId, 'comments_posts.json', 'SELECT * FROM `msz_comments_posts` WHERE `user_id` = :user_id'); - db_to_zip($archive, $currentUserId, 'comments_votes.json', 'SELECT * FROM `msz_comments_votes` WHERE `user_id` = :user_id'); - db_to_zip($archive, $currentUserId, 'forum_permissions.json', 'SELECT * FROM `msz_forum_permissions` WHERE `user_id` = :user_id'); - db_to_zip($archive, $currentUserId, 'forum_posts.json', 'SELECT *, INET6_NTOA(`post_ip`) AS `post_ip` FROM `msz_forum_posts` WHERE `user_id` = :user_id'); - db_to_zip($archive, $currentUserId, 'forum_topics.json', 'SELECT * FROM `msz_forum_topics` WHERE `user_id` = :user_id'); - db_to_zip($archive, $currentUserId, 'forum_topics_track.json', 'SELECT * FROM `msz_forum_topics_track` WHERE `user_id` = :user_id'); - db_to_zip($archive, $currentUserId, 'login_attempts.json', 'SELECT *, INET6_NTOA(`attempt_ip`) AS `attempt_ip` FROM `msz_login_attempts` WHERE `user_id` = :user_id'); - db_to_zip($archive, $currentUserId, 'news_posts.json', 'SELECT * FROM `msz_news_posts` WHERE `user_id` = :user_id'); - db_to_zip($archive, $currentUserId, 'permissions.json', 'SELECT * FROM `msz_permissions` WHERE `user_id` = :user_id'); - db_to_zip($archive, $currentUserId, 'profile_fields_values.json', 'SELECT * FROM `msz_profile_fields_values` WHERE `user_id` = :user_id'); - db_to_zip($archive, $currentUserId, 'sessions.json', 'SELECT *, INET6_NTOA(`session_ip`) AS `session_ip`, INET6_NTOA(`session_ip_last`) AS `session_ip_last` FROM `msz_sessions` WHERE `user_id` = :user_id'); - db_to_zip($archive, $currentUserId, 'users.json', 'SELECT *, NULL AS `password`, NULL AS `user_totp_key`, INET6_NTOA(`register_ip`) AS `register_ip`, INET6_NTOA(`last_ip`) AS `last_ip` FROM `msz_users` WHERE `user_id` = :user_id'); - db_to_zip($archive, $currentUserId, 'users_password_resets.json', 'SELECT *, INET6_NTOA(`reset_ip`) AS `reset_ip` FROM `msz_users_password_resets` WHERE `user_id` = :user_id'); - db_to_zip($archive, $currentUserId, 'user_roles.json', 'SELECT * FROM `msz_user_roles` WHERE `user_id` = :user_id'); - db_to_zip($archive, $currentUserId, 'user_warnings.json', 'SELECT *, INET6_NTOA(`user_ip`) AS `user_ip`, NULL AS `issuer_id`, NULL AS `issuer_ip`, NULL AS `warning_note_private` FROM `msz_user_warnings` WHERE `user_id` = :user_id'); + $tmpFiles = []; - $archive->close(); + try { + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'audit_log', ['user_id:s:n', 'log_action:s', 'log_params:j', 'log_created:t', 'log_ip:a:n', 'log_country:s']); + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'auth_tfa', ['user_id:s', 'tfa_token:n', 'tfa_created:t']); + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'changelog_changes', ['change_id:s', 'user_id:s:n', 'change_action:s:n', 'change_created:t', 'change_log:s', 'change_text:s:n']); + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'comments_categories', ['category_id:s', 'category_name:s', 'owner_id:s:n', 'category_created:t', 'category_locked:t:n'], 'owner_id'); + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'comments_posts', ['comment_id:s', 'category_id:s', 'user_id:s:n', 'comment_reply_to:s:n', 'comment_text:s', 'comment_created:t', 'comment_pinned:t:n', 'comment_edited:t:n', 'comment_deleted:t:n']); + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'comments_votes', ['comment_id:s', 'user_id:s', 'comment_vote:i']); + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_permissions', ['user_id:s:n', 'role_id:s:n', 'forum_id:s', 'forum_perms_allow:i', 'forum_perms_deny:i']); + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_posts', ['post_id:s', 'topic_id:s', 'forum_id:s', 'user_id:s:n', 'post_ip:a', 'post_text:s', 'post_parse:i', 'post_display_signature:b', 'post_created:t', 'post_edited:t:n', 'post_deleted:t:n']); + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_topics', ['topic_id:s', 'forum_id:s', 'user_id:s:n', 'topic_type:i', 'topic_title:s', 'topic_count_views:i', 'topic_created:t', 'topic_bumped:t', 'topic_deleted:t:n', 'topic_locked:t:n']); + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_topics_redirects', ['topic_id:s', 'user_id:s:n', 'topic_redir_url:s', 'topic_redir_created:t']); + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_topics_track', ['user_id:s', 'topic_id:s', 'forum_id:s', 'track_last_read:t']); + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'login_attempts', ['user_id:s:n', 'attempt_success:b', 'attempt_ip:a', 'attempt_country:s', 'attempt_created:t', 'attempt_user_agent:s']); + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'news_posts', ['post_id:s', 'category_id:s', 'user_id:s:n', 'comment_section_id:s:n', 'post_is_featured:b', 'post_title:s', 'post_text:s', 'post_scheduled:t', 'post_created:t', 'post_updated:t', 'post_deleted:t:n']); + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'permissions', ['user_id:s:n', 'role_id:s:n', 'general_perms_allow:i', 'general_perms_deny:i', 'user_perms_allow:i', 'user_perms_deny:i', 'changelog_perms_allow:i', 'changelog_perms_deny:i', 'news_perms_allow:i', 'news_perms_deny:i', 'forum_perms_allow:i', 'forum_perms_deny:i', 'comments_perms_allow:i', 'comments_perms_deny:i']); + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'profile_fields_values', ['field_id:s', 'user_id:s', 'format_id:s', 'field_value:s']); + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'sessions', ['session_id:s', 'user_id:s', 'session_key:n', 'session_ip:a', 'session_ip_last:a:n', 'session_user_agent:s', 'session_country:s', 'session_expires:t', 'session_expires_bump:b', 'session_created:t', 'session_active:t:n']); + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'users', ['user_id:s', 'username:s', 'password:n', 'email:s', 'register_ip:a', 'last_ip:a', 'user_super:b', 'user_country:s', 'user_colour:i:n', 'user_created:t', 'user_active:t:n', 'user_deleted:t:n', 'display_role:s:n', 'user_totp_key:n', 'user_about_content:s:n', 'user_about_parser:i', 'user_signature_content:s:n', 'user_signature_parser:i', 'user_birthdate:s:n', 'user_background_settings:i:n', 'user_title:s:n']); + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'users_password_resets', ['user_id:s', 'reset_ip:a', 'reset_requested:t', 'verification_code:n']); + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'user_roles', ['user_id:s', 'role_id:s']); + $tmpFiles[] = db_to_zip($archive, $currentUserId, 'user_warnings', ['warning_id:s', 'user_id:s', 'user_ip:a', 'issuer_id:n', 'issuer_ip:n', 'warning_created:t', 'warning_duration:t:n', 'warning_type:i', 'warning_note:s', 'warning_note_private:s:n']); + + $archive->close(); + } finally { + foreach($tmpFiles as $tmpFile) + if(is_file($tmpFile)) + unlink($tmpFile); + } } else { $errors[] = 'Something went wrong while creating your account archive.'; break; diff --git a/templates/settings/data.twig b/templates/settings/data.twig index 3a32e6b..b4e75b1 100644 --- a/templates/settings/data.twig +++ b/templates/settings/data.twig @@ -10,7 +10,7 @@ {{ input_csrf() }}
-

Here you can request raw json files containing pretty much all data relating to your account. Moderator identities are concealed and password hashes are removed from the output.

+

Here you can request raw json files containing pretty much all data relating to your account. Moderator identities are concealed and password hashes are removed from the output. Files in the archive will be in JSON Lines format.