Fixed account data export memory usage.

This commit is contained in:
flash 2023-07-17 19:28:13 +00:00
parent 1a11a8f8ba
commit e5d9128cd0
3 changed files with 115 additions and 30 deletions

@ -1 +1 @@
Subproject commit 939dcd10fe7f33d567e4bf327b284dcb8a9bdd63 Subproject commit ccf75ede4ae02aa184a51f565ee0b4f0ace29c6f

View file

@ -2,6 +2,8 @@
namespace Misuzu; namespace Misuzu;
use ZipArchive; use ZipArchive;
use Index\XString;
use Index\IO\FileStream;
use Misuzu\Users\User; use Misuzu\Users\User;
use Misuzu\Users\UserSession; use Misuzu\Users\UserSession;
@ -12,18 +14,91 @@ if(!UserSession::hasCurrent()) {
return; return;
} }
function db_to_zip(ZipArchive $archive, int $userId, string $filename, string $query, int $params = 1): void { $dbConn = $msz->getDbConn();
$prepare = DB::prepare($query);
if($params < 2) { function db_to_zip(ZipArchive $archive, int $userId, string $baseName, array $fieldInfos, string $userIdField = 'user_id'): string {
$prepare->bind('user_id', $userId); global $dbConn;
} else {
for($i = 1; $i <= $params; $i++) { $fields = [];
$prepare->bind('user_id_' . $i, $userId);
foreach($fieldInfos as $key => $fieldInfo) {
$fieldInfo = explode(':', $fieldInfo);
$fieldInfo = [
'name' => $fieldInfo[0],
'type' => $fieldInfo[1] ?? 'n',
'nullable' => ($fieldInfo[2] ?? '') === 'n',
];
$fieldInfo['is_null'] = $fieldInfo['type'] === 'n';
if(!$fieldInfo['is_null']) {
$selectField = $fieldInfo['name'];
if($fieldInfo['type'] === 'a')
$selectField = sprintf('INET6_NTOA(%s)', $selectField);
elseif($fieldInfo['type'] === 't')
$selectField = sprintf('UNIX_TIMESTAMP(%s)', $selectField);
$fields[] = $selectField;
} }
$fieldInfos[$key] = $fieldInfo;
} }
$archive->addFromString($filename, json_encode($prepare->fetchAll(), JSON_PRETTY_PRINT)); $tmpName = sys_get_temp_dir() . DIRECTORY_SEPARATOR . sprintf('msz-user-data-%d-%s-%s.tmp', $userId, $baseName, XString::random(8));
$tmpStream = FileStream::newWrite($tmpName);
try {
$stmt = $dbConn->prepare(sprintf('SELECT %s FROM msz_%s WHERE %s = ?', implode(', ', $fields), $baseName, $userIdField));
$stmt->addParameter(1, $userId);
$stmt->execute();
$result = $stmt->getResult();
while($result->next()) {
$args = -1;
$row = [];
foreach($fieldInfos as $fieldInfo) {
$fieldValue = null;
if(!$fieldInfo['is_null']) {
++$args;
if(!$fieldInfo['nullable'] || !$result->isNull($args)) {
if($fieldInfo['type'] === 's' || $fieldInfo['type'] === 'a' || $fieldInfo['type'] === 'j') {
$fieldValue = $result->getString($args);
if($fieldInfo['type'] === 'j')
$fieldValue = json_decode($fieldValue);
} elseif($fieldInfo['type'] === 't' || $fieldInfo['type'] === 'b' || $fieldInfo['type'] === 'i') {
$fieldValue = $result->getInteger($args);
if($fieldInfo['type'] === 'b')
$fieldValue = $fieldValue !== 0;
elseif($fieldInfo['type'] === 't') {
$fieldValue = date(DATE_ATOM, $fieldValue);
if(str_ends_with($fieldValue, '+00:00'))
$fieldValue = substr($fieldValue, 0, -6) . 'Z';
}
} else {
$fieldValue = '?';
}
}
}
$row[$fieldInfo['name']] = $fieldValue;
}
$tmpStream->write(json_encode($row, JSON_INVALID_UTF8_SUBSTITUTE));
$tmpStream->write("\n");
}
} finally {
$tmpStream->flush();
$tmpStream->close();
}
$archive->addFile($tmpName, $baseName . '.jsonl');
return $tmpName;
} }
$errors = []; $errors = [];
@ -32,7 +107,7 @@ $currentUserId = $currentUser->getId();
if(isset($_POST['action']) && is_string($_POST['action'])) { if(isset($_POST['action']) && is_string($_POST['action'])) {
if(isset($_POST['password']) && is_string($_POST['password']) if(isset($_POST['password']) && is_string($_POST['password'])
&& $currentUser->checkPassword($_POST['password'] ?? '')) { && ($currentUser->checkPassword($_POST['password'] ?? ''))) {
switch($_POST['action']) { switch($_POST['action']) {
case 'data': case 'data':
$msz->createAuditLog('PERSONAL_DATA_DOWNLOAD'); $msz->createAuditLog('PERSONAL_DATA_DOWNLOAD');
@ -44,26 +119,36 @@ if(isset($_POST['action']) && is_string($_POST['action'])) {
if(!is_file($filePath)) { if(!is_file($filePath)) {
if($archive->open($filePath, ZipArchive::CREATE | ZIPARCHIVE::OVERWRITE) === true) { if($archive->open($filePath, ZipArchive::CREATE | ZIPARCHIVE::OVERWRITE) === true) {
db_to_zip($archive, $currentUserId, 'audit_log.json', 'SELECT *, INET6_NTOA(`log_ip`) AS `log_ip` FROM `msz_audit_log` WHERE `user_id` = :user_id'); $tmpFiles = [];
db_to_zip($archive, $currentUserId, 'auth_tfa.json', 'SELECT * FROM `msz_auth_tfa` WHERE `user_id` = :user_id');
db_to_zip($archive, $currentUserId, 'changelog_changes.json', 'SELECT * FROM `msz_changelog_changes` WHERE `user_id` = :user_id');
db_to_zip($archive, $currentUserId, 'comments_posts.json', 'SELECT * FROM `msz_comments_posts` WHERE `user_id` = :user_id');
db_to_zip($archive, $currentUserId, 'comments_votes.json', 'SELECT * FROM `msz_comments_votes` WHERE `user_id` = :user_id');
db_to_zip($archive, $currentUserId, 'forum_permissions.json', 'SELECT * FROM `msz_forum_permissions` WHERE `user_id` = :user_id');
db_to_zip($archive, $currentUserId, 'forum_posts.json', 'SELECT *, INET6_NTOA(`post_ip`) AS `post_ip` FROM `msz_forum_posts` WHERE `user_id` = :user_id');
db_to_zip($archive, $currentUserId, 'forum_topics.json', 'SELECT * FROM `msz_forum_topics` WHERE `user_id` = :user_id');
db_to_zip($archive, $currentUserId, 'forum_topics_track.json', 'SELECT * FROM `msz_forum_topics_track` WHERE `user_id` = :user_id');
db_to_zip($archive, $currentUserId, 'login_attempts.json', 'SELECT *, INET6_NTOA(`attempt_ip`) AS `attempt_ip` FROM `msz_login_attempts` WHERE `user_id` = :user_id');
db_to_zip($archive, $currentUserId, 'news_posts.json', 'SELECT * FROM `msz_news_posts` WHERE `user_id` = :user_id');
db_to_zip($archive, $currentUserId, 'permissions.json', 'SELECT * FROM `msz_permissions` WHERE `user_id` = :user_id');
db_to_zip($archive, $currentUserId, 'profile_fields_values.json', 'SELECT * FROM `msz_profile_fields_values` WHERE `user_id` = :user_id');
db_to_zip($archive, $currentUserId, 'sessions.json', 'SELECT *, INET6_NTOA(`session_ip`) AS `session_ip`, INET6_NTOA(`session_ip_last`) AS `session_ip_last` FROM `msz_sessions` WHERE `user_id` = :user_id');
db_to_zip($archive, $currentUserId, 'users.json', 'SELECT *, NULL AS `password`, NULL AS `user_totp_key`, INET6_NTOA(`register_ip`) AS `register_ip`, INET6_NTOA(`last_ip`) AS `last_ip` FROM `msz_users` WHERE `user_id` = :user_id');
db_to_zip($archive, $currentUserId, 'users_password_resets.json', 'SELECT *, INET6_NTOA(`reset_ip`) AS `reset_ip` FROM `msz_users_password_resets` WHERE `user_id` = :user_id');
db_to_zip($archive, $currentUserId, 'user_roles.json', 'SELECT * FROM `msz_user_roles` WHERE `user_id` = :user_id');
db_to_zip($archive, $currentUserId, 'user_warnings.json', 'SELECT *, INET6_NTOA(`user_ip`) AS `user_ip`, NULL AS `issuer_id`, NULL AS `issuer_ip`, NULL AS `warning_note_private` FROM `msz_user_warnings` WHERE `user_id` = :user_id');
$archive->close(); try {
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'audit_log', ['user_id:s:n', 'log_action:s', 'log_params:j', 'log_created:t', 'log_ip:a:n', 'log_country:s']);
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'auth_tfa', ['user_id:s', 'tfa_token:n', 'tfa_created:t']);
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'changelog_changes', ['change_id:s', 'user_id:s:n', 'change_action:s:n', 'change_created:t', 'change_log:s', 'change_text:s:n']);
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'comments_categories', ['category_id:s', 'category_name:s', 'owner_id:s:n', 'category_created:t', 'category_locked:t:n'], 'owner_id');
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'comments_posts', ['comment_id:s', 'category_id:s', 'user_id:s:n', 'comment_reply_to:s:n', 'comment_text:s', 'comment_created:t', 'comment_pinned:t:n', 'comment_edited:t:n', 'comment_deleted:t:n']);
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'comments_votes', ['comment_id:s', 'user_id:s', 'comment_vote:i']);
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_permissions', ['user_id:s:n', 'role_id:s:n', 'forum_id:s', 'forum_perms_allow:i', 'forum_perms_deny:i']);
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_posts', ['post_id:s', 'topic_id:s', 'forum_id:s', 'user_id:s:n', 'post_ip:a', 'post_text:s', 'post_parse:i', 'post_display_signature:b', 'post_created:t', 'post_edited:t:n', 'post_deleted:t:n']);
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_topics', ['topic_id:s', 'forum_id:s', 'user_id:s:n', 'topic_type:i', 'topic_title:s', 'topic_count_views:i', 'topic_created:t', 'topic_bumped:t', 'topic_deleted:t:n', 'topic_locked:t:n']);
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_topics_redirects', ['topic_id:s', 'user_id:s:n', 'topic_redir_url:s', 'topic_redir_created:t']);
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_topics_track', ['user_id:s', 'topic_id:s', 'forum_id:s', 'track_last_read:t']);
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'login_attempts', ['user_id:s:n', 'attempt_success:b', 'attempt_ip:a', 'attempt_country:s', 'attempt_created:t', 'attempt_user_agent:s']);
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'news_posts', ['post_id:s', 'category_id:s', 'user_id:s:n', 'comment_section_id:s:n', 'post_is_featured:b', 'post_title:s', 'post_text:s', 'post_scheduled:t', 'post_created:t', 'post_updated:t', 'post_deleted:t:n']);
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'permissions', ['user_id:s:n', 'role_id:s:n', 'general_perms_allow:i', 'general_perms_deny:i', 'user_perms_allow:i', 'user_perms_deny:i', 'changelog_perms_allow:i', 'changelog_perms_deny:i', 'news_perms_allow:i', 'news_perms_deny:i', 'forum_perms_allow:i', 'forum_perms_deny:i', 'comments_perms_allow:i', 'comments_perms_deny:i']);
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'profile_fields_values', ['field_id:s', 'user_id:s', 'format_id:s', 'field_value:s']);
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'sessions', ['session_id:s', 'user_id:s', 'session_key:n', 'session_ip:a', 'session_ip_last:a:n', 'session_user_agent:s', 'session_country:s', 'session_expires:t', 'session_expires_bump:b', 'session_created:t', 'session_active:t:n']);
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'users', ['user_id:s', 'username:s', 'password:n', 'email:s', 'register_ip:a', 'last_ip:a', 'user_super:b', 'user_country:s', 'user_colour:i:n', 'user_created:t', 'user_active:t:n', 'user_deleted:t:n', 'display_role:s:n', 'user_totp_key:n', 'user_about_content:s:n', 'user_about_parser:i', 'user_signature_content:s:n', 'user_signature_parser:i', 'user_birthdate:s:n', 'user_background_settings:i:n', 'user_title:s:n']);
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'users_password_resets', ['user_id:s', 'reset_ip:a', 'reset_requested:t', 'verification_code:n']);
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'user_roles', ['user_id:s', 'role_id:s']);
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'user_warnings', ['warning_id:s', 'user_id:s', 'user_ip:a', 'issuer_id:n', 'issuer_ip:n', 'warning_created:t', 'warning_duration:t:n', 'warning_type:i', 'warning_note:s', 'warning_note_private:s:n']);
$archive->close();
} finally {
foreach($tmpFiles as $tmpFile)
if(is_file($tmpFile))
unlink($tmpFile);
}
} else { } else {
$errors[] = 'Something went wrong while creating your account archive.'; $errors[] = 'Something went wrong while creating your account archive.';
break; break;

View file

@ -10,7 +10,7 @@
{{ input_csrf() }} {{ input_csrf() }}
<div class="settings__description"> <div class="settings__description">
<p>Here you can request raw json files containing pretty much all data relating to your account. Moderator identities are concealed and password hashes are removed from the output.</p> <p>Here you can request raw json files containing pretty much all data relating to your account. Moderator identities are concealed and password hashes are removed from the output. Files in the archive will be in <a href="https://jsonlines.org/" target="_blank" rel="noopener" class="link">JSON Lines</a> format.</p>
</div> </div>
<div class="settings__data__content"> <div class="settings__data__content">