Fixed account data export memory usage.
This commit is contained in:
parent
1a11a8f8ba
commit
e5d9128cd0
3 changed files with 115 additions and 30 deletions
|
|
@ -1 +1 @@
|
|||
Subproject commit 939dcd10fe7f33d567e4bf327b284dcb8a9bdd63
|
||||
Subproject commit ccf75ede4ae02aa184a51f565ee0b4f0ace29c6f
|
||||
|
|
@ -2,6 +2,8 @@
|
|||
namespace Misuzu;
|
||||
|
||||
use ZipArchive;
|
||||
use Index\XString;
|
||||
use Index\IO\FileStream;
|
||||
use Misuzu\Users\User;
|
||||
use Misuzu\Users\UserSession;
|
||||
|
||||
|
|
@ -12,18 +14,91 @@ if(!UserSession::hasCurrent()) {
|
|||
return;
|
||||
}
|
||||
|
||||
function db_to_zip(ZipArchive $archive, int $userId, string $filename, string $query, int $params = 1): void {
|
||||
$prepare = DB::prepare($query);
|
||||
$dbConn = $msz->getDbConn();
|
||||
|
||||
if($params < 2) {
|
||||
$prepare->bind('user_id', $userId);
|
||||
} else {
|
||||
for($i = 1; $i <= $params; $i++) {
|
||||
$prepare->bind('user_id_' . $i, $userId);
|
||||
function db_to_zip(ZipArchive $archive, int $userId, string $baseName, array $fieldInfos, string $userIdField = 'user_id'): string {
|
||||
global $dbConn;
|
||||
|
||||
$fields = [];
|
||||
|
||||
foreach($fieldInfos as $key => $fieldInfo) {
|
||||
$fieldInfo = explode(':', $fieldInfo);
|
||||
$fieldInfo = [
|
||||
'name' => $fieldInfo[0],
|
||||
'type' => $fieldInfo[1] ?? 'n',
|
||||
'nullable' => ($fieldInfo[2] ?? '') === 'n',
|
||||
];
|
||||
$fieldInfo['is_null'] = $fieldInfo['type'] === 'n';
|
||||
|
||||
if(!$fieldInfo['is_null']) {
|
||||
$selectField = $fieldInfo['name'];
|
||||
|
||||
if($fieldInfo['type'] === 'a')
|
||||
$selectField = sprintf('INET6_NTOA(%s)', $selectField);
|
||||
elseif($fieldInfo['type'] === 't')
|
||||
$selectField = sprintf('UNIX_TIMESTAMP(%s)', $selectField);
|
||||
|
||||
$fields[] = $selectField;
|
||||
}
|
||||
|
||||
$fieldInfos[$key] = $fieldInfo;
|
||||
}
|
||||
|
||||
$archive->addFromString($filename, json_encode($prepare->fetchAll(), JSON_PRETTY_PRINT));
|
||||
$tmpName = sys_get_temp_dir() . DIRECTORY_SEPARATOR . sprintf('msz-user-data-%d-%s-%s.tmp', $userId, $baseName, XString::random(8));
|
||||
$tmpStream = FileStream::newWrite($tmpName);
|
||||
|
||||
try {
|
||||
$stmt = $dbConn->prepare(sprintf('SELECT %s FROM msz_%s WHERE %s = ?', implode(', ', $fields), $baseName, $userIdField));
|
||||
$stmt->addParameter(1, $userId);
|
||||
$stmt->execute();
|
||||
$result = $stmt->getResult();
|
||||
|
||||
while($result->next()) {
|
||||
$args = -1;
|
||||
$row = [];
|
||||
|
||||
foreach($fieldInfos as $fieldInfo) {
|
||||
$fieldValue = null;
|
||||
|
||||
if(!$fieldInfo['is_null']) {
|
||||
++$args;
|
||||
|
||||
if(!$fieldInfo['nullable'] || !$result->isNull($args)) {
|
||||
if($fieldInfo['type'] === 's' || $fieldInfo['type'] === 'a' || $fieldInfo['type'] === 'j') {
|
||||
$fieldValue = $result->getString($args);
|
||||
|
||||
if($fieldInfo['type'] === 'j')
|
||||
$fieldValue = json_decode($fieldValue);
|
||||
} elseif($fieldInfo['type'] === 't' || $fieldInfo['type'] === 'b' || $fieldInfo['type'] === 'i') {
|
||||
$fieldValue = $result->getInteger($args);
|
||||
|
||||
if($fieldInfo['type'] === 'b')
|
||||
$fieldValue = $fieldValue !== 0;
|
||||
elseif($fieldInfo['type'] === 't') {
|
||||
$fieldValue = date(DATE_ATOM, $fieldValue);
|
||||
if(str_ends_with($fieldValue, '+00:00'))
|
||||
$fieldValue = substr($fieldValue, 0, -6) . 'Z';
|
||||
}
|
||||
} else {
|
||||
$fieldValue = '?';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$row[$fieldInfo['name']] = $fieldValue;
|
||||
}
|
||||
|
||||
$tmpStream->write(json_encode($row, JSON_INVALID_UTF8_SUBSTITUTE));
|
||||
$tmpStream->write("\n");
|
||||
}
|
||||
} finally {
|
||||
$tmpStream->flush();
|
||||
$tmpStream->close();
|
||||
}
|
||||
|
||||
$archive->addFile($tmpName, $baseName . '.jsonl');
|
||||
|
||||
return $tmpName;
|
||||
}
|
||||
|
||||
$errors = [];
|
||||
|
|
@ -32,7 +107,7 @@ $currentUserId = $currentUser->getId();
|
|||
|
||||
if(isset($_POST['action']) && is_string($_POST['action'])) {
|
||||
if(isset($_POST['password']) && is_string($_POST['password'])
|
||||
&& $currentUser->checkPassword($_POST['password'] ?? '')) {
|
||||
&& ($currentUser->checkPassword($_POST['password'] ?? ''))) {
|
||||
switch($_POST['action']) {
|
||||
case 'data':
|
||||
$msz->createAuditLog('PERSONAL_DATA_DOWNLOAD');
|
||||
|
|
@ -44,26 +119,36 @@ if(isset($_POST['action']) && is_string($_POST['action'])) {
|
|||
|
||||
if(!is_file($filePath)) {
|
||||
if($archive->open($filePath, ZipArchive::CREATE | ZIPARCHIVE::OVERWRITE) === true) {
|
||||
db_to_zip($archive, $currentUserId, 'audit_log.json', 'SELECT *, INET6_NTOA(`log_ip`) AS `log_ip` FROM `msz_audit_log` WHERE `user_id` = :user_id');
|
||||
db_to_zip($archive, $currentUserId, 'auth_tfa.json', 'SELECT * FROM `msz_auth_tfa` WHERE `user_id` = :user_id');
|
||||
db_to_zip($archive, $currentUserId, 'changelog_changes.json', 'SELECT * FROM `msz_changelog_changes` WHERE `user_id` = :user_id');
|
||||
db_to_zip($archive, $currentUserId, 'comments_posts.json', 'SELECT * FROM `msz_comments_posts` WHERE `user_id` = :user_id');
|
||||
db_to_zip($archive, $currentUserId, 'comments_votes.json', 'SELECT * FROM `msz_comments_votes` WHERE `user_id` = :user_id');
|
||||
db_to_zip($archive, $currentUserId, 'forum_permissions.json', 'SELECT * FROM `msz_forum_permissions` WHERE `user_id` = :user_id');
|
||||
db_to_zip($archive, $currentUserId, 'forum_posts.json', 'SELECT *, INET6_NTOA(`post_ip`) AS `post_ip` FROM `msz_forum_posts` WHERE `user_id` = :user_id');
|
||||
db_to_zip($archive, $currentUserId, 'forum_topics.json', 'SELECT * FROM `msz_forum_topics` WHERE `user_id` = :user_id');
|
||||
db_to_zip($archive, $currentUserId, 'forum_topics_track.json', 'SELECT * FROM `msz_forum_topics_track` WHERE `user_id` = :user_id');
|
||||
db_to_zip($archive, $currentUserId, 'login_attempts.json', 'SELECT *, INET6_NTOA(`attempt_ip`) AS `attempt_ip` FROM `msz_login_attempts` WHERE `user_id` = :user_id');
|
||||
db_to_zip($archive, $currentUserId, 'news_posts.json', 'SELECT * FROM `msz_news_posts` WHERE `user_id` = :user_id');
|
||||
db_to_zip($archive, $currentUserId, 'permissions.json', 'SELECT * FROM `msz_permissions` WHERE `user_id` = :user_id');
|
||||
db_to_zip($archive, $currentUserId, 'profile_fields_values.json', 'SELECT * FROM `msz_profile_fields_values` WHERE `user_id` = :user_id');
|
||||
db_to_zip($archive, $currentUserId, 'sessions.json', 'SELECT *, INET6_NTOA(`session_ip`) AS `session_ip`, INET6_NTOA(`session_ip_last`) AS `session_ip_last` FROM `msz_sessions` WHERE `user_id` = :user_id');
|
||||
db_to_zip($archive, $currentUserId, 'users.json', 'SELECT *, NULL AS `password`, NULL AS `user_totp_key`, INET6_NTOA(`register_ip`) AS `register_ip`, INET6_NTOA(`last_ip`) AS `last_ip` FROM `msz_users` WHERE `user_id` = :user_id');
|
||||
db_to_zip($archive, $currentUserId, 'users_password_resets.json', 'SELECT *, INET6_NTOA(`reset_ip`) AS `reset_ip` FROM `msz_users_password_resets` WHERE `user_id` = :user_id');
|
||||
db_to_zip($archive, $currentUserId, 'user_roles.json', 'SELECT * FROM `msz_user_roles` WHERE `user_id` = :user_id');
|
||||
db_to_zip($archive, $currentUserId, 'user_warnings.json', 'SELECT *, INET6_NTOA(`user_ip`) AS `user_ip`, NULL AS `issuer_id`, NULL AS `issuer_ip`, NULL AS `warning_note_private` FROM `msz_user_warnings` WHERE `user_id` = :user_id');
|
||||
$tmpFiles = [];
|
||||
|
||||
$archive->close();
|
||||
try {
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'audit_log', ['user_id:s:n', 'log_action:s', 'log_params:j', 'log_created:t', 'log_ip:a:n', 'log_country:s']);
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'auth_tfa', ['user_id:s', 'tfa_token:n', 'tfa_created:t']);
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'changelog_changes', ['change_id:s', 'user_id:s:n', 'change_action:s:n', 'change_created:t', 'change_log:s', 'change_text:s:n']);
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'comments_categories', ['category_id:s', 'category_name:s', 'owner_id:s:n', 'category_created:t', 'category_locked:t:n'], 'owner_id');
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'comments_posts', ['comment_id:s', 'category_id:s', 'user_id:s:n', 'comment_reply_to:s:n', 'comment_text:s', 'comment_created:t', 'comment_pinned:t:n', 'comment_edited:t:n', 'comment_deleted:t:n']);
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'comments_votes', ['comment_id:s', 'user_id:s', 'comment_vote:i']);
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_permissions', ['user_id:s:n', 'role_id:s:n', 'forum_id:s', 'forum_perms_allow:i', 'forum_perms_deny:i']);
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_posts', ['post_id:s', 'topic_id:s', 'forum_id:s', 'user_id:s:n', 'post_ip:a', 'post_text:s', 'post_parse:i', 'post_display_signature:b', 'post_created:t', 'post_edited:t:n', 'post_deleted:t:n']);
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_topics', ['topic_id:s', 'forum_id:s', 'user_id:s:n', 'topic_type:i', 'topic_title:s', 'topic_count_views:i', 'topic_created:t', 'topic_bumped:t', 'topic_deleted:t:n', 'topic_locked:t:n']);
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_topics_redirects', ['topic_id:s', 'user_id:s:n', 'topic_redir_url:s', 'topic_redir_created:t']);
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_topics_track', ['user_id:s', 'topic_id:s', 'forum_id:s', 'track_last_read:t']);
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'login_attempts', ['user_id:s:n', 'attempt_success:b', 'attempt_ip:a', 'attempt_country:s', 'attempt_created:t', 'attempt_user_agent:s']);
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'news_posts', ['post_id:s', 'category_id:s', 'user_id:s:n', 'comment_section_id:s:n', 'post_is_featured:b', 'post_title:s', 'post_text:s', 'post_scheduled:t', 'post_created:t', 'post_updated:t', 'post_deleted:t:n']);
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'permissions', ['user_id:s:n', 'role_id:s:n', 'general_perms_allow:i', 'general_perms_deny:i', 'user_perms_allow:i', 'user_perms_deny:i', 'changelog_perms_allow:i', 'changelog_perms_deny:i', 'news_perms_allow:i', 'news_perms_deny:i', 'forum_perms_allow:i', 'forum_perms_deny:i', 'comments_perms_allow:i', 'comments_perms_deny:i']);
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'profile_fields_values', ['field_id:s', 'user_id:s', 'format_id:s', 'field_value:s']);
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'sessions', ['session_id:s', 'user_id:s', 'session_key:n', 'session_ip:a', 'session_ip_last:a:n', 'session_user_agent:s', 'session_country:s', 'session_expires:t', 'session_expires_bump:b', 'session_created:t', 'session_active:t:n']);
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'users', ['user_id:s', 'username:s', 'password:n', 'email:s', 'register_ip:a', 'last_ip:a', 'user_super:b', 'user_country:s', 'user_colour:i:n', 'user_created:t', 'user_active:t:n', 'user_deleted:t:n', 'display_role:s:n', 'user_totp_key:n', 'user_about_content:s:n', 'user_about_parser:i', 'user_signature_content:s:n', 'user_signature_parser:i', 'user_birthdate:s:n', 'user_background_settings:i:n', 'user_title:s:n']);
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'users_password_resets', ['user_id:s', 'reset_ip:a', 'reset_requested:t', 'verification_code:n']);
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'user_roles', ['user_id:s', 'role_id:s']);
|
||||
$tmpFiles[] = db_to_zip($archive, $currentUserId, 'user_warnings', ['warning_id:s', 'user_id:s', 'user_ip:a', 'issuer_id:n', 'issuer_ip:n', 'warning_created:t', 'warning_duration:t:n', 'warning_type:i', 'warning_note:s', 'warning_note_private:s:n']);
|
||||
|
||||
$archive->close();
|
||||
} finally {
|
||||
foreach($tmpFiles as $tmpFile)
|
||||
if(is_file($tmpFile))
|
||||
unlink($tmpFile);
|
||||
}
|
||||
} else {
|
||||
$errors[] = 'Something went wrong while creating your account archive.';
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@
|
|||
{{ input_csrf() }}
|
||||
|
||||
<div class="settings__description">
|
||||
<p>Here you can request raw json files containing pretty much all data relating to your account. Moderator identities are concealed and password hashes are removed from the output.</p>
|
||||
<p>Here you can request raw json files containing pretty much all data relating to your account. Moderator identities are concealed and password hashes are removed from the output. Files in the archive will be in <a href="https://jsonlines.org/" target="_blank" rel="noopener" class="link">JSON Lines</a> format.</p>
|
||||
</div>
|
||||
|
||||
<div class="settings__data__content">
|
||||
|
|
|
|||
Loading…
Reference in a new issue