Fixed account data export memory usage.

lib/index

@@ -1 +1 @@
-Subproject commit 939dcd10fe7f33d567e4bf327b284dcb8a9bdd63
+Subproject commit ccf75ede4ae02aa184a51f565ee0b4f0ace29c6f

public/settings/data.php

@@ -2,6 +2,8 @@
 namespace Misuzu;
 
 use ZipArchive;
+use Index\XString;
+use Index\IO\FileStream;
 use Misuzu\Users\User;
 use Misuzu\Users\UserSession;
 
@@ -12,18 +14,91 @@ if(!UserSession::hasCurrent()) {
     return;
 }
 
-function db_to_zip(ZipArchive $archive, int $userId, string $filename, string $query, int $params = 1): void {
+$dbConn = $msz->getDbConn();
-    $prepare = DB::prepare($query);
 
-    if($params < 2) {
+function db_to_zip(ZipArchive $archive, int $userId, string $baseName, array $fieldInfos, string $userIdField = 'user_id'): string {
-        $prepare->bind('user_id', $userId);
+    global $dbConn;
-    } else {
+
-        for($i = 1; $i <= $params; $i++) {
+    $fields = [];
-            $prepare->bind('user_id_' . $i, $userId);
+
+    foreach($fieldInfos as $key => $fieldInfo) {
+        $fieldInfo = explode(':', $fieldInfo);
+        $fieldInfo = [
+            'name' => $fieldInfo[0],
+            'type' => $fieldInfo[1] ?? 'n',
+            'nullable' => ($fieldInfo[2] ?? '') === 'n',
+        ];
+        $fieldInfo['is_null'] = $fieldInfo['type'] === 'n';
+
+        if(!$fieldInfo['is_null']) {
+            $selectField = $fieldInfo['name'];
+
+            if($fieldInfo['type'] === 'a')
+                $selectField = sprintf('INET6_NTOA(%s)', $selectField);
+            elseif($fieldInfo['type'] === 't')
+                $selectField = sprintf('UNIX_TIMESTAMP(%s)', $selectField);
+
+            $fields[] = $selectField;
         }
+
+        $fieldInfos[$key] = $fieldInfo;
     }
 
-    $archive->addFromString($filename, json_encode($prepare->fetchAll(), JSON_PRETTY_PRINT));
+    $tmpName = sys_get_temp_dir() . DIRECTORY_SEPARATOR . sprintf('msz-user-data-%d-%s-%s.tmp', $userId, $baseName, XString::random(8));
+    $tmpStream = FileStream::newWrite($tmpName);
+
+    try {
+        $stmt = $dbConn->prepare(sprintf('SELECT %s FROM msz_%s WHERE %s = ?', implode(', ', $fields), $baseName, $userIdField));
+        $stmt->addParameter(1, $userId);
+        $stmt->execute();
+        $result = $stmt->getResult();
+
+        while($result->next()) {
+            $args = -1;
+            $row = [];
+
+            foreach($fieldInfos as $fieldInfo) {
+                $fieldValue = null;
+
+                if(!$fieldInfo['is_null']) {
+                    ++$args;
+
+                    if(!$fieldInfo['nullable'] || !$result->isNull($args)) {
+                        if($fieldInfo['type'] === 's' || $fieldInfo['type'] === 'a' || $fieldInfo['type'] === 'j') {
+                            $fieldValue = $result->getString($args);
+
+                            if($fieldInfo['type'] === 'j')
+                                $fieldValue = json_decode($fieldValue);
+                        } elseif($fieldInfo['type'] === 't' || $fieldInfo['type'] === 'b' || $fieldInfo['type'] === 'i') {
+                            $fieldValue = $result->getInteger($args);
+
+                            if($fieldInfo['type'] === 'b')
+                                $fieldValue = $fieldValue !== 0;
+                            elseif($fieldInfo['type'] === 't') {
+                                $fieldValue = date(DATE_ATOM, $fieldValue);
+                                if(str_ends_with($fieldValue, '+00:00'))
+                                    $fieldValue = substr($fieldValue, 0, -6) . 'Z';
+                            }
+                        } else {
+                            $fieldValue = '?';
+                        }
+                    }
+                }
+
+                $row[$fieldInfo['name']] = $fieldValue;
+            }
+
+            $tmpStream->write(json_encode($row, JSON_INVALID_UTF8_SUBSTITUTE));
+            $tmpStream->write("\n");
+        }
+    } finally {
+        $tmpStream->flush();
+        $tmpStream->close();
+    }
+
+    $archive->addFile($tmpName, $baseName . '.jsonl');
+
+    return $tmpName;
 }
 
 $errors = [];
@@ -32,7 +107,7 @@ $currentUserId = $currentUser->getId();
 
 if(isset($_POST['action']) && is_string($_POST['action'])) {
     if(isset($_POST['password']) && is_string($_POST['password'])
-        && $currentUser->checkPassword($_POST['password'] ?? '')) {
+        && ($currentUser->checkPassword($_POST['password'] ?? ''))) {
         switch($_POST['action']) {
             case 'data':
                 $msz->createAuditLog('PERSONAL_DATA_DOWNLOAD');
@@ -44,26 +119,36 @@ if(isset($_POST['action']) && is_string($_POST['action'])) {
 
                 if(!is_file($filePath)) {
                     if($archive->open($filePath, ZipArchive::CREATE | ZIPARCHIVE::OVERWRITE) === true) {
-                        db_to_zip($archive, $currentUserId, 'audit_log.json',               'SELECT *, INET6_NTOA(`log_ip`) AS `log_ip`                                                                                             FROM `msz_audit_log`                WHERE `user_id` = :user_id');
+                        $tmpFiles = [];
-                        db_to_zip($archive, $currentUserId, 'auth_tfa.json',                'SELECT *                                                                                                                               FROM `msz_auth_tfa`                 WHERE `user_id` = :user_id');
-                        db_to_zip($archive, $currentUserId, 'changelog_changes.json',       'SELECT *                                                                                                                               FROM `msz_changelog_changes`        WHERE `user_id` = :user_id');
-                        db_to_zip($archive, $currentUserId, 'comments_posts.json',          'SELECT *                                                                                                                               FROM `msz_comments_posts`           WHERE `user_id` = :user_id');
-                        db_to_zip($archive, $currentUserId, 'comments_votes.json',          'SELECT *                                                                                                                               FROM `msz_comments_votes`           WHERE `user_id` = :user_id');
-                        db_to_zip($archive, $currentUserId, 'forum_permissions.json',       'SELECT *                                                                                                                               FROM `msz_forum_permissions`        WHERE `user_id` = :user_id');
-                        db_to_zip($archive, $currentUserId, 'forum_posts.json',             'SELECT *, INET6_NTOA(`post_ip`) AS `post_ip`                                                                                           FROM `msz_forum_posts`              WHERE `user_id` = :user_id');
-                        db_to_zip($archive, $currentUserId, 'forum_topics.json',            'SELECT *                                                                                                                               FROM `msz_forum_topics`             WHERE `user_id` = :user_id');
-                        db_to_zip($archive, $currentUserId, 'forum_topics_track.json',      'SELECT *                                                                                                                               FROM `msz_forum_topics_track`       WHERE `user_id` = :user_id');
-                        db_to_zip($archive, $currentUserId, 'login_attempts.json',          'SELECT *, INET6_NTOA(`attempt_ip`) AS `attempt_ip`                                                                                     FROM `msz_login_attempts`           WHERE `user_id` = :user_id');
-                        db_to_zip($archive, $currentUserId, 'news_posts.json',              'SELECT *                                                                                                                               FROM `msz_news_posts`               WHERE `user_id` = :user_id');
-                        db_to_zip($archive, $currentUserId, 'permissions.json',             'SELECT *                                                                                                                               FROM `msz_permissions`              WHERE `user_id` = :user_id');
-                        db_to_zip($archive, $currentUserId, 'profile_fields_values.json',   'SELECT *                                                                                                                               FROM `msz_profile_fields_values`    WHERE `user_id` = :user_id');
-                        db_to_zip($archive, $currentUserId, 'sessions.json',                'SELECT *, INET6_NTOA(`session_ip`) AS `session_ip`, INET6_NTOA(`session_ip_last`) AS `session_ip_last`                                 FROM `msz_sessions`                 WHERE `user_id` = :user_id');
-                        db_to_zip($archive, $currentUserId, 'users.json',                   'SELECT *, NULL AS `password`, NULL AS `user_totp_key`, INET6_NTOA(`register_ip`) AS `register_ip`, INET6_NTOA(`last_ip`) AS `last_ip`  FROM `msz_users`                    WHERE `user_id` = :user_id');
-                        db_to_zip($archive, $currentUserId, 'users_password_resets.json',   'SELECT *, INET6_NTOA(`reset_ip`) AS `reset_ip`                                                                                         FROM `msz_users_password_resets`    WHERE `user_id` = :user_id');
-                        db_to_zip($archive, $currentUserId, 'user_roles.json',              'SELECT *                                                                                                                               FROM `msz_user_roles`               WHERE `user_id` = :user_id');
-                        db_to_zip($archive, $currentUserId, 'user_warnings.json',           'SELECT *, INET6_NTOA(`user_ip`) AS `user_ip`, NULL AS `issuer_id`, NULL AS `issuer_ip`, NULL AS `warning_note_private`                 FROM `msz_user_warnings`            WHERE `user_id` = :user_id');
 
-                        $archive->close();
+                        try {
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'audit_log',              ['user_id:s:n', 'log_action:s', 'log_params:j', 'log_created:t', 'log_ip:a:n', 'log_country:s']);
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'auth_tfa',               ['user_id:s', 'tfa_token:n', 'tfa_created:t']);
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'changelog_changes',      ['change_id:s', 'user_id:s:n', 'change_action:s:n', 'change_created:t', 'change_log:s', 'change_text:s:n']);
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'comments_categories',    ['category_id:s', 'category_name:s', 'owner_id:s:n', 'category_created:t', 'category_locked:t:n'], 'owner_id');
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'comments_posts',         ['comment_id:s', 'category_id:s', 'user_id:s:n', 'comment_reply_to:s:n', 'comment_text:s', 'comment_created:t', 'comment_pinned:t:n', 'comment_edited:t:n', 'comment_deleted:t:n']);
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'comments_votes',         ['comment_id:s', 'user_id:s', 'comment_vote:i']);
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_permissions',      ['user_id:s:n', 'role_id:s:n', 'forum_id:s', 'forum_perms_allow:i', 'forum_perms_deny:i']);
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_posts',            ['post_id:s', 'topic_id:s', 'forum_id:s', 'user_id:s:n', 'post_ip:a', 'post_text:s', 'post_parse:i', 'post_display_signature:b', 'post_created:t', 'post_edited:t:n', 'post_deleted:t:n']);
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_topics',           ['topic_id:s', 'forum_id:s', 'user_id:s:n', 'topic_type:i', 'topic_title:s', 'topic_count_views:i', 'topic_created:t', 'topic_bumped:t', 'topic_deleted:t:n', 'topic_locked:t:n']);
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_topics_redirects', ['topic_id:s', 'user_id:s:n', 'topic_redir_url:s', 'topic_redir_created:t']);
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'forum_topics_track',     ['user_id:s', 'topic_id:s', 'forum_id:s', 'track_last_read:t']);
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'login_attempts',         ['user_id:s:n', 'attempt_success:b', 'attempt_ip:a', 'attempt_country:s', 'attempt_created:t', 'attempt_user_agent:s']);
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'news_posts',             ['post_id:s', 'category_id:s', 'user_id:s:n', 'comment_section_id:s:n', 'post_is_featured:b', 'post_title:s', 'post_text:s', 'post_scheduled:t', 'post_created:t', 'post_updated:t', 'post_deleted:t:n']);
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'permissions',            ['user_id:s:n', 'role_id:s:n', 'general_perms_allow:i', 'general_perms_deny:i', 'user_perms_allow:i', 'user_perms_deny:i', 'changelog_perms_allow:i', 'changelog_perms_deny:i', 'news_perms_allow:i', 'news_perms_deny:i', 'forum_perms_allow:i', 'forum_perms_deny:i', 'comments_perms_allow:i', 'comments_perms_deny:i']);
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'profile_fields_values',  ['field_id:s', 'user_id:s', 'format_id:s', 'field_value:s']);
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'sessions',               ['session_id:s', 'user_id:s', 'session_key:n', 'session_ip:a', 'session_ip_last:a:n', 'session_user_agent:s', 'session_country:s', 'session_expires:t', 'session_expires_bump:b', 'session_created:t', 'session_active:t:n']);
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'users',                  ['user_id:s', 'username:s', 'password:n', 'email:s', 'register_ip:a', 'last_ip:a', 'user_super:b', 'user_country:s', 'user_colour:i:n', 'user_created:t', 'user_active:t:n', 'user_deleted:t:n', 'display_role:s:n', 'user_totp_key:n', 'user_about_content:s:n', 'user_about_parser:i', 'user_signature_content:s:n', 'user_signature_parser:i', 'user_birthdate:s:n', 'user_background_settings:i:n', 'user_title:s:n']);
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'users_password_resets',  ['user_id:s', 'reset_ip:a', 'reset_requested:t', 'verification_code:n']);
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'user_roles',             ['user_id:s', 'role_id:s']);
+                            $tmpFiles[] = db_to_zip($archive, $currentUserId, 'user_warnings',          ['warning_id:s', 'user_id:s', 'user_ip:a', 'issuer_id:n', 'issuer_ip:n', 'warning_created:t', 'warning_duration:t:n', 'warning_type:i', 'warning_note:s', 'warning_note_private:s:n']);
+
+                            $archive->close();
+                        } finally {
+                            foreach($tmpFiles as $tmpFile)
+                                if(is_file($tmpFile))
+                                    unlink($tmpFile);
+                        }
                     } else {
                         $errors[] = 'Something went wrong while creating your account archive.';
                         break;

templates/settings/data.twig

@@ -10,7 +10,7 @@
         {{ input_csrf() }}
 
         <div class="settings__description">
-            <p>Here you can request raw json files containing pretty much all data relating to your account. Moderator identities are concealed and password hashes are removed from the output.</p>
+            <p>Here you can request raw json files containing pretty much all data relating to your account. Moderator identities are concealed and password hashes are removed from the output. Files in the archive will be in <a href="https://jsonlines.org/" target="_blank" rel="noopener" class="link">JSON Lines</a> format.</p>
         </div>
 
         <div class="settings__data__content">