misuzu/tools/cron

#!/usr/bin/env php
<?php
namespace Misuzu;

use stdClass;
use Exception;
use Misuzu\Storage\Uploads\{UploadInfo,UploadsContext};
use Misuzu\Pools\PoolInfo;
use Misuzu\Pools\Rules\{
    EnsureVariantRule,EnsureVariantRuleThumb,RemoveStaleRule,ScanForumRule
};
use Misuzu\Storage\Files\{FilesContext,FileInfo};

require_once __DIR__ . '/../misuzu.php';

$schedTasks = [];
$runSlow = false; // by what metric? i made it up.

function msz_sched_task_sql(string $name, bool $slow, string $command): void {
    global $schedTasks, $runSlow;

    if($slow && !$runSlow)
        return;

    $schedTasks[] = $task = new stdClass;
    $task->name = $name;
    $task->type = 'sql';
    $task->command = $command;
}

function msz_sched_task_func(string $name, bool $slow, callable $command): void {
    global $schedTasks, $runSlow;

    if($slow && !$runSlow)
        return;

    $schedTasks[] = $task = new stdClass;
    $task->name = $name;
    $task->type = 'func';
    $task->command = $command;
}

for($i = 1; $i < count($argv); ++$i) {
    $argvi = $argv[$i];
    if($argvi === 'slow')
        $runSlow = true;
    else die('Unknown argument specified.' . PHP_EOL);
}

if($runSlow)
    echo 'Also running slow tasks!' . PHP_EOL;
else
    echo 'Only running quick tasks!' . PHP_EOL;
echo PHP_EOL;

$cronPath = sys_get_temp_dir() . '/msz-cron-' . hash('sha256', __DIR__) . '.lock';
if(is_file($cronPath))
    die('Misuzu cron script is already running.' . PHP_EOL);

touch($cronPath);
try {
    msz_sched_task_sql('Ensure main role exists.', true,
        'INSERT IGNORE INTO msz_roles (role_id, role_name, role_rank, role_colour, role_description, role_created) VALUES (1, "Member", 1, 1073741824, NULL, NOW())');

    msz_sched_task_sql('Ensure all users have the main role.', true,
        'INSERT INTO msz_users_roles (user_id, role_id) SELECT user_id, 1 FROM msz_users AS u WHERE NOT EXISTS (SELECT 1 FROM msz_users_roles AS ur WHERE role_id = 1 AND u.user_id = ur.user_id)');

    msz_sched_task_sql('Ensure all user_display_role_id field values are correct with msz_users_roles.', true,
        'UPDATE msz_users AS u SET user_display_role_id = (SELECT ur.role_id FROM msz_users_roles AS ur LEFT JOIN msz_roles AS r ON r.role_id = ur.role_id WHERE ur.user_id = u.user_id ORDER BY role_rank DESC LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM msz_users_roles AS ur WHERE ur.role_id = u.user_display_role_id AND ur.user_id = u.user_id)');

    msz_sched_task_sql('Remove expired sessions.', false,
        'DELETE FROM msz_sessions WHERE session_expires < NOW()');

    msz_sched_task_sql('Remove old password reset tokens.', false,
        'DELETE FROM msz_users_password_resets WHERE reset_requested < NOW() - INTERVAL 1 WEEK');

    msz_sched_task_sql('Remove old login attempt logs.', false,
        'DELETE FROM msz_login_attempts WHERE attempt_created < NOW() - INTERVAL 1 MONTH');

    msz_sched_task_sql('Remove old audit log entries.', false,
        'DELETE FROM msz_audit_log WHERE log_created < NOW() - INTERVAL 3 MONTH');

    msz_sched_task_sql('Remove stale forum tracking entries.', false,
        'DELETE tt FROM msz_forum_topics_track AS tt LEFT JOIN msz_forum_topics AS t ON t.topic_id = tt.topic_id WHERE t.topic_bumped < NOW() - INTERVAL 1 MONTH');

    msz_sched_task_sql('Synchronise forum_id.', true,
        'UPDATE msz_forum_posts AS p INNER JOIN msz_forum_topics AS t ON t.topic_id = p.topic_id SET p.forum_id = t.forum_id');

    msz_sched_task_func('Recount forum topics and posts.', true, function() use ($msz) {
        $msz->forumCtx->categories->syncForumCounters();
    });

    msz_sched_task_sql('Clean up expired 2fa tokens.', false,
        'DELETE FROM msz_auth_tfa WHERE tfa_created < NOW() - INTERVAL 15 MINUTE');

    msz_sched_task_func('Clean up OAuth2 related data.', false, function() use ($msz) {
        $msz->oauth2Ctx->pruneExpired(function(string $format, ...$args) {
            echo sprintf($format, ...$args) . PHP_EOL;
        });
    });

    // very heavy stuff that should

    msz_sched_task_func('Resync statistics counters.', true, function() use ($msz) {
        $stats = [
            'users:total'                  => 'SELECT COUNT(*) FROM msz_users',
            'users:active'                 => 'SELECT COUNT(*) FROM msz_users WHERE user_active IS NOT NULL AND user_deleted IS NULL',
            'users:inactive'               => 'SELECT COUNT(*) FROM msz_users WHERE user_active IS NULL OR user_deleted IS NOT NULL',
            'users:online:recent'          => 'SELECT COUNT(*) FROM msz_users WHERE user_active >= NOW() - INTERVAL 1 HOUR', // used to be 5 minutes, but this script runs every hour soooo
            'users:online:today'           => 'SELECT COUNT(*) FROM msz_users WHERE user_active >= NOW() - INTERVAL 24 HOUR',
            'auditlogs:total'              => 'SELECT COUNT(*) FROM msz_audit_log',
            'changelog:changes:total'      => 'SELECT COUNT(*) FROM msz_changelog_changes',
            'changelog:tags:total'         => 'SELECT COUNT(*) FROM msz_changelog_tags',
            'comments:cats:total'          => 'SELECT COUNT(*) FROM msz_comments_categories',
            'comments:cats:locked'         => 'SELECT COUNT(*) FROM msz_comments_categories WHERE category_locked IS NOT NULL',
            'comments:posts:total'         => 'SELECT COUNT(*) FROM msz_comments_posts',
            'comments:posts:visible'       => 'SELECT COUNT(*) FROM msz_comments_posts WHERE comment_deleted IS NULL',
            'comments:posts:deleted'       => 'SELECT COUNT(*) FROM msz_comments_posts WHERE comment_deleted IS NOT NULL',
            'comments:posts:replies'       => 'SELECT COUNT(*) FROM msz_comments_posts WHERE comment_reply_to IS NOT NULL',
            'comments:posts:pinned'        => 'SELECT COUNT(*) FROM msz_comments_posts WHERE comment_pinned IS NOT NULL',
            'comments:posts:edited'        => 'SELECT COUNT(*) FROM msz_comments_posts WHERE comment_edited IS NOT NULL',
            'comments:votes:likes'         => 'SELECT COUNT(*) FROM msz_comments_votes WHERE comment_vote > 0',
            'comments:votes:dislikes'      => 'SELECT COUNT(*) FROM msz_comments_votes WHERE comment_vote < 0',
            'forum:posts:total'            => 'SELECT COUNT(*) FROM msz_forum_posts',
            'forum:posts:visible'          => 'SELECT COUNT(*) FROM msz_forum_posts WHERE post_deleted IS NULL',
            'forum:posts:deleted'          => 'SELECT COUNT(*) FROM msz_forum_posts WHERE post_deleted IS NOT NULL',
            'forum:posts:edited'           => 'SELECT COUNT(*) FROM msz_forum_posts WHERE post_edited IS NOT NULL',
            'forum:posts:parse:plain'      => "SELECT COUNT(*) FROM msz_forum_posts WHERE post_text_format = ''",
            'forum:posts:parse:bbcode'     => "SELECT COUNT(*) FROM msz_forum_posts WHERE post_text_format = 'bb'",
            'forum:posts:parse:markdown'   => "SELECT COUNT(*) FROM msz_forum_posts WHERE post_text_format = 'md'",
            'forum:posts:signature'        => 'SELECT COUNT(*) FROM msz_forum_posts WHERE post_display_signature <> 0',
            'forum:topics:total'           => 'SELECT COUNT(*) FROM msz_forum_topics',
            'forum:topics:type:normal'     => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_type = 0',
            'forum:topics:type:pinned'     => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_type = 1',
            'forum:topics:type:announce'   => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_type = 2',
            'forum:topics:type:global'     => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_type = 3',
            'forum:topics:visible'         => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_deleted IS NULL',
            'forum:topics:deleted'         => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_deleted IS NOT NULL',
            'forum:topics:locked'          => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_locked IS NOT NULL',
            'auth:attempts:total'          => 'SELECT COUNT(*) FROM msz_login_attempts',
            'auth:attempts:failed'         => 'SELECT COUNT(*) FROM msz_login_attempts WHERE attempt_success = 0',
            'auth:sessions:total'          => 'SELECT COUNT(*) FROM msz_sessions',
            'auth:tfasessions:total'       => 'SELECT COUNT(*) FROM msz_auth_tfa',
            'auth:recovery:total'          => 'SELECT COUNT(*) FROM msz_users_password_resets',
            'users:modnotes:total'         => 'SELECT COUNT(*) FROM msz_users_modnotes',
            'users:warnings:total'         => 'SELECT COUNT(*) FROM msz_users_warnings',
            'users:warnings:visible'       => 'SELECT COUNT(*) FROM msz_users_warnings WHERE warn_created > NOW() - INTERVAL 90 DAY',
            'users:bans:total'             => 'SELECT COUNT(*) FROM msz_users_bans',
            'users:bans:active'            => 'SELECT COUNT(*) FROM msz_users_bans WHERE ban_expires IS NULL OR ban_expires > NOW()',
            'pms:msgs:total'               => 'SELECT COUNT(*) FROM msz_messages',
            'pms:msgs:messages'            => 'SELECT COUNT(DISTINCT msg_id) FROM msz_messages',
            'pms:msgs:replies'             => 'SELECT COUNT(*) FROM msz_messages WHERE msg_reply_to IS NULL',
            'pms:msgs:drafts'              => 'SELECT COUNT(*) FROM msz_messages WHERE msg_sent IS NULL',
            'pms:msgs:unread'              => 'SELECT COUNT(*) FROM msz_messages WHERE msg_read IS NULL',
            'pms:msgs:deleted'             => 'SELECT COUNT(*) FROM msz_messages WHERE msg_deleted IS NOT NULL',
            'pms:msgs:plain'               => "SELECT COUNT(*) FROM msz_messages WHERE msg_body_format = ''",
            'pms:msgs:bbcode'              => "SELECT COUNT(*) FROM msz_messages WHERE msg_body_format = 'bb'",
            'pms:msgs:markdown'            => "SELECT COUNT(*) FROM msz_messages WHERE msg_body_format = 'md'",
        ];

        foreach($stats as $name => $query) {
            $result = $msz->dbCtx->conn->query($query);
            $msz->counters->set($name, $result->next() ? $result->getInteger(0) : 0);
        }
    });

    msz_sched_task_func('Recalculate permissions (maybe)...', false, function() use ($msz) {
        $needsRecalc = $msz->config->getBoolean('perms.needsRecalc');
        if(!$needsRecalc)
            return;

        $msz->config->removeValues('perms.needsRecalc');
        $msz->perms->precalculatePermissions($msz->forumCtx->categories);
    });

    msz_sched_task_func('Omega disliking comments...', true, function() use ($msz) {
        $commentIds = $msz->config->getArray('comments.omegadislike');
        if(!is_array($commentIds))
            return;

        $stmt = $msz->dbCtx->conn->prepare('REPLACE INTO msz_comments_votes (comment_id, user_id, comment_vote) SELECT ?, user_id, -1 FROM msz_users');
        foreach($commentIds as $commentId) {
            $stmt->addParameter(1, $commentId);
            $stmt->execute();
        }
    });

    msz_sched_task_func('Announcing random topics...', true, function() use ($msz) {
        $categoryIds = $msz->config->getArray('forum.rngannounce');
        if(!is_array($categoryIds))
            return;

        $stmtRevert = $msz->dbCtx->conn->prepare('UPDATE msz_forum_topics SET topic_type = 0 WHERE forum_id = ? AND topic_type = 2');
        $stmtRandom = $msz->dbCtx->conn->prepare('SELECT topic_id FROM msz_forum_topics WHERE forum_id = ? AND topic_deleted IS NULL ORDER BY RAND() LIMIT 1');
        $stmtAnnounce = $msz->dbCtx->conn->prepare('UPDATE msz_forum_topics SET topic_type = 2 WHERE topic_id = ?');
        foreach($categoryIds as $categoryId) {
            $stmtRevert->addParameter(1, $categoryId);
            $stmtRevert->execute();

            $stmtRandom->addParameter(1, $categoryId);
            $stmtRandom->execute();

            $resultRandom = $stmtRandom->getResult();
            if($resultRandom->next()) {
                $stmtAnnounce->addParameter(1, $resultRandom->getInteger(0));
                $stmtAnnounce->execute();
            }
        }
    });

    msz_sched_task_func('Changing category icons...', false, function() use ($msz) {
        $categoryIds = $msz->config->getArray('forum.rngicon');
        if(!is_array($categoryIds))
            return;

        $stmtIcon = $msz->dbCtx->conn->prepare('UPDATE msz_forum_categories SET forum_icon = COALESCE(?, forum_icon), forum_name = COALESCE(?, forum_name), forum_colour = ((ROUND(RAND() * 255) << 16) | (ROUND(RAND() * 255) << 8) | ROUND(RAND() * 255)) WHERE forum_id = ?');
        $stmtUnlock = $msz->dbCtx->conn->prepare('UPDATE msz_forum_topics SET topic_locked = IF(?, NULL, COALESCE(topic_locked, NOW())) WHERE topic_id = ?');

        foreach($categoryIds as $categoryId) {
            $scoped = $msz->config->scopeTo(sprintf('forum.rngicon.fc%d', $categoryId));

            $icons = $scoped->getArray('icons');
            $icon = $icons[array_rand($icons)];

            $name = null;
            $names = $scoped->getArray('names');
            for($i = 0; $i < count($names); $i += 2)
                if($names[$i] === $icon) {
                    $name = $names[$i + 1] ?? null;
                    break;
                }
            $name ??= $scoped->getString('name');

            $stmtIcon->addParameter(1, $icon);
            $stmtIcon->addParameter(2, empty($name) ? null : $name);
            $stmtIcon->addParameter(3, $categoryId);
            $stmtIcon->execute();

            $unlockModes = $scoped->getArray('unlock');
            foreach($unlockModes as $unlockMode) {
                $unlockScoped = $scoped->scopeTo(sprintf('unlock.%s', $unlockMode));
                $topicId = $unlockScoped->getInteger('topic');
                $match = $unlockScoped->getArray('match');
                $matches = in_array($icon, $match);

                $stmtUnlock->addParameter(1, $matches ? 1 : 0);
                $stmtUnlock->addParameter(2, $topicId);
                $stmtUnlock->execute();
            }
        }
    });

    msz_sched_task_func('Removing stale entries from storage pools...', false, function() use ($msz) {
        $pools = [];
        $getPool = fn($ruleRaw) => array_key_exists($ruleRaw->poolId, $pools) ? $pools[$ruleRaw->poolId] : (
            $pools[$ruleRaw->poolId] = $msz->storageCtx->poolsCtx->pools->getPool($ruleRaw->poolId)
        );

        // Run cleanup adjacent tasks first
        $rules = $msz->storageCtx->poolsCtx->pools->getPoolRules(types: ['remove_stale', 'scan_forum']);
        foreach($rules as $ruleRaw) {
            $poolInfo = $getPool($ruleRaw);
            $rule = $msz->storageCtx->poolsCtx->rules->create($ruleRaw);

            if($rule instanceof RemoveStaleRule) {
                printf('Removing stale entries for pool #%d...%s', $poolInfo->id, PHP_EOL);
                $msz->storageCtx->uploadsCtx->uploads->deleteStaleUploads(
                    $poolInfo,
                    $rule->inactiveForSeconds,
                    $rule->ignoreUploadIds
                );
            } elseif($rule instanceof ScanForumRule) {
                //var_dump('scan_forum', $rule);
                // necessary stuff for this doesn't exist yet so this can remain a no-op
            }
        }
    });

    msz_sched_task_func('Purging denylisted files...', true, function() use ($msz) {
        $fileInfos = $msz->storageCtx->filesCtx->data->getFiles(denied: true);
        foreach($fileInfos as $fileInfo) {
            printf('Deleting file #%d...%s', $fileInfo->id, PHP_EOL);
            $msz->storageCtx->filesCtx->deleteFile($fileInfo);
        }
    });

    msz_sched_task_func('Purging orphaned files...', true, function() use ($msz) {
        $fileInfos = $msz->storageCtx->filesCtx->data->getFiles(orphaned: true);
        foreach($fileInfos as $fileInfo) {
            printf('Deleting file #%d...%s', $fileInfo->id, PHP_EOL);
            $msz->storageCtx->filesCtx->deleteFile($fileInfo);
        }
    });

    msz_sched_task_func('Ensuring alternate variants exist...', true, function() use ($msz) {
        $pools = [];
        $getPool = fn($ruleRaw) => array_key_exists($ruleRaw->poolId, $pools) ? $pools[$ruleRaw->poolId] : (
            $pools[$ruleRaw->poolId] = $msz->storageCtx->poolsCtx->pools->getPool($ruleRaw->poolId)
        );

        $rules = $msz->storageCtx->poolsCtx->pools->getPoolRules(types: ['ensure_variant']);
        foreach($rules as $ruleRaw) {
            $poolInfo = $getPool($ruleRaw);
            $rule = $msz->storageCtx->poolsCtx->rules->create($ruleRaw);

            if($rule instanceof EnsureVariantRule) {
                if(!$rule->onCron)
                    continue;

                $createVariant = null; // ensure this wasn't previously assigned
                if($rule->params instanceof EnsureVariantRuleThumb) {
                    $createVariant = function(
                        FilesContext $filesCtx,
                        UploadsContext $uploadsCtx,
                        PoolInfo $poolInfo,
                        EnsureVariantRule $rule,
                        EnsureVariantRuleThumb $ruleInfo,
                        FileInfo $originalInfo,
                        UploadInfo $uploadInfo
                    ) {
                        $uploadsCtx->uploads->createUploadVariant(
                            $uploadInfo,
                            $rule->variant,
                            $filesCtx->createThumbnailFromRule(
                                $originalInfo,
                                $ruleInfo
                            )
                        );
                    };
                }

                if(!isset($createVariant) || !is_callable($createVariant)) {
                    printf('!!! Could not create a constructor for "%s" variants for pool #%d !!! Skipping for now...%s', $rule->variant, $poolInfo->id, PHP_EOL);
                    continue;
                }

                printf('Ensuring existence of "%s" variants for pool #%d...%s', $rule->variant, $poolInfo->id, PHP_EOL);

                $uploads = $msz->storageCtx->uploadsCtx->uploads->getUploads(
                    poolInfo: $poolInfo,
                    variant: $rule->variant,
                    variantExists: false
                );

                $processed = 0;
                foreach($uploads as $uploadInfo)
                    try {
                        $createVariant(
                            $msz->storageCtx->filesCtx,
                            $msz->storageCtx->uploadsCtx,
                            $poolInfo,
                            $rule,
                            $rule->params,
                            $msz->storageCtx->filesCtx->data->getFile(
                                $msz->storageCtx->uploadsCtx->uploads->getUploadVariant($uploadInfo, '')->fileId
                            ),
                            $uploadInfo
                        );
                    } catch(Exception $ex) {
                        printf('Exception thrown while processing "%s" variant for upload #%d in pool #%d:%s', $rule->variant, $uploadInfo->id, $poolInfo->id, PHP_EOL);
                        printf('%s%s', $ex->getMessage(), PHP_EOL);
                    } finally {
                        ++$processed;
                    }

                printf('Processed %d records!%s%s', $processed, PHP_EOL);
            }
        }
    });

    echo 'Running ' . count($schedTasks) . ' tasks...' . PHP_EOL;

    foreach($schedTasks as $task) {
        echo '=> ' . $task->name . PHP_EOL;
        $success = true;

        try {
            switch($task->type) {
                case 'sql':
                    $affected = $msz->dbCtx->conn->execute($task->command);
                    echo $affected . ' rows affected. ' . PHP_EOL;
                    break;

                case 'func':
                    $result = ($task->command)();
                    if(is_bool($result))
                        $success = $result;
                    break;

                default:
                    $success = false;
                    echo 'Unknown task type.' . PHP_EOL;
                    break;
            }
        } catch(Exception $ex) {
            $success = false;
            echo (string)$ex;
        } finally {
            if($success)
                echo 'Done!';
            else
                echo '!!! FAILED !!!';
            echo PHP_EOL;
        }

        echo PHP_EOL;
    }
} finally {
    unlink($cronPath);
    printf('Took %.5fms.%s', ((float)hrtime(true) - MSZ_STARTUP) / 1000000, PHP_EOL);
}