flashii/misuzu
flashii/misuzu
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Shoved EEPROM into Misuzu.

Browse source
This commit is contained in:
flash 2025-03-27 00:44:42 +00:00
parent 0b7031959b
commit 5f6133c007
Signed by: flash
GPG key ID: 2C9C2C574D47FE3E
63 changed files with 3893 additions and 222 deletions

1
.gitignore vendored
View file

@ -21,6 +21,7 @@
/.migrating
# Storage
/storage
/store
# OS specific

3
build.js
View file

@ -37,9 +37,12 @@ const fs = require('fs');
twig: [
{ source: 'errors/400', target: '/', name: 'error-400.html', },
{ source: 'errors/401', target: '/', name: 'error-401.html', },
{ source: 'errors/402', target: '/', name: 'error-402.html', },
{ source: 'errors/403', target: '/', name: 'error-403.html', },
{ source: 'errors/404', target: '/', name: 'error-404.html', },
{ source: 'errors/405', target: '/', name: 'error-405.html', },
{ source: 'errors/410', target: '/', name: 'error-410.html', },
{ source: 'errors/451', target: '/', name: 'error-451.html', },
{ source: 'errors/500', target: '/', name: 'error-500.html', },
{ source: 'errors/502', target: '/', name: 'error-502.html', },
{ source: 'errors/503', target: '/', name: 'error-503.html', },

6
composer.lock generated
View file

@ -501,11 +501,11 @@
},
{
"name": "flashwave/index",
"version": "v0.2503.251417",
"version": "v0.2503.260138",
"source": {
"type": "git",
"url": "https://patchii.net/flash/index.git",
"reference": "0f3501acc6a13f1cee351a1ff015b97b45f24b55"
"reference": "ea549dd0eb7cc7e7348bfcfb0e95da880dd2c039"
},
"require": {
"ext-mbstring": "*",
@ -554,7 +554,7 @@
],
"description": "Composer package for the common library for my projects.",
"homepage": "https://railgun.sh/index",
"time": "2025-03-25T14:17:15+00:00"
"time": "2025-03-26T01:40:42+00:00"
},
{
"name": "graham-campbell/result-type",

165
database/2025_03_26_164049_create_storage_tables.php Normal file
View file

@ -0,0 +1,165 @@
<?php
use Index\Db\DbConnection;
use Index\Db\Migration\DbMigration;
final class CreateStorageTables_20250326_164049 implements DbMigration {
public function migrate(DbConnection $conn): void {
$conn->execute(<<<SQL
CREATE TABLE msz_storage_denylist (
deny_id INT(10) UNSIGNED NOT NULL AUTO_INCREMENT,
deny_hash BINARY(32) NOT NULL,
deny_reason ENUM('copyright','rules','other') NOT NULL COLLATE 'ascii_general_ci',
deny_created TIMESTAMP NOT NULL DEFAULT current_timestamp(),
PRIMARY KEY (deny_id),
UNIQUE KEY msz_storage_denylist_hash_unique (deny_hash)
) COLLATE='utf8mb4_bin' ENGINE=InnoDB;
SQL);
$conn->execute(<<<SQL
CREATE TABLE msz_storage_files (
file_id BIGINT(20) UNSIGNED NOT NULL,
file_hash BINARY(32) NOT NULL,
file_type VARCHAR(255) NOT NULL COLLATE 'ascii_general_ci',
file_size INT(11) NOT NULL,
file_created TIMESTAMP NOT NULL DEFAULT current_timestamp(),
PRIMARY KEY (file_id),
UNIQUE KEY msz_storage_files_hash_unique (file_hash),
KEY msz_storage_files_created_index (file_created),
KEY msz_storage_files_type_index (file_type),
KEY msz_storage_files_size_index (file_size)
) COLLATE='utf8mb4_bin' ENGINE=InnoDB;
SQL);
$conn->execute(<<<SQL
CREATE TABLE msz_storage_pools (
pool_id INT(10) UNSIGNED NOT NULL AUTO_INCREMENT,
pool_name VARCHAR(32) NOT NULL COLLATE 'ascii_general_ci',
pool_secret VARCHAR(255) NULL DEFAULT NULL COLLATE 'ascii_bin',
pool_created TIMESTAMP NOT NULL DEFAULT current_timestamp(),
pool_deprecated TIMESTAMP NULL DEFAULT NULL,
PRIMARY KEY (pool_id),
UNIQUE KEY msz_storage_pools_name_unique (pool_name),
KEY msz_storage_pools_created_index (pool_created),
KEY msz_storage_pools_deprecated_index (pool_deprecated)
) COLLATE='utf8mb4_bin' ENGINE=InnoDB;
SQL);
$conn->execute(<<<SQL
CREATE TABLE msz_storage_pools_rules (
rule_id INT(10) UNSIGNED NOT NULL AUTO_INCREMENT,
pool_id INT(10) UNSIGNED NOT NULL,
rule_type VARCHAR(64) NOT NULL COLLATE 'ascii_general_ci',
rule_params LONGTEXT NOT NULL DEFAULT '{}' COLLATE 'utf8mb4_bin',
PRIMARY KEY (rule_id),
KEY msz_storage_pools_rules_pool_foreign (pool_id),
KEY msz_storage_pools_rules_type_index (rule_type),
CONSTRAINT msz_storage_pools_rules_pool_foreign
FOREIGN KEY (pool_id)
REFERENCES msz_storage_pools (pool_id)
ON UPDATE CASCADE
ON DELETE CASCADE,
CONSTRAINT rule_params
CHECK (json_valid(rule_params)),
CONSTRAINT msz_storage_pools_rules_params_ensure_object
CHECK (json_type(rule_params) = 'OBJECT')
) COLLATE='utf8mb4_bin' ENGINE=InnoDB;
SQL);
$conn->execute(<<<SQL
CREATE TABLE msz_storage_tasks (
task_id BIGINT(20) UNSIGNED NOT NULL,
task_secret CHAR(16) NOT NULL COLLATE 'ascii_bin',
user_id INT(10) UNSIGNED NOT NULL,
pool_id INT(10) UNSIGNED NOT NULL,
task_state ENUM('pending','complete','error') NOT NULL DEFAULT 'pending' COLLATE 'ascii_general_ci',
task_ip VARBINARY(16) NOT NULL,
task_name VARCHAR(255) NOT NULL COLLATE 'utf8mb4_unicode_520_ci',
task_size INT(10) UNSIGNED NOT NULL,
task_type VARCHAR(255) NOT NULL COLLATE 'ascii_general_ci',
task_hash BINARY(32) NOT NULL,
task_error TEXT NOT NULL DEFAULT '' COLLATE 'utf8mb4_bin',
task_created TIMESTAMP NOT NULL DEFAULT current_timestamp(),
PRIMARY KEY (task_id),
KEY msz_storage_tasks_user_foreign (user_id),
KEY msz_storage_tasks_pool_foreign (pool_id),
KEY msz_storage_tasks_created_index (task_created),
KEY msz_storage_tasks_state_index (task_state),
CONSTRAINT msz_storage_tasks_user_foreign
FOREIGN KEY (user_id)
REFERENCES msz_users (user_id)
ON UPDATE CASCADE
ON DELETE CASCADE,
CONSTRAINT msz_storage_tasks_pool_foreign
FOREIGN KEY (pool_id)
REFERENCES msz_storage_pools (pool_id)
ON UPDATE CASCADE
ON DELETE CASCADE
) COLLATE='utf8mb4_bin' ENGINE=InnoDB;
SQL);
$conn->execute(<<<SQL
CREATE TABLE msz_storage_uploads (
upload_id BIGINT(20) UNSIGNED NOT NULL,
user_id INT(10) UNSIGNED NOT NULL,
pool_id INT(10) UNSIGNED NOT NULL,
upload_secret CHAR(4) NOT NULL COLLATE 'ascii_bin',
upload_name VARCHAR(255) NOT NULL COLLATE 'utf8mb4_unicode_520_ci',
upload_ip VARBINARY(16) NOT NULL,
upload_created TIMESTAMP NOT NULL DEFAULT current_timestamp(),
upload_accessed TIMESTAMP NOT NULL DEFAULT current_timestamp(),
PRIMARY KEY (upload_id),
KEY msz_storage_uploads_ip_index (upload_ip),
KEY msz_storage_uploads_created_index (upload_created),
KEY msz_storage_uploads_accessed_index (upload_accessed),
KEY msz_storage_uploads_user_foreign (user_id),
KEY msz_storage_uploads_pool_foreign (pool_id),
CONSTRAINT msz_storage_uploads_user_foreign
FOREIGN KEY (user_id)
REFERENCES msz_users (user_id)
ON UPDATE CASCADE
ON DELETE CASCADE,
CONSTRAINT msz_storage_uploads_pool_foreign
FOREIGN KEY (pool_id)
REFERENCES msz_storage_pools (pool_id)
ON UPDATE CASCADE
ON DELETE CASCADE
) COLLATE='utf8mb4_bin' ENGINE=InnoDB;
SQL);
$conn->execute(<<<SQL
CREATE TABLE msz_storage_uploads_files (
upload_id BIGINT(20) UNSIGNED NOT NULL,
upload_variant VARCHAR(255) NOT NULL COLLATE 'ascii_general_ci',
file_id BIGINT(20) UNSIGNED NOT NULL,
file_type VARCHAR(255) NULL DEFAULT NULL COLLATE 'ascii_general_ci',
PRIMARY KEY (upload_id, upload_variant),
KEY msz_storage_uploads_files_upload_foreign (upload_id),
KEY msz_storage_uploads_files_file_foreign (file_id),
CONSTRAINT msz_storage_uploads_files_upload_foreign
FOREIGN KEY (upload_id)
REFERENCES msz_storage_uploads (upload_id)
ON UPDATE CASCADE
ON DELETE CASCADE,
CONSTRAINT msz_storage_uploads_files_file_foreign
FOREIGN KEY (file_id)
REFERENCES msz_storage_files (file_id)
ON UPDATE CASCADE
ON DELETE RESTRICT
) COLLATE='utf8mb4_bin' ENGINE=InnoDB;
SQL);
$conn->execute(<<<SQL
CREATE TABLE msz_storage_uploads_legacy (
upload_id_legacy BINARY(32) NOT NULL,
upload_id BIGINT(20) UNSIGNED NOT NULL,
UNIQUE KEY msz_storage_uploads_legacy_unique (upload_id_legacy),
KEY msz_storage_uploads_legacy_foreign (upload_id),
CONSTRAINT msz_storage_uploads_legacy_foreign
FOREIGN KEY (upload_id)
REFERENCES msz_storage_uploads (upload_id)
ON UPDATE CASCADE
ON DELETE CASCADE
) COLLATE='utf8mb4_bin' ENGINE=InnoDB;
SQL);
}
}

4
misuzu.php
View file

@ -24,5 +24,7 @@ if(!empty($_ENV['SENTRY_DSN']))
$msz = new MisuzuContext(
$_ENV['DATABASE_DSN'] ?? 'null:',
$_ENV['DOMAIN_ROLES'] ?? 'localhost=main,redirect:/go'
$_ENV['DOMAIN_ROLES'] ?? 'localhost=main,redirect,storage',
$_ENV['STORAGE_PATH_LOCAL'] ?? Misuzu::PATH_STORAGE,
$_ENV['STORAGE_PATH_REMOTE'] ?? '/_storage',
);

1
src/Auth/AuthApiRoutes.php
View file

@ -31,6 +31,7 @@ final class AuthApiRoutes implements RouteHandler {
/** @return void|array{error: string, error_description?: string} */
#[PrefixFilter('/api/v1')]
#[PrefixFilter('/oauth2')]
#[PrefixFilter('/uploads')]
public function handleAuthorization(HttpResponseBuilder $response, HttpRequest $request) {
if($this->authInfo->loggedIn)
return;

1
src/Misuzu.php
View file

@ -22,6 +22,7 @@ final class Misuzu {
public const string PATH_PUBLIC_LEGACY = self::PATH_ROOT . DIRECTORY_SEPARATOR . 'public-legacy';
public const string PATH_SOURCE = self::PATH_ROOT . DIRECTORY_SEPARATOR . 'src';
public const string PATH_STORE = self::PATH_ROOT . DIRECTORY_SEPARATOR . 'store';
public const string PATH_STORAGE = self::PATH_ROOT . DIRECTORY_SEPARATOR . 'storage';
public const string PATH_TEMPLATES = self::PATH_ROOT . DIRECTORY_SEPARATOR . 'templates';
public const string PATH_VERSION = self::PATH_ROOT . DIRECTORY_SEPARATOR . 'VERSION';

71
src/MisuzuContext.php
View file

@ -7,6 +7,7 @@ use Index\Config\Db\DbConfig;
use Index\Db\DbConnection;
use Index\Db\Migration\{DbMigrationManager,DbMigrationRepo,FsDbMigrationRepo};
use Index\Http\HttpRequest;
use Index\Snowflake\{BinarySnowflake,RandomSnowflake,SnowflakeGenerator};
use Index\Templating\TplEnvironment;
use Index\Urls\UrlRegistry;
use Misuzu\Routing\RoutingContext;
@ -19,6 +20,10 @@ class MisuzuContext {
public private(set) DomainRoles $domainRoles;
public private(set) TplEnvironment $templating;
public private(set) SnowflakeGenerator $snowflake;
public private(set) BinarySnowflake $sfBinary;
public private(set) RandomSnowflake $sfRandom;
public private(set) AuditLog\AuditLogData $auditLog;
public private(set) Counters\CountersData $counters;
@ -34,6 +39,7 @@ class MisuzuContext {
public private(set) Messages\MessagesContext $messagesCtx;
public private(set) OAuth2\OAuth2Context $oauth2Ctx;
public private(set) Profile\ProfileContext $profileCtx;
public private(set) Storage\StorageContext $storageCtx;
public private(set) Users\UsersContext $usersCtx;
public private(set) Redirects\RedirectsContext $redirectsCtx;
@ -47,7 +53,9 @@ class MisuzuContext {
public function __construct(
#[\SensitiveParameter] string $dsn,
string $domainRoles
string $domainRoles,
string $storageLocalPath,
string $storageRemotePath,
) {
$this->deps = new Dependencies;
$this->deps->register($this);
@ -56,29 +64,60 @@ class MisuzuContext {
$this->deps->register($this->dbCtx = new DatabaseContext($dsn));
$this->deps->register($this->dbCtx->conn);
$this->deps->register($this->config = $this->deps->constructLazy(DbConfig::class, tableName: 'msz_config'));
$this->deps->register($this->domainRoles = $this->deps->constructLazy(DomainRoles::class, domainRoles: $domainRoles));
$this->deps->register($this->config = $this->deps->constructLazy(
DbConfig::class,
tableName: 'msz_config',
));
$this->deps->register($this->domainRoles = $this->deps->constructLazy(
DomainRoles::class,
domainRoles: $domainRoles,
));
$this->deps->register($this->snowflake = $this->deps->constructLazy(SnowflakeGenerator::class));
$this->deps->register($this->sfBinary = $this->deps->constructLazy(BinarySnowflake::class));
$this->deps->register($this->sfRandom = $this->deps->constructLazy(RandomSnowflake::class));
$this->deps->register($this->perms = $this->deps->constructLazy(Perms\PermissionsData::class));
$this->deps->register($this->authInfo = $this->deps->constructLazy(Auth\AuthInfo::class));
$this->deps->register($this->siteInfo = $this->deps->constructLazy(SiteInfo::class, config: $this->config->scopeTo('site')));
$this->deps->register($this->assetInfo = $this->deps->constructLazy(AssetInfo::class, pathOrAssets: AssetInfo::CURRENT_PATH));
$this->deps->register($this->siteInfo = $this->deps->constructLazy(
SiteInfo::class,
config: $this->config->scopeTo('site'),
));
$this->deps->register($this->assetInfo = $this->deps->constructLazy(
AssetInfo::class,
pathOrAssets: AssetInfo::CURRENT_PATH,
));
$this->deps->register($this->appsCtx = $this->deps->constructLazy(Apps\AppsContext::class));
$this->deps->register($this->authCtx = $this->deps->constructLazy(Auth\AuthContext::class, config: $this->config->scopeTo('auth')));
$this->deps->register($this->authCtx = $this->deps->constructLazy(
Auth\AuthContext::class,
config: $this->config->scopeTo('auth'),
));
$this->deps->register($this->commentsCtx = $this->deps->constructLazy(Comments\CommentsContext::class));
$this->deps->register($this->forumCtx = $this->deps->constructLazy(Forum\ForumContext::class));
$this->deps->register($this->messagesCtx = $this->deps->constructLazy(Messages\MessagesContext::class));
$this->deps->register($this->oauth2Ctx = $this->deps->constructLazy(OAuth2\OAuth2Context::class, config: $this->config->scopeTo('oauth2')));
$this->deps->register($this->oauth2Ctx = $this->deps->constructLazy(
OAuth2\OAuth2Context::class,
config: $this->config->scopeTo('oauth2'),
));
$this->deps->register($this->profileCtx = $this->deps->constructLazy(Profile\ProfileContext::class));
$this->deps->register($this->usersCtx = $this->deps->constructLazy(Users\UsersContext::class));
$this->deps->register($this->redirectsCtx = $this->deps->constructLazy(Redirects\RedirectsContext::class, config: $this->config->scopeTo('redirects')));
$this->deps->register($this->redirectsCtx = $this->deps->constructLazy(
Redirects\RedirectsContext::class,
config: $this->config->scopeTo('redirects'),
));
$this->deps->register($this->auditLog = $this->deps->constructLazy(AuditLog\AuditLogData::class));
$this->deps->register($this->changelog = $this->deps->constructLazy(Changelog\ChangelogData::class));
$this->deps->register($this->counters = $this->deps->constructLazy(Counters\CountersData::class));
$this->deps->register($this->emotes = $this->deps->constructLazy(Emoticons\EmotesData::class));
$this->deps->register($this->news = $this->deps->constructLazy(News\NewsData::class));
$this->deps->register($this->storageCtx = $this->deps->construct(
Storage\StorageContext::class,
localPath: $storageLocalPath,
remotePath: $storageRemotePath,
));
}
public function initMailer(): void {
@ -141,14 +180,17 @@ class MisuzuContext {
$host = $request->getHeaderLine('Host');
$roles = $this->domainRoles->getRoles($host);
$routingCtx = new RoutingContext;
$routingCtx = $this->deps->construct(RoutingContext::class);
$this->deps->register($this->urls = $routingCtx->urls);
if(in_array('main', $roles))
$this->registerMainRoutes($routingCtx);
if(in_array('redirect', $roles))
$this->registerRedirectorRoutes($routingCtx);
$this->registerRedirectRoutes($routingCtx);
if(in_array('storage', $roles))
$this->registerStorageRoutes($routingCtx);
return $routingCtx;
}
@ -181,6 +223,9 @@ class MisuzuContext {
config: $this->config->scopeTo('messages')
));
$routingCtx->register($this->deps->constructLazy(Storage\Tasks\TasksRoutes::class));
$routingCtx->register($this->deps->constructLazy(Storage\Uploads\UploadsLegacyRoutes::class));
$routingCtx->register($this->deps->constructLazy(OAuth2\OAuth2ApiRoutes::class));
$routingCtx->register($this->deps->constructLazy(OAuth2\OAuth2WebRoutes::class));
$routingCtx->register($this->deps->constructLazy(WebFinger\WebFingerRoutes::class));
@ -204,11 +249,15 @@ class MisuzuContext {
$routingCtx->register($this->deps->constructLazy(LegacyRoutes::class));
}
public function registerRedirectorRoutes(RoutingContext $routingCtx): void {
public function registerRedirectRoutes(RoutingContext $routingCtx): void {
$routingCtx->register($this->deps->constructLazy(Redirects\LandingRedirectsRoutes::class));
$routingCtx->register($this->deps->constructLazy(Redirects\AliasRedirectsRoutes::class));
$routingCtx->register($this->deps->constructLazy(Redirects\IncrementalRedirectsRoutes::class));
$routingCtx->register($this->deps->constructLazy(Redirects\SocialRedirectsRoutes::class));
$routingCtx->register($this->deps->constructLazy(Redirects\NamedRedirectsRoutes::class));
}
public function registerStorageRoutes(RoutingContext $routingCtx): void {
$routingCtx->register($this->deps->constructLazy(Storage\Uploads\UploadsViewRoutes::class));
}
}

26
src/Perm.php
View file

@ -7,7 +7,8 @@ use stdClass;
// To avoid future conflicts, unused/deprecated permissions should remain defined for any given category
final class Perm {
// GLOBAL ALLOCATION:
// 0bXXXXX_XXXXXXXX_CCCCCCCC_MMMMFFFF_NNNNNNNN_LLLLLLLL_GGGGGGGG
// 0bXXXXX_XXXXSSSS_CCCCCCCC_MMMMFFFF_NNNNNNNN_LLLLLLLL_GGGGGGGG
// S -> Legacy storage permissions
// M -> Messages permissions
// G -> General global permissions
// L -> Changelog permissions
@ -47,6 +48,11 @@ final class Perm {
public const G_COMMENTS_LOCK = 0b00000_00000000_01000000_00000000_00000000_00000000_00000000;
public const G_COMMENTS_VOTE = 0b00000_00000000_10000000_00000000_00000000_00000000_00000000;
public const G_STORAGE_LEGACY_UPLOAD = 0b00000_00000001_00000000_00000000_00000000_00000000_00000000;
public const G_STORAGE_LEGACY_DOUBLE = 0b00000_00000010_00000000_00000000_00000000_00000000_00000000;
public const G_STORAGE_LEGACY_DELETE_OWN = 0b00000_00000100_00000000_00000000_00000000_00000000_00000000;
public const G_STORAGE_LEGACY_DELETE_ANY = 0b00000_00001000_00000000_00000000_00000000_00000000_00000000;
// USER ALLOCATION:
// There's no rules here, manage perms started abouts halfway through the 31-bit integer
// Maybe formally define the octets regardless later?
@ -119,7 +125,7 @@ final class Perm {
public const INFO_FOR_ROLE = self::INFO_FOR_USER; // just alias for now, no clue if this will ever desync
public const INFO_FOR_FORUM_CATEGORY = ['forum'];
public const LISTS_FOR_USER = ['global:general', 'global:changelog', 'global:news', 'global:forum', 'global:messages', 'global:comments', 'user:personal', 'user:manage', 'chat:general'];
public const LISTS_FOR_USER = ['global:general', 'global:changelog', 'global:news', 'global:forum', 'global:messages', 'global:comments', 'global:storage', 'user:personal', 'user:manage', 'chat:general'];
public const LISTS_FOR_ROLE = self::LISTS_FOR_USER; // idem
public const LISTS_FOR_FORUM_CATEGORY = ['forum:category', 'forum:topic', 'forum:post'];
@ -189,6 +195,17 @@ final class Perm {
],
],
'global:storage' => [
'title' => 'Storage Permissions',
'perms' => [
'global',
self::G_STORAGE_LEGACY_UPLOAD,
self::G_STORAGE_LEGACY_DOUBLE,
self::G_STORAGE_LEGACY_DELETE_OWN,
self::G_STORAGE_LEGACY_DELETE_ANY,
],
],
'user:personal' => [
'title' => 'User Permissions',
'perms' => [
@ -314,6 +331,11 @@ final class Perm {
self::G_COMMENTS_PIN => 'Can pin commments.',
self::G_COMMENTS_LOCK => 'Can lock comment categories.',
self::G_COMMENTS_VOTE => 'Can vote (like or dislike) on comments.',
self::G_STORAGE_LEGACY_UPLOAD => 'Can upload files through the legacy endpoint, if the pool allows it.',
self::G_STORAGE_LEGACY_DOUBLE => 'Can upload files double the maximum file size through the legacy endpoint, if the pool allows it.',
self::G_STORAGE_LEGACY_DELETE_OWN => 'Can delete own files through the legacy endpoint, if the pool allows it.',
self::G_STORAGE_LEGACY_DELETE_ANY => 'Can delete ANY files through the legacy endpoint, if the pool allows it.',
],
'user' => [

48
src/Routing/RoutingAccessControlHandler.php Normal file
View file

@ -0,0 +1,48 @@
<?php
namespace Misuzu\Routing;
use Index\Config\Config;
use Index\Http\HttpUri;
use Index\Http\Routing\HandlerContext;
use Index\Http\Routing\Routes\RouteInfo;
use Index\Http\Routing\AccessControl\{AccessControl,SimpleAccessControlHandler};
class RoutingAccessControlHandler extends SimpleAccessControlHandler {
public function __construct(
private Config $config,
) {}
/** @param string[] $origins */
public static function filterOrigin(array $origins, HttpUri $origin): ?string {
$host = '.' . $origin->host;
foreach($origins as $allowed)
if(str_ends_with($host, '.' . $allowed))
return (string)$origin;
return null;
}
#[\Override]
public function checkAccess(
HandlerContext $context,
AccessControl $accessControl,
HttpUri $origin,
?RouteInfo $routeInfo = null,
): string|bool {
if($accessControl->credentials) {
$result = null;
if($context->request->requestTarget === '/_sockchat'
|| str_starts_with($context->request->requestTarget, '/_sockchat/')) {
$result = self::filterOrigin($this->config->getArray('sockChat.origins'), $origin);
} elseif($context->request->requestTarget === '/storage'
|| str_starts_with($context->request->requestTarget, '/storage/')) {
$result = self::filterOrigin($this->config->getArray('storage.origins'), $origin);
}
if($result !== null)
return $result;
}
return true;
}
}

13
src/Routing/RoutingContext.php
View file

@ -1,6 +1,7 @@
<?php
namespace Misuzu\Routing;
use Index\Config\Config;
use Index\Http\{HttpRequest,HttpResponseBuilder};
use Index\Http\Routing\{Router,RouteHandler};
use Index\Http\Routing\Filters\FilterInfo;
@ -10,12 +11,12 @@ class RoutingContext {
public private(set) Router $router;
public private(set) UrlRegistry $urls;
public function __construct(
?Router $router = null,
?UrlRegistry $urls = null
) {
$this->urls = $urls ?? new ArrayUrlRegistry;
$this->router = $router ?? new Router(errorHandler: new RoutingErrorHandler);
public function __construct(Config $config) {
$this->urls = new ArrayUrlRegistry;
$this->router = new Router(
new RoutingErrorHandler,
new RoutingAccessControlHandler($config),
);
$this->router->filter(FilterInfo::prefix('/', fn(HttpResponseBuilder $response) => $response->setPoweredBy('Misuzu')));
}

9
src/SiteInfo.php
View file

@ -1,16 +1,17 @@
<?php
namespace Misuzu;
use RuntimeException;
use Index\Config\Config;
class SiteInfo {
private array $props; // @phpstan-ignore-line: Seems PHPStan doesn't support property hooks yet :)
/** @var array<string, string> */
private array $props;
public function __construct(Config $config) {
$this->props = $config->getValues([
['name:s', 'Misuzu'],
'desc:s',
'domain:s',
'url:s',
'email:s',
'bsky:s',
@ -38,6 +39,10 @@ class SiteInfo {
get => $this->props['bsky'];
}
public string $domain {
get => $this->props['domain'];
}
public string $externalLogo {
get => $this->props['ext_logo'];
}

29
src/Storage/Denylist/DenylistContext.php Normal file
View file

@ -0,0 +1,29 @@
<?php
namespace Misuzu\Storage\Denylist;
use Index\Db\DbConnection;
use Misuzu\Storage\Files\FileInfo;
class DenylistContext {
public private(set) DenylistData $list;
public function __construct(DbConnection $dbConn) {
$this->list = new DenylistData($dbConn);
}
public function denyFile(FileInfo|string $infoOrHash, DenylistReason|string $reason): void {
if($infoOrHash instanceof FileInfo)
$infoOrHash = $infoOrHash->hash;
if(is_string($reason))
$reason = DenylistReason::from($reason);
$this->list->createDenylistEntry($infoOrHash, $reason);
}
public function getDenylistEntry(FileInfo|string $infoOrHash): ?DenylistInfo {
if($infoOrHash instanceof FileInfo)
$infoOrHash = $infoOrHash->hash;
return $this->list->getDenylistEntry($infoOrHash);
}
}

32
src/Storage/Denylist/DenylistData.php Normal file
View file

@ -0,0 +1,32 @@
<?php
namespace Misuzu\Storage\Denylist;
use InvalidArgumentException;
use Index\Db\{DbConnection,DbStatementCache};
class DenylistData {
private DbStatementCache $cache;
public function __construct(DbConnection $dbConn) {
$this->cache = new DbStatementCache($dbConn);
}
public function getDenylistEntry(string $hash): ?DenylistInfo {
$stmt = $this->cache->get('SELECT deny_id, deny_hash, deny_reason, UNIX_TIMESTAMP(deny_created) FROM msz_storage_denylist WHERE deny_hash = ?');
$stmt->nextParameter($hash);
$stmt->execute();
$result = $stmt->getResult();
return $result->next() ? DenylistInfo::fromResult($result) : null;
}
public function createDenylistEntry(string $hash, DenylistReason $reason): void {
if(strlen($hash) !== 32)
throw new InvalidArgumentException('$hash must be 32 bytes.');
$stmt = $this->cache->get('INSERT INTO msz_storage_denylist (deny_hash, deny_reason) VALUES (?, ?)');
$stmt->nextParameter($hash);
$stmt->nextParameter($reason->value);
$stmt->execute();
}
}

35
src/Storage/Denylist/DenylistInfo.php Normal file
View file

@ -0,0 +1,35 @@
<?php
namespace Misuzu\Storage\Denylist;
use Carbon\CarbonImmutable;
use Index\Db\DbResult;
class DenylistInfo {
public function __construct(
public private(set) string $id,
public private(set) string $hash,
public private(set) DenylistReason $reason,
public private(set) int $createdTime,
) {}
public static function fromResult(DbResult $result): DenylistInfo {
return new DenylistInfo(
id: $result->getString(0),
hash: $result->getString(1),
reason: DenylistReason::tryFrom($result->getString(2)) ?? DenylistReason::Other,
createdTime: $result->getInteger(3),
);
}
public bool $isCopyrightTakedown {
get => $this->reason === DenylistReason::Copyright;
}
public bool $isRulesViolation {
get => $this->reason === DenylistReason::Rules;
}
public CarbonImmutable $createdAt {
get => CarbonImmutable::createFromTimestampUTC($this->createdTime);
}
}

8
src/Storage/Denylist/DenylistReason.php Normal file
View file

@ -0,0 +1,8 @@
<?php
namespace Misuzu\Storage\Denylist;
enum DenylistReason: string {
case Copyright = 'copyright';
case Rules = 'rules';
case Other = 'other';
}

7
src/Storage/Files/FileImportMode.php Normal file
View file

@ -0,0 +1,7 @@
<?php
namespace Misuzu\Storage\Files;
enum FileImportMode {
case Copy;
case Move;
}

50
src/Storage/Files/FileInfo.php Normal file
View file

@ -0,0 +1,50 @@
<?php
namespace Misuzu\Storage\Files;
use Carbon\CarbonImmutable;
use Index\UriBase64;
use Index\Db\DbResult;
class FileInfo {
public function __construct(
public private(set) string $id,
public private(set) string $hash,
public private(set) string $type,
public private(set) int $size,
public private(set) int $createdTime,
) {}
public static function fromResult(DbResult $result): FileInfo {
return new FileInfo(
id: $result->getString(0),
hash: $result->getString(1),
type: $result->getString(2),
size: $result->getInteger(3),
createdTime: $result->getInteger(4),
);
}
public string $hashString {
get => UriBase64::encode($this->hash);
}
public string $hashHexString {
get => bin2hex($this->hash);
}
public CarbonImmutable $createdAt {
get => CarbonImmutable::createFromTimestampUTC($this->createdTime);
}
public bool $isImage {
get => str_starts_with($this->type, 'image/');
}
public bool $isVideo {
get => str_starts_with($this->type, 'video/');
}
public bool $isAudio {
get => str_starts_with($this->type, 'audio/');
}
}

7
src/Storage/Files/FileInfoGetFileField.php Normal file
View file

@ -0,0 +1,7 @@
<?php
namespace Misuzu\Storage\Files;
enum FileInfoGetFileField {
case Id;
case Hash;
}

221
src/Storage/Files/FilesContext.php Normal file
View file

@ -0,0 +1,221 @@
<?php
namespace Misuzu\Storage\Files;
use Imagick;
use ImagickException;
use InvalidArgumentException;
use RuntimeException;
use Index\Db\DbConnection;
use Index\Snowflake\BinarySnowflake;
use Misuzu\Storage\Pools\Rules\EnsureVariantRuleThumb;
use Misuzu\Storage\Uploads\UploadVariantInfo;
class FilesContext {
public private(set) FilesData $data;
public private(set) FilesStorage $storage;
public function __construct(
string $localPath,
string $remotePath,
DbConnection $dbConn,
BinarySnowflake $snowflake
) {
$this->storage = new FilesStorage($localPath, $remotePath);
$this->data = new FilesData($dbConn, $snowflake);
}
public function deleteFile(
FileInfo|string $fileInfo,
bool $ensureOrphan = true
): void {
if(!($fileInfo instanceof FileInfo))
$fileInfo = $this->data->getFile($fileInfo);
if($ensureOrphan)
$this->data->unlinkFileUploads($fileInfo);
if($this->storage->delete($fileInfo))
$this->data->deleteFile($fileInfo);
}
public function importFile(
string $path,
FileImportMode $mode = FileImportMode::Move,
?string $type = null,
?string $hash = null
): FileInfo {
if(!is_file($path))
throw new InvalidArgumentException('$path is not a file that exists');
$type ??= mime_content_type($path);
$size = filesize($path);
$hash = $this->storage->import($path, $mode, $hash);
try {
return $this->data->createFile($hash, $type, $size);
} catch(RuntimeException $ex) {
return $this->data->getFile($hash, FileInfoGetFileField::Hash);
}
}
public static function supportsThumbnailing(FileInfo $fileInfo): bool {
return $fileInfo->isImage
|| $fileInfo->isAudio
|| $fileInfo->isVideo;
}
public static function supportsSquareCropping(FileInfo $fileInfo): bool {
return $fileInfo->isImage;
}
public function readIntoImagick(Imagick $imagick, FileInfo $fileInfo): void {
try {
$path = $this->storage->getLocalPath($fileInfo);
if($fileInfo->isImage) {
try {
$file = fopen($path, 'rb');
$imagick->readImageFile($file);
} finally {
if(isset($file) && is_resource($file))
fclose($file);
}
return;
}
if($fileInfo->isAudio) {
try {
$file = popen(sprintf('ffmpeg -i %s -an -f image2pipe -c:v copy -frames:v 1 2>/dev/null -', escapeshellarg($path)), 'rb');
$imagick->readImageBlob(stream_get_contents($file));
} finally {
if(isset($file) && is_resource($file))
pclose($file);
}
return;
}
if($fileInfo->isVideo) {
try {
$file = popen(sprintf('ffmpeg -i %s -f image2pipe -c:v png -frames:v 1 2>/dev/null -', escapeshellarg($path)), 'rb');
$imagick->readImageBlob(stream_get_contents($file));
} finally {
if(isset($file) && is_resource($file))
pclose($file);
}
}
} catch(ImagickException $ex) {
if($ex->getCode() !== 1)
throw $ex;
$imagick->newImage(1, 1, 'black', 'bmp');
}
}
public function createThumbnailFromRule(
FileInfo $original,
EnsureVariantRuleThumb $rules
): FileInfo {
return $this->createThumbnail(
$original,
$rules->width,
$rules->height,
$rules->quality,
$rules->format,
$rules->background
);
}
public function createThumbnail(
FileInfo $original,
int $width = 300,
int $height = 300,
int $quality = 80,
string $format = 'jpg',
string $background = 'black'
): FileInfo {
if(!self::supportsThumbnailing($original))
throw new InvalidArgumentException('$original does not support thumbnailing');
if($width < 1)
throw new InvalidArgumentException('$width must be greater than 0');
if($height < 1)
throw new InvalidArgumentException('$height must be greater than 0');
if($quality < 0 || $quality > 100)
throw new InvalidArgumentException('$quality must be a positive integer, not greater than 100');
$format = strtoupper($format);
if(!in_array($format, Imagick::queryFormats()))
throw new InvalidArgumentException('$format is not a supported format');
$type = 'application/octet-stream';
$path = tempnam(sys_get_temp_dir(), 'eeprom-thumbnail-');
$imagick = new Imagick;
try {
$this->readIntoImagick($imagick, $original);
$imagick->setImageFormat($format);
$imagick->setImageCompressionQuality($quality);
$imagick->setImageBackgroundColor($background);
$imagick->thumbnailImage($width, $height, true, true);
$type = $imagick->getImageMimeType();
$imagick->writeImage(sprintf('%s:%s', $format, $path));
} finally {
$imagick->clear();
}
if(!is_file($path))
throw new RuntimeException('failed to create thumbnail file');
return $this->importFile($path, FileImportMode::Move, $type);
}
public function createSquareCropped(FileInfo $original, int $dimensions): FileInfo {
if($dimensions < 1)
throw new InvalidArgumentException('$dimensions must be greater than 0');
$sourcePath = $this->storage->getLocalPath($original);
if(!is_file($sourcePath))
throw new RuntimeException('local data for provided $original does not exist');
$targetPath = tempnam(sys_get_temp_dir(), sprintf('eeprom-crop-%d-', $dimensions));
$imagick = new Imagick;
try {
if($original->isImage)
try {
$file = fopen($sourcePath, 'rb');
$imagick->readImageFile($file);
} finally {
if(isset($file) && is_resource($file))
fclose($file);
}
$width = $imagick->getImageWidth();
$height = $imagick->getImageHeight();
if($width > $height) {
$width = (int)($width * $dimensions / $height);
$height = $dimensions;
} else {
$width = $dimensions;
$height = (int)($height * $dimensions / $width);
}
do {
$imagick->resizeImage($width, $height, Imagick::FILTER_LANCZOS, 0.9);
$imagick->cropImage(
$dimensions,
$dimensions,
(int)ceil(($width - $dimensions) / 2),
(int)ceil(($height - $dimensions) / 2)
);
$imagick->setImagePage($dimensions, $dimensions, 0, 0);
} while($imagick->nextImage());
$imagick->writeImage($targetPath);
} finally {
$imagick->clear();
}
if(!is_file($targetPath))
throw new RuntimeException('failed to create cropped file');
return $this->importFile($targetPath, FileImportMode::Move, $original->type);
}
}

154
src/Storage/Files/FilesData.php Normal file
View file

@ -0,0 +1,154 @@
<?php
namespace Misuzu\Storage\Files;
use InvalidArgumentException;
use RuntimeException;
use Index\Db\{DbConnection,DbStatementCache};
use Index\Snowflake\BinarySnowflake;
class FilesData {
private DbStatementCache $cache;
public function __construct(
DbConnection $dbConn,
private BinarySnowflake $snowflake
) {
$this->cache = new DbStatementCache($dbConn);
}
/** @return iterable<FileInfo> */
public function getFiles(
?bool $orphaned = null,
?bool $denied = null
): iterable {
$hasOrphaned = $orphaned !== null;
$hasDenied = $denied !== null;
$query = <<<SQL
SELECT f.file_id, f.file_hash, f.file_type, f.file_size, UNIX_TIMESTAMP(f.file_created)
FROM msz_storage_files AS f
SQL;
if($hasOrphaned)
$query .= <<<SQL
LEFT JOIN msz_storage_uploads_files AS uf ON f.file_id = uf.file_id
SQL;
if($hasDenied)
$query .= <<<SQL
LEFT JOIN msz_storage_denylist AS dl ON f.file_hash = dl.deny_hash
SQL;
$args = 0;
if($orphaned !== null) {
++$args;
$query .= sprintf(
' WHERE uf.file_id %s NULL',
$orphaned ? 'IS' : 'IS NOT'
);
}
if($denied !== null)
$query .= sprintf(
' %s dl.deny_hash %s NULL',
++$args > 1 ? 'OR' : 'WHERE',
$denied ? 'IS NOT' : 'IS'
);
$stmt = $this->cache->get($query);
$stmt->execute();
return $stmt->getResult()->getIterator(FileInfo::fromResult(...));
}
public function getFile(
string $value,
FileInfoGetFileField $field = FileInfoGetFileField::Id
): FileInfo {
$field = match($field) {
FileInfoGetFileField::Id => 'file_id',
FileInfoGetFileField::Hash => 'file_hash',
};
$stmt = $this->cache->get(<<<SQL
SELECT file_id, file_hash, file_type, file_size, UNIX_TIMESTAMP(file_created)
FROM msz_storage_files
WHERE {$field} = ?
SQL);
$stmt->nextParameter($value);
$stmt->execute();
$result = $stmt->getResult();
if(!$result->next())
throw new RuntimeException('could not find that file');
return FileInfo::fromResult($result);
}
public function createFile(
string $hash,
string $type,
int $size
): FileInfo {
if(strlen($hash) !== 32)
throw new InvalidArgumentException('$hash must be a 32 byte hash string');
if(mb_strlen($type, 'UTF-8') > 255)
throw new InvalidArgumentException('$type is too long');
if($size < 0)
throw new InvalidArgumentException('$size may not be negative');
$fileId = (string)$this->snowflake->next($hash);
$stmt = $this->cache->get(<<<SQL
INSERT INTO msz_storage_files (
file_id, file_hash, file_type, file_size
) VALUES (?, ?, ?, ?)
SQL);
$stmt->nextParameter($fileId);
$stmt->nextParameter($hash);
$stmt->nextParameter($type);
$stmt->nextParameter($size);
$stmt->execute();
return $this->getFile($fileId);
}
public function deleteFile(
FileInfo|string $value,
FileInfoGetFileField $field = FileInfoGetFileField::Hash
): void {
if($value instanceof FileInfo) {
$value = $value->id;
$field = 'file_id';
} else
$field = match($field) {
FileInfoGetFileField::Id => 'file_id',
FileInfoGetFileField::Hash => 'file_hash',
};
$stmt = $this->cache->get(<<<SQL
DELETE FROM msz_storage_files
WHERE {$field} = ?
SQL);
$stmt->nextParameter($value);
$stmt->execute();
}
public function unlinkFileUploads(
FileInfo|string $value,
FileInfoGetFileField $field = FileInfoGetFileField::Hash
): void {
if($value instanceof FileInfo) {
$field = '?';
$value = $value->id;
} else
$field = match($field) {
FileInfoGetFileField::Id => '?',
FileInfoGetFileField::Hash => '(SELECT file_id FROM msz_storage_files WHERE file_hash = ?)',
};
$stmt = $this->cache->get(<<<SQL
DELETE FROM msz_storage_uploads_files
WHERE file_id = {$field}
SQL);
$stmt->nextParameter($value);
$stmt->execute();
}
}

101
src/Storage/Files/FilesStorage.php Normal file
View file

@ -0,0 +1,101 @@
<?php
namespace Misuzu\Storage\Files;
use InvalidArgumentException;
use RuntimeException;
use Misuzu\Storage\HashHelpers;
class FilesStorage {
public function __construct(
private string $localPath,
private string $remotePath,
) {}
public static function stringifyHash(FileInfo|string $infoOrHash): string {
if($infoOrHash instanceof FileInfo)
return $infoOrHash->hashString;
return HashHelpers::toUriBase64String($infoOrHash);
}
public function getLocalPath(FileInfo|string $infoOrHash, bool $ensurePath = false): string {
$hash = self::stringifyHash($infoOrHash);
$dir1 = substr($hash, 0, 2);
$dir2 = substr($hash, 2, 2);
$dir = sprintf(
'%s%s%s%s%s',
$this->localPath,
DIRECTORY_SEPARATOR,
$dir1,
DIRECTORY_SEPARATOR,
$dir2
);
if($ensurePath && !is_dir($dir))
mkdir($dir, 0775, true);
return sprintf(
'%s%s%s',
$dir,
DIRECTORY_SEPARATOR,
$hash
);
}
public function getRemotePath(FileInfo|string $infoOrHash): string {
$hash = self::stringifyHash($infoOrHash);
$dir1 = substr($hash, 0, 2);
$dir2 = substr($hash, 2, 2);
return sprintf(
'%s%s%s%s%s%s%s',
$this->remotePath,
DIRECTORY_SEPARATOR,
$dir1,
DIRECTORY_SEPARATOR,
$dir2,
DIRECTORY_SEPARATOR,
$hash
);
}
public function exists(FileInfo|string $infoOrHash): bool {
return is_file($this->getLocalPath($infoOrHash));
}
public function import(
string $path,
FileImportMode $mode = FileImportMode::Move,
?string $hash = null
): string {
if(!is_file($path))
throw new InvalidArgumentException('$path does not exist or is not a file');
if($hash === null)
$hash = hash_file('sha256', $path, true);
elseif(strlen($hash) !== 32)
throw new InvalidArgumentException('$hash must be null or 32 bytes');
$target = $this->getLocalPath($hash, true);
if(is_file($target)) {
if($mode === FileImportMode::Move)
unlink($path);
} else {
if($mode === FileImportMode::Move)
$success = rename($path, $target);
elseif($mode === FileImportMode::Copy)
$success = copy($path, $target);
if(!$success)
throw new RuntimeException('unable to move or copy $path using given $mode');
}
return $hash;
}
public function delete(FileInfo|string $infoOrHash): bool {
$path = $this->getLocalPath($infoOrHash);
return !is_file($path) || unlink($path);
}
}

43
src/Storage/HashHelpers.php Normal file
View file

@ -0,0 +1,43 @@
<?php
namespace Misuzu\Storage;
use InvalidArgumentException;
use Index\UriBase64;
final class HashHelpers {
public static function toUriBase64String(string $hash): string {
// already a base64uri string, hopefully
if(strlen($hash) === 43 && ctype_print($hash))
return $hash;
if(strlen($hash) === 66 && str_starts_with(strtolower($hash), '0x'))
$hash = substr($hash, 2);
// hex base64, decode
if(strlen($hash) === 64 && ctype_xdigit($hash))
$hash = hex2bin($hash);
// SHA256 hash, encode
if(strlen($hash) === 32)
return UriBase64::encode($hash);
throw new InvalidArgumentException('$hash is not an acceptable format');
}
public static function toBytes(string $hash): string {
// already a raw sha256 hash!
if(strlen($hash) === 32)
return $hash;
// base64uri string
if(strlen($hash) === 43 && ctype_print($hash))
return UriBase64::decode($hash);
// hex, possible 0x prefixed
if(strlen($hash) === 66 && str_starts_with(strtolower($hash), '0x'))
$hash = substr($hash, 2);
if(strlen($hash) === 64)
return hex2bin($hash);
throw new InvalidArgumentException('$hash is not an acceptable format');
}
}

14
src/Storage/Pools/CheckRuleException.php Normal file
View file

@ -0,0 +1,14 @@
<?php
namespace Misuzu\Storage\Pools;
use RuntimeException;
class CheckRuleException extends RuntimeException {
/** @param array<string, array<scalar>|scalar> $args */
public function __construct(
public private(set) string $rule,
public private(set) array $args
) {
parent::__construct(sprintf('Check for rule "%s" failed.', $rule));
}
}

7
src/Storage/Pools/CheckablePoolRule.php Normal file
View file

@ -0,0 +1,7 @@
<?php
namespace Misuzu\Storage\Pools;
interface CheckablePoolRule extends PoolRule {
/** @throws CheckRuleException */
public function checkRule(CheckablePoolRuleArgs $args): void;
}

11
src/Storage/Pools/CheckablePoolRuleArgs.php Normal file
View file

@ -0,0 +1,11 @@
<?php
namespace Misuzu\Storage\Pools;
final class CheckablePoolRuleArgs {
public function __construct(
public private(set) string $fileName,
public private(set) int $fileSize,
public private(set) string $fileType,
public private(set) string $fileHash
) {}
}

52
src/Storage/Pools/PoolInfo.php Normal file
View file

@ -0,0 +1,52 @@
<?php
namespace Misuzu\Storage\Pools;
use Carbon\CarbonImmutable;
use Index\Db\DbResult;
class PoolInfo {
public function __construct(
public private(set) string $id,
public private(set) string $name,
#[\SensitiveParameter] private ?string $secret,
public private(set) int $createdTime,
public private(set) ?int $deprecatedTime
) {}
public static function fromResult(DbResult $result): PoolInfo {
return new PoolInfo(
id: $result->getString(0),
name: $result->getString(1),
secret: $result->getStringOrNull(2),
createdTime: $result->getInteger(3),
deprecatedTime: $result->getIntegerOrNull(4),
);
}
public CarbonImmutable $createdAt {
get => CarbonImmutable::createFromTimestampUTC($this->createdTime);
}
public bool $deprecated {
get => $this->deprecatedTime !== null;
}
public ?CarbonImmutable $deprecatedAt {
get => $this->deprecatedTime === null ? null : CarbonImmutable::createFromTimestampUTC($this->deprecatedTime);
}
public bool $protected {
get => $this->secret !== null;
}
public function verifyHash(#[\SensitiveParameter] string $data, string $hash, string $algo = 'sha256'): bool {
// not sure how you'd end up here but lets err on the side of caution for it :)
if($this->secret === null)
return false;
return hash_equals(
hash_hmac($algo, $data, $this->secret, true),
$hash
);
}
}

8
src/Storage/Pools/PoolInfoGetField.php Normal file
View file

@ -0,0 +1,8 @@
<?php
namespace Misuzu\Storage\Pools;
enum PoolInfoGetField {
case Id;
case Name;
case UploadId;
}

6
src/Storage/Pools/PoolRule.php Normal file
View file

@ -0,0 +1,6 @@
<?php
namespace Misuzu\Storage\Pools;
interface PoolRule {
public PoolRuleInfo $info { get; }
}

23
src/Storage/Pools/PoolRuleInfo.php Normal file
View file

@ -0,0 +1,23 @@
<?php
namespace Misuzu\Storage\Pools;
use Index\Db\DbResult;
class PoolRuleInfo {
/** @param mixed[] $params */
public function __construct(
public private(set) string $id,
public private(set) string $poolId,
public private(set) string $type,
public private(set) array $params
) {}
public static function fromResult(DbResult $result): PoolRuleInfo {
return new PoolRuleInfo(
id: $result->getString(0),
poolId: $result->getString(1),
type: $result->getString(2),
params: json_decode($result->getString(3), true),
);
}
}

44
src/Storage/Pools/PoolRules.php Normal file
View file

@ -0,0 +1,44 @@
<?php
namespace Misuzu\Storage\Pools;
use Index\XArray;
class PoolRules {
/** @param PoolRuleInfo[] $rulesRaw */
public function __construct(
public private(set) PoolInfo $pool,
public private(set) array $rulesRaw,
private PoolRulesRegistry $registry
) {}
/** @var PoolRule[] */
public array $rules {
get => XArray::select($this->rulesRaw, fn($rule) => $this->registry->create($rule));
}
public function getRuleRaw(string $type): ?PoolRuleInfo {
return XArray::first($this->rulesRaw, fn($rule) => strcasecmp($rule->type, $type) === 0);
}
public function getRule(string $type): ?PoolRule {
return $this->registry->create($this->getRuleRaw($type));
}
/** @return PoolRuleInfo[] */
public function getRulesRaw(string $type): array {
return XArray::where($this->rulesRaw, fn($rule) => strcasecmp($rule->type, $type) === 0);
}
/** @return PoolRule[] */
public function getRules(string $type): array {
return XArray::select($this->getRulesRaw($type), fn($rule) => $this->registry->create($rule));
}
/** @return CheckablePoolRule[] */
public function getCheckableRules(?string $type = null): array {
return XArray::where( // @phpstan-ignore return.type
$type === null ? $this->rules : $this->getRules($type),
fn($rule) => $rule instanceof CheckablePoolRule
);
}
}

27
src/Storage/Pools/PoolRulesRegistry.php Normal file
View file

@ -0,0 +1,27 @@
<?php
namespace Misuzu\Storage\Pools;
use RuntimeException;
class PoolRulesRegistry {
/** @var array<string, callable(PoolRuleInfo): PoolRule> */
private array $constructors = [];
/** @param callable(PoolRuleInfo): PoolRule $constructor */
public function register(string $type, callable $constructor): void {
if(array_key_exists($type, $this->constructors))
throw new RuntimeException('$type already registered');
$this->constructors[$type] = $constructor;
}
public function create(?PoolRuleInfo $ruleInfo): ?PoolRule {
if($ruleInfo === null)
return null;
if(!array_key_exists($ruleInfo->type, $this->constructors))
throw new RuntimeException('no constructor for a rule of the given type registered');
return $this->constructors[$ruleInfo->type]($ruleInfo);
}
}

95
src/Storage/Pools/PoolsContext.php Normal file
View file

@ -0,0 +1,95 @@
<?php
namespace Misuzu\Storage\Pools;
use RuntimeException;
use Index\XArray;
use Index\Db\DbConnection;
use Misuzu\Storage\Uploads\UploadVariantInfo;
class PoolsContext {
public private(set) PoolsData $pools;
public private(set) PoolRulesRegistry $rules;
/** @var array<string, PoolInfo> */
private array $names = [];
public function __construct(DbConnection $dbConn) {
$this->pools = new PoolsData($dbConn);
$this->rules = new PoolRulesRegistry;
$this->rules->register(Rules\AcceptTypesRule::TYPE, Rules\AcceptTypesRule::create(...));
$this->rules->register(Rules\ConstrainSizeRule::TYPE, Rules\ConstrainSizeRule::create(...));
$this->rules->register(Rules\EnsureVariantRule::TYPE, Rules\EnsureVariantRule::create(...));
$this->rules->register(Rules\OnePerUserRule::TYPE, Rules\OnePerUserRule::create(...));
$this->rules->register(Rules\RemoveStaleRule::TYPE, Rules\RemoveStaleRule::create(...));
$this->rules->register(Rules\ScanForumRule::TYPE, Rules\ScanForumRule::create(...));
}
public function getPoolRules(PoolInfo|string $poolInfo): PoolRules {
if(!($poolInfo instanceof PoolInfo))
$poolInfo = $this->pools->getPool($poolInfo);
return new PoolRules(
$poolInfo,
iterator_to_array($this->pools->getPoolRules(poolInfo: $poolInfo)),
$this->rules
);
}
public function getPoolFromVariant(UploadVariantInfo $variantInfo): PoolInfo {
return $this->pools->getPool($variantInfo->fileId, PoolInfoGetField::Id);
}
public function getPoolName(string $poolId): ?string {
if(!array_key_exists($poolId, $this->names))
$this->names[$poolId] = $this->pools->getPoolName($poolId);
return $this->names[$poolId];
}
/**
* @return array{
* name: string,
* protected?: bool,
* created_at: string,
* deprecated_at?: string,
* accept_types?: string[],
* max_bytes?: int,
* idle_time?: int,
* variants?: string[],
* }
*/
public function extractPool(PoolInfo $poolInfo): array {
$body = [];
$body['name'] = $poolInfo->name;
if($poolInfo->protected)
$body['protected'] = true;
$body['created_at'] = $poolInfo->createdAt->toIso8601ZuluString();
if($poolInfo->deprecated)
$body['deprecated_at'] = $poolInfo->deprecatedAt->toIso8601ZuluString();
$rules = $this->getPoolRules($poolInfo);
$acceptTypesRule = $rules->getRule(Rules\AcceptTypesRule::TYPE);
if($acceptTypesRule instanceof Rules\AcceptTypesRule)
$body['accept_types'] = $acceptTypesRule->types;
$maxSizeRule = $rules->getRule(Rules\ConstrainSizeRule::TYPE);
if($maxSizeRule instanceof Rules\ConstrainSizeRule && $maxSizeRule->maxSize)
$body['max_bytes'] = $maxSizeRule->maxSize;
$removeStaleRule = $rules->getRule(Rules\RemoveStaleRule::TYPE);
if($removeStaleRule instanceof Rules\RemoveStaleRule && $removeStaleRule->inactiveForSeconds > 0)
$body['idle_time'] = $removeStaleRule->inactiveForSeconds;
$ensureVariantRules = XArray::select(
$rules->getRules(Rules\EnsureVariantRule::TYPE),
fn($ensureVariantRule) => $ensureVariantRule->variant // @phpstan-ignore property.notFound
);
if(!empty($ensureVariantRules))
$body['variants'] = $ensureVariantRules;
return $body;
}
}

117
src/Storage/Pools/PoolsData.php Normal file
View file

@ -0,0 +1,117 @@
<?php
namespace Misuzu\Storage\Pools;
use InvalidArgumentException;
use RuntimeException;
use Index\Db\{DbConnection,DbStatementCache,DbTools};
class PoolsData {
private DbStatementCache $cache;
public function __construct(DbConnection $dbConn) {
$this->cache = new DbStatementCache($dbConn);
}
/** @return iterable<PoolInfo> */
public function getPools(
?bool $protected = null,
?bool $deprecated = null
): iterable {
$args = 0;
$query = <<<SQL
SELECT pool_id, pool_name, pool_secret, UNIX_TIMESTAMP(pool_created), UNIX_TIMESTAMP(pool_deprecated)
FROM msz_storage_pools
SQL;
if($protected !== null) {
++$args;
$query .= sprintf(
' WHERE pool_secret %s NULL',
$protected ? 'IS NOT' : 'IS'
);
}
if($deprecated !== null)
$query .= sprintf(
' %s pool_deprecated %s NULL',
++$args > 1 ? 'AND' : 'WHERE',
$deprecated ? 'IS NOT' : 'IS'
);
$stmt = $this->cache->get($query);
$stmt->execute();
return $stmt->getResult()->getIterator(PoolInfo::fromResult(...));
}
public function getPool(
string $value,
PoolInfoGetField $field = PoolInfoGetField::Id
): PoolInfo {
$field = match($field) {
PoolInfoGetField::Id => 'pool_id = ?',
PoolInfoGetField::Name => 'pool_name = ?',
PoolInfoGetField::UploadId => 'pool_id = (SELECT pool_id FROM msz_storage_uploads WHERE upload_id = ?)',
};
$stmt = $this->cache->get(<<<SQL
SELECT pool_id, pool_name, pool_secret, UNIX_TIMESTAMP(pool_created), UNIX_TIMESTAMP(pool_deprecated)
FROM msz_storage_pools
WHERE {$field}
SQL);
$stmt->nextParameter($value);
$stmt->execute();
$result = $stmt->getResult();
if(!$result->next())
throw new RuntimeException('could not find that file record');
return PoolInfo::fromResult($result);
}
public function getPoolName(string $poolId): ?string {
$stmt = $this->cache->get('SELECT pool_name FROM msz_storage_pools WHERE pool_id = ?');
$stmt->nextParameter($poolId);
$stmt->execute();
$result = $stmt->getResult();
return $result->next() ? $result->getStringOrNull(0) : null;
}
/**
* @param string|null|string[] $types
* @return iterable<PoolRuleInfo>
*/
public function getPoolRules(
PoolInfo|string|null $poolInfo = null,
string|array|null $types = null
): iterable {
$hasPoolInfo = $poolInfo !== null;
$hasTypes = $types !== null;
if(is_string($types))
$types = [$types];
$args = 0;
$query = <<<SQL
SELECT rule_id, pool_id, rule_type, rule_params
FROM msz_storage_pools_rules
SQL;
if($hasPoolInfo) {
++$args;
$query .= ' WHERE pool_id = ?';
}
if($hasTypes)
$query .= sprintf(' %s rule_type IN (%s)', ++$args > 1 ? 'AND' : 'WHERE', DbTools::prepareListString($types));
$stmt = $this->cache->get($query);
if($hasPoolInfo)
$stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo);
if($hasTypes)
foreach($types as $type) {
if(!is_string($type))
throw new InvalidArgumentException('all entries of $types but be a string');
$stmt->nextParameter($type);
}
$stmt->execute();
return $stmt->getResult()->getIterator(PoolRuleInfo::fromResult(...));
}
}

33
src/Storage/Pools/Rules/AcceptTypesRule.php Normal file
View file

@ -0,0 +1,33 @@
<?php
namespace Misuzu\Storage\Pools\Rules;
use Misuzu\Storage\Pools\{CheckablePoolRule,CheckablePoolRuleArgs,CheckRuleException,PoolRuleInfo};
class AcceptTypesRule implements CheckablePoolRule {
public const string TYPE = 'accept_types';
/** @param string[] $types */
public function __construct(
public private(set) PoolRuleInfo $info,
public private(set) array $types
) {}
public function isAccepted(string $type): bool {
return in_array($type, $this->types);
}
public function checkRule(CheckablePoolRuleArgs $args): void {
if(!$this->isAccepted($args->fileType))
throw new CheckRuleException(self::TYPE, [
'accepted' => $this->types,
'given' => $args->fileType,
]);
}
public static function create(PoolRuleInfo $info): AcceptTypesRule {
return new AcceptTypesRule(
$info,
isset($info->params['types']) && is_array($info->params['types']) ? $info->params['types'] : [],
);
}
}

31
src/Storage/Pools/Rules/ConstrainSizeRule.php Normal file
View file

@ -0,0 +1,31 @@
<?php
namespace Misuzu\Storage\Pools\Rules;
use Misuzu\Storage\Pools\{CheckablePoolRule,CheckablePoolRuleArgs,CheckRuleException,PoolRuleInfo};
class ConstrainSizeRule implements CheckablePoolRule {
public const string TYPE = 'constrain_size';
public function __construct(
public private(set) PoolRuleInfo $info,
public private(set) int $maxSize,
public private(set) bool $allowLegacyMultiplier = false
) {}
public function checkRule(CheckablePoolRuleArgs $args): void {
// note: this intentionally ignores allowLegacyMultiplier
if($args->fileSize > $this->maxSize)
throw new CheckRuleException(self::TYPE, [
'max' => $this->maxSize,
'given' => $args->fileSize,
]);
}
public static function create(PoolRuleInfo $info): ConstrainSizeRule {
return new ConstrainSizeRule(
$info,
isset($info->params['max_size']) && is_int($info->params['max_size']) ? max(0, $info->params['max_size']) : 0,
isset($info->params['allow_legacy_multiplier']) && is_bool($info->params['allow_legacy_multiplier']) && $info->params['allow_legacy_multiplier']
);
}
}

53
src/Storage/Pools/Rules/EnsureVariantRule.php Normal file
View file

@ -0,0 +1,53 @@
<?php
namespace Misuzu\Storage\Pools\Rules;
use RuntimeException;
use Misuzu\Storage\Pools\{PoolRule,PoolRuleInfo};
class EnsureVariantRule implements PoolRule {
public const string TYPE = 'ensure_variant';
/** @var array<string, callable(mixed[]): EnsureVariantRuleParams> */
private static array $infoTypes;
public static function init(): void {
self::$infoTypes = [
'crop' => EnsureVariantRuleCrop::create(...),
'thumb' => EnsureVariantRuleThumb::create(...),
];
}
/** @param string[] $when */
public function __construct(
public private(set) PoolRuleInfo $info,
public private(set) string $variant,
public private(set) array $when,
public private(set) EnsureVariantRuleParams $params
) {}
public bool $onAccess {
get => in_array('access', $this->when);
}
public bool $onCron {
get => in_array('cron', $this->when);
}
public static function create(PoolRuleInfo $info): EnsureVariantRule {
if(!isset($info->params['info']) || !is_array($info->params['info']))
throw new RuntimeException('missing info key in params');
$params = $info->params['info'];
if(!isset($params['$kind']) || !is_string($params['$kind']) || !array_key_exists($params['$kind'], self::$infoTypes))
throw new RuntimeException('unsupported info kind in params');
return new EnsureVariantRule(
$info,
isset($info->params['name']) && is_string($info->params['name']) ? $info->params['name'] : '',
isset($info->params['when']) && is_array($info->params['when']) ? $info->params['when'] : [],
self::$infoTypes[$params['$kind']]($params)
);
}
}
EnsureVariantRule::init();

26
src/Storage/Pools/Rules/EnsureVariantRuleCrop.php Normal file
View file

@ -0,0 +1,26 @@
<?php
namespace Misuzu\Storage\Pools\Rules;
use InvalidArgumentException;
class EnsureVariantRuleCrop implements EnsureVariantRuleParams {
public const string KIND = 'crop';
public function __construct(
public private(set) int $width,
public private(set) int $height
) {
if($width < 1)
throw new InvalidArgumentException('$width must be greater than 0');
if($height < 1)
throw new InvalidArgumentException('$height must be greater than 0');
}
/** @param mixed[] $params */
public static function create(array $params): EnsureVariantRuleCrop {
return new EnsureVariantRuleCrop(
isset($params['w']) && is_int($params['w']) ? $params['w'] : 0,
isset($params['h']) && is_int($params['h']) ? $params['h'] : 0,
);
}
}

4
src/Storage/Pools/Rules/EnsureVariantRuleParams.php Normal file
View file

@ -0,0 +1,4 @@
<?php
namespace Misuzu\Storage\Pools\Rules;
interface EnsureVariantRuleParams {}

34
src/Storage/Pools/Rules/EnsureVariantRuleThumb.php Normal file
View file

@ -0,0 +1,34 @@
<?php
namespace Misuzu\Storage\Pools\Rules;
use InvalidArgumentException;
class EnsureVariantRuleThumb implements EnsureVariantRuleParams {
public const string KIND = 'thumb';
public function __construct(
public private(set) int $width,
public private(set) int $height,
public private(set) int $quality,
public private(set) string $format,
public private(set) string $background
) {
if($width < 1)
throw new InvalidArgumentException('$width must be greater than 0');
if($height < 1)
throw new InvalidArgumentException('$height must be greater than 0');
if($quality < 0 || $quality > 100)
throw new InvalidArgumentException('$quality must be a positive integer, not greater than 100');
}
/** @param mixed[] $params */
public static function create(array $params): EnsureVariantRuleThumb {
return new EnsureVariantRuleThumb(
isset($params['w']) && is_int($params['w']) ? $params['w'] : 0,
isset($params['h']) && is_int($params['h']) ? $params['h'] : 0,
isset($params['q']) && is_int($params['q']) ? $params['q'] : 80,
isset($params['f']) && is_string($params['f']) ? $params['f'] : 'jpg',
isset($params['b']) && is_string($params['b']) ? $params['b'] : 'black'
);
}
}

16
src/Storage/Pools/Rules/OnePerUserRule.php Normal file
View file

@ -0,0 +1,16 @@
<?php
namespace Misuzu\Storage\Pools\Rules;
use Misuzu\Storage\Pools\{PoolRule,PoolRuleInfo};
class OnePerUserRule implements PoolRule {
public const string TYPE = 'one_per_user';
public function __construct(
public private(set) PoolRuleInfo $info
) {}
public static function create(PoolRuleInfo $info): OnePerUserRule {
return new OnePerUserRule($info);
}
}

31
src/Storage/Pools/Rules/RemoveStaleRule.php Normal file
View file

@ -0,0 +1,31 @@
<?php
namespace Misuzu\Storage\Pools\Rules;
use Misuzu\Storage\Pools\{PoolRule,PoolRuleInfo};
use Misuzu\Storage\Uploads\UploadInfo;
class RemoveStaleRule implements PoolRule {
public const string TYPE = 'remove_stale';
/** @param string[] $ignoreUploadIds */
public function __construct(
public private(set) PoolRuleInfo $info,
public private(set) int $inactiveForSeconds,
public private(set) array $ignoreUploadIds
) {}
public function isIgnored(UploadInfo|string $uploadId): bool {
if($uploadId instanceof UploadInfo)
$uploadId = $uploadId->id;
return in_array($uploadId, $this->ignoreUploadIds);
}
public static function create(PoolRuleInfo $info): RemoveStaleRule {
return new RemoveStaleRule(
$info,
isset($info->params['inactive_for']) && is_int($info->params['inactive_for']) ? max(0, $info->params['inactive_for']) : 0,
isset($info->params['ignore']) && is_array($info->params['ignore']) ? $info->params['ignore'] : []
);
}
}

16
src/Storage/Pools/Rules/ScanForumRule.php Normal file
View file

@ -0,0 +1,16 @@
<?php
namespace Misuzu\Storage\Pools\Rules;
use Misuzu\Storage\Pools\{PoolRule,PoolRuleInfo};
class ScanForumRule implements PoolRule {
public const string TYPE = 'scan_forum';
public function __construct(
public private(set) PoolRuleInfo $info
) {}
public static function create(PoolRuleInfo $info): ScanForumRule {
return new ScanForumRule($info);
}
}

33
src/Storage/StorageContext.php Normal file
View file

@ -0,0 +1,33 @@
<?php
namespace Misuzu\Storage;
use Index\Dependencies;
use Index\Config\Config;
class StorageContext {
public private(set) Denylist\DenylistContext $denylistCtx;
public private(set) Files\FilesContext $filesCtx;
public private(set) Pools\PoolsContext $poolsCtx;
public private(set) Tasks\TasksContext $tasksCtx;
public private(set) Uploads\UploadsContext $uploadsCtx;
public function __construct(
Dependencies $deps,
Config $config,
string $localPath,
string $remotePath,
) {
$deps->register($this->denylistCtx = $deps->constructLazy(Denylist\DenylistContext::class));
$deps->register($this->filesCtx = $deps->constructLazy(
Files\FilesContext::class,
localPath: $localPath,
remotePath: $remotePath,
));
$deps->register($this->poolsCtx = $deps->constructLazy(Pools\PoolsContext::class));
$deps->register($this->tasksCtx = $deps->constructLazy(Tasks\TasksContext::class));
$deps->register($this->uploadsCtx = $deps->constructLazy(
Uploads\UploadsContext::class,
config: $config->scopeTo('uploads'),
));
}
}

76
src/Storage/Tasks/TaskInfo.php Normal file
View file

@ -0,0 +1,76 @@
<?php
namespace Misuzu\Storage\Tasks;
use Carbon\CarbonImmutable;
use Index\{UriBase64,XNumber};
use Index\Db\DbResult;
class TaskInfo {
public const int LIFETIME = 10 * 60;
public function __construct(
public private(set) string $id,
public private(set) string $secret,
public private(set) string $userId,
public private(set) string $poolId,
public private(set) TaskState $state,
public private(set) string $remoteAddress,
public private(set) string $name,
public private(set) int $size,
public private(set) string $type,
public private(set) string $hash,
public private(set) int $createdTime
) {}
public static function fromResult(DbResult $result): TaskInfo {
return new TaskInfo(
id: $result->getString(0),
secret: $result->getString(1),
userId: $result->getString(2),
poolId: $result->getString(3),
state: TaskState::tryFrom($result->getString(4)) ?? TaskState::Pending,
remoteAddress: $result->getString(5),
name: $result->getString(6),
size: $result->getInteger(7),
type: $result->getString(8),
hash: $result->getString(9),
createdTime: $result->getInteger(10),
);
}
public string $id62 {
get => XNumber::toBase62((int)$this->id);
}
public string $idWeb {
get => $this->id62 . $this->secret;
}
public string $uploadSecret {
get => $this->secret[0] . $this->secret[4] . $this->secret[8] . $this->secret[12];
}
public string $uploadIdWeb {
get => $this->id62 . $this->uploadSecret;
}
public string $hashString {
get => UriBase64::encode($this->hash);
}
public CarbonImmutable $createdAt {
get => CarbonImmutable::createFromTimestampUTC($this->createdTime);
}
public int $expiresTime {
get => $this->createdTime + self::LIFETIME;
}
public CarbonImmutable $expiresAt {
get => CarbonImmutable::createFromTimestampUTC($this->expiresTime);
}
public function verifySecret(#[\SensitiveParameter] string $secret): bool {
return hash_equals($this->secret, $secret);
}
}

8
src/Storage/Tasks/TaskState.php Normal file
View file

@ -0,0 +1,8 @@
<?php
namespace Misuzu\Storage\Tasks;
enum TaskState: string {
case Pending = 'pending';
case Complete = 'complete';
case Error = 'error';
}

34
src/Storage/Tasks/TasksContext.php Normal file
View file

@ -0,0 +1,34 @@
<?php
namespace Misuzu\Storage\Tasks;
use Index\Config\Config;
use Index\Db\DbConnection;
use Index\Snowflake\RandomSnowflake;
class TasksContext {
public const int CHUNK_SIZE = 4 * 1024 * 1024;
public private(set) TasksData $tasks;
public function __construct(
private Config $config,
DbConnection $dbConn,
RandomSnowflake $snowflake
) {
$this->tasks = new TasksData($dbConn, $snowflake);
}
public function getTaskUrl(TaskInfo|string $idWeb): string {
if($idWeb instanceof TaskInfo)
$idWeb = $idWeb->idWeb;
return sprintf('https://%s/uploads/%s', $this->config->getString('api'), $idWeb);
}
public function getTaskWorkingPath(TaskInfo|string $taskInfo): string {
if($taskInfo instanceof TaskInfo)
$taskInfo = $taskInfo->id;
return implode(DIRECTORY_SEPARATOR, [sys_get_temp_dir(), sprintf('eeprom-upload-%s', $taskInfo)]);
}
}

125
src/Storage/Tasks/TasksData.php Normal file
View file

@ -0,0 +1,125 @@
<?php
namespace Misuzu\Storage\Tasks;
use InvalidArgumentException;
use RuntimeException;
use Stringable;
use Index\XString;
use Index\Db\{DbConnection,DbStatementCache,DbTools};
use Index\Snowflake\RandomSnowflake;
use Misuzu\Storage\Pools\PoolInfo;
class TasksData {
private DbStatementCache $cache;
public function __construct(
DbConnection $dbConn,
private RandomSnowflake $snowflake
) {
$this->cache = new DbStatementCache($dbConn);
}
/** @return iterable<TaskInfo> */
public function getTasks(
?TaskState $state = null,
): iterable {
$hasState = $state !== null;
$args = 0;
$query = <<<SQL
SELECT task_id, task_secret, user_id, pool_id, task_state, INET6_NTOA(task_ip),
task_name, task_size, task_type, task_hash, UNIX_TIMESTAMP(task_created)
FROM msz_storage_tasks
SQL;
if($hasState) {
++$args;
$query .= ' WHERE task_state = ?';
}
$stmt = $this->cache->get($query);
if($hasState)
$stmt->nextParameter($state->value);
$stmt->execute();
return $stmt->getResult()->getIterator(TaskInfo::fromResult(...));
}
public function getTask(
string $taskId,
): TaskInfo {
$stmt = $this->cache->get(<<<SQL
SELECT task_id, task_secret, user_id, pool_id, task_state, INET6_NTOA(task_ip),
task_name, task_size, task_type, task_hash, UNIX_TIMESTAMP(task_created)
FROM msz_storage_tasks
WHERE task_id = ?
SQL);
$stmt->nextParameter($taskId);
$stmt->execute();
$result = $stmt->getResult();
if(!$result->next())
throw new RuntimeException('task not found');
return TaskInfo::fromResult($result);
}
public function getTaskError(TaskInfo|string $taskInfo): string {
$stmt = $this->cache->get('SELECT task_error FROM msz_storage_tasks WHERE task_id = ?');
$stmt->nextParameter($taskInfo instanceof TaskInfo ? $taskInfo->id : $taskInfo);
$stmt->execute();
$result = $stmt->getResult();
return $result->next() ? $result->getString(0) : '';
}
public function createTask(
PoolInfo|string $poolInfo,
string $userId,
string $remoteAddr,
string $name,
int $size,
string $type,
string $hash
): TaskInfo {
$taskId = (string)$this->snowflake->next();
$stmt = $this->cache->get(<<<SQL
INSERT INTO msz_storage_tasks (
task_id, task_secret, user_id, pool_id, task_ip,
task_name, task_size, task_type, task_hash
) VALUES (?, ?, ?, ?, INET6_ATON(?), ?, ?, ?, ?)
SQL);
$stmt->nextParameter($taskId);
$stmt->nextParameter(XString::random(16));
$stmt->nextParameter($userId);
$stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo);
$stmt->nextParameter($remoteAddr);
$stmt->nextParameter($name);
$stmt->nextParameter($size);
$stmt->nextParameter($type);
$stmt->nextParameter($hash);
$stmt->execute();
return $this->getTask($taskId);
}
public function deleteTask(TaskInfo|string $taskInfo): void {
$stmt = $this->cache->get('DELETE FROM msz_storage_tasks WHERE task_id = ?');
$stmt->nextParameter($taskInfo instanceof TaskInfo ? $taskInfo->id : $taskInfo);
$stmt->execute();
}
public function setTaskState(TaskInfo|string $taskInfo, TaskState $state): void {
$stmt = $this->cache->get('UPDATE msz_storage_tasks SET task_state = ? WHERE task_id = ? AND task_state != "error"');
$stmt->nextParameter($state->value);
$stmt->nextParameter($taskInfo instanceof TaskInfo ? $taskInfo->id : $taskInfo);
$stmt->execute();
}
public function setTaskError(TaskInfo|string $taskInfo, Stringable|string $error): void {
$stmt = $this->cache->get('UPDATE msz_storage_tasks SET task_state = "error", task_error = ? WHERE task_id = ?');
$stmt->nextParameter((string)$error);
$stmt->nextParameter($taskInfo instanceof TaskInfo ? $taskInfo->id : $taskInfo);
$stmt->execute();
}
}

203
src/Storage/Tasks/TasksRoutes.php Normal file
View file

@ -0,0 +1,203 @@
<?php
namespace Misuzu\Storage\Tasks;
use RuntimeException;
use Index\{ByteFormat,XNumber};
use Index\Http\{HttpResponseBuilder,HttpRequest};
use Index\Http\Routing\{HttpPut,RouteHandler,RouteHandlerCommon};
use Index\Http\Routing\AccessControl\AccessControl;
use Index\Http\Routing\Routes\PatternRoute;
class TasksRoutes implements RouteHandler {
use RouteHandlerCommon;
public function __construct(
private TasksContext $tasksCtx,
) {}
/** @return ''|array{error: string, english: string} */
#[AccessControl(allowHeaders: ['X-Content-Index'])]
#[PatternRoute('PUT', '/storage/tasks/([A-Za-z0-9]+)(?:-([a-z0-9]+))?(?:\.([A-Za-z0-9\-_]+))?')]
public function putUpload(
HttpResponseBuilder $response,
HttpRequest $request,
string $taskId,
string $variant = '',
string $extension = '',
) {
if($request->getHeaderLine('Content-Type') !== 'application/octet-stream') {
$response->statusCode = 400;
return [
'error' => 'bad_type',
'english' => 'Content-Type must be application/octet-stream.',
];
}
$content = (string)$request->getBody();
if(strlen($taskId) < 5) {
$response->statusCode = 404;
return [
'error' => 'not_found',
'english' => 'Requested upload task does not exist.',
];
}
if($variant !== '') {
$response->statusCode = 405;
$response->setAllow(['HEAD', 'GET']);
return [
'error' => 'method_not_allowed',
'english' => 'PUT requests are not supported for upload variants.',
];
}
$length = (int)$request->getHeaderLine('Content-Length');
if($length < 1) {
$response->statusCode = 411;
return [
'error' => 'length_required',
'english' => 'A Content-Length header with the size of the request body is required.',
];
}
if(strlen($content) !== $length) {
$response->statusCode = 400;
return [
'error' => 'length_mismatch',
'english' => 'Length specified in Content-Length differs from actual request body length.',
];
}
if($length > TasksContext::CHUNK_SIZE) {
$response->statusCode = 413;
return [
'error' => 'content_too_large',
'english' => sprintf(
'Chunks may not larger than %s, the given chunk was %s.',
ByteFormat::format(TasksContext::CHUNK_SIZE, false),
ByteFormat::format($length, false)
),
'chunk_size' => $length,
'max_size' => TasksContext::CHUNK_SIZE,
];
}
if($request->hasHeader('X-Content-Index')) {
if($request->hasParam('index')) {
$response->statusCode = 400;
return [
'error' => 'bad_param',
'english' => 'the index query parameter may not be used at the same time as the X-Content-Index header.',
];
}
$index = (int)filter_var($request->getHeaderLine('X-Content-Index'), FILTER_SANITIZE_NUMBER_INT);
} elseif($request->hasParam('index')) {
$index = (int)$request->getFilteredParam('index', FILTER_SANITIZE_NUMBER_INT);
} else $index = 0;
if($index < 0) {
$response->statusCode = 400;
return [
'error' => 'bad_index',
'english' => 'index parameter must be greater than or equal to 0.',
];
}
$taskSecret = substr($taskId, -16);
$taskId = (string)XNumber::fromBase62(substr($taskId, 0, -16));
try {
$taskInfo = $this->tasksCtx->tasks->getTask($taskId);
} catch(RuntimeException $ex) {
$response->statusCode = 404;
return [
'error' => 'not_found',
'english' => 'Requested upload task does not exist.',
];
}
if(!$taskInfo->verifySecret($taskSecret)) {
$response->statusCode = 404;
return [
'error' => 'not_found',
'english' => 'Requested upload task does not exist.',
];
}
if($taskInfo->state !== TaskState::Pending) {
$response->statusCode = 409;
return [
'error' => 'not_pending',
'english' => 'Requested upload task is no longer accepting data.',
];
}
$offset = TasksContext::CHUNK_SIZE * $index;
if($offset >= $taskInfo->size) {
$response->statusCode = 400;
return [
'error' => 'offset_too_large',
'english' => 'Provided offset is greater than file size.',
];
}
$path = $this->tasksCtx->getTaskWorkingPath($taskInfo);
if(!is_file($path)) {
$this->tasksCtx->tasks->setTaskError($taskInfo, sprintf('working file despawned during PUT request: %s', $path));
$response->statusCode = 500;
return [
'error' => 'server_error',
'english' => 'Working file no longer exist on the server, the upload process cannot continue.',
];
}
$handle = fopen($path, 'rb+');
if($handle === false) {
$this->tasksCtx->tasks->setTaskError($taskInfo, sprintf('failed to open during PUT request: %s', $path));
$response->statusCode = 500;
return [
'error' => 'server_error',
'english' => 'Was not able to open working file, the upload process cannot continue.',
];
}
if(fseek($handle, $offset, SEEK_SET) < 0) {
$this->tasksCtx->tasks->setTaskError($taskInfo, sprintf('was unable to seek during PUT request: %s', $path));
$response->statusCode = 500;
return [
'error' => 'server_error',
'english' => 'Was not able to seek to provided offset, the upload process cannot continue.',
];
}
if(fwrite($handle, $content, $length) === false) {
$this->tasksCtx->tasks->setTaskError($taskInfo, sprintf('was unable to write during PUT request: %s', $path));
$response->statusCode = 500;
return [
'error' => 'server_error',
'english' => 'Was not able to seek to write to file, the upload process cannot continue.',
];
}
if(!fflush($handle)) {
$this->tasksCtx->tasks->setTaskError($taskInfo, sprintf('failed to flush file contents during PUT request: %s', $path));
$response->statusCode = 500;
return [
'error' => 'server_error',
'english' => 'Was not able to flush file properly, the upload process cannot continue.',
];
}
if(!fclose($handle)) {
$this->tasksCtx->tasks->setTaskError($taskInfo, sprintf('failed to close file contents during PUT request: %s', $path));
$response->statusCode = 500;
return [
'error' => 'server_error',
'english' => 'Was not able to close file properly, the upload process cannot continue.',
];
}
$response->statusCode = 202;
return '';
}
}

52
src/Storage/Uploads/UploadInfo.php Normal file
View file

@ -0,0 +1,52 @@
<?php
namespace Misuzu\Storage\Uploads;
use Carbon\CarbonImmutable;
use Index\{MediaType,XNumber};
use Index\Db\DbResult;
class UploadInfo {
public function __construct(
public private(set) string $id,
public private(set) string $userId,
public private(set) string $poolId,
public private(set) string $secret,
public private(set) string $name,
public private(set) string $remoteAddress,
public private(set) int $createdTime,
public private(set) int $accessedTime,
) {}
public static function fromResult(DbResult $result): UploadInfo {
return new UploadInfo(
id: $result->getString(0),
userId: $result->getString(1),
poolId: $result->getString(2),
secret: $result->getString(3),
name: $result->getString(4),
remoteAddress: $result->getString(5),
createdTime: $result->getInteger(6),
accessedTime: $result->getInteger(7),
);
}
public string $id62 {
get => XNumber::toBase62((int)$this->id);
}
public string $idWeb {
get => $this->id62 . $this->secret;
}
public CarbonImmutable $createdAt {
get => CarbonImmutable::createFromTimestampUTC($this->createdTime);
}
public CarbonImmutable $accessedAt {
get => CarbonImmutable::createFromTimestampUTC($this->accessedTime);
}
public bool $accessed {
get => $this->accessedTime > $this->createdTime;
}
}

22
src/Storage/Uploads/UploadVariantInfo.php Normal file
View file

@ -0,0 +1,22 @@
<?php
namespace Misuzu\Storage\Uploads;
use Index\Db\DbResult;
class UploadVariantInfo {
public function __construct(
public private(set) string $uploadId,
public private(set) string $variant,
public private(set) string $fileId,
public private(set) ?string $type
) {}
public static function fromResult(DbResult $result): UploadVariantInfo {
return new UploadVariantInfo(
uploadId: $result->getString(0),
variant: $result->getString(1),
fileId: $result->getString(2),
type: $result->getStringOrNull(3),
);
}
}

185
src/Storage/Uploads/UploadsContext.php Normal file
View file

@ -0,0 +1,185 @@
<?php
namespace Misuzu\Storage\Uploads;
use DateTimeInterface;
use Stringable;
use Index\Config\Config;
use Index\Db\DbConnection;
use Index\Http\HttpUri;
use Index\Snowflake\RandomSnowflake;
use Index\Urls\UrlRegistry;
use Misuzu\SiteInfo;
use Misuzu\Storage\Files\{FilesContext,FileInfo};
use Misuzu\Storage\Pools\PoolsContext;
use Misuzu\Storage\Tasks\TaskInfo;
class UploadsContext {
public private(set) UploadsData $uploads;
public function __construct(
private Config $config,
DbConnection $dbConn,
RandomSnowflake $snowflake,
private PoolsContext $poolsCtx,
private FilesContext $filesCtx,
private ?UrlRegistry $urls,
private SiteInfo $siteInfo,
) {
$this->uploads = new UploadsData($dbConn, $snowflake);
}
/** @param array<string, Stringable|scalar|null|list<Stringable|scalar|null>> $args */
public function getUploadUrl(
UploadInfo|TaskInfo|string $idWeb,
string $variant = '',
string $extension = '',
bool $includeProto = true,
bool $forceApiDomain = false,
bool $useArgForVariant = false,
array $args = [],
): string {
if($idWeb instanceof UploadInfo)
$idWeb = $idWeb->idWeb;
elseif($idWeb instanceof TaskInfo)
$idWeb = $idWeb->uploadIdWeb;
$proto = $includeProto ? 'https:' : '';
if($variant !== '') {
if($useArgForVariant) {
$args['v'] = $variant;
$variant = '';
} else
$variant = sprintf('-%s', $variant);
}
if($extension !== '')
$extension = sprintf('.%s', $extension);
$args = empty($args) ? '' : sprintf('?%s', HttpUri::buildQueryString($args));
if($forceApiDomain || !$this->config->hasValues('short')) {
$domain = $this->siteInfo->domain;
$path = rtrim($this->urls?->format('storage-upload') ?? '/uploads', '/');
} else {
$domain = $this->config->getString('short');
$path = '';
}
return sprintf('%s//%s%s/%s%s%s%s', $proto, $domain, $path, $idWeb, $variant, $extension, $args);
}
/**
* @param array<string, ?scalar> $overrides
* @return array<string, ?scalar>
*/
public function convertToClientJsonV1(
UploadInfo $uploadInfo,
UploadVariantInfo $variantInfo,
FileInfo $fileInfo,
array $overrides = []
): array {
$ext = pathinfo($uploadInfo->name, PATHINFO_EXTENSION);
return array_merge([
'id' => $uploadInfo->idWeb,
'url' => $this->getUploadUrl($uploadInfo, extension: $ext, includeProto: false),
'urlf' => $this->getUploadUrl($uploadInfo, extension: $ext, includeProto: false, forceApiDomain: true),
'thumb' => $this->getUploadUrl($uploadInfo, 't', 'jpg', includeProto: false),
'name' => $uploadInfo->name,
'type' => $variantInfo->type ?? $fileInfo->type,
'size' => $fileInfo->size,
'user' => (int)$uploadInfo->userId,
'user_str' => $uploadInfo->userId,
'appl' => (int)$uploadInfo->poolId,
'appl_str' => $uploadInfo->poolId,
'hash' => $fileInfo->hashHexString,
'created' => $uploadInfo->createdAt->toIso8601ZuluString(),
'accessed' => $uploadInfo->accessedAt->toIso8601ZuluString(),
'expires' => $uploadInfo->accessedAt->add('weeks', 2)->toIso8601ZuluString(),
// These can never be reached, and in situation where they technically could it's because of an outdated local record
'deleted' => null,
'dmca' => null,
], $overrides);
}
/**
* @return array{
* name: string,
* type: string,
* size: int,
* hash: string,
* url: string,
* }
*/
public function extractUploadVariant(
UploadInfo $uploadInfo,
UploadVariantInfo $variantInfo,
FileInfo $fileInfo
): array {
$body = [];
$body['name'] = $variantInfo->variant;
$body['type'] = $variantInfo->type ?? $fileInfo->type;
$body['size'] = $fileInfo->size;
$body['hash'] = $fileInfo->hashString;
$body['url'] = $this->getUploadUrl($uploadInfo, $variantInfo->variant);
return $body;
}
/**
* @param ?UploadVariantInfo[] $variantInfos
* @return array{
* id: string,
* user_id?: string,
* pool?: string,
* name: string,
* created_at: string,
* accessed_at?: string,
* variants?: array{
* type: string,
* size: int,
* hash: string,
* url: string,
* }[]
* }
*/
public function extractUpload(
UploadInfo $uploadInfo,
bool $includeUserId = true,
bool|string $poolName = true,
?array $variantInfos = null
): array {
$body = [];
$body['id'] = $uploadInfo->idWeb;
if($includeUserId)
$body['user_id'] = $uploadInfo->userId;
if($poolName !== false)
$body['pool'] = $poolName === true ? $this->poolsCtx->getPoolName($uploadInfo->poolId) : $poolName;
$body['name'] = $uploadInfo->name;
$body['created_at'] = $uploadInfo->createdAt->toIso8601ZuluString();
if($uploadInfo->accessed)
$body['accessed_at'] = $uploadInfo->accessedAt->toIso8601ZuluString();
$variants = [];
$variantInfos ??= $this->uploads->getUploadVariants($uploadInfo);
foreach($variantInfos as $variantInfo) {
$fileInfo = $this->filesCtx->data->getFile($variantInfo->fileId);
if($variantInfo->variant === '') {
$body['type'] = $variantInfo->type ?? $fileInfo->type;
$body['size'] = $fileInfo->size;
$body['hash'] = $fileInfo->hashString;
$body['url'] = $this->getUploadUrl($uploadInfo);
continue;
}
$variants[] = $this->extractUploadVariant($uploadInfo, $variantInfo, $fileInfo);
}
if(!empty($variants))
$body['variants'] = $variants;
return $body;
}
}

378
src/Storage/Uploads/UploadsData.php Normal file
View file

@ -0,0 +1,378 @@
<?php
namespace Misuzu\Storage\Uploads;
use InvalidArgumentException;
use RuntimeException;
use Misuzu\Storage\Files\FileInfo;
use Misuzu\Storage\Pools\PoolInfo;
use Index\XString;
use Index\Db\{DbConnection,DbStatementCache,DbTools};
use Index\Snowflake\RandomSnowflake;
class UploadsData {
private DbStatementCache $cache;
public function __construct(
DbConnection $dbConn,
private RandomSnowflake $snowflake
) {
$this->cache = new DbStatementCache($dbConn);
}
/** @return iterable<UploadInfo> */
public function getUploads(
PoolInfo|string|null $poolInfo = null,
?string $userId = null,
?string $variant = null,
?bool $variantExists = null,
?int $beforeId = null,
?int $limit = null,
?bool $ascending = null
): iterable {
$hasPoolInfo = $poolInfo !== null;
$hasUserId = $userId !== null;
$hasVariant = $variant !== null;
$hasVariantExists = $variantExists !== null;
$hasBeforeId = $beforeId !== null;
$hasLimit = $limit !== null;
$query = <<<SQL
SELECT u.upload_id, u.user_id, u.pool_id, u.upload_secret, u.upload_name,
INET6_NTOA(u.upload_ip), UNIX_TIMESTAMP(u.upload_created), UNIX_TIMESTAMP(u.upload_accessed)
FROM msz_storage_uploads AS u
SQL;
if($hasVariant)
$query .= <<<SQL
LEFT JOIN msz_storage_uploads_files AS uf
ON u.upload_id = uf.upload_id AND uf.upload_variant = ?
SQL;
$args = 0;
if($hasPoolInfo) {
++$args;
$query .= ' WHERE u.pool_id = ?';
}
if($hasUserId)
$query .= sprintf(' %s u.user_id = ?', ++$args > 1 ? 'AND' : 'WHERE');
if($hasVariantExists)
$query .= sprintf(
' %s uf.upload_id %s NULL',
++$args > 1 ? 'AND' : 'WHERE',
$variantExists ? 'IS NOT' : 'IS'
);
if($hasBeforeId)
$query .= sprintf(' %s u.upload_id < ?', ++$args > 1 ? 'AND' : 'WHERE');
if($ascending !== null)
$query .= sprintf(
' ORDER BY u.upload_id %s',
$ascending ? 'ASC' : 'DESC'
);
if($hasLimit)
$query .= ' LIMIT ?';
$stmt = $this->cache->get($query);
if($hasVariant)
$stmt->nextParameter($variant);
if($hasPoolInfo)
$stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo);
if($hasUserId)
$stmt->nextParameter($userId);
if($hasBeforeId)
$stmt->nextParameter($beforeId);
if($hasLimit)
$stmt->nextParameter($limit);
$stmt->execute();
return $stmt->getResult()->getIterator(UploadInfo::fromResult(...));
}
public function getUpload(
?string $uploadId = null,
PoolInfo|string|null $poolInfo = null,
?string $userId = null,
?string $secret = null,
): UploadInfo {
$hasUploadId = $uploadId !== null;
$hasPoolInfo = $poolInfo !== null;
$hasUserId = $userId !== null;
$hasSecret = $secret !== null;
$args = 0;
$query = 'SELECT upload_id, user_id, pool_id, upload_secret, upload_name, INET6_NTOA(upload_ip), UNIX_TIMESTAMP(upload_created), UNIX_TIMESTAMP(upload_accessed) FROM msz_storage_uploads';
if($hasUploadId) {
++$args;
$query .= ' WHERE upload_id = ?';
}
if($hasPoolInfo)
$query .= sprintf(' %s pool_id = ?', ++$args > 1 ? 'AND' : 'WHERE');
if($hasUserId)
$query .= sprintf(' %s user_id = ?', ++$args > 1 ? 'AND' : 'WHERE');
if($hasSecret)
$query .= sprintf(' %s upload_secret = ?', ++$args > 1 ? 'AND' : 'WHERE');
$stmt = $this->cache->get($query);
if($hasUploadId)
$stmt->nextParameter($uploadId);
if($hasPoolInfo)
$stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo);
if($hasUserId)
$stmt->nextParameter($userId);
if($hasSecret)
$stmt->nextParameter($secret);
$stmt->execute();
$result = $stmt->getResult();
if(!$result->next())
throw new RuntimeException('upload not found');
return UploadInfo::fromResult($result);
}
public function resolveUploadFromVariant(
PoolInfo|string $poolInfo,
string $userId,
FileInfo|string $fileInfo,
string $variant
): string {
$stmt = $this->cache->get(<<<SQL
SELECT u.upload_id
FROM msz_storage_uploads AS u
LEFT JOIN msz_storage_uploads_files AS uf
ON uf.upload_id = u.upload_id
WHERE u.pool_id = ?
AND u.user_id = ?
AND uf.file_id = ?
AND uf.upload_variant = ?
SQL);
$stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo);
$stmt->nextParameter($userId);
$stmt->nextParameter($fileInfo instanceof FileInfo ? $fileInfo->id : $fileInfo);
$stmt->nextParameter($variant);
$stmt->execute();
$result = $stmt->getResult();
if(!$result->next())
throw new RuntimeException('upload could not be resolved');
return $result->getString(0);
}
public function createUpload(
PoolInfo|string $poolInfo,
string $userId,
string $fileName,
string $remoteAddr,
?string $uploadId = null,
?string $secret = null,
?int $createdTime = null
): UploadInfo {
$uploadId ??= (string)$this->snowflake->next();
$secret ??= XString::random(4);
$stmt = $this->cache->get('INSERT INTO msz_storage_uploads (upload_id, pool_id, user_id, upload_secret, upload_name, upload_ip, upload_created) VALUES (?, ?, ?, ?, ?, INET6_ATON(?), FROM_UNIXTIME(?))');
$stmt->nextParameter($uploadId);
$stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo);
$stmt->nextParameter($userId);
$stmt->nextParameter($secret);
$stmt->nextParameter($fileName);
$stmt->nextParameter($remoteAddr);
$stmt->nextParameter($createdTime);
$stmt->execute();
return $this->getUpload(uploadId: $uploadId);
}
public function updateUpload(
UploadInfo|string $uploadInfo,
?string $fileName = null,
int|bool $accessedAt = false,
): void {
$fields = [];
$values = [];
if($fileName !== null) {
$fields[] = 'upload_name = ?';
$values[] = $fileName;
}
if($accessedAt !== false) {
$fields[] = sprintf('upload_accessed = %s', $accessedAt === true ? 'NOW()' : 'FROM_UNIXTIME(?)');
if($accessedAt !== true)
$values[] = $accessedAt;
}
if(empty($fields))
return;
$stmt = $this->cache->get(sprintf('UPDATE msz_storage_uploads SET %s WHERE upload_id = ?', implode(', ', $fields)));
foreach($values as $value)
$stmt->nextParameter($value);
$stmt->nextParameter($uploadInfo instanceof UploadInfo ? $uploadInfo->id : $uploadInfo);
$stmt->execute();
}
public function deleteUpload(UploadInfo|string $uploadInfo): void {
$stmt = $this->cache->get('DELETE FROM msz_storage_uploads WHERE upload_id = ?');
$stmt->nextParameter($uploadInfo instanceof UploadInfo ? $uploadInfo->id : $uploadInfo);
$stmt->execute();
}
public function deleteUploadsByUser(
string $userId
): void {
$stmt = $this->cache->get('DELETE FROM msz_storage_uploads WHERE user_id = ?');
$stmt->nextParameter($userId);
$stmt->execute();
}
/** @param ?array<string|UploadInfo> $ignore */
public function deleteStaleUploads(
PoolInfo|string $poolInfo,
int $staleSeconds,
?array $ignore = null
): void {
$hasIgnore = !empty($ignore);
$query = <<<SQL
DELETE FROM msz_storage_uploads
WHERE pool_id = ?
AND upload_accessed <= (NOW() - INTERVAL ? SECOND)
SQL;
if($hasIgnore)
$query .= sprintf(
' AND upload_id NOT IN (%s)',
DbTools::prepareListString($ignore)
);
$stmt = $this->cache->get($query);
$stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo);
$stmt->nextParameter($staleSeconds);
if($hasIgnore)
foreach($ignore as $item) {
if($item instanceof UploadInfo)
$stmt->nextParameter($item->id);
elseif(is_string($item))
$stmt->nextParameter($item);
else
throw new InvalidArgumentException('$ignore contains entries with unsupported types');
}
$stmt->execute();
}
/** @return iterable<UploadVariantInfo> */
public function getUploadVariants(
UploadInfo|string $uploadInfo
): iterable {
$stmt = $this->cache->get(<<<SQL
SELECT upload_id, upload_variant, file_id, file_type
FROM msz_storage_uploads_files
WHERE upload_id = ?
SQL);
$stmt->nextParameter($uploadInfo instanceof UploadInfo ? $uploadInfo->id : $uploadInfo);
$stmt->execute();
return $stmt->getResult()->getIterator(UploadVariantInfo::fromResult(...));
}
public function getUploadVariant(
UploadInfo|string $uploadInfo,
string $variant
): UploadVariantInfo {
$stmt = $this->cache->get(<<<SQL
SELECT upload_id, upload_variant, file_id, file_type
FROM msz_storage_uploads_files
WHERE upload_id = ? AND upload_variant = ?
SQL);
$stmt->nextParameter($uploadInfo instanceof UploadInfo ? $uploadInfo->id : $uploadInfo);
$stmt->nextParameter($variant);
$stmt->execute();
$result = $stmt->getResult();
if(!$result->next())
throw new RuntimeException('upload variant not found');
return UploadVariantInfo::fromResult($result);
}
public function resolveUploadVariant(
PoolInfo|string $poolInfo,
string $userId,
FileInfo|string $fileInfo,
string $variant
): UploadVariantInfo {
$stmt = $this->cache->get(<<<SQL
SELECT uf.upload_id, uf.upload_variant, uf.file_id, uf.file_type
FROM msz_storage_uploads_files AS uf
LEFT JOIN msz_storage_uploads AS u ON u.upload_id = uf.upload_id
WHERE u.pool_id = ?
AND u.user_id = ?
AND uf.file_id = ?
AND uf.upload_variant = ?
SQL);
$stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo);
$stmt->nextParameter($userId);
$stmt->nextParameter($fileInfo instanceof FileInfo ? $fileInfo->id : $fileInfo);
$stmt->nextParameter($variant);
$stmt->execute();
$result = $stmt->getResult();
if(!$result->next())
throw new RuntimeException('upload variant could not be resolved');
return UploadVariantInfo::fromResult($result);
}
public function createUploadVariant(
UploadInfo|string $uploadInfo,
string $variant,
FileInfo|string $fileInfo,
?string $fileType = null
): UploadVariantInfo {
$stmt = $this->cache->get(<<<SQL
INSERT INTO msz_storage_uploads_files (
upload_id, upload_variant, file_id, file_type
) VALUES (?, ?, ?, ?)
SQL);
$stmt->nextParameter($uploadInfo instanceof UploadInfo ? $uploadInfo->id : $uploadInfo);
$stmt->nextParameter($variant);
$stmt->nextParameter($fileInfo instanceof FileInfo ? $fileInfo->id : $fileInfo);
$stmt->nextParameter(!($fileInfo instanceof FileInfo) || $fileInfo->type !== $fileType ? $fileType : null);
$stmt->execute();
return $this->getUploadVariant($uploadInfo, $variant);
}
public function deleteUploadVariant(
UploadVariantInfo|string $uploadIdOrVariantInfo,
?string $variant = null
): void {
if($uploadIdOrVariantInfo instanceof UploadVariantInfo) {
if($variant !== null)
throw new InvalidArgumentException('$variant must be null if $uploadIdOrVariantInfo is an instance of UploadVariantInfo');
$uploadId = $uploadIdOrVariantInfo->uploadId;
$variant = $uploadIdOrVariantInfo->variant;
} else {
if($variant === null)
throw new InvalidArgumentException('$variant may not be null if $uploadIdOrVariantInfo is a string');
$uploadId = $uploadIdOrVariantInfo;
}
$stmt = $this->cache->get('DELETE FROM msz_storage_uploads_files WHERE upload_id = ? AND upload_variant = ?');
$stmt->nextParameter($uploadId);
$stmt->nextParameter($variant);
$stmt->execute();
}
public function resolveLegacyId(string $legacyId): ?string {
$stmt = $this->cache->get('SELECT upload_id FROM msz_storage_uploads_legacy WHERE upload_id_legacy = ?');
$stmt->nextParameter($legacyId);
$stmt->execute();
$result = $stmt->getResult();
return $result->next() ? $result->getStringOrNull(0) : null;
}
}

383
src/Storage/Uploads/UploadsLegacyRoutes.php Normal file
View file

@ -0,0 +1,383 @@
<?php
namespace Misuzu\Storage\Uploads;
use RuntimeException;
use Index\{ByteFormat,XArray,XNumber};
use Index\Http\{HttpResponseBuilder,HttpRequest};
use Index\Http\Content\MultipartFormContent;
use Index\Http\Content\Multipart\FileMultipartFormData;
use Index\Http\Routing\{HttpDelete,HttpOptions,HttpPost,RouteHandler,RouteHandlerCommon};
use Index\Http\Routing\AccessControl\AccessControl;
use Index\Http\Routing\Processors\Before;
use Index\Http\Routing\Routes\{ExactRoute,PatternRoute};
use Index\Urls\{UrlFormat,UrlSource,UrlSourceCommon};
use Misuzu\Perm;
use Misuzu\Auth\AuthInfo;
use Misuzu\Storage\Denylist\{DenylistContext,DenylistReason};
use Misuzu\Storage\Pools\PoolsContext;
use Misuzu\Storage\Pools\Rules\{ConstrainSizeRule,EnsureVariantRule,EnsureVariantRuleThumb};
use Misuzu\Storage\Files\{FilesContext,FileImportMode,FileInfoGetFileField};
use Misuzu\Users\UsersContext;
class UploadsLegacyRoutes implements RouteHandler, UrlSource {
use RouteHandlerCommon, UrlSourceCommon;
public function __construct(
private UsersContext $usersCtx,
private PoolsContext $poolsCtx,
private UploadsContext $uploadsCtx,
private FilesContext $filesCtx,
private DenylistContext $denylistCtx,
private AuthInfo $authInfo,
) {}
/** @return void|int|array<string, ?scalar> */
#[AccessControl]
#[PatternRoute('GET', '/uploads/([A-Za-z0-9]+|[A-Za-z0-9\-_]{32})(?:-([a-z0-9]+))?(?:\.([A-Za-z0-9\-_]+))?')]
#[UrlFormat('storage-upload', '/uploads/<upload>')]
public function getUpload(
HttpResponseBuilder $response,
HttpRequest $request,
string $uploadId,
string $variant = '',
string $extension = ''
) {
if(strlen($uploadId) < 5)
return 404;
$variantQuery = false;
if($variant === '' && $request->hasParam('v')) {
$variantQuery = true;
$variant = (string)$request->getParam('v');
}
$isV1Json = ($variantQuery || $variant === '') && $extension === 'json';
if($isV1Json)
$variant = '';
if(strlen($uploadId) === 32) {
$uploadId = $this->uploadsCtx->uploads->resolveLegacyId($uploadId) ?? $uploadId;
$uploadSecret = null;
} else {
$uploadSecret = substr($uploadId, -4);
$uploadId = XNumber::fromBase62(substr($uploadId, 0, -4));
}
try {
$uploadInfo = $this->uploadsCtx->uploads->getUpload(uploadId: $uploadId, secret: $uploadSecret);
} catch(RuntimeException $ex) {
return 404;
}
try {
$variantInfo = $this->uploadsCtx->uploads->getUploadVariant($uploadInfo, $variant);
} catch(RuntimeException $ex) {
if($variant === '')
return 404;
try {
$poolInfo = $this->poolsCtx->pools->getPool($uploadInfo->poolId);
$originalInfo = $this->filesCtx->data->getFile(
$this->uploadsCtx->uploads->getUploadVariant($uploadInfo, '')->fileId
);
$rule = XArray::first(
$this->poolsCtx->getPoolRules($poolInfo)->getRules(EnsureVariantRule::TYPE),
fn($rule) => $rule->variant === $variant // @phpstan-ignore property.notFound
);
if($rule instanceof EnsureVariantRule && $rule->onAccess) {
if($rule->params instanceof EnsureVariantRuleThumb) {
$fileInfo = $this->filesCtx->createThumbnailFromRule($originalInfo, $rule->params);
$variantInfo = $this->uploadsCtx->uploads->createUploadVariant($uploadInfo, $rule->variant, $fileInfo);
}
}
} catch(RuntimeException $ex) {
return 404;
}
}
if(!isset($variantInfo))
return 404;
if(!isset($fileInfo))
try {
$fileInfo = $this->filesCtx->data->getFile($variantInfo->fileId);
} catch(RuntimeException $ex) {
return 404;
}
$this->uploadsCtx->uploads->updateUpload(
$uploadInfo,
accessedAt: true,
);
$denyInfo = $this->denylistCtx->getDenylistEntry($fileInfo);
if($denyInfo !== null)
return $denyInfo->isCopyrightTakedown ? 451 : 410;
if($isV1Json)
return $this->uploadsCtx->convertToClientJsonV1($uploadInfo, $variantInfo, $fileInfo);
$fileName = $uploadInfo->name;
if($variantInfo->variant !== '') // FAIRLY SIGNIFICANT TODO: obviously this should support more file exts than .jpg...
$fileName = sprintf('%s-%s.jpg', pathinfo($fileName, PATHINFO_FILENAME), $variantInfo->variant);
$contentType = $variantInfo->type ?? $fileInfo->type;
if($contentType === 'application/octet-stream' || str_starts_with($contentType, 'text/'))
$contentType = 'text/plain';
$response->accelRedirect($this->filesCtx->storage->getRemotePath($fileInfo));
$response->setContentType($contentType);
$response->setFileName(addslashes($fileName));
}
/** @return array<string, ?scalar> */
#[AccessControl(credentials: true, allowHeaders: ['Authorization'], exposeHeaders: ['X-EEPROM-Max-Size'])]
#[Before('input:multipart')]
#[ExactRoute('POST', '/uploads')]
public function postUpload(HttpResponseBuilder $response, HttpRequest $request, MultipartFormContent $content) {
if(!$this->authInfo->loggedIn) {
$response->statusCode = 401;
return [
'error' => 'auth',
'english' => "You must be logged in to upload files. If you are and you're getting this error anyway, try refreshing the page!",
];
}
if($this->usersCtx->hasActiveBan($this->authInfo->userInfo)) {
$response->statusCode = 403;
return [
'error' => 'restricted',
'english' => 'You are currently banned and are not allowed to upload or manage files.',
];
}
$perms = $this->authInfo->getPerms('global');
if(!$perms->check(Perm::G_STORAGE_LEGACY_UPLOAD)) {
$response->statusCode = 403;
return [
'error' => 'permission',
'english' => 'You are not allowed to upload files.',
];
}
$file = $content->getParamData('file');
if(!($file instanceof FileMultipartFormData)) {
$response->statusCode = 400;
return [
'error' => 'file',
'english' => 'File upload is missing from request body.',
];
}
$fileSize = $file->getSize();
if($fileSize < 1) {
$response->statusCode = 400;
return [
'error' => 'empty',
'english' => 'The file you attempted to upload is empty.',
];
}
try {
$poolInfo = $this->poolsCtx->pools->getPool((string)$content->getFilteredParam('src', FILTER_VALIDATE_INT));
if($poolInfo->protected) {
$response->statusCode = 404;
return [
'error' => 'pool_incompatible',
'english' => 'The target upload pool is not compatible with this endpoint.',
];
}
} catch(RuntimeException $ex) {
$response->statusCode = 404;
return [
'error' => 'pool',
'english' => 'The target upload pool does not exist.',
];
}
$rules = $this->poolsCtx->getPoolRules($poolInfo);
// If there's no constrain_size rule on this pool its likely not meant to be used through this API!
$maxSizeRule = $rules->getRule('constrain_size');
if(!($maxSizeRule instanceof ConstrainSizeRule)) {
$response->statusCode = 500;
return [
'error' => 'size_rule_missing',
'english' => 'A pool size constraint rule is required any this pool does not specify one.',
];
}
$maxFileSize = $maxSizeRule->maxSize;
if($maxSizeRule->allowLegacyMultiplier && $perms->check(Perm::G_STORAGE_LEGACY_DOUBLE))
$maxFileSize *= 2;
if($fileSize > $maxFileSize) {
$response->statusCode = 413;
$response->setHeader('X-EEPROM-Max-Size', (string)$maxFileSize);
return [
'error' => 'size',
'english' => sprintf(
'Uploads may not be larger than %s, the file you uploaded was %s.',
ByteFormat::format($maxFileSize),
ByteFormat::format($fileSize)
),
'file_size' => $fileSize,
'max_size' => $maxFileSize,
];
}
$tmpFile = tempnam(sys_get_temp_dir(), 'eeprom-upload-');
$file->moveTo($tmpFile);
$hash = hash_file('sha256', $tmpFile, true);
$denyInfo = $this->denylistCtx->getDenylistEntry($hash);
if($denyInfo !== null) {
$response->statusCode = 451;
return [
'error' => 'takedown',
'english' => sprintf(
'This file is blocked from being uploaded because of %s.',
match($denyInfo->reason) {
DenylistReason::Copyright => 'a copyright takedown request',
DenylistReason::Rules => 'a violation of the rules',
default => 'reasons beyond understanding',
}
),
'reason' => $denyInfo->reason->value,
];
}
$fileName = $file->getClientFilename();
$mediaType = $file->getClientMediaType();
if($mediaType === null || $mediaType === 'application/octet-stream')
$mediaType = mime_content_type($tmpFile);
try {
$fileInfo = $this->filesCtx->data->getFile($hash, FileInfoGetFileField::Hash);
} catch(RuntimeException $ex) {
$fileInfo = $this->filesCtx->importFile(
$tmpFile,
FileImportMode::Move,
$mediaType
);
}
try {
// TODO: Okay so we're reading from the msz_storage_uploads_files table, then fetching the info from msz_storage_uploads,
// and then fetching the info again from the msz_storage_uploads_files table... this can be done better but this will do for now
$uploadInfo = $this->uploadsCtx->uploads->getUpload(
uploadId: $this->uploadsCtx->uploads->resolveUploadFromVariant($poolInfo, $this->authInfo->userId, $fileInfo, '')
);
$variantInfo = $this->uploadsCtx->uploads->getUploadVariant($uploadInfo, '');
$this->uploadsCtx->uploads->updateUpload(
$uploadInfo,
fileName: $fileName,
accessedAt: true,
);
} catch(RuntimeException $ex) {
$uploadInfo = $this->uploadsCtx->uploads->createUpload(
$poolInfo,
$this->authInfo->userId,
$fileName,
$request->remoteAddress
);
$variantInfo = $this->uploadsCtx->uploads->createUploadVariant(
$uploadInfo, '', $fileInfo, $mediaType
);
}
$response->statusCode = 201;
$response->setHeader('Content-Type', 'application/json; charset=utf-8');
return $this->uploadsCtx->convertToClientJsonV1($uploadInfo, $variantInfo, $fileInfo, [
'name' => $fileName,
]);
}
/** @return int|array{error: string, english: string} */
#[AccessControl(credentials: true, allowHeaders: ['Authorization'])]
#[PatternRoute('DELETE', '/uploads/([A-Za-z0-9]+|[A-Za-z0-9\-_]{32})(?:-([a-z0-9]+))?(?:\.([A-Za-z0-9\-_]+))?')]
public function deleteUpload(HttpResponseBuilder $response, HttpRequest $request, string $uploadId) {
if(!$this->authInfo->loggedIn) {
$response->statusCode = 401;
return [
'error' => 'auth',
'english' => "You must be logged in to delete uploaded files. If you are and you're getting this error anyway, try refreshing the page!",
];
}
if($this->usersCtx->hasActiveBan($this->authInfo->userInfo)) {
$response->statusCode = 403;
return [
'error' => 'restricted',
'english' => 'You are currently banned and are not allowed to upload or manage files.',
];
}
$perms = $this->authInfo->getPerms('global');
$canDeleteAny = $perms->check(Perm::G_STORAGE_LEGACY_DELETE_ANY);
if(!$canDeleteAny && !$perms->check(Perm::G_STORAGE_LEGACY_DELETE_OWN)) {
$response->statusCode = 403;
return [
'error' => 'permission',
'english' => 'You are not allowed to delete files.',
];
}
if(strlen($uploadId) < 5) {
$response->statusCode = 404;
return [
'error' => 'none',
'english' => 'That file does not exist.',
];
}
if(strlen($uploadId) === 32) {
$uploadId = $this->uploadsCtx->uploads->resolveLegacyId($uploadId) ?? $uploadId;
$uploadSecret = null;
} else {
$uploadSecret = substr($uploadId, -4);
$uploadId = XNumber::fromBase62(substr($uploadId, 0, -4));
}
try {
$uploadInfo = $this->uploadsCtx->uploads->getUpload(uploadId: $uploadId, secret: $uploadSecret);
} catch(RuntimeException $ex) {
$response->statusCode = 404;
return [
'error' => 'none',
'english' => 'That file does not exist.',
];
}
try {
$poolInfo = $this->poolsCtx->pools->getPool($uploadInfo->poolId);
if($poolInfo->protected) {
$response->statusCode = 404;
return [
'error' => 'pool_incompatible',
'english' => 'The target upload pool is not compatible with this endpoint.',
];
}
} catch(RuntimeException $ex) {
$response->statusCode = 404;
return [
'error' => 'pool',
'english' => 'The target upload pool does not exist.',
];
}
if(!$canDeleteAny && $this->authInfo->userId !== $uploadInfo->userId) {
$response->statusCode = 403;
return [
'error' => 'owner',
'english' => "You aren't allowed to delete files uploaded by others.",
];
}
$this->uploadsCtx->uploads->deleteUpload($uploadInfo);
return 204;
}
}

116
src/Storage/Uploads/UploadsViewRoutes.php Normal file
View file

@ -0,0 +1,116 @@
<?php
namespace Misuzu\Storage\Uploads;
use RuntimeException;
use Index\{XArray,XNumber};
use Index\Http\{HttpResponseBuilder,HttpRequest};
use Index\Http\Routing\{Router,RouteHandler,RouteHandlerCommon};
use Index\Http\Routing\AccessControl\AccessControl;
use Index\Http\Routing\Routes\PatternRoute;
use Misuzu\Storage\Denylist\DenylistContext;
use Misuzu\Storage\Pools\PoolsContext;
use Misuzu\Storage\Pools\Rules\{EnsureVariantRule,EnsureVariantRuleThumb};
use Misuzu\Storage\Files\FilesContext;
class UploadsViewRoutes implements RouteHandler {
use RouteHandlerCommon;
public function __construct(
private PoolsContext $poolsCtx,
private UploadsContext $uploadsCtx,
private FilesContext $filesCtx,
private DenylistContext $denylistCtx
) {}
/** @return int|void */
#[AccessControl]
#[PatternRoute('GET', '/([A-Za-z0-9]+|[A-Za-z0-9\-_]{32})(?:-([a-z0-9]+))?(?:\.([A-Za-z0-9\-_]+))?')]
public function getUpload(
HttpResponseBuilder $response,
HttpRequest $request,
string $uploadId,
string $variant = '',
string $extension = ''
) {
if(strlen($uploadId) < 5)
return 404;
$variantQuery = false;
if($variant === '' && $request->hasParam('v')) {
$variantQuery = true;
$variant = (string)$request->getParam('v');
}
if(strlen($uploadId) === 32) {
$uploadId = $this->uploadsCtx->uploads->resolveLegacyId($uploadId) ?? $uploadId;
$uploadSecret = null;
} else {
$uploadSecret = substr($uploadId, -4);
$uploadId = XNumber::fromBase62(substr($uploadId, 0, -4));
}
try {
$uploadInfo = $this->uploadsCtx->uploads->getUpload(uploadId: $uploadId, secret: $uploadSecret);
} catch(RuntimeException $ex) {
return 404;
}
try {
$variantInfo = $this->uploadsCtx->uploads->getUploadVariant($uploadInfo, $variant);
} catch(RuntimeException $ex) {
if($variant === '')
return 404;
try {
$poolInfo = $this->poolsCtx->pools->getPool($uploadInfo->poolId);
$originalInfo = $this->filesCtx->data->getFile(
$this->uploadsCtx->uploads->getUploadVariant($uploadInfo, '')->fileId
);
$rule = XArray::first(
$this->poolsCtx->getPoolRules($poolInfo)->getRules(EnsureVariantRule::TYPE),
fn($rule) => $rule->variant === $variant // @phpstan-ignore property.notFound
);
if($rule instanceof EnsureVariantRule && $rule->onAccess) {
if($rule->params instanceof EnsureVariantRuleThumb) {
$fileInfo = $this->filesCtx->createThumbnailFromRule($originalInfo, $rule->params);
$variantInfo = $this->uploadsCtx->uploads->createUploadVariant($uploadInfo, $rule->variant, $fileInfo);
}
}
} catch(RuntimeException $ex) {
return 404;
}
}
if(!isset($variantInfo))
return 404;
if(!isset($fileInfo))
try {
$fileInfo = $this->filesCtx->data->getFile($variantInfo->fileId);
} catch(RuntimeException $ex) {
return 404;
}
$this->uploadsCtx->uploads->updateUpload(
$uploadInfo,
accessedAt: true,
);
$denyInfo = $this->denylistCtx->getDenylistEntry($fileInfo);
if($denyInfo !== null)
return $denyInfo->isCopyrightTakedown ? 451 : 410;
$fileName = $uploadInfo->name;
if($variantInfo->variant !== '') // FAIRLY SIGNIFICANT TODO: obviously this should support more file exts than .jpg...
$fileName = sprintf('%s-%s.jpg', pathinfo($fileName, PATHINFO_FILENAME), $variantInfo->variant);
$contentType = $variantInfo->type ?? $fileInfo->type;
if($contentType === 'application/octet-stream' || str_starts_with($contentType, 'text/'))
$contentType = 'text/plain';
$response->accelRedirect($this->filesCtx->storage->getRemotePath($fileInfo));
$response->setContentType($contentType);
$response->setFileName(addslashes($fileName));
}
}

7
templates/errors/402.twig Normal file
View file

@ -0,0 +1,7 @@
{% extends 'errors/master.twig' %}
{% set error_code = 402 %}
{% set error_text = 'Payment Required' %}
{% set error_icon = 'fas fa-euro-sign' %}
{% set error_blerb = 'You must be Tenshi to view this page.' %}
{% set error_colour = '#ffcc00' %}

7
templates/errors/410.twig Normal file
View file

@ -0,0 +1,7 @@
{% extends 'errors/master.twig' %}
{% set error_code = 410 %}
{% set error_text = 'Gone' %}
{% set error_icon = 'fas fa-border-none' %}
{% set error_blerb = "What you're looking for is no longer available." %}
{% set error_colour = '#9475b2' %}

7
templates/errors/451.twig Normal file
View file

@ -0,0 +1,7 @@
{% extends 'errors/master.twig' %}
{% set error_code = 451 %}
{% set error_text = 'Unavailable for Legal Reasons' %}
{% set error_icon = 'fas fa-gavel' %}
{% set error_blerb = "What you're looking for is no longer available." %}
{% set error_colour = '#a41c3c' %}

526
tools/cron
View file

@ -4,6 +4,12 @@ namespace Misuzu;
use stdClass;
use Exception;
use Misuzu\Storage\Uploads\{UploadInfo,UploadsContext};
use Misuzu\Pools\PoolInfo;
use Misuzu\Pools\Rules\{
EnsureVariantRule,EnsureVariantRuleThumb,RemoveStaleRule,ScanForumRule
};
use Misuzu\Storage\Files\{FilesContext,FileInfo};
require_once __DIR__ . '/../misuzu.php';
@ -47,233 +53,359 @@ else
echo 'Only running quick tasks!' . PHP_EOL;
echo PHP_EOL;
msz_sched_task_sql('Ensure main role exists.', true,
'INSERT IGNORE INTO msz_roles (role_id, role_name, role_rank, role_colour, role_description, role_created) VALUES (1, "Member", 1, 1073741824, NULL, NOW())');
$cronPath = sys_get_temp_dir() . '/msz-cron-' . hash('sha256', __DIR__) . '.lock';
if(is_file($cronPath))
die('Misuzu cron script is already running.' . PHP_EOL);
msz_sched_task_sql('Ensure all users have the main role.', true,
'INSERT INTO msz_users_roles (user_id, role_id) SELECT user_id, 1 FROM msz_users AS u WHERE NOT EXISTS (SELECT 1 FROM msz_users_roles AS ur WHERE role_id = 1 AND u.user_id = ur.user_id)');
touch($cronPath);
try {
msz_sched_task_sql('Ensure main role exists.', true,
'INSERT IGNORE INTO msz_roles (role_id, role_name, role_rank, role_colour, role_description, role_created) VALUES (1, "Member", 1, 1073741824, NULL, NOW())');
msz_sched_task_sql('Ensure all user_display_role_id field values are correct with msz_users_roles.', true,
'UPDATE msz_users AS u SET user_display_role_id = (SELECT ur.role_id FROM msz_users_roles AS ur LEFT JOIN msz_roles AS r ON r.role_id = ur.role_id WHERE ur.user_id = u.user_id ORDER BY role_rank DESC LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM msz_users_roles AS ur WHERE ur.role_id = u.user_display_role_id AND ur.user_id = u.user_id)');
msz_sched_task_sql('Ensure all users have the main role.', true,
'INSERT INTO msz_users_roles (user_id, role_id) SELECT user_id, 1 FROM msz_users AS u WHERE NOT EXISTS (SELECT 1 FROM msz_users_roles AS ur WHERE role_id = 1 AND u.user_id = ur.user_id)');
msz_sched_task_sql('Remove expired sessions.', false,
'DELETE FROM msz_sessions WHERE session_expires < NOW()');
msz_sched_task_sql('Ensure all user_display_role_id field values are correct with msz_users_roles.', true,
'UPDATE msz_users AS u SET user_display_role_id = (SELECT ur.role_id FROM msz_users_roles AS ur LEFT JOIN msz_roles AS r ON r.role_id = ur.role_id WHERE ur.user_id = u.user_id ORDER BY role_rank DESC LIMIT 1) WHERE NOT EXISTS (SELECT 1 FROM msz_users_roles AS ur WHERE ur.role_id = u.user_display_role_id AND ur.user_id = u.user_id)');
msz_sched_task_sql('Remove old password reset tokens.', false,
'DELETE FROM msz_users_password_resets WHERE reset_requested < NOW() - INTERVAL 1 WEEK');
msz_sched_task_sql('Remove expired sessions.', false,
'DELETE FROM msz_sessions WHERE session_expires < NOW()');
msz_sched_task_sql('Remove old login attempt logs.', false,
'DELETE FROM msz_login_attempts WHERE attempt_created < NOW() - INTERVAL 1 MONTH');
msz_sched_task_sql('Remove old password reset tokens.', false,
'DELETE FROM msz_users_password_resets WHERE reset_requested < NOW() - INTERVAL 1 WEEK');
msz_sched_task_sql('Remove old audit log entries.', false,
'DELETE FROM msz_audit_log WHERE log_created < NOW() - INTERVAL 3 MONTH');
msz_sched_task_sql('Remove old login attempt logs.', false,
'DELETE FROM msz_login_attempts WHERE attempt_created < NOW() - INTERVAL 1 MONTH');
msz_sched_task_sql('Remove stale forum tracking entries.', false,
'DELETE tt FROM msz_forum_topics_track AS tt LEFT JOIN msz_forum_topics AS t ON t.topic_id = tt.topic_id WHERE t.topic_bumped < NOW() - INTERVAL 1 MONTH');
msz_sched_task_sql('Remove old audit log entries.', false,
'DELETE FROM msz_audit_log WHERE log_created < NOW() - INTERVAL 3 MONTH');
msz_sched_task_sql('Synchronise forum_id.', true,
'UPDATE msz_forum_posts AS p INNER JOIN msz_forum_topics AS t ON t.topic_id = p.topic_id SET p.forum_id = t.forum_id');
msz_sched_task_sql('Remove stale forum tracking entries.', false,
'DELETE tt FROM msz_forum_topics_track AS tt LEFT JOIN msz_forum_topics AS t ON t.topic_id = tt.topic_id WHERE t.topic_bumped < NOW() - INTERVAL 1 MONTH');
msz_sched_task_func('Recount forum topics and posts.', true, function() use ($msz) {
$msz->forumCtx->categories->syncForumCounters();
});
msz_sched_task_sql('Synchronise forum_id.', true,
'UPDATE msz_forum_posts AS p INNER JOIN msz_forum_topics AS t ON t.topic_id = p.topic_id SET p.forum_id = t.forum_id');
msz_sched_task_sql('Clean up expired 2fa tokens.', false,
'DELETE FROM msz_auth_tfa WHERE tfa_created < NOW() - INTERVAL 15 MINUTE');
msz_sched_task_func('Clean up OAuth2 related data.', false, function() use ($msz) {
$msz->oauth2Ctx->pruneExpired(function(string $format, ...$args) {
echo sprintf($format, ...$args) . PHP_EOL;
msz_sched_task_func('Recount forum topics and posts.', true, function() use ($msz) {
$msz->forumCtx->categories->syncForumCounters();
});
});
// very heavy stuff that should
msz_sched_task_sql('Clean up expired 2fa tokens.', false,
'DELETE FROM msz_auth_tfa WHERE tfa_created < NOW() - INTERVAL 15 MINUTE');
msz_sched_task_func('Resync statistics counters.', true, function() use ($msz) {
$stats = [
'users:total' => 'SELECT COUNT(*) FROM msz_users',
'users:active' => 'SELECT COUNT(*) FROM msz_users WHERE user_active IS NOT NULL AND user_deleted IS NULL',
'users:inactive' => 'SELECT COUNT(*) FROM msz_users WHERE user_active IS NULL OR user_deleted IS NOT NULL',
'users:online:recent' => 'SELECT COUNT(*) FROM msz_users WHERE user_active >= NOW() - INTERVAL 1 HOUR', // used to be 5 minutes, but this script runs every hour soooo
'users:online:today' => 'SELECT COUNT(*) FROM msz_users WHERE user_active >= NOW() - INTERVAL 24 HOUR',
'auditlogs:total' => 'SELECT COUNT(*) FROM msz_audit_log',
'changelog:changes:total' => 'SELECT COUNT(*) FROM msz_changelog_changes',
'changelog:tags:total' => 'SELECT COUNT(*) FROM msz_changelog_tags',
'comments:cats:total' => 'SELECT COUNT(*) FROM msz_comments_categories',
'comments:cats:locked' => 'SELECT COUNT(*) FROM msz_comments_categories WHERE category_locked IS NOT NULL',
'comments:posts:total' => 'SELECT COUNT(*) FROM msz_comments_posts',
'comments:posts:visible' => 'SELECT COUNT(*) FROM msz_comments_posts WHERE comment_deleted IS NULL',
'comments:posts:deleted' => 'SELECT COUNT(*) FROM msz_comments_posts WHERE comment_deleted IS NOT NULL',
'comments:posts:replies' => 'SELECT COUNT(*) FROM msz_comments_posts WHERE comment_reply_to IS NOT NULL',
'comments:posts:pinned' => 'SELECT COUNT(*) FROM msz_comments_posts WHERE comment_pinned IS NOT NULL',
'comments:posts:edited' => 'SELECT COUNT(*) FROM msz_comments_posts WHERE comment_edited IS NOT NULL',
'comments:votes:likes' => 'SELECT COUNT(*) FROM msz_comments_votes WHERE comment_vote > 0',
'comments:votes:dislikes' => 'SELECT COUNT(*) FROM msz_comments_votes WHERE comment_vote < 0',
'forum:posts:total' => 'SELECT COUNT(*) FROM msz_forum_posts',
'forum:posts:visible' => 'SELECT COUNT(*) FROM msz_forum_posts WHERE post_deleted IS NULL',
'forum:posts:deleted' => 'SELECT COUNT(*) FROM msz_forum_posts WHERE post_deleted IS NOT NULL',
'forum:posts:edited' => 'SELECT COUNT(*) FROM msz_forum_posts WHERE post_edited IS NOT NULL',
'forum:posts:parse:plain' => "SELECT COUNT(*) FROM msz_forum_posts WHERE post_text_format = ''",
'forum:posts:parse:bbcode' => "SELECT COUNT(*) FROM msz_forum_posts WHERE post_text_format = 'bb'",
'forum:posts:parse:markdown' => "SELECT COUNT(*) FROM msz_forum_posts WHERE post_text_format = 'md'",
'forum:posts:signature' => 'SELECT COUNT(*) FROM msz_forum_posts WHERE post_display_signature <> 0',
'forum:topics:total' => 'SELECT COUNT(*) FROM msz_forum_topics',
'forum:topics:type:normal' => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_type = 0',
'forum:topics:type:pinned' => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_type = 1',
'forum:topics:type:announce' => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_type = 2',
'forum:topics:type:global' => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_type = 3',
'forum:topics:visible' => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_deleted IS NULL',
'forum:topics:deleted' => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_deleted IS NOT NULL',
'forum:topics:locked' => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_locked IS NOT NULL',
'auth:attempts:total' => 'SELECT COUNT(*) FROM msz_login_attempts',
'auth:attempts:failed' => 'SELECT COUNT(*) FROM msz_login_attempts WHERE attempt_success = 0',
'auth:sessions:total' => 'SELECT COUNT(*) FROM msz_sessions',
'auth:tfasessions:total' => 'SELECT COUNT(*) FROM msz_auth_tfa',
'auth:recovery:total' => 'SELECT COUNT(*) FROM msz_users_password_resets',
'users:modnotes:total' => 'SELECT COUNT(*) FROM msz_users_modnotes',
'users:warnings:total' => 'SELECT COUNT(*) FROM msz_users_warnings',
'users:warnings:visible' => 'SELECT COUNT(*) FROM msz_users_warnings WHERE warn_created > NOW() - INTERVAL 90 DAY',
'users:bans:total' => 'SELECT COUNT(*) FROM msz_users_bans',
'users:bans:active' => 'SELECT COUNT(*) FROM msz_users_bans WHERE ban_expires IS NULL OR ban_expires > NOW()',
'pms:msgs:total' => 'SELECT COUNT(*) FROM msz_messages',
'pms:msgs:messages' => 'SELECT COUNT(DISTINCT msg_id) FROM msz_messages',
'pms:msgs:replies' => 'SELECT COUNT(*) FROM msz_messages WHERE msg_reply_to IS NULL',
'pms:msgs:drafts' => 'SELECT COUNT(*) FROM msz_messages WHERE msg_sent IS NULL',
'pms:msgs:unread' => 'SELECT COUNT(*) FROM msz_messages WHERE msg_read IS NULL',
'pms:msgs:deleted' => 'SELECT COUNT(*) FROM msz_messages WHERE msg_deleted IS NOT NULL',
'pms:msgs:plain' => "SELECT COUNT(*) FROM msz_messages WHERE msg_body_format = ''",
'pms:msgs:bbcode' => "SELECT COUNT(*) FROM msz_messages WHERE msg_body_format = 'bb'",
'pms:msgs:markdown' => "SELECT COUNT(*) FROM msz_messages WHERE msg_body_format = 'md'",
];
msz_sched_task_func('Clean up OAuth2 related data.', false, function() use ($msz) {
$msz->oauth2Ctx->pruneExpired(function(string $format, ...$args) {
echo sprintf($format, ...$args) . PHP_EOL;
});
});
foreach($stats as $name => $query) {
$result = $msz->dbCtx->conn->query($query);
$msz->counters->set($name, $result->next() ? $result->getInteger(0) : 0);
}
});
// very heavy stuff that should
msz_sched_task_func('Recalculate permissions (maybe)...', false, function() use ($msz) {
$needsRecalc = $msz->config->getBoolean('perms.needsRecalc');
if(!$needsRecalc)
return;
msz_sched_task_func('Resync statistics counters.', true, function() use ($msz) {
$stats = [
'users:total' => 'SELECT COUNT(*) FROM msz_users',
'users:active' => 'SELECT COUNT(*) FROM msz_users WHERE user_active IS NOT NULL AND user_deleted IS NULL',
'users:inactive' => 'SELECT COUNT(*) FROM msz_users WHERE user_active IS NULL OR user_deleted IS NOT NULL',
'users:online:recent' => 'SELECT COUNT(*) FROM msz_users WHERE user_active >= NOW() - INTERVAL 1 HOUR', // used to be 5 minutes, but this script runs every hour soooo
'users:online:today' => 'SELECT COUNT(*) FROM msz_users WHERE user_active >= NOW() - INTERVAL 24 HOUR',
'auditlogs:total' => 'SELECT COUNT(*) FROM msz_audit_log',
'changelog:changes:total' => 'SELECT COUNT(*) FROM msz_changelog_changes',
'changelog:tags:total' => 'SELECT COUNT(*) FROM msz_changelog_tags',
'comments:cats:total' => 'SELECT COUNT(*) FROM msz_comments_categories',
'comments:cats:locked' => 'SELECT COUNT(*) FROM msz_comments_categories WHERE category_locked IS NOT NULL',
'comments:posts:total' => 'SELECT COUNT(*) FROM msz_comments_posts',
'comments:posts:visible' => 'SELECT COUNT(*) FROM msz_comments_posts WHERE comment_deleted IS NULL',
'comments:posts:deleted' => 'SELECT COUNT(*) FROM msz_comments_posts WHERE comment_deleted IS NOT NULL',
'comments:posts:replies' => 'SELECT COUNT(*) FROM msz_comments_posts WHERE comment_reply_to IS NOT NULL',
'comments:posts:pinned' => 'SELECT COUNT(*) FROM msz_comments_posts WHERE comment_pinned IS NOT NULL',
'comments:posts:edited' => 'SELECT COUNT(*) FROM msz_comments_posts WHERE comment_edited IS NOT NULL',
'comments:votes:likes' => 'SELECT COUNT(*) FROM msz_comments_votes WHERE comment_vote > 0',
'comments:votes:dislikes' => 'SELECT COUNT(*) FROM msz_comments_votes WHERE comment_vote < 0',
'forum:posts:total' => 'SELECT COUNT(*) FROM msz_forum_posts',
'forum:posts:visible' => 'SELECT COUNT(*) FROM msz_forum_posts WHERE post_deleted IS NULL',
'forum:posts:deleted' => 'SELECT COUNT(*) FROM msz_forum_posts WHERE post_deleted IS NOT NULL',
'forum:posts:edited' => 'SELECT COUNT(*) FROM msz_forum_posts WHERE post_edited IS NOT NULL',
'forum:posts:parse:plain' => "SELECT COUNT(*) FROM msz_forum_posts WHERE post_text_format = ''",
'forum:posts:parse:bbcode' => "SELECT COUNT(*) FROM msz_forum_posts WHERE post_text_format = 'bb'",
'forum:posts:parse:markdown' => "SELECT COUNT(*) FROM msz_forum_posts WHERE post_text_format = 'md'",
'forum:posts:signature' => 'SELECT COUNT(*) FROM msz_forum_posts WHERE post_display_signature <> 0',
'forum:topics:total' => 'SELECT COUNT(*) FROM msz_forum_topics',
'forum:topics:type:normal' => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_type = 0',
'forum:topics:type:pinned' => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_type = 1',
'forum:topics:type:announce' => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_type = 2',
'forum:topics:type:global' => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_type = 3',
'forum:topics:visible' => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_deleted IS NULL',
'forum:topics:deleted' => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_deleted IS NOT NULL',
'forum:topics:locked' => 'SELECT COUNT(*) FROM msz_forum_topics WHERE topic_locked IS NOT NULL',
'auth:attempts:total' => 'SELECT COUNT(*) FROM msz_login_attempts',
'auth:attempts:failed' => 'SELECT COUNT(*) FROM msz_login_attempts WHERE attempt_success = 0',
'auth:sessions:total' => 'SELECT COUNT(*) FROM msz_sessions',
'auth:tfasessions:total' => 'SELECT COUNT(*) FROM msz_auth_tfa',
'auth:recovery:total' => 'SELECT COUNT(*) FROM msz_users_password_resets',
'users:modnotes:total' => 'SELECT COUNT(*) FROM msz_users_modnotes',
'users:warnings:total' => 'SELECT COUNT(*) FROM msz_users_warnings',
'users:warnings:visible' => 'SELECT COUNT(*) FROM msz_users_warnings WHERE warn_created > NOW() - INTERVAL 90 DAY',
'users:bans:total' => 'SELECT COUNT(*) FROM msz_users_bans',
'users:bans:active' => 'SELECT COUNT(*) FROM msz_users_bans WHERE ban_expires IS NULL OR ban_expires > NOW()',
'pms:msgs:total' => 'SELECT COUNT(*) FROM msz_messages',
'pms:msgs:messages' => 'SELECT COUNT(DISTINCT msg_id) FROM msz_messages',
'pms:msgs:replies' => 'SELECT COUNT(*) FROM msz_messages WHERE msg_reply_to IS NULL',
'pms:msgs:drafts' => 'SELECT COUNT(*) FROM msz_messages WHERE msg_sent IS NULL',
'pms:msgs:unread' => 'SELECT COUNT(*) FROM msz_messages WHERE msg_read IS NULL',
'pms:msgs:deleted' => 'SELECT COUNT(*) FROM msz_messages WHERE msg_deleted IS NOT NULL',
'pms:msgs:plain' => "SELECT COUNT(*) FROM msz_messages WHERE msg_body_format = ''",
'pms:msgs:bbcode' => "SELECT COUNT(*) FROM msz_messages WHERE msg_body_format = 'bb'",
'pms:msgs:markdown' => "SELECT COUNT(*) FROM msz_messages WHERE msg_body_format = 'md'",
];
$msz->config->removeValues('perms.needsRecalc');
$msz->perms->precalculatePermissions($msz->forumCtx->categories);
});
msz_sched_task_func('Omega disliking comments...', true, function() use ($msz) {
$commentIds = $msz->config->getArray('comments.omegadislike');
if(!is_array($commentIds))
return;
$stmt = $msz->dbCtx->conn->prepare('REPLACE INTO msz_comments_votes (comment_id, user_id, comment_vote) SELECT ?, user_id, -1 FROM msz_users');
foreach($commentIds as $commentId) {
$stmt->addParameter(1, $commentId);
$stmt->execute();
}
});
msz_sched_task_func('Announcing random topics...', true, function() use ($msz) {
$categoryIds = $msz->config->getArray('forum.rngannounce');
if(!is_array($categoryIds))
return;
$stmtRevert = $msz->dbCtx->conn->prepare('UPDATE msz_forum_topics SET topic_type = 0 WHERE forum_id = ? AND topic_type = 2');
$stmtRandom = $msz->dbCtx->conn->prepare('SELECT topic_id FROM msz_forum_topics WHERE forum_id = ? AND topic_deleted IS NULL ORDER BY RAND() LIMIT 1');
$stmtAnnounce = $msz->dbCtx->conn->prepare('UPDATE msz_forum_topics SET topic_type = 2 WHERE topic_id = ?');
foreach($categoryIds as $categoryId) {
$stmtRevert->addParameter(1, $categoryId);
$stmtRevert->execute();
$stmtRandom->addParameter(1, $categoryId);
$stmtRandom->execute();
$resultRandom = $stmtRandom->getResult();
if($resultRandom->next()) {
$stmtAnnounce->addParameter(1, $resultRandom->getInteger(0));
$stmtAnnounce->execute();
foreach($stats as $name => $query) {
$result = $msz->dbCtx->conn->query($query);
$msz->counters->set($name, $result->next() ? $result->getInteger(0) : 0);
}
}
});
});
msz_sched_task_func('Changing category icons...', false, function() use ($msz) {
$categoryIds = $msz->config->getArray('forum.rngicon');
if(!is_array($categoryIds))
return;
msz_sched_task_func('Recalculate permissions (maybe)...', false, function() use ($msz) {
$needsRecalc = $msz->config->getBoolean('perms.needsRecalc');
if(!$needsRecalc)
return;
$stmtIcon = $msz->dbCtx->conn->prepare('UPDATE msz_forum_categories SET forum_icon = COALESCE(?, forum_icon), forum_name = COALESCE(?, forum_name), forum_colour = ((ROUND(RAND() * 255) << 16) | (ROUND(RAND() * 255) << 8) | ROUND(RAND() * 255)) WHERE forum_id = ?');
$stmtUnlock = $msz->dbCtx->conn->prepare('UPDATE msz_forum_topics SET topic_locked = IF(?, NULL, COALESCE(topic_locked, NOW())) WHERE topic_id = ?');
$msz->config->removeValues('perms.needsRecalc');
$msz->perms->precalculatePermissions($msz->forumCtx->categories);
});
foreach($categoryIds as $categoryId) {
$scoped = $msz->config->scopeTo(sprintf('forum.rngicon.fc%d', $categoryId));
msz_sched_task_func('Omega disliking comments...', true, function() use ($msz) {
$commentIds = $msz->config->getArray('comments.omegadislike');
if(!is_array($commentIds))
return;
$icons = $scoped->getArray('icons');
$icon = $icons[array_rand($icons)];
$stmt = $msz->dbCtx->conn->prepare('REPLACE INTO msz_comments_votes (comment_id, user_id, comment_vote) SELECT ?, user_id, -1 FROM msz_users');
foreach($commentIds as $commentId) {
$stmt->addParameter(1, $commentId);
$stmt->execute();
}
});
$name = null;
$names = $scoped->getArray('names');
for($i = 0; $i < count($names); $i += 2)
if($names[$i] === $icon) {
$name = $names[$i + 1] ?? null;
break;
msz_sched_task_func('Announcing random topics...', true, function() use ($msz) {
$categoryIds = $msz->config->getArray('forum.rngannounce');
if(!is_array($categoryIds))
return;
$stmtRevert = $msz->dbCtx->conn->prepare('UPDATE msz_forum_topics SET topic_type = 0 WHERE forum_id = ? AND topic_type = 2');
$stmtRandom = $msz->dbCtx->conn->prepare('SELECT topic_id FROM msz_forum_topics WHERE forum_id = ? AND topic_deleted IS NULL ORDER BY RAND() LIMIT 1');
$stmtAnnounce = $msz->dbCtx->conn->prepare('UPDATE msz_forum_topics SET topic_type = 2 WHERE topic_id = ?');
foreach($categoryIds as $categoryId) {
$stmtRevert->addParameter(1, $categoryId);
$stmtRevert->execute();
$stmtRandom->addParameter(1, $categoryId);
$stmtRandom->execute();
$resultRandom = $stmtRandom->getResult();
if($resultRandom->next()) {
$stmtAnnounce->addParameter(1, $resultRandom->getInteger(0));
$stmtAnnounce->execute();
}
$name ??= $scoped->getString('name');
$stmtIcon->addParameter(1, $icon);
$stmtIcon->addParameter(2, empty($name) ? null : $name);
$stmtIcon->addParameter(3, $categoryId);
$stmtIcon->execute();
$unlockModes = $scoped->getArray('unlock');
foreach($unlockModes as $unlockMode) {
$unlockScoped = $scoped->scopeTo(sprintf('unlock.%s', $unlockMode));
$topicId = $unlockScoped->getInteger('topic');
$match = $unlockScoped->getArray('match');
$matches = in_array($icon, $match);
$stmtUnlock->addParameter(1, $matches ? 1 : 0);
$stmtUnlock->addParameter(2, $topicId);
$stmtUnlock->execute();
}
}
});
});
echo 'Running ' . count($schedTasks) . ' tasks...' . PHP_EOL;
msz_sched_task_func('Changing category icons...', false, function() use ($msz) {
$categoryIds = $msz->config->getArray('forum.rngicon');
if(!is_array($categoryIds))
return;
foreach($schedTasks as $task) {
echo '=> ' . $task->name . PHP_EOL;
$success = true;
$stmtIcon = $msz->dbCtx->conn->prepare('UPDATE msz_forum_categories SET forum_icon = COALESCE(?, forum_icon), forum_name = COALESCE(?, forum_name), forum_colour = ((ROUND(RAND() * 255) << 16) | (ROUND(RAND() * 255) << 8) | ROUND(RAND() * 255)) WHERE forum_id = ?');
$stmtUnlock = $msz->dbCtx->conn->prepare('UPDATE msz_forum_topics SET topic_locked = IF(?, NULL, COALESCE(topic_locked, NOW())) WHERE topic_id = ?');
try {
switch($task->type) {
case 'sql':
$affected = $msz->dbCtx->conn->execute($task->command);
echo $affected . ' rows affected. ' . PHP_EOL;
break;
foreach($categoryIds as $categoryId) {
$scoped = $msz->config->scopeTo(sprintf('forum.rngicon.fc%d', $categoryId));
case 'func':
$result = ($task->command)();
if(is_bool($result))
$success = $result;
break;
$icons = $scoped->getArray('icons');
$icon = $icons[array_rand($icons)];
default:
$success = false;
echo 'Unknown task type.' . PHP_EOL;
break;
$name = null;
$names = $scoped->getArray('names');
for($i = 0; $i < count($names); $i += 2)
if($names[$i] === $icon) {
$name = $names[$i + 1] ?? null;
break;
}
$name ??= $scoped->getString('name');
$stmtIcon->addParameter(1, $icon);
$stmtIcon->addParameter(2, empty($name) ? null : $name);
$stmtIcon->addParameter(3, $categoryId);
$stmtIcon->execute();
$unlockModes = $scoped->getArray('unlock');
foreach($unlockModes as $unlockMode) {
$unlockScoped = $scoped->scopeTo(sprintf('unlock.%s', $unlockMode));
$topicId = $unlockScoped->getInteger('topic');
$match = $unlockScoped->getArray('match');
$matches = in_array($icon, $match);
$stmtUnlock->addParameter(1, $matches ? 1 : 0);
$stmtUnlock->addParameter(2, $topicId);
$stmtUnlock->execute();
}
}
} catch(Exception $ex) {
$success = false;
echo (string)$ex;
} finally {
if($success)
echo 'Done!';
else
echo '!!! FAILED !!!';
});
msz_sched_task_func('Removing stale entries from storage pools...', false, function() use ($msz) {
$pools = [];
$getPool = fn($ruleRaw) => array_key_exists($ruleRaw->poolId, $pools) ? $pools[$ruleRaw->poolId] : (
$pools[$ruleRaw->poolId] = $msz->storageCtx->poolsCtx->pools->getPool($ruleRaw->poolId)
);
// Run cleanup adjacent tasks first
$rules = $msz->storageCtx->poolsCtx->pools->getPoolRules(types: ['remove_stale', 'scan_forum']);
foreach($rules as $ruleRaw) {
$poolInfo = $getPool($ruleRaw);
$rule = $msz->storageCtx->poolsCtx->rules->create($ruleRaw);
if($rule instanceof RemoveStaleRule) {
printf('Removing stale entries for pool #%d...%s', $poolInfo->id, PHP_EOL);
$msz->storageCtx->uploadsCtx->uploads->deleteStaleUploads(
$poolInfo,
$rule->inactiveForSeconds,
$rule->ignoreUploadIds
);
} elseif($rule instanceof ScanForumRule) {
//var_dump('scan_forum', $rule);
// necessary stuff for this doesn't exist yet so this can remain a no-op
}
}
});
msz_sched_task_func('Purging denylisted files...', true, function() use ($msz) {
$fileInfos = $msz->storageCtx->filesCtx->data->getFiles(denied: true);
foreach($fileInfos as $fileInfo) {
printf('Deleting file #%d...%s', $fileInfo->id, PHP_EOL);
$msz->storageCtx->filesCtx->deleteFile($fileInfo);
}
});
msz_sched_task_func('Purging orphaned files...', true, function() use ($msz) {
$fileInfos = $msz->storageCtx->filesCtx->data->getFiles(orphaned: true);
foreach($fileInfos as $fileInfo) {
printf('Deleting file #%d...%s', $fileInfo->id, PHP_EOL);
$msz->storageCtx->filesCtx->deleteFile($fileInfo);
}
});
msz_sched_task_func('Ensuring alternate variants exist...', true, function() use ($msz) {
$pools = [];
$getPool = fn($ruleRaw) => array_key_exists($ruleRaw->poolId, $pools) ? $pools[$ruleRaw->poolId] : (
$pools[$ruleRaw->poolId] = $msz->storageCtx->poolsCtx->pools->getPool($ruleRaw->poolId)
);
$rules = $msz->storageCtx->poolsCtx->pools->getPoolRules(types: ['ensure_variant']);
foreach($rules as $ruleRaw) {
$poolInfo = $getPool($ruleRaw);
$rule = $msz->storageCtx->poolsCtx->rules->create($ruleRaw);
if($rule instanceof EnsureVariantRule) {
if(!$rule->onCron)
continue;
$createVariant = null; // ensure this wasn't previously assigned
if($rule->params instanceof EnsureVariantRuleThumb) {
$createVariant = function(
FilesContext $filesCtx,
UploadsContext $uploadsCtx,
PoolInfo $poolInfo,
EnsureVariantRule $rule,
EnsureVariantRuleThumb $ruleInfo,
FileInfo $originalInfo,
UploadInfo $uploadInfo
) {
$uploadsCtx->uploads->createUploadVariant(
$uploadInfo,
$rule->variant,
$filesCtx->createThumbnailFromRule(
$originalInfo,
$ruleInfo
)
);
};
}
if(!isset($createVariant) || !is_callable($createVariant)) {
printf('!!! Could not create a constructor for "%s" variants for pool #%d !!! Skipping for now...%s', $rule->variant, $poolInfo->id, PHP_EOL);
continue;
}
printf('Ensuring existence of "%s" variants for pool #%d...%s', $rule->variant, $poolInfo->id, PHP_EOL);
$uploads = $msz->storageCtx->uploadsCtx->uploads->getUploads(
poolInfo: $poolInfo,
variant: $rule->variant,
variantExists: false
);
$processed = 0;
foreach($uploads as $uploadInfo)
try {
$createVariant(
$msz->storageCtx->filesCtx,
$msz->storageCtx->uploadsCtx,
$poolInfo,
$rule,
$rule->params,
$msz->storageCtx->filesCtx->data->getFile(
$msz->storageCtx->uploadsCtx->uploads->getUploadVariant($uploadInfo, '')->fileId
),
$uploadInfo
);
} catch(Exception $ex) {
printf('Exception thrown while processing "%s" variant for upload #%d in pool #%d:%s', $rule->variant, $uploadInfo->id, $poolInfo->id, PHP_EOL);
printf('%s%s', $ex->getMessage(), PHP_EOL);
} finally {
++$processed;
}
printf('Processed %d records!%s%s', $processed, PHP_EOL);
}
}
});
echo 'Running ' . count($schedTasks) . ' tasks...' . PHP_EOL;
foreach($schedTasks as $task) {
echo '=> ' . $task->name . PHP_EOL;
$success = true;
try {
switch($task->type) {
case 'sql':
$affected = $msz->dbCtx->conn->execute($task->command);
echo $affected . ' rows affected. ' . PHP_EOL;
break;
case 'func':
$result = ($task->command)();
if(is_bool($result))
$success = $result;
break;
default:
$success = false;
echo 'Unknown task type.' . PHP_EOL;
break;
}
} catch(Exception $ex) {
$success = false;
echo (string)$ex;
} finally {
if($success)
echo 'Done!';
else
echo '!!! FAILED !!!';
echo PHP_EOL;
}
echo PHP_EOL;
}
echo PHP_EOL;
} finally {
unlink($cronPath);
echo 'Took ' . number_format(microtime(true) - MSZ_STARTUP, 5) . 'ms.' . PHP_EOL;
}
echo 'Took ' . number_format(microtime(true) - MSZ_STARTUP, 5) . 'ms.' . PHP_EOL;

77
tools/delete-upload Executable file
View file

@ -0,0 +1,77 @@
#!/usr/bin/env php
<?php
use Index\XNumber;
require_once __DIR__ . '/../misuzu.php';
$options = getopt('s', [
'base62',
], $restIndex);
if($argc <= $restIndex)
die(sprintf('No upload ID specified.%s', PHP_EOL));
$uploadId = $argv[$restIndex];
$isBase62 = isset($options['s']) || isset($options['base62']);
if($isBase62)
$uploadId = XNumber::fromBase62($uploadId);
try {
$uploadInfo = $msz->uploadsCtx->uploads->getUpload(uploadId: $uploadId);
} catch(RuntimeException $ex) {
$hint = $isBase62 ? 'trim last four characters from base62 id' : 'specify -s for base62 ids';
die(sprintf('The requested upload does not exist. (hint: %s)%s', $hint, PHP_EOL));
}
printf(
'Found upload #%d (%s) in pool #%d uploaded by user #%d (%s) at %s with canonical name "%s".%s',
$uploadInfo->id,
$uploadInfo->id62,
$uploadInfo->poolId,
$uploadInfo->userId,
$uploadInfo->remoteAddress,
$uploadInfo->createdAt->toIso8601ZuluString(),
htmlspecialchars($uploadInfo->name),
PHP_EOL
);
$variants = [];
foreach($msz->uploadsCtx->uploads->getUploadVariants($uploadInfo) as $variantInfo) {
$variants[] = $variantInfo;
printf(
'Found variant "%s" linking to file #%d with media type %s.%s',
$variantInfo->variant,
$variantInfo->fileId,
htmlspecialchars($variantInfo->type ?? '(none)'),
PHP_EOL
);
}
$displayStorageDeleteHint = false;
if(count($variants) < 1) {
printf('This upload has no variants to delete.%s', PHP_EOL);
} else {
$variants = array_reverse($variants);
foreach($variants as $variantInfo)
try {
printf('Deleting variant "%s" for upload #%d...%s', $variantInfo->variant, $variantInfo->uploadId, PHP_EOL);
$msz->storageCtx->uploadsCtx->uploads->deleteUploadVariant($variantInfo);
printf('Attempting to delete file...%s', PHP_EOL);
$msz->storageCtx->filesCtx->deleteFile($variantInfo->fileId, false);
} catch(Exception $ex) {
printf('Exception occurred while deleting variant:%s', PHP_EOL);
printf('%s%s', $ex->getMessage(), PHP_EOL);
}
}
try {
printf('Deleting upload #%d...%s', $uploadInfo->id, PHP_EOL);
$msz->uploadsCtx->uploads->deleteUpload($uploadInfo);
} catch(Exception $ex) {
printf('Exception occurred while deleting upload:%s%s%s', PHP_EOL, $ex->getMessage(), PHP_EOL);
}
if($displayStorageDeleteHint)
printf('NOTE: if the exception(s) that occurred while deleting variants were related to database constraints, an upload linking to the same file exists, if they occurred due to filesystem permissions, the file will linger but will be removed during the next scheduled clean up.%s', PHP_EOL);

93
tools/deny-file Executable file
View file

@ -0,0 +1,93 @@
#!/usr/bin/env php
<?php
use Index\{UriBase64,XArray,XNumber};
use Misuzu\Storage\Denylist\DenylistReason;
use Misuzu\Storage\Files\FileInfoGetFileField;
require_once __DIR__ . '/../misuzu.php';
$options = getopt('hsur:', [
'hash',
'base62',
'upload',
'reason:',
], $restIndex);
if($argc <= $restIndex)
die(sprintf('No file ID specified.%s', PHP_EOL));
$fileId = $argv[$restIndex];
$fileField = FileInfoGetFileField::Id;
$uploadId = null;
$isHash = isset($options['h']) || isset($options['hash']);
$isBase62 = isset($options['s']) || isset($options['base62']);
$isUploadId = isset($options['u']) || isset($options['upload']);
try {
$reason = DenylistReason::from($options['r'] ?? $options['reason'] ?? '');
} catch(ValueError $ex) {
die(sprintf(
'Reason value is not supported, it must be one of the following values: %s.%s',
implode(', ', XArray::select(DenylistReason::cases(), fn($case) => $case->value)),
PHP_EOL
));
}
if($isHash) {
$fileId = match(strlen($fileId)) {
32 => $fileId,
43 => UriBase64::decode($fileId),
64 => hex2bin($fileId),
66 => hex2bin(substr($fileId, 2)),
default => null,
};
$fileField = FileInfoGetFileField::Hash;
} else {
if($isBase62)
$fileId = XNumber::fromBase62($fileId);
if($isUploadId) {
$uploadId = $fileId;
$fileId = null;
try {
$variantInfo = $msz->storageCtx->uploadsCtx->uploads->getUploadVariant($uploadId, '');
} catch(RuntimeException $ex) {
$hint = $isBase62 ? 'trim last four characters from base62 id' : 'specify -s for base62 ids';
die(sprintf('The requested upload does not exist. (hint: %s)%s', $hint, PHP_EOL));
}
$fileId = $variantInfo->fileId;
}
}
if($fileId === null)
die(sprintf('Could not resolve file ID from the provided inputs.%s', PHP_EOL));
try {
$fileInfo = $msz->storageCtx->filesCtx->data->getFile($fileId, $fileField);
} catch(RuntimeException $ex) {
$hint = $isUploadId ? (
$isBase62 ? 'trim last four characters from base62 id' : 'specify -s for base62 ids'
) : 'specify -h for hashes or -u to use upload ids';
die(sprintf('The requested file does not exist. (hint: %s)%s', $hint, PHP_EOL));
}
printf(
'Found file #%d of %d bytes with type %s at %s with SHA256 hash %s.%s',
$fileInfo->id,
$fileInfo->size,
htmlspecialchars($fileInfo->type),
$fileInfo->createdAt->toIso8601ZuluString(),
UriBase64::encode($fileInfo->hash),
PHP_EOL
);
try {
printf('Attempting to deny file...%s', PHP_EOL);
$msz->storageCtx->denylistCtx->denyFile($fileInfo, $reason);
} catch(Exception $ex) {
printf('Exception occurred while deleting file:%s', PHP_EOL);
printf('%s%s', $ex->getMessage(), PHP_EOL);
}

83
tools/nuke-file Executable file
View file

@ -0,0 +1,83 @@
#!/usr/bin/env php
<?php
use Index\{UriBase64,XNumber};
use Misuzu\Storage\Files\FileInfoGetFileField;
require_once __DIR__ . '/../misuzu.php';
$options = getopt('hsuv:', [
'hash',
'base62',
'upload',
'variant:',
], $restIndex);
if($argc <= $restIndex)
die(sprintf('No file ID specified.%s', PHP_EOL));
$fileId = $argv[$restIndex];
$fileField = FileInfoGetFileField::Id;
$uploadId = null;
$isHash = isset($options['h']) || isset($options['hash']);
$isBase62 = isset($options['s']) || isset($options['base62']);
$isUploadId = isset($options['u']) || isset($options['upload']);
$variant = $options['variant'] ?? $options['v'] ?? '';
if($isHash) {
$fileId = match(strlen($fileId)) {
32 => $fileId,
43 => UriBase64::decode($fileId),
64 => hex2bin($fileId),
66 => hex2bin(substr($fileId, 2)),
default => null,
};
$fileField = FileInfoGetFileField::Hash;
} else {
if($isBase62)
$fileId = XNumber::fromBase62($fileId);
if($isUploadId) {
$uploadId = $fileId;
$fileId = null;
try {
$variantInfo = $msz->storageCtx->uploadsCtx->uploads->getUploadVariant($uploadId, $variant);
} catch(RuntimeException $ex) {
$hint = $isBase62 ? 'trim last four characters from base62 id' : 'specify -s for base62 ids';
die(sprintf('The requested upload variant does not exist. (hint: %s)%s', $hint, PHP_EOL));
}
$fileId = $variantInfo->fileId;
}
}
if($fileId === null)
die(sprintf('Could not resolve file ID from the provided inputs.%s', PHP_EOL));
try {
$fileInfo = $msz->storageCtx->filesCtx->data->getFile($fileId, $fileField);
} catch(RuntimeException $ex) {
$hint = $isUploadId ? (
$isBase62 ? 'trim last four characters from base62 id' : 'specify -s for base62 ids'
) : 'specify -h for hashes or -u to use upload ids, with -v for variant selection';
die(sprintf('The requested file does not exist. (hint: %s)%s', $hint, PHP_EOL));
}
printf(
'Found file #%d of %d bytes with type %s at %s with SHA256 hash %s.%s',
$fileInfo->id,
$fileInfo->size,
htmlspecialchars($fileInfo->type),
$fileInfo->createdAt->toIso8601ZuluString(),
UriBase64::encode($fileInfo->hash),
PHP_EOL
);
try {
printf('Attempting to delete file...%s', PHP_EOL);
$msz->storageCtx->filesCtx->deleteFile($fileInfo);
} catch(Exception $ex) {
printf('Exception occurred while deleting file:%s', PHP_EOL);
printf('%s%s', $ex->getMessage(), PHP_EOL);
}

17
tools/purge-user-uploads Executable file
View file

@ -0,0 +1,17 @@
#!/usr/bin/env php
<?php
require_once __DIR__ . '/../misuzu.php';
if($argc <= 1)
die(sprintf('No uploader ID specified.%s', PHP_EOL));
$userId = $argv[1];
try {
printf('Attempting to delete uploads by user #%s...%s', $userId, PHP_EOL);
$msz->storageCtx->uploadsCtx->uploads->deleteUploadsByUser($userId);
printf('Done! Actual purging of data in storage will be handled by the next cron cycle.%s', PHP_EOL);
} catch(Exception $ex) {
printf('Exception occurred while trying to delete user uploads:%s', PHP_EOL);
printf('%s%s', $ex->getMessage(), PHP_EOL);
}