flashii/eeprom
flashii/eeprom
2
0
Fork 1
Code Pull requests Releases Activity

Initial implementation of the pools system.

Browse source
This commit is contained in:
flash 2025-01-03 03:32:56 +00:00
parent e43da58f68
commit e7e00a438a
43 changed files with 1457 additions and 447 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
cron.php
View file

@ -1,23 +0,0 @@
<?php
namespace EEPROM;
$cronPath = sys_get_temp_dir() . '/eeprom-cron-' . hash('sha256', __DIR__) . '.lock';
if(is_file($cronPath))
die('EEPROM cron script is already running.' . PHP_EOL);
touch($cronPath);
try {
require_once __DIR__ . '/eeprom.php';
$eeprom->uploadsCtx->uploadsData->deleteExpiredUploads();
$records = $eeprom->storageCtx->records->getFiles(orphaned: true);
foreach($records as $record)
$eeprom->storageCtx->deleteFile($record);
$records = $eeprom->storageCtx->records->getFiles(denied: true);
foreach($records as $record)
$eeprom->storageCtx->deleteFile($record);
} finally {
unlink($cronPath);
}

51
database/2024_12_25_123334_pools_system_with_rules.php Normal file
View file

@ -0,0 +1,51 @@
<?php
use Index\Db\DbConnection;
use Index\Db\Migration\DbMigration;
final class PoolsSystemWithRules_20241225_123334 implements DbMigration {
public function migrate(DbConnection $conn): void {
// Set NULL access timestamps to the created timestamp
$conn->execute('UPDATE prm_uploads SET upload_accessed = upload_created WHERE upload_accessed IS NULL');
// this has to happen first
$conn->execute(<<<SQL
ALTER TABLE prm_uploads
DROP FOREIGN KEY prm_uploads_application_foreign
SQL);
// Remove stuff that'll now be handled by pool rules
$conn->execute(<<<SQL
ALTER TABLE prm_uploads
CHANGE COLUMN app_id pool_id INT(10) UNSIGNED NOT NULL AFTER user_id,
CHANGE COLUMN upload_accessed upload_accessed TIMESTAMP NOT NULL DEFAULT current_timestamp() AFTER upload_created,
DROP COLUMN upload_bump,
DROP COLUMN upload_expires,
DROP INDEX prm_uploads_application_foreign,
ADD INDEX prm_uploads_pool_foreign (pool_id) USING BTREE,
ADD CONSTRAINT prm_uploads_pool_foreign
FOREIGN KEY (pool_id)
REFERENCES prm_pools (pool_id)
ON UPDATE CASCADE
ON DELETE CASCADE;
SQL);
// bye bye apps
$conn->execute('DROP TABLE prm_applications');
// shh
$conn->execute('RENAME TABLE prm_blacklist TO prm_denylist');
$conn->execute(<<<SQL
ALTER TABLE prm_denylist
CHANGE COLUMN bl_hash deny_hash BINARY(32) NOT NULL FIRST,
CHANGE COLUMN bl_reason deny_reason ENUM('copyright','rules','other') NOT NULL COLLATE 'ascii_general_ci' AFTER deny_hash,
CHANGE COLUMN bl_created deny_created TIMESTAMP NOT NULL DEFAULT current_timestamp() AFTER deny_reason,
DROP PRIMARY KEY,
ADD UNIQUE INDEX prm_denylist_hash_unique (deny_hash);
SQL);
$conn->execute(<<<SQL
ALTER TABLE prm_denylist
ADD COLUMN deny_id INT UNSIGNED NOT NULL AUTO_INCREMENT FIRST,
ADD PRIMARY KEY (deny_id)
SQL);
}
}

31
src/Apps/AppInfo.php
View file

@ -1,31 +0,0 @@
<?php
namespace EEPROM\Apps;
use Carbon\CarbonImmutable;
use Index\Db\DbResult;
final class AppInfo {
public function __construct(
public private(set) string $id,
public private(set) string $name,
public private(set) int $createdTime,
public private(set) int $dataSizeLimit,
public private(set) bool $allowSizeMultiplier,
public private(set) int $bumpAmount,
) {}
public static function fromResult(DbResult $result): AppInfo {
return new AppInfo(
id: $result->getString(0),
name: $result->getString(1),
createdTime: $result->getInteger(2),
dataSizeLimit: $result->getInteger(3),
allowSizeMultiplier: $result->getBoolean(4),
bumpAmount: $result->getInteger(5),
);
}
public CarbonImmutable $createdAt {
get => CarbonImmutable::createFromTimestampUTC($this->createdTime);
}
}

21
src/Apps/AppsContext.php
View file

@ -1,21 +0,0 @@
<?php
namespace EEPROM\Apps;
use RuntimeException;
use Index\Db\DbConnection;
class AppsContext {
public private(set) AppsData $appsData;
public function __construct(DbConnection $dbConn) {
$this->appsData = new AppsData($dbConn);
}
public function getApp(string $appId): AppInfo {
$appInfo = $this->appsData->getApp($appId);
if($appInfo === null)
throw new RuntimeException('Not application with this ID exists.');
return $appInfo;
}
}

23
src/Apps/AppsData.php
View file

@ -1,23 +0,0 @@
<?php
namespace EEPROM\Apps;
use Index\Db\{DbConnection,DbStatementCache};
// this system is to be deprecated and replaced by pools system
class AppsData {
private DbStatementCache $cache;
public function __construct(DbConnection $dbConn) {
$this->cache = new DbStatementCache($dbConn);
}
public function getApp(string $appId): ?AppInfo {
$stmt = $this->cache->get('SELECT app_id, app_name, UNIX_TIMESTAMP(app_created), app_size_limit, app_allow_size_multiplier, app_expiry FROM prm_applications WHERE app_id = ?');
$stmt->nextParameter($appId);
$stmt->execute();
$result = $stmt->getResult();
return $result->next() ? AppInfo::fromResult($result) : null;
}
}

27
src/Blacklist/BlacklistContext.php
View file

@ -1,27 +0,0 @@
<?php
namespace EEPROM\Blacklist;
use EEPROM\Storage\StorageRecord;
use Index\Db\DbConnection;
class BlacklistContext {
public private(set) BlacklistData $blacklistData;
public function __construct(DbConnection $dbConn) {
$this->blacklistData = new BlacklistData($dbConn);
}
public function createBlacklistEntry(StorageRecord|string $recordOrHash, string $reason): void {
if($recordOrHash instanceof StorageRecord)
$recordOrHash = $recordOrHash->hash;
$this->blacklistData->createBlacklistEntry($recordOrHash, $reason);
}
public function getBlacklistEntry(StorageRecord|string $recordOrHash): ?BlacklistInfo {
if($recordOrHash instanceof StorageRecord)
$recordOrHash = $recordOrHash->hash;
return $this->blacklistData->getBlacklistEntry($recordOrHash);
}
}

34
src/Blacklist/BlacklistData.php
View file

@ -1,34 +0,0 @@
<?php
namespace EEPROM\Blacklist;
use InvalidArgumentException;
use Index\Db\{DbConnection,DbStatementCache};
class BlacklistData {
private DbStatementCache $cache;
public function __construct(DbConnection $dbConn) {
$this->cache = new DbStatementCache($dbConn);
}
public function getBlacklistEntry(string $hash): ?BlacklistInfo {
$stmt = $this->cache->get('SELECT bl_hash, bl_reason, UNIX_TIMESTAMP(bl_created) FROM prm_blacklist WHERE bl_hash = ?');
$stmt->nextParameter($hash);
$stmt->execute();
$result = $stmt->getResult();
return $result->next() ? BlacklistInfo::fromResult($result) : null;
}
public function createBlacklistEntry(string $hash, string $reason): void {
if(strlen($hash) !== 32)
throw new InvalidArgumentException('$hash must be 32 bytes.');
if(!in_array($reason, BlacklistInfo::REASONS))
throw new InvalidArgumentException('$reason is not a valid reason.');
$stmt = $this->cache->get('INSERT INTO prm_blacklist (bl_hash, bl_reason) VALUES (?, ?)');
$stmt->nextParameter($hash);
$stmt->nextParameter($reason);
$stmt->execute();
}
}

39
src/Blacklist/BlacklistInfo.php
View file

@ -1,39 +0,0 @@
<?php
namespace EEPROM\Blacklist;
use Carbon\CarbonImmutable;
use Index\Db\DbResult;
class BlacklistInfo {
public const REASONS = [
'copyright',
'rules',
'other',
];
public function __construct(
public private(set) string $hash,
public private(set) string $reason,
public private(set) int $createdTime,
) {}
public static function fromResult(DbResult $result): BlacklistInfo {
return new BlacklistInfo(
hash: $result->getString(0),
reason: $result->getString(1),
createdTime: $result->getInteger(2),
);
}
public bool $isCopyrightTakedown {
get => $this->reason === 'copyright';
}
public bool $isRulesViolation {
get => $this->reason === 'rules';
}
public CarbonImmutable $createdAt {
get => CarbonImmutable::createFromTimestampUTC($this->createdTime);
}
}

29
src/Denylist/DenylistContext.php Normal file
View file

@ -0,0 +1,29 @@
<?php
namespace EEPROM\Denylist;
use EEPROM\Storage\StorageRecord;
use Index\Db\DbConnection;
class DenylistContext {
public private(set) DenylistData $list;
public function __construct(DbConnection $dbConn) {
$this->list = new DenylistData($dbConn);
}
public function denyFile(StorageRecord|string $recordOrHash, DenylistReason|string $reason): void {
if($recordOrHash instanceof StorageRecord)
$recordOrHash = $recordOrHash->hash;
if(is_string($reason))
$reason = DenylistReason::from($reason);
$this->list->createDenylistEntry($recordOrHash, $reason);
}
public function getDenylistEntry(StorageRecord|string $recordOrHash): ?DenylistInfo {
if($recordOrHash instanceof StorageRecord)
$recordOrHash = $recordOrHash->hash;
return $this->list->getDenylistEntry($recordOrHash);
}
}

32
src/Denylist/DenylistData.php Normal file
View file

@ -0,0 +1,32 @@
<?php
namespace EEPROM\Denylist;
use InvalidArgumentException;
use Index\Db\{DbConnection,DbStatementCache};
class DenylistData {
private DbStatementCache $cache;
public function __construct(DbConnection $dbConn) {
$this->cache = new DbStatementCache($dbConn);
}
public function getDenylistEntry(string $hash): ?DenylistInfo {
$stmt = $this->cache->get('SELECT deny_id, deny_hash, deny_reason, UNIX_TIMESTAMP(deny_created) FROM prm_denylist WHERE deny_hash = ?');
$stmt->nextParameter($hash);
$stmt->execute();
$result = $stmt->getResult();
return $result->next() ? DenylistInfo::fromResult($result) : null;
}
public function createDenylistEntry(string $hash, DenylistReason $reason): void {
if(strlen($hash) !== 32)
throw new InvalidArgumentException('$hash must be 32 bytes.');
$stmt = $this->cache->get('INSERT INTO prm_denylist (deny_hash, deny_reason) VALUES (?, ?)');
$stmt->nextParameter($hash);
$stmt->nextParameter($reason->value);
$stmt->execute();
}
}

35
src/Denylist/DenylistInfo.php Normal file
View file

@ -0,0 +1,35 @@
<?php
namespace EEPROM\Denylist;
use Carbon\CarbonImmutable;
use Index\Db\DbResult;
class DenylistInfo {
public function __construct(
public private(set) string $id,
public private(set) string $hash,
public private(set) DenylistReason $reason,
public private(set) int $createdTime,
) {}
public static function fromResult(DbResult $result): DenylistInfo {
return new DenylistInfo(
id: $result->getString(0),
hash: $result->getString(1),
reason: DenylistReason::tryFrom($result->getString(2)) ?? DenylistReason::Other,
createdTime: $result->getInteger(3),
);
}
public bool $isCopyrightTakedown {
get => $this->reason === DenylistReason::Copyright;
}
public bool $isRulesViolation {
get => $this->reason === DenylistReason::Rules;
}
public CarbonImmutable $createdAt {
get => CarbonImmutable::createFromTimestampUTC($this->createdTime);
}
}

8
src/Denylist/DenylistReason.php Normal file
View file

@ -0,0 +1,8 @@
<?php
namespace EEPROM\Denylist;
enum DenylistReason: string {
case Copyright = 'copyright';
case Rules = 'rules';
case Other = 'other';
}

14
src/EEPROMContext.php
View file

@ -14,8 +14,8 @@ class EEPROMContext {
private AuthInfo $authInfo;
private Apps\AppsContext $appsCtx;
private Blacklist\BlacklistContext $blacklistCtx;
public private(set) Denylist\DenylistContext $denylistCtx;
public private(set) Pools\PoolsContext $poolsCtx;
public private(set) Storage\StorageContext $storageCtx;
public private(set) Uploads\UploadsContext $uploadsCtx;
private Users\UsersContext $usersCtx;
@ -29,10 +29,10 @@ class EEPROMContext {
$this->authInfo = new AuthInfo;
$this->appsCtx = new Apps\AppsContext($dbConn);
$this->blacklistCtx = new Blacklist\BlacklistContext($dbConn);
$this->denylistCtx = new Denylist\DenylistContext($dbConn);
$this->poolsCtx = new Pools\PoolsContext($dbConn);
$this->storageCtx = new Storage\StorageContext($config->scopeTo('storage'), $dbConn, $this->snowflake);
$this->uploadsCtx = new Uploads\UploadsContext($config, $dbConn, $this->snowflake);
$this->uploadsCtx = new Uploads\UploadsContext($config->scopeTo('domain'), $dbConn, $this->snowflake);
$this->usersCtx = new Users\UsersContext($dbConn);
}
@ -57,10 +57,10 @@ class EEPROMContext {
$routingCtx->register(new Uploads\UploadsRoutes(
$this->authInfo,
$this->appsCtx,
$this->poolsCtx,
$this->uploadsCtx,
$this->storageCtx,
$this->blacklistCtx,
$this->denylistCtx,
$isApiDomain
));

52
src/Pools/PoolInfo.php Normal file
View file

@ -0,0 +1,52 @@
<?php
namespace EEPROM\Pools;
use Carbon\CarbonImmutable;
use Index\Db\DbResult;
class PoolInfo {
public function __construct(
public private(set) string $id,
public private(set) string $name,
#[\SensitiveParameter] private ?string $secret,
public private(set) int $createdTime,
public private(set) ?int $deprecatedTime
) {}
public static function fromResult(DbResult $result): PoolInfo {
return new PoolInfo(
id: $result->getString(0),
name: $result->getString(1),
secret: $result->getStringOrNull(2),
createdTime: $result->getInteger(3),
deprecatedTime: $result->getIntegerOrNull(4),
);
}
public CarbonImmutable $createdAt {
get => CarbonImmutable::createFromTimestampUTC($this->createdTime);
}
public bool $deprecated {
get => $this->deprecatedTime !== null;
}
public ?CarbonImmutable $deprecatedAt {
get => $this->deprecatedTime === null ? null : CarbonImmutable::createFromTimestampUTC($this->deprecatedTime);
}
public bool $hasSecret {
get => $this->secret !== null;
}
public function verifyHash(string $data, string $hash, string $algo = 'sha256'): bool {
// not sure how you'd end up here but lets err on the side of caution for it :)
if($this->secret === null)
return false;
return hash_equals(
hash_hmac($algo, $data, $this->secret, true),
$hash
);
}
}

8
src/Pools/PoolInfoGetField.php Normal file
View file

@ -0,0 +1,8 @@
<?php
namespace EEPROM\Pools;
enum PoolInfoGetField {
case Id;
case Name;
case UploadId;
}

4
src/Pools/PoolRule.php Normal file
View file

@ -0,0 +1,4 @@
<?php
namespace EEPROM\Pools;
interface PoolRule {}

22
src/Pools/PoolRuleInfo.php Normal file
View file

@ -0,0 +1,22 @@
<?php
namespace EEPROM\Pools;
use Index\Db\DbResult;
class PoolRuleInfo {
public function __construct(
public private(set) string $id,
public private(set) string $poolId,
public private(set) string $type,
public private(set) array $params
) {}
public static function fromResult(DbResult $result): PoolRuleInfo {
return new PoolRuleInfo(
id: $result->getString(0),
poolId: $result->getString(1),
type: $result->getString(2),
params: json_decode($result->getString(3), true),
);
}
}

32
src/Pools/PoolRules.php Normal file
View file

@ -0,0 +1,32 @@
<?php
namespace EEPROM\Pools;
use Index\XArray;
class PoolRules {
public function __construct(
public private(set) PoolInfo $pool,
public private(set) array $rulesRaw,
private PoolRulesRegistry $registry
) {}
public array $rules {
get => XArray::select($this->rulesRaw, fn($rule) => $this->registry->create($rule));
}
public function getRuleRaw(string $type): ?PoolRuleInfo {
return XArray::first($this->rulesRaw, fn($rule) => strcasecmp($rule->type, $type) === 0);
}
public function getRule(string $type): ?PoolRule {
return $this->registry->create($this->getRuleRaw($type));
}
public function getRulesRaw(string $type): array {
return XArray::where($this->rulesRaw, fn($rule) => strcasecmp($rule->type, $type) === 0);
}
public function getRules(string $type): array {
return XArray::select($this->getRulesRaw($type), fn($rule) => $this->registry->create($rule));
}
}

25
src/Pools/PoolRulesRegistry.php Normal file
View file

@ -0,0 +1,25 @@
<?php
namespace EEPROM\Pools;
use RuntimeException;
class PoolRulesRegistry {
private array $constructors = [];
public function register(string $type, callable $constructor): void {
if(array_key_exists($type, $this->constructors))
throw new RuntimeException('$type already registered');
$this->constructors[$type] = $constructor;
}
public function create(?PoolRuleInfo $ruleInfo): ?PoolRule {
if($ruleInfo === null)
return null;
if(!array_key_exists($ruleInfo->type, $this->constructors))
throw new RuntimeException('no constructor for a rule of the given type registered');
return $this->constructors[$ruleInfo->type]($ruleInfo->params);
}
}

38
src/Pools/PoolsContext.php Normal file
View file

@ -0,0 +1,38 @@
<?php
namespace EEPROM\Pools;
use RuntimeException;
use EEPROM\Uploads\UploadVariantInfo;
use Index\Db\DbConnection;
class PoolsContext {
public private(set) PoolsData $pools;
public private(set) PoolRulesRegistry $rules;
public function __construct(DbConnection $dbConn) {
$this->pools = new PoolsData($dbConn);
$this->rules = new PoolRulesRegistry;
$this->rules->register(Rules\AcceptTypesRule::TYPE, Rules\AcceptTypesRule::create(...));
$this->rules->register(Rules\ConstrainSizeRule::TYPE, Rules\ConstrainSizeRule::create(...));
$this->rules->register(Rules\EnsureVariantRule::TYPE, Rules\EnsureVariantRule::create(...));
$this->rules->register(Rules\OnePerUserRule::TYPE, Rules\OnePerUserRule::create(...));
$this->rules->register(Rules\RemoveStaleRule::TYPE, Rules\RemoveStaleRule::create(...));
$this->rules->register(Rules\ScanForumRule::TYPE, Rules\ScanForumRule::create(...));
}
public function getPoolRules(PoolInfo|string $poolInfo): PoolRules {
if(!($poolInfo instanceof PoolInfo))
$poolInfo = $this->pools->getPool($poolInfo);
return new PoolRules(
$poolInfo,
iterator_to_array($this->pools->getPoolRules(poolInfo: $poolInfo)),
$this->rules
);
}
public function getPoolFromVariant(UploadVariantInfo $variantInfo): StorageRecord {
return $this->pools->getPool($variantInfo->fileId, PoolInfoGetField::Id);
}
}

73
src/Pools/PoolsData.php Normal file
View file

@ -0,0 +1,73 @@
<?php
namespace EEPROM\Pools;
use InvalidArgumentException;
use RuntimeException;
use Index\Db\{DbConnection,DbStatementCache,DbTools};
class PoolsData {
private DbStatementCache $cache;
public function __construct(DbConnection $dbConn) {
$this->cache = new DbStatementCache($dbConn);
}
public function getPool(
string $value,
PoolInfoGetField $field = PoolInfoGetField::Id
): PoolInfo {
$field = match($field) {
PoolInfoGetField::Id => 'pool_id = ?',
PoolInfoGetField::Name => 'pool_name = ?',
PoolInfoGetField::UploadId => 'pool_id = (SELECT pool_id FROM prm_uploads WHERE upload_id = ?)',
default => throw new InvalidArgumentException('$field is not an acceptable value'),
};
$stmt = $this->cache->get(<<<SQL
SELECT pool_id, pool_name, pool_secret, UNIX_TIMESTAMP(pool_created), UNIX_TIMESTAMP(pool_deprecated)
FROM prm_pools
WHERE {$field}
SQL);
$stmt->nextParameter($value);
$stmt->execute();
$result = $stmt->getResult();
if(!$result->next())
throw new RuntimeException('could not find that file record');
return PoolInfo::fromResult($result);
}
public function getPoolRules(
PoolInfo|string|null $poolInfo = null,
string|array|null $types = null
): iterable {
$hasPoolInfo = $poolInfo !== null;
$hasTypes = $types !== null;
if(is_string($types))
$types = [$types];
$args = 0;
$query = <<<SQL
SELECT rule_id, pool_id, rule_type, rule_params
FROM prm_pools_rules
SQL;
if($hasPoolInfo)
$query .= sprintf(' %s pool_id = ?', ++$args > 1 ? 'AND' : 'WHERE');
if($hasTypes)
$query .= sprintf(' %s rule_type IN (%s)', ++$args > 1 ? 'AND' : 'WHERE', DbTools::prepareListString($types));
$stmt = $this->cache->get($query);
if($hasPoolInfo)
$stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo);
if($hasTypes)
foreach($types as $type) {
if(!is_string($type))
throw new InvalidArgumentException('all entries of $types but be a string');
$stmt->nextParameter($type);
}
$stmt->execute();
return $stmt->getResult()->getIterator(PoolRuleInfo::fromResult(...));
}
}

12
src/Pools/Rules/AcceptTypesRule.php Normal file
View file

@ -0,0 +1,12 @@
<?php
namespace EEPROM\Pools\Rules;
use EEPROM\Pools\PoolRule;
class AcceptTypesRule implements PoolRule {
public const string TYPE = 'accept_types';
public static function create(array $params): AcceptTypesRule {
return new AcceptTypesRule;
}
}

20
src/Pools/Rules/ConstrainSizeRule.php Normal file
View file

@ -0,0 +1,20 @@
<?php
namespace EEPROM\Pools\Rules;
use EEPROM\Pools\PoolRule;
class ConstrainSizeRule implements PoolRule {
public const string TYPE = 'constrain_size';
public function __construct(
public private(set) int $maxSize,
public private(set) bool $allowLegacyMultiplier = false
) {}
public static function create(array $params): ConstrainSizeRule {
return new ConstrainSizeRule(
maxSize: isset($params['max_size']) && is_int($params['max_size']) ? max(0, $params['max_size']) : 0,
allowLegacyMultiplier: isset($params['allow_legacy_multiplier']) && is_bool($params['allow_legacy_multiplier']) && $params['allow_legacy_multiplier']
);
}
}

49
src/Pools/Rules/EnsureVariantRule.php Normal file
View file

@ -0,0 +1,49 @@
<?php
namespace EEPROM\Pools\Rules;
use RuntimeException;
use EEPROM\Pools\PoolRule;
class EnsureVariantRule implements PoolRule {
public const string TYPE = 'ensure_variant';
private static array $infoTypes;
public static function init(): void {
self::$infoTypes = [
'crop' => EnsureVariantRuleCrop::create(...),
'thumb' => EnsureVariantRuleThumb::create(...),
];
}
public function __construct(
public private(set) string $variant,
public private(set) array $when,
public private(set) EnsureVariantRuleInfo $info
) {}
public bool $onAccess {
get => in_array('access', $this->when);
}
public bool $onCron {
get => in_array('cron', $this->when);
}
public static function create(array $params): EnsureVariantRule {
if(!isset($params['info']) || !is_array($params['info']))
throw new RuntimeException('missing info key in $params');
$info = $params['info'];
if(!isset($info['$kind']) || !is_string($info['$kind']) || !array_key_exists($info['$kind'], self::$infoTypes))
throw new RuntimeException('unsupported info kind in $params');
return new EnsureVariantRule(
isset($params['name']) && is_string($params['name']) ? $params['name'] : '',
isset($params['when']) && is_array($params['when']) ? $params['when'] : [],
self::$infoTypes[$info['$kind']]($info)
);
}
}
EnsureVariantRule::init();

25
src/Pools/Rules/EnsureVariantRuleCrop.php Normal file
View file

@ -0,0 +1,25 @@
<?php
namespace EEPROM\Pools\Rules;
use InvalidArgumentException;
class EnsureVariantRuleCrop implements EnsureVariantRuleInfo {
public const string KIND = 'crop';
public function __construct(
public private(set) int $width,
public private(set) int $height
) {
if($width < 1)
throw new InvalidArgumentException('$width must be greater than 0');
if($height < 1)
throw new InvalidArgumentException('$height must be greater than 0');
}
public static function create(array $params): EnsureVariantRuleCrop {
return new EnsureVariantRuleCrop(
isset($params['w']) && is_int($params['w']) ? $params['w'] : 0,
isset($params['h']) && is_int($params['h']) ? $params['h'] : 0,
);
}
}

4
src/Pools/Rules/EnsureVariantRuleInfo.php Normal file
View file

@ -0,0 +1,4 @@
<?php
namespace EEPROM\Pools\Rules;
interface EnsureVariantRuleInfo {}

33
src/Pools/Rules/EnsureVariantRuleThumb.php Normal file
View file

@ -0,0 +1,33 @@
<?php
namespace EEPROM\Pools\Rules;
use InvalidArgumentException;
class EnsureVariantRuleThumb implements EnsureVariantRuleInfo {
public const string KIND = 'thumb';
public function __construct(
public private(set) int $width,
public private(set) int $height,
public private(set) int $quality,
public private(set) string $format,
public private(set) string $background
) {
if($width < 1)
throw new InvalidArgumentException('$width must be greater than 0');
if($height < 1)
throw new InvalidArgumentException('$height must be greater than 0');
if($quality < 0 || $quality > 100)
throw new InvalidArgumentException('$quality must be a positive integer, not greater than 100');
}
public static function create(array $params): EnsureVariantRuleThumb {
return new EnsureVariantRuleThumb(
isset($params['w']) && is_int($params['w']) ? $params['w'] : 0,
isset($params['h']) && is_int($params['h']) ? $params['h'] : 0,
isset($params['q']) && is_int($params['q']) ? $params['q'] : 80,
isset($params['f']) && is_string($params['f']) ? $params['f'] : 'jpg',
isset($params['b']) && is_string($params['b']) ? $params['b'] : 'black'
);
}
}

12
src/Pools/Rules/OnePerUserRule.php Normal file
View file

@ -0,0 +1,12 @@
<?php
namespace EEPROM\Pools\Rules;
use EEPROM\Pools\PoolRule;
class OnePerUserRule implements PoolRule {
public const string TYPE = 'one_per_user';
public static function create(array $params): OnePerUserRule {
return new OnePerUserRule;
}
}

28
src/Pools/Rules/RemoveStaleRule.php Normal file
View file

@ -0,0 +1,28 @@
<?php
namespace EEPROM\Pools\Rules;
use EEPROM\Pools\PoolRule;
use EEPROM\Uploads\UploadInfo;
class RemoveStaleRule implements PoolRule {
public const string TYPE = 'remove_stale';
public function __construct(
public private(set) int $inactiveForSeconds,
public private(set) array $ignoreUploadIds
) {}
public function isIgnored(UploadInfo|string $uploadId): bool {
if($uploadId instanceof UploadInfo)
$uploadId = $uploadId->id;
return in_array($uploadId, $this->ignoreUploadIds);
}
public static function create(array $params): RemoveStaleRule {
return new RemoveStaleRule(
isset($params['inactive_for']) && is_int($params['inactive_for']) ? max(0, $params['inactive_for']) : 0,
isset($params['ignore']) && is_array($params['ignore']) ? $params['ignore'] : []
);
}
}

12
src/Pools/Rules/ScanForumRule.php Normal file
View file

@ -0,0 +1,12 @@
<?php
namespace EEPROM\Pools\Rules;
use EEPROM\Pools\PoolRule;
class ScanForumRule implements PoolRule {
public const string TYPE = 'scan_forum';
public static function create(array $params): ScanForumRule {
return new ScanForumRule;
}
}

177
src/Storage/StorageContext.php
View file

@ -2,9 +2,11 @@
namespace EEPROM\Storage;
use Imagick;
use ImagickException;
use InvalidArgumentException;
use RuntimeException;
use EEPROM\SnowflakeGenerator;
use EEPROM\Pools\Rules\EnsureVariantRuleThumb;
use EEPROM\Uploads\UploadVariantInfo;
use Index\Config\Config;
use Index\Db\DbConnection;
@ -25,17 +27,12 @@ class StorageContext {
$this->records = new StorageRecords($dbConn, $snowflake);
}
public function getFileRecordFromVariant(UploadVariantInfo $variantInfo): StorageRecord {
return $this->records->getFile($variantInfo->fileId, StorageRecordGetFileField::Id);
}
public function deleteFile(
StorageRecord|string $recordInfo,
bool $ensureOrphan = true
): void {
// Does taking ID here make sense or should it be hash?
if(!($recordInfo instanceof StorageRecord))
$recordInfo = $this->records->getFile($recordInfo, StorageRecordGetFileField::Id);
$recordInfo = $this->records->getFile($recordInfo);
if($ensureOrphan)
$this->records->unlinkFileUploads($recordInfo);
@ -59,90 +56,168 @@ class StorageContext {
try {
return $this->records->createFile($hash, $type, $size);
} catch(RuntimeException $ex) {
return $this->records->getFile($hash);
return $this->records->getFile($hash, StorageRecordGetFileField::Hash);
}
}
public function supportsThumbnailing(StorageRecord $fileInfo): bool {
public static function supportsThumbnailing(StorageRecord $fileInfo): bool {
return $fileInfo->isImage
|| $fileInfo->isAudio
|| $fileInfo->isVideo;
}
public function createThumbnail(StorageRecord $fileInfo): StorageRecord {
$filePath = $this->files->getLocalPath($fileInfo);
if(!is_file($filePath))
throw new RuntimeException('local data for provided $fileInfo does not exist');
public static function supportsSquareCropping(StorageRecord $fileInfo): bool {
return $fileInfo->isImage;
}
$tmpPath = tempnam(sys_get_temp_dir(), 'eeprom-thumbnail-');
$imagick = new Imagick;
public function readIntoImagick(Imagick $imagick, StorageRecord $fileInfo): void {
try {
$path = $this->files->getLocalPath($fileInfo);
if($fileInfo->isImage) {
try {
$file = fopen($filePath, 'rb');
$file = fopen($path, 'rb');
$imagick->readImageFile($file);
} finally {
if(isset($file) && is_resource($file))
fclose($file);
}
} elseif($fileInfo->isAudio) {
return;
}
if($fileInfo->isAudio) {
try {
$file = popen(sprintf('ffmpeg -i %s -an -f image2pipe -c:v copy -frames:v 1 2>/dev/null -', escapeshellarg($filePath)), 'rb');
$file = popen(sprintf('ffmpeg -i %s -an -f image2pipe -c:v copy -frames:v 1 2>/dev/null -', escapeshellarg($path)), 'rb');
$imagick->readImageBlob(stream_get_contents($file));
} finally {
if(isset($file) && is_resource($file))
pclose($file);
}
} elseif($fileInfo->isVideo) {
return;
}
if($fileInfo->isVideo) {
try {
$file = popen(sprintf('ffmpeg -i %s -f image2pipe -c:v png -frames:v 1 2>/dev/null -', escapeshellarg($filePath)), 'rb');
$file = popen(sprintf('ffmpeg -i %s -f image2pipe -c:v png -frames:v 1 2>/dev/null -', escapeshellarg($path)), 'rb');
$imagick->readImageBlob(stream_get_contents($file));
} finally {
if(isset($file) && is_resource($file))
pclose($file);
}
}
} catch(ImagickException $ex) {
if($ex->getCode() !== 1)
throw $ex;
$imagick->setImageFormat('jpg');
$imagick->setImageCompressionQuality(80);
$imagick->newImage(1, 1, 'black', 'bmp');
}
}
$width = $imagick->getImageWidth();
$height = $imagick->getImageHeight();
$thumbRes = min(300, $width, $height);
public function createThumbnailFromRule(
StorageRecord $original,
EnsureVariantRuleThumb $rules
): StorageRecord {
return $this->createThumbnail(
$original,
$rules->width,
$rules->height,
$rules->quality,
$rules->format,
$rules->background
);
}
if($width === $height) {
$resizeWidth = $resizeHeight = $thumbRes;
} elseif($width > $height) {
$resizeWidth = $width * $thumbRes / $height;
$resizeHeight = $thumbRes;
} else {
$resizeWidth = $thumbRes;
$resizeHeight = $height * $thumbRes / $width;
}
public function createThumbnail(
StorageRecord $original,
int $width = 300,
int $height = 300,
int $quality = 80,
string $format = 'jpg',
string $background = 'black'
): StorageRecord {
if(!self::supportsThumbnailing($original))
throw new InvalidArgumentException('$original does not support thumbnailing');
if($width < 1)
throw new InvalidArgumentException('$width must be greater than 0');
if($height < 1)
throw new InvalidArgumentException('$height must be greater than 0');
if($quality < 0 || $quality > 100)
throw new InvalidArgumentException('$quality must be a positive integer, not greater than 100');
$format = strtoupper($format);
if(!in_array($format, Imagick::queryFormats()))
throw new InvalidArgumentException('$format is not a supported format');
$resizeWidth = (int)$resizeWidth;
$resizeHeight = (int)$resizeHeight;
$imagick->resizeImage(
$resizeWidth, $resizeHeight,
Imagick::FILTER_GAUSSIAN, 0.7
);
$imagick->cropImage(
$thumbRes,
$thumbRes,
(int)ceil(($resizeWidth - $thumbRes) / 2),
(int)ceil(($resizeHeight - $thumbRes) / 2)
);
$imagick->writeImage($tmpPath);
$type = 'application/octet-stream';
$path = tempnam(sys_get_temp_dir(), 'eeprom-thumbnail-');
$imagick = new Imagick;
try {
$this->readIntoImagick($imagick, $original);
$imagick->setImageFormat($format);
$imagick->setImageCompressionQuality($quality);
$imagick->setImageBackgroundColor($background);
$imagick->thumbnailImage($width, $height, true, true);
$type = $imagick->getImageMimeType();
$imagick->writeImage(sprintf('%s:%s', $format, $path));
} finally {
$imagick->clear();
}
if(!is_file($tmpPath))
if(!is_file($path))
throw new RuntimeException('failed to create thumbnail file');
return $this->importFile($tmpPath, StorageImportMode::Move, 'image/jpeg');
return $this->importFile($path, StorageImportMode::Move, $type);
}
public function createSquareCropped(StorageRecord $original, int $dimensions): StorageRecord {
if($dimensions < 1)
throw new InvalidArgumentException('$dimensions must be greater than 0');
$sourcePath = $this->files->getLocalPath($original);
if(!is_file($sourcePath))
throw new RuntimeException('local data for provided $original does not exist');
$targetPath = tempnam(sys_get_temp_dir(), sprintf('eeprom-crop-%d-', $dimensions));
$imagick = new Imagick;
try {
if($original->isImage)
try {
$file = fopen($sourcePath, 'rb');
$imagick->readImageFile($file);
} finally {
if(isset($file) && is_resource($file))
fclose($file);
}
$width = $imagick->getImageWidth();
$height = $imagick->getImageHeight();
if($width > $height) {
$width = (int)($width * $dimensions / $height);
$height = $dimensions;
} else {
$width = $dimensions;
$height = (int)($height * $dimensions / $width);
}
do {
$imagick->resizeImage($width, $height, Imagick::FILTER_LANCZOS, 0.9);
$imagick->cropImage(
$dimensions,
$dimensions,
(int)ceil(($width - $dimensions) / 2),
(int)ceil(($height - $dimensions) / 2)
);
$imagick->setImagePage($dimensions, $dimensions, 0, 0);
} while($imagick->nextImage());
$imagick->writeImage($targetPath);
} finally {
$imagick->clear();
}
if(!is_file($targetPath))
throw new RuntimeException('failed to create cropped file');
return $this->importFile($targetPath, StorageImportMode::Move, $original->type);
}
}

8
src/Storage/StorageRecords.php
View file

@ -33,7 +33,7 @@ class StorageRecords {
SQL;
if($hasDenied)
$query .= <<<SQL
LEFT JOIN prm_blacklist AS dl ON f.file_hash = dl.bl_hash
LEFT JOIN prm_denylist AS dl ON f.file_hash = dl.deny_hash
SQL;
$args = 0;
@ -45,7 +45,7 @@ class StorageRecords {
);
if($denied !== null)
$query .= sprintf(
' %s dl.bl_hash %s NULL',
' %s dl.deny_hash %s NULL',
++$args > 1 ? 'OR' : 'WHERE',
$denied ? 'IS NOT' : 'IS'
);
@ -58,7 +58,7 @@ class StorageRecords {
public function getFile(
string $value,
StorageRecordGetFileField $field = StorageRecordGetFileField::Hash // wdym verbose?
StorageRecordGetFileField $field = StorageRecordGetFileField::Id
): StorageRecord {
$field = match($field) {
StorageRecordGetFileField::Id => 'file_id',
@ -106,7 +106,7 @@ class StorageRecords {
$stmt->nextParameter($size);
$stmt->execute();
return $this->getFile($fileId, StorageRecordGetFileField::Id);
return $this->getFile($fileId);
}
public function deleteFile(

32
src/Uploads/UploadInfo.php
View file

@ -9,28 +9,24 @@ class UploadInfo {
public function __construct(
public private(set) string $id,
public private(set) ?string $userId,
public private(set) ?string $appId,
public private(set) string $poolId,
public private(set) string $secret,
public private(set) string $name,
public private(set) int $bumpAmount,
public private(set) string $remoteAddress,
public private(set) int $createdTime,
public private(set) ?int $accessedTime,
public private(set) ?int $expiresTime,
public private(set) int $accessedTime,
) {}
public static function fromResult(DbResult $result): UploadInfo {
return new UploadInfo(
id: $result->getString(0),
userId: $result->getStringOrNull(1),
appId: $result->getStringOrNull(2),
poolId: $result->getString(2),
secret: $result->getString(3),
name: $result->getString(4),
bumpAmount: $result->getInteger(5),
remoteAddress: $result->getString(6),
createdTime: $result->getInteger(7),
accessedTime: $result->getIntegerOrNull(8),
expiresTime: $result->getIntegerOrNull(9),
remoteAddress: $result->getString(5),
createdTime: $result->getInteger(6),
accessedTime: $result->getInteger(7),
);
}
@ -42,19 +38,7 @@ class UploadInfo {
get => CarbonImmutable::createFromTimestampUTC($this->createdTime);
}
public ?CarbonImmutable $accessedAt {
get => $this->accessedTime === null ? null : CarbonImmutable::createFromTimestampUTC($this->accessedTime);
}
public bool $expired {
get => $this->expiresTime !== null && $this->expiresTime <= time();
}
public ?CarbonImmutable $expiredAt {
get => $this->expiresTime === null ? null : CarbonImmutable::createFromTimestampUTC($this->expiresTime);
}
public ?int $bumpAmountForUpdate {
get => $this->expiresTime !== null && $this->bumpAmount > 0 ? $this->bumpAmount : null;
public CarbonImmutable $accessedAt {
get => CarbonImmutable::createFromTimestampUTC($this->accessedTime);
}
}

16
src/Uploads/UploadsContext.php
View file

@ -8,21 +8,21 @@ use EEPROM\{FFMPEG,SnowflakeGenerator};
use EEPROM\Storage\StorageRecord;
class UploadsContext {
public private(set) UploadsData $uploadsData;
public private(set) UploadsData $uploads;
public function __construct(
private Config $config,
DbConnection $dbConn,
SnowflakeGenerator $snowflake
) {
$this->uploadsData = new UploadsData($dbConn, $snowflake);
$this->uploads = new UploadsData($dbConn, $snowflake);
}
public function getFileUrlV1(UploadInfo $uploadInfo, bool $forceApiDomain = false): string {
if(!$forceApiDomain && $this->config->hasValues('domain:short'))
return sprintf('//%s/%s%s', $this->config->getString('domain:short'), $uploadInfo->id62, $uploadInfo->secret);
if(!$forceApiDomain && $this->config->hasValues('short'))
return sprintf('//%s/%s%s', $this->config->getString('short'), $uploadInfo->id62, $uploadInfo->secret);
return sprintf('//%s/uploads/%s%s', $this->config->getString('domain:api'), $uploadInfo->id62, $uploadInfo->secret);
return sprintf('//%s/uploads/%s%s', $this->config->getString('api'), $uploadInfo->id62, $uploadInfo->secret);
}
public function getThumbnailUrlV1(UploadInfo $uploadInfo, bool $forceApiDomain = false): string {
@ -44,11 +44,11 @@ class UploadsContext {
'type' => $variantInfo->type ?? $fileInfo->type,
'size' => $fileInfo->size,
'user' => $uploadInfo->userId === null ? null : (int)$uploadInfo->userId,
'appl' => $uploadInfo->appId === null ? null : (int)$uploadInfo->appId,
'appl' => (int)$uploadInfo->poolId,
'hash' => $fileInfo->hashString,
'created' => $uploadInfo->createdAt->toIso8601ZuluString(),
'accessed' => $uploadInfo->accessedAt?->toIso8601ZuluString(),
'expires' => $uploadInfo->expiredAt?->toIso8601ZuluString(),
'accessed' => $uploadInfo->accessedAt->toIso8601ZuluString(),
'expires' => $uploadInfo->accessedAt->add(2, 'weeks')->toIso8601ZuluString(),
// These can never be reached, and in situation where they technically could it's because of an outdated local record
'deleted' => null,

168
src/Uploads/UploadsData.php
View file

@ -4,9 +4,9 @@ namespace EEPROM\Uploads;
use InvalidArgumentException;
use RuntimeException;
use Index\XString;
use Index\Db\{DbConnection,DbStatementCache};
use Index\Db\{DbConnection,DbStatementCache,DbTools};
use EEPROM\SnowflakeGenerator;
use EEPROM\Apps\AppInfo;
use EEPROM\Pools\PoolInfo;
use EEPROM\Storage\StorageRecord;
use EEPROM\Users\UserInfo;
@ -21,20 +21,41 @@ class UploadsData {
}
public function getUploads(
?bool $expired = null
PoolInfo|string|null $poolInfo = null,
?string $variant = null,
?bool $variantExists = null
): iterable {
$hasExpired = $expired !== null;
$hasPoolInfo = $poolInfo !== null;
$hasVariant = $variant !== null;
$hasVariantExists = $variantExists !== null;
$query = <<<SQL
SELECT u.upload_id, u.user_id, u.pool_id, u.upload_secret, u.upload_name,
INET6_NTOA(u.upload_ip), UNIX_TIMESTAMP(u.upload_created), UNIX_TIMESTAMP(u.upload_accessed)
FROM prm_uploads AS u
SQL;
if($hasVariant)
$query .= <<<SQL
LEFT JOIN prm_uploads_files AS uf
ON u.upload_id = uf.upload_id AND uf.upload_variant = ?
SQL;
$args = 0;
$query = 'SELECT upload_id, user_id, app_id, upload_secret, upload_name, upload_bump, INET6_NTOA(upload_ip), UNIX_TIMESTAMP(upload_created), UNIX_TIMESTAMP(upload_accessed), UNIX_TIMESTAMP(upload_expires) FROM prm_uploads';
if($hasExpired) {
if($expired)
$query .= sprintf(' %s upload_expires IS NOT NULL AND upload_expires <= NOW()', ++$args > 1 ? 'AND' : 'WHERE');
else
$query .= sprintf(' %s (upload_expires IS NULL OR upload_expires > NOW())', ++$args > 1 ? 'AND' : 'WHERE');
}
if($hasPoolInfo)
$query .= sprintf(' %s u.pool_id = ?', ++$args > 1 ? 'AND' : 'WHERE');
if($hasVariantExists)
$query .= sprintf(
' %s uf.upload_id %s NULL',
++$args > 1 ? 'AND' : 'WHERE',
$variantExists ? 'IS NOT' : 'IS'
);
$stmt = $this->cache->get($query);
if($hasVariant)
$stmt->nextParameter($variant);
if($hasPoolInfo)
$stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo);
$stmt->execute();
return $stmt->getResult()->getIterator(UploadInfo::fromResult(...));
@ -42,41 +63,33 @@ class UploadsData {
public function getUpload(
?string $uploadId = null,
AppInfo|string|null $appInfo = null,
PoolInfo|string|null $poolInfo = null,
UserInfo|string|null $userInfo = null,
?string $secret = null,
?bool $expired = null,
): UploadInfo {
$hasUploadId = $uploadId !== null;
$hasAppInfo = $appInfo !== null;
$hasPoolInfo = $poolInfo !== null;
$hasUserInfo = $userInfo !== null;
$hasSecret = $secret !== null;
$hasExpired = $expired !== null;
$args = 0;
$query = 'SELECT upload_id, user_id, app_id, upload_secret, upload_name, upload_bump, INET6_NTOA(upload_ip), UNIX_TIMESTAMP(upload_created), UNIX_TIMESTAMP(upload_accessed), UNIX_TIMESTAMP(upload_expires) FROM prm_uploads';
$query = 'SELECT upload_id, user_id, pool_id, upload_secret, upload_name, INET6_NTOA(upload_ip), UNIX_TIMESTAMP(upload_created), UNIX_TIMESTAMP(upload_accessed) FROM prm_uploads';
if($hasUploadId) {
++$args;
$query .= ' WHERE upload_id = ?';
}
if($hasAppInfo)
$query .= sprintf(' %s app_id = ?', ++$args > 1 ? 'AND' : 'WHERE');
if($hasPoolInfo)
$query .= sprintf(' %s pool_id = ?', ++$args > 1 ? 'AND' : 'WHERE');
if($hasUserInfo)
$query .= sprintf(' %s user_id = ?', ++$args > 1 ? 'AND' : 'WHERE');
if($hasSecret)
$query .= sprintf(' %s upload_secret = ?', ++$args > 1 ? 'AND' : 'WHERE');
if($hasExpired) {
if($expired)
$query .= sprintf(' %s upload_expires IS NOT NULL AND upload_expires <= NOW()', ++$args > 1 ? 'AND' : 'WHERE');
else
$query .= sprintf(' %s (upload_expires IS NULL OR upload_expires > NOW())', ++$args > 1 ? 'AND' : 'WHERE');
}
$stmt = $this->cache->get($query);
if($hasUploadId)
$stmt->nextParameter($uploadId);
if($hasAppInfo)
$stmt->nextParameter($appInfo instanceof AppInfo ? $appInfo->id : $appInfo);
if($hasPoolInfo)
$stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo);
if($hasUserInfo)
$stmt->nextParameter($userInfo instanceof UserInfo ? $userInfo->id : $userInfo);
if($hasSecret)
@ -91,7 +104,7 @@ class UploadsData {
}
public function resolveUploadFromVariant(
AppInfo|string $appInfo,
PoolInfo|string $poolInfo,
UserInfo|string $userInfo,
StorageRecord|string $fileInfo,
string $variant
@ -101,16 +114,12 @@ class UploadsData {
FROM prm_uploads AS u
LEFT JOIN prm_uploads_files AS uf
ON uf.upload_id = u.upload_id
WHERE u.app_id = ?
WHERE u.pool_id = ?
AND u.user_id = ?
AND uf.file_id = ?
AND uf.upload_variant = ?
AND (
u.upload_expires IS NULL
OR u.upload_expires > NOW()
)
SQL);
$stmt->nextParameter($appInfo instanceof AppInfo ? $appInfo->id : $appInfo);
$stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo);
$stmt->nextParameter($userInfo instanceof UserInfo ? $userInfo->id : $userInfo);
$stmt->nextParameter($fileInfo instanceof StorageRecord ? $fileInfo->id : $fileInfo);
$stmt->nextParameter($variant);
@ -124,27 +133,20 @@ class UploadsData {
}
public function createUpload(
AppInfo|string $appInfo,
PoolInfo|string $poolInfo,
UserInfo|string $userInfo,
string $fileName,
string $remoteAddr,
int $fileExpiry,
bool $bumpExpiry
string $remoteAddr
): UploadInfo {
if($fileExpiry < 0)
throw new InvalidArgumentException('$fileExpiry must be a positive integer.');
$uploadId = $this->snowflake->nextRandom();
$stmt = $this->cache->get('INSERT INTO prm_uploads (upload_id, app_id, user_id, upload_secret, upload_name, upload_ip, upload_expires, upload_bump) VALUES (?, ?, ?, ?, ?, INET6_ATON(?), FROM_UNIXTIME(?), ?)');
$stmt = $this->cache->get('INSERT INTO prm_uploads (upload_id, pool_id, user_id, upload_secret, upload_name, upload_ip) VALUES (?, ?, ?, ?, ?, INET6_ATON(?))');
$stmt->nextParameter($uploadId);
$stmt->nextParameter($appInfo instanceof AppInfo ? $appInfo->id : $appInfo);
$stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo);
$stmt->nextParameter($userInfo instanceof UserInfo ? $userInfo->id : $userInfo);
$stmt->nextParameter(XString::random(4));
$stmt->nextParameter($fileName);
$stmt->nextParameter($remoteAddr);
$stmt->nextParameter($fileExpiry > 0 ? (time() + $fileExpiry) : 0);
$stmt->nextParameter($bumpExpiry ? $fileExpiry : 0);
$stmt->execute();
return $this->getUpload(uploadId: $uploadId);
@ -153,8 +155,7 @@ class UploadsData {
public function updateUpload(
UploadInfo|string $uploadInfo,
?string $fileName = null,
int|null|bool $accessedAt = false,
int|null|false $expiresAt = false
int|bool $accessedAt = false,
): void {
$fields = [];
$values = [];
@ -170,11 +171,6 @@ class UploadsData {
$values[] = $accessedAt;
}
if($expiresAt !== false) {
$fields[] = 'upload_expires = NOW() + INTERVAL ? SECOND';
$values[] = $expiresAt;
}
if(empty($fields))
return;
@ -191,12 +187,51 @@ class UploadsData {
$stmt->execute();
}
public function deleteExpiredUploads(): void {
$this->dbConn->execute(<<<SQL
public function deleteStaleUploads(
PoolInfo|string $poolInfo,
int $staleSeconds,
UploadInfo|string|array|null $ignore = null
): void {
$hasIgnore = !empty($ignore);
$query = <<<SQL
DELETE FROM prm_uploads
WHERE upload_expires IS NOT NULL
AND upload_expires <= NOW()
WHERE pool_id = ?
AND upload_accessed <= (NOW() - INTERVAL ? SECOND)
SQL;
if($hasIgnore)
$query .= sprintf(
' AND upload_id NOT IN (%s)',
DbTools::prepareListString($ignore)
);
$stmt = $this->cache->get($query);
$stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo);
$stmt->nextParameter($staleSeconds);
if($hasIgnore)
foreach($ignore as $item) {
if($item instanceof UploadInfo)
$stmt->nextParameter($item->id);
elseif(is_string($item))
$stmt->nextParameter($item);
else
throw new InvalidArgumentException('$ignore contains entries with unsupported types');
}
$stmt->execute();
}
public function getUploadVariants(
UploadInfo|string $uploadInfo
): iterable {
$stmt = $this->cache->get(<<<SQL
SELECT upload_id, upload_variant, file_id, file_type
FROM prm_uploads_files
WHERE upload_id = ?
SQL);
$stmt->nextParameter($uploadInfo instanceof UploadInfo ? $uploadInfo->id : $uploadInfo);
$stmt->execute();
return $stmt->getResult()->getIterator(UploadVariantInfo::fromResult(...));
}
public function getUploadVariant(
@ -239,6 +274,29 @@ class UploadsData {
return $this->getUploadVariant($uploadInfo, $variant);
}
public function deleteUploadVariant(
UploadVariantInfo|string $uploadIdOrVariantInfo,
?string $variant = null
): void {
if($uploadIdOrVariantInfo instanceof UploadVariantInfo) {
if($variant !== null)
throw new InvalidArgumentException('$variant must be null if $uploadIdOrVariantInfo is an instance of UploadVariantInfo');
$uploadId = $uploadIdOrVariantInfo->uploadId;
$variant = $uploadIdOrVariantInfo->variant;
} else {
if($variant === null)
throw new InvalidArgumentException('$variant may not be null if $uploadIdOrVariantInfo is a string');
$uploadId = $uploadIdOrVariantInfo;
}
$stmt = $this->cache->get('DELETE FROM prm_uploads_files WHERE upload_id = ? AND upload_variant = ?');
$stmt->nextParameter($uploadId);
$stmt->nextParameter($variant);
$stmt->execute();
}
public function resolveLegacyId(string $legacyId): ?string {
$stmt = $this->cache->get('SELECT upload_id FROM prm_uploads_legacy WHERE upload_id_legacy = ?');
$stmt->nextParameter($legacyId);

196
src/Uploads/UploadsRoutes.php
View file

@ -2,21 +2,22 @@
namespace EEPROM\Uploads;
use RuntimeException;
use Index\XNumber;
use Index\{XArray,XNumber};
use Index\Http\Routing\{HandlerAttribute,HttpDelete,HttpGet,HttpOptions,HttpPost,Router,RouteHandler};
use EEPROM\Apps\AppsContext;
use EEPROM\Auth\AuthInfo;
use EEPROM\Blacklist\BlacklistContext;
use EEPROM\Storage\{StorageContext,StorageImportMode};
use EEPROM\Denylist\{DenylistContext,DenylistReason};
use EEPROM\Pools\PoolsContext;
use EEPROM\Pools\Rules\{EnsureVariantRule,EnsureVariantRuleThumb};
use EEPROM\Storage\{StorageContext,StorageImportMode,StorageRecordGetFileField};
use EEPROM\Uploads\UploadsContext;
class UploadsRoutes implements RouteHandler {
public function __construct(
private AuthInfo $authInfo,
private AppsContext $appsCtx,
private PoolsContext $poolsCtx,
private UploadsContext $uploadsCtx,
private StorageContext $storageCtx,
private BlacklistContext $blacklistCtx,
private DenylistContext $denylistCtx,
private bool $isApiDomain
) {}
@ -31,7 +32,7 @@ class UploadsRoutes implements RouteHandler {
#[HttpOptions('/uploads/([A-Za-z0-9]+|[A-Za-z0-9\-_]{32})(?:\.(t|json))?')]
#[HttpGet('/uploads/([A-Za-z0-9]+|[A-Za-z0-9\-_]{32})(?:\.(t|json))?')]
public function getUpload($response, $request, string $fileId, string $fileExt = '') {
public function getUpload($response, $request, string $uploadId, string $uploadVariant = '') {
if($this->isApiDomain) {
if($request->hasHeader('Origin'))
$response->setHeader('Access-Control-Allow-Credentials', 'true');
@ -45,87 +46,87 @@ class UploadsRoutes implements RouteHandler {
if($request->getMethod() === 'OPTIONS')
return 204;
$isData = $fileExt === '';
$isThumbnail = $fileExt === 't';
$isJson = $this->isApiDomain && $fileExt === 'json';
$isOriginal = $uploadVariant === '';
$isJson = $this->isApiDomain && $uploadVariant === 'json';
if(!$isData && !$isThumbnail && !$isJson)
return 404;
$uploadsData = $this->uploadsCtx->uploadsData;
if(strlen($fileId) === 32) {
$fileId = $uploadsData->resolveLegacyId($fileId) ?? $fileId;
$fileSecret = null;
if(strlen($uploadId) === 32) {
$uploadId = $this->uploadsCtx->uploads->resolveLegacyId($uploadId) ?? $uploadId;
$uploadSecret = null;
} else {
if(strlen($fileId) < 5) {
if(strlen($uploadId) < 5) {
$response->setContent('File not found.');
return 404;
}
$fileSecret = substr($fileId, -4);
$fileId = XNumber::fromBase62(substr($fileId, 0, -4));
$uploadSecret = substr($uploadId, -4);
$uploadId = XNumber::fromBase62(substr($uploadId, 0, -4));
}
try {
$uploadInfo = $uploadsData->getUpload(uploadId: $fileId, expired: false, secret: $fileSecret);
$uploadInfo = $this->uploadsCtx->uploads->getUpload(uploadId: $uploadId, secret: $uploadSecret);
} catch(RuntimeException $ex) {
$response->setContent('File not found.');
return 404;
}
if($isThumbnail) {
try {
$originalInfo = $this->storageCtx->getFileRecordFromVariant(
$uploadsData->getUploadVariant($uploadInfo, '')
);
} catch(RuntimeException $ex) {
try {
$variantInfo = $this->uploadsCtx->uploads->getUploadVariant($uploadInfo, $uploadVariant);
} catch(RuntimeException $ex) {
if($isOriginal) {
$response->setContent('Data is missing.');
return 404;
}
if(!$this->storageCtx->supportsThumbnailing($originalInfo))
return 404;
try {
$variantInfo = $uploadsData->getUploadVariant($uploadInfo, 'thumb');
} catch(RuntimeException $ex) {
$storageInfo = $this->storageCtx->createThumbnail($originalInfo);
$uploadsData->createUploadVariant($uploadInfo, 'thumb', $storageInfo);
}
} else {
try {
$variantInfo = $uploadsData->getUploadVariant($uploadInfo, '');
} catch(RuntimeException $ex) {
$response->setContent('Data is missing.');
return 404;
}
if(!$isJson)
$uploadsData->updateUpload(
$uploadInfo,
accessedAt: true,
expiresAt: $uploadInfo->bumpAmountForUpdate,
$poolInfo = $this->poolsCtx->pools->getPool($uploadInfo->poolId);
$originalInfo = $this->storageCtx->records->getFile(
$this->uploadsCtx->uploads->getUploadVariant($uploadInfo, '')->fileId
);
$rule = XArray::first(
$this->poolsCtx->getPoolRules($poolInfo)->getRules(EnsureVariantRule::TYPE),
fn($rule) => $rule->variant === $uploadVariant
);
if($rule instanceof EnsureVariantRule && $rule->onAccess) {
if($rule->info instanceof EnsureVariantRuleThumb) {
$storageInfo = $this->storageCtx->createThumbnailFromRule($originalInfo, $rule->info);
$variantInfo = $this->uploadsCtx->uploads->createUploadVariant($uploadInfo, $rule->variant, $storageInfo);
}
}
} catch(RuntimeException $ex) {
$response->setContent('Original data is missing.');
return 404;
}
}
if(!isset($variantInfo)) {
$response->setContent('Variant not available.');
return 404;
}
if($isOriginal)
$this->uploadsCtx->uploads->updateUpload(
$uploadInfo,
accessedAt: true,
);
if(!isset($storageInfo))
try {
$storageInfo = $this->storageCtx->getFileRecordFromVariant($variantInfo);
$storageInfo = $this->storageCtx->records->getFile($variantInfo->fileId);
} catch(RuntimeException $ex) {
$response->setContent('Data is missing.');
return 404;
}
$blInfo = $this->blacklistCtx->getBlacklistEntry($storageInfo);
if($blInfo !== null) {
$response->setContent(match($blInfo->reason) {
'copyright' => 'File is unavailable for copyright reasons.',
'rules' => 'File was in violation of the rules.',
$denyInfo = $this->denylistCtx->getDenylistEntry($storageInfo);
if($denyInfo !== null) {
$response->setContent(match($denyInfo->reason) {
DenylistReason::Copyright => 'File is unavailable for copyright reasons.',
DenylistReason::Rules => 'File was in violation of the rules.',
default => 'File was removed for reasons beyond understanding.',
});
return $blInfo->isCopyrightTakedown ? 451 : 410;
return $denyInfo->isCopyrightTakedown ? 451 : 410;
}
if($isJson)
@ -156,17 +157,6 @@ class UploadsRoutes implements RouteHandler {
if($request->getMethod() === 'OPTIONS')
return 204;
if(!$request->isFormContent())
return 400;
$content = $request->getContent();
try {
$appInfo = $this->appsCtx->getApp((string)$content->getParam('src', FILTER_VALIDATE_INT));
} catch(RuntimeException $ex) {
return 404;
}
if(!$this->authInfo->loggedIn)
return 401;
@ -174,6 +164,11 @@ class UploadsRoutes implements RouteHandler {
if($userInfo->restricted)
return 403;
if(!$request->isFormContent())
return 400;
$content = $request->getContent();
try {
$file = $content->getUploadedFile('file');
} catch(RuntimeException $ex) {
@ -185,8 +180,23 @@ class UploadsRoutes implements RouteHandler {
if($fileSize < 1)
return 400;
$maxFileSize = $appInfo->dataSizeLimit;
if($appInfo->allowSizeMultiplier)
try {
$poolInfo = $this->poolsCtx->pools->getPool((string)$content->getParam('src', FILTER_VALIDATE_INT));
if($poolInfo->hasSecret)
return 404;
} catch(RuntimeException $ex) {
return 404;
}
$rules = $this->poolsCtx->getPoolRules($poolInfo);
// If there's no constrain_size rule on this pool its likely not meant to be used through this API!
$maxSizeRule = $rules->getRule('constrain_size');
if($maxSizeRule === null)
return 500;
$maxFileSize = $maxSizeRule->maxSize;
if($maxSizeRule->allowLegacyMultiplier)
$maxFileSize *= $userInfo->dataSizeMultiplier;
if($file->getSize() !== $fileSize || $fileSize > $maxFileSize) {
@ -195,11 +205,10 @@ class UploadsRoutes implements RouteHandler {
return 413;
}
$uploadsData = $this->uploadsCtx->uploadsData;
$hash = hash_file('sha256', $localFile, true);
$blInfo = $this->blacklistCtx->getBlacklistEntry($hash);
if($blInfo !== null)
$denyInfo = $this->denylistCtx->getDenylistEntry($hash);
if($denyInfo !== null)
return 451;
$fileName = $file->getSuggestedFileName();
@ -208,7 +217,7 @@ class UploadsRoutes implements RouteHandler {
$mediaType = mime_content_type($localFile);
try {
$storageInfo = $this->storageCtx->records->getFile($hash);
$storageInfo = $this->storageCtx->records->getFile($hash, StorageRecordGetFileField::Hash);
} catch(RuntimeException $ex) {
$storageInfo = $this->storageCtx->importFile(
$file->getLocalFileName(),
@ -220,27 +229,24 @@ class UploadsRoutes implements RouteHandler {
try {
// TODO: Okay so we're reading from the prm_uploads_files table, then fetching the info from prm_uploads,
// and then fetching the info again from the prm_uploads_files table... this can be done better but this will do for now
$uploadInfo = $uploadsData->getUpload(
uploadId: $uploadsData->resolveUploadFromVariant($appInfo, $userInfo, $storageInfo, ''),
expired: false
$uploadInfo = $this->uploadsCtx->uploads->getUpload(
uploadId: $this->uploadsCtx->uploads->resolveUploadFromVariant($poolInfo, $userInfo, $storageInfo, '')
);
$variantInfo = $uploadsData->getUploadVariant($uploadInfo, '');
$variantInfo = $this->uploadsCtx->uploads->getUploadVariant($uploadInfo, '');
$uploadsData->updateUpload(
$this->uploadsCtx->uploads->updateUpload(
$uploadInfo,
fileName: $fileName,
expiresAt: $uploadInfo->bumpAmountForUpdate,
accessedAt: true,
);
} catch(RuntimeException $ex) {
$uploadInfo = $uploadsData->createUpload(
$appInfo,
$uploadInfo = $this->uploadsCtx->uploads->createUpload(
$poolInfo,
$userInfo,
$fileName,
$request->getRemoteAddress(),
$appInfo->bumpAmount,
true
$request->getRemoteAddress()
);
$variantInfo = $uploadsData->createUploadVariant(
$variantInfo = $this->uploadsCtx->uploads->createUploadVariant(
$uploadInfo, '', $storageInfo, $mediaType
);
}
@ -254,7 +260,7 @@ class UploadsRoutes implements RouteHandler {
}
#[HttpDelete('/uploads/([A-Za-z0-9]+|[A-Za-z0-9\-_]{32})')]
public function deleteUpload($response, $request, string $fileId) {
public function deleteUpload($response, $request, string $uploadId) {
if($request->hasHeader('Origin'))
$response->setHeader('Access-Control-Allow-Credentials', 'true');
@ -264,23 +270,21 @@ class UploadsRoutes implements RouteHandler {
if(!$this->authInfo->loggedIn)
return 401;
$uploadsData = $this->uploadsCtx->uploadsData;
if(strlen($fileId) === 32) {
$fileId = $uploadsData->resolveLegacyId($fileId) ?? $fileId;
$fileSecret = null;
if(strlen($uploadId) === 32) {
$uploadId = $this->uploadsCtx->uploads->resolveLegacyId($uploadId) ?? $uploadId;
$uploadSecret = null;
} else {
if(strlen($fileId) < 5) {
if(strlen($uploadId) < 5) {
$response->setContent('File not found.');
return 404;
}
$fileSecret = substr($fileId, -4);
$fileId = XNumber::fromBase62(substr($fileId, 0, -4));
$uploadSecret = substr($uploadId, -4);
$uploadId = XNumber::fromBase62(substr($uploadId, 0, -4));
}
try {
$uploadInfo = $uploadsData->getUpload(uploadId: $fileId, expired: false, secret: $fileSecret);
$uploadInfo = $this->uploadsCtx->uploads->getUpload(uploadId: $uploadId, secret: $uploadSecret);
} catch(RuntimeException $ex) {
$response->setContent('File not found.');
return 404;
@ -290,7 +294,7 @@ class UploadsRoutes implements RouteHandler {
if($userInfo->restricted || $userInfo->id !== $uploadInfo->userId)
return 403;
$uploadsData->deleteUpload($uploadInfo);
$this->uploadsCtx->uploads->deleteUpload($uploadInfo);
return 204;
}
}

8
src/Users/UsersContext.php
View file

@ -4,14 +4,14 @@ namespace EEPROM\Users;
use Index\Db\DbConnection;
class UsersContext {
public private(set) UsersData $usersData;
public private(set) UsersData $users;
public function __construct(DbConnection $dbConn) {
$this->usersData = new UsersData($dbConn);
$this->users = new UsersData($dbConn);
}
public function getUser(string $userId): UserInfo {
$this->usersData->ensureUserExists($userId);
return $this->usersData->getUser($userId);
$this->users->ensureUserExists($userId);
return $this->users->getUser($userId);
}
}

78
tools/README.md Normal file
View file

@ -0,0 +1,78 @@
# EEPROM Tools
Below a short description of every script in this directory, what it does and what arguments it takes, etc.
## `cron`
Usage: `cron`
This script performs scheduled maintenance, primarily cleaning up orphaned records and executing other rule stuff.
It accepts no arguments BUT you should be running the script as the same user as the webserver/PHP FPM to ensure that you won't run into permission issues.
## `delete-upload`
Usage: `delete-upload [OPTION]... [UPLOAD]`
This script deletes a single upload and all of its variants from the database.
It also attempts to purge the associated files, but that can only be done if they're not used by other uploads.
This tool accepts one option `-s`/`--base62` which causes `[UPLOAD]` to be interpreted as a base62 number to make administration easier.
Make sure to remove the trailing four secret characters when copying from a URL!
## `deny-file`
Usage: `nuke-file [OPTION]... [FILE]`
Prevents a file from being uploaded in the future and marks the existing copy for removal.
This tool accepts three options:
- `-r`/`--reason`: Required!!! Reason for denying the file. Currently supports `copyright`, `rules` and `other`.
- `-h`/`--hash`: Treat the provided ID as a SHA256 hash, can be hex, URI-safe base64 or raw if you're feeling feisty.
- `-s`/`--base62`: Interpret provided ID as a base62 number.
- `-u`/`--upload`: Interpret provided ID as an upload ID instead of a file ID.
## `migrate`
Usage: `migrate`
Runs database migrations, part of the Index framework.
Does not take any arguments but does write a lockfile to `/tmp` (or whatever the system temporary file directory is) to ensure this script is only run once.
## `migrate-override-toggle`
Usage: `migrate-override-toggle`
Toggles the migration lock file manually.
Can be useful to recover from a crash without having to dig through `/tmp` or to just manually enable maintenance mode.
## `new-migration`
Usage: `new-migration [NAME PART]...`
Creates a new database migration, part of the Index framework.
A new file will be written to the `database` folder within the project following a template embedded in the Index library source code.
All arguments of passed to the command will be concatenated to form the name of the migration.
## `nuke-file`
Usage: `nuke-file [OPTION]... [FILE]`
The nuclear approach.
Deletes a file from the storage and takes any associated uploads down with it.
This tool accepts three options:
- `-h`/`--hash`: Treat the provided ID as a SHA256 hash, can be hex, URI-safe base64 or raw if you're feeling feisty.
- `-s`/`--base62`: Interpret provided ID as a base62 number.
- `-u`/`--upload`: Interpret provided ID as an upload ID instead of a file ID.
- `-v`/`--variant=VARIANT`: If using `-u`/`--upload`, specify the name of the variant. If left omitted the all variants will be removed.

140
tools/cron Executable file
View file

@ -0,0 +1,140 @@
#!/usr/bin/env php
<?php
use EEPROM\EEPROMContext;
use EEPROM\Uploads\{UploadInfo,UploadsContext};
use EEPROM\Pools\PoolInfo;
use EEPROM\Pools\Rules\{
EnsureVariantRule,EnsureVariantRuleThumb,RemoveStaleRule,ScanForumRule
};
use EEPROM\Storage\{StorageContext,StorageRecord};
require_once __DIR__ . '/../eeprom.php';
$cronPath = sys_get_temp_dir() . '/eeprom-cron-' . hash('sha256', __DIR__) . '.lock';
if(is_file($cronPath))
die('EEPROM cron script is already running.' . PHP_EOL);
touch($cronPath);
try {
$pools = [];
$getPool = fn($ruleRaw) => array_key_exists($ruleRaw->poolId, $pools) ? $pools[$ruleRaw->poolId] : (
$pools[$ruleRaw->poolId] = $eeprom->poolsCtx->pools->getPool($ruleRaw->poolId)
);
// Run cleanup adjacent tasks first
$rules = $eeprom->poolsCtx->pools->getPoolRules(types: ['remove_stale', 'scan_forum']);
foreach($rules as $ruleRaw) {
$poolInfo = $getPool($ruleRaw);
$rule = $eeprom->poolsCtx->rules->create($ruleRaw);
if($rule instanceof RemoveStaleRule) {
printf('Removing stale entries for pool #%d...%s', $poolInfo->id, PHP_EOL);
$eeprom->uploadsCtx->uploads->deleteStaleUploads(
$poolInfo,
$rule->inactiveForSeconds,
$rule->ignoreUploadIds
);
} elseif($rule instanceof ScanForumRule) {
//var_dump('scan_forum', $rule);
// necessary stuff for this doesn't exist yet so this can remain a no-op
}
}
echo PHP_EOL;
printf('Purging denylisted file records...%s', PHP_EOL);
$records = $eeprom->storageCtx->records->getFiles(denied: true);
foreach($records as $record) {
printf('Deleting file #%d...%s', $record->id, PHP_EOL);
$eeprom->storageCtx->deleteFile($record);
}
echo PHP_EOL;
printf('Purging uploads without an original variant...%s', PHP_EOL);
$records = $eeprom->uploadsCtx->uploads->getUploads(variant: '', variantExists: false);
foreach($records as $record) {
printf('Deleting upload #%d...%s', $record->id, PHP_EOL);
$eeprom->uploadsCtx->uploads->deleteUpload($record);
}
echo PHP_EOL;
printf('Purging orphaned file records...%s', PHP_EOL);
$records = $eeprom->storageCtx->records->getFiles(orphaned: true);
foreach($records as $record) {
printf('Deleting file #%d...%s', $record->id, PHP_EOL);
$eeprom->storageCtx->deleteFile($record);
}
echo PHP_EOL;
// Now run the ones that make new things
$rules = $eeprom->poolsCtx->pools->getPoolRules(types: ['ensure_variant']);
foreach($rules as $ruleRaw) {
$poolInfo = $getPool($ruleRaw);
$rule = $eeprom->poolsCtx->rules->create($ruleRaw);
if($rule instanceof EnsureVariantRule) {
if(!$rule->onCron)
continue;
$createVariant = null; // ensure this wasn't previously assigned
if($rule->info instanceof EnsureVariantRuleThumb) {
$createVariant = function(
StorageContext $storageCtx,
UploadsContext $uploadsCtx,
PoolInfo $poolInfo,
EnsureVariantRule $rule,
EnsureVariantRuleThumb $ruleInfo,
StorageRecord $originalInfo,
UploadInfo $uploadInfo
) {
$uploadsCtx->uploads->createUploadVariant(
$uploadInfo,
$rule->variant,
$storageCtx->createThumbnailFromRule(
$originalInfo,
$ruleInfo
)
);
};
}
if(!isset($createVariant) || !is_callable($createVariant)) {
printf('!!! Could not create a constructor for "%s" variants for pool #%d !!! Skipping for now...%s', $rule->variant, $poolInfo->id, PHP_EOL);
continue;
}
printf('Ensuring existence of "%s" variants for pool #%d...%s', $rule->variant, $poolInfo->id, PHP_EOL);
$uploads = $eeprom->uploadsCtx->uploads->getUploads(
poolInfo: $poolInfo,
variant: $rule->variant,
variantExists: false
);
$processed = 0;
foreach($uploads as $uploadInfo)
try {
$createVariant(
$eeprom->storageCtx,
$eeprom->uploadsCtx,
$poolInfo,
$rule,
$rule->info,
$eeprom->storageCtx->records->getFile(
$eeprom->uploadsCtx->uploads->getUploadVariant($uploadInfo, '')->fileId
),
$uploadInfo
);
} catch(Exception $ex) {
printf('Exception thrown while processing "%s" variant for upload #%d in pool #%d:%s', $rule->variant, $uploadInfo->id, $poolInfo->id, PHP_EOL);
printf('%s%s', $ex->getMessage(), PHP_EOL);
} finally {
++$processed;
}
printf('Processed %d records!%s%s', $processed, PHP_EOL, PHP_EOL);
}
}
echo PHP_EOL;
} finally {
unlink($cronPath);
}

77
tools/delete-upload Executable file
View file

@ -0,0 +1,77 @@
#!/usr/bin/env php
<?php
use Index\XNumber;
require_once __DIR__ . '/../eeprom.php';
$options = getopt('s', [
'base62',
], $restIndex);
if($argc <= $restIndex)
die(sprintf('No upload ID specified.%s', PHP_EOL));
$uploadId = $argv[$restIndex];
$isBase62 = isset($options['s']) || isset($options['base62']);
if($isBase62)
$uploadId = XNumber::fromBase62($uploadId);
try {
$uploadInfo = $eeprom->uploadsCtx->uploads->getUpload(uploadId: $uploadId);
} catch(RuntimeException $ex) {
$hint = $isBase62 ? 'trim last four characters from base62 id' : 'specify -s for base62 ids';
die(sprintf('The requested upload does not exist. (hint: %s)%s', $hint, PHP_EOL));
}
printf(
'Found upload #%d (%s) in pool #%d uploaded by user #%d (%s) at %s with canonical name "%s".%s',
$uploadInfo->id,
$uploadInfo->id62,
$uploadInfo->poolId,
$uploadInfo->userId,
$uploadInfo->remoteAddress,
$uploadInfo->createdAt->toIso8601ZuluString(),
htmlspecialchars($uploadInfo->name),
PHP_EOL
);
$variants = [];
foreach($eeprom->uploadsCtx->uploads->getUploadVariants($uploadInfo) as $variantInfo) {
$variants[] = $variantInfo;
printf(
'Found variant "%s" linking to file #%d with media type %s.%s',
$variantInfo->variant,
$variantInfo->fileId,
htmlspecialchars($variantInfo->type ?? '(none)'),
PHP_EOL
);
}
$displayStorageDeleteHint = false;
if(count($variants) < 1) {
printf('This upload has no variants to delete.%s', PHP_EOL);
} else {
$variants = array_reverse($variants);
foreach($variants as $variantInfo)
try {
printf('Deleting variant "%s" for upload #%d...%s', $variantInfo->variant, $variantInfo->uploadId, PHP_EOL);
$eeprom->uploadsCtx->uploads->deleteUploadVariant($variantInfo);
printf('Attempting to delete file...%s', PHP_EOL);
$eeprom->storageCtx->deleteFile($variantInfo->fileId, false);
} catch(Exception $ex) {
printf('Exception occurred while deleting variant:%s', PHP_EOL);
printf('%s%s', $ex->getMessage(), PHP_EOL);
}
}
try {
printf('Deleting upload #%d...%s', $uploadInfo->id, PHP_EOL);
$eeprom->uploadsCtx->uploads->deleteUpload($uploadInfo);
} catch(Exception $ex) {
printf('Exception occurred while deleting upload:%s%s%s', PHP_EOL, $ex->getMessage(), PHP_EOL);
}
if($displayStorageDeleteHint)
printf('NOTE: if the exception(s) that occurred while deleting variants were related to database constraints, an upload linking to the same file exists, if they occurred due to filesystem permissions, the file will linger but will be removed during the next scheduled clean up.%s', PHP_EOL);

93
tools/deny-file Executable file
View file

@ -0,0 +1,93 @@
#!/usr/bin/env php
<?php
use EEPROM\Denylist\DenylistReason;
use EEPROM\Storage\StorageRecordGetFileField;
use Index\{UriBase64,XArray,XNumber};
require_once __DIR__ . '/../eeprom.php';
$options = getopt('hsur:', [
'hash',
'base62',
'upload',
'reason:',
], $restIndex);
if($argc <= $restIndex)
die(sprintf('No file ID specified.%s', PHP_EOL));
$fileId = $argv[$restIndex];
$fileField = StorageRecordGetFileField::Id;
$uploadId = null;
$isHash = isset($options['h']) || isset($options['hash']);
$isBase62 = isset($options['s']) || isset($options['base62']);
$isUploadId = isset($options['u']) || isset($options['upload']);
try {
$reason = DenylistReason::from($options['r'] ?? $options['reason'] ?? '');
} catch(ValueError $ex) {
die(sprintf(
'Reason value is not supported, it must be one of the following values: %s.%s',
implode(', ', XArray::select(DenylistReason::cases(), fn($case) => $case->value)),
PHP_EOL
));
}
if($isHash) {
$fileId = match(strlen($fileId)) {
32 => $fileId,
43 => UriBase64::decode($fileId),
64 => hex2bin($fileId),
66 => hex2bin(substr($fileId, 2)),
default => null,
};
$fileField = StorageRecordGetFileField::Hash;
} else {
if($isBase62)
$fileId = XNumber::fromBase62($fileId);
if($isUploadId) {
$uploadId = $fileId;
$fileId = null;
try {
$variantInfo = $eeprom->uploadsCtx->uploads->getUploadVariant($uploadId, '');
} catch(RuntimeException $ex) {
$hint = $isBase62 ? 'trim last four characters from base62 id' : 'specify -s for base62 ids';
die(sprintf('The requested upload does not exist. (hint: %s)%s', $hint, PHP_EOL));
}
$fileId = $variantInfo->fileId;
}
}
if($fileId === null)
die(sprintf('Could not resolve file ID from the provided inputs.%s', PHP_EOL));
try {
$storageInfo = $eeprom->storageCtx->records->getFile($fileId, $fileField);
} catch(RuntimeException $ex) {
$hint = $isUploadId ? (
$isBase62 ? 'trim last four characters from base62 id' : 'specify -s for base62 ids'
) : 'specify -h for hashes or -u to use upload ids';
die(sprintf('The requested file does not exist. (hint: %s)%s', $hint, PHP_EOL));
}
printf(
'Found file #%d of %d bytes with type %s at %s with SHA256 hash %s.%s',
$storageInfo->id,
$storageInfo->size,
htmlspecialchars($storageInfo->type),
$storageInfo->createdAt->toIso8601ZuluString(),
UriBase64::encode($storageInfo->hash),
PHP_EOL
);
try {
printf('Attempting to deny file...%s', PHP_EOL);
$eeprom->denylistCtx->denyFile($storageInfo, $reason);
} catch(Exception $ex) {
printf('Exception occurred while deleting file:%s', PHP_EOL);
printf('%s%s', $ex->getMessage(), PHP_EOL);
}

12
tools/migrate-override-toggle Executable file
View file

@ -0,0 +1,12 @@
#!/usr/bin/env php
<?php
require_once __DIR__ . '/../eeprom.php';
$lockPath = $eeprom->database->getMigrateLockPath();
if(is_file($lockPath)) {
printf('Removing migration lock...%s', PHP_EOL);
unlink($lockPath);
} else {
printf('Setting migration lock...%s', PHP_EOL);
touch($lockPath);
}

83
tools/nuke-file Executable file
View file

@ -0,0 +1,83 @@
#!/usr/bin/env php
<?php
use EEPROM\Storage\StorageRecordGetFileField;
use Index\{UriBase64,XNumber};
require_once __DIR__ . '/../eeprom.php';
$options = getopt('hsuv:', [
'hash',
'base62',
'upload',
'variant:',
], $restIndex);
if($argc <= $restIndex)
die(sprintf('No file ID specified.%s', PHP_EOL));
$fileId = $argv[$restIndex];
$fileField = StorageRecordGetFileField::Id;
$uploadId = null;
$isHash = isset($options['h']) || isset($options['hash']);
$isBase62 = isset($options['s']) || isset($options['base62']);
$isUploadId = isset($options['u']) || isset($options['upload']);
$variant = $options['variant'] ?? $options['v'] ?? '';
if($isHash) {
$fileId = match(strlen($fileId)) {
32 => $fileId,
43 => UriBase64::decode($fileId),
64 => hex2bin($fileId),
66 => hex2bin(substr($fileId, 2)),
default => null,
};
$fileField = StorageRecordGetFileField::Hash;
} else {
if($isBase62)
$fileId = XNumber::fromBase62($fileId);
if($isUploadId) {
$uploadId = $fileId;
$fileId = null;
try {
$variantInfo = $eeprom->uploadsCtx->uploads->getUploadVariant($uploadId, $variant);
} catch(RuntimeException $ex) {
$hint = $isBase62 ? 'trim last four characters from base62 id' : 'specify -s for base62 ids';
die(sprintf('The requested upload variant does not exist. (hint: %s)%s', $hint, PHP_EOL));
}
$fileId = $variantInfo->fileId;
}
}
if($fileId === null)
die(sprintf('Could not resolve file ID from the provided inputs.%s', PHP_EOL));
try {
$storageInfo = $eeprom->storageCtx->records->getFile($fileId, $fileField);
} catch(RuntimeException $ex) {
$hint = $isUploadId ? (
$isBase62 ? 'trim last four characters from base62 id' : 'specify -s for base62 ids'
) : 'specify -h for hashes or -u to use upload ids, with -v for variant selection';
die(sprintf('The requested file does not exist. (hint: %s)%s', $hint, PHP_EOL));
}
printf(
'Found file #%d of %d bytes with type %s at %s with SHA256 hash %s.%s',
$storageInfo->id,
$storageInfo->size,
htmlspecialchars($storageInfo->type),
$storageInfo->createdAt->toIso8601ZuluString(),
UriBase64::encode($storageInfo->hash),
PHP_EOL
);
try {
printf('Attempting to delete file...%s', PHP_EOL);
$eeprom->storageCtx->deleteFile($storageInfo);
} catch(Exception $ex) {
printf('Exception occurred while deleting file:%s', PHP_EOL);
printf('%s%s', $ex->getMessage(), PHP_EOL);
}