diff --git a/cron.php b/cron.php deleted file mode 100644 index bd3bbc8..0000000 --- a/cron.php +++ /dev/null @@ -1,23 +0,0 @@ -uploadsCtx->uploadsData->deleteExpiredUploads(); - - $records = $eeprom->storageCtx->records->getFiles(orphaned: true); - foreach($records as $record) - $eeprom->storageCtx->deleteFile($record); - - $records = $eeprom->storageCtx->records->getFiles(denied: true); - foreach($records as $record) - $eeprom->storageCtx->deleteFile($record); -} finally { - unlink($cronPath); -} diff --git a/database/2024_12_25_123334_pools_system_with_rules.php b/database/2024_12_25_123334_pools_system_with_rules.php new file mode 100644 index 0000000..4766142 --- /dev/null +++ b/database/2024_12_25_123334_pools_system_with_rules.php @@ -0,0 +1,51 @@ +execute('UPDATE prm_uploads SET upload_accessed = upload_created WHERE upload_accessed IS NULL'); + + // this has to happen first + $conn->execute(<<execute(<<execute('DROP TABLE prm_applications'); + + // shh + $conn->execute('RENAME TABLE prm_blacklist TO prm_denylist'); + $conn->execute(<<execute(<<getString(0), - name: $result->getString(1), - createdTime: $result->getInteger(2), - dataSizeLimit: $result->getInteger(3), - allowSizeMultiplier: $result->getBoolean(4), - bumpAmount: $result->getInteger(5), - ); - } - - public CarbonImmutable $createdAt { - get => CarbonImmutable::createFromTimestampUTC($this->createdTime); - } -} diff --git a/src/Apps/AppsContext.php b/src/Apps/AppsContext.php deleted file mode 100644 index bbba63d..0000000 --- a/src/Apps/AppsContext.php +++ /dev/null @@ -1,21 +0,0 @@ -appsData = new AppsData($dbConn); - } - - public function getApp(string $appId): AppInfo { - $appInfo = $this->appsData->getApp($appId); - if($appInfo === null) - throw new RuntimeException('Not application with this ID exists.'); - - return $appInfo; - } -} diff --git a/src/Apps/AppsData.php b/src/Apps/AppsData.php deleted file mode 100644 index cfe9929..0000000 --- a/src/Apps/AppsData.php +++ /dev/null @@ -1,23 +0,0 @@ -cache = new DbStatementCache($dbConn); - } - - public function getApp(string $appId): ?AppInfo { - $stmt = $this->cache->get('SELECT app_id, app_name, UNIX_TIMESTAMP(app_created), app_size_limit, app_allow_size_multiplier, app_expiry FROM prm_applications WHERE app_id = ?'); - $stmt->nextParameter($appId); - $stmt->execute(); - - $result = $stmt->getResult(); - return $result->next() ? AppInfo::fromResult($result) : null; - } -} diff --git a/src/Blacklist/BlacklistContext.php b/src/Blacklist/BlacklistContext.php deleted file mode 100644 index d84a8b3..0000000 --- a/src/Blacklist/BlacklistContext.php +++ /dev/null @@ -1,27 +0,0 @@ -blacklistData = new BlacklistData($dbConn); - } - - public function createBlacklistEntry(StorageRecord|string $recordOrHash, string $reason): void { - if($recordOrHash instanceof StorageRecord) - $recordOrHash = $recordOrHash->hash; - - $this->blacklistData->createBlacklistEntry($recordOrHash, $reason); - } - - public function getBlacklistEntry(StorageRecord|string $recordOrHash): ?BlacklistInfo { - if($recordOrHash instanceof StorageRecord) - $recordOrHash = $recordOrHash->hash; - - return $this->blacklistData->getBlacklistEntry($recordOrHash); - } -} diff --git a/src/Blacklist/BlacklistData.php b/src/Blacklist/BlacklistData.php deleted file mode 100644 index d4b63f0..0000000 --- a/src/Blacklist/BlacklistData.php +++ /dev/null @@ -1,34 +0,0 @@ -cache = new DbStatementCache($dbConn); - } - - public function getBlacklistEntry(string $hash): ?BlacklistInfo { - $stmt = $this->cache->get('SELECT bl_hash, bl_reason, UNIX_TIMESTAMP(bl_created) FROM prm_blacklist WHERE bl_hash = ?'); - $stmt->nextParameter($hash); - $stmt->execute(); - - $result = $stmt->getResult(); - return $result->next() ? BlacklistInfo::fromResult($result) : null; - } - - public function createBlacklistEntry(string $hash, string $reason): void { - if(strlen($hash) !== 32) - throw new InvalidArgumentException('$hash must be 32 bytes.'); - if(!in_array($reason, BlacklistInfo::REASONS)) - throw new InvalidArgumentException('$reason is not a valid reason.'); - - $stmt = $this->cache->get('INSERT INTO prm_blacklist (bl_hash, bl_reason) VALUES (?, ?)'); - $stmt->nextParameter($hash); - $stmt->nextParameter($reason); - $stmt->execute(); - } -} diff --git a/src/Blacklist/BlacklistInfo.php b/src/Blacklist/BlacklistInfo.php deleted file mode 100644 index f6089e2..0000000 --- a/src/Blacklist/BlacklistInfo.php +++ /dev/null @@ -1,39 +0,0 @@ -getString(0), - reason: $result->getString(1), - createdTime: $result->getInteger(2), - ); - } - - public bool $isCopyrightTakedown { - get => $this->reason === 'copyright'; - } - - public bool $isRulesViolation { - get => $this->reason === 'rules'; - } - - public CarbonImmutable $createdAt { - get => CarbonImmutable::createFromTimestampUTC($this->createdTime); - } -} diff --git a/src/Denylist/DenylistContext.php b/src/Denylist/DenylistContext.php new file mode 100644 index 0000000..1e41746 --- /dev/null +++ b/src/Denylist/DenylistContext.php @@ -0,0 +1,29 @@ +list = new DenylistData($dbConn); + } + + public function denyFile(StorageRecord|string $recordOrHash, DenylistReason|string $reason): void { + if($recordOrHash instanceof StorageRecord) + $recordOrHash = $recordOrHash->hash; + if(is_string($reason)) + $reason = DenylistReason::from($reason); + + $this->list->createDenylistEntry($recordOrHash, $reason); + } + + public function getDenylistEntry(StorageRecord|string $recordOrHash): ?DenylistInfo { + if($recordOrHash instanceof StorageRecord) + $recordOrHash = $recordOrHash->hash; + + return $this->list->getDenylistEntry($recordOrHash); + } +} diff --git a/src/Denylist/DenylistData.php b/src/Denylist/DenylistData.php new file mode 100644 index 0000000..5e7eba9 --- /dev/null +++ b/src/Denylist/DenylistData.php @@ -0,0 +1,32 @@ +cache = new DbStatementCache($dbConn); + } + + public function getDenylistEntry(string $hash): ?DenylistInfo { + $stmt = $this->cache->get('SELECT deny_id, deny_hash, deny_reason, UNIX_TIMESTAMP(deny_created) FROM prm_denylist WHERE deny_hash = ?'); + $stmt->nextParameter($hash); + $stmt->execute(); + + $result = $stmt->getResult(); + return $result->next() ? DenylistInfo::fromResult($result) : null; + } + + public function createDenylistEntry(string $hash, DenylistReason $reason): void { + if(strlen($hash) !== 32) + throw new InvalidArgumentException('$hash must be 32 bytes.'); + + $stmt = $this->cache->get('INSERT INTO prm_denylist (deny_hash, deny_reason) VALUES (?, ?)'); + $stmt->nextParameter($hash); + $stmt->nextParameter($reason->value); + $stmt->execute(); + } +} diff --git a/src/Denylist/DenylistInfo.php b/src/Denylist/DenylistInfo.php new file mode 100644 index 0000000..0cf3291 --- /dev/null +++ b/src/Denylist/DenylistInfo.php @@ -0,0 +1,35 @@ +getString(0), + hash: $result->getString(1), + reason: DenylistReason::tryFrom($result->getString(2)) ?? DenylistReason::Other, + createdTime: $result->getInteger(3), + ); + } + + public bool $isCopyrightTakedown { + get => $this->reason === DenylistReason::Copyright; + } + + public bool $isRulesViolation { + get => $this->reason === DenylistReason::Rules; + } + + public CarbonImmutable $createdAt { + get => CarbonImmutable::createFromTimestampUTC($this->createdTime); + } +} diff --git a/src/Denylist/DenylistReason.php b/src/Denylist/DenylistReason.php new file mode 100644 index 0000000..8ae639a --- /dev/null +++ b/src/Denylist/DenylistReason.php @@ -0,0 +1,8 @@ +authInfo = new AuthInfo; - $this->appsCtx = new Apps\AppsContext($dbConn); - $this->blacklistCtx = new Blacklist\BlacklistContext($dbConn); + $this->denylistCtx = new Denylist\DenylistContext($dbConn); + $this->poolsCtx = new Pools\PoolsContext($dbConn); $this->storageCtx = new Storage\StorageContext($config->scopeTo('storage'), $dbConn, $this->snowflake); - $this->uploadsCtx = new Uploads\UploadsContext($config, $dbConn, $this->snowflake); + $this->uploadsCtx = new Uploads\UploadsContext($config->scopeTo('domain'), $dbConn, $this->snowflake); $this->usersCtx = new Users\UsersContext($dbConn); } @@ -57,10 +57,10 @@ class EEPROMContext { $routingCtx->register(new Uploads\UploadsRoutes( $this->authInfo, - $this->appsCtx, + $this->poolsCtx, $this->uploadsCtx, $this->storageCtx, - $this->blacklistCtx, + $this->denylistCtx, $isApiDomain )); diff --git a/src/Pools/PoolInfo.php b/src/Pools/PoolInfo.php new file mode 100644 index 0000000..d5e5730 --- /dev/null +++ b/src/Pools/PoolInfo.php @@ -0,0 +1,52 @@ +getString(0), + name: $result->getString(1), + secret: $result->getStringOrNull(2), + createdTime: $result->getInteger(3), + deprecatedTime: $result->getIntegerOrNull(4), + ); + } + + public CarbonImmutable $createdAt { + get => CarbonImmutable::createFromTimestampUTC($this->createdTime); + } + + public bool $deprecated { + get => $this->deprecatedTime !== null; + } + + public ?CarbonImmutable $deprecatedAt { + get => $this->deprecatedTime === null ? null : CarbonImmutable::createFromTimestampUTC($this->deprecatedTime); + } + + public bool $hasSecret { + get => $this->secret !== null; + } + + public function verifyHash(string $data, string $hash, string $algo = 'sha256'): bool { + // not sure how you'd end up here but lets err on the side of caution for it :) + if($this->secret === null) + return false; + + return hash_equals( + hash_hmac($algo, $data, $this->secret, true), + $hash + ); + } +} diff --git a/src/Pools/PoolInfoGetField.php b/src/Pools/PoolInfoGetField.php new file mode 100644 index 0000000..44bb7ca --- /dev/null +++ b/src/Pools/PoolInfoGetField.php @@ -0,0 +1,8 @@ +getString(0), + poolId: $result->getString(1), + type: $result->getString(2), + params: json_decode($result->getString(3), true), + ); + } +} diff --git a/src/Pools/PoolRules.php b/src/Pools/PoolRules.php new file mode 100644 index 0000000..f2d5aec --- /dev/null +++ b/src/Pools/PoolRules.php @@ -0,0 +1,32 @@ + XArray::select($this->rulesRaw, fn($rule) => $this->registry->create($rule)); + } + + public function getRuleRaw(string $type): ?PoolRuleInfo { + return XArray::first($this->rulesRaw, fn($rule) => strcasecmp($rule->type, $type) === 0); + } + + public function getRule(string $type): ?PoolRule { + return $this->registry->create($this->getRuleRaw($type)); + } + + public function getRulesRaw(string $type): array { + return XArray::where($this->rulesRaw, fn($rule) => strcasecmp($rule->type, $type) === 0); + } + + public function getRules(string $type): array { + return XArray::select($this->getRulesRaw($type), fn($rule) => $this->registry->create($rule)); + } +} diff --git a/src/Pools/PoolRulesRegistry.php b/src/Pools/PoolRulesRegistry.php new file mode 100644 index 0000000..7086de5 --- /dev/null +++ b/src/Pools/PoolRulesRegistry.php @@ -0,0 +1,25 @@ +constructors)) + throw new RuntimeException('$type already registered'); + + $this->constructors[$type] = $constructor; + } + + public function create(?PoolRuleInfo $ruleInfo): ?PoolRule { + if($ruleInfo === null) + return null; + + if(!array_key_exists($ruleInfo->type, $this->constructors)) + throw new RuntimeException('no constructor for a rule of the given type registered'); + + return $this->constructors[$ruleInfo->type]($ruleInfo->params); + } +} diff --git a/src/Pools/PoolsContext.php b/src/Pools/PoolsContext.php new file mode 100644 index 0000000..e89769a --- /dev/null +++ b/src/Pools/PoolsContext.php @@ -0,0 +1,38 @@ +pools = new PoolsData($dbConn); + + $this->rules = new PoolRulesRegistry; + $this->rules->register(Rules\AcceptTypesRule::TYPE, Rules\AcceptTypesRule::create(...)); + $this->rules->register(Rules\ConstrainSizeRule::TYPE, Rules\ConstrainSizeRule::create(...)); + $this->rules->register(Rules\EnsureVariantRule::TYPE, Rules\EnsureVariantRule::create(...)); + $this->rules->register(Rules\OnePerUserRule::TYPE, Rules\OnePerUserRule::create(...)); + $this->rules->register(Rules\RemoveStaleRule::TYPE, Rules\RemoveStaleRule::create(...)); + $this->rules->register(Rules\ScanForumRule::TYPE, Rules\ScanForumRule::create(...)); + } + + public function getPoolRules(PoolInfo|string $poolInfo): PoolRules { + if(!($poolInfo instanceof PoolInfo)) + $poolInfo = $this->pools->getPool($poolInfo); + + return new PoolRules( + $poolInfo, + iterator_to_array($this->pools->getPoolRules(poolInfo: $poolInfo)), + $this->rules + ); + } + + public function getPoolFromVariant(UploadVariantInfo $variantInfo): StorageRecord { + return $this->pools->getPool($variantInfo->fileId, PoolInfoGetField::Id); + } +} diff --git a/src/Pools/PoolsData.php b/src/Pools/PoolsData.php new file mode 100644 index 0000000..6643d4e --- /dev/null +++ b/src/Pools/PoolsData.php @@ -0,0 +1,73 @@ +cache = new DbStatementCache($dbConn); + } + + public function getPool( + string $value, + PoolInfoGetField $field = PoolInfoGetField::Id + ): PoolInfo { + $field = match($field) { + PoolInfoGetField::Id => 'pool_id = ?', + PoolInfoGetField::Name => 'pool_name = ?', + PoolInfoGetField::UploadId => 'pool_id = (SELECT pool_id FROM prm_uploads WHERE upload_id = ?)', + default => throw new InvalidArgumentException('$field is not an acceptable value'), + }; + + $stmt = $this->cache->get(<<nextParameter($value); + $stmt->execute(); + + $result = $stmt->getResult(); + if(!$result->next()) + throw new RuntimeException('could not find that file record'); + + return PoolInfo::fromResult($result); + } + + public function getPoolRules( + PoolInfo|string|null $poolInfo = null, + string|array|null $types = null + ): iterable { + $hasPoolInfo = $poolInfo !== null; + $hasTypes = $types !== null; + if(is_string($types)) + $types = [$types]; + + $args = 0; + $query = << 1 ? 'AND' : 'WHERE'); + if($hasTypes) + $query .= sprintf(' %s rule_type IN (%s)', ++$args > 1 ? 'AND' : 'WHERE', DbTools::prepareListString($types)); + + $stmt = $this->cache->get($query); + if($hasPoolInfo) + $stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo); + if($hasTypes) + foreach($types as $type) { + if(!is_string($type)) + throw new InvalidArgumentException('all entries of $types but be a string'); + $stmt->nextParameter($type); + } + $stmt->execute(); + + return $stmt->getResult()->getIterator(PoolRuleInfo::fromResult(...)); + } +} diff --git a/src/Pools/Rules/AcceptTypesRule.php b/src/Pools/Rules/AcceptTypesRule.php new file mode 100644 index 0000000..b70ee32 --- /dev/null +++ b/src/Pools/Rules/AcceptTypesRule.php @@ -0,0 +1,12 @@ + EnsureVariantRuleCrop::create(...), + 'thumb' => EnsureVariantRuleThumb::create(...), + ]; + } + + public function __construct( + public private(set) string $variant, + public private(set) array $when, + public private(set) EnsureVariantRuleInfo $info + ) {} + + public bool $onAccess { + get => in_array('access', $this->when); + } + + public bool $onCron { + get => in_array('cron', $this->when); + } + + public static function create(array $params): EnsureVariantRule { + if(!isset($params['info']) || !is_array($params['info'])) + throw new RuntimeException('missing info key in $params'); + + $info = $params['info']; + if(!isset($info['$kind']) || !is_string($info['$kind']) || !array_key_exists($info['$kind'], self::$infoTypes)) + throw new RuntimeException('unsupported info kind in $params'); + + return new EnsureVariantRule( + isset($params['name']) && is_string($params['name']) ? $params['name'] : '', + isset($params['when']) && is_array($params['when']) ? $params['when'] : [], + self::$infoTypes[$info['$kind']]($info) + ); + } +} + +EnsureVariantRule::init(); diff --git a/src/Pools/Rules/EnsureVariantRuleCrop.php b/src/Pools/Rules/EnsureVariantRuleCrop.php new file mode 100644 index 0000000..6d3b005 --- /dev/null +++ b/src/Pools/Rules/EnsureVariantRuleCrop.php @@ -0,0 +1,25 @@ + 100) + throw new InvalidArgumentException('$quality must be a positive integer, not greater than 100'); + } + + public static function create(array $params): EnsureVariantRuleThumb { + return new EnsureVariantRuleThumb( + isset($params['w']) && is_int($params['w']) ? $params['w'] : 0, + isset($params['h']) && is_int($params['h']) ? $params['h'] : 0, + isset($params['q']) && is_int($params['q']) ? $params['q'] : 80, + isset($params['f']) && is_string($params['f']) ? $params['f'] : 'jpg', + isset($params['b']) && is_string($params['b']) ? $params['b'] : 'black' + ); + } +} diff --git a/src/Pools/Rules/OnePerUserRule.php b/src/Pools/Rules/OnePerUserRule.php new file mode 100644 index 0000000..23cf71c --- /dev/null +++ b/src/Pools/Rules/OnePerUserRule.php @@ -0,0 +1,12 @@ +id; + + return in_array($uploadId, $this->ignoreUploadIds); + } + + public static function create(array $params): RemoveStaleRule { + return new RemoveStaleRule( + isset($params['inactive_for']) && is_int($params['inactive_for']) ? max(0, $params['inactive_for']) : 0, + isset($params['ignore']) && is_array($params['ignore']) ? $params['ignore'] : [] + ); + } +} diff --git a/src/Pools/Rules/ScanForumRule.php b/src/Pools/Rules/ScanForumRule.php new file mode 100644 index 0000000..1e6ac17 --- /dev/null +++ b/src/Pools/Rules/ScanForumRule.php @@ -0,0 +1,12 @@ +records = new StorageRecords($dbConn, $snowflake); } - public function getFileRecordFromVariant(UploadVariantInfo $variantInfo): StorageRecord { - return $this->records->getFile($variantInfo->fileId, StorageRecordGetFileField::Id); - } - public function deleteFile( StorageRecord|string $recordInfo, bool $ensureOrphan = true ): void { - // Does taking ID here make sense or should it be hash? if(!($recordInfo instanceof StorageRecord)) - $recordInfo = $this->records->getFile($recordInfo, StorageRecordGetFileField::Id); + $recordInfo = $this->records->getFile($recordInfo); if($ensureOrphan) $this->records->unlinkFileUploads($recordInfo); @@ -59,90 +56,168 @@ class StorageContext { try { return $this->records->createFile($hash, $type, $size); } catch(RuntimeException $ex) { - return $this->records->getFile($hash); + return $this->records->getFile($hash, StorageRecordGetFileField::Hash); } } - public function supportsThumbnailing(StorageRecord $fileInfo): bool { + public static function supportsThumbnailing(StorageRecord $fileInfo): bool { return $fileInfo->isImage || $fileInfo->isAudio || $fileInfo->isVideo; } - public function createThumbnail(StorageRecord $fileInfo): StorageRecord { - $filePath = $this->files->getLocalPath($fileInfo); - if(!is_file($filePath)) - throw new RuntimeException('local data for provided $fileInfo does not exist'); + public static function supportsSquareCropping(StorageRecord $fileInfo): bool { + return $fileInfo->isImage; + } - $tmpPath = tempnam(sys_get_temp_dir(), 'eeprom-thumbnail-'); - $imagick = new Imagick; + public function readIntoImagick(Imagick $imagick, StorageRecord $fileInfo): void { try { + $path = $this->files->getLocalPath($fileInfo); + if($fileInfo->isImage) { try { - $file = fopen($filePath, 'rb'); + $file = fopen($path, 'rb'); $imagick->readImageFile($file); } finally { if(isset($file) && is_resource($file)) fclose($file); } - } elseif($fileInfo->isAudio) { + return; + } + + if($fileInfo->isAudio) { try { - $file = popen(sprintf('ffmpeg -i %s -an -f image2pipe -c:v copy -frames:v 1 2>/dev/null -', escapeshellarg($filePath)), 'rb'); + $file = popen(sprintf('ffmpeg -i %s -an -f image2pipe -c:v copy -frames:v 1 2>/dev/null -', escapeshellarg($path)), 'rb'); $imagick->readImageBlob(stream_get_contents($file)); } finally { if(isset($file) && is_resource($file)) pclose($file); } - } elseif($fileInfo->isVideo) { + return; + } + + if($fileInfo->isVideo) { try { - $file = popen(sprintf('ffmpeg -i %s -f image2pipe -c:v png -frames:v 1 2>/dev/null -', escapeshellarg($filePath)), 'rb'); + $file = popen(sprintf('ffmpeg -i %s -f image2pipe -c:v png -frames:v 1 2>/dev/null -', escapeshellarg($path)), 'rb'); $imagick->readImageBlob(stream_get_contents($file)); } finally { if(isset($file) && is_resource($file)) pclose($file); } } + } catch(ImagickException $ex) { + if($ex->getCode() !== 1) + throw $ex; - $imagick->setImageFormat('jpg'); - $imagick->setImageCompressionQuality(80); + $imagick->newImage(1, 1, 'black', 'bmp'); + } + } - $width = $imagick->getImageWidth(); - $height = $imagick->getImageHeight(); - $thumbRes = min(300, $width, $height); + public function createThumbnailFromRule( + StorageRecord $original, + EnsureVariantRuleThumb $rules + ): StorageRecord { + return $this->createThumbnail( + $original, + $rules->width, + $rules->height, + $rules->quality, + $rules->format, + $rules->background + ); + } - if($width === $height) { - $resizeWidth = $resizeHeight = $thumbRes; - } elseif($width > $height) { - $resizeWidth = $width * $thumbRes / $height; - $resizeHeight = $thumbRes; - } else { - $resizeWidth = $thumbRes; - $resizeHeight = $height * $thumbRes / $width; - } + public function createThumbnail( + StorageRecord $original, + int $width = 300, + int $height = 300, + int $quality = 80, + string $format = 'jpg', + string $background = 'black' + ): StorageRecord { + if(!self::supportsThumbnailing($original)) + throw new InvalidArgumentException('$original does not support thumbnailing'); + if($width < 1) + throw new InvalidArgumentException('$width must be greater than 0'); + if($height < 1) + throw new InvalidArgumentException('$height must be greater than 0'); + if($quality < 0 || $quality > 100) + throw new InvalidArgumentException('$quality must be a positive integer, not greater than 100'); + $format = strtoupper($format); + if(!in_array($format, Imagick::queryFormats())) + throw new InvalidArgumentException('$format is not a supported format'); - $resizeWidth = (int)$resizeWidth; - $resizeHeight = (int)$resizeHeight; - - $imagick->resizeImage( - $resizeWidth, $resizeHeight, - Imagick::FILTER_GAUSSIAN, 0.7 - ); - - $imagick->cropImage( - $thumbRes, - $thumbRes, - (int)ceil(($resizeWidth - $thumbRes) / 2), - (int)ceil(($resizeHeight - $thumbRes) / 2) - ); - - $imagick->writeImage($tmpPath); + $type = 'application/octet-stream'; + $path = tempnam(sys_get_temp_dir(), 'eeprom-thumbnail-'); + $imagick = new Imagick; + try { + $this->readIntoImagick($imagick, $original); + $imagick->setImageFormat($format); + $imagick->setImageCompressionQuality($quality); + $imagick->setImageBackgroundColor($background); + $imagick->thumbnailImage($width, $height, true, true); + $type = $imagick->getImageMimeType(); + $imagick->writeImage(sprintf('%s:%s', $format, $path)); } finally { $imagick->clear(); } - if(!is_file($tmpPath)) + if(!is_file($path)) throw new RuntimeException('failed to create thumbnail file'); - return $this->importFile($tmpPath, StorageImportMode::Move, 'image/jpeg'); + return $this->importFile($path, StorageImportMode::Move, $type); + } + + public function createSquareCropped(StorageRecord $original, int $dimensions): StorageRecord { + if($dimensions < 1) + throw new InvalidArgumentException('$dimensions must be greater than 0'); + + $sourcePath = $this->files->getLocalPath($original); + if(!is_file($sourcePath)) + throw new RuntimeException('local data for provided $original does not exist'); + + $targetPath = tempnam(sys_get_temp_dir(), sprintf('eeprom-crop-%d-', $dimensions)); + $imagick = new Imagick; + try { + if($original->isImage) + try { + $file = fopen($sourcePath, 'rb'); + $imagick->readImageFile($file); + } finally { + if(isset($file) && is_resource($file)) + fclose($file); + } + + $width = $imagick->getImageWidth(); + $height = $imagick->getImageHeight(); + + if($width > $height) { + $width = (int)($width * $dimensions / $height); + $height = $dimensions; + } else { + $width = $dimensions; + $height = (int)($height * $dimensions / $width); + } + + do { + $imagick->resizeImage($width, $height, Imagick::FILTER_LANCZOS, 0.9); + $imagick->cropImage( + $dimensions, + $dimensions, + (int)ceil(($width - $dimensions) / 2), + (int)ceil(($height - $dimensions) / 2) + ); + $imagick->setImagePage($dimensions, $dimensions, 0, 0); + } while($imagick->nextImage()); + + $imagick->writeImage($targetPath); + } finally { + $imagick->clear(); + } + + if(!is_file($targetPath)) + throw new RuntimeException('failed to create cropped file'); + + return $this->importFile($targetPath, StorageImportMode::Move, $original->type); } } diff --git a/src/Storage/StorageRecords.php b/src/Storage/StorageRecords.php index 6e9337b..44fe23b 100644 --- a/src/Storage/StorageRecords.php +++ b/src/Storage/StorageRecords.php @@ -33,7 +33,7 @@ class StorageRecords { SQL; if($hasDenied) $query .= << 1 ? 'OR' : 'WHERE', $denied ? 'IS NOT' : 'IS' ); @@ -58,7 +58,7 @@ class StorageRecords { public function getFile( string $value, - StorageRecordGetFileField $field = StorageRecordGetFileField::Hash // wdym verbose? + StorageRecordGetFileField $field = StorageRecordGetFileField::Id ): StorageRecord { $field = match($field) { StorageRecordGetFileField::Id => 'file_id', @@ -106,7 +106,7 @@ class StorageRecords { $stmt->nextParameter($size); $stmt->execute(); - return $this->getFile($fileId, StorageRecordGetFileField::Id); + return $this->getFile($fileId); } public function deleteFile( diff --git a/src/Uploads/UploadInfo.php b/src/Uploads/UploadInfo.php index e8274cd..6ec1a29 100644 --- a/src/Uploads/UploadInfo.php +++ b/src/Uploads/UploadInfo.php @@ -9,28 +9,24 @@ class UploadInfo { public function __construct( public private(set) string $id, public private(set) ?string $userId, - public private(set) ?string $appId, + public private(set) string $poolId, public private(set) string $secret, public private(set) string $name, - public private(set) int $bumpAmount, public private(set) string $remoteAddress, public private(set) int $createdTime, - public private(set) ?int $accessedTime, - public private(set) ?int $expiresTime, + public private(set) int $accessedTime, ) {} public static function fromResult(DbResult $result): UploadInfo { return new UploadInfo( id: $result->getString(0), userId: $result->getStringOrNull(1), - appId: $result->getStringOrNull(2), + poolId: $result->getString(2), secret: $result->getString(3), name: $result->getString(4), - bumpAmount: $result->getInteger(5), - remoteAddress: $result->getString(6), - createdTime: $result->getInteger(7), - accessedTime: $result->getIntegerOrNull(8), - expiresTime: $result->getIntegerOrNull(9), + remoteAddress: $result->getString(5), + createdTime: $result->getInteger(6), + accessedTime: $result->getInteger(7), ); } @@ -42,19 +38,7 @@ class UploadInfo { get => CarbonImmutable::createFromTimestampUTC($this->createdTime); } - public ?CarbonImmutable $accessedAt { - get => $this->accessedTime === null ? null : CarbonImmutable::createFromTimestampUTC($this->accessedTime); - } - - public bool $expired { - get => $this->expiresTime !== null && $this->expiresTime <= time(); - } - - public ?CarbonImmutable $expiredAt { - get => $this->expiresTime === null ? null : CarbonImmutable::createFromTimestampUTC($this->expiresTime); - } - - public ?int $bumpAmountForUpdate { - get => $this->expiresTime !== null && $this->bumpAmount > 0 ? $this->bumpAmount : null; + public CarbonImmutable $accessedAt { + get => CarbonImmutable::createFromTimestampUTC($this->accessedTime); } } diff --git a/src/Uploads/UploadsContext.php b/src/Uploads/UploadsContext.php index fd5915a..01f2db8 100644 --- a/src/Uploads/UploadsContext.php +++ b/src/Uploads/UploadsContext.php @@ -8,21 +8,21 @@ use EEPROM\{FFMPEG,SnowflakeGenerator}; use EEPROM\Storage\StorageRecord; class UploadsContext { - public private(set) UploadsData $uploadsData; + public private(set) UploadsData $uploads; public function __construct( private Config $config, DbConnection $dbConn, SnowflakeGenerator $snowflake ) { - $this->uploadsData = new UploadsData($dbConn, $snowflake); + $this->uploads = new UploadsData($dbConn, $snowflake); } public function getFileUrlV1(UploadInfo $uploadInfo, bool $forceApiDomain = false): string { - if(!$forceApiDomain && $this->config->hasValues('domain:short')) - return sprintf('//%s/%s%s', $this->config->getString('domain:short'), $uploadInfo->id62, $uploadInfo->secret); + if(!$forceApiDomain && $this->config->hasValues('short')) + return sprintf('//%s/%s%s', $this->config->getString('short'), $uploadInfo->id62, $uploadInfo->secret); - return sprintf('//%s/uploads/%s%s', $this->config->getString('domain:api'), $uploadInfo->id62, $uploadInfo->secret); + return sprintf('//%s/uploads/%s%s', $this->config->getString('api'), $uploadInfo->id62, $uploadInfo->secret); } public function getThumbnailUrlV1(UploadInfo $uploadInfo, bool $forceApiDomain = false): string { @@ -44,11 +44,11 @@ class UploadsContext { 'type' => $variantInfo->type ?? $fileInfo->type, 'size' => $fileInfo->size, 'user' => $uploadInfo->userId === null ? null : (int)$uploadInfo->userId, - 'appl' => $uploadInfo->appId === null ? null : (int)$uploadInfo->appId, + 'appl' => (int)$uploadInfo->poolId, 'hash' => $fileInfo->hashString, 'created' => $uploadInfo->createdAt->toIso8601ZuluString(), - 'accessed' => $uploadInfo->accessedAt?->toIso8601ZuluString(), - 'expires' => $uploadInfo->expiredAt?->toIso8601ZuluString(), + 'accessed' => $uploadInfo->accessedAt->toIso8601ZuluString(), + 'expires' => $uploadInfo->accessedAt->add(2, 'weeks')->toIso8601ZuluString(), // These can never be reached, and in situation where they technically could it's because of an outdated local record 'deleted' => null, diff --git a/src/Uploads/UploadsData.php b/src/Uploads/UploadsData.php index f1eb008..ab0a065 100644 --- a/src/Uploads/UploadsData.php +++ b/src/Uploads/UploadsData.php @@ -4,9 +4,9 @@ namespace EEPROM\Uploads; use InvalidArgumentException; use RuntimeException; use Index\XString; -use Index\Db\{DbConnection,DbStatementCache}; +use Index\Db\{DbConnection,DbStatementCache,DbTools}; use EEPROM\SnowflakeGenerator; -use EEPROM\Apps\AppInfo; +use EEPROM\Pools\PoolInfo; use EEPROM\Storage\StorageRecord; use EEPROM\Users\UserInfo; @@ -21,20 +21,41 @@ class UploadsData { } public function getUploads( - ?bool $expired = null + PoolInfo|string|null $poolInfo = null, + ?string $variant = null, + ?bool $variantExists = null ): iterable { - $hasExpired = $expired !== null; + $hasPoolInfo = $poolInfo !== null; + $hasVariant = $variant !== null; + $hasVariantExists = $variantExists !== null; + + $query = << 1 ? 'AND' : 'WHERE'); - else - $query .= sprintf(' %s (upload_expires IS NULL OR upload_expires > NOW())', ++$args > 1 ? 'AND' : 'WHERE'); - } + if($hasPoolInfo) + $query .= sprintf(' %s u.pool_id = ?', ++$args > 1 ? 'AND' : 'WHERE'); + if($hasVariantExists) + $query .= sprintf( + ' %s uf.upload_id %s NULL', + ++$args > 1 ? 'AND' : 'WHERE', + $variantExists ? 'IS NOT' : 'IS' + ); $stmt = $this->cache->get($query); + if($hasVariant) + $stmt->nextParameter($variant); + if($hasPoolInfo) + $stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo); $stmt->execute(); return $stmt->getResult()->getIterator(UploadInfo::fromResult(...)); @@ -42,41 +63,33 @@ class UploadsData { public function getUpload( ?string $uploadId = null, - AppInfo|string|null $appInfo = null, + PoolInfo|string|null $poolInfo = null, UserInfo|string|null $userInfo = null, ?string $secret = null, - ?bool $expired = null, ): UploadInfo { $hasUploadId = $uploadId !== null; - $hasAppInfo = $appInfo !== null; + $hasPoolInfo = $poolInfo !== null; $hasUserInfo = $userInfo !== null; $hasSecret = $secret !== null; - $hasExpired = $expired !== null; $args = 0; - $query = 'SELECT upload_id, user_id, app_id, upload_secret, upload_name, upload_bump, INET6_NTOA(upload_ip), UNIX_TIMESTAMP(upload_created), UNIX_TIMESTAMP(upload_accessed), UNIX_TIMESTAMP(upload_expires) FROM prm_uploads'; + $query = 'SELECT upload_id, user_id, pool_id, upload_secret, upload_name, INET6_NTOA(upload_ip), UNIX_TIMESTAMP(upload_created), UNIX_TIMESTAMP(upload_accessed) FROM prm_uploads'; if($hasUploadId) { ++$args; $query .= ' WHERE upload_id = ?'; } - if($hasAppInfo) - $query .= sprintf(' %s app_id = ?', ++$args > 1 ? 'AND' : 'WHERE'); + if($hasPoolInfo) + $query .= sprintf(' %s pool_id = ?', ++$args > 1 ? 'AND' : 'WHERE'); if($hasUserInfo) $query .= sprintf(' %s user_id = ?', ++$args > 1 ? 'AND' : 'WHERE'); if($hasSecret) $query .= sprintf(' %s upload_secret = ?', ++$args > 1 ? 'AND' : 'WHERE'); - if($hasExpired) { - if($expired) - $query .= sprintf(' %s upload_expires IS NOT NULL AND upload_expires <= NOW()', ++$args > 1 ? 'AND' : 'WHERE'); - else - $query .= sprintf(' %s (upload_expires IS NULL OR upload_expires > NOW())', ++$args > 1 ? 'AND' : 'WHERE'); - } $stmt = $this->cache->get($query); if($hasUploadId) $stmt->nextParameter($uploadId); - if($hasAppInfo) - $stmt->nextParameter($appInfo instanceof AppInfo ? $appInfo->id : $appInfo); + if($hasPoolInfo) + $stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo); if($hasUserInfo) $stmt->nextParameter($userInfo instanceof UserInfo ? $userInfo->id : $userInfo); if($hasSecret) @@ -91,7 +104,7 @@ class UploadsData { } public function resolveUploadFromVariant( - AppInfo|string $appInfo, + PoolInfo|string $poolInfo, UserInfo|string $userInfo, StorageRecord|string $fileInfo, string $variant @@ -101,16 +114,12 @@ class UploadsData { FROM prm_uploads AS u LEFT JOIN prm_uploads_files AS uf ON uf.upload_id = u.upload_id - WHERE u.app_id = ? + WHERE u.pool_id = ? AND u.user_id = ? AND uf.file_id = ? AND uf.upload_variant = ? - AND ( - u.upload_expires IS NULL - OR u.upload_expires > NOW() - ) SQL); - $stmt->nextParameter($appInfo instanceof AppInfo ? $appInfo->id : $appInfo); + $stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo); $stmt->nextParameter($userInfo instanceof UserInfo ? $userInfo->id : $userInfo); $stmt->nextParameter($fileInfo instanceof StorageRecord ? $fileInfo->id : $fileInfo); $stmt->nextParameter($variant); @@ -124,27 +133,20 @@ class UploadsData { } public function createUpload( - AppInfo|string $appInfo, + PoolInfo|string $poolInfo, UserInfo|string $userInfo, string $fileName, - string $remoteAddr, - int $fileExpiry, - bool $bumpExpiry + string $remoteAddr ): UploadInfo { - if($fileExpiry < 0) - throw new InvalidArgumentException('$fileExpiry must be a positive integer.'); - $uploadId = $this->snowflake->nextRandom(); - $stmt = $this->cache->get('INSERT INTO prm_uploads (upload_id, app_id, user_id, upload_secret, upload_name, upload_ip, upload_expires, upload_bump) VALUES (?, ?, ?, ?, ?, INET6_ATON(?), FROM_UNIXTIME(?), ?)'); + $stmt = $this->cache->get('INSERT INTO prm_uploads (upload_id, pool_id, user_id, upload_secret, upload_name, upload_ip) VALUES (?, ?, ?, ?, ?, INET6_ATON(?))'); $stmt->nextParameter($uploadId); - $stmt->nextParameter($appInfo instanceof AppInfo ? $appInfo->id : $appInfo); + $stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo); $stmt->nextParameter($userInfo instanceof UserInfo ? $userInfo->id : $userInfo); $stmt->nextParameter(XString::random(4)); $stmt->nextParameter($fileName); $stmt->nextParameter($remoteAddr); - $stmt->nextParameter($fileExpiry > 0 ? (time() + $fileExpiry) : 0); - $stmt->nextParameter($bumpExpiry ? $fileExpiry : 0); $stmt->execute(); return $this->getUpload(uploadId: $uploadId); @@ -153,8 +155,7 @@ class UploadsData { public function updateUpload( UploadInfo|string $uploadInfo, ?string $fileName = null, - int|null|bool $accessedAt = false, - int|null|false $expiresAt = false + int|bool $accessedAt = false, ): void { $fields = []; $values = []; @@ -170,11 +171,6 @@ class UploadsData { $values[] = $accessedAt; } - if($expiresAt !== false) { - $fields[] = 'upload_expires = NOW() + INTERVAL ? SECOND'; - $values[] = $expiresAt; - } - if(empty($fields)) return; @@ -191,12 +187,51 @@ class UploadsData { $stmt->execute(); } - public function deleteExpiredUploads(): void { - $this->dbConn->execute(<<cache->get($query); + $stmt->nextParameter($poolInfo instanceof PoolInfo ? $poolInfo->id : $poolInfo); + $stmt->nextParameter($staleSeconds); + if($hasIgnore) + foreach($ignore as $item) { + if($item instanceof UploadInfo) + $stmt->nextParameter($item->id); + elseif(is_string($item)) + $stmt->nextParameter($item); + else + throw new InvalidArgumentException('$ignore contains entries with unsupported types'); + } + $stmt->execute(); + } + + public function getUploadVariants( + UploadInfo|string $uploadInfo + ): iterable { + $stmt = $this->cache->get(<<nextParameter($uploadInfo instanceof UploadInfo ? $uploadInfo->id : $uploadInfo); + $stmt->execute(); + + return $stmt->getResult()->getIterator(UploadVariantInfo::fromResult(...)); } public function getUploadVariant( @@ -239,6 +274,29 @@ class UploadsData { return $this->getUploadVariant($uploadInfo, $variant); } + public function deleteUploadVariant( + UploadVariantInfo|string $uploadIdOrVariantInfo, + ?string $variant = null + ): void { + if($uploadIdOrVariantInfo instanceof UploadVariantInfo) { + if($variant !== null) + throw new InvalidArgumentException('$variant must be null if $uploadIdOrVariantInfo is an instance of UploadVariantInfo'); + + $uploadId = $uploadIdOrVariantInfo->uploadId; + $variant = $uploadIdOrVariantInfo->variant; + } else { + if($variant === null) + throw new InvalidArgumentException('$variant may not be null if $uploadIdOrVariantInfo is a string'); + + $uploadId = $uploadIdOrVariantInfo; + } + + $stmt = $this->cache->get('DELETE FROM prm_uploads_files WHERE upload_id = ? AND upload_variant = ?'); + $stmt->nextParameter($uploadId); + $stmt->nextParameter($variant); + $stmt->execute(); + } + public function resolveLegacyId(string $legacyId): ?string { $stmt = $this->cache->get('SELECT upload_id FROM prm_uploads_legacy WHERE upload_id_legacy = ?'); $stmt->nextParameter($legacyId); diff --git a/src/Uploads/UploadsRoutes.php b/src/Uploads/UploadsRoutes.php index 4c7f636..ddb9a4e 100644 --- a/src/Uploads/UploadsRoutes.php +++ b/src/Uploads/UploadsRoutes.php @@ -2,21 +2,22 @@ namespace EEPROM\Uploads; use RuntimeException; -use Index\XNumber; +use Index\{XArray,XNumber}; use Index\Http\Routing\{HandlerAttribute,HttpDelete,HttpGet,HttpOptions,HttpPost,Router,RouteHandler}; -use EEPROM\Apps\AppsContext; use EEPROM\Auth\AuthInfo; -use EEPROM\Blacklist\BlacklistContext; -use EEPROM\Storage\{StorageContext,StorageImportMode}; +use EEPROM\Denylist\{DenylistContext,DenylistReason}; +use EEPROM\Pools\PoolsContext; +use EEPROM\Pools\Rules\{EnsureVariantRule,EnsureVariantRuleThumb}; +use EEPROM\Storage\{StorageContext,StorageImportMode,StorageRecordGetFileField}; use EEPROM\Uploads\UploadsContext; class UploadsRoutes implements RouteHandler { public function __construct( private AuthInfo $authInfo, - private AppsContext $appsCtx, + private PoolsContext $poolsCtx, private UploadsContext $uploadsCtx, private StorageContext $storageCtx, - private BlacklistContext $blacklistCtx, + private DenylistContext $denylistCtx, private bool $isApiDomain ) {} @@ -31,7 +32,7 @@ class UploadsRoutes implements RouteHandler { #[HttpOptions('/uploads/([A-Za-z0-9]+|[A-Za-z0-9\-_]{32})(?:\.(t|json))?')] #[HttpGet('/uploads/([A-Za-z0-9]+|[A-Za-z0-9\-_]{32})(?:\.(t|json))?')] - public function getUpload($response, $request, string $fileId, string $fileExt = '') { + public function getUpload($response, $request, string $uploadId, string $uploadVariant = '') { if($this->isApiDomain) { if($request->hasHeader('Origin')) $response->setHeader('Access-Control-Allow-Credentials', 'true'); @@ -45,87 +46,87 @@ class UploadsRoutes implements RouteHandler { if($request->getMethod() === 'OPTIONS') return 204; - $isData = $fileExt === ''; - $isThumbnail = $fileExt === 't'; - $isJson = $this->isApiDomain && $fileExt === 'json'; + $isOriginal = $uploadVariant === ''; + $isJson = $this->isApiDomain && $uploadVariant === 'json'; - if(!$isData && !$isThumbnail && !$isJson) - return 404; - - $uploadsData = $this->uploadsCtx->uploadsData; - - if(strlen($fileId) === 32) { - $fileId = $uploadsData->resolveLegacyId($fileId) ?? $fileId; - $fileSecret = null; + if(strlen($uploadId) === 32) { + $uploadId = $this->uploadsCtx->uploads->resolveLegacyId($uploadId) ?? $uploadId; + $uploadSecret = null; } else { - if(strlen($fileId) < 5) { + if(strlen($uploadId) < 5) { $response->setContent('File not found.'); return 404; } - $fileSecret = substr($fileId, -4); - $fileId = XNumber::fromBase62(substr($fileId, 0, -4)); + $uploadSecret = substr($uploadId, -4); + $uploadId = XNumber::fromBase62(substr($uploadId, 0, -4)); } try { - $uploadInfo = $uploadsData->getUpload(uploadId: $fileId, expired: false, secret: $fileSecret); + $uploadInfo = $this->uploadsCtx->uploads->getUpload(uploadId: $uploadId, secret: $uploadSecret); } catch(RuntimeException $ex) { $response->setContent('File not found.'); return 404; } - if($isThumbnail) { - try { - $originalInfo = $this->storageCtx->getFileRecordFromVariant( - $uploadsData->getUploadVariant($uploadInfo, '') - ); - } catch(RuntimeException $ex) { + try { + $variantInfo = $this->uploadsCtx->uploads->getUploadVariant($uploadInfo, $uploadVariant); + } catch(RuntimeException $ex) { + if($isOriginal) { $response->setContent('Data is missing.'); return 404; } - if(!$this->storageCtx->supportsThumbnailing($originalInfo)) - return 404; - try { - $variantInfo = $uploadsData->getUploadVariant($uploadInfo, 'thumb'); - } catch(RuntimeException $ex) { - $storageInfo = $this->storageCtx->createThumbnail($originalInfo); - $uploadsData->createUploadVariant($uploadInfo, 'thumb', $storageInfo); - } - } else { - try { - $variantInfo = $uploadsData->getUploadVariant($uploadInfo, ''); - } catch(RuntimeException $ex) { - $response->setContent('Data is missing.'); - return 404; - } - - if(!$isJson) - $uploadsData->updateUpload( - $uploadInfo, - accessedAt: true, - expiresAt: $uploadInfo->bumpAmountForUpdate, + $poolInfo = $this->poolsCtx->pools->getPool($uploadInfo->poolId); + $originalInfo = $this->storageCtx->records->getFile( + $this->uploadsCtx->uploads->getUploadVariant($uploadInfo, '')->fileId ); + + $rule = XArray::first( + $this->poolsCtx->getPoolRules($poolInfo)->getRules(EnsureVariantRule::TYPE), + fn($rule) => $rule->variant === $uploadVariant + ); + if($rule instanceof EnsureVariantRule && $rule->onAccess) { + if($rule->info instanceof EnsureVariantRuleThumb) { + $storageInfo = $this->storageCtx->createThumbnailFromRule($originalInfo, $rule->info); + $variantInfo = $this->uploadsCtx->uploads->createUploadVariant($uploadInfo, $rule->variant, $storageInfo); + } + } + } catch(RuntimeException $ex) { + $response->setContent('Original data is missing.'); + return 404; + } } + if(!isset($variantInfo)) { + $response->setContent('Variant not available.'); + return 404; + } + + if($isOriginal) + $this->uploadsCtx->uploads->updateUpload( + $uploadInfo, + accessedAt: true, + ); + if(!isset($storageInfo)) try { - $storageInfo = $this->storageCtx->getFileRecordFromVariant($variantInfo); + $storageInfo = $this->storageCtx->records->getFile($variantInfo->fileId); } catch(RuntimeException $ex) { $response->setContent('Data is missing.'); return 404; } - $blInfo = $this->blacklistCtx->getBlacklistEntry($storageInfo); - if($blInfo !== null) { - $response->setContent(match($blInfo->reason) { - 'copyright' => 'File is unavailable for copyright reasons.', - 'rules' => 'File was in violation of the rules.', + $denyInfo = $this->denylistCtx->getDenylistEntry($storageInfo); + if($denyInfo !== null) { + $response->setContent(match($denyInfo->reason) { + DenylistReason::Copyright => 'File is unavailable for copyright reasons.', + DenylistReason::Rules => 'File was in violation of the rules.', default => 'File was removed for reasons beyond understanding.', }); - return $blInfo->isCopyrightTakedown ? 451 : 410; + return $denyInfo->isCopyrightTakedown ? 451 : 410; } if($isJson) @@ -156,17 +157,6 @@ class UploadsRoutes implements RouteHandler { if($request->getMethod() === 'OPTIONS') return 204; - if(!$request->isFormContent()) - return 400; - - $content = $request->getContent(); - - try { - $appInfo = $this->appsCtx->getApp((string)$content->getParam('src', FILTER_VALIDATE_INT)); - } catch(RuntimeException $ex) { - return 404; - } - if(!$this->authInfo->loggedIn) return 401; @@ -174,6 +164,11 @@ class UploadsRoutes implements RouteHandler { if($userInfo->restricted) return 403; + if(!$request->isFormContent()) + return 400; + + $content = $request->getContent(); + try { $file = $content->getUploadedFile('file'); } catch(RuntimeException $ex) { @@ -185,8 +180,23 @@ class UploadsRoutes implements RouteHandler { if($fileSize < 1) return 400; - $maxFileSize = $appInfo->dataSizeLimit; - if($appInfo->allowSizeMultiplier) + try { + $poolInfo = $this->poolsCtx->pools->getPool((string)$content->getParam('src', FILTER_VALIDATE_INT)); + if($poolInfo->hasSecret) + return 404; + } catch(RuntimeException $ex) { + return 404; + } + + $rules = $this->poolsCtx->getPoolRules($poolInfo); + + // If there's no constrain_size rule on this pool its likely not meant to be used through this API! + $maxSizeRule = $rules->getRule('constrain_size'); + if($maxSizeRule === null) + return 500; + + $maxFileSize = $maxSizeRule->maxSize; + if($maxSizeRule->allowLegacyMultiplier) $maxFileSize *= $userInfo->dataSizeMultiplier; if($file->getSize() !== $fileSize || $fileSize > $maxFileSize) { @@ -195,11 +205,10 @@ class UploadsRoutes implements RouteHandler { return 413; } - $uploadsData = $this->uploadsCtx->uploadsData; $hash = hash_file('sha256', $localFile, true); - $blInfo = $this->blacklistCtx->getBlacklistEntry($hash); - if($blInfo !== null) + $denyInfo = $this->denylistCtx->getDenylistEntry($hash); + if($denyInfo !== null) return 451; $fileName = $file->getSuggestedFileName(); @@ -208,7 +217,7 @@ class UploadsRoutes implements RouteHandler { $mediaType = mime_content_type($localFile); try { - $storageInfo = $this->storageCtx->records->getFile($hash); + $storageInfo = $this->storageCtx->records->getFile($hash, StorageRecordGetFileField::Hash); } catch(RuntimeException $ex) { $storageInfo = $this->storageCtx->importFile( $file->getLocalFileName(), @@ -220,27 +229,24 @@ class UploadsRoutes implements RouteHandler { try { // TODO: Okay so we're reading from the prm_uploads_files table, then fetching the info from prm_uploads, // and then fetching the info again from the prm_uploads_files table... this can be done better but this will do for now - $uploadInfo = $uploadsData->getUpload( - uploadId: $uploadsData->resolveUploadFromVariant($appInfo, $userInfo, $storageInfo, ''), - expired: false + $uploadInfo = $this->uploadsCtx->uploads->getUpload( + uploadId: $this->uploadsCtx->uploads->resolveUploadFromVariant($poolInfo, $userInfo, $storageInfo, '') ); - $variantInfo = $uploadsData->getUploadVariant($uploadInfo, ''); + $variantInfo = $this->uploadsCtx->uploads->getUploadVariant($uploadInfo, ''); - $uploadsData->updateUpload( + $this->uploadsCtx->uploads->updateUpload( $uploadInfo, fileName: $fileName, - expiresAt: $uploadInfo->bumpAmountForUpdate, + accessedAt: true, ); } catch(RuntimeException $ex) { - $uploadInfo = $uploadsData->createUpload( - $appInfo, + $uploadInfo = $this->uploadsCtx->uploads->createUpload( + $poolInfo, $userInfo, $fileName, - $request->getRemoteAddress(), - $appInfo->bumpAmount, - true + $request->getRemoteAddress() ); - $variantInfo = $uploadsData->createUploadVariant( + $variantInfo = $this->uploadsCtx->uploads->createUploadVariant( $uploadInfo, '', $storageInfo, $mediaType ); } @@ -254,7 +260,7 @@ class UploadsRoutes implements RouteHandler { } #[HttpDelete('/uploads/([A-Za-z0-9]+|[A-Za-z0-9\-_]{32})')] - public function deleteUpload($response, $request, string $fileId) { + public function deleteUpload($response, $request, string $uploadId) { if($request->hasHeader('Origin')) $response->setHeader('Access-Control-Allow-Credentials', 'true'); @@ -264,23 +270,21 @@ class UploadsRoutes implements RouteHandler { if(!$this->authInfo->loggedIn) return 401; - $uploadsData = $this->uploadsCtx->uploadsData; - - if(strlen($fileId) === 32) { - $fileId = $uploadsData->resolveLegacyId($fileId) ?? $fileId; - $fileSecret = null; + if(strlen($uploadId) === 32) { + $uploadId = $this->uploadsCtx->uploads->resolveLegacyId($uploadId) ?? $uploadId; + $uploadSecret = null; } else { - if(strlen($fileId) < 5) { + if(strlen($uploadId) < 5) { $response->setContent('File not found.'); return 404; } - $fileSecret = substr($fileId, -4); - $fileId = XNumber::fromBase62(substr($fileId, 0, -4)); + $uploadSecret = substr($uploadId, -4); + $uploadId = XNumber::fromBase62(substr($uploadId, 0, -4)); } try { - $uploadInfo = $uploadsData->getUpload(uploadId: $fileId, expired: false, secret: $fileSecret); + $uploadInfo = $this->uploadsCtx->uploads->getUpload(uploadId: $uploadId, secret: $uploadSecret); } catch(RuntimeException $ex) { $response->setContent('File not found.'); return 404; @@ -290,7 +294,7 @@ class UploadsRoutes implements RouteHandler { if($userInfo->restricted || $userInfo->id !== $uploadInfo->userId) return 403; - $uploadsData->deleteUpload($uploadInfo); + $this->uploadsCtx->uploads->deleteUpload($uploadInfo); return 204; } } diff --git a/src/Users/UsersContext.php b/src/Users/UsersContext.php index a0ce786..f5e8bf3 100644 --- a/src/Users/UsersContext.php +++ b/src/Users/UsersContext.php @@ -4,14 +4,14 @@ namespace EEPROM\Users; use Index\Db\DbConnection; class UsersContext { - public private(set) UsersData $usersData; + public private(set) UsersData $users; public function __construct(DbConnection $dbConn) { - $this->usersData = new UsersData($dbConn); + $this->users = new UsersData($dbConn); } public function getUser(string $userId): UserInfo { - $this->usersData->ensureUserExists($userId); - return $this->usersData->getUser($userId); + $this->users->ensureUserExists($userId); + return $this->users->getUser($userId); } } diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 0000000..32ac185 --- /dev/null +++ b/tools/README.md @@ -0,0 +1,78 @@ +# EEPROM Tools + +Below a short description of every script in this directory, what it does and what arguments it takes, etc. + + +## `cron` + +Usage: `cron` + +This script performs scheduled maintenance, primarily cleaning up orphaned records and executing other rule stuff. + +It accepts no arguments BUT you should be running the script as the same user as the webserver/PHP FPM to ensure that you won't run into permission issues. + + +## `delete-upload` + +Usage: `delete-upload [OPTION]... [UPLOAD]` + +This script deletes a single upload and all of its variants from the database. +It also attempts to purge the associated files, but that can only be done if they're not used by other uploads. + +This tool accepts one option `-s`/`--base62` which causes `[UPLOAD]` to be interpreted as a base62 number to make administration easier. +Make sure to remove the trailing four secret characters when copying from a URL! + + +## `deny-file` + +Usage: `nuke-file [OPTION]... [FILE]` + +Prevents a file from being uploaded in the future and marks the existing copy for removal. + +This tool accepts three options: + - `-r`/`--reason`: Required!!! Reason for denying the file. Currently supports `copyright`, `rules` and `other`. + - `-h`/`--hash`: Treat the provided ID as a SHA256 hash, can be hex, URI-safe base64 or raw if you're feeling feisty. + - `-s`/`--base62`: Interpret provided ID as a base62 number. + - `-u`/`--upload`: Interpret provided ID as an upload ID instead of a file ID. + + +## `migrate` + +Usage: `migrate` + +Runs database migrations, part of the Index framework. + +Does not take any arguments but does write a lockfile to `/tmp` (or whatever the system temporary file directory is) to ensure this script is only run once. + + +## `migrate-override-toggle` + +Usage: `migrate-override-toggle` + +Toggles the migration lock file manually. + +Can be useful to recover from a crash without having to dig through `/tmp` or to just manually enable maintenance mode. + + +## `new-migration` + +Usage: `new-migration [NAME PART]...` + +Creates a new database migration, part of the Index framework. + +A new file will be written to the `database` folder within the project following a template embedded in the Index library source code. +All arguments of passed to the command will be concatenated to form the name of the migration. + + +## `nuke-file` + +Usage: `nuke-file [OPTION]... [FILE]` + +The nuclear approach. +Deletes a file from the storage and takes any associated uploads down with it. + +This tool accepts three options: + - `-h`/`--hash`: Treat the provided ID as a SHA256 hash, can be hex, URI-safe base64 or raw if you're feeling feisty. + - `-s`/`--base62`: Interpret provided ID as a base62 number. + - `-u`/`--upload`: Interpret provided ID as an upload ID instead of a file ID. + - `-v`/`--variant=VARIANT`: If using `-u`/`--upload`, specify the name of the variant. If left omitted the all variants will be removed. diff --git a/tools/cron b/tools/cron new file mode 100755 index 0000000..b61b618 --- /dev/null +++ b/tools/cron @@ -0,0 +1,140 @@ +#!/usr/bin/env php + array_key_exists($ruleRaw->poolId, $pools) ? $pools[$ruleRaw->poolId] : ( + $pools[$ruleRaw->poolId] = $eeprom->poolsCtx->pools->getPool($ruleRaw->poolId) + ); + + // Run cleanup adjacent tasks first + $rules = $eeprom->poolsCtx->pools->getPoolRules(types: ['remove_stale', 'scan_forum']); + foreach($rules as $ruleRaw) { + $poolInfo = $getPool($ruleRaw); + $rule = $eeprom->poolsCtx->rules->create($ruleRaw); + + if($rule instanceof RemoveStaleRule) { + printf('Removing stale entries for pool #%d...%s', $poolInfo->id, PHP_EOL); + $eeprom->uploadsCtx->uploads->deleteStaleUploads( + $poolInfo, + $rule->inactiveForSeconds, + $rule->ignoreUploadIds + ); + } elseif($rule instanceof ScanForumRule) { + //var_dump('scan_forum', $rule); + // necessary stuff for this doesn't exist yet so this can remain a no-op + } + } + echo PHP_EOL; + + printf('Purging denylisted file records...%s', PHP_EOL); + $records = $eeprom->storageCtx->records->getFiles(denied: true); + foreach($records as $record) { + printf('Deleting file #%d...%s', $record->id, PHP_EOL); + $eeprom->storageCtx->deleteFile($record); + } + echo PHP_EOL; + + printf('Purging uploads without an original variant...%s', PHP_EOL); + $records = $eeprom->uploadsCtx->uploads->getUploads(variant: '', variantExists: false); + foreach($records as $record) { + printf('Deleting upload #%d...%s', $record->id, PHP_EOL); + $eeprom->uploadsCtx->uploads->deleteUpload($record); + } + echo PHP_EOL; + + printf('Purging orphaned file records...%s', PHP_EOL); + $records = $eeprom->storageCtx->records->getFiles(orphaned: true); + foreach($records as $record) { + printf('Deleting file #%d...%s', $record->id, PHP_EOL); + $eeprom->storageCtx->deleteFile($record); + } + echo PHP_EOL; + + // Now run the ones that make new things + $rules = $eeprom->poolsCtx->pools->getPoolRules(types: ['ensure_variant']); + foreach($rules as $ruleRaw) { + $poolInfo = $getPool($ruleRaw); + $rule = $eeprom->poolsCtx->rules->create($ruleRaw); + + if($rule instanceof EnsureVariantRule) { + if(!$rule->onCron) + continue; + + $createVariant = null; // ensure this wasn't previously assigned + if($rule->info instanceof EnsureVariantRuleThumb) { + $createVariant = function( + StorageContext $storageCtx, + UploadsContext $uploadsCtx, + PoolInfo $poolInfo, + EnsureVariantRule $rule, + EnsureVariantRuleThumb $ruleInfo, + StorageRecord $originalInfo, + UploadInfo $uploadInfo + ) { + $uploadsCtx->uploads->createUploadVariant( + $uploadInfo, + $rule->variant, + $storageCtx->createThumbnailFromRule( + $originalInfo, + $ruleInfo + ) + ); + }; + } + + if(!isset($createVariant) || !is_callable($createVariant)) { + printf('!!! Could not create a constructor for "%s" variants for pool #%d !!! Skipping for now...%s', $rule->variant, $poolInfo->id, PHP_EOL); + continue; + } + + printf('Ensuring existence of "%s" variants for pool #%d...%s', $rule->variant, $poolInfo->id, PHP_EOL); + + $uploads = $eeprom->uploadsCtx->uploads->getUploads( + poolInfo: $poolInfo, + variant: $rule->variant, + variantExists: false + ); + + $processed = 0; + foreach($uploads as $uploadInfo) + try { + $createVariant( + $eeprom->storageCtx, + $eeprom->uploadsCtx, + $poolInfo, + $rule, + $rule->info, + $eeprom->storageCtx->records->getFile( + $eeprom->uploadsCtx->uploads->getUploadVariant($uploadInfo, '')->fileId + ), + $uploadInfo + ); + } catch(Exception $ex) { + printf('Exception thrown while processing "%s" variant for upload #%d in pool #%d:%s', $rule->variant, $uploadInfo->id, $poolInfo->id, PHP_EOL); + printf('%s%s', $ex->getMessage(), PHP_EOL); + } finally { + ++$processed; + } + + printf('Processed %d records!%s%s', $processed, PHP_EOL, PHP_EOL); + } + } + echo PHP_EOL; +} finally { + unlink($cronPath); +} diff --git a/tools/delete-upload b/tools/delete-upload new file mode 100755 index 0000000..a9dcc3d --- /dev/null +++ b/tools/delete-upload @@ -0,0 +1,77 @@ +#!/usr/bin/env php +uploadsCtx->uploads->getUpload(uploadId: $uploadId); +} catch(RuntimeException $ex) { + $hint = $isBase62 ? 'trim last four characters from base62 id' : 'specify -s for base62 ids'; + die(sprintf('The requested upload does not exist. (hint: %s)%s', $hint, PHP_EOL)); +} + +printf( + 'Found upload #%d (%s) in pool #%d uploaded by user #%d (%s) at %s with canonical name "%s".%s', + $uploadInfo->id, + $uploadInfo->id62, + $uploadInfo->poolId, + $uploadInfo->userId, + $uploadInfo->remoteAddress, + $uploadInfo->createdAt->toIso8601ZuluString(), + htmlspecialchars($uploadInfo->name), + PHP_EOL +); + +$variants = []; +foreach($eeprom->uploadsCtx->uploads->getUploadVariants($uploadInfo) as $variantInfo) { + $variants[] = $variantInfo; + printf( + 'Found variant "%s" linking to file #%d with media type %s.%s', + $variantInfo->variant, + $variantInfo->fileId, + htmlspecialchars($variantInfo->type ?? '(none)'), + PHP_EOL + ); +} + +$displayStorageDeleteHint = false; + +if(count($variants) < 1) { + printf('This upload has no variants to delete.%s', PHP_EOL); +} else { + $variants = array_reverse($variants); + foreach($variants as $variantInfo) + try { + printf('Deleting variant "%s" for upload #%d...%s', $variantInfo->variant, $variantInfo->uploadId, PHP_EOL); + $eeprom->uploadsCtx->uploads->deleteUploadVariant($variantInfo); + + printf('Attempting to delete file...%s', PHP_EOL); + $eeprom->storageCtx->deleteFile($variantInfo->fileId, false); + } catch(Exception $ex) { + printf('Exception occurred while deleting variant:%s', PHP_EOL); + printf('%s%s', $ex->getMessage(), PHP_EOL); + } +} + +try { + printf('Deleting upload #%d...%s', $uploadInfo->id, PHP_EOL); + $eeprom->uploadsCtx->uploads->deleteUpload($uploadInfo); +} catch(Exception $ex) { + printf('Exception occurred while deleting upload:%s%s%s', PHP_EOL, $ex->getMessage(), PHP_EOL); +} + +if($displayStorageDeleteHint) + printf('NOTE: if the exception(s) that occurred while deleting variants were related to database constraints, an upload linking to the same file exists, if they occurred due to filesystem permissions, the file will linger but will be removed during the next scheduled clean up.%s', PHP_EOL); diff --git a/tools/deny-file b/tools/deny-file new file mode 100755 index 0000000..af9831f --- /dev/null +++ b/tools/deny-file @@ -0,0 +1,93 @@ +#!/usr/bin/env php + $case->value)), + PHP_EOL + )); +} + +if($isHash) { + $fileId = match(strlen($fileId)) { + 32 => $fileId, + 43 => UriBase64::decode($fileId), + 64 => hex2bin($fileId), + 66 => hex2bin(substr($fileId, 2)), + default => null, + }; + $fileField = StorageRecordGetFileField::Hash; +} else { + if($isBase62) + $fileId = XNumber::fromBase62($fileId); + + if($isUploadId) { + $uploadId = $fileId; + $fileId = null; + + try { + $variantInfo = $eeprom->uploadsCtx->uploads->getUploadVariant($uploadId, ''); + } catch(RuntimeException $ex) { + $hint = $isBase62 ? 'trim last four characters from base62 id' : 'specify -s for base62 ids'; + die(sprintf('The requested upload does not exist. (hint: %s)%s', $hint, PHP_EOL)); + } + + $fileId = $variantInfo->fileId; + } +} + +if($fileId === null) + die(sprintf('Could not resolve file ID from the provided inputs.%s', PHP_EOL)); + +try { + $storageInfo = $eeprom->storageCtx->records->getFile($fileId, $fileField); +} catch(RuntimeException $ex) { + $hint = $isUploadId ? ( + $isBase62 ? 'trim last four characters from base62 id' : 'specify -s for base62 ids' + ) : 'specify -h for hashes or -u to use upload ids'; + die(sprintf('The requested file does not exist. (hint: %s)%s', $hint, PHP_EOL)); +} + +printf( + 'Found file #%d of %d bytes with type %s at %s with SHA256 hash %s.%s', + $storageInfo->id, + $storageInfo->size, + htmlspecialchars($storageInfo->type), + $storageInfo->createdAt->toIso8601ZuluString(), + UriBase64::encode($storageInfo->hash), + PHP_EOL +); + +try { + printf('Attempting to deny file...%s', PHP_EOL); + $eeprom->denylistCtx->denyFile($storageInfo, $reason); +} catch(Exception $ex) { + printf('Exception occurred while deleting file:%s', PHP_EOL); + printf('%s%s', $ex->getMessage(), PHP_EOL); +} diff --git a/tools/migrate-override-toggle b/tools/migrate-override-toggle new file mode 100755 index 0000000..877d51e --- /dev/null +++ b/tools/migrate-override-toggle @@ -0,0 +1,12 @@ +#!/usr/bin/env php +database->getMigrateLockPath(); +if(is_file($lockPath)) { + printf('Removing migration lock...%s', PHP_EOL); + unlink($lockPath); +} else { + printf('Setting migration lock...%s', PHP_EOL); + touch($lockPath); +} diff --git a/tools/nuke-file b/tools/nuke-file new file mode 100755 index 0000000..810d79e --- /dev/null +++ b/tools/nuke-file @@ -0,0 +1,83 @@ +#!/usr/bin/env php + $fileId, + 43 => UriBase64::decode($fileId), + 64 => hex2bin($fileId), + 66 => hex2bin(substr($fileId, 2)), + default => null, + }; + $fileField = StorageRecordGetFileField::Hash; +} else { + if($isBase62) + $fileId = XNumber::fromBase62($fileId); + + if($isUploadId) { + $uploadId = $fileId; + $fileId = null; + + try { + $variantInfo = $eeprom->uploadsCtx->uploads->getUploadVariant($uploadId, $variant); + } catch(RuntimeException $ex) { + $hint = $isBase62 ? 'trim last four characters from base62 id' : 'specify -s for base62 ids'; + die(sprintf('The requested upload variant does not exist. (hint: %s)%s', $hint, PHP_EOL)); + } + + $fileId = $variantInfo->fileId; + } +} + +if($fileId === null) + die(sprintf('Could not resolve file ID from the provided inputs.%s', PHP_EOL)); + +try { + $storageInfo = $eeprom->storageCtx->records->getFile($fileId, $fileField); +} catch(RuntimeException $ex) { + $hint = $isUploadId ? ( + $isBase62 ? 'trim last four characters from base62 id' : 'specify -s for base62 ids' + ) : 'specify -h for hashes or -u to use upload ids, with -v for variant selection'; + die(sprintf('The requested file does not exist. (hint: %s)%s', $hint, PHP_EOL)); +} + +printf( + 'Found file #%d of %d bytes with type %s at %s with SHA256 hash %s.%s', + $storageInfo->id, + $storageInfo->size, + htmlspecialchars($storageInfo->type), + $storageInfo->createdAt->toIso8601ZuluString(), + UriBase64::encode($storageInfo->hash), + PHP_EOL +); + +try { + printf('Attempting to delete file...%s', PHP_EOL); + $eeprom->storageCtx->deleteFile($storageInfo); +} catch(Exception $ex) { + printf('Exception occurred while deleting file:%s', PHP_EOL); + printf('%s%s', $ex->getMessage(), PHP_EOL); +}