flashii/eeprom
flashii/eeprom
Archived
3
0
Fork 1
Code Pull requests Releases Activity

Storage system overhaul.

Browse source
This commit is contained in:
flash 2024-12-23 00:52:14 +00:00
parent 4129a03239
commit 5a22ec3167
27 changed files with 1112 additions and 402 deletions

1
.gitignore vendored
View file

@ -11,3 +11,4 @@
/vendor
/.migrating
/node_modules
/storage

16
composer.lock generated
View file

@ -114,11 +114,11 @@
},
{
"name": "flashwave/index",
"version": "v0.2410.630140",
"version": "v0.2410.830205",
"source": {
"type": "git",
"url": "https://patchii.net/flash/index.git",
"reference": "469391f9b601bf30553252470f175588744d4c18"
"reference": "416c716b2efab9619d14be02a20cd6540e18a83f"
},
"require": {
"ext-mbstring": "*",
@ -165,7 +165,7 @@
],
"description": "Composer package for the common library for my projects.",
"homepage": "https://railgun.sh/index",
"time": "2024-12-02T01:41:44+00:00"
"time": "2024-12-22T02:05:46+00:00"
},
{
"name": "guzzlehttp/psr7",
@ -344,16 +344,16 @@
},
{
"name": "nesbot/carbon",
"version": "3.8.2",
"version": "3.8.3",
"source": {
"type": "git",
"url": "https://github.com/briannesbitt/Carbon.git",
"reference": "e1268cdbc486d97ce23fef2c666dc3c6b6de9947"
"reference": "f01cfa96468f4c38325f507ab81a4f1d2cd93cfe"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/briannesbitt/Carbon/zipball/e1268cdbc486d97ce23fef2c666dc3c6b6de9947",
"reference": "e1268cdbc486d97ce23fef2c666dc3c6b6de9947",
"url": "https://api.github.com/repos/briannesbitt/Carbon/zipball/f01cfa96468f4c38325f507ab81a4f1d2cd93cfe",
"reference": "f01cfa96468f4c38325f507ab81a4f1d2cd93cfe",
"shasum": ""
},
"require": {
@ -446,7 +446,7 @@
"type": "tidelift"
}
],
"time": "2024-11-07T17:46:48+00:00"
"time": "2024-12-21T18:03:19+00:00"
},
{
"name": "psr/clock",

36
cron.php
View file

@ -1,35 +1,23 @@
<?php
namespace EEPROM;
if(!defined('EEPROM_SEM_NAME'))
define('EEPROM_SEM_NAME', 'c');
if(!defined('EEPROM_SFM_PATH'))
define('EEPROM_SFM_PATH', sys_get_temp_dir() . DIRECTORY_SEPARATOR . '{6bf19abb-ae7e-4a1d-85f9-00dfb7c90264}');
if(!is_file(EEPROM_SFM_PATH))
touch(EEPROM_SFM_PATH);
$ftok = ftok(EEPROM_SFM_PATH, EEPROM_SEM_NAME);
$semaphore = sem_get($ftok, 1);
if(!sem_acquire($semaphore))
die('Failed to acquire semaphore.' . PHP_EOL);
$cronPath = sys_get_temp_dir() . '/eeprom-cron-' . hash('sha256', __DIR__) . '.lock';
if(is_file($cronPath))
die('EEPROM cron script is already running.' . PHP_EOL);
touch($cronPath);
try {
require_once __DIR__ . '/eeprom.php';
// Mark expired as deleted
$expired = $eeprom->uploadsCtx->uploadsData->getUploads(expired: true, deleted: false);
foreach($expired as $uploadInfo)
$eeprom->uploadsCtx->uploadsData->deleteUpload($uploadInfo);
$eeprom->uploadsCtx->uploadsData->deleteExpiredUploads();
// Hard delete soft deleted files
$deleted = $eeprom->uploadsCtx->uploadsData->getUploads(deleted: true);
foreach($deleted as $uploadInfo) {
$eeprom->uploadsCtx->deleteUploadData($uploadInfo);
$eeprom->uploadsCtx->uploadsData->nukeUpload($uploadInfo);
}
$records = $eeprom->storageCtx->records->getFiles(orphaned: true);
foreach($records as $record)
$eeprom->storageCtx->deleteFile($record);
// new storage format should store by hashes again, ensure blacklisted data is no longer saved
$records = $eeprom->storageCtx->records->getFiles(denied: true);
foreach($records as $record)
$eeprom->storageCtx->deleteFile($record);
} finally {
sem_release($semaphore);
unlink($cronPath);
}

84
database/2024_12_20_043832_convert_upload_ids_to_snowflakes.php Normal file
View file

@ -0,0 +1,84 @@
<?php
use Index\XString;
use Index\Db\DbConnection;
use Index\Db\Migration\DbMigration;
final class ConvertUploadIdsToSnowflakes_20241220_043832 implements DbMigration {
public const string OLD_UPLOADS = PRM_PUBLIC . '/data';
public const string OLD_THUMBS = PRM_PUBLIC . '/thumb';
public function migrate(DbConnection $conn): void {
// Create new upload id field and rename old one
$conn->execute(<<<SQL
ALTER TABLE prm_uploads
ADD COLUMN upload_id BIGINT UNSIGNED NOT NULL DEFAULT 0 FIRST,
ADD COLUMN upload_secret CHAR(4) NOT NULL DEFAULT '' COLLATE 'ascii_bin' AFTER upload_hash,
CHANGE COLUMN upload_id upload_id_legacy BINARY(32) NOT NULL AFTER upload_id,
DROP PRIMARY KEY,
ADD PRIMARY KEY (upload_id_legacy) USING BTREE;
SQL);
// Generate snowflakes for all existing entries
$snowflaker = $GLOBALS['eeprom']->snowflake;
$stmt = $conn->prepare('UPDATE prm_uploads SET upload_id = ?, upload_secret = ? WHERE upload_id_legacy = ?');
$result = $conn->query('SELECT upload_id_legacy, UNIX_TIMESTAMP(upload_created) FROM prm_uploads');
while($result->next()) {
$legacyId = $result->getString(0);
$timestamp = $result->getInteger(1) * 1000;
$snowflake = $snowflaker->from($timestamp, (ord($legacyId[0]) << 8) | ord($legacyId[1]));
$uploadPath = realpath(sprintf('%s/%s', self::OLD_UPLOADS, $legacyId));
if($uploadPath !== false)
rename(
$uploadPath,
sprintf('%s%s%s', self::OLD_UPLOADS, DIRECTORY_SEPARATOR, $snowflake)
);
$thumbPath = realpath(sprintf('%s/%s', self::OLD_THUMBS, $legacyId));
if($thumbPath !== false)
rename(
$thumbPath,
sprintf('%s%s%s', self::OLD_THUMBS, DIRECTORY_SEPARATOR, $snowflake)
);
$stmt->reset();
$stmt->nextParameter($snowflake);
$stmt->nextParameter(XString::random(4));
$stmt->nextParameter($legacyId);
$stmt->execute();
}
// Change PRIMARY KEY
$conn->execute(<<<SQL
ALTER TABLE prm_uploads
DROP PRIMARY KEY,
ADD PRIMARY KEY (upload_id) USING BTREE;
SQL);
// Change redirect table
$conn->execute(<<<SQL
CREATE TABLE prm_uploads_legacy (
upload_id_legacy BINARY(32) NOT NULL,
upload_id BIGINT(20) UNSIGNED NOT NULL,
UNIQUE INDEX prm_uploads_legacy_unique (upload_id_legacy) USING BTREE,
INDEX prm_uploads_legacy_foreign (upload_id) USING BTREE,
CONSTRAINT prm_uploads_legacy_foreign
FOREIGN KEY (upload_id)
REFERENCES prm_uploads (upload_id)
ON UPDATE CASCADE
ON DELETE CASCADE
) ENGINE=InnoDB;
SQL);
// Import legacy ids
$conn->execute('INSERT prm_uploads_legacy (upload_id_legacy, upload_id) SELECT upload_id_legacy, upload_id FROM prm_uploads');
// Drop upload_id_legacy column and make new ones non-optional
$conn->execute(<<<SQL
ALTER TABLE prm_uploads
CHANGE COLUMN upload_id upload_id BIGINT(20) UNSIGNED NOT NULL FIRST,
CHANGE COLUMN upload_secret upload_secret CHAR(4) NOT NULL COLLATE 'ascii_bin' AFTER upload_hash,
DROP COLUMN upload_id_legacy
SQL);
}
}

93
database/2024_12_20_043833_store_all_files_by_hash.php Normal file
View file

@ -0,0 +1,93 @@
<?php
use Index\Db\DbConnection;
use Index\Db\Migration\DbMigration;
final class StoreAllFilesByHash_20241220_043833 implements DbMigration {
public const string OLD_UPLOADS = PRM_PUBLIC . '/data';
public const string OLD_THUMBS = PRM_PUBLIC . '/thumb';
public function migrate(DbConnection $conn): void {
$conn->execute(<<<SQL
CREATE TABLE prm_files (
file_id BIGINT(20) UNSIGNED NOT NULL,
file_hash BINARY(32) NOT NULL,
file_type VARCHAR(255) NOT NULL COLLATE 'ascii_general_ci',
file_size INT(11) NOT NULL,
file_created TIMESTAMP NOT NULL DEFAULT current_timestamp(),
PRIMARY KEY (file_id) USING BTREE,
UNIQUE INDEX prm_files_hash_unique (file_hash) USING BTREE,
INDEX prm_files_created_index (file_created) USING BTREE,
INDEX prm_files_type_index (file_type) USING BTREE,
INDEX prm_files_size_index (file_size) USING BTREE
) ENGINE=InnoDB COLLATE='utf8mb4_bin';
SQL);
$conn->execute(<<<SQL
CREATE TABLE prm_uploads_files (
upload_id BIGINT(20) UNSIGNED NOT NULL,
upload_variant VARCHAR(255) NOT NULL COLLATE 'ascii_general_ci',
file_id BIGINT(20) UNSIGNED NOT NULL,
file_type VARCHAR(255) NULL DEFAULT NULL COLLATE 'ascii_general_ci',
PRIMARY KEY (upload_id, upload_variant) USING BTREE,
INDEX prm_uploads_files_upload_foreign (upload_id) USING BTREE,
INDEX prm_uploads_files_file_foreign (file_id) USING BTREE,
CONSTRAINT prm_uploads_files_file_foreign
FOREIGN KEY (file_id)
REFERENCES prm_files (file_id)
ON UPDATE CASCADE
ON DELETE RESTRICT,
CONSTRAINT prm_uploads_files_upload_foreign
FOREIGN KEY (upload_id)
REFERENCES prm_uploads (upload_id)
ON UPDATE CASCADE
ON DELETE CASCADE
) ENGINE=InnoDB COLLATE='utf8mb4_bin';
SQL);
$storageCtx = $GLOBALS['eeprom']->storageCtx;
$stmt = $conn->prepare('INSERT INTO prm_uploads_files (upload_id, upload_variant, file_id, file_type) VALUES (?, ?, ?, ?)');
$result = $conn->query('SELECT upload_id, upload_type FROM prm_uploads');
while($result->next()) {
$stmt->reset();
$uploadId = $result->getString(0);
$uploadType = $result->getString(1);
$stmt->addParameter(1, $uploadId);
$uploadPath = realpath(sprintf('%s/%s', self::OLD_UPLOADS, $uploadId));
if($uploadPath !== false) {
$record = $storageCtx->importFile($uploadPath, type: $uploadType);
$stmt->addParameter(2, '');
$stmt->addParameter(3, $record->id);
$stmt->addParameter(4, $record->type === $uploadType ? null : $uploadType);
$stmt->execute();
}
$thumbPath = realpath(sprintf('%s/%s', self::OLD_THUMBS, $uploadId));
if($thumbPath !== false) {
$record = $storageCtx->importFile($thumbPath, type: 'image/jpeg');
$stmt->addParameter(2, 'thumb');
$stmt->addParameter(3, $record->id);
$stmt->addParameter(4, null);
$stmt->execute();
}
}
$conn->execute(<<<SQL
ALTER TABLE prm_uploads
CHANGE COLUMN upload_name upload_name VARCHAR(255) NOT NULL COLLATE 'utf8mb4_unicode_520_ci' AFTER upload_secret,
CHANGE COLUMN upload_bump upload_bump INT(10) UNSIGNED NOT NULL DEFAULT '0' AFTER upload_name,
DROP COLUMN upload_hash,
DROP COLUMN upload_type,
DROP COLUMN upload_size,
DROP COLUMN upload_deleted,
DROP INDEX prm_uploads_hash_index,
DROP INDEX prm_uploads_deleted_index,
DROP INDEX prm_uploads_unique;
SQL);
}
}

7
eeprom.php
View file

@ -11,8 +11,6 @@ define('PRM_DEBUG', is_file(PRM_ROOT . '/.debug'));
define('PRM_PUBLIC', PRM_ROOT . '/public');
define('PRM_SOURCE', PRM_ROOT . '/src');
define('PRM_MIGRATIONS', PRM_ROOT . '/database');
define('PRM_UPLOADS', PRM_PUBLIC . '/data');
define('PRM_THUMBS', PRM_PUBLIC . '/thumb');
require_once PRM_ROOT . '/vendor/autoload.php';
@ -35,11 +33,6 @@ if($cfg->hasValues('sentry:dsn'))
});
})($cfg->scopeTo('sentry'));
if(!is_dir(PRM_UPLOADS))
mkdir(PRM_UPLOADS, 0775, true);
if(!is_dir(PRM_THUMBS))
mkdir(PRM_THUMBS, 0775, true);
$db = DbBackends::create($cfg->getString('database:dsn', 'null:'));
$db->execute('SET SESSION time_zone = \'+00:00\', sql_mode = \'STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION\';');

32
public/assets/script.js Normal file
View file

@ -0,0 +1,32 @@
(() => {
const $i = document.getElementById.bind(document);
const $ffs = (size, binary) => {
if(size < 1)
return 'Zero Bytes';
var div = binary ? 1024 : 1000,
exp = Math.floor(Math.log(size) / Math.log(div)),
size = size / Math.pow(div, exp);
var symbol = ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'][exp];
return size.toFixed(2) + ' ' + symbol + (binary ? 'i' : '') + 'B';
};
const update = async () => {
const response = await fetch('/stats.json');
if(!response.ok)
console.error('stats request failed!!!!');
const stats = await response.json();
$i('-eeprom-stat-size-fmt').textContent = $ffs(stats.size);
$i('-eeprom-stat-size-byte').textContent = `${stats.size.toLocaleString()} bytes`;
$i('-eeprom-stat-files').textContent = stats.files.toLocaleString();
$i('-eeprom-stat-uploads').textContent = stats.uploads.toLocaleString();
$i('-eeprom-stat-types').textContent = stats.types.toLocaleString();
$i('-eeprom-stat-members').textContent = stats.members.toLocaleString();
};
setInterval(update, 5 * 60 * 1000);
update();
})();

5
public/assets/style.css
View file

@ -33,14 +33,13 @@ body {
.eeprom-header {
margin: 20px 0;
padding: 0 20px;
text-align: center;
}
.eeprom-logo {
max-width: 590px;
width: 100%;
}
.eeprom-description {
text-align: center;
display: inline-block;
}
.eeprom-stats {

55
public/index.html
View file

@ -1,17 +1,17 @@
<!doctype html>
<html>
<head>
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>EEPROM</title>
<link href="/assets/style.css" type="text/css" rel="stylesheet"/>
<link href="/assets/style.css" type="text/css" rel="stylesheet">
</head>
<body>
<div class="eeprom">
<div class="eeprom-inner">
<div class="eeprom-header">
<div class="eeprom-logo">
<img src="/assets/eeprom-logo.png" alt="EEPROM" class="eeprom-logo" />
<img src="/assets/eeprom-logo.png" alt="EEPROM" class="eeprom-logo">
</div>
<div class="eeprom-description">
<p>File Upload Service</p>
@ -28,6 +28,15 @@
</div>
</div>
<div class="eeprom-divider"></div>
<div class="eeprom-stat">
<div class="eeprom-stat-num">
<div id="-eeprom-stat-uploads" class="eeprom-stat-num-big">---</div>
</div>
<div class="eeprom-stat-title">
Uploads
</div>
</div>
<div class="eeprom-divider"></div>
<div class="eeprom-stat">
<div class="eeprom-stat-num">
<div id="-eeprom-stat-files" class="eeprom-stat-num-big">---</div>
@ -60,42 +69,6 @@
</div>
</div>
</div>
<script type="text/javascript">
var sSizeFmt = document.getElementById('-eeprom-stat-size-fmt'),
sSizeByte = document.getElementById('-eeprom-stat-size-byte'),
sFiles = document.getElementById('-eeprom-stat-files'),
sTypes = document.getElementById('-eeprom-stat-types'),
sMembers = document.getElementById('-eeprom-stat-members');
Number.prototype.formatFileSize = function(binary) {
if(this < 1)
return 'Zero Bytes';
var div = binary ? 1024 : 1000,
exp = Math.floor(Math.log(this) / Math.log(div)),
size = this / Math.pow(div, exp);
var symbol = ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'][exp];
return size.toFixed(2) + ' ' + symbol + (binary ? 'i' : '') + 'B';
};
var eUpdateStats = function() {
var xhr = new XMLHttpRequest;
xhr.onload = function() {
var stats = JSON.parse(xhr.responseText);
sSizeFmt.textContent = stats.size.formatFileSize();
sSizeByte.textContent = stats.size.toLocaleString() + ' bytes';
sFiles.textContent = stats.files.toLocaleString();
sTypes.textContent = stats.types.toLocaleString();
sMembers.textContent = stats.members.toLocaleString();
};
xhr.open('GET', '/stats.json');
xhr.send();
};
eUpdateStats();
setInterval(eUpdateStats, 5 * 60 * 1000);
</script>
<script src="/assets/script.js"></script>
</body>
</html>

19
src/Blacklist/BlacklistContext.php
View file

@ -1,7 +1,7 @@
<?php
namespace EEPROM\Blacklist;
use EEPROM\Uploads\UploadInfo;
use EEPROM\Storage\StorageRecord;
use Index\Db\DbConnection;
class BlacklistContext {
@ -11,18 +11,17 @@ class BlacklistContext {
$this->blacklistData = new BlacklistData($dbConn);
}
public function createBlacklistEntry(UploadInfo|string $item, string $reason): void {
if($item instanceof UploadInfo)
$item = hex2bin($item->hashString);
public function createBlacklistEntry(StorageRecord|string $recordOrHash, string $reason): void {
if($recordOrHash instanceof StorageRecord)
$recordOrHash = $recordOrHash->hash;
$this->blacklistData->createBlacklistEntry($item, $reason);
$this->blacklistData->createBlacklistEntry($recordOrHash, $reason);
}
public function getBlacklistEntry(UploadInfo|string $item): ?BlacklistInfo {
// will this ever be useful? who knows!
if($item instanceof UploadInfo)
$item = hex2bin($item->hashString);
public function getBlacklistEntry(StorageRecord|string $recordOrHash): ?BlacklistInfo {
if($recordOrHash instanceof StorageRecord)
$recordOrHash = $recordOrHash->hash;
return $this->blacklistData->getBlacklistEntry($item);
return $this->blacklistData->getBlacklistEntry($recordOrHash);
}
}

15
src/DatabaseContext.php
View file

@ -3,8 +3,11 @@ namespace EEPROM;
use Index\Db\DbConnection;
use Index\Db\Migration\{DbMigrationManager,DbMigrationRepo,FsDbMigrationRepo};
use Index\Http\Routing\{HttpMiddleware,RouteHandler,RouteHandlerTrait};
class DatabaseContext implements RouteHandler {
use RouteHandlerTrait;
class DatabaseContext {
public function __construct(
public private(set) DbConnection $connection
) {}
@ -21,4 +24,14 @@ class DatabaseContext {
public function createMigrationRepo(): DbMigrationRepo {
return new FsDbMigrationRepo(PRM_MIGRATIONS);
}
public function getMigrateLockPath(): string {
return sys_get_temp_dir() . '/eeprom-migrate-' . hash('sha256', PRM_ROOT) . '.lock';
}
#[HttpMiddleware('/')]
public function middleware() {
if(is_file($this->getMigrateLockPath()))
return 503;
}
}

9
src/EEPROMContext.php
View file

@ -8,10 +8,13 @@ use EEPROM\Auth\AuthInfo;
class EEPROMContext {
public private(set) DatabaseContext $database;
public private(set) SnowflakeGenerator $snowflake;
private AuthInfo $authInfo;
private Apps\AppsContext $appsCtx;
private Blacklist\BlacklistContext $blacklistCtx;
public private(set) Storage\StorageContext $storageCtx;
public private(set) Uploads\UploadsContext $uploadsCtx;
private Users\UsersContext $usersCtx;
@ -19,18 +22,21 @@ class EEPROMContext {
private Config $config,
DbConnection $dbConn
) {
$this->snowflake = new SnowflakeGenerator;
$this->database = new DatabaseContext($dbConn);
$this->authInfo = new AuthInfo;
$this->appsCtx = new Apps\AppsContext($dbConn);
$this->blacklistCtx = new Blacklist\BlacklistContext($dbConn);
$this->uploadsCtx = new Uploads\UploadsContext($config, $dbConn);
$this->storageCtx = new Storage\StorageContext($config->scopeTo('storage'), $dbConn, $this->snowflake);
$this->uploadsCtx = new Uploads\UploadsContext($config, $dbConn, $this->snowflake);
$this->usersCtx = new Users\UsersContext($dbConn);
}
public function createRouting(bool $isApiDomain): RoutingContext {
$routingCtx = new RoutingContext($this->config->scopeTo('cors'));
$routingCtx->register($this->database);
if($isApiDomain) {
$routingCtx->register(new Auth\AuthRoutes(
@ -46,6 +52,7 @@ class EEPROMContext {
$this->authInfo,
$this->appsCtx,
$this->uploadsCtx,
$this->storageCtx,
$this->blacklistCtx,
$isApiDomain
));

15
src/LandingRoutes.php
View file

@ -20,22 +20,27 @@ class LandingRoutes implements RouteHandler {
#[HttpGet('/stats.json')]
public function getStats() {
$stats = new stdClass;
$stats->uploads = 0;
$stats->files = 0;
$stats->size = 0;
$stats->types = 0;
$stats->members = 0;
$result = $this->dbCtx->connection->query('SELECT COUNT(upload_id), SUM(upload_size), COUNT(DISTINCT upload_type) FROM prm_uploads WHERE upload_deleted IS NULL');
$result = $this->dbCtx->connection->query('SELECT COUNT(upload_id) FROM prm_uploads');
if($result->next())
$stats->uploads = $result->getInteger(0);
$result = $this->dbCtx->connection->query('SELECT COUNT(user_id) FROM prm_users WHERE user_restricted IS NULL');
if($result->next())
$stats->members = $result->getInteger(0);
$result = $this->dbCtx->connection->query('SELECT COUNT(*), SUM(file_size), COUNT(DISTINCT file_type) FROM prm_files');
if($result->next()) {
$stats->files = $result->getInteger(0);
$stats->size = $result->getInteger(1);
$stats->types = $result->getInteger(2);
}
$result = $this->dbCtx->connection->query('SELECT COUNT(user_id) FROM prm_users WHERE user_restricted IS NULL');
if($result->next())
$stats->members = $result->getInteger(0);
return $stats;
}

47
src/SnowflakeGenerator.php Normal file
View file

@ -0,0 +1,47 @@
<?php
namespace EEPROM;
class SnowflakeGenerator {
public const int EPOCH = 1356998400000;
private const int TS_MASK = 0x7FFFFFFFFFFF;
private const int TS_SHIFT = 16;
private const int FP_MASK = 0xFFFF;
private const int FP_BYTES = 2;
private int $counter = -1;
public function __construct(
private int $epoch = self::EPOCH
) {}
public static function now(): int {
return (int)date_create()->format('Uv');
}
public function from(int $timestamp, int $snowflake): int {
return ($snowflake & self::FP_MASK)
| ((($timestamp - $this->epoch) & self::TS_MASK) << self::TS_SHIFT);
}
public function next(int $snowflake): int {
return $this->from(self::now(), $snowflake);
}
public function nextRandom(): int {
return $this->next(random_int(0, self::FP_MASK));
}
public function nextIncrement(): int {
if($this->counter >= self::FP_MASK)
$this->counter = -1;
return $this->next(++$this->counter);
}
public function nextHash(string $hash): int {
return $this->next(
hexdec(bin2hex(substr(str_pad($hash, self::FP_BYTES, "\0"), 0, self::FP_BYTES)))
);
}
}

148
src/Storage/StorageContext.php Normal file
View file

@ -0,0 +1,148 @@
<?php
namespace EEPROM\Storage;
use Imagick;
use InvalidArgumentException;
use RuntimeException;
use EEPROM\SnowflakeGenerator;
use EEPROM\Uploads\UploadVariantInfo;
use Index\Config\Config;
use Index\Db\DbConnection;
class StorageContext {
public private(set) StorageRecords $records;
public private(set) StorageFiles $files;
public function __construct(
Config $config,
DbConnection $dbConn,
SnowflakeGenerator $snowflake
) {
$this->files = new StorageFiles(
$config->getString('local', PRM_ROOT . '/storage'),
$config->getString('remote', '/_storage%s')
);
$this->records = new StorageRecords($dbConn, $snowflake);
}
public function getFileRecordFromVariant(UploadVariantInfo $variantInfo): StorageRecord {
return $this->records->getFile($variantInfo->fileId, StorageRecordGetFileField::Id);
}
public function deleteFile(
StorageRecord|string $recordInfo,
bool $ensureOrphan = true
): void {
// Does taking ID here make sense or should it be hash?
if(!($recordInfo instanceof StorageRecord))
$recordInfo = $this->records->getFile($recordInfo, StorageRecordGetFileField::Id);
if($ensureOrphan)
$this->records->unlinkFileUploads($recordInfo);
if($this->files->delete($recordInfo))
$this->records->deleteFile($recordInfo);
}
public function importFile(
string $path,
StorageImportMode $mode = StorageImportMode::Move,
?string $type = null
): StorageRecord {
if(!is_file($path))
throw new InvalidArgumentException('$path is not a file that exists');
$type ??= mime_content_type($path);
$size = filesize($path);
$hash = $this->files->import($path, $mode);
try {
return $this->records->createFile($hash, $type, $size);
} catch(RuntimeException $ex) {
return $this->records->getFile($hash);
}
}
public function supportsThumbnailing(StorageRecord $fileInfo): bool {
return $fileInfo->isImage
|| $fileInfo->isAudio
|| $fileInfo->isVideo;
}
public function createThumbnail(StorageRecord $fileInfo): StorageRecord {
$filePath = $this->files->getLocalPath($fileInfo);
if(!is_file($filePath))
throw new RuntimeException('local data for provided $fileInfo does not exist');
$tmpPath = tempnam(sys_get_temp_dir(), 'eeprom-thumbnail-');
$imagick = new Imagick;
try {
if($fileInfo->isImage) {
try {
$file = fopen($filePath, 'rb');
$imagick->readImageFile($file);
} finally {
if(isset($file) && is_resource($file))
fclose($file);
}
} elseif($fileInfo->isAudio) {
try {
$file = popen(sprintf('ffmpeg -i %s -an -f image2pipe -c:v copy -frames:v 1 2>/dev/null -', escapeshellarg($filePath)), 'rb');
$imagick->readImageBlob(stream_get_contents($file));
} finally {
if(isset($file) && is_resource($file))
pclose($file);
}
} elseif($fileInfo->isVideo) {
try {
$file = popen(sprintf('ffmpeg -i %s -f image2pipe -c:v png -frames:v 1 2>/dev/null -', escapeshellarg($filePath)), 'rb');
$imagick->readImageBlob(stream_get_contents($file));
} finally {
if(isset($file) && is_resource($file))
pclose($file);
}
}
$imagick->setImageFormat('jpg');
$imagick->setImageCompressionQuality(80);
$width = $imagick->getImageWidth();
$height = $imagick->getImageHeight();
$thumbRes = min(300, $width, $height);
if($width === $height) {
$resizeWidth = $resizeHeight = $thumbRes;
} elseif($width > $height) {
$resizeWidth = $width * $thumbRes / $height;
$resizeHeight = $thumbRes;
} else {
$resizeWidth = $thumbRes;
$resizeHeight = $height * $thumbRes / $width;
}
$resizeWidth = (int)$resizeWidth;
$resizeHeight = (int)$resizeHeight;
$imagick->resizeImage(
$resizeWidth, $resizeHeight,
Imagick::FILTER_GAUSSIAN, 0.7
);
$imagick->cropImage(
$thumbRes,
$thumbRes,
(int)ceil(($resizeWidth - $thumbRes) / 2),
(int)ceil(($resizeHeight - $thumbRes) / 2)
);
$imagick->writeImage($tmpPath);
} finally {
$imagick->clear();
}
if(!is_file($tmpPath))
throw new RuntimeException('failed to create thumbnail file');
return $this->importFile($tmpPath, StorageImportMode::Move, 'image/jpeg');
}
}

102
src/Storage/StorageFiles.php Normal file
View file

@ -0,0 +1,102 @@
<?php
namespace EEPROM\Storage;
use InvalidArgumentException;
use RuntimeException;
use Index\UriBase64;
class StorageFiles {
public function __construct(
private string $localPath,
private string $remotePath
) {}
public static function stringifyHash(StorageRecord|string $hashOrRecord): string {
if($hashOrRecord instanceof StorageRecord)
return $hashOrRecord->hashString;
if(strlen($hashOrRecord) === 64 && ctype_xdigit($hashOrRecord)) // hex base64, decode
$hashOrRecord = hex2bin($hashOrRecord);
if(strlen($hashOrRecord) === 32) // SHA256 hash, encode
return UriBase64::encode($hashOrRecord);
if(strlen($hashOrRecord) === 43 && ctype_print($hashOrRecord)) // already a base64uri string, hopefully
return $hashOrRecord;
throw new InvalidArgumentException('$hashOrRecord must be a 32 byte long binary string or a 43 byte long printable ascii string or a 64 byte long hexadecimal string or an instance of StorageRecord');
}
public function getLocalPath(StorageRecord|string $hashOrRecord, bool $ensurePath = false): string {
$hash = self::stringifyHash($hashOrRecord);
$dir1 = substr($hash, 0, 2);
$dir2 = substr($hash, 2, 2);
$dir = sprintf(
'%s%s%s%s%s',
$this->localPath,
DIRECTORY_SEPARATOR,
$dir1,
DIRECTORY_SEPARATOR,
$dir2
);
if($ensurePath && !is_dir($dir))
mkdir($dir, 0775, true);
return sprintf(
'%s%s%s',
$dir,
DIRECTORY_SEPARATOR,
$hash
);
}
public function getRemotePath(StorageRecord|string $hashOrRecord): string {
$hash = self::stringifyHash($hashOrRecord);
$dir1 = substr($hash, 0, 2);
$dir2 = substr($hash, 2, 2);
return sprintf($this->remotePath, sprintf(
'%s%s%s%s%s%s',
DIRECTORY_SEPARATOR,
$dir1,
DIRECTORY_SEPARATOR,
$dir2,
DIRECTORY_SEPARATOR,
$hash
));
}
public function exists(StorageRecord|string $hashOrRecord): bool {
return is_file($this->getLocalPath($hashOrRecord));
}
public function import(string $path, StorageImportMode $mode = StorageImportMode::Move): string {
if(!is_file($path))
throw new InvalidArgumentException('$path does not exist or is not a file');
$hash = hash_file('sha256', $path, true);
$target = $this->getLocalPath($hash, true);
if(is_file($target)) {
if($mode === StorageImportMode::Move || $mode === StorageImportMode::Upload)
unlink($path);
} else {
if($mode === StorageImportMode::Move)
$success = rename($path, $target);
elseif($mode === StorageImportMode::Copy)
$success = copy($path, $target);
elseif($mode === StorageImportMode::Upload)
$success = move_uploaded_file($path, $target);
else
$success = false;
if(!$success)
throw new RuntimeException('unable to move or copy $path using given $mode');
}
return $hash;
}
public function delete(StorageRecord|string $hashOrRecord): bool {
$path = $this->getLocalPath($hashOrRecord);
return !is_file($path) || unlink($path);
}
}

8
src/Storage/StorageImportMode.php Normal file
View file

@ -0,0 +1,8 @@
<?php
namespace EEPROM\Storage;
enum StorageImportMode {
case Copy;
case Move;
case Upload;
}

46
src/Storage/StorageRecord.php Normal file
View file

@ -0,0 +1,46 @@
<?php
namespace EEPROM\Storage;
use Carbon\CarbonImmutable;
use Index\UriBase64;
use Index\Db\DbResult;
class StorageRecord {
public function __construct(
public private(set) string $id,
public private(set) string $hash,
public private(set) string $type,
public private(set) int $size,
public private(set) int $createdTime,
) {}
public static function fromResult(DbResult $result): StorageRecord {
return new StorageRecord(
id: $result->getString(0),
hash: $result->getString(1),
type: $result->getString(2),
size: $result->getInteger(3),
createdTime: $result->getInteger(4),
);
}
public string $hashString {
get => UriBase64::encode($this->hash);
}
public CarbonImmutable $createdAt {
get => CarbonImmutable::createFromTimestampUTC($this->createdTime);
}
public bool $isImage {
get => str_starts_with($this->type, 'image/');
}
public bool $isVideo {
get => str_starts_with($this->type, 'video/');
}
public bool $isAudio {
get => str_starts_with($this->type, 'audio/');
}
}

7
src/Storage/StorageRecordGetFileField.php Normal file
View file

@ -0,0 +1,7 @@
<?php
namespace EEPROM\Storage;
enum StorageRecordGetFileField {
case Id;
case Hash;
}

155
src/Storage/StorageRecords.php Normal file
View file

@ -0,0 +1,155 @@
<?php
namespace EEPROM\Storage;
use InvalidArgumentException;
use RuntimeException;
use EEPROM\SnowflakeGenerator;
use Index\Db\{DbConnection,DbStatementCache};
class StorageRecords {
private DbStatementCache $cache;
public function __construct(
DbConnection $dbConn,
private SnowflakeGenerator $snowflake
) {
$this->cache = new DbStatementCache($dbConn);
}
public function getFiles(
?bool $orphaned = null,
?bool $denied = null
): iterable {
$hasOrphaned = $orphaned !== null;
$hasDenied = $denied !== null;
$query = <<<SQL
SELECT f.file_id, f.file_hash, f.file_type, f.file_size, UNIX_TIMESTAMP(f.file_created)
FROM prm_files AS f
SQL;
if($hasOrphaned)
$query .= <<<SQL
LEFT JOIN prm_uploads_files AS uf ON f.file_id = uf.file_id
SQL;
if($hasDenied)
$query .= <<<SQL
LEFT JOIN prm_blacklist AS dl ON f.file_hash = dl.bl_hash
SQL;
$args = 0;
if($orphaned !== null)
$query .= sprintf(
' %s uf.file_id %s NULL',
++$args > 1 ? 'OR' : 'WHERE',
$orphaned ? 'IS' : 'IS NOT'
);
if($denied !== null)
$query .= sprintf(
' %s dl.bl_hash %s NULL',
++$args > 1 ? 'OR' : 'WHERE',
$denied ? 'IS NOT' : 'IS'
);
$stmt = $this->cache->get($query);
$stmt->execute();
return $stmt->getResult()->getIterator(StorageRecord::fromResult(...));
}
public function getFile(
string $value,
StorageRecordGetFileField $field = StorageRecordGetFileField::Hash // wdym verbose?
): StorageRecord {
$field = match($field) {
StorageRecordGetFileField::Id => 'file_id',
StorageRecordGetFileField::Hash => 'file_hash',
default => throw new InvalidArgumentException('$field is not an acceptable value'),
};
$stmt = $this->cache->get(<<<SQL
SELECT file_id, file_hash, file_type, file_size, UNIX_TIMESTAMP(file_created)
FROM prm_files
WHERE {$field} = ?
SQL);
$stmt->nextParameter($value);
$stmt->execute();
$result = $stmt->getResult();
if(!$result->next())
throw new RuntimeException('could not find that file record');
return StorageRecord::fromResult($result);
}
public function createFile(
string $hash,
string $type,
int $size
): StorageRecord {
if(strlen($hash) !== 32)
throw new InvalidArgumentException('$hash must be a 32 byte hash string');
if(mb_strlen($type, 'UTF-8') > 255)
throw new InvalidArgumentException('$type is too long');
if($size < 0)
throw new InvalidArgumentException('$size may not be negative');
$fileId = $this->snowflake->nextHash($hash);
$stmt = $this->cache->get(<<<SQL
INSERT INTO prm_files (
file_id, file_hash, file_type, file_size
) VALUES (?, ?, ?, ?)
SQL);
$stmt->nextParameter($fileId); // generate snowflake
$stmt->nextParameter($hash);
$stmt->nextParameter($type);
$stmt->nextParameter($size);
$stmt->execute();
return $this->getFile($fileId, StorageRecordGetFileField::Id);
}
public function deleteFile(
StorageRecord|string $value,
StorageRecordGetFileField $field = StorageRecordGetFileField::Hash
): void {
if($value instanceof StorageRecord) {
$value = $value->id;
$field = 'file_id';
} else
$field = match($field) {
StorageRecordGetFileField::Id => 'file_id',
StorageRecordGetFileField::Hash => 'file_hash',
default => throw new InvalidArgumentException('$field is not an acceptable value'),
};
$stmt = $this->cache->get(<<<SQL
DELETE FROM prm_files
WHERE {$field} = ?
SQL);
$stmt->nextParameter($value);
$stmt->execute();
}
public function unlinkFileUploads(
StorageRecord|string $value,
StorageRecordGetFileField $field = StorageRecordGetFileField::Hash
): void {
if($value instanceof StorageRecord) {
$field = '?';
$value = $value->id;
} else
$field = match($field) {
StorageRecordGetFileField::Id => '?',
StorageRecordGetFileField::Hash => '(SELECT file_id FROM prm_files WHERE file_hash = ?)',
default => throw new InvalidArgumentException('$field is not an acceptable value'),
};
$stmt = $this->cache->get(<<<SQL
DELETE FROM prm_uploads_files
WHERE file_id = {$field}
SQL);
$stmt->nextParameter($value);
$stmt->execute();
}
}

56
src/Uploads/UploadInfo.php
View file

@ -2,7 +2,7 @@
namespace EEPROM\Uploads;
use Carbon\CarbonImmutable;
use Index\MediaType;
use Index\{MediaType,XNumber};
use Index\Db\DbResult;
class UploadInfo {
@ -10,16 +10,13 @@ class UploadInfo {
public private(set) string $id,
public private(set) ?string $userId,
public private(set) ?string $appId,
public private(set) string $hashString,
public private(set) string $secret,
public private(set) string $name,
public private(set) int $bumpAmount,
public private(set) string $remoteAddress,
public private(set) int $createdTime,
public private(set) ?int $accessedTime,
public private(set) ?int $expiresTime,
public private(set) ?int $deletedTime,
public private(set) int $bumpAmount,
public private(set) string $name,
public private(set) string $mediaTypeString,
public private(set) int $dataSize,
) {}
public static function fromResult(DbResult $result): UploadInfo {
@ -27,19 +24,20 @@ class UploadInfo {
id: $result->getString(0),
userId: $result->getStringOrNull(1),
appId: $result->getStringOrNull(2),
hashString: $result->getString(3),
remoteAddress: $result->getString(4),
createdTime: $result->getInteger(5),
accessedTime: $result->getIntegerOrNull(6),
expiresTime: $result->getIntegerOrNull(7),
deletedTime: $result->getIntegerOrNull(8),
bumpAmount: $result->getInteger(9),
name: $result->getString(10),
mediaTypeString: $result->getString(11),
dataSize: $result->getInteger(12),
secret: $result->getString(3),
name: $result->getString(4),
bumpAmount: $result->getInteger(5),
remoteAddress: $result->getString(6),
createdTime: $result->getInteger(7),
accessedTime: $result->getIntegerOrNull(8),
expiresTime: $result->getIntegerOrNull(9),
);
}
public string $id62 {
get => XNumber::toBase62((int)$this->id);
}
public CarbonImmutable $createdAt {
get => CarbonImmutable::createFromTimestampUTC($this->createdTime);
}
@ -56,31 +54,7 @@ class UploadInfo {
get => $this->expiresTime === null ? null : CarbonImmutable::createFromTimestampUTC($this->expiresTime);
}
public bool $deleted {
get => $this->deletedTime !== null;
}
public ?CarbonImmutable $deletedAt {
get => $this->deletedTime === null ? null : CarbonImmutable::createFromTimestampUTC($this->deletedTime);
}
public ?int $bumpAmountForUpdate {
get => $this->expiresTime !== null && $this->bumpAmount > 0 ? $this->bumpAmount : null;
}
public MediaType $mediaType {
get => MediaType::parse($this->mediaTypeString);
}
public bool $isImage {
get => str_starts_with($this->mediaTypeString, 'image/');
}
public bool $isVideo {
get => str_starts_with($this->mediaTypeString, 'video/');
}
public bool $isAudio {
get => str_starts_with($this->mediaTypeString, 'audio/');
}
}

22
src/Uploads/UploadVariantInfo.php Normal file
View file

@ -0,0 +1,22 @@
<?php
namespace EEPROM\Uploads;
use Index\Db\DbResult;
class UploadVariantInfo {
public function __construct(
public private(set) string $uploadId,
public private(set) string $variant,
public private(set) string $fileId,
public private(set) ?string $type
) {}
public static function fromResult(DbResult $result): UploadVariantInfo {
return new UploadVariantInfo(
uploadId: $result->getString(0),
variant: $result->getString(1),
fileId: $result->getString(2),
type: $result->getStringOrNull(3),
);
}
}

162
src/Uploads/UploadsContext.php
View file

@ -2,179 +2,57 @@
namespace EEPROM\Uploads;
use DateTimeInterface;
use Imagick;
use Index\Config\Config;
use Index\Db\DbConnection;
use EEPROM\FFMPEG;
use EEPROM\{FFMPEG,SnowflakeGenerator};
use EEPROM\Storage\StorageRecord;
class UploadsContext {
public private(set) UploadsData $uploadsData;
public function __construct(
private Config $config,
DbConnection $dbConn
DbConnection $dbConn,
SnowflakeGenerator $snowflake
) {
$this->uploadsData = new UploadsData($dbConn);
}
public function getFileDataPath(UploadInfo $uploadInfo): string {
return sprintf('%s%s%s', PRM_UPLOADS, DIRECTORY_SEPARATOR, $uploadInfo->id);
}
public function getThumbnailDataPath(UploadInfo $uploadInfo): string {
return sprintf('%s%s%s', PRM_THUMBS, DIRECTORY_SEPARATOR, $uploadInfo->id);
}
public function getFileDataRedirectPath(UploadInfo $uploadInfo): string {
return sprintf('%s%s%s', str_replace(PRM_PUBLIC, '', PRM_UPLOADS), DIRECTORY_SEPARATOR, $uploadInfo->id);
}
public function getThumbnailDataRedirectPath(UploadInfo $uploadInfo): string {
return sprintf('%s%s%s', str_replace(PRM_PUBLIC, '', PRM_THUMBS), DIRECTORY_SEPARATOR, $uploadInfo->id);
}
public function getThumbnailDataPathOrCreate(UploadInfo $uploadInfo): string {
if(!$this->supportsThumbnailing($uploadInfo))
return '';
$thumbPath = $this->getThumbnailDataPath($uploadInfo);
if(is_file($thumbPath))
return $thumbPath;
return $this->createThumbnailInternal(
$uploadInfo,
$this->getFileDataPath($uploadInfo),
$thumbPath
);
}
public function getThumbnailDataRedirectPathOrCreate(UploadInfo $uploadInfo): string {
return str_replace(PRM_PUBLIC, '', $this->getThumbnailDataPathOrCreate($uploadInfo));
$this->uploadsData = new UploadsData($dbConn, $snowflake);
}
public function getFileUrlV1(UploadInfo $uploadInfo, bool $forceApiDomain = false): string {
if(!$forceApiDomain && $this->config->hasValues('domain:short'))
return sprintf('//%s/%s', $this->config->getString('domain:short'), $uploadInfo->id);
return sprintf('//%s/%s%s', $this->config->getString('domain:short'), $uploadInfo->id62, $uploadInfo->secret);
return sprintf('//%s/uploads/%s', $this->config->getString('domain:api'), $uploadInfo->id);
return sprintf('//%s/uploads/%s%s', $this->config->getString('domain:api'), $uploadInfo->id62, $uploadInfo->secret);
}
public function getThumbnailUrlV1(UploadInfo $uploadInfo, bool $forceApiDomain = false): string {
return sprintf('%s.t', $this->getFileUrlV1($uploadInfo, $forceApiDomain));
}
public function convertToClientJsonV1(UploadInfo $uploadInfo, array $overrides = []): array {
public function convertToClientJsonV1(
UploadInfo $uploadInfo,
UploadVariantInfo $variantInfo,
StorageRecord $fileInfo,
array $overrides = []
): array {
return array_merge([
'id' => $uploadInfo->id,
'id' => $uploadInfo->id62 . $uploadInfo->secret,
'url' => $this->getFileUrlV1($uploadInfo),
'urlf' => $this->getFileUrlV1($uploadInfo, true),
'thumb' => $this->getThumbnailUrlV1($uploadInfo),
'name' => $uploadInfo->name,
'type' => $uploadInfo->mediaTypeString,
'size' => $uploadInfo->dataSize,
'type' => $variantInfo->type ?? $fileInfo->type,
'size' => $fileInfo->size,
'user' => $uploadInfo->userId === null ? null : (int)$uploadInfo->userId,
'appl' => $uploadInfo->appId === null ? null : (int)$uploadInfo->appId,
'hash' => $uploadInfo->hashString,
'created' => str_replace('+00:00', 'Z', $uploadInfo->createdAt->format(DateTimeInterface::ATOM)),
'accessed' => $uploadInfo->accessedTime === null ? null : str_replace('+00:00', 'Z', $uploadInfo->accessedAt->format(DateTimeInterface::ATOM)),
'expires' => $uploadInfo->expiresTime === null ? null : str_replace('+00:00', 'Z', $uploadInfo->expiredAt->format(DateTimeInterface::ATOM)),
'hash' => $fileInfo->hashString,
'created' => $uploadInfo->createdAt->toIso8601ZuluString(),
'accessed' => $uploadInfo->accessedAt?->toIso8601ZuluString(),
'expires' => $uploadInfo->expiredAt?->toIso8601ZuluString(),
// These can never be reached, and in situation where they technically could it's because of an outdated local record
'deleted' => null,
'dmca' => null,
], $overrides);
}
public function supportsThumbnailing(UploadInfo $uploadInfo): bool {
return $uploadInfo->isImage
|| $uploadInfo->isAudio
|| $uploadInfo->isVideo;
}
public function createThumbnail(UploadInfo $uploadInfo): string {
if(!$this->supportsThumbnailing($uploadInfo))
return '';
return $this->createThumbnailInternal(
$uploadInfo,
$this->getFileDataPath($uploadInfo),
$this->getThumbnailDataPath($uploadInfo)
);
}
private function createThumbnailInternal(UploadInfo $uploadInfo, string $filePath, string $thumbPath): string {
$imagick = new Imagick;
if($uploadInfo->isImage) {
try {
$file = fopen($filePath, 'rb');
$imagick->readImageFile($file);
} finally {
if(isset($file) && is_resource($file))
fclose($file);
}
} elseif($uploadInfo->isAudio) {
try {
$file = popen(sprintf('ffmpeg -i %s -an -f image2pipe -c:v copy -frames:v 1 2>/dev/null -', escapeshellarg($filePath)), 'rb');
$imagick->readImageBlob(stream_get_contents($file));
} finally {
if(isset($file) && is_resource($file))
pclose($file);
}
} elseif($uploadInfo->isVideo) {
try {
$file = popen(sprintf('ffmpeg -i %s -f image2pipe -c:v png -frames:v 1 2>/dev/null -', escapeshellarg($filePath)), 'rb');
$imagick->readImageBlob(stream_get_contents($file));
} finally {
if(isset($file) && is_resource($file))
pclose($file);
}
}
$imagick->setImageFormat('jpg');
$imagick->setImageCompressionQuality($this->config->getInteger('thumb:quality', 80));
$width = $imagick->getImageWidth();
$height = $imagick->getImageHeight();
$thumbRes = min($this->config->getInteger('thumb:dimensions', 300), $width, $height);
if($width === $height) {
$resizeWidth = $resizeHeight = $thumbRes;
} elseif($width > $height) {
$resizeWidth = $width * $thumbRes / $height;
$resizeHeight = $thumbRes;
} else {
$resizeWidth = $thumbRes;
$resizeHeight = $height * $thumbRes / $width;
}
$resizeWidth = (int)$resizeWidth;
$resizeHeight = (int)$resizeHeight;
$imagick->resizeImage(
$resizeWidth, $resizeHeight,
Imagick::FILTER_GAUSSIAN, 0.7
);
$imagick->cropImage(
$thumbRes,
$thumbRes,
(int)ceil(($resizeWidth - $thumbRes) / 2),
(int)ceil(($resizeHeight - $thumbRes) / 2)
);
$imagick->writeImage($thumbPath);
return is_file($thumbPath) ? $thumbPath : '';
}
public function deleteUploadData(UploadInfo|string $uploadInfo): void {
$filePath = $this->getFileDataPath($uploadInfo);
if(is_file($filePath))
unlink($filePath);
$thumbPath = $this->getThumbnailDataPath($uploadInfo);
if(is_file($thumbPath))
unlink($thumbPath);
}
}

171
src/Uploads/UploadsData.php
View file

@ -2,33 +2,37 @@
namespace EEPROM\Uploads;
use InvalidArgumentException;
use RuntimeException;
use Index\XString;
use Index\Db\{DbConnection,DbStatementCache};
use EEPROM\SnowflakeGenerator;
use EEPROM\Apps\AppInfo;
use EEPROM\Storage\StorageRecord;
use EEPROM\Users\UserInfo;
class UploadsData {
private DbStatementCache $cache;
public function __construct(DbConnection $dbConn) {
public function __construct(
private DbConnection $dbConn,
private SnowflakeGenerator $snowflake
) {
$this->cache = new DbStatementCache($dbConn);
}
public function getUploads(
?bool $deleted = null,
?bool $expired = null
): iterable {
$hasDeleted = $deleted !== null;
$hasExpired = $expired !== null;
$args = 0;
$query = 'SELECT upload_id, user_id, app_id, LOWER(HEX(upload_hash)), INET6_NTOA(upload_ip), UNIX_TIMESTAMP(upload_created), UNIX_TIMESTAMP(upload_accessed), UNIX_TIMESTAMP(upload_expires), UNIX_TIMESTAMP(upload_deleted), upload_bump, upload_name, upload_type, upload_size FROM prm_uploads';
if($hasDeleted) {
++$args;
$query .= sprintf(' WHERE upload_deleted %s NULL', $deleted ? 'IS NOT' : 'IS');
$query = 'SELECT upload_id, user_id, app_id, upload_secret, upload_name, upload_bump, INET6_NTOA(upload_ip), UNIX_TIMESTAMP(upload_created), UNIX_TIMESTAMP(upload_accessed), UNIX_TIMESTAMP(upload_expires) FROM prm_uploads';
if($hasExpired) {
if($expired)
$query .= sprintf(' %s upload_expires IS NOT NULL AND upload_expires <= NOW()', ++$args > 1 ? 'AND' : 'WHERE');
else
$query .= sprintf(' %s (upload_expires IS NULL OR upload_expires > NOW())', ++$args > 1 ? 'AND' : 'WHERE');
}
if($hasExpired)
$query .= sprintf(' %s upload_expires %s NOW()', ++$args > 1 ? 'AND' : 'WHERE', $expired ? '<=' : '>');
$stmt = $this->cache->get($query);
$stmt->execute();
@ -38,73 +42,106 @@ class UploadsData {
public function getUpload(
?string $uploadId = null,
?string $hashString = null,
AppInfo|string|null $appInfo = null,
UserInfo|string|null $userInfo = null,
): ?UploadInfo {
?string $secret = null,
?bool $expired = null,
): UploadInfo {
$hasUploadId = $uploadId !== null;
$hasHashString = $hashString !== null;
$hasAppInfo = $appInfo !== null;
$hasUserInfo = $userInfo !== null;
$hasSecret = $secret !== null;
$hasExpired = $expired !== null;
$args = 0;
$query = 'SELECT upload_id, user_id, app_id, LOWER(HEX(upload_hash)), INET6_NTOA(upload_ip), UNIX_TIMESTAMP(upload_created), UNIX_TIMESTAMP(upload_accessed), UNIX_TIMESTAMP(upload_expires), UNIX_TIMESTAMP(upload_deleted), upload_bump, upload_name, upload_type, upload_size FROM prm_uploads';
$query = 'SELECT upload_id, user_id, app_id, upload_secret, upload_name, upload_bump, INET6_NTOA(upload_ip), UNIX_TIMESTAMP(upload_created), UNIX_TIMESTAMP(upload_accessed), UNIX_TIMESTAMP(upload_expires) FROM prm_uploads';
if($hasUploadId) {
++$args;
$query .= ' WHERE upload_id = ?';
}
if($hasHashString)
$query .= sprintf(' %s upload_hash = UNHEX(?)', ++$args > 1 ? 'AND' : 'WHERE');
if($hasAppInfo)
$query .= sprintf(' %s app_id = ?', ++$args > 1 ? 'AND' : 'WHERE');
if($hasUserInfo)
$query .= sprintf(' %s user_id = ?', ++$args > 1 ? 'AND' : 'WHERE');
if($hasSecret)
$query .= sprintf(' %s upload_secret = ?', ++$args > 1 ? 'AND' : 'WHERE');
if($hasExpired) {
if($expired)
$query .= sprintf(' %s upload_expires IS NOT NULL AND upload_expires <= NOW()', ++$args > 1 ? 'AND' : 'WHERE');
else
$query .= sprintf(' %s (upload_expires IS NULL OR upload_expires > NOW())', ++$args > 1 ? 'AND' : 'WHERE');
}
$stmt = $this->cache->get($query);
if($hasUploadId)
$stmt->nextParameter($uploadId);
if($hasHashString)
$stmt->nextParameter($hashString);
if($hasAppInfo)
$stmt->nextParameter($appInfo instanceof AppInfo ? $appInfo->id : $appInfo);
if($hasUserInfo)
$stmt->nextParameter($userInfo instanceof UserInfo ? $userInfo->id : $userInfo);
if($hasSecret)
$stmt->nextParameter($secret);
$stmt->execute();
$result = $stmt->getResult();
return $result->next() ? UploadInfo::fromResult($result) : null;
if(!$result->next())
throw new RuntimeException('upload not found');
return UploadInfo::fromResult($result);
}
public function resolveUploadFromVariant(
AppInfo|string $appInfo,
UserInfo|string $userInfo,
StorageRecord|string $fileInfo,
string $variant
): string {
$stmt = $this->cache->get(<<<SQL
SELECT u.upload_id
FROM prm_uploads AS u
LEFT JOIN prm_uploads_files AS uf
ON uf.upload_id = u.upload_id
WHERE u.app_id = ?
AND u.user_id = ?
AND uf.file_id = ?
AND uf.upload_variant = ?
AND (
u.upload_expires IS NULL
OR u.upload_expires > NOW()
)
SQL);
$stmt->nextParameter($appInfo instanceof AppInfo ? $appInfo->id : $appInfo);
$stmt->nextParameter($userInfo instanceof UserInfo ? $userInfo->id : $userInfo);
$stmt->nextParameter($fileInfo instanceof StorageRecord ? $fileInfo->id : $fileInfo);
$stmt->nextParameter($variant);
$stmt->execute();
$result = $stmt->getResult();
if(!$result->next())
throw new RuntimeException('upload could not be resolved');
return $result->getString(0);
}
public function createUpload(
AppInfo|string $appInfo,
UserInfo|string $userInfo,
string $remoteAddr,
string $fileName,
string $fileType,
int $fileSize,
string $fileHash,
string $remoteAddr,
int $fileExpiry,
bool $bumpExpiry
): UploadInfo {
if(strpos($fileType, '/') === false)
throw new InvalidArgumentException('$fileType must contain a /');
if($fileSize < 1)
throw new InvalidArgumentException('$fileSize must be more than 0.');
if(strlen($fileHash) !== 64)
throw new InvalidArgumentException('$fileHash must be 64 characters.');
if($fileExpiry < 0)
throw new InvalidArgumentException('$fileExpiry must be a positive integer.');
$uploadId = XString::random(32);
$uploadId = $this->snowflake->nextRandom();
$stmt = $this->cache->get('INSERT INTO prm_uploads (upload_id, app_id, user_id, upload_name, upload_type, upload_size, upload_hash, upload_ip, upload_expires, upload_bump) VALUES (?, ?, ?, ?, ?, ?, UNHEX(?), INET6_ATON(?), FROM_UNIXTIME(?), ?)');
$stmt = $this->cache->get('INSERT INTO prm_uploads (upload_id, app_id, user_id, upload_secret, upload_name, upload_ip, upload_expires, upload_bump) VALUES (?, ?, ?, ?, ?, INET6_ATON(?), FROM_UNIXTIME(?), ?)');
$stmt->nextParameter($uploadId);
$stmt->nextParameter($appInfo instanceof AppInfo ? $appInfo->id : $appInfo);
$stmt->nextParameter($userInfo instanceof UserInfo ? $userInfo->id : $userInfo);
$stmt->nextParameter(XString::random(4));
$stmt->nextParameter($fileName);
$stmt->nextParameter($fileType);
$stmt->nextParameter($fileSize);
$stmt->nextParameter($fileHash);
$stmt->nextParameter($remoteAddr);
$stmt->nextParameter($fileExpiry > 0 ? (time() + $fileExpiry) : 0);
$stmt->nextParameter($bumpExpiry ? $fileExpiry : 0);
@ -148,21 +185,65 @@ class UploadsData {
$stmt->execute();
}
public function restoreUpload(UploadInfo|string $uploadInfo): void {
$stmt = $this->cache->get('UPDATE prm_uploads SET upload_deleted = NULL WHERE upload_id = ?');
$stmt->nextParameter($uploadInfo instanceof UploadInfo ? $uploadInfo->id : $uploadInfo);
$stmt->execute();
}
public function deleteUpload(UploadInfo|string $uploadInfo): void {
$stmt = $this->cache->get('UPDATE prm_uploads SET upload_deleted = COALESCE(upload_deleted, NOW()) WHERE upload_id = ?');
$stmt->nextParameter($uploadInfo instanceof UploadInfo ? $uploadInfo->id : $uploadInfo);
$stmt->execute();
}
public function nukeUpload(UploadInfo|string $uploadInfo): void {
$stmt = $this->cache->get('DELETE FROM prm_uploads WHERE upload_id = ?');
$stmt->nextParameter($uploadInfo instanceof UploadInfo ? $uploadInfo->id : $uploadInfo);
$stmt->execute();
}
public function deleteExpiredUploads(): void {
$this->dbConn->execute(<<<SQL
DELETE FROM prm_uploads
WHERE upload_expires IS NOT NULL
AND upload_expires <= NOW()
SQL);
}
public function getUploadVariant(
UploadInfo|string $uploadInfo,
string $variant
): UploadVariantInfo {
$stmt = $this->cache->get(<<<SQL
SELECT upload_id, upload_variant, file_id, file_type
FROM prm_uploads_files
WHERE upload_id = ? AND upload_variant = ?
SQL);
$stmt->nextParameter($uploadInfo instanceof UploadInfo ? $uploadInfo->id : $uploadInfo);
$stmt->nextParameter($variant);
$stmt->execute();
$result = $stmt->getResult();
if(!$result->next())
throw new RuntimeException('upload variant not found');
return UploadVariantInfo::fromResult($result);
}
public function createUploadVariant(
UploadInfo|string $uploadInfo,
string $variant,
StorageRecord|string $fileInfo,
?string $fileType = null
): UploadVariantInfo {
$stmt = $this->cache->get(<<<SQL
INSERT INTO prm_uploads_files (
upload_id, upload_variant, file_id, file_type
) VALUES (?, ?, ?, ?)
SQL);
$stmt->nextParameter($uploadInfo instanceof UploadInfo ? $uploadInfo->id : $uploadInfo);
$stmt->nextParameter($variant);
$stmt->nextParameter($fileInfo instanceof StorageRecord ? $fileInfo->id : $fileInfo);
$stmt->nextParameter(!($fileInfo instanceof StorageRecord) || $fileInfo->type !== $fileType ? $fileType : null);
$stmt->execute();
return $this->getUploadVariant($uploadInfo, $variant);
}
public function resolveLegacyId(string $legacyId): ?string {
$stmt = $this->cache->get('SELECT upload_id FROM prm_uploads_legacy WHERE upload_id_legacy = ?');
$stmt->nextParameter($legacyId);
$stmt->execute();
$result = $stmt->getResult();
return $result->next() ? $result->getStringOrNull(0) : null;
}
}

182
src/Uploads/UploadsRoutes.php
View file

@ -2,10 +2,12 @@
namespace EEPROM\Uploads;
use RuntimeException;
use Index\XNumber;
use Index\Http\Routing\{HandlerAttribute,HttpDelete,HttpGet,HttpOptions,HttpPost,Router,RouteHandler};
use EEPROM\Apps\AppsContext;
use EEPROM\Auth\AuthInfo;
use EEPROM\Blacklist\BlacklistContext;
use EEPROM\Storage\{StorageContext,StorageImportMode};
use EEPROM\Uploads\UploadsContext;
class UploadsRoutes implements RouteHandler {
@ -13,6 +15,7 @@ class UploadsRoutes implements RouteHandler {
private AuthInfo $authInfo,
private AppsContext $appsCtx,
private UploadsContext $uploadsCtx,
private StorageContext $storageCtx,
private BlacklistContext $blacklistCtx,
private bool $isApiDomain
) {}
@ -22,12 +25,12 @@ class UploadsRoutes implements RouteHandler {
HandlerAttribute::register($router, $this);
} else {
$router->options('/', $this->getUpload(...));
$router->get('/([A-Za-z0-9\-_]+)(?:\.(t))?', $this->getUpload(...));
$router->get('/([A-Za-z0-9]+|[A-Za-z0-9\-_]{32})(?:\.(t))?', $this->getUpload(...));
}
}
#[HttpOptions('/uploads/([A-Za-z0-9\-_]+)(?:\.(t|json))?')]
#[HttpGet('/uploads/([A-Za-z0-9\-_]+)(?:\.(t|json))?')]
#[HttpOptions('/uploads/([A-Za-z0-9]+|[A-Za-z0-9\-_]{32})(?:\.(t|json))?')]
#[HttpGet('/uploads/([A-Za-z0-9]+|[A-Za-z0-9\-_]{32})(?:\.(t|json))?')]
public function getUpload($response, $request, string $fileId, string $fileExt = '') {
if($this->isApiDomain) {
if($request->hasHeader('Origin'))
@ -50,13 +53,71 @@ class UploadsRoutes implements RouteHandler {
return 404;
$uploadsData = $this->uploadsCtx->uploadsData;
$uploadInfo = $uploadsData->getUpload(uploadId: $fileId);
if($uploadInfo === null) {
if(strlen($fileId) === 32) {
$fileId = $uploadsData->resolveLegacyId($fileId) ?? $fileId;
$fileSecret = null;
} else {
if(strlen($fileId) < 5) {
$response->setContent('File not found.');
return 404;
}
$fileSecret = substr($fileId, -4);
$fileId = XNumber::fromBase62(substr($fileId, 0, -4));
}
try {
$uploadInfo = $uploadsData->getUpload(uploadId: $fileId, expired: false, secret: $fileSecret);
} catch(RuntimeException $ex) {
$response->setContent('File not found.');
return 404;
}
$blInfo = $this->blacklistCtx->getBlacklistEntry($uploadInfo);
if($isThumbnail) {
try {
$originalInfo = $this->storageCtx->getFileRecordFromVariant(
$uploadsData->getUploadVariant($uploadInfo, '')
);
} catch(RuntimeException $ex) {
$response->setContent('Data is missing.');
return 404;
}
if(!$this->storageCtx->supportsThumbnailing($originalInfo))
return 404;
try {
$variantInfo = $uploadsData->getUploadVariant($uploadInfo, 'thumb');
} catch(RuntimeException $ex) {
$storageInfo = $this->storageCtx->createThumbnail($originalInfo);
$uploadsData->createUploadVariant($uploadInfo, 'thumb', $storageInfo);
}
} else {
try {
$variantInfo = $uploadsData->getUploadVariant($uploadInfo, '');
} catch(RuntimeException $ex) {
$response->setContent('Data is missing.');
return 404;
}
if(!$isJson)
$uploadsData->updateUpload(
$uploadInfo,
accessedAt: true,
expiresAt: $uploadInfo->bumpAmountForUpdate,
);
}
if(!isset($storageInfo))
try {
$storageInfo = $this->storageCtx->getFileRecordFromVariant($variantInfo);
} catch(RuntimeException $ex) {
$response->setContent('Data is missing.');
return 404;
}
$blInfo = $this->blacklistCtx->getBlacklistEntry($storageInfo);
if($blInfo !== null) {
$response->setContent(match($blInfo->reason) {
'copyright' => 'File is unavailable for copyright reasons.',
@ -67,45 +128,18 @@ class UploadsRoutes implements RouteHandler {
return $blInfo->isCopyrightTakedown ? 451 : 410;
}
if($uploadInfo->deleted || $uploadInfo->expired) {
$response->setContent('File not found.');
return 404;
}
if($isJson)
return $this->uploadsCtx->convertToClientJsonV1($uploadInfo);
$filePath = $this->uploadsCtx->getFileDataPath($uploadInfo);
if(!is_file($filePath)) {
$response->setContent('Data is missing.');
return 404;
}
if(!$isThumbnail)
$uploadsData->updateUpload(
$uploadInfo,
accessedAt: true,
expiresAt: $uploadInfo->bumpAmountForUpdate,
);
return $this->uploadsCtx->convertToClientJsonV1($uploadInfo, $variantInfo, $storageInfo);
$fileName = $uploadInfo->name;
$contentType = $uploadInfo->mediaTypeString;
if($variantInfo->variant !== '') // FAIRLY SIGNIFICANT TODO: obviously this should support more file exts than .jpg...
$fileName = sprintf('%s-%s.jpg', pathinfo($fileName, PATHINFO_FILENAME), $variantInfo->variant);
$contentType = $variantInfo->type ?? $storageInfo->type;
if($contentType === 'application/octet-stream' || str_starts_with($contentType, 'text/'))
$contentType = 'text/plain';
if($isThumbnail) {
if(!$this->uploadsCtx->supportsThumbnailing($uploadInfo))
return 404;
$contentType = 'image/jpeg';
$accelRedirectPath = $this->uploadsCtx->getThumbnailDataRedirectPathOrCreate($uploadInfo);
$fileName = pathinfo($fileName, PATHINFO_FILENAME) . '-thumb.jpg';
} else {
$accelRedirectPath = $this->uploadsCtx->getFileDataRedirectPath($uploadInfo);
}
$response->accelRedirect($accelRedirectPath);
$response->accelRedirect($this->storageCtx->files->getRemotePath($storageInfo));
$response->setContentType($contentType);
$response->setFileName(addslashes($fileName));
}
@ -160,43 +194,64 @@ class UploadsRoutes implements RouteHandler {
}
$uploadsData = $this->uploadsCtx->uploadsData;
$hash = hash_file('sha256', $localFile);
$hash = hash_file('sha256', $localFile, true);
$blInfo = $this->blacklistCtx->getBlacklistEntry(hex2bin($hash));
$blInfo = $this->blacklistCtx->getBlacklistEntry($hash);
if($blInfo !== null)
return 451;
$fileName = $file->getSuggestedFileName();
$uploadInfo = $uploadsData->getUpload(appInfo: $appInfo, userInfo: $userInfo, hashString: $hash);
if($uploadInfo === null) {
$uploadInfo = $uploadsData->createUpload(
$appInfo, $userInfo, $request->getRemoteAddress(),
$fileName, (string)$file->getSuggestedMediaType(),
$fileSize, $hash, $appInfo->bumpAmount, true
$mediaType = (string)$file->getSuggestedMediaType();
if($mediaType === '/')
$mediaType = mime_content_type($localFile);
try {
$storageInfo = $this->storageCtx->records->getFile($hash);
} catch(RuntimeException $ex) {
$storageInfo = $this->storageCtx->importFile(
$file->getLocalFileName(),
StorageImportMode::Upload,
$mediaType
);
$filePath = $this->uploadsCtx->getFileDataPath($uploadInfo);
$file->moveTo($filePath);
} else {
$filePath = $this->uploadsCtx->getFileDataPath($uploadInfo);
if($uploadInfo->deleted)
$uploadsData->restoreUpload($uploadInfo);
}
try {
// TODO: Okay so we're reading from the prm_uploads_files table, then fetching the info from prm_uploads,
// and then fetching the info again from the prm_uploads_files table... this can be done better but this will do for now
$uploadInfo = $uploadsData->getUpload(
uploadId: $uploadsData->resolveUploadFromVariant($appInfo, $userInfo, $storageInfo, ''),
expired: false
);
$variantInfo = $uploadsData->getUploadVariant($uploadInfo, '');
$uploadsData->updateUpload(
$uploadInfo,
fileName: $fileName,
expiresAt: $uploadInfo->bumpAmountForUpdate,
);
} catch(RuntimeException $ex) {
$uploadInfo = $uploadsData->createUpload(
$appInfo,
$userInfo,
$fileName,
$request->getRemoteAddress(),
$appInfo->bumpAmount,
true
);
$variantInfo = $uploadsData->createUploadVariant(
$uploadInfo, '', $storageInfo, $mediaType
);
}
$response->setStatusCode(201);
$response->setHeader('Content-Type', 'application/json; charset=utf-8');
return $this->uploadsCtx->convertToClientJsonV1($uploadInfo, [
return $this->uploadsCtx->convertToClientJsonV1($uploadInfo, $variantInfo, $storageInfo, [
'name' => $fileName,
]);
}
#[HttpDelete('/uploads/([A-Za-z0-9\-_]+)')]
#[HttpDelete('/uploads/([A-Za-z0-9]+|[A-Za-z0-9\-_]{32})')]
public function deleteUpload($response, $request, string $fileId) {
if($request->hasHeader('Origin'))
$response->setHeader('Access-Control-Allow-Credentials', 'true');
@ -209,13 +264,22 @@ class UploadsRoutes implements RouteHandler {
$uploadsData = $this->uploadsCtx->uploadsData;
$uploadInfo = $uploadsData->getUpload(uploadId: $fileId);
if($uploadInfo === null) {
$response->setContent('File not found.');
return 404;
if(strlen($fileId) === 32) {
$fileId = $uploadsData->resolveLegacyId($fileId) ?? $fileId;
$fileSecret = null;
} else {
if(strlen($fileId) < 5) {
$response->setContent('File not found.');
return 404;
}
$fileSecret = substr($fileId, -4);
$fileId = XNumber::fromBase62(substr($fileId, 0, -4));
}
if($uploadInfo->deleted || $uploadInfo->expired) {
try {
$uploadInfo = $uploadsData->getUpload(uploadId: $fileId, expired: false, secret: $fileSecret);
} catch(RuntimeException $ex) {
$response->setContent('File not found.');
return 404;
}

10
tools/migrate
View file

@ -2,10 +2,12 @@
<?php
require_once __DIR__ . '/../eeprom.php';
try {
touch(PRM_ROOT . '/.migrating');
chmod(PRM_ROOT . '/.migrating', 0777);
$lockPath = $eeprom->database->getMigrateLockPath();
if(is_file($lockPath))
die('A migration script is already running.' . PHP_EOL);
touch($lockPath);
try {
echo 'Creating migration manager...' . PHP_EOL;
$manager = $eeprom->database->createMigrationManager();
@ -28,5 +30,5 @@ try {
echo PHP_EOL;
} finally {
unlink(PRM_ROOT . '/.migrating');
unlink($lockPath);
}

11
tools/migrate-storage
View file

@ -1,11 +0,0 @@
#!/usr/bin/env php
<?php
require_once __DIR__ . '/../eeprom.php';
try {
touch(PRM_ROOT . '/.migrating');
chmod(PRM_ROOT . '/.migrating', 0777);
} finally {
unlink(PRM_ROOT . '/.migrating');
}