flashii/eeprom
Archived
3
0
Fork 1

Switch to Index Snowflake implementation.

This commit is contained in:
flash 2025-03-26 01:51:51 +00:00
parent edca7b49bd
commit ebb654067d
Signed by: flash
GPG key ID: 2C9C2C574D47FE3E
10 changed files with 29 additions and 69 deletions

6
composer.lock generated
View file

@ -116,11 +116,11 @@
}, },
{ {
"name": "flashwave/index", "name": "flashwave/index",
"version": "v0.2503.230355", "version": "v0.2503.260138",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://patchii.net/flash/index.git", "url": "https://patchii.net/flash/index.git",
"reference": "2372a113d26380176994f64ab99c42aaf2e9d98e" "reference": "ea549dd0eb7cc7e7348bfcfb0e95da880dd2c039"
}, },
"require": { "require": {
"ext-mbstring": "*", "ext-mbstring": "*",
@ -169,7 +169,7 @@
], ],
"description": "Composer package for the common library for my projects.", "description": "Composer package for the common library for my projects.",
"homepage": "https://railgun.sh/index", "homepage": "https://railgun.sh/index",
"time": "2025-03-23T03:45:47+00:00" "time": "2025-03-26T01:40:42+00:00"
}, },
{ {
"name": "guzzlehttp/guzzle", "name": "guzzlehttp/guzzle",

View file

@ -19,13 +19,13 @@ final class ConvertUploadIdsToSnowflakes_20241220_043832 implements DbMigration
SQL); SQL);
// Generate snowflakes for all existing entries // Generate snowflakes for all existing entries
$snowflaker = $GLOBALS['eeprom']->snowflake; $snowflaker = $GLOBALS['eeprom']->sfBinary;
$stmt = $conn->prepare('UPDATE prm_uploads SET upload_id = ?, upload_secret = ? WHERE upload_id_legacy = ?'); $stmt = $conn->prepare('UPDATE prm_uploads SET upload_id = ?, upload_secret = ? WHERE upload_id_legacy = ?');
$result = $conn->query('SELECT upload_id_legacy, UNIX_TIMESTAMP(upload_created) FROM prm_uploads'); $result = $conn->query('SELECT upload_id_legacy, UNIX_TIMESTAMP(upload_created) FROM prm_uploads');
while($result->next()) { while($result->next()) {
$legacyId = $result->getString(0); $legacyId = $result->getString(0);
$timestamp = $result->getInteger(1) * 1000; $timestamp = $result->getInteger(1) * 1000;
$snowflake = $snowflaker->from($timestamp, (ord($legacyId[0]) << 8) | ord($legacyId[1])); $snowflake = $snowflaker->next((ord($legacyId[0]) << 8) | ord($legacyId[1]), $timestamp);
$uploadPath = realpath(sprintf('%s/%s', self::OLD_UPLOADS, $legacyId)); $uploadPath = realpath(sprintf('%s/%s', self::OLD_UPLOADS, $legacyId));
if($uploadPath !== false) if($uploadPath !== false)

View file

@ -4,12 +4,16 @@ namespace EEPROM;
use Index\Dependencies; use Index\Dependencies;
use Index\Config\Config; use Index\Config\Config;
use Index\Db\DbConnection; use Index\Db\DbConnection;
use Index\Snowflake\{BinarySnowflake,RandomSnowflake,SnowflakeGenerator};
class EEPROMContext { class EEPROMContext {
public private(set) Dependencies $deps; public private(set) Dependencies $deps;
public private(set) DatabaseContext $dbCtx; public private(set) DatabaseContext $dbCtx;
public private(set) SnowflakeGenerator $snowflake; public private(set) SnowflakeGenerator $snowflake;
public private(set) BinarySnowflake $sfBinary;
public private(set) RandomSnowflake $sfRandom;
public private(set) Auth\AuthContext $authCtx; public private(set) Auth\AuthContext $authCtx;
public private(set) Denylist\DenylistContext $denylistCtx; public private(set) Denylist\DenylistContext $denylistCtx;
@ -30,6 +34,8 @@ class EEPROMContext {
$this->deps->register($this->dbCtx->conn); $this->deps->register($this->dbCtx->conn);
$this->deps->register($this->snowflake = $this->deps->constructLazy(SnowflakeGenerator::class)); $this->deps->register($this->snowflake = $this->deps->constructLazy(SnowflakeGenerator::class));
$this->deps->register($this->sfBinary = $this->deps->constructLazy(BinarySnowflake::class));
$this->deps->register($this->sfRandom = $this->deps->constructLazy(RandomSnowflake::class));
$this->deps->register($this->authCtx = $this->deps->constructLazy(Auth\AuthContext::class, $config->scopeTo('apii'))); $this->deps->register($this->authCtx = $this->deps->constructLazy(Auth\AuthContext::class, $config->scopeTo('apii')));
$this->deps->register($this->denylistCtx = $this->deps->constructLazy(Denylist\DenylistContext::class)); $this->deps->register($this->denylistCtx = $this->deps->constructLazy(Denylist\DenylistContext::class));

View file

@ -1,47 +0,0 @@
<?php
namespace EEPROM;
class SnowflakeGenerator {
public const int EPOCH = 1356998400000;
private const int TS_MASK = 0x7FFFFFFFFFFF;
private const int TS_SHIFT = 16;
private const int FP_MASK = 0xFFFF;
private const int FP_BYTES = 2;
private int $counter = -1;
public function __construct(
private int $epoch = self::EPOCH
) {}
public static function now(): int {
return (int)date_create()->format('Uv');
}
public function from(int $timestamp, int $snowflake): int {
return ($snowflake & self::FP_MASK)
| ((($timestamp - $this->epoch) & self::TS_MASK) << self::TS_SHIFT);
}
public function next(int $snowflake): int {
return $this->from(self::now(), $snowflake);
}
public function nextRandom(): int {
return $this->next(random_int(0, self::FP_MASK));
}
public function nextIncrement(): int {
if($this->counter >= self::FP_MASK)
$this->counter = -1;
return $this->next(++$this->counter);
}
public function nextHash(string $hash): int {
return $this->next(
hexdec(bin2hex(substr(str_pad($hash, self::FP_BYTES, "\0"), 0, self::FP_BYTES)))
);
}
}

View file

@ -5,11 +5,11 @@ use Imagick;
use ImagickException; use ImagickException;
use InvalidArgumentException; use InvalidArgumentException;
use RuntimeException; use RuntimeException;
use EEPROM\SnowflakeGenerator;
use EEPROM\Pools\Rules\EnsureVariantRuleThumb; use EEPROM\Pools\Rules\EnsureVariantRuleThumb;
use EEPROM\Uploads\UploadVariantInfo; use EEPROM\Uploads\UploadVariantInfo;
use Index\Config\Config; use Index\Config\Config;
use Index\Db\DbConnection; use Index\Db\DbConnection;
use Index\Snowflake\BinarySnowflake;
class StorageContext { class StorageContext {
public private(set) StorageRecords $records; public private(set) StorageRecords $records;
@ -18,7 +18,7 @@ class StorageContext {
public function __construct( public function __construct(
Config $config, Config $config,
DbConnection $dbConn, DbConnection $dbConn,
SnowflakeGenerator $snowflake BinarySnowflake $snowflake
) { ) {
$this->files = new StorageFiles( $this->files = new StorageFiles(
$config->getString('local', PRM_ROOT . '/storage'), $config->getString('local', PRM_ROOT . '/storage'),

View file

@ -3,15 +3,15 @@ namespace EEPROM\Storage;
use InvalidArgumentException; use InvalidArgumentException;
use RuntimeException; use RuntimeException;
use EEPROM\SnowflakeGenerator;
use Index\Db\{DbConnection,DbStatementCache}; use Index\Db\{DbConnection,DbStatementCache};
use Index\Snowflake\BinarySnowflake;
class StorageRecords { class StorageRecords {
private DbStatementCache $cache; private DbStatementCache $cache;
public function __construct( public function __construct(
DbConnection $dbConn, DbConnection $dbConn,
private SnowflakeGenerator $snowflake private BinarySnowflake $snowflake
) { ) {
$this->cache = new DbStatementCache($dbConn); $this->cache = new DbStatementCache($dbConn);
} }
@ -93,7 +93,7 @@ class StorageRecords {
if($size < 0) if($size < 0)
throw new InvalidArgumentException('$size may not be negative'); throw new InvalidArgumentException('$size may not be negative');
$fileId = (string)$this->snowflake->nextHash($hash); $fileId = (string)$this->snowflake->next($hash);
$stmt = $this->cache->get(<<<SQL $stmt = $this->cache->get(<<<SQL
INSERT INTO prm_files ( INSERT INTO prm_files (

View file

@ -1,9 +1,9 @@
<?php <?php
namespace EEPROM\Tasks; namespace EEPROM\Tasks;
use EEPROM\SnowflakeGenerator;
use Index\Config\Config; use Index\Config\Config;
use Index\Db\DbConnection; use Index\Db\DbConnection;
use Index\Snowflake\RandomSnowflake;
class TasksContext { class TasksContext {
public const int CHUNK_SIZE = 4 * 1024 * 1024; public const int CHUNK_SIZE = 4 * 1024 * 1024;
@ -13,7 +13,7 @@ class TasksContext {
public function __construct( public function __construct(
private Config $config, private Config $config,
DbConnection $dbConn, DbConnection $dbConn,
SnowflakeGenerator $snowflake RandomSnowflake $snowflake
) { ) {
$this->tasks = new TasksData($dbConn, $snowflake); $this->tasks = new TasksData($dbConn, $snowflake);
} }

View file

@ -4,17 +4,17 @@ namespace EEPROM\Tasks;
use InvalidArgumentException; use InvalidArgumentException;
use RuntimeException; use RuntimeException;
use Stringable; use Stringable;
use EEPROM\SnowflakeGenerator;
use EEPROM\Pools\PoolInfo; use EEPROM\Pools\PoolInfo;
use Index\XString; use Index\XString;
use Index\Db\{DbConnection,DbStatementCache,DbTools}; use Index\Db\{DbConnection,DbStatementCache,DbTools};
use Index\Snowflake\RandomSnowflake;
class TasksData { class TasksData {
private DbStatementCache $cache; private DbStatementCache $cache;
public function __construct( public function __construct(
DbConnection $dbConn, DbConnection $dbConn,
private SnowflakeGenerator $snowflake private RandomSnowflake $snowflake
) { ) {
$this->cache = new DbStatementCache($dbConn); $this->cache = new DbStatementCache($dbConn);
} }
@ -80,7 +80,7 @@ class TasksData {
string $type, string $type,
string $hash string $hash
): TaskInfo { ): TaskInfo {
$taskId = (string)$this->snowflake->nextRandom(); $taskId = (string)$this->snowflake->next();
$stmt = $this->cache->get(<<<SQL $stmt = $this->cache->get(<<<SQL
INSERT INTO prm_tasks ( INSERT INTO prm_tasks (

View file

@ -2,12 +2,13 @@
namespace EEPROM\Uploads; namespace EEPROM\Uploads;
use DateTimeInterface; use DateTimeInterface;
use Index\Config\Config; use EEPROM\FFMPEG;
use Index\Db\DbConnection;
use EEPROM\{FFMPEG,SnowflakeGenerator};
use EEPROM\Pools\PoolsContext; use EEPROM\Pools\PoolsContext;
use EEPROM\Storage\{StorageContext,StorageRecord}; use EEPROM\Storage\{StorageContext,StorageRecord};
use EEPROM\Tasks\TaskInfo; use EEPROM\Tasks\TaskInfo;
use Index\Config\Config;
use Index\Db\DbConnection;
use Index\Snowflake\RandomSnowflake;
class UploadsContext { class UploadsContext {
public private(set) UploadsData $uploads; public private(set) UploadsData $uploads;
@ -15,7 +16,7 @@ class UploadsContext {
public function __construct( public function __construct(
private Config $config, private Config $config,
DbConnection $dbConn, DbConnection $dbConn,
SnowflakeGenerator $snowflake, RandomSnowflake $snowflake,
private PoolsContext $poolsCtx, private PoolsContext $poolsCtx,
private StorageContext $storageCtx private StorageContext $storageCtx
) { ) {

View file

@ -3,18 +3,18 @@ namespace EEPROM\Uploads;
use InvalidArgumentException; use InvalidArgumentException;
use RuntimeException; use RuntimeException;
use EEPROM\SnowflakeGenerator;
use EEPROM\Pools\PoolInfo; use EEPROM\Pools\PoolInfo;
use EEPROM\Storage\StorageRecord; use EEPROM\Storage\StorageRecord;
use Index\XString; use Index\XString;
use Index\Db\{DbConnection,DbStatementCache,DbTools}; use Index\Db\{DbConnection,DbStatementCache,DbTools};
use Index\Snowflake\RandomSnowflake;
class UploadsData { class UploadsData {
private DbStatementCache $cache; private DbStatementCache $cache;
public function __construct( public function __construct(
DbConnection $dbConn, DbConnection $dbConn,
private SnowflakeGenerator $snowflake private RandomSnowflake $snowflake
) { ) {
$this->cache = new DbStatementCache($dbConn); $this->cache = new DbStatementCache($dbConn);
} }
@ -168,7 +168,7 @@ class UploadsData {
?string $secret = null, ?string $secret = null,
?int $createdTime = null ?int $createdTime = null
): UploadInfo { ): UploadInfo {
$uploadId ??= (string)$this->snowflake->nextRandom(); $uploadId ??= (string)$this->snowflake->next();
$secret ??= XString::random(4); $secret ??= XString::random(4);
$stmt = $this->cache->get('INSERT INTO prm_uploads (upload_id, pool_id, user_id, upload_secret, upload_name, upload_ip, upload_created) VALUES (?, ?, ?, ?, ?, INET6_ATON(?), FROM_UNIXTIME(?))'); $stmt = $this->cache->get('INSERT INTO prm_uploads (upload_id, pool_id, user_id, upload_secret, upload_name, upload_ip, upload_created) VALUES (?, ?, ?, ?, ?, INET6_ATON(?), FROM_UNIXTIME(?))');