Switch to Index Snowflake implementation.
This commit is contained in:
parent
edca7b49bd
commit
ebb654067d
10 changed files with 29 additions and 69 deletions
6
composer.lock
generated
6
composer.lock
generated
|
@ -116,11 +116,11 @@
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "flashwave/index",
|
"name": "flashwave/index",
|
||||||
"version": "v0.2503.230355",
|
"version": "v0.2503.260138",
|
||||||
"source": {
|
"source": {
|
||||||
"type": "git",
|
"type": "git",
|
||||||
"url": "https://patchii.net/flash/index.git",
|
"url": "https://patchii.net/flash/index.git",
|
||||||
"reference": "2372a113d26380176994f64ab99c42aaf2e9d98e"
|
"reference": "ea549dd0eb7cc7e7348bfcfb0e95da880dd2c039"
|
||||||
},
|
},
|
||||||
"require": {
|
"require": {
|
||||||
"ext-mbstring": "*",
|
"ext-mbstring": "*",
|
||||||
|
@ -169,7 +169,7 @@
|
||||||
],
|
],
|
||||||
"description": "Composer package for the common library for my projects.",
|
"description": "Composer package for the common library for my projects.",
|
||||||
"homepage": "https://railgun.sh/index",
|
"homepage": "https://railgun.sh/index",
|
||||||
"time": "2025-03-23T03:45:47+00:00"
|
"time": "2025-03-26T01:40:42+00:00"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "guzzlehttp/guzzle",
|
"name": "guzzlehttp/guzzle",
|
||||||
|
|
|
@ -19,13 +19,13 @@ final class ConvertUploadIdsToSnowflakes_20241220_043832 implements DbMigration
|
||||||
SQL);
|
SQL);
|
||||||
|
|
||||||
// Generate snowflakes for all existing entries
|
// Generate snowflakes for all existing entries
|
||||||
$snowflaker = $GLOBALS['eeprom']->snowflake;
|
$snowflaker = $GLOBALS['eeprom']->sfBinary;
|
||||||
$stmt = $conn->prepare('UPDATE prm_uploads SET upload_id = ?, upload_secret = ? WHERE upload_id_legacy = ?');
|
$stmt = $conn->prepare('UPDATE prm_uploads SET upload_id = ?, upload_secret = ? WHERE upload_id_legacy = ?');
|
||||||
$result = $conn->query('SELECT upload_id_legacy, UNIX_TIMESTAMP(upload_created) FROM prm_uploads');
|
$result = $conn->query('SELECT upload_id_legacy, UNIX_TIMESTAMP(upload_created) FROM prm_uploads');
|
||||||
while($result->next()) {
|
while($result->next()) {
|
||||||
$legacyId = $result->getString(0);
|
$legacyId = $result->getString(0);
|
||||||
$timestamp = $result->getInteger(1) * 1000;
|
$timestamp = $result->getInteger(1) * 1000;
|
||||||
$snowflake = $snowflaker->from($timestamp, (ord($legacyId[0]) << 8) | ord($legacyId[1]));
|
$snowflake = $snowflaker->next((ord($legacyId[0]) << 8) | ord($legacyId[1]), $timestamp);
|
||||||
|
|
||||||
$uploadPath = realpath(sprintf('%s/%s', self::OLD_UPLOADS, $legacyId));
|
$uploadPath = realpath(sprintf('%s/%s', self::OLD_UPLOADS, $legacyId));
|
||||||
if($uploadPath !== false)
|
if($uploadPath !== false)
|
||||||
|
|
|
@ -4,12 +4,16 @@ namespace EEPROM;
|
||||||
use Index\Dependencies;
|
use Index\Dependencies;
|
||||||
use Index\Config\Config;
|
use Index\Config\Config;
|
||||||
use Index\Db\DbConnection;
|
use Index\Db\DbConnection;
|
||||||
|
use Index\Snowflake\{BinarySnowflake,RandomSnowflake,SnowflakeGenerator};
|
||||||
|
|
||||||
class EEPROMContext {
|
class EEPROMContext {
|
||||||
public private(set) Dependencies $deps;
|
public private(set) Dependencies $deps;
|
||||||
|
|
||||||
public private(set) DatabaseContext $dbCtx;
|
public private(set) DatabaseContext $dbCtx;
|
||||||
|
|
||||||
public private(set) SnowflakeGenerator $snowflake;
|
public private(set) SnowflakeGenerator $snowflake;
|
||||||
|
public private(set) BinarySnowflake $sfBinary;
|
||||||
|
public private(set) RandomSnowflake $sfRandom;
|
||||||
|
|
||||||
public private(set) Auth\AuthContext $authCtx;
|
public private(set) Auth\AuthContext $authCtx;
|
||||||
public private(set) Denylist\DenylistContext $denylistCtx;
|
public private(set) Denylist\DenylistContext $denylistCtx;
|
||||||
|
@ -30,6 +34,8 @@ class EEPROMContext {
|
||||||
$this->deps->register($this->dbCtx->conn);
|
$this->deps->register($this->dbCtx->conn);
|
||||||
|
|
||||||
$this->deps->register($this->snowflake = $this->deps->constructLazy(SnowflakeGenerator::class));
|
$this->deps->register($this->snowflake = $this->deps->constructLazy(SnowflakeGenerator::class));
|
||||||
|
$this->deps->register($this->sfBinary = $this->deps->constructLazy(BinarySnowflake::class));
|
||||||
|
$this->deps->register($this->sfRandom = $this->deps->constructLazy(RandomSnowflake::class));
|
||||||
|
|
||||||
$this->deps->register($this->authCtx = $this->deps->constructLazy(Auth\AuthContext::class, $config->scopeTo('apii')));
|
$this->deps->register($this->authCtx = $this->deps->constructLazy(Auth\AuthContext::class, $config->scopeTo('apii')));
|
||||||
$this->deps->register($this->denylistCtx = $this->deps->constructLazy(Denylist\DenylistContext::class));
|
$this->deps->register($this->denylistCtx = $this->deps->constructLazy(Denylist\DenylistContext::class));
|
||||||
|
|
|
@ -1,47 +0,0 @@
|
||||||
<?php
|
|
||||||
namespace EEPROM;
|
|
||||||
|
|
||||||
class SnowflakeGenerator {
|
|
||||||
public const int EPOCH = 1356998400000;
|
|
||||||
|
|
||||||
private const int TS_MASK = 0x7FFFFFFFFFFF;
|
|
||||||
private const int TS_SHIFT = 16;
|
|
||||||
private const int FP_MASK = 0xFFFF;
|
|
||||||
private const int FP_BYTES = 2;
|
|
||||||
|
|
||||||
private int $counter = -1;
|
|
||||||
|
|
||||||
public function __construct(
|
|
||||||
private int $epoch = self::EPOCH
|
|
||||||
) {}
|
|
||||||
|
|
||||||
public static function now(): int {
|
|
||||||
return (int)date_create()->format('Uv');
|
|
||||||
}
|
|
||||||
|
|
||||||
public function from(int $timestamp, int $snowflake): int {
|
|
||||||
return ($snowflake & self::FP_MASK)
|
|
||||||
| ((($timestamp - $this->epoch) & self::TS_MASK) << self::TS_SHIFT);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function next(int $snowflake): int {
|
|
||||||
return $this->from(self::now(), $snowflake);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function nextRandom(): int {
|
|
||||||
return $this->next(random_int(0, self::FP_MASK));
|
|
||||||
}
|
|
||||||
|
|
||||||
public function nextIncrement(): int {
|
|
||||||
if($this->counter >= self::FP_MASK)
|
|
||||||
$this->counter = -1;
|
|
||||||
|
|
||||||
return $this->next(++$this->counter);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function nextHash(string $hash): int {
|
|
||||||
return $this->next(
|
|
||||||
hexdec(bin2hex(substr(str_pad($hash, self::FP_BYTES, "\0"), 0, self::FP_BYTES)))
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -5,11 +5,11 @@ use Imagick;
|
||||||
use ImagickException;
|
use ImagickException;
|
||||||
use InvalidArgumentException;
|
use InvalidArgumentException;
|
||||||
use RuntimeException;
|
use RuntimeException;
|
||||||
use EEPROM\SnowflakeGenerator;
|
|
||||||
use EEPROM\Pools\Rules\EnsureVariantRuleThumb;
|
use EEPROM\Pools\Rules\EnsureVariantRuleThumb;
|
||||||
use EEPROM\Uploads\UploadVariantInfo;
|
use EEPROM\Uploads\UploadVariantInfo;
|
||||||
use Index\Config\Config;
|
use Index\Config\Config;
|
||||||
use Index\Db\DbConnection;
|
use Index\Db\DbConnection;
|
||||||
|
use Index\Snowflake\BinarySnowflake;
|
||||||
|
|
||||||
class StorageContext {
|
class StorageContext {
|
||||||
public private(set) StorageRecords $records;
|
public private(set) StorageRecords $records;
|
||||||
|
@ -18,7 +18,7 @@ class StorageContext {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
Config $config,
|
Config $config,
|
||||||
DbConnection $dbConn,
|
DbConnection $dbConn,
|
||||||
SnowflakeGenerator $snowflake
|
BinarySnowflake $snowflake
|
||||||
) {
|
) {
|
||||||
$this->files = new StorageFiles(
|
$this->files = new StorageFiles(
|
||||||
$config->getString('local', PRM_ROOT . '/storage'),
|
$config->getString('local', PRM_ROOT . '/storage'),
|
||||||
|
|
|
@ -3,15 +3,15 @@ namespace EEPROM\Storage;
|
||||||
|
|
||||||
use InvalidArgumentException;
|
use InvalidArgumentException;
|
||||||
use RuntimeException;
|
use RuntimeException;
|
||||||
use EEPROM\SnowflakeGenerator;
|
|
||||||
use Index\Db\{DbConnection,DbStatementCache};
|
use Index\Db\{DbConnection,DbStatementCache};
|
||||||
|
use Index\Snowflake\BinarySnowflake;
|
||||||
|
|
||||||
class StorageRecords {
|
class StorageRecords {
|
||||||
private DbStatementCache $cache;
|
private DbStatementCache $cache;
|
||||||
|
|
||||||
public function __construct(
|
public function __construct(
|
||||||
DbConnection $dbConn,
|
DbConnection $dbConn,
|
||||||
private SnowflakeGenerator $snowflake
|
private BinarySnowflake $snowflake
|
||||||
) {
|
) {
|
||||||
$this->cache = new DbStatementCache($dbConn);
|
$this->cache = new DbStatementCache($dbConn);
|
||||||
}
|
}
|
||||||
|
@ -93,7 +93,7 @@ class StorageRecords {
|
||||||
if($size < 0)
|
if($size < 0)
|
||||||
throw new InvalidArgumentException('$size may not be negative');
|
throw new InvalidArgumentException('$size may not be negative');
|
||||||
|
|
||||||
$fileId = (string)$this->snowflake->nextHash($hash);
|
$fileId = (string)$this->snowflake->next($hash);
|
||||||
|
|
||||||
$stmt = $this->cache->get(<<<SQL
|
$stmt = $this->cache->get(<<<SQL
|
||||||
INSERT INTO prm_files (
|
INSERT INTO prm_files (
|
||||||
|
|
|
@ -1,9 +1,9 @@
|
||||||
<?php
|
<?php
|
||||||
namespace EEPROM\Tasks;
|
namespace EEPROM\Tasks;
|
||||||
|
|
||||||
use EEPROM\SnowflakeGenerator;
|
|
||||||
use Index\Config\Config;
|
use Index\Config\Config;
|
||||||
use Index\Db\DbConnection;
|
use Index\Db\DbConnection;
|
||||||
|
use Index\Snowflake\RandomSnowflake;
|
||||||
|
|
||||||
class TasksContext {
|
class TasksContext {
|
||||||
public const int CHUNK_SIZE = 4 * 1024 * 1024;
|
public const int CHUNK_SIZE = 4 * 1024 * 1024;
|
||||||
|
@ -13,7 +13,7 @@ class TasksContext {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private Config $config,
|
private Config $config,
|
||||||
DbConnection $dbConn,
|
DbConnection $dbConn,
|
||||||
SnowflakeGenerator $snowflake
|
RandomSnowflake $snowflake
|
||||||
) {
|
) {
|
||||||
$this->tasks = new TasksData($dbConn, $snowflake);
|
$this->tasks = new TasksData($dbConn, $snowflake);
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,17 +4,17 @@ namespace EEPROM\Tasks;
|
||||||
use InvalidArgumentException;
|
use InvalidArgumentException;
|
||||||
use RuntimeException;
|
use RuntimeException;
|
||||||
use Stringable;
|
use Stringable;
|
||||||
use EEPROM\SnowflakeGenerator;
|
|
||||||
use EEPROM\Pools\PoolInfo;
|
use EEPROM\Pools\PoolInfo;
|
||||||
use Index\XString;
|
use Index\XString;
|
||||||
use Index\Db\{DbConnection,DbStatementCache,DbTools};
|
use Index\Db\{DbConnection,DbStatementCache,DbTools};
|
||||||
|
use Index\Snowflake\RandomSnowflake;
|
||||||
|
|
||||||
class TasksData {
|
class TasksData {
|
||||||
private DbStatementCache $cache;
|
private DbStatementCache $cache;
|
||||||
|
|
||||||
public function __construct(
|
public function __construct(
|
||||||
DbConnection $dbConn,
|
DbConnection $dbConn,
|
||||||
private SnowflakeGenerator $snowflake
|
private RandomSnowflake $snowflake
|
||||||
) {
|
) {
|
||||||
$this->cache = new DbStatementCache($dbConn);
|
$this->cache = new DbStatementCache($dbConn);
|
||||||
}
|
}
|
||||||
|
@ -80,7 +80,7 @@ class TasksData {
|
||||||
string $type,
|
string $type,
|
||||||
string $hash
|
string $hash
|
||||||
): TaskInfo {
|
): TaskInfo {
|
||||||
$taskId = (string)$this->snowflake->nextRandom();
|
$taskId = (string)$this->snowflake->next();
|
||||||
|
|
||||||
$stmt = $this->cache->get(<<<SQL
|
$stmt = $this->cache->get(<<<SQL
|
||||||
INSERT INTO prm_tasks (
|
INSERT INTO prm_tasks (
|
||||||
|
|
|
@ -2,12 +2,13 @@
|
||||||
namespace EEPROM\Uploads;
|
namespace EEPROM\Uploads;
|
||||||
|
|
||||||
use DateTimeInterface;
|
use DateTimeInterface;
|
||||||
use Index\Config\Config;
|
use EEPROM\FFMPEG;
|
||||||
use Index\Db\DbConnection;
|
|
||||||
use EEPROM\{FFMPEG,SnowflakeGenerator};
|
|
||||||
use EEPROM\Pools\PoolsContext;
|
use EEPROM\Pools\PoolsContext;
|
||||||
use EEPROM\Storage\{StorageContext,StorageRecord};
|
use EEPROM\Storage\{StorageContext,StorageRecord};
|
||||||
use EEPROM\Tasks\TaskInfo;
|
use EEPROM\Tasks\TaskInfo;
|
||||||
|
use Index\Config\Config;
|
||||||
|
use Index\Db\DbConnection;
|
||||||
|
use Index\Snowflake\RandomSnowflake;
|
||||||
|
|
||||||
class UploadsContext {
|
class UploadsContext {
|
||||||
public private(set) UploadsData $uploads;
|
public private(set) UploadsData $uploads;
|
||||||
|
@ -15,7 +16,7 @@ class UploadsContext {
|
||||||
public function __construct(
|
public function __construct(
|
||||||
private Config $config,
|
private Config $config,
|
||||||
DbConnection $dbConn,
|
DbConnection $dbConn,
|
||||||
SnowflakeGenerator $snowflake,
|
RandomSnowflake $snowflake,
|
||||||
private PoolsContext $poolsCtx,
|
private PoolsContext $poolsCtx,
|
||||||
private StorageContext $storageCtx
|
private StorageContext $storageCtx
|
||||||
) {
|
) {
|
||||||
|
|
|
@ -3,18 +3,18 @@ namespace EEPROM\Uploads;
|
||||||
|
|
||||||
use InvalidArgumentException;
|
use InvalidArgumentException;
|
||||||
use RuntimeException;
|
use RuntimeException;
|
||||||
use EEPROM\SnowflakeGenerator;
|
|
||||||
use EEPROM\Pools\PoolInfo;
|
use EEPROM\Pools\PoolInfo;
|
||||||
use EEPROM\Storage\StorageRecord;
|
use EEPROM\Storage\StorageRecord;
|
||||||
use Index\XString;
|
use Index\XString;
|
||||||
use Index\Db\{DbConnection,DbStatementCache,DbTools};
|
use Index\Db\{DbConnection,DbStatementCache,DbTools};
|
||||||
|
use Index\Snowflake\RandomSnowflake;
|
||||||
|
|
||||||
class UploadsData {
|
class UploadsData {
|
||||||
private DbStatementCache $cache;
|
private DbStatementCache $cache;
|
||||||
|
|
||||||
public function __construct(
|
public function __construct(
|
||||||
DbConnection $dbConn,
|
DbConnection $dbConn,
|
||||||
private SnowflakeGenerator $snowflake
|
private RandomSnowflake $snowflake
|
||||||
) {
|
) {
|
||||||
$this->cache = new DbStatementCache($dbConn);
|
$this->cache = new DbStatementCache($dbConn);
|
||||||
}
|
}
|
||||||
|
@ -168,7 +168,7 @@ class UploadsData {
|
||||||
?string $secret = null,
|
?string $secret = null,
|
||||||
?int $createdTime = null
|
?int $createdTime = null
|
||||||
): UploadInfo {
|
): UploadInfo {
|
||||||
$uploadId ??= (string)$this->snowflake->nextRandom();
|
$uploadId ??= (string)$this->snowflake->next();
|
||||||
$secret ??= XString::random(4);
|
$secret ??= XString::random(4);
|
||||||
|
|
||||||
$stmt = $this->cache->get('INSERT INTO prm_uploads (upload_id, pool_id, user_id, upload_secret, upload_name, upload_ip, upload_created) VALUES (?, ?, ?, ?, ?, INET6_ATON(?), FROM_UNIXTIME(?))');
|
$stmt = $this->cache->get('INSERT INTO prm_uploads (upload_id, pool_id, user_id, upload_secret, upload_name, upload_ip, upload_created) VALUES (?, ?, ?, ?, ?, INET6_ATON(?), FROM_UNIXTIME(?))');
|
||||||
|
|
Reference in a new issue