cphipps/watchstate
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Add ValidateCommand for validating backend reference IDs

Browse Source
This commit is contained in:
arabcoders
2025-05-09 23:58:14 +03:00
parent 1d8550f309
commit b47194bad4
6 changed files with 440 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
FAQ.md
View File

@@ -420,7 +420,7 @@ These environment variables relates to the tool itself, You should manage them v
> [!IMPORTANT]
> for environment variables that has `{TASK}` tag, you **MUST** replace it with one of `IMPORT`, `EXPORT`, `BACKUP`,
`PRUNE`, `INDEXES`.
`PRUNE`, `INDEXES` or `VALIDATE`.
## Add tool specific environment variables

21
config/config.php
View File

@@ -9,6 +9,7 @@ use App\Commands\Events\DispatchCommand;
use App\Commands\State\BackupCommand;
use App\Commands\State\ExportCommand;
use App\Commands\State\ImportCommand;
use App\Commands\State\ValidateCommand;
use App\Commands\System\IndexCommand;
use App\Commands\System\PruneCommand;
use App\Libs\Mappers\Import\MemoryMapper;
@@ -17,13 +18,13 @@ use Monolog\Level;
return (function () {
$inContainer = inContainer();
$progressTimeCheck = fn (int $v, int $d): int => 0 === $v || $v >= 180 ? $v : $d;
$progressTimeCheck = fn(int $v, int $d): int => 0 === $v || $v >= 180 ? $v : $d;
$config = [
'name' => 'WatchState',
'version' => '$(version_via_ci)',
'tz' => env('WS_TZ', env('TZ', 'UTC')),
'path' => fixPath(env('WS_DATA_PATH', fn () => $inContainer ? '/config' : __DIR__ . '/../var')),
'path' => fixPath(env('WS_DATA_PATH', fn() => $inContainer ? '/config' : __DIR__ . '/../var')),
'logs' => [
'context' => (bool)env('WS_LOGS_CONTEXT', false),
'prune' => [
@@ -44,7 +45,7 @@ return (function () {
'encode' => JSON_INVALID_UTF8_IGNORE | JSON_HEX_TAG | JSON_HEX_APOS | JSON_HEX_AMP | JSON_HEX_QUOT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE,
'headers' => [
'Content-Type' => 'application/json',
'X-Application-Version' => fn () => getAppVersion(),
'X-Application-Version' => fn() => getAppVersion(),
'Access-Control-Allow-Origin' => '*',
],
],
@@ -68,7 +69,7 @@ return (function () {
],
'episodes' => [
'enable' => [
'guid' => (bool)env('WS_EPISODES_ENABLE_GUID', false),
'guid' => (bool)env('WS_EPISODES_ENABLE_GUID', false),
],
],
'ignore' => [],
@@ -159,14 +160,14 @@ return (function () {
$config['profiler'] = [
'save' => (bool)env('WS_PROFILER_SAVE', true),
'path' => env('WS_PROFILER_PATH', fn () => ag($config, 'tmpDir') . '/profiler'),
'path' => env('WS_PROFILER_PATH', fn() => ag($config, 'tmpDir') . '/profiler'),
'collector' => env('WS_PROFILER_COLLECTOR', null),
];
$config['cache'] = [
'prefix' => env('WS_CACHE_PREFIX', null),
'url' => env('WS_CACHE_URL', 'redis://127.0.0.1:6379'),
'path' => env('WS_CACHE_PATH', fn () => ag($config, 'tmpDir') . '/cache'),
'path' => env('WS_CACHE_PATH', fn() => ag($config, 'tmpDir') . '/cache'),
];
$config['logger'] = [
@@ -310,6 +311,14 @@ return (function () {
'timer' => $checkTaskTimer((string)env('WS_CRON_INDEXES_AT', '0 3 * * 3'), '0 3 * * 3'),
'args' => env('WS_CRON_INDEXES_ARGS', '-v'),
],
ValidateCommand::TASK_NAME => [
'command' => ValidateCommand::ROUTE,
'name' => ValidateCommand::TASK_NAME,
'info' => 'Validate stored backends reference id against the backends.',
'enabled' => (bool)env('WS_CRON_VALIDATE', true),
'timer' => $checkTaskTimer((string)env('WS_CRON_VALIDATE_AT', '0 4 */14 * *'), '0 4 */14 * *'),
'args' => env('WS_CRON_VALIDATE_ARGS', '-v'),
],
DispatchCommand::TASK_NAME => [
'command' => DispatchCommand::ROUTE,
'name' => DispatchCommand::TASK_NAME,

2
config/env.spec.php
View File

@@ -258,7 +258,7 @@ return (function () {
};
// -- Do not forget to update the tasks list if you add a new task.
$tasks = ['import', 'export', 'backup', 'prune', 'indexes'];
$tasks = ['import', 'export', 'backup', 'prune', 'indexes', 'validate'];
$task_env = [
[
'key' => 'WS_CRON_{TASK}',

12
frontend/pages/history/[id]/index.vue
View File

@@ -330,8 +330,10 @@
</div>
<div class="card-header-icon">
<div class="field is-grouped">
<div class="control" v-if="false === item?.validated">
<NuxtLink @click="deleteMetadata(key)">
<div class="control"
v-if="false === item?.validated">
<NuxtLink
@click="Object.keys(data.metadata).length > 1 ? deleteMetadata(data,key) : deleteItem(data)">
<span class="icon-text has-text-danger">
<span class="icon"><i class="fas fa-trash"/></span>
<span>Delete</span>
@@ -691,7 +693,7 @@ const deleteItem = async (item) => {
return
}
if (!confirm(`Are you sure you want to delete '${makeName(item)}'?`)) {
if (!confirm(`Delete '${makeName(item)}' local record?`)) {
return
}
@@ -763,8 +765,8 @@ const validateItem = async () => {
} catch (e) {
}
}
const deleteMetadata = async backend => {
if (!confirm(`Remove metadata from '${backend}'?`)) {
const deleteMetadata = async (item, backend) => {
if (!confirm(`Remove '${backend}' metadata from '${makeName(item)}' data?`)) {
return
}

12
src/API/History/Index.php
View File

@@ -767,7 +767,7 @@ final class Index
}
#[Delete(self::URL . '/{id:\d+}/metadata/{backend}[/]', name: 'history.metadata.delete')]
public function delete_item_metadata(iCache $cache, iRequest $request, string $id, string $backend): iResponse
public function delete_item_metadata(iRequest $request, string $id, string $backend): iResponse
{
try {
$userContext = $this->getUserContext(request: $request, mapper: $this->mapper, logger: $this->logger);
@@ -785,10 +785,16 @@ final class Index
return api_error('Item metadata not found.', Status::NOT_FOUND);
}
$item->metadata = ag_delete($item->getMetadata(), $backend);
if (count($item->removeMetadata($backend)) < 1) {
return api_error('Item metadata not found.', Status::NOT_FOUND);
}
if (count($item->getMetadata()) < 1) {
$userContext->db->remove($item);
return api_message('Record deleted.', Status::OK);
}
$userContext->db->update($item);
return $this->read($request, $id);
}

407
src/Commands/State/ValidateCommand.php Normal file
View File

@@ -0,0 +1,407 @@
<?php
declare(strict_types=1);
namespace App\Commands\State;
use App\Command;
use App\Libs\Attributes\DI\Inject;
use App\Libs\Attributes\Route\Cli;
use App\Libs\Entity\StateInterface as iState;
use App\Libs\Extends\StreamLogHandler;
use App\Libs\LogSuppressor;
use App\Libs\Mappers\Import\DirectMapper;
use App\Libs\Mappers\ImportInterface as iImport;
use App\Libs\Stream;
use App\Libs\UserContext;
use Monolog\Level;
use Monolog\Logger;
use Psr\Log\LoggerInterface as iLogger;
use Symfony\Component\Console\Helper\Table;
use Symfony\Component\Console\Helper\TableSeparator;
use Symfony\Component\Console\Input\InputInterface as iInput;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface as iOutput;
use Symfony\Component\Console\Style\SymfonyStyle;
use Throwable;
/**
* Class ValidateCommand
*
* This command Validate local databases against the backends metadata, and validate whether the reference ID
* is still valid and exists in the backend.
*/
#[Cli(command: self::ROUTE)]
class ValidateCommand extends Command
{
public const string ROUTE = 'state:validate';
public const string TASK_NAME = 'validate';
/**
* @var array<array-key,array<string,bool>> Store the status of item from backend in-case we have multiple sub-users.
*/
private array $cache = [];
private const array TO_VERBOSITY = [
Level::Emergency->value => iOutput::VERBOSITY_SILENT,
Level::Critical->value => iOutput::VERBOSITY_QUIET,
Level::Alert->value => iOutput::VERBOSITY_NORMAL,
Level::Error->value => iOutput::VERBOSITY_NORMAL,
Level::Warning->value => iOutput::VERBOSITY_NORMAL,
Level::Notice->value => iOutput::VERBOSITY_VERBOSE,
Level::Info->value => iOutput::VERBOSITY_VERY_VERBOSE,
Level::Debug->value => iOutput::VERBOSITY_DEBUG,
];
private array $perRun = [];
/**
* Class Constructor.
*
* @param iImport $mapper The import interface object.
* @param iLogger $logger The logger interface object.
* @param LogSuppressor $suppressor The log suppressor object.
*
*/
public function __construct(
#[Inject(DirectMapper::class)]
private iImport $mapper,
private iLogger $logger,
private LogSuppressor $suppressor,
) {
set_time_limit(0);
ini_set('memory_limit', '-1');
parent::__construct();
}
/**
* Configure the method.
*/
protected function configure(): void
{
$this->setName(self::ROUTE)
->setDescription('Validate stored backends reference id against the backends.')
->addOption('user', 'u', InputOption::VALUE_REQUIRED, 'Select user. Default all users.')
->addOption(
'logfile',
null,
InputOption::VALUE_REQUIRED,
'Save console output to file. Will not work with progress bar.'
)
->addOption('progress', null, InputOption::VALUE_NONE, 'Show progress bar.');
}
protected function runCommand(iInput $input, iOutput $output): int
{
return $this->single(fn(): int => $this->process($input, $output), $output, [
iLogger::class => $this->logger,
Level::class => Level::Error,
]);
}
/**
* Validate the local databases against the backends reference ID.
*
* @param iInput $input The input interface object.
* @param iOutput $output The output interface object.
*
* @return int The return status code.
*/
protected function process(iInput $input, iOutput $output): int
{
if (null !== ($logfile = $input->getOption('logfile')) && true === ($this->logger instanceof Logger)) {
$this->logger->setHandlers([
$this->suppressor->withHandler(new StreamLogHandler(new Stream($logfile, 'w'), $output))
]);
}
$logIO = null;
$io = null;
if ($input->getOption('progress') && method_exists($output, 'section')) {
$logIO = new SymfonyStyle($input, $output->section());
$io = new SymfonyStyle($input, $output->section());
}
$users = getUsersContext(mapper: $this->mapper, logger: $this->logger);
if (null !== ($user = $input->getOption('user'))) {
$users = array_filter($users, fn($k) => $k === $user, mode: ARRAY_FILTER_USE_KEY);
if (empty($users)) {
$output->writeln(r("<error>User '{user}' not found.</error>", ['user' => $user]));
return self::FAILURE;
}
}
$start_time = microtime(true);
foreach ($users as $userContext) {
$userStart = microtime(true);
$this->output(Level::Notice, "SYSTEM: Validating '{user}' local database metadata reference ids.", [
'user' => $userContext->name,
], $logIO);
$this->validate($userContext, $io, $logIO);
$this->output(Level::Notice, "SYSTEM: Completed '{user}' local database validation in '{duration}'s.", [
'user' => $userContext->name,
'duration' => round(microtime(true) - $userStart, 4),
], $logIO);
}
$this->output(Level::Notice, "SYSTEM: Completed local databases validation in '{duration}'s.", [
'duration' => round(microtime(true) - $start_time, 4),
], $logIO);
$this->renderStatus($output);
return self::SUCCESS;
}
private function validate(
UserContext $userContext,
SymfonyStyle|null $progBar = null,
SymfonyStyle|null $logIO = null
): void {
$clients = [];
foreach ($userContext->config->getAll() as $backend => $config) {
$clients[$backend] = makeBackend($config, $backend, [UserContext::class => $userContext]);
}
$records = $userContext->db->getTotal();
if (null !== $progBar) {
$progBar->progressStart($records);
$progBar->newLine();
}
$this->perRun[$userContext->name] = [
'updated' => 0,
'removed' => 0,
'no_change' => 0,
'backends' => array_map(fn() => ['found' => 0, 'removed' => 0], $userContext->config->getAll()),
];
$ref = &$this->perRun[$userContext->name];
$progressUpdate = 0;
$recordsCount = number_format($records);
foreach ($userContext->db->fetch() as $item) {
try {
if (count($item->getMetadata()) < 1) {
$this->output(
Level::Warning,
"SYSTEM: No metadata found for item '{user}: #{id}' Removing record.",
[
'id' => $item->id,
'user' => $userContext->name,
],
$logIO,
);
$userContext->db->remove($item);
$ref['removed']++;
continue;
}
$meta = $item->getMetadata();
$this->output(
Level::Debug,
"SYSTEM: Validating '{user}: #{id}' - '{title}' reference ID for '{backends}'.",
[
'id' => $item->id,
'user' => $userContext->name,
'title' => $item->getName(),
'backends' => implode(', ', array_keys($meta)),
],
$logIO,
);
foreach ($meta as $backend => $metadata) {
$id = ag($metadata, iState::COLUMN_ID);
$this->output(
Level::Debug,
"SYSTEM: Validating '{user}@{backend}: #{id} - {item_id}' '{title}' reference ID.",
[
'id' => $item->id,
'item_id' => $id,
'user' => $userContext->name,
'title' => $item->getName(),
'backend' => $backend,
],
$logIO,
);
if (null === $id) {
$this->output(
Level::Notice,
"SYSTEM: No reference ID found for item '{user}@{backend}: #{id}' Removing reference ID.",
[
'id' => $item->id,
'backend' => $backend,
'user' => $userContext->name,
],
$logIO,
);
$ref['removed']++;
$item->removeMetadata($backend);
continue;
}
if (null === ($clients[$backend] ?? null)) {
$this->output(
Level::Warning,
"SYSTEM: '{user}: #{id}' has reference to '{backend}' which doesn't exists. Removing reference ID.",
[
'id' => $item->id,
'user' => $userContext->name,
'backend' => $backend,
],
$logIO,
);
$item->removeMetadata($backend);
continue;
}
$sub_ref = &$this->perRun[$userContext->name]['backends'][$backend];
$cacheKey = $clients[$backend]->getContext()->backendUrl . $id;
try {
if (isset($this->cache[$cacheKey])) {
$data = $this->cache[$cacheKey];
} else {
$data = $clients[$backend]->getMetadata($id);
$data = !(count($data) < 1);
$this->cache[$cacheKey] = $data;
}
if (false === $data) {
$this->output(
Level::Notice,
"SYSTEM: Request for '{user}@{backend}: #{id} - {item_id}' didnt return any data. Removing reference ID.",
[
'id' => $item->id,
'item_id' => $id,
'user' => $userContext->name,
'backend' => $backend,
],
$logIO,
);
$sub_ref['removed']++;
$item->removeMetadata($backend);
continue;
}
$sub_ref['found']++;
} catch (Throwable $e) {
$this->output(
Level::Notice,
"SYSTEM: Request for '{user}@{backend}: #{id} - {item_id}'. returned with error. {error}. Removing reference ID.",
[
'id' => $item->id,
'item_id' => $id,
'user' => $userContext->name,
'backend' => $backend,
'error' => $e->getMessage(),
],
$logIO,
);
$sub_ref['removed']++;
$this->cache[$cacheKey] = false;
$item->removeMetadata($backend);
continue;
}
}
if (count($item->metadata) < 1) {
$this->output(
Level::Notice,
"SYSTEM: Item '{user}: #{id}' no longer have any reference ID. Removing record.",
[
'id' => $item->id,
'user' => $userContext->name,
],
$logIO,
);
$ref['removed']++;
$userContext->db->remove($item);
continue;
}
if ($item->diff()) {
$ref['updated']++;
$userContext->db->update($item);
} else {
$ref['no_change']++;
}
} finally {
if (null === $progBar) {
$progressUpdate++;
if (0 === ($progressUpdate % 500)) {
$this->output(Level::Info, "SYSTEM: Processed '{progress}/{total}' %{percent}.", [
'progress' => number_format($progressUpdate),
'total' => $recordsCount,
'percent' => round(($progressUpdate / $records) * 100, 3),
], $logIO);
}
} else {
$progBar->progressAdvance();
}
}
}
if (null !== $progBar) {
$progBar->progressFinish();
$progBar->newLine();
}
}
private function output(Level $level, string $message, array $context = [], SymfonyStyle|null $io = null): void
{
if (null !== $io) {
$io->writeln(r($message, $context), self::TO_VERBOSITY[$level->value] ?? iOutput::VERBOSITY_NORMAL);
return;
}
$this->logger->log($level, $message, $context);
}
private function renderStatus(iOutput $output): void
{
foreach ($this->perRun as $user => $data) {
$this->logger->notice("User '{user}' local database, had {u} updated, {r} removed, {n} no change.", [
'user' => $user,
'u' => $data['updated'],
'r' => $data['removed'],
'n' => $data['no_change'],
]);
$tbl = [];
$total = count($data['backends']);
$i = 0;
foreach ($data['backends'] as $backend => $backendData) {
$i++;
$tbl[] = [
'Backend' => $backend,
'Reference Found' => $backendData['found'],
'Reference Removed' => $backendData['removed'],
];
if ($i < $total) {
$tbl[] = new TableSeparator();
}
}
$output->writeln('');
new Table($output)->setHeaders(array_keys($tbl[0]))->setStyle('box')->setRows(array_values($tbl))->render();
$output->writeln('');
}
}
}