From dd7e79f5152ffaee2b76fbc3fffa044b0d3b467d Mon Sep 17 00:00:00 2001 From: "Abdulmhsen B. A. A" Date: Mon, 27 Jun 2022 17:11:50 +0300 Subject: [PATCH 01/11] updated last remaining exception catchers without trace support. --- src/Backends/Jellyfin/Action/Import.php | 1 + src/Backends/Plex/Action/Import.php | 1 + src/Commands/State/PushCommand.php | 1 + src/Libs/Initializer.php | 2 ++ src/Libs/Storage/PDO/PDOAdapter.php | 44 +++++++++++++++++++++---- 5 files changed, 43 insertions(+), 6 deletions(-) diff --git a/src/Backends/Jellyfin/Action/Import.php b/src/Backends/Jellyfin/Action/Import.php index 53a13505..4b9d5cd0 100644 --- a/src/Backends/Jellyfin/Action/Import.php +++ b/src/Backends/Jellyfin/Action/Import.php @@ -139,6 +139,7 @@ class Import return []; } catch (JsonException $e) { $this->logger->error('Request for [%(backend)] libraries returned with invalid body.', [ + 'backend' => $context->backendName, 'exception' => [ 'file' => $e->getFile(), 'line' => $e->getLine(), diff --git a/src/Backends/Plex/Action/Import.php b/src/Backends/Plex/Action/Import.php index 5c297ca4..d462bc38 100644 --- a/src/Backends/Plex/Action/Import.php +++ b/src/Backends/Plex/Action/Import.php @@ -140,6 +140,7 @@ class Import return []; } catch (JsonException $e) { $this->logger->error('Request for [%(backend)] libraries returned with invalid body.', [ + 'backend' => $context->backendName, 'exception' => [ 'file' => $e->getFile(), 'line' => $e->getLine(), diff --git a/src/Commands/State/PushCommand.php b/src/Commands/State/PushCommand.php index 74f79405..f25ed7c0 100644 --- a/src/Commands/State/PushCommand.php +++ b/src/Commands/State/PushCommand.php @@ -199,6 +199,7 @@ class PushCommand extends Command 'line' => $e->getLine(), 'kind' => get_class($e), 'message' => $e->getMessage(), + 'trace' => $e->getTrace(), ], ] ); diff --git a/src/Libs/Initializer.php b/src/Libs/Initializer.php index f9a7ed58..db8e019a 100644 --- a/src/Libs/Initializer.php +++ b/src/Libs/Initializer.php @@ -157,6 +157,7 @@ final class Initializer 'file' => $e->getFile(), 'line' => $e->getLine(), 'kind' => get_class($e), + 'trace' => $e->getTrace(), ] ); $response = new Response(500); @@ -206,6 +207,7 @@ final class Initializer 'line' => $e->getLine(), 'kind' => get_class($e), 'message' => $e->getMessage(), + 'trace' => $e->getTrace(), ] ]); continue; diff --git a/src/Libs/Storage/PDO/PDOAdapter.php b/src/Libs/Storage/PDO/PDOAdapter.php index 1ec26a93..50bee045 100644 --- a/src/Libs/Storage/PDO/PDOAdapter.php +++ b/src/Libs/Storage/PDO/PDOAdapter.php @@ -95,7 +95,15 @@ final class PDOAdapter implements StorageInterface } catch (PDOException $e) { $this->stmt['insert'] = null; if (false === $this->viaTransaction && false === $this->singleTransaction) { - $this->logger->error($e->getMessage(), $entity->getAll()); + $this->logger->error($e->getMessage(), [ + 'entity' => $entity->getAll(), + 'exception' => [ + 'file' => $e->getFile(), + 'line' => $e->getLine(), + 'message' => $e->getMessage(), + 'trace' => $e->getTrace(), + ], + ]); return $entity; } throw $e; @@ -241,7 +249,15 @@ final class PDOAdapter implements StorageInterface } catch (PDOException $e) { $this->stmt['update'] = null; if (false === $this->viaTransaction && false === $this->singleTransaction) { - $this->logger->error($e->getMessage(), $entity->getAll()); + $this->logger->error($e->getMessage(), [ + 'entity' => $entity->getAll(), + 'exception' => [ + 'file' => $e->getFile(), + 'line' => $e->getLine(), + 'message' => $e->getMessage(), + 'trace' => $e->getTrace(), + ] + ]); return $entity; } throw $e; @@ -268,7 +284,15 @@ final class PDOAdapter implements StorageInterface $this->query(sprintf('DELETE FROM state WHERE %s = %d', iFace::COLUMN_ID, (int)$id)); } catch (PDOException $e) { - $this->logger->error($e->getMessage()); + $this->logger->error($e->getMessage(), [ + 'entity' => $entity->getAll(), + 'exception' => [ + 'file' => $e->getFile(), + 'line' => $e->getLine(), + 'message' => $e->getMessage(), + 'trace' => $e->getTrace(), + ], + ]); return false; } @@ -295,7 +319,15 @@ final class PDOAdapter implements StorageInterface } } catch (PDOException $e) { $actions['failed']++; - $this->logger->error($e->getMessage(), $entity->getAll()); + $this->logger->error($e->getMessage(), [ + 'entity' => $entity->getAll(), + 'exception' => [ + 'file' => $e->getFile(), + 'line' => $e->getLine(), + 'message' => $e->getMessage(), + 'trace' => $e->getTrace(), + ], + ]); } } @@ -519,7 +551,7 @@ final class PDOAdapter implements StorageInterface try { return $stmt->execute($cond); } catch (PDOException $e) { - if (false !== stripos($e->getMessage(), 'database is locked')) { + if (true === str_contains(strtolower($e->getMessage()), 'database is locked')) { if ($i >= self::LOCK_RETRY) { throw $e; } @@ -545,7 +577,7 @@ final class PDOAdapter implements StorageInterface try { return $this->pdo->query($sql); } catch (PDOException $e) { - if (false !== stripos($e->getMessage(), 'database is locked')) { + if (true === str_contains(strtolower($e->getMessage()), 'database is locked')) { if ($i >= self::LOCK_RETRY) { throw $e; } From a0d77c77b3b9a2903c2518691e7ec7cab0f493b1 Mon Sep 17 00:00:00 2001 From: "Abdulmhsen B. A. A" Date: Mon, 27 Jun 2022 17:18:52 +0300 Subject: [PATCH 02/11] updated mappers to reflect new guids changes. --- src/Libs/Mappers/Import/DirectMapper.php | 27 ++++++++++++++---------- src/Libs/Mappers/Import/MemoryMapper.php | 2 -- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/Libs/Mappers/Import/DirectMapper.php b/src/Libs/Mappers/Import/DirectMapper.php index 73f1be3f..4573744e 100644 --- a/src/Libs/Mappers/Import/DirectMapper.php +++ b/src/Libs/Mappers/Import/DirectMapper.php @@ -507,7 +507,11 @@ final class DirectMapper implements ImportInterface protected function addPointers(iFace $entity, string|int $pointer): ImportInterface { - foreach ([...$entity->getPointers(), ...$entity->getRelativePointers()] as $key) { + foreach ($entity->getRelativePointers() as $key) { + $this->pointers[$key] = $pointer; + } + + foreach ($entity->getPointers() as $key) { $this->pointers[$key . '/' . $entity->type] = $pointer; } @@ -527,18 +531,12 @@ final class DirectMapper implements ImportInterface return $entity->id; } - // -- Prioritize relative ids for episodes, External ids are often incorrect for episodes. - if (true === $entity->isEpisode()) { - foreach ($entity->getRelativePointers() as $key) { - $lookup = $key . '/' . $entity->type; - if (null !== ($this->pointers[$lookup] ?? null)) { - return $this->pointers[$lookup]; - } + foreach ($entity->getRelativePointers() as $key) { + if (null !== ($this->pointers[$key] ?? null)) { + return $this->pointers[$key]; } } - // -- look up movies based on guid. - // -- if episode didn't have any match using relative id then fallback to external ids. foreach ($entity->getPointers() as $key) { $lookup = $key . '/' . $entity->type; if (null !== ($this->pointers[$lookup] ?? null)) { @@ -559,12 +557,19 @@ final class DirectMapper implements ImportInterface protected function removePointers(iFace $entity): ImportInterface { - foreach ([...$entity->getPointers(), ...$entity->getRelativePointers()] as $key) { + foreach ($entity->getPointers() as $key) { $lookup = $key . '/' . $entity->type; if (null !== ($this->pointers[$lookup] ?? null)) { unset($this->pointers[$lookup]); } } + + foreach ($entity->getRelativePointers() as $key) { + if (null !== ($this->pointers[$key] ?? null)) { + unset($this->pointers[$key]); + } + } + return $this; } } diff --git a/src/Libs/Mappers/Import/MemoryMapper.php b/src/Libs/Mappers/Import/MemoryMapper.php index 2425cf44..fa971367 100644 --- a/src/Libs/Mappers/Import/MemoryMapper.php +++ b/src/Libs/Mappers/Import/MemoryMapper.php @@ -466,14 +466,12 @@ final class MemoryMapper implements ImportInterface return self::GUID . $entity->id; } - // -- Prioritize relative ids for episodes, External ids are often incorrect for episodes. foreach ($entity->getRelativePointers() as $key) { if (null !== ($this->pointers[$key] ?? null)) { return $this->pointers[$key]; } } - // -- fallback to guids for movies and episode in case there was no relative id match. foreach ($entity->getPointers() as $key) { $lookup = $key . '/' . $entity->type; if (null !== ($this->pointers[$lookup] ?? null)) { From 2ad601e46ca82a4a2ef031a1ef7f6f166408a663 Mon Sep 17 00:00:00 2001 From: "Abdulmhsen B. A. A" Date: Mon, 27 Jun 2022 18:26:01 +0300 Subject: [PATCH 03/11] MemoryMapper: report changed data more accurately. --- src/Libs/Mappers/Import/MemoryMapper.php | 69 +++++++++++++----------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/src/Libs/Mappers/Import/MemoryMapper.php b/src/Libs/Mappers/Import/MemoryMapper.php index fa971367..efbaa1df 100644 --- a/src/Libs/Mappers/Import/MemoryMapper.php +++ b/src/Libs/Mappers/Import/MemoryMapper.php @@ -202,14 +202,18 @@ final class MemoryMapper implements ImportInterface fields: array_merge($keys, [iFace::COLUMN_EXTRA]) )->markAsUnplayed(backend: $entity); - $this->logger->notice('MAPPER: [%(backend)] marked [%(title)] as unplayed.', [ - 'id' => $cloned->id, - 'backend' => $entity->via, - 'title' => $cloned->getName(), - 'changes' => $this->objects[$pointer]->diff( - array_merge($keys, [iFace::COLUMN_WATCHED, iFace::COLUMN_UPDATED]) - ), - ]); + $changes = $this->objects[$pointer]->diff( + array_merge($keys, [iFace::COLUMN_WATCHED, iFace::COLUMN_UPDATED]) + ); + + if (count($changes) >= 1) { + $this->logger->notice('MAPPER: [%(backend)] marked [%(title)] as unplayed.', [ + 'id' => $cloned->id, + 'backend' => $entity->via, + 'title' => $cloned->getName(), + 'changes' => $changes, + ]); + } return $this; } @@ -236,16 +240,20 @@ final class MemoryMapper implements ImportInterface $this->removePointers($cloned)->addPointers($this->objects[$pointer], $pointer); - $this->logger->notice('MAPPER: [%(backend)] updated [%(title)] metadata.', [ - 'id' => $cloned->id, - 'backend' => $entity->via, - 'title' => $cloned->getName(), - 'changes' => $cloned::fromArray($cloned->getAll())->apply( - entity: $entity, - fields: $localFields - )->diff(fields: $keys), - 'fields' => implode(',', $localFields), - ]); + $changes = $cloned::fromArray($cloned->getAll())->apply( + entity: $entity, + fields: $localFields + )->diff(fields: $keys); + + if (count($changes) >= 1) { + $this->logger->notice('MAPPER: [%(backend)] updated [%(title)] metadata.', [ + 'id' => $cloned->id, + 'backend' => $entity->via, + 'title' => $cloned->getName(), + 'changes' => $changes, + 'fields' => implode(',', $localFields), + ]); + } return $this; } @@ -274,18 +282,19 @@ final class MemoryMapper implements ImportInterface ); $this->removePointers($cloned)->addPointers($this->objects[$pointer], $pointer); - $this->logger->notice('MAPPER: [%(backend)] Updated [%(title)].', [ - 'id' => $cloned->id, - 'backend' => $entity->via, - 'title' => $cloned->getName(), - 'changes' => $cloned::fromArray($cloned->getAll())->apply( - entity: $entity, - fields: $keys - )->diff( - fields: $keys - ), - 'fields' => implode(', ', $keys), - ]); + $changes = $cloned::fromArray($cloned->getAll())->apply(entity: $entity, fields: $keys)->diff( + fields: $keys + ); + + if (count($changes) >= 1) { + $this->logger->notice('MAPPER: [%(backend)] Updated [%(title)].', [ + 'id' => $cloned->id, + 'backend' => $entity->via, + 'title' => $cloned->getName(), + 'changes' => $changes, + 'fields' => implode(', ', $keys), + ]); + } return $this; } From 9b2c54731a8d18b638c39a20f479acc5ebe0e9f2 Mon Sep 17 00:00:00 2001 From: "Abdulmhsen B. A. A" Date: Mon, 27 Jun 2022 18:38:58 +0300 Subject: [PATCH 04/11] Updated build to include new branches. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f85be0f0..58dbe48f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,8 +3,7 @@ name: Build Container Images on: push: branches: - - 'master' - - 'dev' + - '*' tags-ignore: - 'v0*' paths-ignore: @@ -12,6 +11,7 @@ on: pull_request: branches: - 'master' + - 'v1.0' paths-ignore: - '**.md' From 76162a7039ff557ad0e3fd4c1f09fdc6609f650a Mon Sep 17 00:00:00 2001 From: "Abdulmhsen B. A. A" Date: Mon, 27 Jun 2022 18:55:56 +0300 Subject: [PATCH 05/11] Updated build to include new branches. --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 58dbe48f..b76fb25a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,7 +11,6 @@ on: pull_request: branches: - 'master' - - 'v1.0' paths-ignore: - '**.md' From f4afd24f2c763502e397ca7103cd216da201e6fd Mon Sep 17 00:00:00 2001 From: "Abdulmhsen B. A. A" Date: Mon, 27 Jun 2022 19:58:04 +0300 Subject: [PATCH 06/11] Fixed updating last import date for backends. --- src/Commands/State/ExportCommand.php | 13 ++++++++----- src/Commands/State/ImportCommand.php | 9 ++++++++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/Commands/State/ExportCommand.php b/src/Commands/State/ExportCommand.php index 31684752..da14041b 100644 --- a/src/Commands/State/ExportCommand.php +++ b/src/Commands/State/ExportCommand.php @@ -334,12 +334,15 @@ class ExportCommand extends Command continue; } - if (true === (bool)Data::get(sprintf('%s.has_errors', $name))) { - $this->logger->notice( - sprintf('%s: Not updating last export date. Backend reported an error.', $name) - ); - } else { + if (false === (bool)Data::get("{$name}.has_errors", false)) { Config::save(sprintf('servers.%s.export.lastSync', $name), time()); + } else { + $this->logger->warning( + 'SYSTEM: Not updating last export date for [%(backend)]. Backend reported an error.', + [ + 'backend' => $name, + ] + ); } } diff --git a/src/Commands/State/ImportCommand.php b/src/Commands/State/ImportCommand.php index d57d2f98..74cb0014 100644 --- a/src/Commands/State/ImportCommand.php +++ b/src/Commands/State/ImportCommand.php @@ -249,8 +249,15 @@ class ImportCommand extends Command $inDryMode = $this->mapper->inDryRunMode() || ag($server, 'options.' . Options::DRY_RUN); - if (false === Data::get(sprintf('%s.has_errors', $name)) && false === $inDryMode) { + if (false === (bool)Data::get("{$name}.has_errors", false) && false === $inDryMode) { Config::save(sprintf('servers.%s.import.lastSync', $name), time()); + } else { + $this->logger->warning( + 'SYSTEM: Not updating last import date for [%(backend)]. Backend reported an error.', + [ + 'backend' => $name, + ] + ); } } From b6aeadc6b96b7e21969dc5b0b5077bac15bb7fe0 Mon Sep 17 00:00:00 2001 From: "Abdulmhsen B. A. A" Date: Tue, 28 Jun 2022 17:55:18 +0300 Subject: [PATCH 07/11] Code cleanup. --- src/Backends/Emby/Action/Export.php | 9 +++ src/Backends/Emby/Action/GetLibrary.php | 9 +++ src/Backends/Emby/Action/Import.php | 9 +++ src/Backends/Emby/EmbyGuid.php | 4 +- src/Backends/Jellyfin/Action/Export.php | 34 ++++----- src/Backends/Jellyfin/Action/Import.php | 46 +++++++------ src/Backends/Jellyfin/Action/Push.php | 13 ++-- src/Backends/Plex/Action/Export.php | 28 ++++---- src/Backends/Plex/Action/Import.php | 60 ++++++++-------- src/Command.php | 2 +- src/Commands/State/ExportCommand.php | 12 ++-- src/Commands/State/ImportCommand.php | 44 +++++++----- src/Commands/State/PushCommand.php | 6 +- src/Libs/Data.php | 60 ---------------- src/Libs/Mappers/Import/DirectMapper.php | 30 ++++---- src/Libs/Mappers/Import/MemoryMapper.php | 84 +++++++++++------------ src/Libs/Message.php | 76 ++++++++++++++++++++ tests/Mappers/Import/DirectMapperTest.php | 4 +- tests/Mappers/Import/MemoryMapperTest.php | 38 +++++----- 19 files changed, 309 insertions(+), 259 deletions(-) create mode 100644 src/Backends/Emby/Action/Export.php create mode 100644 src/Backends/Emby/Action/GetLibrary.php create mode 100644 src/Backends/Emby/Action/Import.php delete mode 100644 src/Libs/Data.php create mode 100644 src/Libs/Message.php diff --git a/src/Backends/Emby/Action/Export.php b/src/Backends/Emby/Action/Export.php new file mode 100644 index 00000000..0a079142 --- /dev/null +++ b/src/Backends/Emby/Action/Export.php @@ -0,0 +1,9 @@ +processShow(context: $context, guid: $guid, item: $item, logContext: $logContext); return; } + $mappedType = JFC::TYPE_MAPPER[$type] ?? $type; + try { $queue = ag($opts, 'queue', fn() => Container::get(QueueRequests::class)); $after = ag($opts, 'after', null); - Data::increment($context->backendName, $type . '_total'); + Message::increment("{$context->backendName}.{$mappedType}.total"); $logContext['item'] = [ 'id' => ag($item, 'Id'), 'title' => match ($type) { - JellyfinClient::TYPE_MOVIE => sprintf( + JFC::TYPE_MOVIE => sprintf( '%s (%d)', ag($item, ['Name', 'OriginalTitle'], '??'), ag($item, 'ProductionYear', '0000') ), - JellyfinClient::TYPE_EPISODE => trim( + JFC::TYPE_EPISODE => trim( sprintf( '%s - (%sx%s)', ag($item, 'SeriesName', '??'), @@ -79,7 +81,7 @@ class Export extends Import ], ]); - Data::increment($context->backendName, $type . '_ignored_no_date_is_set'); + Message::increment("{$context->backendName}.{$mappedType}.ignored_no_date_is_set"); return; } @@ -107,7 +109,7 @@ class Export extends Import ], ]); - Data::increment($context->backendName, $type . '_ignored_no_supported_guid'); + Message::increment("{$context->backendName}.{$mappedType}.ignored_no_supported_guid"); return; } @@ -125,7 +127,7 @@ class Export extends Import ] ); - Data::increment($context->backendName, $type . '_ignored_date_is_equal_or_higher'); + Message::increment("{$context->backendName}.{$mappedType}.ignored_date_is_equal_or_higher"); return; } } @@ -135,7 +137,7 @@ class Export extends Import 'backend' => $context->backendName, ...$logContext, ]); - Data::increment($context->backendName, $type . '_ignored_not_found_in_db'); + Message::increment("{$context->backendName}.{$mappedType}.ignored_not_found_in_db"); return; } @@ -154,7 +156,7 @@ class Export extends Import ); } - Data::increment($context->backendName, $type . '_ignored_state_unchanged'); + Message::increment("{$context->backendName}.{$mappedType}.ignored_state_unchanged"); return; } @@ -171,7 +173,7 @@ class Export extends Import ] ); - Data::increment($context->backendName, $type . '_ignored_date_is_newer'); + Message::increment("{$context->backendName}.{$mappedType}.ignored_date_is_newer"); return; } diff --git a/src/Backends/Jellyfin/Action/Import.php b/src/Backends/Jellyfin/Action/Import.php index 4b9d5cd0..42a0c80e 100644 --- a/src/Backends/Jellyfin/Action/Import.php +++ b/src/Backends/Jellyfin/Action/Import.php @@ -10,13 +10,13 @@ use App\Backends\Common\GuidInterface as iGuid; use App\Backends\Common\Response; use App\Backends\Jellyfin\JellyfinActionTrait; use App\Backends\Jellyfin\JellyfinClient as JFC; -use App\Libs\Data; -use App\Libs\Entity\StateInterface as iFace; +use App\Libs\Entity\StateInterface as iState; use App\Libs\Guid; -use App\Libs\Mappers\ImportInterface; +use App\Libs\Mappers\ImportInterface as iImport; +use App\Libs\Message; use App\Libs\Options; use Closure; -use DateTimeInterface; +use DateTimeInterface as iDate; use JsonException; use JsonMachine\Items; use JsonMachine\JsonDecoder\DecodingError; @@ -40,8 +40,8 @@ class Import /** * @param Context $context * @param iGuid $guid - * @param ImportInterface $mapper - * @param DateTimeInterface|null $after + * @param iImport $mapper + * @param iDate|null $after * @param array $opts * * @return Response @@ -49,8 +49,8 @@ class Import public function __invoke( Context $context, iGuid $guid, - ImportInterface $mapper, - DateTimeInterface|null $after = null, + iImport $mapper, + iDate|null $after = null, array $opts = [] ): Response { return $this->tryResponse($context, fn() => $this->getLibraries( @@ -104,7 +104,7 @@ class Import 'status_code' => $response->getStatusCode(), ] ); - Data::add($context->backendName, 'has_errors', true); + Message::add("{$context->backendName}.has_errors", true); return []; } @@ -121,7 +121,7 @@ class Import 'backend' => $context->backendName, 'body' => $json, ]); - Data::add($context->backendName, 'has_errors', true); + Message::add("{$context->backendName}.has_errors", true); return []; } } catch (ExceptionInterface $e) { @@ -135,7 +135,7 @@ class Import 'trace' => $context->trace ? $e->getTrace() : [], ], ]); - Data::add($context->backendName, 'has_errors', true); + Message::add("{$context->backendName}.has_errors", true); return []; } catch (JsonException $e) { $this->logger->error('Request for [%(backend)] libraries returned with invalid body.', [ @@ -147,7 +147,7 @@ class Import 'trace' => $context->trace ? $e->getTrace() : [], ], ]); - Data::add($context->backendName, 'has_errors', true); + Message::add("{$context->backendName}.has_errors", true); return []; } @@ -313,7 +313,7 @@ class Import 'unsupported' => $unsupported, ], ]); - Data::add($context->backendName, 'has_errors', true); + Message::add("{$context->backendName}.has_errors", true); return []; } @@ -405,6 +405,8 @@ class Import 'duration' => number_format($end->getTimestamp() - $start->getTimestamp()), ], ]); + + Message::increment('response.size', (int)$response->getInfo('size_download')); } protected function processShow(Context $context, iGuid $guid, array $item, array $logContext = []): void @@ -461,7 +463,7 @@ class Import protected function process( Context $context, iGuid $guid, - ImportInterface $mapper, + iImport $mapper, array $item, array $logContext = [], array $opts = [] @@ -471,8 +473,10 @@ class Import return; } + $mappedType = JFC::TYPE_MAPPER[$type] ?? $type; + try { - Data::increment($context->backendName, $type . '_total'); + Message::increment("{$context->backendName}.{$mappedType}.total"); $logContext['item'] = [ 'id' => ag($item, 'Id'), @@ -514,7 +518,7 @@ class Import 'body' => $item, ]); - Data::increment($context->backendName, $type . '_ignored_no_date_is_set'); + Message::increment("{$context->backendName}.{$mappedType}.ignored_no_date_is_set"); return; } @@ -525,10 +529,10 @@ class Import opts: $opts + [ 'library' => ag($logContext, 'library.id'), 'override' => [ - iFace::COLUMN_EXTRA => [ + iState::COLUMN_EXTRA => [ $context->backendName => [ - iFace::COLUMN_EXTRA_EVENT => 'task.import', - iFace::COLUMN_EXTRA_DATE => makeDate('now'), + iState::COLUMN_EXTRA_EVENT => 'task.import', + iState::COLUMN_EXTRA_DATE => makeDate('now'), ], ], ] @@ -550,12 +554,12 @@ class Import 'guids' => !empty($providerIds) ? $providerIds : 'None' ]); - Data::increment($context->backendName, $type . '_ignored_no_supported_guid'); + Message::increment("{$context->backendName}.{$mappedType}.ignored_no_supported_guid"); return; } $mapper->add(entity: $entity, opts: [ - 'after' => ag($opts, 'after'), + 'after' => ag($opts, 'after', null), Options::IMPORT_METADATA_ONLY => true === (bool)ag($context->options, Options::IMPORT_METADATA_ONLY), ]); } catch (Throwable $e) { diff --git a/src/Backends/Jellyfin/Action/Push.php b/src/Backends/Jellyfin/Action/Push.php index c46816f6..aa378c2a 100644 --- a/src/Backends/Jellyfin/Action/Push.php +++ b/src/Backends/Jellyfin/Action/Push.php @@ -5,10 +5,9 @@ declare(strict_types=1); namespace App\Backends\Jellyfin\Action; use App\Backends\Common\CommonTrait; -use App\Backends\Common\Response; use App\Backends\Common\Context; +use App\Backends\Common\Response; use App\Backends\Jellyfin\JellyfinClient; -use App\Libs\Entity\StateInterface as iFace; use App\Libs\Entity\StateInterface as iState; use App\Libs\Options; use App\Libs\QueueRequests; @@ -52,7 +51,7 @@ class Push $requests = []; foreach ($entities as $key => $entity) { - if (true !== ($entity instanceof iFace)) { + if (true !== ($entity instanceof iState)) { continue; } @@ -72,7 +71,7 @@ class Push ], ]; - if (null === ag($metadata, iFace::COLUMN_ID, null)) { + if (null === ag($metadata, iState::COLUMN_ID, null)) { $this->logger->warning( 'Ignoring [%(item.title)] for [%(backend)]. No metadata was found.', [ @@ -83,11 +82,11 @@ class Push continue; } - $logContext['remote']['id'] = ag($metadata, iFace::COLUMN_ID); + $logContext['remote']['id'] = ag($metadata, iState::COLUMN_ID); try { $url = $context->backendUrl->withPath( - sprintf('/Users/%s/items/%s', $context->backendUser, ag($metadata, iFace::COLUMN_ID)) + sprintf('/Users/%s/items/%s', $context->backendUser, ag($metadata, iState::COLUMN_ID)) )->withQuery( http_build_query( [ @@ -149,7 +148,7 @@ class Push $entity = $entities[$id]; - assert($entity instanceof iFace); + assert($entity instanceof iState); if (200 !== $response->getStatusCode()) { if (404 === $response->getStatusCode()) { diff --git a/src/Backends/Plex/Action/Export.php b/src/Backends/Plex/Action/Export.php index 55e9fa29..a7d2ae79 100644 --- a/src/Backends/Plex/Action/Export.php +++ b/src/Backends/Plex/Action/Export.php @@ -5,11 +5,11 @@ declare(strict_types=1); namespace App\Backends\Plex\Action; use App\Backends\Common\Context; -use App\Backends\Common\GuidInterface; +use App\Backends\Common\GuidInterface as iGuid; use App\Backends\Plex\PlexClient; use App\Libs\Container; -use App\Libs\Data; -use App\Libs\Mappers\ImportInterface; +use App\Libs\Mappers\ImportInterface as iImport; +use App\Libs\Message; use App\Libs\Options; use App\Libs\QueueRequests; use DateTimeInterface; @@ -19,8 +19,8 @@ final class Export extends Import { protected function process( Context $context, - GuidInterface $guid, - ImportInterface $mapper, + iGuid $guid, + iImport $mapper, array $item, array $logContext = [], array $opts = [], @@ -35,9 +35,11 @@ final class Export extends Import return; } + $mappedType = PlexClient::TYPE_MAPPER[$type] ?? $type; + try { - Data::increment($context->backendName, $library . '_total'); - Data::increment($context->backendName, $type . '_total'); + Message::increment("{$context->backendName}.{$library}.total"); + Message::increment("{$context->backendName}.{$mappedType}.total"); $year = (int)ag($item, ['grandParentYear', 'parentYear', 'year'], 0); if (0 === $year && null !== ($airDate = ag($item, 'originallyAvailableAt'))) { @@ -80,7 +82,7 @@ final class Export extends Import ], ]); - Data::increment($context->backendName, $type . '_ignored_no_date_is_set'); + Message::increment("{$context->backendName}.{$mappedType}.ignored_no_date_is_set"); return; } @@ -114,7 +116,7 @@ final class Export extends Import ], ]); - Data::increment($context->backendName, $type . '_ignored_no_supported_guid'); + Message::increment("{$context->backendName}.{$mappedType}.ignored_no_supported_guid"); return; } @@ -132,7 +134,7 @@ final class Export extends Import ] ); - Data::increment($context->backendName, $type . '_ignored_date_is_equal_or_higher'); + Message::increment("{$context->backendName}.{$mappedType}.ignored_date_is_equal_or_higher"); return; } } @@ -142,7 +144,7 @@ final class Export extends Import 'backend' => $context->backendName, ...$logContext, ]); - Data::increment($context->backendName, $type . '_ignored_not_found_in_db'); + Message::increment("{$context->backendName}.{$mappedType}.ignored_not_found_in_db"); return; } @@ -161,7 +163,7 @@ final class Export extends Import ); } - Data::increment($context->backendName, $type . '_ignored_state_unchanged'); + Message::increment("{$context->backendName}.{$mappedType}.ignored_state_unchanged"); return; } @@ -178,7 +180,7 @@ final class Export extends Import ] ); - Data::increment($context->backendName, $type . '_ignored_date_is_newer'); + Message::increment("{$context->backendName}.{$mappedType}.ignored_date_is_newer"); return; } diff --git a/src/Backends/Plex/Action/Import.php b/src/Backends/Plex/Action/Import.php index d462bc38..aff2e689 100644 --- a/src/Backends/Plex/Action/Import.php +++ b/src/Backends/Plex/Action/Import.php @@ -10,13 +10,13 @@ use App\Backends\Common\GuidInterface as iGuid; use App\Backends\Common\Response; use App\Backends\Plex\PlexActionTrait; use App\Backends\Plex\PlexClient; -use App\Libs\Data; -use App\Libs\Entity\StateInterface as iFace; +use App\Libs\Entity\StateInterface as iState; use App\Libs\Guid; -use App\Libs\Mappers\ImportInterface; +use App\Libs\Mappers\ImportInterface as iImport; +use App\Libs\Message; use App\Libs\Options; use Closure; -use DateTimeInterface; +use DateTimeInterface as iDate; use JsonException; use JsonMachine\Items; use JsonMachine\JsonDecoder\DecodingError; @@ -40,8 +40,8 @@ class Import /** * @param Context $context * @param iGuid $guid - * @param ImportInterface $mapper - * @param DateTimeInterface|null $after + * @param iImport $mapper + * @param iDate|null $after * @param array $opts * * @return Response @@ -49,8 +49,8 @@ class Import public function __invoke( Context $context, iGuid $guid, - ImportInterface $mapper, - DateTimeInterface|null $after = null, + iImport $mapper, + iDate|null $after = null, array $opts = [] ): Response { return $this->tryResponse($context, fn() => $this->getLibraries( @@ -105,7 +105,7 @@ class Import ] ); - Data::add($context->backendName, 'has_errors', true); + Message::add("{$context->backendName}.has_errors", true); return []; } @@ -122,7 +122,7 @@ class Import 'backend' => $context->backendName, 'body' => $json, ]); - Data::add($context->backendName, 'has_errors', true); + Message::add("{$context->backendName}.has_errors", true); return []; } } catch (ExceptionInterface $e) { @@ -136,7 +136,7 @@ class Import 'trace' => $context->trace ? $e->getTrace() : [], ], ]); - Data::add($context->backendName, 'has_errors', true); + Message::add("{$context->backendName}.has_errors", true); return []; } catch (JsonException $e) { $this->logger->error('Request for [%(backend)] libraries returned with invalid body.', [ @@ -148,7 +148,7 @@ class Import 'trace' => $context->trace ? $e->getTrace() : [], ], ]); - Data::add($context->backendName, 'has_errors', true); + Message::add("{$context->backendName}.has_errors", true); return []; } @@ -337,7 +337,7 @@ class Import ], ]); - Data::add($context->backendName, 'has_errors', true); + Message::add("{$context->backendName}.has_errors", true); return []; } @@ -361,6 +361,7 @@ class Import return; } + $start = makeDate(); $this->logger->info('Parsing [%(backend)] library [%(library.title)] response.', [ 'backend' => $context->backendName, @@ -429,6 +430,8 @@ class Import 'duration' => number_format($end->getTimestamp() - $start->getTimestamp()), ], ]); + + Message::increment('response.size', (int)$response->getInfo('size_download')); } protected function processShow(Context $context, iGuid $guid, array $item, array $logContext = []): void @@ -503,23 +506,20 @@ class Import protected function process( Context $context, iGuid $guid, - ImportInterface $mapper, + iImport $mapper, array $item, array $logContext = [], array $opts = [] ): void { - $after = ag($opts, 'after', null); - $library = ag($logContext, 'library.id'); - $type = ag($item, 'type'); + if (PlexClient::TYPE_SHOW === ($type = ag($item, 'type'))) { + $this->processShow(context: $context, guid: $guid, item: $item, logContext: $logContext); + return; + } + + $mappedType = PlexClient::TYPE_MAPPER[$type] ?? $type; try { - if (PlexClient::TYPE_SHOW === $type) { - $this->processShow($context, $guid, $item, $logContext); - return; - } - - Data::increment($context->backendName, $library . '_total'); - Data::increment($context->backendName, $type . '_total'); + Message::increment("{$context->backendName}.{$mappedType}.total"); $year = (int)ag($item, ['grandParentYear', 'parentYear', 'year'], 0); if (0 === $year && null !== ($airDate = ag($item, 'originallyAvailableAt'))) { @@ -560,7 +560,7 @@ class Import 'body' => $item, ]); - Data::increment($context->backendName, $type . '_ignored_no_date_is_set'); + Message::increment("{$context->backendName}.{$mappedType}.ignored_no_date_is_set"); return; } @@ -570,10 +570,10 @@ class Import item: $item, opts: $opts + [ 'override' => [ - iFace::COLUMN_EXTRA => [ + iState::COLUMN_EXTRA => [ $context->backendName => [ - iFace::COLUMN_EXTRA_EVENT => 'task.import', - iFace::COLUMN_EXTRA_DATE => makeDate('now'), + iState::COLUMN_EXTRA_EVENT => 'task.import', + iState::COLUMN_EXTRA_DATE => makeDate('now'), ], ], ], @@ -601,12 +601,12 @@ class Import 'guids' => !empty($item['Guid']) ? $item['Guid'] : 'None' ]); - Data::increment($context->backendName, $type . '_ignored_no_supported_guid'); + Message::increment("{$context->backendName}.{$mappedType}.ignored_no_supported_guid"); return; } $mapper->add(entity: $entity, opts: [ - 'after' => $after, + 'after' => ag($opts, 'after', null), Options::IMPORT_METADATA_ONLY => true === (bool)ag($context->options, Options::IMPORT_METADATA_ONLY), ]); } catch (Throwable $e) { diff --git a/src/Command.php b/src/Command.php index fb21a203..f63a1e9c 100644 --- a/src/Command.php +++ b/src/Command.php @@ -255,7 +255,7 @@ class Command extends BaseCommand $suggest = []; - foreach (self::DISPLAY_OUTPUT as $name) { + foreach (static::DISPLAY_OUTPUT as $name) { if (empty($currentValue) || str_starts_with($name, $currentValue)) { $suggest[] = $name; } diff --git a/src/Commands/State/ExportCommand.php b/src/Commands/State/ExportCommand.php index da14041b..68a24a94 100644 --- a/src/Commands/State/ExportCommand.php +++ b/src/Commands/State/ExportCommand.php @@ -6,8 +6,8 @@ namespace App\Commands\State; use App\Command; use App\Libs\Config; -use App\Libs\Data; use App\Libs\Mappers\Import\DirectMapper; +use App\Libs\Message; use App\Libs\Options; use App\Libs\QueueRequests; use App\Libs\Storage\StorageInterface; @@ -142,8 +142,6 @@ class ExportCommand extends Command continue; } - Data::addBucket($name); - $opts = ag($backend, 'options', []); if ($input->getOption('ignore-date')) { @@ -334,7 +332,7 @@ class ExportCommand extends Command continue; } - if (false === (bool)Data::get("{$name}.has_errors", false)) { + if (false === (bool)Message::get("{$name}.has_errors", false)) { Config::save(sprintf('servers.%s.export.lastSync', $name), time()); } else { $this->logger->warning( @@ -452,12 +450,12 @@ class ExportCommand extends Command array_push($requests, ...$backend['class']->export($this->mapper, $this->queue, $after)); if (false === $input->getOption('dry-run')) { - if (true === (bool)Data::get(sprintf('%s.has_errors', $name))) { - $this->logger->notice('Not updating last export date. [%(backend)] report an error.', [ + if (true === (bool)Message::get("{$name}.has_errors")) { + $this->logger->warning('SYSTEM: Not updating last export date. [%(backend)] report an error.', [ 'backend' => $name, ]); } else { - Config::save(sprintf('servers.%s.export.lastSync', $name), time()); + Config::save("servers.{$name}.export.lastSync", time()); } } } diff --git a/src/Commands/State/ImportCommand.php b/src/Commands/State/ImportCommand.php index 74cb0014..dd62e59d 100644 --- a/src/Commands/State/ImportCommand.php +++ b/src/Commands/State/ImportCommand.php @@ -6,10 +6,10 @@ namespace App\Commands\State; use App\Command; use App\Libs\Config; -use App\Libs\Data; -use App\Libs\Entity\StateInterface; +use App\Libs\Entity\StateInterface as iState; use App\Libs\Mappers\Import\DirectMapper; use App\Libs\Mappers\ImportInterface; +use App\Libs\Message; use App\Libs\Options; use App\Libs\Storage\StorageInterface; use Psr\Log\LoggerInterface; @@ -65,6 +65,7 @@ class ImportCommand extends Command InputOption::VALUE_NONE, 'import metadata changes only. Works when there are records in storage.' ) + ->addOption('show-messages', null, InputOption::VALUE_NONE, 'Show internal messages.') ->addOption('config', 'c', InputOption::VALUE_REQUIRED, 'Use Alternative config file.') ->setAliases(['import', 'pull']); } @@ -204,7 +205,6 @@ class ImportCommand extends Command $this->storage->singleTransaction(); foreach ($list as $name => &$server) { - Data::addBucket($name); $metadata = false; $opts = ag($server, 'options', []); @@ -249,15 +249,14 @@ class ImportCommand extends Command $inDryMode = $this->mapper->inDryRunMode() || ag($server, 'options.' . Options::DRY_RUN); - if (false === (bool)Data::get("{$name}.has_errors", false) && false === $inDryMode) { - Config::save(sprintf('servers.%s.import.lastSync', $name), time()); - } else { - $this->logger->warning( - 'SYSTEM: Not updating last import date for [%(backend)]. Backend reported an error.', - [ + if (false === $inDryMode) { + if (true === (bool)Message::get("{$name}.has_errors")) { + $this->logger->warning('SYSTEM: Not updating last import date. [%(backend)] reported an error.', [ 'backend' => $name, - ] - ); + ]); + } else { + Config::save("servers.{$name}.import.lastSync", time()); + } } } @@ -302,6 +301,9 @@ class ImportCommand extends Command 'now' => getMemoryUsage(), 'peak' => getPeakMemoryUsage(), ], + 'responses' => [ + 'size' => fsize((int)Message::get('response.size', 0)), + ], ]); $queue = $requestData = null; @@ -322,17 +324,17 @@ class ImportCommand extends Command $a = [ [ - 'Type' => ucfirst(StateInterface::TYPE_MOVIE), - 'Added' => $operations[StateInterface::TYPE_MOVIE]['added'] ?? '-', - 'Updated' => $operations[StateInterface::TYPE_MOVIE]['updated'] ?? '-', - 'Failed' => $operations[StateInterface::TYPE_MOVIE]['failed'] ?? '-', + 'Type' => ucfirst(iState::TYPE_MOVIE), + 'Added' => $operations[iState::TYPE_MOVIE]['added'] ?? '-', + 'Updated' => $operations[iState::TYPE_MOVIE]['updated'] ?? '-', + 'Failed' => $operations[iState::TYPE_MOVIE]['failed'] ?? '-', ], new TableSeparator(), [ - 'Type' => ucfirst(StateInterface::TYPE_EPISODE), - 'Added' => $operations[StateInterface::TYPE_EPISODE]['added'] ?? '-', - 'Updated' => $operations[StateInterface::TYPE_EPISODE]['updated'] ?? '-', - 'Failed' => $operations[StateInterface::TYPE_EPISODE]['failed'] ?? '-', + 'Type' => ucfirst(iState::TYPE_EPISODE), + 'Added' => $operations[iState::TYPE_EPISODE]['added'] ?? '-', + 'Updated' => $operations[iState::TYPE_EPISODE]['updated'] ?? '-', + 'Failed' => $operations[iState::TYPE_EPISODE]['failed'] ?? '-', ], ]; @@ -346,6 +348,10 @@ class ImportCommand extends Command file_put_contents($config, Yaml::dump(Config::get('servers', []), 8, 2)); } + if ($input->getOption('show-messages')) { + $this->displayContent(Message::getAll(), $output, $input->getOption('output') === 'json' ? 'json' : 'yaml'); + } + return self::SUCCESS; } } diff --git a/src/Commands/State/PushCommand.php b/src/Commands/State/PushCommand.php index f25ed7c0..7c6297f1 100644 --- a/src/Commands/State/PushCommand.php +++ b/src/Commands/State/PushCommand.php @@ -7,8 +7,7 @@ namespace App\Commands\State; use App\Command; use App\Libs\Config; use App\Libs\Container; -use App\Libs\Data; -use App\Libs\Entity\StateInterface; +use App\Libs\Entity\StateInterface as iState; use App\Libs\Options; use App\Libs\QueueRequests; use App\Libs\Storage\StorageInterface; @@ -75,7 +74,7 @@ class PushCommand extends Command $entities = $items = []; foreach ($this->cache->get('queue', []) as $item) { - $items[] = Container::get(StateInterface::class)::fromArray($item); + $items[] = Container::get(iState::class)::fromArray($item); } if (!empty($items)) { @@ -142,7 +141,6 @@ class PushCommand extends Command } foreach ($list as $name => &$server) { - Data::addBucket((string)$name); $opts = ag($server, 'options', []); if ($input->getOption('ignore-date')) { diff --git a/src/Libs/Data.php b/src/Libs/Data.php deleted file mode 100644 index b3181e9f..00000000 --- a/src/Libs/Data.php +++ /dev/null @@ -1,60 +0,0 @@ - $entity->via, 'title' => $entity->getName(), ]); - Data::increment($entity->via, $entity->type . '_failed_no_guid'); + Message::increment("{$entity->via}.{$entity->type}.failed_no_guid"); return $this; } @@ -110,7 +110,7 @@ final class DirectMapper implements ImportInterface if (null === ($local = $this->get($entity))) { if (true === $metadataOnly) { $this->actions[$entity->type]['failed']++; - Data::increment($entity->via, $entity->type . '_failed'); + Message::increment("{$entity->via}.{$entity->type}.failed"); $this->logger->notice('MAPPER: Ignoring [%(backend)] [%(title)]. Does not exist in storage.', [ 'metaOnly' => true, @@ -161,13 +161,13 @@ final class DirectMapper implements ImportInterface if (null === ($this->changed[$entity->id] ?? null)) { $this->actions[$entity->type]['added']++; - Data::increment($entity->via, $entity->type . '_added'); + Message::increment("{$entity->via}.{$entity->type}.added"); } $this->changed[$entity->id] = $this->objects[$entity->id] = $entity->id; } catch (PDOException|Exception $e) { $this->actions[$entity->type]['failed']++; - Data::increment($entity->via, $entity->type . '_failed'); + Message::increment("{$entity->via}.{$entity->type}.failed"); $this->logger->error(sprintf('MAPPER: %s', $e->getMessage()), [ 'backend' => $entity->via, 'title' => $entity->getName(), @@ -210,13 +210,13 @@ final class DirectMapper implements ImportInterface if (null === ($this->changed[$local->id] ?? null)) { $this->actions[$local->type]['updated']++; - Data::increment($entity->via, $local->type . '_updated'); + Message::increment("{$entity->via}.{$local->type}.updated"); } $this->changed[$local->id] = $this->objects[$local->id] = $local->id; } catch (PDOException $e) { $this->actions[$local->type]['failed']++; - Data::increment($entity->via, $local->type . '_failed'); + Message::increment("{$entity->via}.{$local->type}.failed"); $this->logger->error(sprintf('MAPPER: %s', $e->getMessage()), [ 'id' => $cloned->id, 'backend' => $entity->via, @@ -266,13 +266,13 @@ final class DirectMapper implements ImportInterface if (null === ($this->changed[$local->id] ?? null)) { $this->actions[$local->type]['updated']++; - Data::increment($entity->via, $local->type . '_updated'); + Message::increment("{$entity->via}.{$local->type}.updated"); } $this->changed[$local->id] = $this->objects[$local->id] = $local->id; } catch (PDOException $e) { $this->actions[$local->type]['failed']++; - Data::increment($entity->via, $local->type . '_failed'); + Message::increment("{$entity->via}.{$local->type}.failed"); $this->logger->error(sprintf('MAPPER: %s', $e->getMessage()), [ 'id' => $cloned->id, 'backend' => $entity->via, @@ -319,13 +319,13 @@ final class DirectMapper implements ImportInterface if (null === ($this->changed[$local->id] ?? null)) { $this->actions[$local->type]['updated']++; - Data::increment($entity->via, $local->type . '_updated'); + Message::increment("{$entity->via}.{$local->type}.updated"); } $this->changed[$local->id] = $this->objects[$local->id] = $local->id; } catch (PDOException $e) { $this->actions[$local->type]['failed']++; - Data::increment($entity->via, $local->type . '_failed'); + Message::increment("{$entity->via}.{$local->type}.failed"); $this->logger->error(sprintf('MAPPER: %s', $e->getMessage()), [ 'id' => $cloned->id, 'title' => $cloned->getName(), @@ -340,7 +340,7 @@ final class DirectMapper implements ImportInterface } } - Data::increment($entity->via, $entity->type . '_ignored_not_played_since_last_sync'); + Message::increment("{$entity->via}.{$entity->type}.ignored_not_played_since_last_sync"); return $this; } } @@ -370,13 +370,13 @@ final class DirectMapper implements ImportInterface if (null === ($this->changed[$local->id] ?? null)) { $this->actions[$local->type]['updated']++; - Data::increment($entity->via, $entity->type . '_updated'); + Message::increment("{$entity->via}.{$entity->type}.updated"); } $this->changed[$local->id] = $this->objects[$local->id] = $local->id; } catch (PDOException $e) { $this->actions[$local->type]['failed']++; - Data::increment($entity->via, $local->type . '_failed'); + Message::increment("{$entity->via}.{$local->type}.failed"); $this->logger->error(sprintf('MAPPER: %s', $e->getMessage()), [ 'id' => $cloned->id, 'backend' => $entity->via, @@ -403,7 +403,7 @@ final class DirectMapper implements ImportInterface ]); } - Data::increment($entity->via, $entity->type . '_ignored_no_change'); + Message::increment("{$entity->via}.{$entity->type}.ignored_no_change"); return $this; } diff --git a/src/Libs/Mappers/Import/MemoryMapper.php b/src/Libs/Mappers/Import/MemoryMapper.php index efbaa1df..038767d6 100644 --- a/src/Libs/Mappers/Import/MemoryMapper.php +++ b/src/Libs/Mappers/Import/MemoryMapper.php @@ -4,10 +4,10 @@ declare(strict_types=1); namespace App\Libs\Mappers\Import; -use App\Libs\Data; -use App\Libs\Entity\StateInterface as iFace; +use App\Libs\Entity\StateInterface as iState; use App\Libs\Guid; use App\Libs\Mappers\ImportInterface; +use App\Libs\Message; use App\Libs\Options; use App\Libs\Storage\StorageInterface; use DateTimeInterface; @@ -19,7 +19,7 @@ final class MemoryMapper implements ImportInterface protected const GUID = 'local_db://'; /** - * @var array Entities table. + * @var array Entities table. */ protected array $objects = []; @@ -71,7 +71,7 @@ final class MemoryMapper implements ImportInterface return $this; } - public function add(iFace $entity, array $opts = []): self + public function add(iState $entity, array $opts = []): self { if (false === $entity->hasGuids() && false === $entity->hasRelativeGuid()) { $this->logger->warning('MAPPER: Ignoring [%(backend)] [%(title)] no valid/supported external ids.', [ @@ -79,7 +79,7 @@ final class MemoryMapper implements ImportInterface 'backend' => $entity->via, 'title' => $entity->getName(), ]); - Data::increment($entity->via, $entity->type . '_failed_no_guid'); + Message::increment("{$entity->via}.{$entity->type}.failed_no_guid"); return $this; } @@ -91,7 +91,7 @@ final class MemoryMapper implements ImportInterface */ if (false === ($pointer = $this->getPointer($entity))) { if (true === $metadataOnly) { - Data::increment($entity->via, $entity->type . '_failed'); + Message::increment("{$entity->via}.{$entity->type}.failed"); $this->logger->notice('MAPPER: Ignoring [%(backend)] [%(title)]. Does not exist in storage.', [ 'metaOnly' => true, 'backend' => $entity->via, @@ -106,25 +106,25 @@ final class MemoryMapper implements ImportInterface $this->changed[$pointer] = $pointer; - Data::increment($entity->via, $entity->type . '_added'); + Message::increment("{$entity->via}.{$entity->type}.added"); $this->addPointers($this->objects[$pointer], $pointer); if (true === $this->inTraceMode()) { $data = $entity->getAll(); unset($data['id']); - $data[iFace::COLUMN_UPDATED] = makeDate($data[iFace::COLUMN_UPDATED]); - $data[iFace::COLUMN_WATCHED] = 0 === $data[iFace::COLUMN_WATCHED] ? 'No' : 'Yes'; + $data[iState::COLUMN_UPDATED] = makeDate($data[iState::COLUMN_UPDATED]); + $data[iState::COLUMN_WATCHED] = 0 === $data[iState::COLUMN_WATCHED] ? 'No' : 'Yes'; if ($entity->isMovie()) { - unset($data[iFace::COLUMN_SEASON], $data[iFace::COLUMN_EPISODE], $data[iFace::COLUMN_PARENT]); + unset($data[iState::COLUMN_SEASON], $data[iState::COLUMN_EPISODE], $data[iState::COLUMN_PARENT]); } } else { $data = [ - iFace::COLUMN_META_DATA => [ + iState::COLUMN_META_DATA => [ $entity->via => [ - iFace::COLUMN_ID => ag($entity->getMetadata($entity->via), iFace::COLUMN_ID), - iFace::COLUMN_UPDATED => makeDate($entity->updated), - iFace::COLUMN_GUIDS => $entity->getGuids(), - iFace::COLUMN_PARENT => $entity->getParentGuids(), + iState::COLUMN_ID => ag($entity->getMetadata($entity->via), iState::COLUMN_ID), + iState::COLUMN_UPDATED => makeDate($entity->updated), + iState::COLUMN_GUIDS => $entity->getGuids(), + iState::COLUMN_PARENT => $entity->getParentGuids(), ] ], ]; @@ -139,7 +139,7 @@ final class MemoryMapper implements ImportInterface return $this; } - $keys = [iFace::COLUMN_META_DATA]; + $keys = [iState::COLUMN_META_DATA]; /** * DO NOT operate directly on this object it should be cloned. @@ -152,18 +152,18 @@ final class MemoryMapper implements ImportInterface */ if (true === $metadataOnly) { if (true === (clone $cloned)->apply(entity: $entity, fields: $keys)->isChanged(fields: $keys)) { - $localFields = array_merge($keys, [iFace::COLUMN_GUIDS]); + $localFields = array_merge($keys, [iState::COLUMN_GUIDS]); $this->changed[$pointer] = $pointer; - Data::increment($entity->via, $entity->type . '_updated'); + Message::increment("{$entity->via}.{$entity->type}.updated"); $entity->guids = Guid::makeVirtualGuid( $entity->via, - ag($entity->getMetadata($entity->via), iFace::COLUMN_ID) + ag($entity->getMetadata($entity->via), iState::COLUMN_ID) ); $this->objects[$pointer] = $this->objects[$pointer]->apply( entity: $entity, - fields: array_merge($localFields, [iFace::COLUMN_EXTRA]) + fields: array_merge($localFields, [iState::COLUMN_EXTRA]) ); $this->removePointers($cloned)->addPointers($this->objects[$pointer], $pointer); @@ -195,15 +195,15 @@ final class MemoryMapper implements ImportInterface // -- Handle mark as unplayed logic. if (false === $entity->isWatched() && true === $cloned->shouldMarkAsUnplayed(backend: $entity)) { $this->changed[$pointer] = $pointer; - Data::increment($entity->via, $entity->type . '_updated'); + Message::increment("{$entity->via}.{$entity->type}.updated"); $this->objects[$pointer] = $this->objects[$pointer]->apply( entity: $entity, - fields: array_merge($keys, [iFace::COLUMN_EXTRA]) + fields: array_merge($keys, [iState::COLUMN_EXTRA]) )->markAsUnplayed(backend: $entity); $changes = $this->objects[$pointer]->diff( - array_merge($keys, [iFace::COLUMN_WATCHED, iFace::COLUMN_UPDATED]) + array_merge($keys, [iState::COLUMN_WATCHED, iState::COLUMN_UPDATED]) ); if (count($changes) >= 1) { @@ -224,18 +224,18 @@ final class MemoryMapper implements ImportInterface */ if (true === (bool)ag($this->options, Options::MAPPER_ALWAYS_UPDATE_META)) { if (true === (clone $cloned)->apply(entity: $entity, fields: $keys)->isChanged(fields: $keys)) { - $localFields = array_merge($keys, [iFace::COLUMN_GUIDS]); + $localFields = array_merge($keys, [iState::COLUMN_GUIDS]); $this->changed[$pointer] = $pointer; - Data::increment($entity->via, $entity->type . '_updated'); + Message::increment("{$entity->via}.{$entity->type}.updated"); $entity->guids = Guid::makeVirtualGuid( $entity->via, - ag($entity->getMetadata($entity->via), iFace::COLUMN_ID) + ag($entity->getMetadata($entity->via), iState::COLUMN_ID) ); $this->objects[$pointer] = $this->objects[$pointer]->apply( entity: $entity, - fields: array_merge($localFields, [iFace::COLUMN_EXTRA]) + fields: array_merge($localFields, [iState::COLUMN_EXTRA]) ); $this->removePointers($cloned)->addPointers($this->objects[$pointer], $pointer); @@ -259,26 +259,26 @@ final class MemoryMapper implements ImportInterface } } - Data::increment($entity->via, $entity->type . '_ignored_not_played_since_last_sync'); + Message::increment("{$entity->via}.{$entity->type}.ignored_not_played_since_last_sync"); return $this; } } $keys = $opts['diff_keys'] ?? array_flip( array_keys_diff( - base: array_flip(iFace::ENTITY_KEYS), - list: iFace::ENTITY_IGNORE_DIFF_CHANGES, + base: array_flip(iState::ENTITY_KEYS), + list: iState::ENTITY_IGNORE_DIFF_CHANGES, has: false ) ); if (true === (clone $cloned)->apply(entity: $entity, fields: $keys)->isChanged(fields: $keys)) { $this->changed[$pointer] = $pointer; - Data::increment($entity->via, $entity->type . '_updated'); + Message::increment("{$entity->via}.{$entity->type}.updated"); $this->objects[$pointer] = $this->objects[$pointer]->apply( entity: $entity, - fields: array_merge($keys, [iFace::COLUMN_EXTRA]) + fields: array_merge($keys, [iState::COLUMN_EXTRA]) ); $this->removePointers($cloned)->addPointers($this->objects[$pointer], $pointer); @@ -311,17 +311,17 @@ final class MemoryMapper implements ImportInterface ]); } - Data::increment($entity->via, $entity->type . '_ignored_no_change'); + Message::increment("{$entity->via}.{$entity->type}.ignored_no_change"); return $this; } - public function get(iFace $entity): null|iFace + public function get(iState $entity): null|iState { return false === ($pointer = $this->getPointer($entity)) ? null : $this->objects[$pointer]; } - public function remove(iFace $entity): bool + public function remove(iState $entity): bool { if (false === ($pointer = $this->getPointer($entity))) { return false; @@ -344,8 +344,8 @@ final class MemoryMapper implements ImportInterface { $state = $this->storage->transactional(function (StorageInterface $storage) { $list = [ - iFace::TYPE_MOVIE => ['added' => 0, 'updated' => 0, 'failed' => 0], - iFace::TYPE_EPISODE => ['added' => 0, 'updated' => 0, 'failed' => 0], + iState::TYPE_MOVIE => ['added' => 0, 'updated' => 0, 'failed' => 0], + iState::TYPE_EPISODE => ['added' => 0, 'updated' => 0, 'failed' => 0], ]; $count = count($this->changed); @@ -391,7 +391,7 @@ final class MemoryMapper implements ImportInterface return $state; } - public function has(iFace $entity): bool + public function has(iState $entity): bool { return null !== $this->get($entity); } @@ -449,7 +449,7 @@ final class MemoryMapper implements ImportInterface return true === (bool)ag($this->options, Options::DEBUG_TRACE, false); } - protected function addPointers(iFace $entity, string|int $pointer): ImportInterface + protected function addPointers(iState $entity, string|int $pointer): ImportInterface { foreach ($entity->getRelativePointers() as $key) { $this->pointers[$key] = $pointer; @@ -465,11 +465,11 @@ final class MemoryMapper implements ImportInterface /** * Is the object already mapped? * - * @param iFace $entity + * @param iState $entity * * @return int|string|bool int pointer for the object, Or false if not registered. */ - protected function getPointer(iFace $entity): int|string|bool + protected function getPointer(iState $entity): int|string|bool { if (null !== $entity->id && null !== ($this->objects[self::GUID . $entity->id] ?? null)) { return self::GUID . $entity->id; @@ -499,7 +499,7 @@ final class MemoryMapper implements ImportInterface return false; } - protected function removePointers(iFace $entity): ImportInterface + protected function removePointers(iState $entity): ImportInterface { foreach ($entity->getPointers() as $key) { $lookup = $key . '/' . $entity->type; diff --git a/src/Libs/Message.php b/src/Libs/Message.php new file mode 100644 index 00000000..c527bf15 --- /dev/null +++ b/src/Libs/Message.php @@ -0,0 +1,76 @@ +mapper = new DirectMapper($logger, $this->storage); $this->mapper->setOptions(options: ['class' => new StateEntity([])]); - Data::reset(); + Message::reset(); } public function test_add_conditions(): void diff --git a/tests/Mappers/Import/MemoryMapperTest.php b/tests/Mappers/Import/MemoryMapperTest.php index a84fdd41..b5056ac3 100644 --- a/tests/Mappers/Import/MemoryMapperTest.php +++ b/tests/Mappers/Import/MemoryMapperTest.php @@ -4,11 +4,11 @@ declare(strict_types=1); namespace Tests\Mappers\Import; -use App\Libs\Data; use App\Libs\Entity\StateEntity; -use App\Libs\Entity\StateInterface as iFace; +use App\Libs\Entity\StateInterface as iState; use App\Libs\Guid; use App\Libs\Mappers\Import\MemoryMapper; +use App\Libs\Message; use App\Libs\Storage\PDO\PDOAdapter; use App\Libs\Storage\StorageInterface; use Monolog\Handler\TestHandler; @@ -46,7 +46,7 @@ class MemoryMapperTest extends TestCase $this->mapper = new MemoryMapper($logger, $this->storage); $this->mapper->setOptions(options: ['class' => new StateEntity([])]); - Data::reset(); + Message::reset(); } public function test_loadData_null_date_conditions(): void @@ -68,7 +68,7 @@ class MemoryMapperTest extends TestCase { $time = time(); - $this->testEpisode[iFace::COLUMN_UPDATED] = $time; + $this->testEpisode[iState::COLUMN_UPDATED] = $time; $testMovie = new StateEntity($this->testMovie); $testEpisode = new StateEntity($this->testEpisode); @@ -97,8 +97,8 @@ class MemoryMapperTest extends TestCase $this->assertSame( [ - iFace::TYPE_MOVIE => ['added' => 1, 'updated' => 0, 'failed' => 0], - iFace::TYPE_EPISODE => ['added' => 1, 'updated' => 0, 'failed' => 0], + iState::TYPE_MOVIE => ['added' => 1, 'updated' => 0, 'failed' => 0], + iState::TYPE_EPISODE => ['added' => 1, 'updated' => 0, 'failed' => 0], ], $this->mapper->commit() ); @@ -106,7 +106,7 @@ class MemoryMapperTest extends TestCase // -- assert 0 as we have committed the changes to the db, and the state should have been reset. $this->assertCount(0, $this->mapper); - $testEpisode->metadata['home_plex'][iFace::COLUMN_GUIDS][Guid::GUID_TVRAGE] = '2'; + $testEpisode->metadata['home_plex'][iState::COLUMN_GUIDS][Guid::GUID_TVRAGE] = '2'; $this->mapper->add($testEpisode); @@ -114,8 +114,8 @@ class MemoryMapperTest extends TestCase $this->assertSame( [ - iFace::TYPE_MOVIE => ['added' => 0, 'updated' => 0, 'failed' => 0], - iFace::TYPE_EPISODE => ['added' => 0, 'updated' => 1, 'failed' => 0], + iState::TYPE_MOVIE => ['added' => 0, 'updated' => 0, 'failed' => 0], + iState::TYPE_EPISODE => ['added' => 0, 'updated' => 1, 'failed' => 0], ], $this->mapper->commit() ); @@ -128,7 +128,7 @@ class MemoryMapperTest extends TestCase $movie = $this->testMovie; $episode = $this->testEpisode; - foreach (iFace::ENTITY_ARRAY_KEYS as $key) { + foreach (iState::ENTITY_ARRAY_KEYS as $key) { if (null !== ($movie[$key] ?? null)) { ksort($movie[$key]); } @@ -170,7 +170,7 @@ class MemoryMapperTest extends TestCase $this->assertNull($this->mapper->get($testEpisode)); $this->mapper->loadData(makeDate($time - 1)); - $this->assertInstanceOf(iFace::class, $this->mapper->get($testEpisode)); + $this->assertInstanceOf(iState::class, $this->mapper->get($testEpisode)); } public function test_commit_conditions(): void @@ -185,25 +185,25 @@ class MemoryMapperTest extends TestCase $this->assertSame( [ - iFace::TYPE_MOVIE => ['added' => 1, 'updated' => 0, 'failed' => 0], - iFace::TYPE_EPISODE => ['added' => 1, 'updated' => 0, 'failed' => 0], + iState::TYPE_MOVIE => ['added' => 1, 'updated' => 0, 'failed' => 0], + iState::TYPE_EPISODE => ['added' => 1, 'updated' => 0, 'failed' => 0], ], $insert ); - $testMovie->metadata['home_plex'][iFace::COLUMN_GUIDS][Guid::GUID_ANIDB] = '1920'; - $testEpisode->metadata['home_plex'][iFace::COLUMN_GUIDS][Guid::GUID_ANIDB] = '1900'; + $testMovie->metadata['home_plex'][iState::COLUMN_GUIDS][Guid::GUID_ANIDB] = '1920'; + $testEpisode->metadata['home_plex'][iState::COLUMN_GUIDS][Guid::GUID_ANIDB] = '1900'; $this->mapper - ->add($testMovie, ['diff_keys' => iFace::ENTITY_KEYS]) - ->add($testEpisode, ['diff_keys' => iFace::ENTITY_KEYS]); + ->add($testMovie, ['diff_keys' => iState::ENTITY_KEYS]) + ->add($testEpisode, ['diff_keys' => iState::ENTITY_KEYS]); $updated = $this->mapper->commit(); $this->assertSame( [ - iFace::TYPE_MOVIE => ['added' => 0, 'updated' => 1, 'failed' => 0], - iFace::TYPE_EPISODE => ['added' => 0, 'updated' => 1, 'failed' => 0], + iState::TYPE_MOVIE => ['added' => 0, 'updated' => 1, 'failed' => 0], + iState::TYPE_EPISODE => ['added' => 0, 'updated' => 1, 'failed' => 0], ], $updated ); From 2447ecd2914200b0981786184f01d70a7f1a4e4c Mon Sep 17 00:00:00 2001 From: "Abdulmhsen B. A. A" Date: Tue, 28 Jun 2022 18:23:36 +0300 Subject: [PATCH 08/11] Made export smarter by marking items not found if waiting period for metadata exported. --- config/config.php | 2 ++ src/Commands/State/ExportCommand.php | 33 +++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/config/config.php b/config/config.php index 866fadc0..5f2a60dc 100644 --- a/config/config.php +++ b/config/config.php @@ -31,6 +31,8 @@ return (function () { 'export' => [ // -- Trigger full export mode if changes exceed X number. 'threshold' => env('WS_EXPORT_THRESHOLD', 1000), + // -- Extra margin for marking item not found for backend in export mode. Default 3 days. + 'not_found' => env('WS_EXPORT_NOT_FOUND', 259_200), ], 'episodes' => [ 'disable' => [ diff --git a/src/Commands/State/ExportCommand.php b/src/Commands/State/ExportCommand.php index 68a24a94..227c93dc 100644 --- a/src/Commands/State/ExportCommand.php +++ b/src/Commands/State/ExportCommand.php @@ -6,6 +6,7 @@ namespace App\Commands\State; use App\Command; use App\Libs\Config; +use App\Libs\Entity\StateInterface as iState; use App\Libs\Mappers\Import\DirectMapper; use App\Libs\Message; use App\Libs\Options; @@ -224,11 +225,35 @@ class ExportCommand extends Command foreach ($backends as $backend) { $name = ag($backend, 'name'); - if (null === ag($backend, 'export.lastSync', null)) { + if (null === ($lastSync = ag($backend, 'export.lastSync', null))) { continue; } if (false === ag_exists($entity->getMetadata(), $name)) { + $addedDate = ag($entity->getMetadata($entity->via), iState::COLUMN_META_DATA_ADDED_AT); + $extraMargin = (int)Config::get('export.not_found'); + + if (null !== $addedDate && $lastSync > ($addedDate + $extraMargin)) { + $this->logger->info( + 'SYSTEM: Ignoring [%(item.title)] for [%(backend)] waiting period for metadata expired.', + [ + 'backend' => $name, + 'item' => [ + 'id' => $entity->id, + 'title' => $entity->getName(), + ], + 'wait_period' => [ + 'added_at' => makeDate($addedDate), + 'extra_margin' => $extraMargin, + 'last_sync_at' => makeDate($lastSync), + 'diff' => $lastSync - ($addedDate + $extraMargin), + ], + ] + ); + + continue; + } + if (true === ag_exists($push, $name)) { unset($push[$name]); } @@ -241,6 +266,12 @@ class ExportCommand extends Command 'id' => $entity->id, 'title' => $entity->getName(), ], + 'wait_period' => [ + 'added_at' => makeDate($addedDate), + 'extra_margin' => $extraMargin, + 'last_sync_at' => makeDate($lastSync), + 'diff' => $lastSync - ($addedDate + $extraMargin), + ], ] ); From 84b4c9eaf7ec924298edf5f52317220ab0bc8973 Mon Sep 17 00:00:00 2001 From: "Abdulmhsen B. A. A" Date: Tue, 28 Jun 2022 20:06:20 +0300 Subject: [PATCH 09/11] Support adding scope to ignore guid rule. --- src/Backends/Jellyfin/JellyfinGuid.php | 5 +- src/Backends/Plex/PlexGuid.php | 5 +- src/Commands/Backend/Ignore/ListCommand.php | 20 +++- src/Commands/Backend/Ignore/ManageCommand.php | 41 +++++-- src/Libs/Initializer.php | 9 +- src/Libs/helpers.php | 103 +++++++++++++++++- 6 files changed, 163 insertions(+), 20 deletions(-) diff --git a/src/Backends/Jellyfin/JellyfinGuid.php b/src/Backends/Jellyfin/JellyfinGuid.php index 3955b4f1..ba27ca7d 100644 --- a/src/Backends/Jellyfin/JellyfinGuid.php +++ b/src/Backends/Jellyfin/JellyfinGuid.php @@ -79,12 +79,13 @@ class JellyfinGuid implements iGuid } try { + $id = ag($context, 'item.id', null); $type = ag($context, 'item.type', '??'); $type = JellyfinClient::TYPE_MAPPER[$type] ?? $type; - if (true === isIgnoredId($this->context->backendName, $type, $key, $value)) { + if (true === isIgnoredId($this->context->backendName, $type, $key, $value, $id)) { if (true === $log) { - $this->logger->info( + $this->logger->notice( 'Ignoring [%(backend)] external id [%(source)] for %(item.type) [%(item.title)] as requested.', [ 'backend' => $this->context->backendName, diff --git a/src/Backends/Plex/PlexGuid.php b/src/Backends/Plex/PlexGuid.php index 7a04f25d..122e2fc0 100644 --- a/src/Backends/Plex/PlexGuid.php +++ b/src/Backends/Plex/PlexGuid.php @@ -140,12 +140,13 @@ final class PlexGuid implements GuidInterface continue; } + $id = ag($context, 'item.id', null); $type = ag($context, 'item.type', '??'); $type = PlexClient::TYPE_MAPPER[$type] ?? $type; - if (true === isIgnoredId($this->context->backendName, $type, $key, $value)) { + if (true === isIgnoredId($this->context->backendName, $type, $key, $value, $id)) { if (true === $log) { - $this->logger->info( + $this->logger->notice( 'Ignoring [%(backend)] external id [%(source)] for %(item.type) [%(item.title)] as requested.', [ 'backend' => $this->context->backendName, diff --git a/src/Commands/Backend/Ignore/ListCommand.php b/src/Commands/Backend/Ignore/ListCommand.php index 9b0fd30b..745a0681 100644 --- a/src/Commands/Backend/Ignore/ListCommand.php +++ b/src/Commands/Backend/Ignore/ListCommand.php @@ -48,6 +48,12 @@ HELP protected function runCommand(InputInterface $input, OutputInterface $output): int { + $path = Config::get('path') . '/config/ignore.yaml'; + + if (false === file_exists($path)) { + touch($path); + } + $list = []; $fBackend = $input->getOption('backend'); @@ -64,6 +70,7 @@ HELP $type = ag($urlParts, 'scheme'); $db = ag($urlParts, 'user'); $id = ag($urlParts, 'pass'); + $scope = ag($urlParts, 'query'); if (null !== $fBackend && $backend !== $fBackend) { continue; @@ -81,13 +88,24 @@ HELP continue; } - $list[] = [ + $builder = [ 'backend' => $backend, 'type' => $type, 'db' => $db, 'id' => $id, + 'Scoped' => null === $scope ? 'No' : 'Yes', 'created' => makeDate($date), ]; + + if ('table' !== $input->getOption('output')) { + $builder = ['rule' => (string)makeIgnoreId($guid)] + $builder; + $builder['scope'] = []; + if (null !== $scope) { + parse_str($scope, $builder['scope']); + } + } + + $list[] = $builder; } if (empty($list)) { diff --git a/src/Commands/Backend/Ignore/ManageCommand.php b/src/Commands/Backend/Ignore/ManageCommand.php index 0dee06a8..93d444d3 100644 --- a/src/Commands/Backend/Ignore/ManageCommand.php +++ b/src/Commands/Backend/Ignore/ManageCommand.php @@ -34,7 +34,7 @@ This command allow you to ignore specific external id from backend. This helps when there is a conflict between your media servers provided external ids. Generally this should only be used as last resort. You should try to fix the source of the problem. -The id format is: type://db:id@backend_name +The id format is: type://db:id@backend_name[?id=backend_id] ----------------------------- How to Add id to ignore list. @@ -51,6 +51,14 @@ For movies: For episodes: {$cmdContext} servers:ignore episode://tvdb:320234@plex_home +To scope ignore rule to specfic item from backend, You can do the same as and add [?id=backend_id]. + +[backend_id]: + +Refers to the item id from backend. To ignore a specfic guid for item id 1212111 you can do something like this: + +{$cmdContext} servers:ignore episode://tvdb:320234@plex_home?id=1212111 + ---------------------------------- How to Remove id from ignore list. ---------------------------------- @@ -95,18 +103,37 @@ HELP $output->writeln(sprintf('Removed: id \'%s\' from ignore list.', $id)); } else { $this->checkGuid($id); - if (true === ag_exists($list, $id)) { + + $id = makeIgnoreId($id); + + if (true === ag_exists($list, (string)$id)) { $output->writeln( - sprintf( - 'Id \'%s\' already exists in the ignore list. added at \'%s\'.', - $id, - makeDate(ag($list, $id))->format('Y-m-d H:i:s T') + replacer( + 'ERROR: Cannot add [{id}] as it\'s already exists. added at [{date}].', + [ + 'id' => $id, + 'date' => makeDate(ag($list, (string)$id))->format('Y-m-d H:i:s T'), + ], ) ); return self::FAILURE; } - $list = ag_set($list, $id, makeDate()); + if (true === ag_exists($list, (string)$id->withQuery(''))) { + $output->writeln( + replacer( + 'ERROR: Cannot add [{id}] as [{global}] already exists. added at [{date}].', + [ + 'id' => (string)$id, + 'global' => (string)$id->withQuery(''), + 'date' => makeDate(ag($list, (string)$id->withQuery('')))->format('Y-m-d H:i:s T') + ] + ) + ); + return self::FAILURE; + } + + $list = ag_set($list, (string)$id, makeDate()); $output->writeln(sprintf('Added: id \'%s\' to ignore list.', $id)); } diff --git a/src/Libs/Initializer.php b/src/Libs/Initializer.php index db8e019a..249c282f 100644 --- a/src/Libs/Initializer.php +++ b/src/Libs/Initializer.php @@ -78,9 +78,14 @@ final class Initializer } $path = Config::get('path') . '/config/ignore.yaml'; - if (file_exists($path)) { - Config::save('ignore', Yaml::parseFile($path)); + if (($yaml = Yaml::parseFile($path)) && is_array($yaml)) { + $list = []; + foreach ($yaml as $key => $val) { + $list[(string)makeIgnoreId($key)] = $val; + } + Config::save('ignore', $list); + } } })(); diff --git a/src/Libs/helpers.php b/src/Libs/helpers.php index a86daf33..78146d3d 100644 --- a/src/Libs/helpers.php +++ b/src/Libs/helpers.php @@ -11,6 +11,7 @@ use Nyholm\Psr7\Response; use Nyholm\Psr7\Uri; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ServerRequestInterface; +use Psr\Http\Message\UriInterface; use Psr\Log\LoggerInterface; use Psr\SimpleCache\CacheInterface; use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface; @@ -567,16 +568,106 @@ if (false === function_exists('getPeakMemoryUsage')) { } } -if (false === function_exists('isIgnoredId')) { - function isIgnoredId(string $backend, string $type, string $db, string|int $id): bool +if (false === function_exists('makeIgnoreId')) { + function makeIgnoreId(string $url): UriInterface { + static $filterQuery = null; + + if (null === $filterQuery) { + $filterQuery = function (string $query): string { + $list = $final = []; + $allowed = ['id']; + + parse_str($query, $list); + + foreach ($list as $key => $val) { + if (false === in_array($key, $allowed) || empty($val)) { + continue; + } + + $final[$key] = $val; + } + + return http_build_query($final); + }; + } + + $id = (new Uri($url))->withPath('')->withFragment('')->withPort(null); + return $id->withQuery($filterQuery($id->getQuery())); + } +} + +if (false === function_exists('isIgnoredId')) { + function isIgnoredId( + string $backend, + string $type, + string $db, + string|int $id, + string|int|null $backendId = null + ): bool { if (false === in_array($type, iFace::TYPES_LIST)) { throw new RuntimeException(sprintf('Invalid context type \'%s\' was given.', $type)); } - return ag_exists( - Config::get('ignore', []), - sprintf('%s://%s:%s@%s', $type, $db, $id, $backend) - ); + $list = Config::get('ignore', []); + + $key = makeIgnoreId(sprintf('%s://%s:%s@%s?id=%s', $type, $db, $id, $backend, $backendId)); + + if (null !== ($list[(string)$key->withQuery('')] ?? null)) { + return true; + } + + if (null === $backendId) { + return false; + } + + return null !== ($list[(string)$key] ?? null); + } +} + +if (false === function_exists('replacer')) { + function replacer(string $text, array $context = []): string + { + if (false === str_contains($text, '{') || false === str_contains($text, '}')) { + return $text; + } + + $pattern = '#' . preg_quote('{', '#') . '([\w\d_.]+)' . preg_quote('}', '#') . '#is'; + + $status = preg_match_all($pattern, $text, $matches); + + if (false === $status || $status < 1) { + return $text; + } + + $replacements = []; + + foreach ($matches[1] as $key) { + $placeholder = '{' . $key . '}'; + + if (false === str_contains($text, $placeholder)) { + continue; + } + + if (false === ag_exists($context, $key)) { + continue; + } + + $val = ag($context, $key); + + $context = ag_delete($context, $key); + + if (is_null($val) || is_scalar($val) || (is_object($val) && method_exists($val, '__toString'))) { + $replacements[$placeholder] = $val; + } elseif (is_object($val)) { + $replacements[$placeholder] = implode(',', get_object_vars($val)); + } elseif (is_array($val)) { + $replacements[$placeholder] = implode(',', $val); + } else { + $replacements[$placeholder] = '[' . gettype($val) . ']'; + } + } + + return strtr($text, $replacements); } } From 69158b84ccb2197a4472260146ec3fa7ec4fec31 Mon Sep 17 00:00:00 2001 From: "Abdulmhsen B. A. A" Date: Tue, 28 Jun 2022 21:05:15 +0300 Subject: [PATCH 10/11] use time() instead makeDate for ignorelist created_at --- src/Commands/Backend/Ignore/ManageCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Commands/Backend/Ignore/ManageCommand.php b/src/Commands/Backend/Ignore/ManageCommand.php index 93d444d3..b9cfde9c 100644 --- a/src/Commands/Backend/Ignore/ManageCommand.php +++ b/src/Commands/Backend/Ignore/ManageCommand.php @@ -133,7 +133,7 @@ HELP return self::FAILURE; } - $list = ag_set($list, (string)$id, makeDate()); + $list = ag_set($list, (string)$id, time()); $output->writeln(sprintf('Added: id \'%s\' to ignore list.', $id)); } From cfbb87d0126b787e39ce908fd8e5423b635671e3 Mon Sep 17 00:00:00 2001 From: "Abdulmhsen B. A. A" Date: Tue, 28 Jun 2022 22:19:45 +0300 Subject: [PATCH 11/11] Improved the display of ignorelist to include title of the entity if available. --- src/Commands/Backend/Ignore/ListCommand.php | 104 ++++++++++++++++++-- src/Libs/Entity/StateEntity.php | 6 +- src/Libs/Entity/StateInterface.php | 4 +- 3 files changed, 102 insertions(+), 12 deletions(-) diff --git a/src/Commands/Backend/Ignore/ListCommand.php b/src/Commands/Backend/Ignore/ListCommand.php index 745a0681..c1902102 100644 --- a/src/Commands/Backend/Ignore/ListCommand.php +++ b/src/Commands/Backend/Ignore/ListCommand.php @@ -6,8 +6,14 @@ namespace App\Commands\Backend\Ignore; use App\Command; use App\Libs\Config; -use App\Libs\Entity\StateInterface as iFace; +use App\Libs\Container; +use App\Libs\Entity\StateInterface as iState; use App\Libs\Guid; +use App\Libs\Storage\StorageInterface; +use PDO; +use Psr\Http\Message\UriInterface; +use Psr\SimpleCache\CacheInterface; +use Psr\SimpleCache\InvalidArgumentException; use Symfony\Component\Console\Completion\CompletionInput; use Symfony\Component\Console\Completion\CompletionSuggestions; use Symfony\Component\Console\Input\InputInterface; @@ -16,6 +22,29 @@ use Symfony\Component\Console\Output\OutputInterface; final class ListCommand extends Command { + private const CACHE_KEY = 'ignorelist_titles'; + + private array $cache = []; + + private PDO $db; + private CacheInterface $cacheIO; + + public function __construct(StorageInterface $storage, CacheInterface $cacheIO) + { + $this->cacheIO = $cacheIO; + $this->db = $storage->getPdo(); + + try { + if ($this->cacheIO->has(self::CACHE_KEY)) { + $this->cache = $this->cacheIO->get(self::CACHE_KEY); + } + } catch (InvalidArgumentException) { + $this->cache = []; + } + + parent::__construct(); + } + protected function configure(): void { $cmdContext = trim(commandContext()); @@ -25,6 +54,7 @@ final class ListCommand extends Command ->addOption('backend', null, InputOption::VALUE_REQUIRED, 'Filter based on backend.') ->addOption('db', null, InputOption::VALUE_REQUIRED, 'Filter based on db.') ->addOption('id', null, InputOption::VALUE_REQUIRED, 'Filter based on id.') + ->addOption('with-title', null, InputOption::VALUE_NONE, 'Include entity title in response. Slow operation') ->setDescription('List Ignored external ids.') ->setHelp( << ucfirst($type), 'backend' => $backend, - 'type' => $type, 'db' => $db, 'id' => $id, 'Scoped' => null === $scope ? 'No' : 'Yes', - 'created' => makeDate($date), ]; + if (!empty($this->cache) || $input->getOption('with-title')) { + $builder['title'] = null !== $scope ? ($this->getinfo($rule) ?? 'Unknown') : '** Global Rule **'; + } + if ('table' !== $input->getOption('output')) { - $builder = ['rule' => (string)makeIgnoreId($guid)] + $builder; + $builder = ['rule' => (string)$rule] + $builder; $builder['scope'] = []; if (null !== $scope) { parse_str($scope, $builder['scope']); } + $builder['created'] = makeDate($date); + } else { + $builder['created'] = makeDate($date)->format('Y-m-d H:i:s T'); } - $list[] = $builder; } - + if (empty($list)) { $hasIds = count($ids) >= 1; + $output->writeln( $hasIds ? 'Filters did not return any results.' : 'Ignore list is empty.' ); - if ($hasIds) { + + if (true === $hasIds) { return self::FAILURE; } } @@ -123,6 +162,55 @@ HELP return self::SUCCESS; } + private function getInfo(UriInterface $uri): string|null + { + if (empty($uri->getQuery())) { + return null; + } + + $params = []; + parse_str($uri->getQuery(), $params); + + $key = sprintf('%s://%s@%s', $uri->getScheme(), $uri->getHost(), $params['id']); + + if (true === array_key_exists($key, $this->cache)) { + return $this->cache[$key]; + } + + $sql = sprintf( + "SELECT * FROM state WHERE JSON_EXTRACT(metadata, '$.%s.%s') = :id LIMIT 1", + $uri->getHost(), + $uri->getScheme() === iState::TYPE_SHOW ? 'show' : 'id' + ); + + $stmt = $this->db->prepare($sql); + $stmt->execute(['id' => $params['id']]); + $item = $stmt->fetch(PDO::FETCH_ASSOC); + + if (empty($item)) { + $this->cache[$key] = null; + return null; + } + + $this->cache[$key] = Container::get(iState::class)->fromArray($item)->getName( + iState::TYPE_SHOW === $uri->getScheme() + ); + + return $this->cache[$key]; + } + + public function __destruct() + { + if (empty($this->cache)) { + return; + } + + try { + $this->cacheIO->set(self::CACHE_KEY, $this->cache, new \DateInterval('P3D')); + } catch (InvalidArgumentException) { + } + } + public function complete(CompletionInput $input, CompletionSuggestions $suggestions): void { if ($input->mustSuggestOptionValuesFor('backend')) { @@ -143,7 +231,7 @@ HELP $suggest = []; - foreach (iFace::TYPES_LIST as $name) { + foreach (iState::TYPES_LIST as $name) { if (empty($currentValue) || str_starts_with($name, $currentValue)) { $suggest[] = $name; } diff --git a/src/Libs/Entity/StateEntity.php b/src/Libs/Entity/StateEntity.php index dcdeb2c8..4aa4af63 100644 --- a/src/Libs/Entity/StateEntity.php +++ b/src/Libs/Entity/StateEntity.php @@ -4,9 +4,9 @@ declare(strict_types=1); namespace App\Libs\Entity; +use App\Libs\Entity\StateInterface as iFace; use App\Libs\Guid; use RuntimeException; -use App\Libs\Entity\StateInterface as iFace; final class StateEntity implements iFace { @@ -100,12 +100,12 @@ final class StateEntity implements iFace return $changed; } - public function getName(): string + public function getName(bool $asMovie = false): string { $title = ag($this->data, iFace::COLUMN_TITLE, $this->title); $year = ag($this->data, iFace::COLUMN_YEAR, $this->year); - if ($this->isMovie()) { + if ($this->isMovie() || true === $asMovie) { return sprintf('%s (%s)', $title, $year ?? '0000'); } diff --git a/src/Libs/Entity/StateInterface.php b/src/Libs/Entity/StateInterface.php index ec3a4d80..fe2a8a59 100644 --- a/src/Libs/Entity/StateInterface.php +++ b/src/Libs/Entity/StateInterface.php @@ -199,9 +199,11 @@ interface StateInterface /** * Get constructed name. * + * @param bool $asMovie Return episode title as movie format. + * * @return string */ - public function getName(): string; + public function getName(bool $asMovie = false): string; /** * Get external ids Pointers.