cphipps/watchstate
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

PLEX: Implemented pagination for get library items.

Browse Source
This commit is contained in:
Abdulmhsen B. A. A
2022-07-24 19:42:33 +03:00
parent b745dad3eb
commit 1dae2813a2
5 changed files with 240 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
FAQ.md
View File

@@ -146,21 +146,22 @@ $ docker exec -ti watchstate console system:env
These environment variables relates to the tool itself, you can load them via the recommended methods.
| Key | Type | Description | Default |
|--------------------------|--------|-------------------------------------------------------------------------------|--------------------|
| WS_DATA_PATH | string | Where to store main data. (config, db). | `${BASE_PATH}/var` |
| WS_TMP_DIR | string | Where to store temp data. (logs, cache) | `${WS_DATA_PATH}` |
| WS_TZ | string | Set timezone. | `UTC` |
| WS_CRON_{TASK} | bool | Enable {task} task. Value casted to bool. | `false` |
| WS_CRON_{TASK}_AT | string | When to run {task} task. Valid Cron Expression Expected. | `*/1 * * * *` |
| WS_CRON_{TASK}_ARGS | string | Flags to pass to the {task} command. | `-v` |
| WS_LOGS_CONTEXT | bool | Add context to console output messages. | `false` |
| WS_LOGGER_FILE_ENABLE | bool | Save logs to file. | `true` |
| WS_LOGGER_FILE_LEVEL | string | File Logger Level. | `ERROR` |
| WS_WEBHOOK_DEBUG | bool | If enabled, allow dumping request/webhook using `rdump` & `wdump` parameters. | `false` |
| WS_EPISODES_DISABLE_GUID | bool | Disable external id parsing for episodes and rely on relative ids. | `true` |
| WS_TRUST_PROXY | bool | Trust `WS_TRUST_HEADER` ip. Value casted to bool. | `false` |
| WS_TRUST_HEADER | string | Which header contain user true IP. | `X-Forwarded-For` |
| Key | Type | Description | Default |
|--------------------------|---------|-------------------------------------------------------------------------------|--------------------|
| WS_DATA_PATH | string | Where to store main data. (config, db). | `${BASE_PATH}/var` |
| WS_TMP_DIR | string | Where to store temp data. (logs, cache) | `${WS_DATA_PATH}` |
| WS_TZ | string | Set timezone. | `UTC` |
| WS_CRON_{TASK} | bool | Enable {task} task. Value casted to bool. | `false` |
| WS_CRON_{TASK}_AT | string | When to run {task} task. Valid Cron Expression Expected. | `*/1 * * * *` |
| WS_CRON_{TASK}_ARGS | string | Flags to pass to the {task} command. | `-v` |
| WS_LOGS_CONTEXT | bool | Add context to console output messages. | `false` |
| WS_LOGGER_FILE_ENABLE | bool | Save logs to file. | `true` |
| WS_LOGGER_FILE_LEVEL | string | File Logger Level. | `ERROR` |
| WS_WEBHOOK_DEBUG | bool | If enabled, allow dumping request/webhook using `rdump` & `wdump` parameters. | `false` |
| WS_EPISODES_DISABLE_GUID | bool | Disable external id parsing for episodes and rely on relative ids. | `true` |
| WS_TRUST_PROXY | bool | Trust `WS_TRUST_HEADER` ip. Value casted to bool. | `false` |
| WS_TRUST_HEADER | string | Which header contain user true IP. | `X-Forwarded-For` |
| WS_LIBRARY_SEGMENT | integer | Paginate backend library items request. Per request get total X number. | `8000` |
**Note**: for environment variables that has `{TASK}` tag, you **MUST** replace it with one
of `IMPORT`, `EXPORT`, `PUSH`, `BACKUP`, `PRUNE`, `INDEXES`. To see tasks active settings run

8
config/config.php
View File

@@ -30,11 +30,15 @@ return (function () {
'database' => [
'version' => 'v01',
],
'library' => [
// -- this is used to segment backends requests into pages.
'segment' => (int)env('WS_LIBRARY_SEGMENT', 8_000),
],
'export' => [
// -- Trigger full export mode if changes exceed X number.
'threshold' => env('WS_EXPORT_THRESHOLD', 1000),
'threshold' => (int)env('WS_EXPORT_THRESHOLD', 1_000),
// -- Extra margin for marking item not found for backend in export mode. Default 3 days.
'not_found' => env('WS_EXPORT_NOT_FOUND', 259_200),
'not_found' => (int)env('WS_EXPORT_NOT_FOUND', 259_200),
],
'episodes' => [
'disable' => [

241
src/Backends/Plex/Action/Import.php
View File

@@ -157,7 +157,7 @@ class Import
$ignoreIds = array_map(fn($v) => (int)trim($v), explode(',', (string)$ignoreIds));
}
$requests = [];
$requests = $libraryTotalItems = $total = [];
$ignored = $unsupported = 0;
// -- Get TV shows metadata.
@@ -236,7 +236,134 @@ class Import
}
}
// -- Get Movies/episodes.
// -- Get library items count.
foreach ($listDirs as $section) {
$key = (int)ag($section, 'key');
$logContext = [
'library' => [
'id' => ag($section, 'key'),
'title' => ag($section, 'title', '??'),
'type' => ag($section, 'type', 'unknown'),
],
];
if (true === in_array($key, $ignoreIds ?? [])) {
continue;
}
if (!in_array(ag($logContext, 'library.type'), [PlexClient::TYPE_MOVIE, PlexClient::TYPE_SHOW])) {
continue;
}
$isMovieLibrary = PlexClient::TYPE_MOVIE === ag($logContext, 'library.type');
$url = $context->backendUrl->withPath(sprintf('/library/sections/%d/all', $key))->withQuery(
http_build_query([
'includeGuids' => 1,
'type' => $isMovieLibrary ? 1 : 4,
'sort' => $isMovieLibrary ? 'addedAt' : 'episode.addedAt',
])
);
$logContext['library']['url'] = $url;
$this->logger->debug('Requesting [%(backend)] [%(library.title)] items count.', [
'backend' => $context->backendName,
...$logContext,
]);
try {
$libraryTotalItems[] = $this->http->request(
'GET',
(string)$url,
array_replace_recursive($context->backendHeaders, [
'headers' => [
'X-Plex-Container-Start' => 0,
'X-Plex-Container-Size' => 0,
],
'user_data' => $logContext,
])
);
} catch (ExceptionInterface $e) {
$this->logger->error('Request for [%(backend)] [%(library.title)] items count has failed.', [
'backend' => $context->backendName,
...$logContext,
'exception' => [
'file' => $e->getFile(),
'line' => $e->getLine(),
'kind' => get_class($e),
'message' => $e->getMessage(),
'trace' => $context->trace ? $e->getTrace() : [],
],
]);
continue;
} catch (Throwable $e) {
$this->logger->error(
'Unhandled exception was thrown during [%(backend)] [%(library.title)] items count request.',
[
'backend' => $context->backendName,
...$logContext,
'exception' => [
'file' => $e->getFile(),
'line' => $e->getLine(),
'kind' => get_class($e),
'message' => $e->getMessage(),
'trace' => $context->trace ? $e->getTrace() : [],
],
]
);
continue;
}
}
foreach ($libraryTotalItems as $response) {
$logContext = ag($response->getInfo('user_data'), []);
try {
if (200 !== $response->getStatusCode()) {
$this->logger->error(
'Request for [%(backend)] [%(library.title)] items count returned with unexpected [%(status_code)] status code.',
[
'backend' => $context->backendName,
'status_code' => $response->getStatusCode(),
...$logContext,
]
);
continue;
}
$totalCount = (int)(ag($response->getHeaders(), 'x-plex-container-total-size')[0] ?? 0);
if ($totalCount < 1) {
$this->logger->warning(
'Request for [%(backend)] [%(library.title)] items count returned with total number of 0.',
[
'backend' => $context->backendName,
...$logContext,
'headers' => $response->getHeaders(),
]
);
continue;
}
$total[(int)ag($logContext, 'library.id')] = $totalCount;
} catch (ExceptionInterface $e) {
$this->logger->error('Request for [%(backend)] [%(library.title)] total items has failed.', [
'backend' => $context->backendName,
...$logContext,
'exception' => [
'file' => $e->getFile(),
'line' => $e->getLine(),
'kind' => get_class($e),
'message' => $e->getMessage(),
'trace' => $context->trace ? $e->getTrace() : [],
],
]);
continue;
}
}
foreach ($listDirs as $section) {
$key = (int)ag($section, 'key');
@@ -269,49 +396,85 @@ class Import
continue;
}
$url = $context->backendUrl->withPath(sprintf('/library/sections/%d/all', $key))->withQuery(
http_build_query(
[
'type' => PlexClient::TYPE_MOVIE === ag($logContext, 'library.type') ? 1 : 4,
'includeGuids' => 1,
]
)
);
$logContext['library']['url'] = $url;
$this->logger->debug('Requesting [%(backend)] [%(library.title)] content list.', [
'backend' => $context->backendName,
...$logContext,
]);
try {
$requests[] = $this->http->request(
'GET',
(string)$url,
$context->backendHeaders + [
'user_data' => [
'ok' => $handle($logContext),
'error' => $error($logContext),
]
],
);
} catch (ExceptionInterface $e) {
$this->logger->error('Requesting for [%(backend)] [%(library.title)] content list has failed.', [
if (false === array_key_exists($key, $total)) {
$ignored++;
$this->logger->warning('Ignoring [%(backend)] [%(library.title)]. No items count was found.', [
'backend' => $context->backendName,
...$logContext,
'exception' => [
'file' => $e->getFile(),
'line' => $e->getLine(),
'kind' => get_class($e),
'message' => $e->getMessage(),
'trace' => $context->trace ? $e->getTrace() : [],
],
]);
continue;
}
try {
$segmentTotal = (int)$total[ag($logContext, 'library.id')];
$segmentSize = (int)ag($context->options, Options::LIBRARY_SEGMENT, 1000);
$segmented = ceil($segmentTotal / $segmentSize);
for ($i = 0; $i < $segmented; $i++) {
$logContext['segment'] = [
'number' => $i + 1,
'of' => $segmented,
'size' => $segmentSize,
'total' => $segmentTotal,
];
$isMovieLibrary = PlexClient::TYPE_MOVIE === ag($logContext, 'library.type');
$url = $context->backendUrl->withPath(sprintf('/library/sections/%d/all', $key))->withQuery(
http_build_query(
[
'includeGuids' => 1,
'type' => $isMovieLibrary ? 1 : 4,
'sort' => $isMovieLibrary ? 'addedAt' : 'episode.addedAt',
'X-Plex-Container-Size' => $segmentSize,
'X-Plex-Container-Start' => $i < 1 ? 0 : ($segmentSize * $i),
]
)
);
$logContext['library']['url'] = $url;
$this->logger->debug(
'Requesting [%(backend)] [%(library.title)] [%(segment.number)/%(segment.of)] content list.',
[
'backend' => $context->backendName,
...$logContext,
]
);
$requests[] = $this->http->request(
'GET',
(string)$url,
array_replace_recursive($context->backendHeaders, [
'headers' => [
'X-Plex-Container-Size' => $segmentSize,
'X-Plex-Container-Start' => $i < 1 ? 0 : ($segmentSize * $i),
],
'user_data' => [
'ok' => $handle($logContext),
'error' => $error($logContext),
]
]),
);
}
} catch (ExceptionInterface $e) {
$this->logger->error(
'Request for [%(backend)] [%(library.title)] [%(segment.number)/%(segment.of)] content list has failed.',
[
'backend' => $context->backendName,
...$logContext,
'exception' => [
'file' => $e->getFile(),
'line' => $e->getLine(),
'kind' => get_class($e),
'message' => $e->getMessage(),
'trace' => $context->trace ? $e->getTrace() : [],
],
]
);
continue;
} catch (Throwable $e) {
$this->logger->error(
'Unhandled exception was thrown during [%(backend)] [%(library.title)] content list request.',
'Unhandled exception was thrown during [%(backend)] [%(library.title)] [%(segment.number)/%(segment.of)] content list request.',
[
'backend' => $context->backendName,
...$logContext,

23
src/Backends/Plex/PlexClient.php
View File

@@ -81,18 +81,21 @@ class PlexClient implements iClient
backendId: $context->backendId,
backendToken: $context->backendToken,
backendUser: $context->backendUser,
backendHeaders: array_replace_recursive(
[
'headers' => [
'Accept' => 'application/json',
'X-Plex-Token' => $context->backendToken,
'X-Plex-Container-Size' => 0,
],
backendHeaders: array_replace_recursive([
'headers' => [
'Accept' => 'application/json',
'X-Plex-Token' => $context->backendToken,
'X-Plex-Container-Size' => 0,
],
ag($context->options, 'client', [])
),
], ag($context->options, 'client', [])),
trace: true === ag($context->options, Options::DEBUG_TRACE),
options: $context->options
options: array_replace_recursive($context->options, [
Options::LIBRARY_SEGMENT => (int)ag(
$context->options,
Options::LIBRARY_SEGMENT,
Config::get('library.segment')
),
])
);
$cloned->guid = $cloned->guid->withContext($cloned->context);

5
src/Libs/Options.php
View File

@@ -7,8 +7,8 @@ namespace App\Libs;
final class Options
{
public const DRY_RUN = 'DRY_RUN';
public const NO_CACHE = 'no_cache';
public const CACHE_TTL = 'cache_ttl';
public const NO_CACHE = 'NO_CACHE';
public const CACHE_TTL = 'CACHE_TTL';
public const FORCE_FULL = 'FORCE_FULL';
public const DEBUG_TRACE = 'DEBUG_TRACE';
public const IGNORE_DATE = 'IGNORE_DATE';
@@ -19,6 +19,7 @@ final class Options
public const IMPORT_METADATA_ONLY = 'IMPORT_METADATA_ONLY';
public const MISMATCH_DEEP_SCAN = 'MISMATCH_DEEP_SCAN';
public const DISABLE_GUID = 'DISABLE_GUID';
public const LIBRARY_SEGMENT = 'LIBRARY_SEGMENT';
private function __construct()
{