cphipps/watchstate
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Simplified mismatch, and started moving backends actions into separate commands. update #130

Browse Source
This commit is contained in:
Abdulmhsen B. A. A
2022-05-31 09:31:38 +03:00
parent ffb7b2497e
commit c0b4264680
10 changed files with 765 additions and 593 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
FAQ.md
View File

@@ -141,7 +141,7 @@ can run the same command with `[-h, --help]` to see more options to extend the l
Yes, First run the following command
```bash
$ docker exec -ti watchstate console servers:remote --list-libraries -- [SERVER_NAME]
$ docker exec -ti watchstate console backend:library:list -- [SERVER_NAME]
```
it should show you list of given server libraries, you are mainly interested in the ID column. take note of the library
@@ -223,34 +223,40 @@ $ docker exec -ti console server servers:remote --search-id 2514 -- [SERVER_NAME
---
### Q: Is it possible to look for possible unmatched items?
### Q: Is there anyway to look for possible unmatched items?
Yes, You can use the flag `--search-mismatch '[library_id]'` in `servers:remote`, For example
Yes, You can use the command `backend:library:mismatch`, For example
first get your library id by running the following command
```bash
$ docker exec -ti watchstate console servers:remote --list-libraries -- [SERVER_NAME]
$ docker exec -ti watchstate console backend:library:list -- [SERVER_NAME]
```
it should display something like
| ID | Title | Type | Ignored | Supported |
| Id | Title | Type | Ignored | Supported |
|-----|-------------|--------|---------|-----------|
| 2 | Movies | movie | No | Yes |
| 1 | shows | show | No | Yes |
| 17 | Audio Books | artist | Yes | No |
Then choose the library id that you want to scan, after that run the following command:
Note the library id that you want to scan for possible mis-identified items, then run the following command:
```bash
$ docker exec -ti console server servers:remote --search-mismatch [ID e.g. 2] --search-coef 60 -- [SERVER_NAME]
$ docker exec -ti console server backend:library:mismatch --id [LIBRARY_ID] -- [BACKEND_NAME]
```
### Optional flags that can be used with `--search-mismach`
### Required flags
* `--search-coef` How much in percentage the title has to be in path to be marked as matched item. Defaults to `50.0`.
* `--search-output` Set output style, it can be `yaml` or `json`. Defaults to `json`.
* `[-i, --id]` Library id.
### Optional flags
* `[-p, --percentage]` How much in percentage the title has to be in path to be marked as matched item. Defaults
to `50.0%`.
* `[-o, --output]` Set output mode, it can be `yaml`, `json` or `table`. Defaults to `table`.
* `[-m, --method]` Which algorithm to use, it can be `similarity`, or `levenshtein`. Defaults to `similarity`.
---

4
config/commands.php
View File

@@ -29,4 +29,8 @@ return [
// -- db:
'db:list' => App\Commands\Database\ListCommand::class,
'db:queue' => App\Commands\Database\QueueCommand::class,
// -- backend:library
'backend:library:list' => App\Commands\Backend\Library\ListCommand::class,
'backend:library:mismatch' => App\Commands\Backend\Library\MismatchCommand::class,
];

71
src/Command.php
View File

@@ -5,20 +5,30 @@ declare(strict_types=1);
namespace App;
use App\Libs\Config;
use App\Libs\Servers\ServerInterface;
use DirectoryIterator;
use RuntimeException;
use Symfony\Component\Console\Command\Command as BaseCommand;
use Symfony\Component\Console\Command\LockableTrait;
use Symfony\Component\Console\Completion\CompletionInput;
use Symfony\Component\Console\Completion\CompletionSuggestions;
use Symfony\Component\Console\Helper\Table;
use Symfony\Component\Console\Helper\TableSeparator;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Yaml\Yaml;
use Xhgui\Profiler\Profiler;
class Command extends BaseCommand
{
use LockableTrait;
protected array $outputs = [
'table',
'json',
'yaml',
];
protected function execute(InputInterface $input, OutputInterface $output): int
{
if (!$input->hasOption('profile') || !$input->getOption('profile')) {
@@ -134,6 +144,63 @@ class Command extends BaseCommand
return $config;
}
protected function getBackend(string $name, array $config = []): ServerInterface
{
if (null === Config::get("servers.{$name}.type", null)) {
throw new RuntimeException(sprintf('No backend named \'%s\' was found.', $name));
}
$default = Config::get("servers.{$name}");
$default['name'] = $name;
return makeServer(array_merge_recursive($default, $config), $name);
}
protected function displayContent(array $content, OutputInterface $output, string $mode = 'json'): void
{
if ('json' === $mode) {
$output->writeln(
json_encode(
value: $content,
flags: JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_IGNORE
)
);
} elseif ('table' === $mode) {
$list = [];
$x = 0;
$count = count($content);
foreach ($content as $_ => $item) {
if (false === is_array($item)) {
$item = [$_ => $item];
}
$subItem = [];
foreach ($item as $key => $leaf) {
if (true === is_array($leaf)) {
continue;
}
$subItem[$key] = $leaf;
}
$x++;
$list[] = $subItem;
if ($x < $count) {
$list[] = new TableSeparator();
}
}
if (!empty($list)) {
(new Table($output))->setStyle('box')->setHeaders(
array_map(fn($title) => is_string($title) ? ucfirst($title) : $title, array_keys($list[0]))
)->setRows($list)->render();
}
} else {
$output->writeln(Yaml::dump($content, 8, 2));
}
}
public function complete(CompletionInput $input, CompletionSuggestions $suggestions): void
{
if ($input->mustSuggestOptionValuesFor('config')) {
@@ -154,7 +221,9 @@ class Command extends BaseCommand
$suggestions->suggestValues($suggest);
}
if ($input->mustSuggestOptionValuesFor('servers-filter') || $input->mustSuggestArgumentValuesFor('server')) {
if ($input->mustSuggestOptionValuesFor('servers-filter') ||
$input->mustSuggestArgumentValuesFor('server') ||
$input->mustSuggestArgumentValuesFor('backend')) {
$currentValue = $input->getCompletionValue();
$suggest = [];

95
src/Commands/Backend/Library/ListCommand.php Normal file
View File

@@ -0,0 +1,95 @@
<?php
declare(strict_types=1);
namespace App\Commands\Backend\Library;
use App\Command;
use App\Libs\Config;
use RuntimeException;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Yaml\Yaml;
final class ListCommand extends Command
{
protected function configure(): void
{
$this->setName('backend:library:list')
->setDescription('Get Backend libraries list')
->addOption(
'output',
'o',
InputOption::VALUE_OPTIONAL,
sprintf('Output mode. Can be [%s].', implode(', ', $this->outputs)),
$this->outputs[0],
)
->addOption('config', 'c', InputOption::VALUE_REQUIRED, 'Use Alternative config file.')
->addArgument('backend', InputArgument::REQUIRED, 'Backend name');
}
protected function runCommand(InputInterface $input, OutputInterface $output): int
{
$mode = $input->getOption('output');
// -- Use Custom servers.yaml file.
if (($config = $input->getOption('config'))) {
try {
Config::save('servers', Yaml::parseFile($this->checkCustomServersFile($config)));
} catch (RuntimeException $e) {
$arr = [
'error' => $e->getMessage()
];
$this->displayContent('table' === $mode ? [$arr] : $arr, $output, $mode);
return self::FAILURE;
}
}
try {
$server = $this->getBackend($input->getArgument('backend'));
$libraries = $server->listLibraries();
if (count($libraries) < 1) {
$arr = [
'info' => 'No libraries were found.',
];
$this->displayContent('table' === $mode ? [$arr] : $arr, $output, $mode);
return self::FAILURE;
}
if ('table' === $mode) {
$list = [];
foreach ($libraries as $item) {
foreach ($item as $key => $val) {
if (false === is_bool($val)) {
continue;
}
$item[$key] = $val ? 'Yes' : 'No';
}
$list[] = $item;
}
$libraries = $list;
}
$this->displayContent($libraries, $output, $mode);
return self::SUCCESS;
} catch (RuntimeException $e) {
$arr = [
'error' => $e->getMessage(),
];
if ('table' !== $mode) {
$arr += [
'file' => $e->getFile(),
'line' => $e->getLine(),
];
}
$this->displayContent('table' === $mode ? [$arr] : $arr, $output, $mode);
return self::FAILURE;
}
}
}

440
src/Commands/Backend/Library/MismatchCommand.php Normal file
View File

@@ -0,0 +1,440 @@
<?php
declare(strict_types=1);
namespace App\Commands\Backend\Library;
use App\Command;
use App\Libs\Config;
use RuntimeException;
use Symfony\Component\Console\Completion\CompletionInput;
use Symfony\Component\Console\Completion\CompletionSuggestions;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Yaml\Yaml;
use Throwable;
final class MismatchCommand extends Command
{
protected array $methods = [
'similarity',
'levenshtein',
];
protected function configure(): void
{
$this->setName('backend:library:mismatch')
->setDescription('Find possible mis-matched movies or shows in a specific library.')
->addOption('id', 'i', InputOption::VALUE_REQUIRED, 'Library id.')
->addOption(
'output',
'o',
InputOption::VALUE_OPTIONAL,
sprintf(
'Output mode. Can be [%s]. Modes other than table mode gives more info.',
implode(', ', $this->outputs)
),
$this->outputs[0],
)
->addOption('percentage', 'p', InputOption::VALUE_OPTIONAL, 'Acceptable percentage.', 50.0)
->addOption(
'method',
'm',
InputOption::VALUE_OPTIONAL,
sprintf('Comparison method. Can be [%s].', implode(', ', $this->methods)),
$this->methods[0]
)
->addOption(
'timeout',
null,
InputOption::VALUE_OPTIONAL,
'Request timeout in seconds.',
Config::get('http.default.options.timeout')
)
->addOption('config', 'c', InputOption::VALUE_REQUIRED, 'Use Alternative config file.')
->addArgument('backend', InputArgument::REQUIRED, 'Backend name');
}
protected function runCommand(InputInterface $input, OutputInterface $output): int
{
$mode = $input->getOption('output');
$percentage = $input->getOption('percentage');
// -- Use Custom servers.yaml file.
if (($config = $input->getOption('config'))) {
try {
Config::save('servers', Yaml::parseFile($this->checkCustomServersFile($config)));
} catch (RuntimeException $e) {
$arr = [
'error' => $e->getMessage()
];
$this->displayContent('table' === $mode ? [$arr] : $arr, $output, $mode);
return self::FAILURE;
}
}
if (null === ($id = $input->getOption('id'))) {
$arr = [
'error' => 'Library mismatch search require library id to be passed in [-i, --id].'
];
$this->displayContent('table' === $mode ? [$arr] : $arr, $output, $mode);
return self::FAILURE;
}
try {
$opts = [];
if ($input->getOption('timeout')) {
$opts = ag_set($opts, 'client.timeout', (float)$input->getOption('timeout'));
}
$server = $this->getBackend($input->getArgument('backend'), $opts);
} catch (RuntimeException $e) {
$arr = [
'error' => $e->getMessage(),
];
$this->displayContent('table' === $mode ? [$arr] : $arr, $output, $mode);
return self::FAILURE;
}
try {
$list = [];
foreach ($server->searchMismatch(id: $id) as $item) {
$processed = $this->compare(item: $item, method: $input->getOption('method'));
if (empty($processed)) {
continue;
}
if ($processed['percent'] >= (float)$percentage) {
continue;
}
$list[] = $processed;
}
} catch (Throwable $e) {
$arr = [
'error' => $e->getMessage(),
];
if ('table' !== $mode) {
$arr += [
'file' => $e->getFile(),
'line' => $e->getLine(),
'item' => $item ?? [],
];
}
$this->displayContent('table' === $mode ? [$arr] : $arr, $output, $mode);
return self::FAILURE;
}
if (empty($list)) {
$arr = [
'info' => 'No mis-identified items were found using given parameters.',
];
$this->displayContent('table' === $mode ? [$arr] : $arr, $output, $mode);
return self::SUCCESS;
}
if ('table' === $mode) {
$forTable = [];
foreach ($list as $item) {
$forTable[] = [
'id' => ag($item, 'id'),
'type' => ag($item, 'type'),
'title' => ag($item, 'title'),
'year' => ag($item, 'year'),
'percent' => ag($item, 'percent') . '%',
'path' => ag($item, 'path'),
];
}
$list = $forTable;
}
$this->displayContent($list, $output, $mode);
return self::SUCCESS;
}
private function compare(array $item, string $method): array
{
if (empty($item)) {
return [];
}
if (null === ($paths = ag($item, 'match.paths', [])) || empty($paths)) {
return [];
}
if (null === ($titles = ag($item, 'match.titles', [])) || empty($titles)) {
return [];
}
if (count($item['guids']) < 1) {
$item['guids'] = 'None';
}
$toLower = fn(string $text, bool $isASCII = false) => trim($isASCII ? strtolower($text) : mb_strtolower($text));
$item['percent'] = $percent = 0.0;
foreach ($paths as $path) {
$pathFull = ag($path, 'full');
$pathShort = ag($path, 'short');
if (empty($pathFull) || empty($pathShort)) {
continue;
}
foreach ($titles as $title) {
$isASCII = mb_detect_encoding($pathShort, 'ASCII') && mb_detect_encoding($title, 'ASCII');
$title = $toLower($this->formatName(name: $title), isASCII: $isASCII);
$pathShort = $toLower($this->formatName(name: $pathShort), isASCII: $isASCII);
if (1 === preg_match('/\((\d{4})\)/', basename($pathFull), $match)) {
$withYear = true;
if (ag($item, 'year') && false === str_contains($title, (string)ag($item, 'year'))) {
$title .= ' ' . ag($item, 'year');
}
} else {
$withYear = false;
}
if (true === str_starts_with($pathShort, $title)) {
$percent = 100.0;
}
if (true === $isASCII) {
similar_text($pathShort, $title, $similarity);
$levenshtein = levenshtein($pathShort, $title);
} else {
$this->mb_similar_text($pathShort, $title, $similarity);
$levenshtein = $this->mb_levenshtein($pathShort, $title);
}
$levenshtein = $this->toPercentage($levenshtein, $pathShort, $title);
switch ($method) {
default:
case 'similarity':
if ($similarity > $percent) {
$percent = $similarity;
}
break;
case 'levenshtein':
if ($similarity > $percent) {
$percent = $levenshtein;
}
break;
}
if (round($percent, 3) > $item['percent']) {
$item['percent'] = round($percent, 3);
}
$item['matches'][] = [
'path' => $pathShort,
'title' => $title,
'type' => $isASCII ? 'ascii' : 'unicode',
'methods' => [
'similarity' => round($similarity, 3),
'levenshtein' => round($levenshtein, 3),
'startWith' => str_starts_with($pathShort, $title),
],
'year' => [
'inPath' => $withYear,
'parsed' => isset($match[1]) ? (int)$match[1] : 'No',
'source' => ag($item, 'year', 'No'),
],
];
}
}
if (count($paths) <= 2 && null !== ($paths[0]['full'] ?? null)) {
$item['path'] = basename($paths[0]['full']);
}
return $item;
}
public function complete(CompletionInput $input, CompletionSuggestions $suggestions): void
{
parent::complete($input, $suggestions);
$methods = [
'output' => 'outputs',
'method' => 'methods',
];
foreach ($methods as $key => $of) {
if ($input->mustSuggestOptionValuesFor($key)) {
$currentValue = $input->getCompletionValue();
$suggest = [];
foreach ($this->{$of} as $name) {
if (empty($currentValue) || str_starts_with($name, $currentValue)) {
$suggest[] = $name;
}
}
$suggestions->suggestValues($suggest);
}
}
}
private function formatName(string $name): string
{
return trim(
preg_replace(
'/\s+/',
' ',
str_replace(
[
'?',
':',
'(',
'[',
']',
')',
',',
'|',
'%',
'.',
'–',
'-',
"'",
'"',
'+',
'/',
';',
'&',
'_',
'!',
'*',
],
' ',
$name
)
)
);
}
/**
* Implementation of `mb_similar_text()`.
*
* (c) Antal Áron <antalaron@antalaron.hu>
*
* @see http://php.net/manual/en/function.similar-text.php
* @see http://locutus.io/php/strings/similar_text/
*
* @param string $str1
* @param string $str2
* @param float|null $percent
*
* @return int
*/
private function mb_similar_text(string $str1, string $str2, float|null &$percent = null): int
{
if (0 === mb_strlen($str1) + mb_strlen($str2)) {
$percent = 0.0;
return 0;
}
$pos1 = $pos2 = $max = 0;
$l1 = mb_strlen($str1);
$l2 = mb_strlen($str2);
for ($p = 0; $p < $l1; ++$p) {
for ($q = 0; $q < $l2; ++$q) {
/** @noinspection LoopWhichDoesNotLoopInspection */
/** @noinspection MissingOrEmptyGroupStatementInspection */
for (
$l = 0; ($p + $l < $l1) && ($q + $l < $l2) && mb_substr($str1, $p + $l, 1) === mb_substr(
$str2,
$q + $l,
1
); ++$l
) {
// nothing to do
}
if ($l > $max) {
$max = $l;
$pos1 = $p;
$pos2 = $q;
}
}
}
$similarity = $max;
if ($similarity) {
if ($pos1 && $pos2) {
$similarity += $this->mb_similar_text(mb_substr($str1, 0, $pos1), mb_substr($str2, 0, $pos2));
}
if (($pos1 + $max < $l1) && ($pos2 + $max < $l2)) {
$similarity += $this->mb_similar_text(
mb_substr($str1, $pos1 + $max, $l1 - $pos1 - $max),
mb_substr($str2, $pos2 + $max, $l2 - $pos2 - $max)
);
}
}
$percent = ($similarity * 200.0) / ($l1 + $l2);
return $similarity;
}
private function mb_levenshtein(string $str1, string $str2)
{
$length1 = mb_strlen($str1, 'UTF-8');
$length2 = mb_strlen($str2, 'UTF-8');
if ($length1 < $length2) {
return $this->mb_levenshtein($str2, $str1);
}
if (0 === $length1) {
return $length2;
}
if ($str1 === $str2) {
return 0;
}
$prevRow = range(0, $length2);
for ($i = 0; $i < $length1; $i++) {
$currentRow = [];
$currentRow[0] = $i + 1;
$c1 = mb_substr($str1, $i, 1, 'UTF-8');
for ($j = 0; $j < $length2; $j++) {
$c2 = mb_substr($str2, $j, 1, 'UTF-8');
$insertions = $prevRow[$j + 1] + 1;
$deletions = $currentRow[$j] + 1;
$substitutions = $prevRow[$j] + (($c1 !== $c2) ? 1 : 0);
$currentRow[] = min($insertions, $deletions, $substitutions);
}
$prevRow = $currentRow;
}
return $prevRow[$length2];
}
private function toPercentage($base, $str1, $str2, bool $isASCII = false): float
{
$length = fn(string $text) => $isASCII ? mb_strlen($text, 'UTF-8') : strlen($text);
return (1 - $base / max($length($str1), $length($str2))) * 100;
}
}

138
src/Commands/Servers/RemoteCommand.php
View File

@@ -20,7 +20,6 @@ use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Yaml\Yaml;
use Symfony\Contracts\HttpClient\Exception\ExceptionInterface;
use Throwable;
final class RemoteCommand extends Command
{
@@ -28,7 +27,6 @@ final class RemoteCommand extends Command
{
$this->setName('servers:remote')
->setDescription('Get info from the remote server.')
->addOption('list-libraries', null, InputOption::VALUE_NONE, 'List Server Libraries.')
->addOption('list-users', null, InputOption::VALUE_NONE, 'List Server users.')
->addOption('list-users-with-tokens', null, InputOption::VALUE_NONE, 'Show users list with tokens.')
->addOption('use-token', null, InputOption::VALUE_REQUIRED, 'Override server config token.')
@@ -37,15 +35,7 @@ final class RemoteCommand extends Command
->addOption('search-raw', null, InputOption::VALUE_NONE, 'Return Unfiltered results.')
->addOption('search-limit', null, InputOption::VALUE_REQUIRED, 'Search limit.', 25)
->addOption('search-output', null, InputOption::VALUE_REQUIRED, 'Search output style [json,yaml].', 'json')
->addOption('search-mismatch', null, InputOption::VALUE_REQUIRED, 'Search library for possible mismatch.')
->addOption('search-coef', null, InputOption::VALUE_OPTIONAL, 'Mismatch similar text percentage.', 50.0)
->addOption(
'timeout',
null,
InputOption::VALUE_OPTIONAL,
'Request timeout in seconds.',
Config::get('http.default.options.timeout')
)
->addOption('timeout', null, InputOption::VALUE_OPTIONAL, 'Request timeout in seconds.')
->addOption('config', 'c', InputOption::VALUE_REQUIRED, 'Use Alternative config file.')
->addArgument('server', InputArgument::REQUIRED, 'Server name');
}
@@ -101,10 +91,6 @@ final class RemoteCommand extends Command
$this->listUsers($input, $output, $server);
}
if ($input->getOption('list-libraries')) {
$this->listLibraries($output, $server);
}
if ($input->getOption('search') && $input->getOption('search-limit')) {
$this->search($server, $output, $input);
}
@@ -117,10 +103,6 @@ final class RemoteCommand extends Command
$this->searchId($server, $output, $input);
}
if ($input->getOption('search-mismatch')) {
return $this->searchMismatch($server, $output, $input);
}
return self::SUCCESS;
}
@@ -162,33 +144,6 @@ final class RemoteCommand extends Command
(new Table($output))->setStyle('box')->setHeaders(array_keys($users[0]))->setRows($list)->render();
}
private function listLibraries(
OutputInterface $output,
ServerInterface $server,
): void {
$libraries = $server->listLibraries();
if (count($libraries) < 1) {
$output->writeln('<comment>No users reported by server.</comment>');
return;
}
$list = [];
$x = 0;
$count = count($libraries);
foreach ($libraries as $user) {
$x++;
$values = array_values($user);
$list[] = $values;
if ($x < $count) {
$list[] = new TableSeparator();
}
}
(new Table($output))->setStyle('box')->setHeaders(array_keys($libraries[0]))->setRows($list)->render();
}
private function search(ServerInterface $server, OutputInterface $output, InputInterface $input): void
{
$result = $server->search(
@@ -236,97 +191,6 @@ final class RemoteCommand extends Command
}
}
private function searchMismatch(ServerInterface $server, OutputInterface $output, InputInterface $input): int
{
$id = $input->getOption('search-mismatch');
$percentage = (float)$input->getOption('search-coef');
$mode = $input->getOption('search-output');
try {
$result = $server->searchMismatch(id: $id, opts: ['coef' => $percentage]);
} catch (Throwable $e) {
$this->setOutputContent(
[
[
'error' => $e->getMessage(),
'file' => $e->getFile(),
'line' => $e->getLine()
],
],
$output,
$mode
);
return self::FAILURE;
}
if (empty($result)) {
$this->setOutputContent(
[
[
'info' => sprintf(
'We are %1$02.2f%3$s sure there are no mis-identified items in library \'%2$s\'.',
$percentage,
$id,
'%',
)
]
],
$output,
$mode
);
return self::SUCCESS;
}
$this->setOutputContent($result, $output, $mode);
return self::SUCCESS;
}
private function setOutputContent(array $content, OutputInterface $output, string $mode = 'json'): void
{
if ('json' === $mode) {
$output->writeln(
json_encode(
value: $content,
flags: JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_IGNORE
)
);
} elseif ('table' === $mode) {
$list = [];
$x = 0;
$count = count($content);
foreach ($content as $_ => $item) {
if (false === is_array($item)) {
$item = [$_ => $item];
}
$subItem = [];
foreach ($item as $key => $leaf) {
if (true === is_array($leaf)) {
continue;
}
$subItem[$key] = $leaf;
}
$x++;
$list[] = $subItem;
if ($x < $count) {
$list[] = new TableSeparator();
}
}
if (!empty($list)) {
(new Table($output))->setStyle('box')->setHeaders(
array_map(fn($title) => is_string($title) ? ucfirst($title) : $title, array_keys($list[0]))
)->setRows($list)->render();
}
} else {
$output->writeln(Yaml::dump($content, 8, 2));
}
}
public function complete(CompletionInput $input, CompletionSuggestions $suggestions): void
{
parent::complete($input, $suggestions);

174
src/Libs/Servers/JellyfinServer.php
View File

@@ -17,6 +17,7 @@ use App\Libs\QueueRequests;
use Closure;
use DateInterval;
use DateTimeInterface;
use Generator;
use JsonException;
use JsonMachine\Exception\PathNotFoundException;
use JsonMachine\Items;
@@ -511,13 +512,10 @@ class JellyfinServer implements ServerInterface
/**
* @throws Throwable
*/
public function searchMismatch(string|int $id, array $opts = []): array
public function searchMismatch(string|int $id, array $opts = []): Generator
{
$list = [];
$this->checkConfig();
// -- Get Content type.
$url = $this->url->withPath(sprintf('/Users/%s/items/', $this->user))->withQuery(
http_build_query(
[
@@ -537,9 +535,8 @@ class JellyfinServer implements ServerInterface
if (200 !== $response->getStatusCode()) {
throw new RuntimeException(
sprintf(
'%s: Get metadata request for id \'%s\' responded with unexpected http status code \'%d\'.',
'%s: Get libraries list request responded with unexpected http status code \'%d\'.',
$this->name,
$id,
$response->getStatusCode()
)
);
@@ -594,7 +591,7 @@ class JellyfinServer implements ServerInterface
if (200 !== $response->getStatusCode()) {
throw new RuntimeException(
sprintf(
'%s: Get metadata request for id \'%s\' responded with unexpected http status code \'%d\'.',
'%s: Request to get library content for id \'%s\' responded with unexpected http status code \'%d\'.',
$this->name,
$id,
$response->getStatusCode()
@@ -602,126 +599,69 @@ class JellyfinServer implements ServerInterface
);
}
$handleRequest = function (string $type, array $item) use (&$list, $opts) {
$this->logger->debug(sprintf('%s: Processing %s \'%s\'.', $this->name, $type, ag($item, 'Name')));
$handleRequest = function (string $type, array $item): array {
$url = $this->url->withPath(sprintf('/Users/%s/items/%s', $this->user, ag($item, 'Id')));
$year = ag($item, 'ProductionYear', null);
$parseYear = '/\((\d{4})\)/';
$possibleTitles = $locations = $guids = $matches = [];
$this->logger->debug(
sprintf('%s: Processing %s \'%s\'.', $this->name, $type, ag($item, 'Name')),
[
'url' => (string)$url,
]
);
$possibleTitlesList = ['Name', 'OriginalTitle', 'SortName', 'ForcedSortName'];
foreach ($possibleTitlesList as $title) {
if (null === ($title = ag($item, $title))) {
continue;
}
$title = formatName($title);
if (true === in_array($type, $possibleTitles)) {
continue;
}
$possibleTitles[] = formatName($title);
}
if (null !== ($path = ag($item, 'Path'))) {
if (ag($item, 'Type') === 'Movie') {
$locations[] = [
'l' => dirname($path),
'n' => formatName(basename(dirname($path))),
];
}
$locations[] = [
'l' => $path,
'n' => formatName(basename($path)),
];
}
if (null !== ($providerIds = ag($item, 'ProviderIds'))) {
$guids = $providerIds;
}
foreach ($locations ?? [] as $location) {
foreach ($possibleTitles as $title) {
if (true === str_contains($location['n'], $title)) {
return;
}
$isASCII = mb_detect_encoding($location['n'], 'ASCII', true) && mb_detect_encoding(
$title,
'ASCII',
true
);
if (1 === preg_match($parseYear, basename($location['l']), $match)) {
$withYear = true;
$title = !empty($year) && !str_contains($title, (string)$year) ? $title . ' ' . $year : $title;
} else {
$withYear = false;
}
if (true === $isASCII) {
similar_text($location['n'], $title, $percent);
} else {
mb_similar_text($location['n'], $title, $percent);
}
if ($percent >= $opts['coef']) {
return;
}
$matches[] = [
'path' => $location['n'],
'title' => $title,
'type' => $isASCII ? 'ascii' : 'unicode',
'methods' => [
'similarity' => round($percent, 3),
'levenshtein' => round(
$isASCII ? levenshtein($location['n'], $title) : mb_levenshtein($location['n'], $title),
3
),
'compareStrings' => compareStrings($location['n'], $title),
],
'year' => [
'inPath' => $withYear,
'parsed' => isset($match[1]) ? (int)$match[1] : 'No',
'source' => $year ?? 'No',
],
];
}
}
$year = ag($item, 'ProductionYear', null);
$metadata = [
'id' => ag($item, 'Id'),
'type' => ucfirst($type),
'url' => [(string)$url],
'title' => ag($item, $possibleTitlesList, '??'),
'year' => $year ?? 0000,
'guids' => $guids,
'path' => array_column($locations, 'l') ?? [],
'matching' => $matches,
'comments' => (empty($locations)) ? 'No path found.' : 'Title does not match path.',
'year' => $year,
'guids' => [],
'match' => [
'titles' => [],
'paths' => [],
],
];
if (empty($guids)) {
$metadata['guids'] = 'None.';
foreach ($possibleTitlesList as $title) {
if (null === ($title = ag($item, $title))) {
continue;
}
$isASCII = mb_detect_encoding($title, 'ASCII', true);
$title = trim($isASCII ? strtolower($title) : mb_strtolower($title));
if (true === in_array($title, $metadata['match']['titles'])) {
continue;
}
$metadata['match']['titles'][] = $title;
}
if (!empty($metadata['path']) && count($metadata['path']) <= 1) {
$metadata['path'] = $metadata['path'][0];
if (null !== ($path = ag($item, 'Path'))) {
$metadata['match']['paths'][] = [
'full' => $path,
'short' => basename($path),
];
if (ag($item, 'Type') === 'Movie') {
if (false === str_starts_with(basename($path), basename(dirname($path)))) {
$metadata['match']['paths'][] = [
'full' => $path,
'short' => basename($path),
];
}
}
}
if ('movie' === $type && 2 === count($metadata['path'])) {
$metadata['path'] = $metadata['path'][1];
if (null !== ($providerIds = ag($item, 'ProviderIds'))) {
$metadata['guids'] = $providerIds;
}
$list[] = $metadata;
return $metadata;
};
$it = Items::fromIterable(
@@ -771,6 +711,10 @@ class JellyfinServer implements ServerInterface
);
}
if (empty($requests)) {
throw new RuntimeException('No requests were made as the library is empty.');
}
foreach ($requests as $response) {
if (200 !== $response->getStatusCode()) {
$this->logger->error(
@@ -784,7 +728,7 @@ class JellyfinServer implements ServerInterface
continue;
}
$handleRequest(
yield $handleRequest(
$response->getInfo('user_data')['type'],
json_decode(
json: $response->getContent(),
@@ -793,8 +737,6 @@ class JellyfinServer implements ServerInterface
)
);
}
return $list;
}
public function listLibraries(): array
@@ -877,11 +819,11 @@ class JellyfinServer implements ServerInterface
$type = ag($section, 'CollectionType', 'unknown');
$list[] = [
'ID' => $key,
'Title' => ag($section, 'Name', '???'),
'Type' => $type,
'Ignored' => null !== $ignoreIds && in_array($key, $ignoreIds) ? 'Yes' : 'No',
'Supported' => 'movies' !== $type && 'tvshows' !== $type ? 'No' : 'Yes',
'id' => $key,
'title' => ag($section, 'Name', '???'),
'type' => $type,
'ignored' => null !== $ignoreIds && in_array($key, $ignoreIds),
'supported' => 'movies' === $type || 'tvshows' === $type,
];
}

207
src/Libs/Servers/PlexServer.php
View File

@@ -17,6 +17,7 @@ use App\Libs\QueueRequests;
use Closure;
use DateInterval;
use DateTimeInterface;
use Generator;
use JsonException;
use JsonMachine\Exception\PathNotFoundException;
use JsonMachine\Items;
@@ -545,27 +546,21 @@ class PlexServer implements ServerInterface
/**
* @throws Throwable
*/
public function searchMismatch(string|int $id, array $opts = []): array
public function searchMismatch(string|int $id, array $opts = []): Generator
{
$list = [];
$this->checkConfig();
// -- Get Content type.
$url = $this->url->withPath('/library/sections/');
$this->logger->debug(sprintf('%s: Sending get sections list.', $this->name), [
'url' => $url
]);
$this->logger->debug(sprintf('%s: Requesting list of backend libraries.', $this->name), ['url' => $url]);
$response = $this->http->request('GET', (string)$url, $this->getHeaders());
if (200 !== $response->getStatusCode()) {
throw new RuntimeException(
sprintf(
'%s: Get metadata request for id \'%s\' responded with unexpected http status code \'%d\'.',
'%s: Get libraries list request responded with unexpected http status code \'%d\'.',
$this->name,
$id,
$response->getStatusCode()
)
);
@@ -617,7 +612,7 @@ class PlexServer implements ServerInterface
if (200 !== $response->getStatusCode()) {
throw new RuntimeException(
sprintf(
'%s: Get metadata request for id \'%s\' responded with unexpected http status code \'%d\'.',
'%s: Request to get library content for id \'%s\' responded with unexpected http status code \'%d\'.',
$this->name,
$id,
$response->getStatusCode()
@@ -625,138 +620,88 @@ class PlexServer implements ServerInterface
);
}
$handleRequest = function (string $type, array $item) use (&$list, $opts) {
$this->logger->debug(sprintf('%s: Processing %s \'%s\'.', $this->name, $type, ag($item, 'title')));
$handleRequest = function (string $type, array $item): array {
$url = $this->url->withPath(sprintf('/library/metadata/%d', ag($item, 'ratingKey')));
$this->logger->debug(
sprintf('%s: Processing %s \'%s\'.', $this->name, $type, ag($item, 'Name')),
[
'url' => (string)$url,
]
);
$possibleTitlesList = ['title', 'originalTitle', 'titleSort'];
$year = ag($item, 'year');
$parseYear = '/\((\d{4})\)/';
$metadata = [
'id' => (int)ag($item, 'ratingKey'),
'type' => ucfirst($type),
'url' => (string)$url,
'title' => ag($item, $possibleTitlesList, '??'),
'year' => $year,
'guids' => [],
'match' => [
'titles' => [],
'paths' => [],
],
];
$locations = [];
$possibleTitles = $guids = $matches = [];
$possibleTitlesList = ['title', 'originalTitle', 'titleSort'];
foreach ($possibleTitlesList as $title) {
if (null === ($title = ag($item, $title))) {
continue;
}
$title = formatName($title);
$isASCII = mb_detect_encoding($title, 'ASCII', true);
$title = trim($isASCII ? strtolower($title) : mb_strtolower($title));
if (true === in_array($title, $possibleTitles)) {
if (true === in_array($title, $metadata['match']['titles'])) {
continue;
}
$possibleTitles[] = formatName($title);
$metadata['match']['titles'][] = $title;
}
foreach (ag($item, 'Location', []) as $path) {
$location = ag($path, 'path');
$locations[] = [
'l' => $location,
'n' => formatName(basename($location)),
];
switch ($type) {
case 'show':
foreach (ag($item, 'Location', []) as $path) {
$path = ag($path, 'path');
$metadata['match']['paths'][] = [
'full' => $path,
'short' => basename($path),
];
}
break;
case 'movie':
foreach (ag($item, 'Media', []) as $leaf) {
foreach (ag($leaf, 'Part', []) as $path) {
$path = ag($path, 'file');
$dir = dirname($path);
$metadata['match']['paths'][] = [
'full' => $path,
'short' => basename($path),
];
if (false === str_starts_with(basename($path), basename($dir))) {
$metadata['match']['paths'][] = [
'full' => $path,
'short' => basename($dir),
];
}
}
}
break;
default:
throw new RuntimeException(sprintf('Invalid library item type \'%s\' was given.', $type));
}
foreach (ag($item, 'Media', []) as $leaf) {
foreach (ag($leaf, 'Part', []) as $path) {
$location = ag($path, 'file');
$locations[] = [
'l' => dirname($location),
'n' => formatName(basename(dirname($location))),
];
$locations[] = [
'l' => $location,
'n' => formatName(basename($location)),
];
}
}
$guids[] = ag($item, 'guid', []);
$metadata['guids'][] = ag($item, 'guid', []);
foreach (ag($item, 'Guid', []) as $guid) {
$guids[] = ag($guid, 'id');
$metadata['guids'][] = ag($guid, 'id');
}
foreach ($locations ?? [] as $location) {
foreach ($possibleTitles as $title) {
if (true === str_contains($location['n'], $title)) {
return;
}
$isASCII = mb_detect_encoding($location['n'], 'ASCII', true) && mb_detect_encoding(
$title,
'ASCII',
true
);
if (1 === preg_match($parseYear, basename($location['l']), $match)) {
$withYear = true;
$title = !empty($year) && !str_contains($title, (string)$year) ? $title . ' ' . $year : $title;
} else {
$withYear = false;
}
if (true === $isASCII) {
similar_text($location['n'], $title, $percent);
} else {
mb_similar_text($location['n'], $title, $percent);
}
if ($percent >= $opts['coef']) {
return;
}
$matches[] = [
'path' => $location['n'],
'title' => $title,
'type' => $isASCII ? 'ascii' : 'unicode',
'methods' => [
'similarity' => round($percent, 3),
'levenshtein' => round(
$isASCII ? levenshtein($location['n'], $title) : mb_levenshtein($location['n'], $title),
3
),
'compareStrings' => compareStrings($location['n'], $title),
],
'year' => [
'inPath' => $withYear,
'parsed' => isset($match[1]) ? (int)$match[1] : 'No',
'source' => $year ?? 'No',
],
];
}
}
$metadata = [
'id' => (int)ag($item, 'ratingKey'),
'type' => ucfirst($type),
'url' => [(string)$url],
'title' => ag($item, $possibleTitlesList, '??'),
'year' => $year ?? 0000,
'guids' => $guids,
'path' => array_column($locations, 'l') ?? [],
'matching' => $matches,
'comments' => (empty($locations)) ? 'No path found.' : 'Title does not match path.',
];
if (empty($guids)) {
$metadata['guids'] = 'No external ids found.';
}
if (!empty($metadata['path']) && count($metadata['path']) <= 1) {
$metadata['path'] = $metadata['path'][0];
}
if ('movie' === $type && 2 === count($metadata['path'])) {
$metadata['path'] = $metadata['path'][1];
}
$list[] = $metadata;
return $metadata;
};
$it = Items::fromIterable(
@@ -788,7 +733,7 @@ class PlexServer implements ServerInterface
}
if (iFace::TYPE_MOVIE === $type) {
$handleRequest($type, $entity);
yield $handleRequest($type, $entity);
} else {
$url = $this->url->withPath(sprintf('/library/metadata/%d', ag($entity, 'ratingKey')));
@@ -813,6 +758,10 @@ class PlexServer implements ServerInterface
}
}
if (iFace::TYPE_MOVIE !== $type && empty($requests)) {
throw new RuntimeException('No requests were made as the library is empty.');
}
foreach ($requests as $response) {
if (200 !== $response->getStatusCode()) {
$this->logger->error(
@@ -832,13 +781,11 @@ class PlexServer implements ServerInterface
flags: JSON_THROW_ON_ERROR | JSON_INVALID_UTF8_IGNORE
);
$handleRequest(
yield $handleRequest(
$response->getInfo('user_data')['type'],
ag($json, 'MediaContainer.Metadata.0', [])
);
}
return $list;
}
public function listLibraries(): array
@@ -912,11 +859,13 @@ class PlexServer implements ServerInterface
$type = ag($section, 'type', 'unknown');
$list[] = [
'ID' => $key,
'Title' => ag($section, 'title', '???'),
'Type' => $type,
'Ignored' => null !== $ignoreIds && in_array($key, $ignoreIds) ? 'Yes' : 'No',
'Supported' => 'movie' !== $type && 'show' !== $type ? 'No' : 'Yes',
'id' => $key,
'title' => ag($section, 'title', '???'),
'type' => $type,
'ignored' => null !== $ignoreIds && in_array($key, $ignoreIds),
'supported' => 'movie' === $type || 'show' === $type,
'agent' => ag($section, 'agent'),
'scanner' => ag($section, 'scanner'),
];
}

5
src/Libs/Servers/ServerInterface.php
View File

@@ -8,6 +8,7 @@ use App\Libs\Entity\StateInterface;
use App\Libs\Mappers\ImportInterface;
use App\Libs\QueueRequests;
use DateTimeInterface;
use Generator;
use JsonException;
use Psr\Http\Message\ServerRequestInterface;
use Psr\Http\Message\UriInterface;
@@ -133,9 +134,9 @@ interface ServerInterface
* @param string|int $id
* @param array $opts
*
* @return array
* @return Generator
*/
public function searchMismatch(string|int $id, array $opts = []): array;
public function searchMismatch(string|int $id, array $opts = []): Generator;
/**
* Get all persistent data.

198
src/Libs/helpers.php
View File

@@ -595,201 +595,3 @@ if (false === function_exists('getPeakMemoryUsage')) {
return fsize(memory_get_peak_usage() - BASE_PEAK_MEMORY);
}
}
if (false === function_exists('formatName')) {
function formatName(string $name): string
{
return trim(
preg_replace(
'/\s+/',
' ',
str_replace(
[
'?',
':',
'(',
'[',
']',
')',
',',
'|',
'%',
'.',
'–',
'-',
"'",
'"',
'+',
'/',
';',
'&',
'_',
'!',
'*',
],
' ',
strtolower($name)
)
)
);
}
}
if (false === function_exists('mb_similar_text')) {
/**
* Implementation of `mb_similar_text()`.
*
* (c) Antal Áron <antalaron@antalaron.hu>
*
* @see http://php.net/manual/en/function.similar-text.php
* @see http://locutus.io/php/strings/similar_text/
*
* @param string $str1
* @param string $str2
* @param float|null $percent
*
* @return int
*/
function mb_similar_text(string $str1, string $str2, float|null &$percent = null): int
{
if (0 === mb_strlen($str1) + mb_strlen($str2)) {
$percent = 0.0;
return 0;
}
$pos1 = $pos2 = $max = 0;
$l1 = mb_strlen($str1);
$l2 = mb_strlen($str2);
for ($p = 0; $p < $l1; ++$p) {
for ($q = 0; $q < $l2; ++$q) {
/** @noinspection LoopWhichDoesNotLoopInspection */
/** @noinspection MissingOrEmptyGroupStatementInspection */
for (
$l = 0; ($p + $l < $l1) && ($q + $l < $l2) && mb_substr($str1, $p + $l, 1) === mb_substr(
$str2,
$q + $l,
1
); ++$l
) {
// nothing to do
}
if ($l > $max) {
$max = $l;
$pos1 = $p;
$pos2 = $q;
}
}
}
$similarity = $max;
if ($similarity) {
if ($pos1 && $pos2) {
$similarity += mb_similar_text(mb_substr($str1, 0, $pos1), mb_substr($str2, 0, $pos2));
}
if (($pos1 + $max < $l1) && ($pos2 + $max < $l2)) {
$similarity += mb_similar_text(
mb_substr($str1, $pos1 + $max, $l1 - $pos1 - $max),
mb_substr($str2, $pos2 + $max, $l2 - $pos2 - $max)
);
}
}
$percent = ($similarity * 200.0) / ($l1 + $l2);
return $similarity;
}
}
if (false === function_exists('mb_levenshtein')) {
function mb_levenshtein(string $str1, string $str2)
{
$length1 = mb_strlen($str1, 'UTF-8');
$length2 = mb_strlen($str2, 'UTF-8');
if ($length1 < $length2) {
return mb_levenshtein($str2, $str1);
}
if (0 === $length1) {
return $length2;
}
if ($str1 === $str2) {
return 0;
}
$prevRow = range(0, $length2);
for ($i = 0; $i < $length1; $i++) {
$currentRow = [];
$currentRow[0] = $i + 1;
$c1 = mb_substr($str1, $i, 1, 'UTF-8');
for ($j = 0; $j < $length2; $j++) {
$c2 = mb_substr($str2, $j, 1, 'UTF-8');
$insertions = $prevRow[$j + 1] + 1;
$deletions = $currentRow[$j] + 1;
$substitutions = $prevRow[$j] + (($c1 !== $c2) ? 1 : 0);
$currentRow[] = min($insertions, $deletions, $substitutions);
}
$prevRow = $currentRow;
}
return $prevRow[$length2];
}
}
if (false === function_exists('compareStrings')) {
function compareStrings(string $str1, string $str2): float
{
if (empty($str1) || empty($str2)) {
return 0.000;
}
$length1 = mb_strlen($str1, 'UTF-8');
$length2 = mb_strlen($str2, 'UTF-8');
if ($length1 < $length2) {
return compareStrings($str2, $str1);
}
$ar1 = preg_split('/[^\w\-]+/u', strtolower($str1), flags: PREG_SPLIT_NO_EMPTY);
$ar2 = preg_split('/[^\w\-]+/u', strtolower($str2), flags: PREG_SPLIT_NO_EMPTY);
$ar2Copy = array_values($ar2);
$matchedIndices = [];
$wordMap = [];
foreach ($ar1 as $k => $w1) {
if (isset($wordMap[$w1])) {
if ($wordMap[$w1][0] >= $k) {
$matchedIndices[$k] = $wordMap[$w1][0];
}
array_splice($wordMap[$w1], 0, 1);
} else {
$indices = array_keys($ar2Copy, $w1);
$indexCount = count($indices);
if ($indexCount) {
$matchedIndices[$k] = $indices[0];
if (1 === $indexCount) {
// remove the word at given index from second array so that it won't repeat
unset($ar2Copy[$indices[0]]);
} else {
// remove the word at given indices from second array so that it won't repeat
foreach ($indices as $index) {
unset($ar2Copy[$index]);
}
array_splice($indices, 0, 1);
$wordMap[$w1] = $indices;
}
}
}
}
return round(count($matchedIndices) * 100 / count($ar1), 3);
}
}