cphipps/watchstate
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Updated logs file naming to allow for easier pruning.

Browse Source
This commit is contained in:
Abdulmhsen B. A. A
2022-06-05 03:15:19 +03:00
parent d2ca6b2f43
commit db0e153152
6 changed files with 120 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
README.md
View File

@@ -282,28 +282,28 @@ Click `Save Changes`
# Environment variables. # Environment variables.
| Environment key | Value | Description | Default | | Environment key | Value | Description | Default |
|-------------------------|---------|----------------------------------------------------------------------------------|--------------------------------| |-------------------------|---------|----------------------------------------------------------------------------------|------------------------------------|
| WS_DATA_PATH | string | Where key data stored (config, db). | `${BASE_PATH}/var` | | WS_DATA_PATH | string | Where key data stored (config, db). | `${BASE_PATH}/var` |
| WS_TMP_DIR | string | Where temp data stored. (logs, cache). | `${WS_DATA_PATH}` | | WS_TMP_DIR | string | Where temp data stored. (logs, cache). | `${WS_DATA_PATH}` |
| WS_TZ | string | Set timezone. | `UTC` | | WS_TZ | string | Set timezone. | `UTC` |
| WS_WEBHOOK_DEBUG | bool | Store webhook payloads into `${WS_TMP_DIR}/webhooks` | `false` | | WS_WEBHOOK_DEBUG | bool | Store webhook payloads into `${WS_TMP_DIR}/webhooks` | `false` |
| WS_REQUEST_DEBUG | bool | Store request payloads into `${WS_TMP_DIR}/debug` | `false` | | WS_REQUEST_DEBUG | bool | Store request payloads into `${WS_TMP_DIR}/debug` | `false` |
| WS_WEBHOOK_TOKEN_LENGTH | integer | How many bits to use for webhook token generator | `16` | | WS_WEBHOOK_TOKEN_LENGTH | integer | How many bits to use for webhook token generator | `16` |
| WS_LOGGER_FILE_ENABLE | bool | Save logs to file. | `true` | | WS_LOGGER_FILE_ENABLE | bool | Save logs to file. | `true` |
| WS_LOGGER_FILE | string | Full path to log file. | `${WS_TMP_DIR}/logs/app.log` | | WS_LOGGER_FILE | string | Full path to log file. | `${WS_TMP_DIR}/logs/app.(Ymd).log` |
| WS_LOGGER_FILE_LEVEL | string | File Logger Level. | `ERROR` | | WS_LOGGER_FILE_LEVEL | string | File Logger Level. | `ERROR` |
| WS_CRON_IMPORT | bool | Enable import scheduled task. Value casted to bool. | `false` | | WS_CRON_IMPORT | bool | Enable import scheduled task. Value casted to bool. | `false` |
| WS_CRON_IMPORT_AT | string | When to run import scheduled task. Valid Cron Expression Expected. | `'0 */1 * * *` (Every 1h) | | WS_CRON_IMPORT_AT | string | When to run import scheduled task. Valid Cron Expression Expected. | `'0 */1 * * *` (Every 1h) |
| WS_CRON_IMPORT_ARGS | string | Flags to pass to the import command. | `'-v` | | WS_CRON_IMPORT_ARGS | string | Flags to pass to the import command. | `'-v` |
| WS_CRON_EXPORT | bool | Enable export scheduled task. Value casted to bool. | `false` | | WS_CRON_EXPORT | bool | Enable export scheduled task. Value casted to bool. | `false` |
| WS_CRON_EXPORT_AT | string | When to run export scheduled task. Valid Cron Expression Expected. | `'30 */1 * * *` (Every 1h 30m) | | WS_CRON_EXPORT_AT | string | When to run export scheduled task. Valid Cron Expression Expected. | `'30 */1 * * *` (Every 1h 30m) |
| WS_CRON_EXPORT_ARGS | string | Flags to pass to the export command. | `'-v` | | WS_CRON_EXPORT_ARGS | string | Flags to pass to the export command. | `'-v` |
| WS_CRON_PUSH | bool | Enable push scheduled task. Value casted to bool. | `false` | | WS_CRON_PUSH | bool | Enable push scheduled task. Value casted to bool. | `false` |
| WS_CRON_PUSH_AT | string | When to run push scheduled task. Valid Cron Expression Expected. | `'*/10 * * * *` (Every 10m) | | WS_CRON_PUSH_AT | string | When to run push scheduled task. Valid Cron Expression Expected. | `'*/10 * * * *` (Every 10m) |
| WS_CRON_PUSH_ARGS | string | Flags to pass to the push command. | `'-v` | | WS_CRON_PUSH_ARGS | string | Flags to pass to the push command. | `'-v` |
| WS_LOGS_PRUNE_AFTER | string | Delete logs older than specified time. Set to `disable` to disable the pruning. | `'-3 DAYS` | | WS_LOGS_PRUNE_AFTER | string | Delete logs older than specified time. Set to `disable` to disable the pruning. | `'-3 DAYS` |
| WS_DEBUG_IMPORT | bool | Log no valid/externals id during import process. stored in `${WS_TMP_DIR}/debug` | `'false` | | WS_DEBUG_IMPORT | bool | Log no valid/externals id during import process. stored in `${WS_TMP_DIR}/debug` | `'false` |
# Container specific environment variables. # Container specific environment variables.

66
config/config.php
View File

@@ -33,6 +33,8 @@ return (function () {
], ],
]; ];
$config['logs']['affix'] = makeDate()->format('Ymd');
$config['tmpDir'] = fixPath(env('WS_TMP_DIR', ag($config, 'path'))); $config['tmpDir'] = fixPath(env('WS_TMP_DIR', ag($config, 'path')));
$config['storage'] += [ $config['storage'] += [
@@ -85,7 +87,7 @@ return (function () {
'options' => [ 'options' => [
'save.handler' => 'file', 'save.handler' => 'file',
'save.handler.file' => [ 'save.handler.file' => [
'filename' => ag($config, 'tmpDir') . '/profiler/profiler_' . gmdate('Y_m_d_His') . '.json' 'filename' => ag($config, 'tmpDir') . '/profiler/run.' . makeDate()->format('Ymd_His') . '.json'
], ],
], ],
], ],
@@ -103,7 +105,10 @@ return (function () {
'type' => 'stream', 'type' => 'stream',
'enabled' => env('WS_LOGGER_FILE_ENABLE', true), 'enabled' => env('WS_LOGGER_FILE_ENABLE', true),
'level' => env('WS_LOGGER_FILE_LEVEL', Logger::ERROR), 'level' => env('WS_LOGGER_FILE_LEVEL', Logger::ERROR),
'filename' => env('WS_LOGGER_FILE', fn() => ag($config, 'tmpDir') . '/logs/app.log'), 'filename' => env(
'WS_LOGGER_FILE',
fn() => ag($config, 'tmpDir') . '/logs/app.' . ag($config, 'logs.affix') . '.log'
),
], ],
'stderr' => [ 'stderr' => [
'type' => 'stream', 'type' => 'stream',
@@ -171,33 +176,36 @@ return (function () {
]; ];
$config['tasks'] = [ $config['tasks'] = [
ImportCommand::TASK_NAME => [ 'logfile' => ag($config, 'tmpDir') . '/logs/tasks/task.' . ag($config, 'logs.affix') . '.log',
Task::NAME => ImportCommand::TASK_NAME, 'commands' => [
Task::ENABLED => (bool)env('WS_CRON_IMPORT', false), ImportCommand::TASK_NAME => [
Task::RUN_AT => (string)env('WS_CRON_IMPORT_AT', '0 */1 * * *'), Task::NAME => ImportCommand::TASK_NAME,
Task::COMMAND => '@state:import', Task::ENABLED => (bool)env('WS_CRON_IMPORT', false),
Task::ARGS => env('WS_CRON_IMPORT_ARGS', '-v'), Task::RUN_AT => (string)env('WS_CRON_IMPORT_AT', '0 */1 * * *'),
], Task::COMMAND => '@state:import',
ExportCommand::TASK_NAME => [ Task::ARGS => env('WS_CRON_IMPORT_ARGS', '-v'),
Task::NAME => ExportCommand::TASK_NAME, ],
Task::ENABLED => (bool)env('WS_CRON_EXPORT', false), ExportCommand::TASK_NAME => [
Task::RUN_AT => (string)env('WS_CRON_EXPORT_AT', '30 */1 * * *'), Task::NAME => ExportCommand::TASK_NAME,
Task::COMMAND => '@state:export', Task::ENABLED => (bool)env('WS_CRON_EXPORT', false),
Task::ARGS => env('WS_CRON_EXPORT_ARGS', '-v'), Task::RUN_AT => (string)env('WS_CRON_EXPORT_AT', '30 */1 * * *'),
], Task::COMMAND => '@state:export',
PushCommand::TASK_NAME => [ Task::ARGS => env('WS_CRON_EXPORT_ARGS', '-v'),
Task::NAME => PushCommand::TASK_NAME, ],
Task::ENABLED => (bool)env('WS_CRON_PUSH', false), PushCommand::TASK_NAME => [
Task::RUN_AT => (string)env('WS_CRON_PUSH_AT', '*/10 * * * *'), Task::NAME => PushCommand::TASK_NAME,
Task::COMMAND => '@state:push', Task::ENABLED => (bool)env('WS_CRON_PUSH', false),
Task::ARGS => env('WS_CRON_PUSH_ARGS', '-v'), Task::RUN_AT => (string)env('WS_CRON_PUSH_AT', '*/10 * * * *'),
], Task::COMMAND => '@state:push',
PruneCommand::TASK_NAME => [ Task::ARGS => env('WS_CRON_PUSH_ARGS', '-v'),
Task::NAME => PruneCommand::TASK_NAME, ],
Task::ENABLED => 'disable' !== ag($config,'logs.prune.after'), PruneCommand::TASK_NAME => [
Task::RUN_AT => '0 */12 * * *', Task::NAME => PruneCommand::TASK_NAME,
Task::COMMAND => '@config:prune', Task::ENABLED => 'disable' !== ag($config, 'logs.prune.after'),
Task::ARGS => '-v', Task::RUN_AT => (string)env('WS_CRON_PRUNE_AT', '0 */12 * * *'),
Task::COMMAND => '@config:prune',
Task::ARGS => env('WS_CRON_PRUNE_ARGS', '-v'),
],
], ],
]; ];

2
config/directories.php
View File

@@ -5,7 +5,7 @@ declare(strict_types=1);
return [ return [
'%(path)/db/archive', '%(path)/db/archive',
'%(path)/config', '%(path)/config',
'%(tmpDir)/logs/crons', '%(tmpDir)/logs/tasks',
'%(tmpDir)/cache', '%(tmpDir)/cache',
'%(tmpDir)/profiler', '%(tmpDir)/profiler',
'%(tmpDir)/webhooks', '%(tmpDir)/webhooks',

58
src/Commands/Config/PruneCommand.php
View File

@@ -6,6 +6,7 @@ namespace App\Commands\Config;
use App\Command; use App\Command;
use App\Libs\Config; use App\Libs\Config;
use Psr\Log\LoggerInterface;
use SplFileInfo; use SplFileInfo;
use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Input\InputOption;
@@ -15,6 +16,11 @@ final class PruneCommand extends Command
{ {
public const TASK_NAME = 'prune'; public const TASK_NAME = 'prune';
public function __construct(private LoggerInterface $logger)
{
parent::__construct();
}
protected function configure(): void protected function configure(): void
{ {
$this->setName('config:prune') $this->setName('config:prune')
@@ -25,6 +31,7 @@ final class PruneCommand extends Command
'delete files older specified time', 'delete files older specified time',
Config::get('logs.prune.after', '-3 DAYS') Config::get('logs.prune.after', '-3 DAYS')
) )
->addOption('dry-run', null, InputOption::VALUE_NONE, 'Do not take any action.')
->setDescription('Prune old logs files.'); ->setDescription('Prune old logs files.');
} }
@@ -33,30 +40,49 @@ final class PruneCommand extends Command
$time = $input->getOption('older-than'); $time = $input->getOption('older-than');
if ('disable' === $time) { if ('disable' === $time) {
$output->writeln('Pruning is disabled.', OutputInterface::VERBOSITY_DEBUG); $this->logger->notice('Pruning is disabled.');
return self::SUCCESS; return self::SUCCESS;
} }
$expiresAt = strtotime($input->getOption('older-than'), time()); if (false === ($expiresAt = strtotime($time, time()))) {
$this->logger->error('Invalid older than date was given.', [
'date' => $time,
]);
return self::FAILURE;
}
$paths = [ $directories = [
[ [
'path' => Config::get('tmpDir') . '/logs/crons', 'path' => Config::get('tmpDir') . '/logs',
'filter' => '*.log',
],
[
'path' => Config::get('tmpDir') . '/logs/tasks',
'filter' => '*.log', 'filter' => '*.log',
], ],
[ [
'path' => Config::get('tmpDir') . '/webhooks', 'path' => Config::get('tmpDir') . '/webhooks',
'filter' => '*.json', 'filter' => '*.json',
], ],
[
'path' => Config::get('tmpDir') . '/profiler',
'filter' => '*.json',
],
[ [
'path' => Config::get('tmpDir') . '/debug', 'path' => Config::get('tmpDir') . '/debug',
'filter' => '*.json', 'filter' => '*.json',
], ],
]; ];
foreach ($paths as $item) { $inDryRunMode = $input->getOption('dry-run');
if (!is_dir(ag($item, 'path'))) {
$output->writeln(sprintf('Path \'%s\' does not exists.', ag($item, 'path'))); foreach ($directories as $item) {
$path = ag($item, 'path');
if (null === $path || !is_dir($path)) {
$this->logger->warning('Path does not exists.', [
'path' => $path
]);
continue; continue;
} }
@@ -65,16 +91,28 @@ final class PruneCommand extends Command
$fileName = $file->getBasename(); $fileName = $file->getBasename();
if ('.' === $fileName || '..' === $fileName || $file->isDir() || !$file->isFile()) { if ('.' === $fileName || '..' === $fileName || true === $file->isDir() || false === $file->isFile()) {
$this->logger->debug('Path is not considered valid file.', [
'path' => $file->getRealPath(),
]);
continue; continue;
} }
if ($file->getMTime() > $expiresAt) { if ($file->getMTime() > $expiresAt) {
$this->logger->debug('Path Not yet expired.', [
'path' => after($file->getRealPath(), Config::get('tmpDir')),
'ttl' => $file->getMTime() - $expiresAt,
]);
continue; continue;
} }
$output->writeln(sprintf('Deleting %s', $file->getRealPath())); $this->logger->notice('Deleting File.', [
unlink($file->getRealPath()); 'file' => after($file->getRealPath(), Config::get('tmpDir'))
]);
if (false === $inDryRunMode) {
unlink($file->getRealPath());
}
} }
} }

4
src/Commands/Scheduler/RunCommand.php
View File

@@ -115,7 +115,7 @@ final class RunCommand extends Command
if ($input->getOption('save-log')) { if ($input->getOption('save-log')) {
file_put_contents( file_put_contents(
Config::get('tmpDir') . '/logs/crons/' . gmdate('Y_m_d') . '.log', Config::get('tasks.logfile'),
preg_replace('#\R+#', PHP_EOL, implode(PHP_EOL, $this->logs)), preg_replace('#\R+#', PHP_EOL, implode(PHP_EOL, $this->logs)),
FILE_APPEND FILE_APPEND
); );
@@ -142,7 +142,7 @@ final class RunCommand extends Command
{ {
$tasks = []; $tasks = [];
foreach (Config::get('tasks', []) as $i => $task) { foreach (Config::get('tasks.commands', []) as $i => $task) {
$task[Task::NAME] = $task[Task::NAME] ?? 'task_' . ((int)($i) + 1); $task[Task::NAME] = $task[Task::NAME] ?? 'task_' . ((int)($i) + 1);
$tasks[$task[Task::NAME]] = $task; $tasks[$task[Task::NAME]] = $task;
} }

14
src/Libs/Extends/ConsoleHandler.php
View File

@@ -2,6 +2,7 @@
namespace App\Libs\Extends; namespace App\Libs\Extends;
use DateTimeInterface;
use Monolog\Handler\AbstractProcessingHandler; use Monolog\Handler\AbstractProcessingHandler;
use Monolog\Logger; use Monolog\Logger;
use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Output\OutputInterface;
@@ -51,13 +52,18 @@ class ConsoleHandler extends AbstractProcessingHandler
protected function write(array $record): void protected function write(array $record): void
{ {
$date = $record['datetime'] ?? 'No date set';
if (true === ($date instanceof DateTimeInterface)) {
$date = $date->format(DateTimeInterface::ATOM);
}
$message = sprintf( $message = sprintf(
'[%s] %s.%s: %s %s', '[%s] %s: %s %s',
$record['datetime'], $date,
$record['channel'] ?? 'logger',
$record['level_name'] ?? $record['level'] ?? '??', $record['level_name'] ?? $record['level'] ?? '??',
$record['message'], $record['message'],
!empty($record['context']) ? '{' . arrayToString($record['context']) . '}' : '' !empty($record['context']) ? '{ ' . arrayToString($record['context']) . ' }' : ''
); );
$this->output->writeln($message, $this->output->getVerbosity()); $this->output->writeln($message, $this->output->getVerbosity());