cphipps/watchstate
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Updated logs file naming to allow for easier pruning.

Browse Source
This commit is contained in:
Abdulmhsen B. A. A
2022-06-05 03:15:19 +03:00
parent d2ca6b2f43
commit db0e153152
6 changed files with 120 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
README.md
View File

@@ -282,28 +282,28 @@ Click `Save Changes`
# Environment variables.
| Environment key | Value | Description | Default |
|-------------------------|---------|----------------------------------------------------------------------------------|--------------------------------|
| WS_DATA_PATH | string | Where key data stored (config, db). | `${BASE_PATH}/var` |
| WS_TMP_DIR | string | Where temp data stored. (logs, cache). | `${WS_DATA_PATH}` |
| WS_TZ | string | Set timezone. | `UTC` |
| WS_WEBHOOK_DEBUG | bool | Store webhook payloads into `${WS_TMP_DIR}/webhooks` | `false` |
| WS_REQUEST_DEBUG | bool | Store request payloads into `${WS_TMP_DIR}/debug` | `false` |
| WS_WEBHOOK_TOKEN_LENGTH | integer | How many bits to use for webhook token generator | `16` |
| WS_LOGGER_FILE_ENABLE | bool | Save logs to file. | `true` |
| WS_LOGGER_FILE | string | Full path to log file. | `${WS_TMP_DIR}/logs/app.log` |
| WS_LOGGER_FILE_LEVEL | string | File Logger Level. | `ERROR` |
| WS_CRON_IMPORT | bool | Enable import scheduled task. Value casted to bool. | `false` |
| WS_CRON_IMPORT_AT | string | When to run import scheduled task. Valid Cron Expression Expected. | `'0 */1 * * *` (Every 1h) |
| WS_CRON_IMPORT_ARGS | string | Flags to pass to the import command. | `'-v` |
| WS_CRON_EXPORT | bool | Enable export scheduled task. Value casted to bool. | `false` |
| WS_CRON_EXPORT_AT | string | When to run export scheduled task. Valid Cron Expression Expected. | `'30 */1 * * *` (Every 1h 30m) |
| WS_CRON_EXPORT_ARGS | string | Flags to pass to the export command. | `'-v` |
| WS_CRON_PUSH | bool | Enable push scheduled task. Value casted to bool. | `false` |
| WS_CRON_PUSH_AT | string | When to run push scheduled task. Valid Cron Expression Expected. | `'*/10 * * * *` (Every 10m) |
| WS_CRON_PUSH_ARGS | string | Flags to pass to the push command. | `'-v` |
| WS_LOGS_PRUNE_AFTER | string | Delete logs older than specified time. Set to `disable` to disable the pruning. | `'-3 DAYS` |
| WS_DEBUG_IMPORT | bool | Log no valid/externals id during import process. stored in `${WS_TMP_DIR}/debug` | `'false` |
| Environment key | Value | Description | Default |
|-------------------------|---------|----------------------------------------------------------------------------------|------------------------------------|
| WS_DATA_PATH | string | Where key data stored (config, db). | `${BASE_PATH}/var` |
| WS_TMP_DIR | string | Where temp data stored. (logs, cache). | `${WS_DATA_PATH}` |
| WS_TZ | string | Set timezone. | `UTC` |
| WS_WEBHOOK_DEBUG | bool | Store webhook payloads into `${WS_TMP_DIR}/webhooks` | `false` |
| WS_REQUEST_DEBUG | bool | Store request payloads into `${WS_TMP_DIR}/debug` | `false` |
| WS_WEBHOOK_TOKEN_LENGTH | integer | How many bits to use for webhook token generator | `16` |
| WS_LOGGER_FILE_ENABLE | bool | Save logs to file. | `true` |
| WS_LOGGER_FILE | string | Full path to log file. | `${WS_TMP_DIR}/logs/app.(Ymd).log` |
| WS_LOGGER_FILE_LEVEL | string | File Logger Level. | `ERROR` |
| WS_CRON_IMPORT | bool | Enable import scheduled task. Value casted to bool. | `false` |
| WS_CRON_IMPORT_AT | string | When to run import scheduled task. Valid Cron Expression Expected. | `'0 */1 * * *` (Every 1h) |
| WS_CRON_IMPORT_ARGS | string | Flags to pass to the import command. | `'-v` |
| WS_CRON_EXPORT | bool | Enable export scheduled task. Value casted to bool. | `false` |
| WS_CRON_EXPORT_AT | string | When to run export scheduled task. Valid Cron Expression Expected. | `'30 */1 * * *` (Every 1h 30m) |
| WS_CRON_EXPORT_ARGS | string | Flags to pass to the export command. | `'-v` |
| WS_CRON_PUSH | bool | Enable push scheduled task. Value casted to bool. | `false` |
| WS_CRON_PUSH_AT | string | When to run push scheduled task. Valid Cron Expression Expected. | `'*/10 * * * *` (Every 10m) |
| WS_CRON_PUSH_ARGS | string | Flags to pass to the push command. | `'-v` |
| WS_LOGS_PRUNE_AFTER | string | Delete logs older than specified time. Set to `disable` to disable the pruning. | `'-3 DAYS` |
| WS_DEBUG_IMPORT | bool | Log no valid/externals id during import process. stored in `${WS_TMP_DIR}/debug` | `'false` |
# Container specific environment variables.

66
config/config.php
View File

@@ -33,6 +33,8 @@ return (function () {
],
];
$config['logs']['affix'] = makeDate()->format('Ymd');
$config['tmpDir'] = fixPath(env('WS_TMP_DIR', ag($config, 'path')));
$config['storage'] += [
@@ -85,7 +87,7 @@ return (function () {
'options' => [
'save.handler' => 'file',
'save.handler.file' => [
'filename' => ag($config, 'tmpDir') . '/profiler/profiler_' . gmdate('Y_m_d_His') . '.json'
'filename' => ag($config, 'tmpDir') . '/profiler/run.' . makeDate()->format('Ymd_His') . '.json'
],
],
],
@@ -103,7 +105,10 @@ return (function () {
'type' => 'stream',
'enabled' => env('WS_LOGGER_FILE_ENABLE', true),
'level' => env('WS_LOGGER_FILE_LEVEL', Logger::ERROR),
'filename' => env('WS_LOGGER_FILE', fn() => ag($config, 'tmpDir') . '/logs/app.log'),
'filename' => env(
'WS_LOGGER_FILE',
fn() => ag($config, 'tmpDir') . '/logs/app.' . ag($config, 'logs.affix') . '.log'
),
],
'stderr' => [
'type' => 'stream',
@@ -171,33 +176,36 @@ return (function () {
];
$config['tasks'] = [
ImportCommand::TASK_NAME => [
Task::NAME => ImportCommand::TASK_NAME,
Task::ENABLED => (bool)env('WS_CRON_IMPORT', false),
Task::RUN_AT => (string)env('WS_CRON_IMPORT_AT', '0 */1 * * *'),
Task::COMMAND => '@state:import',
Task::ARGS => env('WS_CRON_IMPORT_ARGS', '-v'),
],
ExportCommand::TASK_NAME => [
Task::NAME => ExportCommand::TASK_NAME,
Task::ENABLED => (bool)env('WS_CRON_EXPORT', false),
Task::RUN_AT => (string)env('WS_CRON_EXPORT_AT', '30 */1 * * *'),
Task::COMMAND => '@state:export',
Task::ARGS => env('WS_CRON_EXPORT_ARGS', '-v'),
],
PushCommand::TASK_NAME => [
Task::NAME => PushCommand::TASK_NAME,
Task::ENABLED => (bool)env('WS_CRON_PUSH', false),
Task::RUN_AT => (string)env('WS_CRON_PUSH_AT', '*/10 * * * *'),
Task::COMMAND => '@state:push',
Task::ARGS => env('WS_CRON_PUSH_ARGS', '-v'),
],
PruneCommand::TASK_NAME => [
Task::NAME => PruneCommand::TASK_NAME,
Task::ENABLED => 'disable' !== ag($config,'logs.prune.after'),
Task::RUN_AT => '0 */12 * * *',
Task::COMMAND => '@config:prune',
Task::ARGS => '-v',
'logfile' => ag($config, 'tmpDir') . '/logs/tasks/task.' . ag($config, 'logs.affix') . '.log',
'commands' => [
ImportCommand::TASK_NAME => [
Task::NAME => ImportCommand::TASK_NAME,
Task::ENABLED => (bool)env('WS_CRON_IMPORT', false),
Task::RUN_AT => (string)env('WS_CRON_IMPORT_AT', '0 */1 * * *'),
Task::COMMAND => '@state:import',
Task::ARGS => env('WS_CRON_IMPORT_ARGS', '-v'),
],
ExportCommand::TASK_NAME => [
Task::NAME => ExportCommand::TASK_NAME,
Task::ENABLED => (bool)env('WS_CRON_EXPORT', false),
Task::RUN_AT => (string)env('WS_CRON_EXPORT_AT', '30 */1 * * *'),
Task::COMMAND => '@state:export',
Task::ARGS => env('WS_CRON_EXPORT_ARGS', '-v'),
],
PushCommand::TASK_NAME => [
Task::NAME => PushCommand::TASK_NAME,
Task::ENABLED => (bool)env('WS_CRON_PUSH', false),
Task::RUN_AT => (string)env('WS_CRON_PUSH_AT', '*/10 * * * *'),
Task::COMMAND => '@state:push',
Task::ARGS => env('WS_CRON_PUSH_ARGS', '-v'),
],
PruneCommand::TASK_NAME => [
Task::NAME => PruneCommand::TASK_NAME,
Task::ENABLED => 'disable' !== ag($config, 'logs.prune.after'),
Task::RUN_AT => (string)env('WS_CRON_PRUNE_AT', '0 */12 * * *'),
Task::COMMAND => '@config:prune',
Task::ARGS => env('WS_CRON_PRUNE_ARGS', '-v'),
],
],
];

2
config/directories.php
View File

@@ -5,7 +5,7 @@ declare(strict_types=1);
return [
'%(path)/db/archive',
'%(path)/config',
'%(tmpDir)/logs/crons',
'%(tmpDir)/logs/tasks',
'%(tmpDir)/cache',
'%(tmpDir)/profiler',
'%(tmpDir)/webhooks',

58
src/Commands/Config/PruneCommand.php
View File

@@ -6,6 +6,7 @@ namespace App\Commands\Config;
use App\Command;
use App\Libs\Config;
use Psr\Log\LoggerInterface;
use SplFileInfo;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
@@ -15,6 +16,11 @@ final class PruneCommand extends Command
{
public const TASK_NAME = 'prune';
public function __construct(private LoggerInterface $logger)
{
parent::__construct();
}
protected function configure(): void
{
$this->setName('config:prune')
@@ -25,6 +31,7 @@ final class PruneCommand extends Command
'delete files older specified time',
Config::get('logs.prune.after', '-3 DAYS')
)
->addOption('dry-run', null, InputOption::VALUE_NONE, 'Do not take any action.')
->setDescription('Prune old logs files.');
}
@@ -33,30 +40,49 @@ final class PruneCommand extends Command
$time = $input->getOption('older-than');
if ('disable' === $time) {
$output->writeln('Pruning is disabled.', OutputInterface::VERBOSITY_DEBUG);
$this->logger->notice('Pruning is disabled.');
return self::SUCCESS;
}
$expiresAt = strtotime($input->getOption('older-than'), time());
if (false === ($expiresAt = strtotime($time, time()))) {
$this->logger->error('Invalid older than date was given.', [
'date' => $time,
]);
return self::FAILURE;
}
$paths = [
$directories = [
[
'path' => Config::get('tmpDir') . '/logs/crons',
'path' => Config::get('tmpDir') . '/logs',
'filter' => '*.log',
],
[
'path' => Config::get('tmpDir') . '/logs/tasks',
'filter' => '*.log',
],
[
'path' => Config::get('tmpDir') . '/webhooks',
'filter' => '*.json',
],
[
'path' => Config::get('tmpDir') . '/profiler',
'filter' => '*.json',
],
[
'path' => Config::get('tmpDir') . '/debug',
'filter' => '*.json',
],
];
foreach ($paths as $item) {
if (!is_dir(ag($item, 'path'))) {
$output->writeln(sprintf('Path \'%s\' does not exists.', ag($item, 'path')));
$inDryRunMode = $input->getOption('dry-run');
foreach ($directories as $item) {
$path = ag($item, 'path');
if (null === $path || !is_dir($path)) {
$this->logger->warning('Path does not exists.', [
'path' => $path
]);
continue;
}
@@ -65,16 +91,28 @@ final class PruneCommand extends Command
$fileName = $file->getBasename();
if ('.' === $fileName || '..' === $fileName || $file->isDir() || !$file->isFile()) {
if ('.' === $fileName || '..' === $fileName || true === $file->isDir() || false === $file->isFile()) {
$this->logger->debug('Path is not considered valid file.', [
'path' => $file->getRealPath(),
]);
continue;
}
if ($file->getMTime() > $expiresAt) {
$this->logger->debug('Path Not yet expired.', [
'path' => after($file->getRealPath(), Config::get('tmpDir')),
'ttl' => $file->getMTime() - $expiresAt,
]);
continue;
}
$output->writeln(sprintf('Deleting %s', $file->getRealPath()));
unlink($file->getRealPath());
$this->logger->notice('Deleting File.', [
'file' => after($file->getRealPath(), Config::get('tmpDir'))
]);
if (false === $inDryRunMode) {
unlink($file->getRealPath());
}
}
}

4
src/Commands/Scheduler/RunCommand.php
View File

@@ -115,7 +115,7 @@ final class RunCommand extends Command
if ($input->getOption('save-log')) {
file_put_contents(
Config::get('tmpDir') . '/logs/crons/' . gmdate('Y_m_d') . '.log',
Config::get('tasks.logfile'),
preg_replace('#\R+#', PHP_EOL, implode(PHP_EOL, $this->logs)),
FILE_APPEND
);
@@ -142,7 +142,7 @@ final class RunCommand extends Command
{
$tasks = [];
foreach (Config::get('tasks', []) as $i => $task) {
foreach (Config::get('tasks.commands', []) as $i => $task) {
$task[Task::NAME] = $task[Task::NAME] ?? 'task_' . ((int)($i) + 1);
$tasks[$task[Task::NAME]] = $task;
}

14
src/Libs/Extends/ConsoleHandler.php
View File

@@ -2,6 +2,7 @@
namespace App\Libs\Extends;
use DateTimeInterface;
use Monolog\Handler\AbstractProcessingHandler;
use Monolog\Logger;
use Symfony\Component\Console\Output\OutputInterface;
@@ -51,13 +52,18 @@ class ConsoleHandler extends AbstractProcessingHandler
protected function write(array $record): void
{
$date = $record['datetime'] ?? 'No date set';
if (true === ($date instanceof DateTimeInterface)) {
$date = $date->format(DateTimeInterface::ATOM);
}
$message = sprintf(
'[%s] %s.%s: %s %s',
$record['datetime'],
$record['channel'] ?? 'logger',
'[%s] %s: %s %s',
$date,
$record['level_name'] ?? $record['level'] ?? '??',
$record['message'],
!empty($record['context']) ? '{' . arrayToString($record['context']) . '}' : ''
!empty($record['context']) ? '{ ' . arrayToString($record['context']) . ' }' : ''
);
$this->output->writeln($message, $this->output->getVerbosity());