22

✏️ Editing: slurp_exim_mainlog

#!/usr/local/cpanel/3rdparty/bin/perl

# cpanel - scripts/slurp_exim_mainlog              Copyright 2022 cPanel, L.L.C.
#                                                           All rights reserved.
# copyright@cpanel.net                                         http://cpanel.net
# This code is subject to the cPanel license. Unauthorized copying is prohibited

package scripts::slurp_exim_mainlog;

use strict;
use warnings;

use parent qw( Cpanel::HelpfulScript );

use Cpanel::EximStats::ImportInProgress ();
use Cpanel::Autodie::Unlink             ();
use Cpanel::Daemonizer::Tiny            ();
use Cpanel::Exception                   ();
use Cpanel::EximStats::Retention        ();
use Cpanel::FileUtils::Dir              ();
use Cpanel::FileUtils::TouchFile        ();
use Cpanel::PIDFile                     ();
use Cpanel::Time::ISO                   ();

use Try::Tiny;

=encoding utf-8

=head1 NAME

scripts::slurp_exim_mainlog

=head1 SYNOPSIS

slurp_exim_mainlog ( --reimport | --force | --help )

This command will do an import of the unarchived (non-'.gz') /var/log/exim_mainlog* files that are
newer than the exim retention setting (90 by default) days. The slurp normally will only run once,
but if you pass the --reimport flag it will attempt to import the log files after a 7 day waiting period.
If you pass the --force flag, the import will start with no regard to the last time it ran.

=head1 DESCRIPTION

This script does an import of the exim_mainlog files in /var/log. The import can only happen
once every 7 days

=cut

our $EXIM_LOG_DIR    = '/var/log';
our $PID_FILE        = '/var/run/slurp_exim_mainlog.pid';
our $OUTPUT_LOG_DIR  = '/var/cpanel/logs';
our $OUTPUT_LOG_FILE = 'eximstats_sqlite_import.log';

our $MAX_OUTPUT_LOG_AGE   = 30 * 24 * 60**2;    # 30 days
our $TIME_BETWEEN_IMPORTS = 7 * 24 * 60**2;     # 7 days

sub _OPTIONS {
    return qw( force reimport );
}

__PACKAGE__->new(@ARGV)->script() unless caller();

## CAVEATS re: case 53744
## 1. the eximstats schema handles dedupes, so we process all recent files
## 2. TaskQueue is overkill for this one-time slurp

sub script {
    my ($self) = @_;

my $time = _time();

## skip the slurp of exim_mainlog* if the touch file exists and it hasn't been TIME_BETWEEN_IMPORTS yet
    if ( !$self->getopt('force') ) {
        if ( $self->getopt('reimport') ) {
            my $last_import_time          = ( stat $Cpanel::EximStats::ImportInProgress::IMPORTED_FILE )[9] || 0;
            my $seconds_since_last_import = ( $time - $last_import_time );
            if ( -e $Cpanel::EximStats::ImportInProgress::IMPORTED_FILE && $TIME_BETWEEN_IMPORTS > $seconds_since_last_import ) {
                print "[slurp_exim_mainlog] Skipping re-import because the logs were last imported at " . scalar localtime($last_import_time) . ", and the system only allows imports every $TIME_BETWEEN_IMPORTS seconds without the --force flag.\n";

return;
            }
        }
        else {
            if ( -e $Cpanel::EximStats::ImportInProgress::IMPORTED_FILE ) {
                print "[slurp_exim_mainlog] Skipping import because the logs were already imported.\n";

return;
            }
        }
    }

my $IMPORT_LOG_AGE_LIMIT = 60**2 * 24 * ( int( Cpanel::EximStats::Retention::get_valid_exim_retention_days() ) || 60 );

Cpanel::PIDFile->do(
        $PID_FILE,
        sub {
            my %imports;
            for my $log ( grep { m/^exim_mainlog/ } @{ Cpanel::FileUtils::Dir::get_directory_nodes($EXIM_LOG_DIR) } ) {

# Skip archived logs for this import
                next if substr( $log, -3 ) eq '.gz';

my $age = ( stat("$EXIM_LOG_DIR/$log") )[9];

## eximstats shows info for 90 days by default, but it's configurable
                next if ( $time - $age > $IMPORT_LOG_AGE_LIMIT );

$imports{"$EXIM_LOG_DIR/$log"} = $age;
            }

my @imports = keys %imports ? map { $_->[0] } sort { $b->[1] <=> $a->[1] } map { [ $_, $imports{$_} ] } keys %imports : ();

Cpanel::FileUtils::TouchFile::touchfile($Cpanel::EximStats::ImportInProgress::IMPORTED_FILE);

$self->call_import_exim_data( \@imports );
        }
    );

return;
}

# In a function for tests
sub call_import_exim_data {
    my ( $self, $imports_ar ) = @_;

if ( !scalar @$imports_ar ) {
        print "[slurp_exim_mainlog] there were no exim_mainlog files new enough to import.\n";
        return;
    }

$self->remove_old_output_logs();

my $log_file = "$OUTPUT_LOG_DIR/$OUTPUT_LOG_FILE." . Cpanel::Time::ISO::unix2iso();
    print "[slurp_exim_mainlog] starting import of the exim_mainlog files. Output will be logged to: $log_file\n";

Cpanel::Daemonizer::Tiny::run_as_daemon(
        sub {
            close(STDIN);
            open( STDIN, "<", "/dev/null" );

open( my $log_fh, '>>',  $log_file );
            open( STDOUT,     ">&=", $log_fh );
            open( STDERR,     ">&=", $log_fh );

exec( '/usr/local/cpanel/scripts/import_exim_data', @$imports_ar );
        }
    );

return;
}

sub remove_old_output_logs {
    my ($self) = @_;

my @logs_to_remove;
    for my $log ( grep { m/^\Q$OUTPUT_LOG_FILE\E\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$/ } @{ Cpanel::FileUtils::Dir::get_directory_nodes($OUTPUT_LOG_DIR) } ) {
        my $log_path = "$OUTPUT_LOG_DIR/$log";

next if !-e $log_path || _time() - ( stat(_) )[9] < $MAX_OUTPUT_LOG_AGE;

push @logs_to_remove, $log_path;
    }

try {
        Cpanel::Autodie::Unlink::unlink_if_exists_batch(@logs_to_remove) if scalar @logs_to_remove;
    }
    catch {
        warn "[slurp_exim_mainlog] there was a problem removing old output logs: " . Cpanel::Exception::get_string_no_id($_);
    };

return;
}

# For tests
sub _time {
    return time();
}

MR SKB