#!/usr/bin/perl
# SSP - System Status Probe
# Find and print useful troubleshooting info on cPanel servers

=head1 COPYRIGHT

This software is Copyright 2021 by cPanel, L.L.C..

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.

3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.

THE SOFTWARE LICENSED HEREUNDER IS PROVIDED "AS IS" AND CPANEL HEREBY DISCLAIMS ALL WARRANTIES OF ANY KIND, WHETHER EXPRESS OR IMPLIED, RELATING TO THE SOFTWARE, ITS THIRD PARTY COMPONENTS, AND ANY DATA ACCESSED THEREFROM, OR THE ACCURACY, TIMELINESS, COMPLETENESS, OR ADEQUACY OF THE SOFTWARE, ITS THIRD PARTY COMPONENTS, AND ANY DATA ACCESSED THEREFROM, INCLUDING THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, SATISFACTORY QUALITY, FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. CPANEL DOES NOT WARRANT THAT THE SOFTWARE OR ITS THIRD PARTY COMPONENTS ARE ERROR-FREE OR WILL OPERATE WITHOUT INTERRUPTION. IF THE SOFTWARE, ITS THIRD PARTY COMPONENTS, OR ANY DATA ACCESSED THEREFROM IS DEFECTIVE, YOU ASSUME THE SOLE RESPONSIBILITY FOR THE ENTIRE COST OF ALL REPAIR OR INJURY OF ANY KIND, EVEN IF CPANEL HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DEFECTS OR DAMAGES. NO ORAL OR WRITTEN INFORMATION OR ADVICE GIVEN BY CPANEL, ITS AFFILIATES, LICENSEES, DEALERS, SUB-LICENSORS, AGENTS OR EMPLOYEES SHALL CREATE A WARRANTY OR IN ANY WAY INCREASE THE SCOPE OF ANY WARRANTY.

=cut

package SSP;

use 5.006;
use strict;
use warnings;
use File::Find;
use Socket;
use IO::Socket::INET;
use Sys::Hostname;
use Term::ANSIColor qw(:constants);
use Time::Local qw{timelocal timegm};
use IPC::Open3;
use Cwd qw(abs_path);
use Getopt::Long();

# Application version (The project maintainer will bump this, don't modify it.)
our $VERSION = '4.99.221';

# Global variables that alter application runtime
our $OPT_SKIP_NETWORKING;    # Disable network calls
our $OPT_TIMEOUT;            # How long to wait for system commands to finish executing

# Global variables updated throughout application
our $CRIT_BUFFER;            # Critical output to be printed at the end

# Things that are the same but used many places
our $CPANEL_LICENSE_FILE = '/usr/local/cpanel/cpanel.lisc';
our $CPANEL_VERSION_FILE = '/usr/local/cpanel/version';
our $CPANEL_CONFIG_FILE  = '/var/cpanel/cpanel.config';
our $MYSQL_CONF_FILE     = '/etc/my.cnf';
our $PURE_FTPD_CONF_FILE = '/etc/pure-ftpd.conf';

# Global variables initialized at application initialization
our %CPCONF;                 # cpanel.config
our $ORIGINAL_PATH;
our %SOCKET;                 # Dispatcher for optional Socket module usage
our $RUN_STATE;
our $HTTP_GET_HOST_CACHE;
our %MEMOIZE_CACHE;
our $YUM_OR_DNF;
our $YUM_CONF;

run(@ARGV) unless caller;

# Initialize application by setting loading all global variables
# (except the RPM variables).  That's done within run()
sub init {
    if ( $^O ne 'linux' ) {
        die "Unknown OS: $^O (only Linux is supported)";
    }

    if ( $< != 0 ) {
        die "SSP must be run as root\n";
    }

    $ORIGINAL_PATH = $ENV{'PATH'};
    ## no critic (LocalizedPunctuationVars)
    $ENV{'PATH'} = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin';
    $| = 1;
    ## use critic
    $Term::ANSIColor::AUTORESET = 1;

    # It is only helpful to memoize something that is used at least twice.
    # There is some memoize overhead, but it is a safe bet for anything with unpredictable runtimes (network, heavy disk I/O, external processes).
    _memoize(
        qw (
          check_for_non_default_permissions
          check_roots_cron_for_certain_commands
          find_httpd_bin
          get_apache_modules_href
          get_apache_version_href
          get_clock_skew
          get_cpanel_license_file_info_href
          get_cpuinfo_href
          get_cpupdate_conf
          get_ea3_php_conf_href
          get_exim_localopts_href
          get_external_ip
          get_external_license_ip
          get_hostinfo_href
          get_hostname
          get_installed_ea4_php_href
          get_ipcs_href
          get_license_info
          get_local_ipaddrs_aref
          get_lsof_port_href
          get_lsws_version_aref
          get_meminfo
          get_mysql_conf_href
          get_mysql_full_version
          get_mysql_numeric_version
          get_new_backup_conf_href
          get_old_backup_conf_href
          get_phpini_aref
          get_process_pid_href
          get_rpm_href
          get_tiers_file
          get_tiers_json_href
          )
    );

    _populate_run_state();

    if ( i_am_one_of( 'cpanel', 'dnsonly' ) ) {
        %CPCONF = get_cpanel_conf();
    }

    ## no critic (StringyEval)
    # This avoids compile-time errors on old Perl where Socket::get(addr|name)info and related constants don't exist.
    eval q(  # Perl 5.14+
        # The import is redundant but guarantees the desirable failure of this eval at run-time if anything is missing.
        Socket->import(qw(getaddrinfo getnameinfo NI_NAMEREQD NI_NUMERICHOST NIx_NOSERV SOCK_RAW));
        %SOCKET = (
            'getaddrinfo'    => \&Socket::getaddrinfo,
            'getnameinfo'    => \&Socket::getnameinfo,
            'NI_NAMEREQD'    => Socket::NI_NAMEREQD,
            'NI_NUMERICHOST' => Socket::NI_NUMERICHOST,
            'NIx_NOSERV'     => Socket::NIx_NOSERV,
            'SOCK_RAW'       => Socket::SOCK_RAW,
        );
        return 1;
    );
    ## use critic
    $SIG{'INT'} = sub {    ## no critic (LocalizedPunctuationVars)
        print "\n\nJust being impatient, or did SSP actually hang? What if it didn't get a chance to check something really important?\n";
        print "\nIf you really want out of here and it didn't work the first time then you interrupted a child and not the parent process. Keep hitting CTRL+C.\n";
        if ($CRIT_BUFFER) {
            print_magenta("\nThere was critical-level output above, here it is again:");
            print $CRIT_BUFFER;
        }
        die;
    };

    return 1;
}

sub run {
    local @ARGV = @_;    # Because GetOptionsFromArray available in Getopt::Long 2.36 and later only, Perl 5.8.8 on CentOS 5.11 includes 2.35

    Getopt::Long::GetOptions(
        'help'       => sub { exit print_help(); },    ## no critic (NoExitsFromSubroutines)
        'bugreport'  => sub { init(); exit print_bug_report(); },
        'csi'        => sub { init(); exit csi_checks_only(); },
        'docreport'  => sub { init(); exit print_doc_bug_report(); },
        'no-network' => \$OPT_SKIP_NETWORKING,
        'no-speed'   => sub { $MEMOIZE_CACHE{'PRECACHE'} = { disabled => 1 }; },
        'profiling'  => sub {
            load_module_with_fallbacks(
                'needed_subs'  => [qw{tv_interval gettimeofday}],
                'modules'      => [qw{Time::HiRes}],
                'fail_warning' => 'Profiling won\'t work without this',
                'fail_fatal'   => 1,
            );
            $MEMOIZE_CACHE{'PROFILING'} = { enabled => 1 };
        },
        'simulatestate=s@' => sub { _simulate_run_state( $_[1] ); },
        'simulatevar=s%'   => sub { _simulate_run_var( $_[1], $_[2] ); },
        'timeout=i'        => \$OPT_TIMEOUT,
    );

    if ($OPT_TIMEOUT) {
        $OPT_TIMEOUT = int $OPT_TIMEOUT;
        if ( $OPT_TIMEOUT < 5 ) {
            $OPT_TIMEOUT = 5;
        }
    }

    init();

    ######################
    ##  END GLOBALS     ##
    ######################

    print "\n";
    for ( 1 .. 3 ) {
        print BOLD GREEN ON_RED "\tPlease DO NOT paste output from SSP into tickets unless it is relevant to an issue" . RESET . "\n";
    }

    if ( i_am('dnsonly') ) {
        print_start("\n\t\tDNSONLY: ");
        print_warning("/var/cpanel/dnsonly or DNSONLY license detected, assuming DNSONLY operation\n");
    }
    unless ( i_am_one_of( 'cpanel', 'dnsonly' ) ) {
        print_critical("\nCPANEL IS NOT INSTALLED ON THIS SERVER! SOME SSP OUTPUT MAY NOT BE RELEVANT!\n");
    }

    print "\n";
    print_tip();
    print_version();
    print "\n";

    ## [CRIT] -- only stuff that we should check as early as possible
    check_for_hacked_server_touchfile();
    check_for_multiple_tech_logins();
    check_for_lve_environment();
    check_for_os_release_5();
    check_for_os_release_32bit();
    check_for_ea3();

    find_httpd_bin();    # Cache result now, it is used by get_apache_* below.  The following must be memoized in init() first.
    _memoize_parallel_populate_cache(
        qw(
          check_for_non_default_permissions
          check_roots_cron_for_certain_commands
          get_apache_modules_href
          get_apache_version_href
          get_clock_skew
          get_external_license_ip
          get_installed_ea4_php_href
          get_license_info
          get_local_ipaddrs_aref
          get_lsof_port_href
          get_process_pid_href
          get_rpm_href
          get_tiers_file
          get_tiers_json_href
          )
    );
    print "\n";

    ## [INFO]
    print_hostname();
    print_os();
    print_kernel_and_cpu();
    print_kernelcare_info();
    print_cpanel_info();
    check_for_cpanel_update();
    check_lts_autofixer();
    print_uptime();
    check_for_license_info();
    check_for_systemd();
    print_apache_info();
    print_lsws_info();
    check_for_lsws_update();
    print_ea3_php_configuration();
    print_ea4_php_configuration();
    check_for_clustering();
    check_sysinfo();
    check_for_remote_mysql();
    print_if_using_other_dns();
    check_for_unbound_stub_resolver();
    check_for_public_resolvers();
    print_mysql_version();
    print_backups_info();
    print_mailserver_info();
    print_ftpserver_info();
    print_exim_info();
    print_roundcube_db();
    check_for_custom_webtemplates();
    check_for_custom_restoremodules();
    check_for_custom_zonetemplates();
    check_selinux_status();    # Can also WARN
    check_imunify_config();

    ## [WARN]
    check_for_license_error();
    check_for_license_creds();
    check_var_cpanel_users();
    check_port_hash();
    check_runlevel();
    check_multiuser_target();
    check_for_missing_root_cron();
    check_for_missing_usr_bin_crontab();
    check_if_upcp_is_running();
    check_valid_upcp();
    check_cpupdate_conf();
    check_interface_lo();
    check_cpanelconfig_filetype();
    check_cpanelsync_exclude_no_chmod();
    check_cpanelsync_exclude();
    check_for_rawopts();
    check_for_rawenv();
    check_for_custom_opt_mods();
    check_for_local_templates();
    check_for_missing_account_suspensions_conf();
    check_for_custom_apache_includes();
    check_for_tomcatoptions();
    check_for_sneaky_htaccess();
    check_ea4_paths_conf();
    check_ea4_missing_cl_repo();
    check_apache_modules();
    check_apache_niceness();
    check_perl_sanity();
    check_for_non_default_permissions();
    check_for_non_default_file_capabilities();
    check_for_non_default_sysctl();
    check_for_stale_lockfiles();
    check_root_suspended();
    check_limitsconf();
    check_disk_space();
    check_disk_inodes();
    check_mounts();
    check_for_hooks_in_scripts_directory();
    check_for_huge_logs();
    check_easy_skip_cpanelsync();
    check_pkgacct_override();
    check_for_yunsuo();
    check_for_gdm();
    check_for_redhat_firewall();
    check_easyapache();
    check_for_missing_ea3_php();
    check_for_ea3_hooks();
    check_for_unsupported_nat();
    check_for_oracle_linux();
    check_for_usr_local_cpanel_hooks();
    check_for_sql_safe_mode();
    check_for_domain_forwarding();
    check_for_empty_apache_templates();
    check_for_empty_postgres_config();
    check_for_empty_easyapache_profiles();
    check_for_missing_timezone_from_phpini();
    check_for_proc_mdstat_recovery();
    check_usr_local_cpanel_path_for_symlinks();
    check_for_system_mem_below_required();
    check_yum_conf();
    check_yum_plugins();
    check_for_cpanel_files();
    check_bash_history_for_certain_commands();
    check_roots_cron_for_certain_commands();
    check_for_missing_or_commented_customlog();
    check_for_cpsources_conf();
    check_for_apache_rlimits();
    check_for_usr_local_lib_libz_so();
    check_for_non_default_modsec_rules();
    check_etc_hosts_sanity();
    check_localhost_resolution();
    check_for_apache_listen_host_is_localhost();
    check_roundcube_mysql_pass_mismatch();
    check_for_hooks_from_manage_hooks();
    check_mysqld_warnings_errors();
    check_mysql_config();
    check_mysql_datadir();
    check_for_extra_mysql_config_files();
    check_perl_version_less_than_588();
    check_for_low_ulimit_for_root();
    check_for_fork_bomb_protection();
    check_for_harmful_php_mode_600_cron();
    check_for_custom_exim_conf_local();
    check_for_maxclients_or_maxrequestworkers_reached();
    check_for_non_default_umask();
    check_for_multiple_imagemagick_installs();
    check_eximstats_size();

    check_for_broken_mysql_tables();
    check_for_clock_skew();
    check_for_zlib_h();
    check_if_httpdconf_ipaddrs_exist();
    check_distcache_and_libapr();
    check_for_custom_repos();
    check_for_rpm_overrides();
    check_var_cpanel_immutable_files();
    check_for_noxsave_in_grub_conf();
    check_for_rpm_dist_ver_unknown();
    check_for_homeloader_php_extension();
    check_for_networkmanager();
    check_for_dhclient();
    check_for_var_cpanel_roundcube_install();
    check_for_missing_etc_localtime();
    check_cpanel_config();
    check_pure_ftpd_conf_for_upload_script_and_dead();
    check_for_perl_env_var();
    check_for_disabled_services();
    check_for_cpbackup_exclude_everything();
    check_for_usr_local_include_jpeglib_h();
    check_for_bw_module_and_more_than_1024_vhosts();
    check_for_uppercase_chars_in_hostname();
    check_for_bad_permissions_on_named_ca();
    check_for_jailshell_additional_mounts_trailing_slash();
    check_for_allow_query_localhost();
    check_for_nocloudlinux_touchfile();
    check_for_stupid_touchfile();
    check_for_phphandler_and_opcode_caching_incompatibility();
    check_for_invalid_HOMEDIR();
    check_for_unsupported_options_in_phpini();    # FB-75397
    check_for_suphp_but_no_fileprotect();
    check_if_hostname_missing_from_localdomains();
    check_for_eximstats_newline();
    check_for_processes_killed_by_lfd();
    check_for_processes_killed_by_oom();
    check_for_processes_killed_by_prm();
    check_for_broken_userdatadomains();
    check_ssl_db_perms();
    check_for_stray_index_php();
    check_for_port_80_not_apache();
    check_for_missing_groups();
    check_for_noquotafs();
    check_for_roundcube_overlay();
    check_for_hostname_park_zoneexists();
    check_for_pgpass_colon_in_password_field();
    check_for_dirs_that_break_ea();
    check_for_extra_uid_0_user();
    check_sudoers_files();
    check_for_easyparams_attributes();
    check_for_allow_update_in_named_conf();
    check_for_broken_mysqldump();
    check_exim_log_sanity();
    check_exim_localopts();
    check_updatelog();
    check_for_readonly_filesystems();
    check_for_cl_unsupported_memory_limits();
    check_for_eblockers();
    check_for_php_selector_incompatibilities();
    check_cloudlinux_sanity();
    check_for_modsec2_stage_files();
    check_for_cron_allow();
    check_for_dev_sandbox();
    check_for_jail_owner();
    check_sshd_config();
    check_for_saltstack();
    check_for_puppet_agent();
    check_for_etc_ubic_dir();
    check_for_acls_cpconf();
    check_for_port_53_dnsmasq();
    check_for_port_21_ftp();
    check_root_dns_resolvers();
    check_cloudlinux_phphandler_file();
    check_ftpusers_file();
    check_mylogincnf();
    check_themesconf();
    check_auto_migrate_ea3_to_ea4();
    check_use_apache_md5_for_htaccess();
    check_for_wordpress_manager_rpms();

    # [3RDP]
    check_smtp_processes();
    check_for_varnish();
    check_for_nginx();
    check_for_mailscanner();
    check_for_apf();
    check_for_csf();
    check_for_prm();
    check_for_les();
    check_for_1h();
    check_for_webmin();
    check_for_symantec();
    check_for_newrelic();
    check_for_multilevel_reseller();
    check_for_cpremote();
    check_for_whmxtra();
    check_for_usr_local_mis();
    check_for_opt_gsi_tools();
    check_for_pyxsoft_antimalware();
    check_for_magicspam();

    # [CRIT] - Anything that requires a pre-defined response to be sent, escalation, or extreme care.
    check_cves_vulnerable_versions();
    check_for_unsupported_php();         # Extreme care!
    check_for_bash_secadv_20140924();    # advisory
    all_malware_checks();

    check_for_additional_rpms();
    check_for_percona_rpms();
    check_for_duplicate_rpms();
    check_for_kernel_headers_rpm();
    check_for_frontpage_rpms();
    check_for_broken_rpm();
    check_for_ea4_mismatch();
    check_mysql_skip_grants();

    print_info2('Done.');

    if ($CRIT_BUFFER) {
        print_magenta("\n\nThere was critical-level output above, here it is again:");
        print $CRIT_BUFFER;
    }

    print_profiling_data();
    return 0;
}

sub print_help {
    print BOLD YELLOW ON_BLACK "System Status Probe $VERSION\n" . RESET;
    (
        my $message = q{
        This software is Copyright 2017 by cPanel, Inc.
        Find and print useful troubleshooting info on cPanel servers.

        Usage: ssp [option]
            --bugreport         Provide bug report template along with environment details (intended for cPanel tech use only)
            --csi               cPanel Security Inspection, run's several security checks
            --docreport         Provide document bug report template (intended for cPanel tech use only)
            --no-network        Skip all network related checks
            --timeout           This overides timeouts of several longer running functions (min: 5s)

        }
    ) =~ s/^ {8}//mg;
    print $message;
}

sub csi_checks_only {
    check_cves_vulnerable_versions();
    check_port_hash();
    check_for_bash_secadv_20140924();    # advisory
    all_malware_checks();
    check_mysql_skip_grants();
    print_info2('SSP checks done.');
}

sub all_malware_checks {
    check_for_UMBREON_rootkit();
    check_for_libms_rootkit();
    check_for_jynx2_rootkit();
    check_for_fritzfrog_botnet();
    check_for_cdorked_A();
    check_for_cdorked_B();
    check_for_libkeyutils_symbols();
    check_for_libkeyutils_filenames();
    check_sha1_sigs_libkeyutils();
    check_sha1_sigs_httpd();
    check_sha1_sigs_named();
    check_sha1_sigs_ssh();
    check_sha1_sigs_ssh_add();
    check_sha1_sigs_sshd();
    check_for_ebury_ssh_G();
    check_for_ebury_ssh_shmem();
    check_for_ebury_root_file();
    check_for_bg_botnet();
    check_for_dragnet();
    check_for_xor_ddos();
    check_for_shellbot();
    check_for_ncom_filenames();
    check_for_cpro();
    check_for_fkcplisc();
    check_for_yoncu();
    check_for_cgls();
    check_for_ctls();
    check_for_xbash();
    check_passwd_hiddenwasp_dirtycow();
    check_for_malicious_root_cron();
    check_for_spoofed_kernel_modules();
    check_for_bad_perms_auth_keys();
    check_for_strange_setuid_binaries();
    check_changepasswd_modules();
    check_for_missing_ps_cmd();
}

sub get_phpini_aref {
    my $phpini = '/usr/local/lib/php.ini';
    my @phpini;
    return () if !-f $phpini;

    if ( open my $fh, '<', $phpini ) {
        while (<$fh>) {
            next if (/^(?:;|$|\[)/);
            chomp;
            push @phpini, $_;
        }
        close $fh;
    }

    return \@phpini;
}

sub find_httpd_bin {
    if ( i_am('ea4') ) {
        return '/usr/sbin/httpd' if -x '/usr/sbin/httpd';
    }
    elsif ( i_am('ea3') ) {
        return '/usr/local/apache/bin/httpd' if -x '/usr/local/apache/bin/httpd';
    }
    return;
}

sub get_apache_version_href {
    return unless my $httpd_bin = find_httpd_bin();
    return unless my @output = split /\n/, timed_run( 0, $httpd_bin, '-v' );
    my %info;
    foreach (@output) {
        if (m{ \A Server \s+ version: \s+ Apache/([^\s]+) \s }xms) {
            $info{'version'} = $1;
        }
        if (m{ \A Server \s+ built: \s+ (.*) \z }xms) {
            $info{'built'} = $1;
            $info{'built'} =~ s/^\s+//g;
        }
        if (m{ \A Cpanel::Easy::Apache \s+ (.*) \z }xms) {
            $info{'ea_version'} = $1;
        }
    }
    if ( i_am('ea4') ) {
        chomp( $info{'ea_version'} = timed_run( 0, 'rpm', '-qf', $httpd_bin ) );
        $info{'ea_version'} =~ s/\.\w\d{1,3}\D+\d+\n//;
    }
    return \%info;
}

sub get_apache_version {
    return unless my $href = get_apache_version_href();
    return unless defined $href->{'version'};
    return $href->{'version'};
}

sub get_apache_modules_href {
    return unless my $httpd_bin = find_httpd_bin();
    my %modules = map { ( split( /\s+/, $_, 3 ) )[1] => 1 } split /\n/, timed_run( 0, $httpd_bin, '-M' );
    return \%modules;
}

sub get_cpanel_license_file_info_href {
    my %license;
    if ( open my $license_fh, '<', $CPANEL_LICENSE_FILE ) {
        my @license_text;
        while (<$license_fh>) {
            last if m{ \A -----BEGIN }xms;
            next unless m{ \A \p{IsPrint}+ \Z }xms;
            chomp;
            push @license_text, $_;
        }
        close $license_fh;
        %license = map { ( split( /:\s+/, $_, 2 ) )[ 0, 1 ] } @license_text;
    }
    return \%license;
}

sub license_file_is_cloudlinux {
    my $href = get_cpanel_license_file_info_href();
    return   if not exists $href->{products};
    return 1 if grep { /cloudlinux/ } $href->{products};
    return 0;
}

sub license_file_is_cpanel {
    my $href = get_cpanel_license_file_info_href();
    return   if not exists $href->{products};
    return 1 if grep { /cpanel/ } $href->{products};
    return 0;
}

sub license_file_is_dnsonly {
    my $href = get_cpanel_license_file_info_href();
    return   if not exists $href->{products};
    return 1 if grep { /dnsonly/ } $href->{products};
    return 0;
}

sub license_file_is_solo {

    # products =~ cpanel and maxusers = 1 indicates Solo.
    my $href = get_cpanel_license_file_info_href();
    return   if not exists $href->{products} or not exists $href->{maxusers};
    return 1 if ( grep { /cpanel/ } $href->{products} and $href->{maxusers} == 1 );
    return 0;
}

sub get_cpanel_conf {
    my %cpconf;
    if ( open( my $cpconf_fh, '<', $CPANEL_CONFIG_FILE ) ) {
        local $/ = undef;
        %cpconf = map { ( split( /=/, $_, 2 ) )[ 0, 1 ] } split( /\n/, readline($cpconf_fh) );
        close $cpconf_fh;
        return %cpconf;
    }
    else {
        print_crit('cpanel.config: ');
        print_critical("$CPANEL_CONFIG_FILE could not be opened.\n");
    }
}

sub get_cpanel_version {
    my $numeric_version;
    my $original_version;

    if ( open my $file_fh, '<', $CPANEL_VERSION_FILE ) {
        $original_version = readline($file_fh);
        close $file_fh;
    }
    return ( 'UNKNOWN', 'UNKNOWN' ) unless defined $original_version;
    chomp $original_version;

    # Parse either 1.2.3.4 or 1.2.3-THING_4 to 1.2.3.4
    $numeric_version = join( '.', split( /\.|-[a-zA-Z]+_/, $original_version ) );
    $numeric_version = 'UNKNOWN' unless $numeric_version =~ /^\d+\.\d+\.\d+\.\d+$/;

    return ( $numeric_version, $original_version );
}

sub _version_cmp {
    my ( $first, $second ) = @_;
    my ( $a1,    $b1, $c1, $d1 ) = split /[\._]/, $first;
    my ( $a2,    $b2, $c2, $d2 ) = split /[\._]/, $second;
    for my $ref ( \$a1, \$b1, \$c1, \$d1, \$a2, \$b2, \$c2, \$d2, ) {    # Fill empties with 0
        $$ref = 0 unless defined $$ref;
    }
    return $a1 <=> $a2 || $b1 <=> $b2 || $c1 <=> $c2 || $d1 <=> $d2;
}

sub version_compare {

    # example: return if version_compare($ver_string, qw( >= 1.2.3.3 ));
    # Must be no more than four version numbers separated by periods and/or underscores.
    my ( $ver1, $mode, $ver2 ) = @_;
    return if ( !defined($ver1) || ( $ver1 =~ /[^\._0-9]/ ) );
    return if ( !defined($ver2) || ( $ver2 =~ /[^\._0-9]/ ) );

    # Shamelessly copied the comparison logic out of Cpanel::Version::Compare
    my %modes = (
        '>' => sub {
            return if $_[0] eq $_[1];
            return _version_cmp(@_) > 0;
        },
        '<' => sub {
            return if $_[0] eq $_[1];
            return _version_cmp(@_) < 0;
        },
        '==' => sub { return $_[0] eq $_[1] || _version_cmp(@_) == 0; },
        '!=' => sub { return $_[0] ne $_[1] && _version_cmp(@_) != 0; },
        '>=' => sub {
            return 1 if $_[0] eq $_[1];
            return _version_cmp(@_) >= 0;
        },
        '<=' => sub {
            return 1 if $_[0] eq $_[1];
            return _version_cmp(@_) <= 0;
        }
    );
    return if ( !exists $modes{$mode} );
    return $modes{$mode}->( $ver1, $ver2 );
}

sub _timedsaferun {    # Borrowed from WHM 66 Cpanel::SafeRun::Timed and modified
                       # We need to be sure to never return undef, return an empty string instead.
    my ( $timer, $stderr_to_stdout, @PROGA ) = @_;
    return '' if ( substr( $PROGA[0], 0, 1 ) eq '/' && !-x $PROGA[0] );
    $timer = $timer       ? $timer       : 25;       # A timer value of 0 means use the default, currently 25.
    $timer = $OPT_TIMEOUT ? $OPT_TIMEOUT : $timer;

    my $output;
    my $complete = 0;
    my $pid;
    my $fh;                                          # FB-63723: must declare $fh before eval block in order to avoid unwanted implicit waitpid on die
    eval {
        local $SIG{'__DIE__'} = 'DEFAULT';
        local $SIG{'ALRM'}    = sub { $output = ''; print RED ON_BLACK 'Timeout while executing: ' . join( ' ', @PROGA ) . "\n"; die; };
        alarm($timer);
        if ( $pid = open( $fh, '-|' ) ) {            ## no critic (BriefOpen)
            local $/;
            $output = readline($fh);
            close($fh);
        }
        elsif ( defined $pid ) {
            open( STDIN, '<', '/dev/null' );         ## no critic (BriefOpen)
            if ($stderr_to_stdout) {
                open( STDERR, '>&', 'STDOUT' );      ## no critic (BriefOpen)
            }
            else {
                open( STDERR, '>', '/dev/null' );    ## no critic (BriefOpen)
            }
            exec(@PROGA) or exit 1;                  ## no critic (NoExitsFromSubroutines)
        }
        else {
            print RED ON_BLACK 'Error while executing: [ ' . join( ' ', @PROGA ) . ' ]: ' . $! . "\n";
            alarm 0;
            die;
        }
        $complete = 1;
        alarm 0;
    };
    alarm 0;
    if ( !$complete && $pid && $pid > 0 ) {
        kill( 15, $pid );                            #TERM
        sleep(2);                                    # Give the process a chance to die 'nicely'
        kill( 9, $pid );                             #KILL
    }
    return defined $output ? $output : '';
}

sub timed_run {
    my ( $timer, @PROGA ) = @_;
    return _timedsaferun( $timer, 0, @PROGA );
}

sub timed_run_trap_stderr {
    my ( $timer, @PROGA ) = @_;
    return _timedsaferun( $timer, 1, @PROGA );
}

sub get_local_ipaddrs_aref {
    my @local_ipaddrs_list;
    my @output;

    unless ( @output = split /\n/, timed_run( 0, 'ip', 'addr' ) ) {
        @output = split /\n/, timed_run( 0, 'ifconfig', '-a' );
    }
    for my $line (@output) {
        if ( $line =~ m{ (\d+\.\d+\.\d+\.\d+) }xms ) {
            push @local_ipaddrs_list, $1;
        }
    }

    return \@local_ipaddrs_list;
}

sub print_version {
    print BOLD YELLOW ON_BLACK "\tSSP $VERSION\n\n";
}

sub print_tip {
    my @tips = (
        '[FB-86549] (Fixed in 11.42.1.1) cPHulk may report root logins to Pure-FTPd despite no evidence being found',
        '[FB-78617] (By design) sysup always installs bind',
        '[FB-75793] (By design) Proxy subdomains are not created for addon domains',
        '[FB-73369] Can\'t log into SquirrelMail, but Horde and Roundcube work? Check if webmail pass contains "odd" characters',
        '[FB-72801] (By design) File Manager creates new files with 0600 perms, even when saving an existing file as a new one',
        '[FB-72733] (By design) File Manager\'s "Compress" feature has a hard coded timeout due to using cPanel\'s form upload logic',
        '[FB-63530] When setting up a remote MySQL server, that server must have the openssh-clients package installed',
        '[FB-63193] File Manager showing "Out of memory" in cPanel error_log? Try renaming $HOME/$USER/.cpanel/datastore/SYSTEMMIME',
        '[FB-62819] "License File Expired: LTD: 1334782495 NOW: 1246416504 FUT!" likely just means the server clock is wrong',
        '[FB-62054] (By design) The "Dedicated IP" box can only be modified when creating a package - not when editing',
        '[FB-61735] (By design) "/u/l/c/whostmgr/bin/whostmgr2 --updatetweaksettings" destroys custom proxy subdomain records. Use WHM >> Tweak Settings instead.',
        '[FB-59450] (By design) Email quotas cannot exceed 2048MB, but they can be unlimited',
        '[FB-58625] Apache 2.0.x links to the wrong PCRE libs. This can cause preg_match*() errors, and "PCRE is not compiled with UTF-8 support"',
        '[FB-57237] (By design) Per ISO 3166-1, the country code for the UK is GB (not UK). Look for this in WHM >> Generate an SSL Certificate [...]',
        '[FB-50745] (By design) The cPanel UI displays differently (more columns than rows) when changing your locale',
        '[FB-46853] Customer complaining that they can\'t log into cPanel as root? Update FB-46853',
        '[FB-44884] upcp resets Mailman lists\' hostnames. pre/postupcp hooks workaround in ticket 3541643',
        '[FB-42027] "Recently Uploaded Cgi Script Mail" scans and sends email alerts about downloaded files too',
        '[FB-21774] Pure-FTPd is not linked against libwrap. As such, Host Access Control does nothing for it',
        'The cpanel-postgresql* packages are for phpPgAdmin. The postgresql-* packages are for PostgreSQL',
        'For a list of obscure issues, see the RareIssues wiki article',
        '11.35+: Use /scripts/check_cpanel_rpms to fix problems in /usr/local/cpanel/3rdparty/  - not checkperlmodules',
        'php.ini for phpMyAdmin, phpPgAdmin, Horde, and RoundCube can be found in /usr/local/cpanel/3rdparty/etc/',
        'If Dovecot/POP/IMAP dies every day around the same time, the server\'s clock could be skewed. Check /var/log/maillog for "moved backwards"',
        '"Allowed memory size of x bytes exhausted" when uploading a db via phpMyAdmin may be resolved by increasing max_allowed_packet',
        'Need to edit php.ini for Horde, RoundCube, phpMyAdmin, or phpPgAdmin? Edit /u/l/c/3rdparty/etc/php.ini, then run /u/l/c/b/install_php_inis',
        'Seeing "domainadmin" errors (e.g. "domainadmin-domainexistsglobal")? Check the Domainadmin-Errors wiki article',
        'Transfers showing "sshcmdpermissiondeny"? Check for modified openssh-clients package (see ticket 3664533)',
        'Learn how cPanel 11.36+ handles rpms: http://go.cpanel.net/rpmversions',
        'Use "rlog <file>" to see a file\'s revision history, and "co -p1.1 <file>" (for example) to see that revision',
        'Files under revision control: fstab, localdomains, named.conf, passwd, shadow, trueuserowners, httpd.conf, php.ini (system and cPanel)',
        'Imagick install issues on PHP 5.4? You may need to run \'pear config-set preferred_state beta\' (see ticket 3754991)',
        'Need to enable ZTS support for PHP? Try \'--enable-maintainer-zts\' (see ticket 3769493)',
        'WHM\'s "Apache mod_userdir Tweak" can be toggled via /scripts/userdirctl',
        'Issues with MySQL for a single user? Check for /home/${USER}/.my.cnf',
        'Services reported as failing while backups are running? chksrvd may be simply timing out due to excessive disk I/O',
        'Blank page in File Manager\'s HTML Editor and iconv "illegal input sequence" in cPanel error_log? Try windows-1251 encoding (see ticket 4088633)',
        'Older CentOS 5.x and CloudLinux 5.x do not support SNI. See the "SNI" wiki article for more info',
        'domlogs are created 0644 by default. cpanellogd changes permissions on them to 0640 a few minutes later',
        'cPanel >> Error Log only searches "recent" logs in Apache\'s error_log . Showing as blank? Maybe there are no recent errors',
        'Horde showing "server configuration did not allow file to be uploaded"? Check disk/inode usage on /tmp',
        'IMAP/webmail showing no email? The cPanel account may have been over its quota. Try renaming dovecot-uidlist, send account an email (see ticket 4314723)',
        'ClamAV not scanning emails? Check if /var/clamd is missing. This will be reflected in Exim\'s logs as well',
        'Use custom_vhost_template_ap(1|2) in userdata files to make changes for an individual vhost',
        'File Manager upload size limits can be adjusted at WHM >> Tweak Settings >> Max HTTP submission size',
        '/var/cpanel/conf/apache/local can potentially cause issues. See ticket 3915299 for an example',
        'System backups are not uploaded via FTP by default, requires manual config. See http://documentation.cpanel.net/display/1144Docs/System+Backups#SystemBackups-Manualconfigurationmethod',
        '$PATH may differ when executing something via cron rather than the command line. See ticket 4419531',
        '"failed to open scan directory /var/spool/exim/scan/[...]: Too many links" could mean a directory has reached limit of 32,000 files/dirs',
        'If innodb_force_recovery is enabled in the MySQL configuration, this can sometimes prevent mysqldump from working (see ticket 5193581).',
        '"Spawned \'ossec-dbd\' with \'/sbin/service restart ossec-hids\'" is from ASL (Atomic Secured Linux). Have customer contact ASL Support if necessary.',
        'You can run SSP with the --bugreport option to print a pre-filled template for submitting a WHM/cPanel bug report.',
        'The path for the modsec_audit.log changes with Mod Ruid2 or MPM ITK installed to /usr/local/apache/conf/modsec_audit/[user]/YYYYMMDD/YYYYMMDD-HHmm/YYYYMMDD-HHmmSS-[unique-id]',
        'LiteSpeed (lsws) does NOT support the Apache web status page - see: http://www.litespeedtech.com/support/forum/threads/solved-cpanel-after-litespeed-installation-whm-server-status-gives-a-404-error.5536/',
        'You can submit new ideas or bug reports for SSP by emailing ssp-requests(at)cpanel.net',
        'You can format json files for more readability: python -m tool.json < file.json | less',
    );

    my $num = int rand scalar $#tips;
    print BOLD WHITE ON_BLACK "\tDid you know? $tips[$num]" . RESET . "\n\n";
}

sub get_tiers_file {    #TODO: Get rid of this in favor of get_tiers_json_href if it works out.
    return _http_get( Host => 'httpupdate.cpanel.net', Path => '/cpanelsync/TIERS' );
}

sub get_process_pid_href {

    # Tested on CentOS 5 through 7.
    # 'ps' is horrible at providing reliably-parseable output.  This is probably as close as we can get.
    # etimes field doesn't exist until CentOS 7 but can be derived from etime.
    my $field_separator = '#^#';                                                   # Any sequence unlikely to occur in normal ps output.  ps will also pad everything with spaces.
    my $ps_format_opt   = join( $field_separator, qw( %p %P %U %t %n %c %a ) );    # like 'pid#^#ppid#^#user#^#etime#^#nice#^#comm#^#args'
    my %hash            = map {
        my ( $pid, $ppid, $user, $etime, $nice, $comm, $args ) = split /\s*\Q$field_separator\E\s*/, $_;
        $pid  =~ s/^\s+//;
        $args =~ s/\s+$//;
        my ( $sec, $min, $hou, $day ) = reverse split( /[:-]/, $etime );
        $day += 0;
        $hou += 0;
        $min += 0;
        $sec += $day * 86400 + $hou * 3600 + $min * 60;
        $pid => {
            'PPID'   => defined $ppid  ? $ppid  : '',
            'USER'   => defined $user  ? $user  : '',
            'ETIME'  => defined $etime ? $etime : '',
            'NICE'   => defined $nice  ? $nice  : '0',
            'COMM'   => defined $comm  ? $comm  : '',
            'ARGS'   => defined $args  ? $args  : '',
            'ETIMES' => $sec,
        }
    } split /\n/, timed_run( 0, 'ps', '--no-headers', '--width=1000', '-eo', $ps_format_opt );
    return \%hash;
}

sub grep_process_cmd {

    # Matches short (COMM) or long (ARGS) command columns
    my ( $pattern, $user ) = @_;
    my $procs = get_process_pid_href();
    my %result;
    for my $pid ( keys %{$procs} ) {
        next                           if defined $user ? $procs->{$pid}->{'USER'} ne $user : 0;
        $result{$pid} = $procs->{$pid} if grep { /$pattern/ } @{ $procs->{$pid} }{ 'COMM', 'ARGS' };
    }
    return %result;
}

sub exists_process_cmd {
    my ( $pattern, $user ) = @_;
    my %procs = grep_process_cmd( $pattern, $user );
    return scalar keys %procs ? 1 : 0;
}

sub get_lsof_port_href {
    my %hash;

    for ( split /\n/, timed_run( 0, 'lsof', '+c15', '-n', '-P', '-i' ) ) {

        # cmd will be max 15 characaters due to lsof limitation
        # Example from CentOS 6:
        # spamd           1781     root    5u  IPv4  10887      0t0  TCP 127.0.0.1:783 (LISTEN)
        # nc      9468 root    3u  IPv6  84415      0t0  TCP [::1]:25 (LISTEN)
        # Example from an older CentOS 5 system (note empty SIZE column):
        # exim       3066  mailnull    3u     IPv6            2566011                 TCP *:smtp (LISTEN)
        my @lsof = split( /\s+/, $_, 10 );
        if ( defined( $lsof[9] ) && $lsof[9] =~ /LISTEN/ ) {
            splice( @lsof, 6, 1 );    # Drop the SIZE/OFF column which can sometimes be blank and throw everything off
        }
        if ( defined( $lsof[8] ) && $lsof[8] =~ /LISTEN/ ) {    # SIZE/OFF column is blank, or has been dropped
            if ( $lsof[7] =~ /^(.*):(\d+)$/ ) {
                my ( $ip, $port ) = ( $1, $2 );
                push @{ $hash{$port} },
                  {
                    'CMD'   => $lsof[0],
                    'PID'   => $lsof[1],
                    'USER'  => $lsof[2],
                    'IPV'   => $lsof[4],
                    'PROTO' => $lsof[6],
                    'IP'    => $ip
                  };
            }
        }
    }

    return \%hash;
}

sub get_ipcs_href {
    my %hash;

    my $header = 0;

    # For now, all we need is shared memory segment owner and creator-pid, but the data structure is extensible.
    # ipcs -m -p
    #
    #------ Shared Memory Creator/Last-op --------
    #shmid      owner      cpid       lpid
    #2228224    root       992        992
    #2588673    root       1309       1315
    #2195458    root       985        985
    #2621443    root       1309       1315
    for ( split /\n/, timed_run( 0, 'ipcs', '-m', '-p' ) ) {
        if ( $header == 0 ) {
            $header = 1 if m/^ shmid \s+ owner \s+ cpid \s+ lpid \s* $/ix;
            next;
        }
        my @ipcs = split( /\s+/, $_, 5 );
        push @{ $hash{ $ipcs[1] }{mp} }, {    # Key by owner, type 'mp' (-m -p output)
            'shmid' => $ipcs[0],
            'cpid'  => $ipcs[2],
            'lpid'  => $ipcs[3]
        };
    }

    return \%hash;
}

sub get_mysql_conf_href {
    return unless open( my $mycnf_fh, '<', $MYSQL_CONF_FILE );
    my %conf;
    my $section = 'unknown';
    while (<$mycnf_fh>) {
        chomp;
        next if /^(#|$)/;
        if (m{ \A \s* \[([^\]]+)] }x) {
            $section = lc($1);
            $section =~ s/^\s*//g;
            $section =~ s/\s*$//g;
            next;
        }
        if (m{ \A \s* ([^=]+?) \s* = \s* (?:["']?) ([^"']*?) (?:["']?) \s* \Z }x) {
            my $key = lc($1);
            $key =~ tr/_-//d;
            $conf{$section}{$key} = [ $1, $2 ];
            next;
        }
        if (m{ \A \s* ([^\s]+) \s* \Z }x) {
            my $key = lc($1);
            $key =~ tr/_-//d;
            $conf{$section}{$key} = [ $1, 'enabled' ];
        }
    }
    close $mycnf_fh;
    return unless scalar keys(%conf);
    return \%conf;
}

sub get_pureftpd_conf_href {
    my %conf;
    if ( open( my $pureftpdconf_fh, '<', $PURE_FTPD_CONF_FILE ) ) {
        while (<$pureftpdconf_fh>) {
            next if /^(#|$)/;
            if (m{ \A \s* ([^\s]+?) \s+ (.*) \Z }x) {
                my $key = lc($1);
                $conf{$key} = { name => $1, value => $2 };
            }
        }
        close $pureftpdconf_fh;
    }
    return \%conf;
}

sub get_proftpd_conf_href {
    my %conf;
    if ( open( my $proftpdconf_fh, '<', '/etc/proftpd.conf' ) ) {
        while (<$proftpdconf_fh>) {
            next if /^(#|$)/;
            if (m{ \A \s* ([^\s]+?) \s+ (.*) \Z }x) {
                my $key = lc($1);
                $conf{$key} = { name => $1, value => $2 };
            }
        }
        close $proftpdconf_fh;
    }
    return \%conf;
}

sub get_exim_localopts_href {
    my %conf;
    if ( open( my $conf_fh, '<', '/etc/exim.conf.localopts' ) ) {
        local $/ = undef;
        %conf = map { ( split( /=/, $_, 2 ) )[ 0, 1 ] } split( /\n/, readline($conf_fh) );
        close $conf_fh;
    }
    return \%conf;
}

sub get_hostinfo_href {
    my $info = {
        'environment'       => get_environment(),
        'hardware'          => timed_run( 0, 'uname', '-i' ),
        'kernel'            => timed_run( 0, 'uname', '-r' ),
        'installtime'       => undef,
        'installtime_epoch' => undef,
    };
    chomp @$info{qw(hardware kernel)};

    my $rpm_name = 'basesystem';
    my $rpms;
    if ( $rpms = get_rpm_href() and exists $rpms->{$rpm_name}->[0]->{'installtime'} ) {
        $info->{'installtime_epoch'} = $rpms->{$rpm_name}->[0]->{'installtime'};
        $info->{'installtime'}       = scalar localtime( $info->{'installtime_epoch'} );
    }

    return $info;
}

sub get_environment {
    my $envtype;
    if ( open my $envtype_fh, '<', '/var/cpanel/envtype' ) {
        $envtype = readline($envtype_fh);
        close $envtype_fh;
    }
    else {
        $envtype = timed_run( 0, '/usr/local/cpanel/bin/envtype' );
    }
    chomp $envtype if $envtype;
    if ( !$envtype ) {
        return 'unknown-envtype';
    }
    return $envtype;
}

sub get_cpuinfo_href {
    my %cpuinfos;

    open my $cpuinfo_fh, '<', '/proc/cpuinfo';
    for my $line ( readline $cpuinfo_fh ) {
        if ( $line =~ /^model name/m ) {
            $line =~ s/^model name\s+:\s+//;
            $line =~ s/\(R\)//g;
            $line =~ s/\(tm\)//g;
            $line =~ s/\s{2,}/ /;
            $line =~ s/\s*\@/ \@/;
            $cpuinfos{'model'} = $line;
            $cpuinfos{'numcores'}++;
        }
        if ( $line =~ /^cpu MHz/m ) {
            $line =~ s/^cpu MHz\s+:\s+//;
            $cpuinfos{'mhz'} = $line;
        }
    }
    close $cpuinfo_fh;

    chomp %cpuinfos;
    return \%cpuinfos;
}

sub get_meminfo {

    # General logic from WHM 56 Cpanel::Sys::Hardware::Memory
    my $proc_meminfo      = '/proc/meminfo';
    my $proc_beancounters = '/proc/user_beancounters';
    my %meminfo;
    my $hostinfo = get_hostinfo_href();

    if ( defined( $hostinfo->{'environment'} ) && $hostinfo->{'environment'} eq 'virtuozzo' ) {

        # https://wiki.openvz.org/UBC_primary_parameters#vmguarpages
        # https://wiki.openvz.org/UBC_secondary_parameters#privvmpages
        if ( open( my $proc_beancounters_fh, '<', $proc_beancounters ) ) {
            while (<$proc_beancounters_fh>) {
                if (m/^\s*(\S+)\s+(.*)/) {
                    my $type = $1;
                    my $parm = $2;
                    chomp($parm);
                    my ( $held, $maxheld, $barrier, $limit, $failcnt ) = split( /\s+/, $parm );
                    next if $held eq '-';

                    # NOTE: VZ uses the # of 4-KiB pages, convert to KiB.
                    # installed value is the lowest of privvmpages, physpages, or vmguarpages barrier (ignoring 0)
                    if ( $type =~ /^(privvmpages|physpages|vmguarpages)$/ ) {
                        unless ( $barrier eq "0" || ( defined( $meminfo{'installed'} ) && $meminfo{'installed'} <= ( $barrier * 4 ) ) ) {
                            $meminfo{'installed'} = $barrier * 4;
                        }
                    }
                    elsif ( $type eq 'oomguarpages' ) {
                        $meminfo{'used'} = $held * 4;
                    }
                    elsif ( $type eq 'swappages' ) {
                        $meminfo{'swapinstalled'} = $limit * 4;
                    }
                }
            }
            close($proc_beancounters_fh);
            $meminfo{'available'} = $meminfo{'installed'} - $meminfo{'used'};
        }
    }
    elsif ( open my $proc_meminfo_fh, '<', $proc_meminfo ) {
        while (<$proc_meminfo_fh>) {
            if (/^\s*([^\:]+):\s+(\d+)/) {
                $meminfo{ lc($1) } = $2;
            }
        }
        close $proc_meminfo_fh;
        $meminfo{'available'}     = $meminfo{'memfree'} + $meminfo{'buffers'} + $meminfo{'cached'};
        $meminfo{'installed'}     = $meminfo{'memtotal'};
        $meminfo{'used'}          = sprintf( '%u', $meminfo{'memtotal'} - $meminfo{'memfree'} );
        $meminfo{'swapinstalled'} = $meminfo{'swaptotal'};
    }

    chomp %meminfo;
    return \%meminfo;
}

sub get_cpupdate_conf {
    my ($conf) = @_;
    $conf = defined($conf) ? $conf : '/etc/cpupdate.conf';
    my %conf;
    if ( open( my $conf_fh, '<', $conf ) ) {
        local $/ = undef;
        %conf = map { ( split( /=/, $_, 2 ) )[ 0, 1 ] } split( /\n/, readline($conf_fh) );
        close $conf_fh;
    }
    return \%conf;
}

sub format_meminfo {
    my ($num) = @_;
    return 'none or unknown' if ( !defined($num) );
    my $hostinfo = get_hostinfo_href();

    # The original values are 9223372036854775807 and 2147483647 4-KiB pages
    if ( defined( $hostinfo->{'environment'} ) && $hostinfo->{'environment'} eq 'virtuozzo' ) {
        return $num = 'unlimited' if $num == '36893488147419103228' + 0;    # KiB
        if ( defined( $hostinfo->{'hardware'} ) && $hostinfo->{'hardware'} eq 'i386' ) {
            return $num = 'unlimited' if $num == '8589934588' + 0;          # KiB
        }
    }
    return int( $num / 1024 ) . "MB";
}

sub get_whm_install_info {
    my $info = {
        'installtime'          => undef,
        'installtime_epoch'    => undef,
        'installversion'       => undef,
        'lastupdatetime'       => undef,
        'lastupdatetime_epoch' => undef,
    };

    my $install_log = '/var/log/cpanel-install.log';

    if ( -f $install_log ) {
        my $birthday_mtime = ( stat($install_log) )[9];
        $info->{'installtime'}       = scalar localtime($birthday_mtime);
        $info->{'installtime_epoch'} = $birthday_mtime;

        if ( open( my $conf_fh, '<', $install_log ) ) {
            while ( readline($conf_fh) ) {
                if (m{Target \s version \s set \s to \s '(\d+\.\d+[^']+)'}xms) {
                    $info->{'installversion'} = $1;
                    if (m{(\d+-\d+-\d+ \s+ \d+:\d+:\d+ (?:\s \-\d+)?)}xms) {
                        $info->{'installtime'} = $1;
                    }
                    last;
                }
                last if $. >= 20_000;
            }
            close $conf_fh;
        }
    }

    if ( my $update_mtime = ( stat($CPANEL_VERSION_FILE) )[9] ) {
        $info->{'lastupdatetime'}       = scalar localtime($update_mtime);
        $info->{'lastupdatetime_epoch'} = $update_mtime;
    }

    return $info;
}

sub print_info {
    my $text = shift;
    print BOLD YELLOW ON_BLACK "[INFO] * $text";
}

sub print_warn {
    my $text = shift;
    print BOLD RED ON_BLACK "[WARN] * $text";
}

sub print_crit {
    my $text = shift;
    $CRIT_BUFFER .= BOLD MAGENTA ON_BLACK '[CRIT] * ' . $text;
    print BOLD MAGENTA ON_BLACK '[CRIT] * ' . $text;
}

sub print_critical {
    my $text = shift;
    $text = $text ? $text : "";
    $CRIT_BUFFER .= BOLD MAGENTA ON_BLACK $text . "\n";
    print BOLD MAGENTA ON_BLACK $text . "\n";
}

sub print_3rdp {
    my $text = shift;
    print BOLD GREEN ON_BLACK "[3RDP] * $text";
}

sub print_3rdp2 {
    my $text = shift;
    print BOLD GREEN ON_BLACK "$text\n";
}

## precedes informational items (e.g., "Hostname:")
sub print_start {
    my $text = shift;
    print BOLD YELLOW ON_BLACK $text;
}
## for informational items (e.g., the server's hostname)
sub print_normal {
    my $text = shift;
    print BOLD CYAN ON_BLACK "$text\n";
}
## for important things (e.g., "Hostname is not a FQDN")
sub print_warning {
    my $text = shift;
    print BOLD RED ON_BLACK "$text\n";
}
## for other imporant things (e.g., "You are in an LVE, do not restart services")
sub print_warning_underline {
    my $text = shift;
    print BOLD UNDERLINE "$text\n";
}

sub print_info2 {
    my $text = shift;
    print BOLD GREEN ON_BLACK "$text\n";
}

sub print_magenta {
    my $text = shift;
    print BOLD MAGENTA ON_BLACK "$text\n";
}

sub print_red {
    my $text = shift;
    print BOLD RED ON_BLACK "$text\n";
}

sub check_for_hacked_server_touchfile {
    return unless i_am('cptech');
    my $docdir = '/usr/share/doc';
    return unless -d $docdir;

    opendir( my $fh, $docdir ) or return;

    # .cp.jeff.2014-04-09_10.5.40.209_1234567
    my @touchfiles = grep { /^\.cp\.([^\d]+)\.(\d{4}-\d{2}-\d{2})_([^_]+)_(\d+)$/ } readdir $fh;
    closedir $fh;

    return if scalar @touchfiles == 0;

    print_generic_hack_predef('HACKED SERVER');
    for my $touchfile (@touchfiles) {
        if ( $touchfile =~ /^\.cp\.([^\d]+)\.(\d{4}-\d{2}-\d{2})_([^_]+)_(\d+)$/ ) {
            my ( $cptech, $date, $ipaddr, $ticket ) = ( $1, $2, $3, $4 );
            $date =~ s#-#/#g;
            $cptech = ucfirst $cptech;
            print_critical("\tL3: $cptech reported this server at $ipaddr as compromised on $date local server time in ticket $ticket");
            if ( !grep { /^$ipaddr$/ } @{ get_local_ipaddrs_aref() } ) {
                print_critical("\t \\_ NOTE: IP address $ipaddr not found on the server!");
            }
        }
    }
    print_critical();
}

sub check_for_multiple_tech_logins {
    return unless i_am('cptech');

    # Prefer 'who' over 'w' because of FROM field length limit in 'w'
    # who -H
    #NAME     LINE         TIME             COMMENT
    #root     pts/0        2014-07-29 07:24 (192.168.130.1)

    # we can sometimes get additional text after the IP or hostname
    #root     pts/2        2014-08-07 07:17 (208.74.121.102:S.0)

    my $who = '/usr/bin/who';
    return if !-x $who;

    my @tech_logins = ();
    my $header      = "";
    my $num_logins  = 0;

    for my $line ( split /\n/, timed_run( 0, $who, '-H' ) ) {
        if ( $line =~ m{ \A NAME\s+ }xms ) {
            $header = $line;
            next;
        }
        if ( $line =~ m{ \((.+)\)\Z }xms ) {
            if (   $1 =~ m{ \A (.*\.)?(cptxoffice\.net|cloudlinux\.com|litespeedtech.com)(:|$) }xms
                || $1 =~ m{ \A (208\.74\.123\.98|184\.94\.197\.[2-6])(:|$) }xms ) {
                push( @tech_logins, $line );
                $num_logins++;
            }
        }
    }

    return if $num_logins <= 1;

    print_critical();
    print_crit('Multiple tech SSH sessions are active (run "ls /var/cpanel/users/ |grep cptkt" for a complete list of ticket users):');
    print_critical("\n");
    print_critical($header) if $header;
    print_critical( join( "\n", @tech_logins ) );
    print_critical();
}

sub check_for_lve_environment {
    my $hostinfo = get_hostinfo_href();

    # pam_lve 0.2 prints this after su or sudo:
    #
    # # /bin/su -
    # Password:
    # ***************************************************************************
    # *                                                                         *
    # *             !!!!  WARNING: YOU ARE INSIDE LVE !!!!                      *
    # *IF YOU RESTART ANY SERVICES STABILITY OF YOUR SYSTEM WILL BE COMPROMIZED *
    # *        CHANGE UID OF THE USER YOU ARE USING TO SU/SUDO                  *
    # *                             MORE INFO:                                  *
    # *http://www.cloudlinux.com/blog/clnews/read-this-if-you-use-su-or-sudo.php*
    # *                                                                         *
    # ***************************************************************************

    # pam_lve 0.3 won't put wheel users in an LVE after su or sudo:
    # http://cloudlinux.com/blog/clnews/read-this-if-you-use-su-or-sudo.php

    if ( $hostinfo->{'kernel'} =~ /\.lve/ and -x '/usr/sbin/lveps' ) {
        if (`/usr/sbin/lveps -p | grep " $$ "`) {    ## no critic (Cpanel::ProhibitQxAndBackticks)
            print_critical();
            print_crit(" You are inside a CloudLinux LVE - DO *NOT* RESTART ANY SERVICES!\n");
            print_critical("          \\_ The pam_lve configuration may not be excluding the wheel group, or your ssh login user was not in the wheel group.");
            print_critical("          \\_ http://docs.cloudlinux.com/index.html?lve_pam_module.html");
            print_critical();
        }
    }
}

sub get_lsws_version_aref {
    my $lshttpd = '/usr/local/lsws/bin/lshttpd';
    return [] unless my @lshttpd_version_output = split /\n/, timed_run( 0, $lshttpd, '-v' );
    my ( $lsws_full_version, $lsws_numeric_version ) = ();
    for (@lshttpd_version_output) {
        if (m{ \A (LiteSpeed/(\d+(?:\.\d+){1,2}).*) }xms) {
            $lsws_full_version    = $1;
            $lsws_numeric_version = $2;
        }
    }

    $lsws_full_version    = "unknown" if !$lsws_full_version;
    $lsws_numeric_version = "unknown" if !$lsws_numeric_version;

    return [ $lsws_full_version, $lsws_numeric_version ];
}

sub check_for_systemd {
    return unless ( -e '/usr/bin/systemctl' or -e '/bin/systemctl' ) and ( -e '/usr/lib/systemd/systemd' or -e '/lib/systemd/systemd' );    # Don't assume /bin or /lib symlinks to /usr are in place
    print_info('Systemd: ');
    print_normal('Use /scripts/restartsrv_* (preferred) or systemctl to restart services -- never use /etc/init.d scripts.');
}

sub check_for_os_release_5 {
    return unless os_version_is(qw( < 6 ));
    print_crit('CentOS/RHEL/CL 5 (or older): ');
    print_critical('This operating system is not supported in WHM 58 and later (OS version 6+ only).');
    print_critical('         \_ Send customer this premade: "MIGRATION - CentOS/RHEL/CL 5 EOL"');
}

sub check_for_os_release_32bit {
    my $hostinfo = get_hostinfo_href();
    return unless ( defined( $hostinfo->{'hardware'} ) && $hostinfo->{'hardware'} eq 'i386' );
    return unless os_version_is(qw( >= 6 ));                                                     # There is an unofficial CentOS 7 i386 build.
    print_crit('CentOS/RHEL/CL i386 (32-bit): ');
    print_critical('This operating system is not supported in WHM 58 and later (x86_64 only).');
    print_critical('         \_ Send customer this premade: "MIGRATION - 32-bit CentOS/RHEL/CL EOL"');
}

sub check_for_ea3 {
    return unless i_am('ea3');
    my $hostinfo = get_hostinfo_href();
    return if ( defined( $hostinfo->{'hardware'} ) && $hostinfo->{'hardware'} eq 'i386' );       # Do not report on 32-bit systems.
    print_crit('EasyApache 3: ');
    print_critical('Support is ending in 2018.');
    print_critical('         \_ Send customer this premade: "MIGRATION - EA3 EOL"');
}

##############################
#  BEGIN [INFO] CHECKS
##############################

sub print_hostname {
    my $hostname            = get_hostname();
    my $hostname_resolves   = check_hostname_resolution($hostname);
    my $external_ip_address = get_external_ip();
    print_info('Hostname: ');
    print_normal($hostname);
    print_warning(qq{\t\\_ too long. More than 60 characters will cause issues with MySQL databases})        if ( length($hostname) > 60 );
    print_warning(qq{\t\\_ may not be a FQDN ( https://support.cpanel.net/hc/en-us/articles/360044460594 )}) if ( $hostname !~ /([\w-]+)\.([\w-]+)\.(\w+)/ );
    print_warning(qq{\t\\_ $hostname does not resolve to licensed IP address [$external_ip_address]})        if ( !$hostname_resolves );
}

sub print_os {
    return unless my $hostinfo = get_hostinfo_href();
    my $install_info = '';
    if ( defined $hostinfo->{'installtime'} ) {
        $install_info = ' [ Installed: ' . $hostinfo->{'installtime'} . ' ]';
    }
    print_info('OS: ');
    print_normal( _get_run_var('os_release') . ' [ ' . $hostinfo->{'environment'} . ' ]' . $install_info );
}

sub print_kernel_and_cpu {
    return unless my $hostinfo = get_hostinfo_href();
    return unless my $cpuinfo  = get_cpuinfo_href();
    print_info('Kernel/CPU: ');
    print_normal("$hostinfo->{'kernel'} $hostinfo->{'hardware'} $hostinfo->{'environment'} $cpuinfo->{'model'} w/ $cpuinfo->{'numcores'} core(s)");
    if ( missing_open_opath_flag() ) {
        print_warning("\t \\_ This kernel does not support O_PATH flag with open() which can cause service failure with systemd, see UPS-177");
    }
    if ( $hostinfo->{'environment'} eq 'virtuozzo' && $hostinfo->{'kernel'} eq '2.6.32-042stab113.11' ) {
        print_warning("\t \\_ This kernel has broken quota support [ https://bugs.openvz.org/browse/OVZ-6661 ]");
    }
}

sub print_kernelcare_info {
    return unless i_am('kernelcare');
    my $kcarectl_path = '/usr/bin/kcarectl';
    my $kcarectl_info = "Installed";
    my $license_output;
    my $uname_output;
    if ( -x $kcarectl_path ) {
        chomp( $license_output = timed_run( 0, $kcarectl_path, '--license-info' ) );
        if ( $license_output =~ /Valid license found/ ) {
            $kcarectl_info .= ' and licensed';
        }
        else {
            $kcarectl_info .= ' (license not detected)';
        }
        chomp( $uname_output = timed_run( 0, $kcarectl_path, '--uname' ) );
        if ( ( $uname_output =~ /^\d+\.\d+\.\d+/ ) && ( $uname_output !~ /\n/ ) ) {
            $kcarectl_info .= ' [ ' . $uname_output . ' ]';
        }
    }
    print_info('KernelCare: ');
    print_normal($kcarectl_info);
}

sub print_cpanel_info {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    my $cpupdate_conf = get_cpupdate_conf();
    my $whm_info      = get_whm_install_info();

    my $get_current_profile_href = get_node_info_href();
    my $profile_id  = exists $get_current_profile_href->{'data'}->{'code'} ? $get_current_profile_href->{'data'}->{'code'}                            : 'UNKNOWN';
    my $cpanel_tier = defined $cpupdate_conf->{CPANEL}                     ? $cpupdate_conf->{CPANEL}                                                 : 'Unknown (could not open/read /etc/cpupdate.conf ?)';
    my $last_update = defined $whm_info->{'lastupdatetime_epoch'}          ? sprintf '%.1f', ( time() - $whm_info->{'lastupdatetime_epoch'} ) / 86400 : 'UNKNOWN';
    my $birthday    = defined $whm_info->{'installversion'}                ? $whm_info->{'installversion'} . ' on '                                   : '';
    $birthday .= defined $whm_info->{'installtime'} ? $whm_info->{'installtime'} : '';

    my $output = _get_run_var('cpanel_original_version') . ' (' . uc($cpanel_tier) . ' tier)' . " ($profile_id profile)" . " Last update: $last_update days ago";
    $output .= " [ Installed: $birthday ]" if length $birthday;

    print_info('cPanel Info: ');
    print_normal($output);

    # Check for expired version
    my ( $parent_ver, $major_ver ) = split( /\./, _get_run_var('cpanel_numeric_version'), 3 );
    my $expire_info;
    if ( defined $parent_ver and defined $major_ver ) {
        $major_ver++ if $major_ver % 2;                      # Bump odd dev versions
        $expire_info = 'has expired.' if $major_ver < 62;    # TIERS.json file only contains 62+
        return unless my $version_href = get_tier_info_for_version_href( $parent_ver . '.' . $major_ver );
        $expire_info = 'has expired (version is not a named or LTS tier in TIERS.json)' if ( !keys %{$version_href} || ( !exists $version_href->{'named'} && !exists $version_href->{'is_lts'} ) );
        if ( exists $version_href->{'expires'} && $version_href->{'expires'} =~ /\A \d+ \Z/xms ) {
            my @months = ( 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec' );
            if ( $version_href->{'expires'} <= time() ) {
                my ( $day, $month, $year ) = ( localtime( $version_href->{'expires'} ) )[ 3, 4, 5 ];
                $expire_info = 'ended on ' . $months[$month] . " " . $day . ", " . ( $year + 1900 );
            }
        }
    }
    if ($expire_info) {
        print_crit('cPanel Info: ');
        print_critical( "Support for this version of WHM/cPanel " . $expire_info );
        print_critical('         \_ Send customer this premade: "EOL version of cPanel"');
        print_critical('         \_ Some SSP output may be irrelevant, incomplete, or inaccurate for EOL versions!');
    }
}

sub get_node_info_href {
    return unless i_am('cpanel');
    return unless cpanel_version_is(qw( >= 11.75.0.0 ));    # call was introduced in version 76

    return unless my $raw = timed_run( 0, '/usr/local/cpanel/bin/whmapi1', 'get_current_profile', '--output=json' );
    my $json_output = get_json_href($raw);
    return $json_output;
}

sub get_json_href {
    my ( $raw, $fail_warning ) = @_;

    return unless defined $raw;
    my $json = load_module_with_fallbacks(
        'needed_subs'  => [qw{new utf8 decode}],
        'modules'      => [qw{Cpanel::JSON::XS JSON::XS JSON::PP}],
        'fail_warning' => $fail_warning,
    );

    return {} unless $json;    # since this expects a href back, we should return an empty set in the event that the json module cannot be loaded

    my $href;
    local $@;
    eval { $href = $json->new->utf8->decode($raw); };    # All or nothing, just be quiet.

    return {} if !$href;                                 # since this expects a href back, we should return an empty set if the raw data passed in was not valid json
    return $href;
}

sub check_for_cpanel_update {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    return unless my $cpupdate_conf = get_cpupdate_conf();
    return unless defined $cpupdate_conf->{CPANEL};
    my ( $available_tier_version, $local_tier_name );
    my $match = 0;

    if ( _get_run_var('cpanel_numeric_version') eq 'UNKNOWN' ) {
        print_info('cPanel update check: ');
        print_warning("unknown or old cPanel version, check $CPANEL_VERSION_FILE");
        return;
    }

    my $tiers = get_tiers_file();
    return unless $tiers;
    my @tiers = split /\n/, $tiers;

    for my $line (@tiers) {
        if ( $line =~ m{ \A (.*) : (\d+\.\d+\.\d+\.\d+) \z }xms ) {
            my $tier = $1;
            $available_tier_version = $2;
            if ( $tier =~ /^$cpupdate_conf->{CPANEL}$/i ) {
                $match = 1;
                last;
            }
        }
    }

    if ( $match == 0 ) {
        print_info('cPanel update check: ');
        print_warning("server is configured to use an unknown tier ($cpupdate_conf->{CPANEL})");
        return;
    }

    if ( cpanel_version_is( '<', $available_tier_version ) ) {
        print_info('cPanel update check: ');
        print_warning( "UPDATE AVAILABLE (" . _get_run_var('cpanel_original_version') . " -> $available_tier_version)" );
    }
}

sub check_lts_autofixer {
    my $file = '/etc/cpupdate.conf.update_config_prefs';
    return unless -f $file;
    my $cpupdate_conf = get_cpupdate_conf($file);
    my $version       = defined( $cpupdate_conf->{'CPANEL'} ) ? $cpupdate_conf->{'CPANEL'} : 'UNKNOWN';
    print_info('LTS Autofixer: ');
    print_warning("updated this server from tier \"$version\", see TECH-848");
}

sub check_perl_version_less_than_588 {
    my $perl_version = $^V;
    if ( $perl_version =~ /^v(.+)$/ ) {
        $perl_version = $1;
    }

    return if !$perl_version;

    if ( version_compare( $perl_version, qw( < 5.8.8 ) ) ) {
        print_warn('Perl Version: ');
        print_warning("less than 5.8.8: [ $perl_version ]");
    }
    if ( version_compare( $perl_version, qw( < 5.14.0 ) ) ) {
        print_warn('Perl Version: ');
        print_warning('better resolver results can be obtained when running SSP with Perl 5.14 or later');
    }
}

sub print_uptime {
    my $uptime = timed_run( 0, 'uptime' );
    chomp $uptime if $uptime;
    $uptime = $uptime ? $uptime : 'UNKNOWN';
    print_info('Uptime: ');
    print_normal($uptime);
}

sub check_for_clustering {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    return unless -e '/var/cpanel/useclusteringdns';
    print_info('DNS Clustering: ');
    print_normal('is enabled');

    my $cluster_dir = '/var/cpanel/cluster/root/config';
    my @dir_contents;
    my @cluster_members;

    if ( -d $cluster_dir ) {
        opendir( my $dir_fh, $cluster_dir );
        @dir_contents = grep { !/^\.\.?$/ } readdir $dir_fh;
        closedir $dir_fh;
    }

    chdir $cluster_dir or return;

    for my $dirent (@dir_contents) {
        my ( $cluster_member, $cluster_member_hostname, $cluster_member_role );
        my %cluster_conf;

        # only active cluster members have -dnsrole files
        if ( $dirent =~ m{ \A (.+)-dnsrole \z }xms ) {
            $cluster_member = $1;

            if ( open my $file_fh, '<', $cluster_member ) {
                local $/;
                %cluster_conf = map { ( split( /=/, $_, 2 ) )[ 0, 1 ] } split( /\n/, readline($file_fh) );
                close $file_fh;
            }
            $cluster_member_hostname = defined $cluster_conf{host} ? $cluster_conf{host} : '?';
            if ( $cluster_member =~ m{ \A (vps\.net|softlayer) \z}xmsi ) {
                $cluster_member_hostname = '';
            }

            if ( open my $file_fh, '<', "${cluster_member}-dnsrole" ) {
                while (<$file_fh>) {
                    $cluster_member_role = $_;
                    chomp $cluster_member_role;
                }
                close $file_fh;
            }
            $cluster_member_role = defined $cluster_member_role ? $cluster_member_role : '?';

            push @cluster_members, $cluster_member_hostname . ' ' . $cluster_member . ' ' . "[" . $cluster_member_role . "]";
        }
    }

    return unless @cluster_members;
    @cluster_members = sort @cluster_members;

    for my $member (@cluster_members) {
        print_magenta( "\t \\_ " . $member );
    }
}

sub print_apache_info {
    return unless i_am_one_of( 'cpanel', 'ea4', 'ea3' );
    my $apache_version = get_apache_version_href();
    my $output;

    $output .= "[ EA4 ] " if i_am('ea4');

    if ( not defined $apache_version->{'version'} or not defined $apache_version->{'built'} or not defined $apache_version->{'ea_version'} ) {
        $output .= 'could not determine Apache info!';
    }
    else {
        $output .= "[ $apache_version->{'version'} ] [ $apache_version->{'built'} w/ $apache_version->{'ea_version'} ]";
    }

    my ( $apache_uptime, $apache_generations );

    my $apache_configured_port = 80;
    my $attempted_port         = ( split( ':', $CPCONF{'apache_port'} ) )[1];
    if ($attempted_port) {
        $apache_configured_port = $attempted_port;
    }

    my $apache_status = _http_get( Host => '127.0.0.1', Port => $apache_configured_port, Path => '/whm-server-status', MultiHomed => 0, Timeout => 5 );

    if ( not $OPT_SKIP_NETWORKING ) {
        if ($apache_status) {
            my @apache_status = split /\n/, $apache_status;

            for my $line (@apache_status) {
                if ( $line =~ m{ Server \s uptime: \s+ (.*) </dt> }xms ) {
                    $apache_uptime = 'Up ' . $1;
                }
                if ( $line =~ m{ Parent \s Server \s Generation: (.*) </dt> }xms ) {
                    $apache_generations = $1 . ' generation(s)';
                }
            }
            $output .= ' [ ' . $apache_uptime . ' ]'      if defined $apache_uptime;
            $output .= ' [ ' . $apache_generations . ' ]' if defined $apache_generations;
        }
        else {
            my $warning = "";
            if ( $apache_configured_port == 80 ) {
                $warning = 'Is Apache up/slow to respond? (failed: http://127.0.0.1/whm-server-status). ';
            }
            else {
                $warning = 'Is Apache up/slow to respond? (failed: http://127.0.0.1:' . $apache_configured_port . '/whm-server-status). ';
            }
            my $ports = get_lsof_port_href();
            if ( exists $ports->{'80'} ) {
                $warning .= 'Something is listening on port 80.';
            }
            else {
                $warning .= 'Nothing is listening on port 80';
            }

            print_info('Apache: ');
            print_warning($warning);
        }
    }

    if ($output) {
        print_info('Apache: ');
        print_normal($output);
    }

    my %apache_ports;
    my %root_httpd;
    my $ports = get_lsof_port_href();
    my $procs = get_process_pid_href();
    while ( my ( $portnum, $aref ) = each(%$ports) ) {
        for my $href (@$aref) {
            next unless $href->{USER} eq "root";
            next unless $href->{CMD} eq "httpd";
            my $pid = $href->{PID};
            if ( defined $procs->{$pid} and $procs->{$pid}->{ETIMES} > 60 ) {
                next if $procs->{$pid}->{ARGS} =~ m{ \A /apache/bin/httpd }xms;    # Ignore these - see TECH-334
                $root_httpd{$pid} = 1;
            }
            $apache_ports{$portnum} = 1;
        }
    }

    if ( scalar keys(%apache_ports) ) {
        print_info('Apache: ');
        print_normal( 'is listening on ports [ ' . join( " ", sort( keys(%apache_ports) ) ) . ' ]' );
    }

    if ( scalar keys(%root_httpd) > 1 ) {
        my $pids = scalar keys(%root_httpd) > 4 ? 'More than 4!' : join( ' ', sort( keys(%root_httpd) ) );
        print_warn('Apache: ');
        print_warning( 'multiple root httpd processes (more than 60 seconds old) found [ ' . $pids . ' ] -- See TECH-314.' );
    }
}

sub get_ea3_php_conf_href {
    return unless i_am('ea3');
    my $phpconf = '/usr/local/apache/conf/php.conf.yaml';
    my %conf;
    if ( open( my $phpconf_fh, '<', $phpconf ) ) {
        while (<$phpconf_fh>) {
            chomp;
            if (/^phpversion: (\d)/) {
                $conf{'phpversion'} = $1;
            }
            if (/^php4:[ \t]+['"]?([^'"]+)/) {
                $conf{'php4handler'} = $1;
            }
            if (/^php5:[ \t]+['"]?([^'"]+)/) {
                $conf{'php5handler'} = $1;
            }
            if (/^suexec:[ \t]+['"]?([^'"]+)/) {
                $conf{'suexec'} = $1;
            }
        }
        close $phpconf_fh;
    }
    else {
        $conf{'php_conf_yaml_missing'} = 1;
    }
    my @php_5_v = split /\n/, timed_run( 0, '/usr/bin/php', '-n', '-v' );
    if ( @php_5_v && $php_5_v[0] =~ /^PHP\s(\S+)\s(\S+)/ ) {
        $conf{'php5version'}       = $1;
        $conf{'php5version_valid'} = 1;
    }
    else {
        $conf{'php5version'}       = '(version unknown)';
        $conf{'php5version_valid'} = 0;
    }
    my @php_4_v = split /\n/, timed_run( 0, '/usr/local/php4/bin/php', '-v' );
    if ( @php_4_v && $php_4_v[0] =~ /^PHP\s(\S+)\s(\S+)/ ) {
        $conf{'php4version'}       = $1;
        $conf{'php4version_valid'} = 1;
    }
    else {
        $conf{'php4version'}       = '(version unknown)';
        $conf{'php4version_valid'} = 0;
    }
    return \%conf;
}

sub print_ea3_php_configuration {
    return unless i_am('ea3');
    my $conf = get_ea3_php_conf_href();
    unless ( defined $conf && !exists $conf->{'php_conf_yaml_missing'} && exists $conf->{'phpversion'} ) {
        print_info('PHP: ');
        print_warning('/usr/local/apache/conf/php.conf.yaml missing or incomplete.  Some PHP checks may be skipped.');
    }

    my $has_ea3_suexec = $conf->{'suexec'} ? 'with suexec' : 'without suexec';

    if ( defined $conf->{'phpversion'} and $conf->{'phpversion'} == 5 ) {
        if ( defined $conf->{'php5version'} and defined $conf->{'php5handler'} ) {
            print_info('PHP Default: ');
            print_normal("PHP $conf->{'php5version'} $conf->{'php5handler'} $has_ea3_suexec");
        }
        if ( defined $conf->{'php4version'} and defined $conf->{'php4handler'} and $conf->{'php4handler'} ne 'none' ) {
            print_info('PHP Secondary: ');
            print_normal("PHP $conf->{'php4version'} $conf->{'php4handler'} $has_ea3_suexec");
        }
    }

    if ( defined $conf->{'phpversion'} and $conf->{'phpversion'} == 4 ) {
        if ( defined $conf->{'php4version'} and defined $conf->{'php4handler'} ) {
            if ( $conf->{'php4handler'} eq 'fcgi' ) {
                print_info('PHP Default: ');
                print_warning("PHP $conf->{'php4version'} $conf->{'php4handler'} $has_ea3_suexec (mod_userdir style URLs don't work with fcgi!)");
            }
            else {
                print_info('PHP Default: ');
                print_normal("PHP $conf->{'php4version'} $conf->{'php4handler'} $has_ea3_suexec");
            }
        }
        if ( defined $conf->{'php5version'} and defined $conf->{'php5handler'} and $conf->{'php5handler'} ne 'none' ) {
            print_info('PHP Secondary: ');
            print_normal("PHP $conf->{'php5version'} $conf->{'php5handler'} $has_ea3_suexec");
        }
    }
}

sub print_ea4_php_configuration {
    return unless i_am('ea4');
    my $info            = 'UNKNOWN';
    my $fpm_jail_toggle = '/var/cpanel/feature_toggles/apachefpmjail';
    my $ea4_php         = get_installed_ea4_php_href();
    my $modules         = get_apache_modules_href();
    print_info('PHP Default: ');
    if ( defined($ea4_php) && defined( $ea4_php->{default} ) && defined( $ea4_php->{ $ea4_php->{default} }->{release_version} ) && defined( $ea4_php->{ $ea4_php->{default} }->{handler} ) ) {
        $info = '[ EA4 ]';
        $info .= " [ $ea4_php->{ $ea4_php->{default} }->{release_version} ( $ea4_php->{default} ) ]";
        $info .= " [ $ea4_php->{ $ea4_php->{default} }->{handler} ]";
    }
    print_normal($info);
    if ( -e $fpm_jail_toggle ) {
        print_info('PHP-FPM: ');
        print_normal( $fpm_jail_toggle . ' exists, PHP-FPM will jail PHP scripts for users that have Jailed or Disabled shells.' );
        if ( defined $modules and not( defined $modules->{'ruid2_module'} and defined $CPCONF{'jailapache'} and $CPCONF{'jailapache'} == 1 ) ) {
            print_warn('PHP-FPM: ');
            print_warning('Jail is enabled without mod_ruid2 and/or Jail Apache Virtual Hosts tweak setting enabled, these MUST also be enabled for proper functioning unless EA-5524 is resolved.');
        }
    }
}

sub get_installed_ea4_php_href {

    # Only supports WHM 54+
    return unless i_am('ea4');
    my $php = {};
    my @available_php;
    my @current_php;
    my ( $available_php, $current_php );
    (@current_php) = split( /\n/, timed_run( 0, '/usr/local/cpanel/bin/rebuild_phpconf', '--current' ) );
    foreach my $line (@current_php) {
        my $pkg;
        if ( $line =~ m{ DEFAULT \s PHP: \s (\S+) }xms ) {
            $pkg                        = $1;
            $php->{$pkg}->{default_php} = 1;
            $php->{default}             = $pkg;
            next;
        }
        if ( $line =~ m{ (\S+) \s SAPI: \s (\S+) }xms ) {
            $pkg = $1;
            $php->{$pkg}->{handler} = $2;
            foreach ( split( /\n/, timed_run( 0, 'scl', 'enable', $pkg, 'php -v' ) ) ) {
                if (m{ PHP \s (\d+\.\S+) \s \(cli\) \s \(built: \s (\w+\s+\d+\s\d+\s\d+:\d+:\d+) }xms) {    # Must accept release version like 7.1.11 or 7.2.0RC5
                    $php->{$pkg}->{release_version} = $1;
                    $php->{$pkg}->{build_time}      = $2;
                    $php->{$pkg}->{build_time} =~ s/  / /;
                }
                if (/Zend\sEngine\sv(\d+\.\d+\.\d+)/) {
                    $php->{$pkg}->{zend} = $1;
                }
            }

            # Gather a list of modules for this given PHP Binary - not used currently, enable when needed.
            # @{ $php->{$pkg}->{module_list} } = grep { !/\[PHP\sModules\]/ && /\w/ && !/Zend/ } split( /\n/, timed_run( 0, 'scl', 'enable', $pkg, 'php -m' ) );
        }
    }
    return $php;
}

sub check_sysinfo {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    return unless my $hostinfo = get_hostinfo_href();

    my $sysinfo_config = '/var/cpanel/sysinfo.config';

    my $rebuild = 0;

    if ( !-e $sysinfo_config ) {
        print_crit('sysinfo: ');
        print_critical('does not exist, run /scripts/gensysinfo to fix');
    }
    else {
        open my $sysinfo_fh, '<', $sysinfo_config;
        while (<$sysinfo_fh>) {
            chomp;
            if (m{ \A rpm_arch=(.*) }xms) {
                if ( $hostinfo->{'hardware'} ne $1 ) {
                    $rebuild = 1;
                }
            }
            if (m{ \A release=(.*) }xms) {
                if ( _get_run_var('os_version') ne $1 ) {
                    $rebuild = 1;
                }
            }
            if (m{ \A ises=(.*) }xms) {
                if ( _get_run_var('os_ises') ne $1 ) {
                    $rebuild = 1;
                }
            }
        }
        close $sysinfo_fh;
    }

    if ( $rebuild == 1 ) {
        print_crit('sysinfo: ');
        print_critical('/var/cpanel/sysinfo.config contains errors -- run /scripts/gensysinfo to fix');
    }
}

sub check_for_remote_mysql {
    my $mysql_host;
    my $mysql_is_local;

    ## obtain mysql host, if exists
    my $my_cnf = '/root/.my.cnf';
    if ( open my $my_cnf_fh, '<', $my_cnf ) {
        while (<$my_cnf_fh>) {
            chomp( my $line = $_ );
            if ( $line =~ m{ \A host \s* = \s* (?:["']?) ([^"']+) }xms ) {
                $mysql_host = $1;
            }
        }
        close $my_cnf_fh;
    }

    if ($mysql_host) {
        if ( $mysql_host =~ m{ ( \d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3} )  }xms ) {
            return if ( $mysql_host eq '127.0.0.1' );
            for my $ipaddr ( @{ get_local_ipaddrs_aref() } ) {
                if ( $ipaddr eq $mysql_host ) {
                    $mysql_is_local = 1;
                    last;
                }
            }
        }
        elsif ( $mysql_host eq 'localhost' or $mysql_host eq get_hostname() ) {
            $mysql_is_local = 1;
        }
        if ( !$mysql_is_local ) {
            print_info('Remote MySQL Host: ');
            print_warning($mysql_host);
        }
    }
}

sub print_if_using_other_dns {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    my %service = (
        '/var/cpanel/usensd'      => 'NSD',
        '/var/cpanel/usemydns'    => 'MyDNS',
        '/var/cpanel/usepowerdns' => 'PowerDNS',
    );
    my @found = grep { -e $_ } keys(%service);
    return unless scalar @found;
    if ( scalar @found > 1 ) {
        print_warn('DNS Service: ');
        print_warning( 'multiple service touchfiles found! [ ' . join( ' ', @found ) . ' ]' );
    }
    for my $found (@found) {
        print_info('DNS Service: ');
        print_normal( $service{$found} );
    }
}

sub print_mysql_version {
    return unless my $mysql_full_version = get_mysql_full_version();
    print_info('MySQL Version: ');
    print_normal($mysql_full_version);
    return unless my $mysql_numeric_version = get_mysql_numeric_version();
    if ( defined $CPCONF{'mysql-version'} ) {
        my $test_version = $CPCONF{'mysql-version'} . '.';
        unless ( index( $mysql_numeric_version, $test_version ) == 0 ) {
            print_warning( "\t \\_ mysql-version=" . $CPCONF{'mysql-version'} . ' in cpanel.config does not match installed version!' );
        }
    }
}

sub get_new_backup_conf_href {
    my $new_backup_config = '/var/cpanel/backups/config';
    return unless -f $new_backup_config;
    return unless open( my $backupconf_fh, '<', $new_backup_config );
    local $/ = undef;
    my $new = { map { ( split( /:\s/, $_, 2 ) )[ 0, 1 ] } split( /\n/, readline($backupconf_fh) ) };
    close $backupconf_fh;
    return $new;
}

sub get_old_backup_conf_href {
    my $old_backup_config = '/etc/cpbackup.conf';
    return unless -f $old_backup_config;
    return unless open( my $backupconf_fh, '<', $old_backup_config );
    local $/ = undef;
    my $old = { map { ( split( /\s/, $_, 2 ) )[ 0, 1 ] } split( /\n/, readline($backupconf_fh) ) };
    close $backupconf_fh;
    return $old;
}

sub print_roundcube_db {
    if ( defined $CPCONF{'roundcube_db'} and $CPCONF{'roundcube_db'} eq 'mysql' ) {
        print_info('Roundcube: ');
        print_normal('using mysql database');
    }
}

sub print_backups_info {
    return unless i_am('cpanel');
    my $old_backup_conf = get_old_backup_conf_href();
    my $new_backup_conf = get_new_backup_conf_href();
    my %new_dest        = ();
    my ( $new_backups_cron, $new_backups_status ) = ( 0, 'No Config' );
    my ( $old_backups_cron, $old_backups_status ) = ( 0, 'No Config' );
    my $warning        = 0;
    my $new_backup_dir = '/var/cpanel/backups/';

    if ( defined $new_backup_conf and defined $new_backup_conf->{'BACKUPENABLE'} and $new_backup_conf->{'BACKUPENABLE'} =~ /yes/ ) {
        my @dir_contents = ();
        if ( opendir( my $dir_fh, $new_backup_dir ) ) {
            @dir_contents = readdir $dir_fh;
            closedir $dir_fh;
        }

        for my $dest (@dir_contents) {
            if ( $dest =~ m{ \.backup_destination \z }xms ) {
                if ( open( my $destconf_fh, '<', $new_backup_dir . $dest ) ) {
                    local $/ = undef;
                    %{ $new_dest{$dest} } = map { ( split( /:\s/, $_, 2 ) )[ 0, 1 ] } split( /\n/, readline($destconf_fh) );
                    close $destconf_fh;
                }
            }
        }
    }

    if ( ( defined $old_backup_conf and defined $old_backup_conf->{'BACKUPENABLE'} and $old_backup_conf->{'BACKUPENABLE'} eq 'yes' ) or ( defined $new_backup_conf and defined $new_backup_conf->{'BACKUPENABLE'} and $new_backup_conf->{'BACKUPENABLE'} =~ /yes/ ) ) {
        if ( open my $file_fh, '<', '/var/spool/cron/root' ) {
            while (<$file_fh>) {
                if (m{ \A [^#] .+ /usr/local/cpanel/scripts/cpbackup }xms) {
                    $old_backups_cron = 1;
                }
                if (m{ \A [^#] .+ /usr/local/cpanel/bin/backup }xms) {
                    $new_backups_cron = 1;
                }
            }
            close $file_fh;
        }
    }

    if ( defined $new_backup_conf and defined $new_backup_conf->{'BACKUPENABLE'} ) {
        if ( $new_backup_conf->{'BACKUPENABLE'} =~ /yes/ ) {
            $new_backups_status = 'Enabled';
            if ( defined( $new_backup_conf->{'BACKUPACCTS'} ) && $new_backup_conf->{'BACKUPACCTS'} =~ /yes/ ) {
                $new_backups_status .= '/WithAccounts';
            }
            elsif ( defined( $new_backup_conf->{'BACKUPACCTS'} ) && $new_backup_conf->{'BACKUPACCTS'} =~ /no/ ) {
                $new_backups_status .= '/NoAccounts';
            }

            if ( defined( $new_backup_conf->{'BACKUPTYPE'} ) && $new_backup_conf->{'BACKUPTYPE'} =~ /uncompressed/ ) {
                $new_backups_status .= '/Uncompressed';
            }
            elsif ( defined( $new_backup_conf->{'BACKUPTYPE'} ) && $new_backup_conf->{'BACKUPTYPE'} =~ /compressed/ ) {
                $new_backups_status .= '/Compressed';
            }
            elsif ( defined( $new_backup_conf->{'BACKUPTYPE'} ) && $new_backup_conf->{'BACKUPTYPE'} =~ /incremental/ ) {
                $new_backups_status .= '/Incremental';
            }
            else {
                $new_backups_status .= '/Unknown';
            }

            if ( $new_backups_cron != 1 ) {
                $new_backups_status .= ' (MISSING CRON!)';
                $warning = 1;
            }
        }
        elsif ( $new_backup_conf->{'BACKUPENABLE'} =~ /no/ ) {
            $new_backups_status = 'Disabled';
        }
    }

    if ( defined $old_backup_conf and defined $old_backup_conf->{'BACKUPENABLE'} ) {
        if ( $old_backup_conf->{'BACKUPENABLE'} eq 'restoreonly' ) {
            $old_backups_status = 'RestoreOnly';
        }
        elsif ( $old_backup_conf->{'BACKUPENABLE'} eq 'yes' ) {
            $old_backups_status = 'Enabled';
            if ( defined( $old_backup_conf->{'BACKUPACCTS'} ) && $old_backup_conf->{'BACKUPACCTS'} eq 'yes' ) {
                $old_backups_status .= '/WithAccounts';
            }
            elsif ( defined( $old_backup_conf->{'BACKUPACCTS'} ) && $old_backup_conf->{'BACKUPACCTS'} eq 'no' ) {
                $old_backups_status .= '/NoAccounts';
            }

            if ( defined( $old_backup_conf->{'BACKUPINC'} ) && $old_backup_conf->{'BACKUPINC'} eq 'yes' ) {
                $old_backups_status .= '/Incremental';
            }
            elsif ( defined( $old_backup_conf->{'COMPRESSACCTS'} ) && $old_backup_conf->{'COMPRESSACCTS'} eq 'yes' ) {
                $old_backups_status .= '/Compressed';
            }
            elsif ( defined( $old_backup_conf->{'COMPRESSACCTS'} ) && $old_backup_conf->{'COMPRESSACCTS'} eq 'no' ) {
                $old_backups_status .= '/Uncompressed';
            }
            else {
                $old_backups_status .= '/Unknown';
            }

            if ( $old_backups_cron != 1 ) {
                $old_backups_status .= ' (MISSING CRON!)';
                $warning = 1;
            }
        }
        elsif ( $old_backup_conf->{'BACKUPENABLE'} eq 'no' ) {
            $old_backups_status = 'Disabled';
        }
    }

    if ( keys(%new_dest) ) {
        if ( defined $new_backup_conf and defined $new_backup_conf->{'KEEPLOCAL'} and $new_backup_conf->{'KEEPLOCAL'} =~ /1/ ) {
            $new_backups_status .= '/RetainLocal';
        }
        else {
            $new_backups_status .= '/NoRetainLocal';
        }
    }

    # Is Jetbackup installed?
    my $JetBackupVersion = "N/A";
    if ( i_am('jetbackup') ) {
        my $raw         = timed_run( 0, 'jetapi', 'backup', '-F', 'manageSettingsGeneral', '-O', 'json' );
        my $json_output = get_json_href($raw);
        if ( defined $json_output->{system}->{version} && defined $json_output->{system}->{tier} ) {
            $JetBackupVersion = $json_output->{system}->{version} . " (" . $json_output->{system}->{tier} . ")";
        }
    }

    print_info('Backups: ');
    if ($warning) {
        print_warning("[New: $new_backups_status] [Legacy: $old_backups_status] [Jetbackup: $JetBackupVersion]");
    }
    else {
        print_normal("[New: $new_backups_status] [Legacy: $old_backups_status] [Jetbackup: $JetBackupVersion]");
    }

    for my $dest ( keys(%new_dest) ) {
        my $type        = exists $new_dest{$dest}->{'type'}    ? $new_dest{$dest}{'type'}                        : 'UNKNOWN';
        my $disabled    = exists $new_dest{$dest}{'disabled'}  ? ( $new_dest{$dest}{'disabled'} ? "Yes" : "No" ) : 'UNKNOWN';
        my $name        = exists $new_dest{$dest}{'name'}      ? $new_dest{$dest}{'name'}                        : 'UNKNOWN';
        my $timeoutdest = exists $new_dest{$dest}->{'timeout'} ? $new_dest{$dest}{'timeout'}                     : 'UNKNOWN';
        print_normal( "\t\t\\_ Remote dest: [Type: " . $type . "] [Disabled: " . $disabled . "] [Name: " . $name . "] [Timeout: " . $timeoutdest . "]" );
        if ( $type eq "SFTP" && exists $new_dest{$dest}{'privatekey'} && exists $new_dest{$dest}{'passphrase'} ) {
            my $key_is_encrypted = 0;
            if ( open my $privatekey_fh, '<', $new_dest{$dest}->{'privatekey'} ) {
                while (<$privatekey_fh>) {
                    if (/ENCRYPTED/) {
                        $key_is_encrypted = 1;
                        last;
                    }
                }
                close $privatekey_fh;
            }
            if ( !$key_is_encrypted ) {
                print_warning("\t\t   \\_ The SFTP private key is not encrypted but the transport config contains a passphrase.  See FB-152341 and FB-152337.");
            }
        }
    }
}

sub print_mailserver_info {
    return unless i_am('cpanel');
    return unless defined $CPCONF{'mailserver'};
    return unless cpanel_version_is(qw( < 11.53.0.0 ));    # 54+ only supports Dovecot

    print_info('Mailserver: ');
    print_normal( $CPCONF{'mailserver'} );
}

sub print_ftpserver_info {
    return unless i_am('cpanel');

    my $external_ip_address = get_external_ip();
    my $pureftpd_conf       = get_pureftpd_conf_href();
    my $proftpd_conf        = get_proftpd_conf_href();

    print_info('FTP Server: ');

    my $passiveports = "";
    my $passiveip    = "";

    if ( defined( $CPCONF{'ftpserver'} ) ) {
        if ( $CPCONF{'ftpserver'} eq 'pure-ftpd' ) {
            if ( defined( $pureftpd_conf->{'passiveportrange'} ) && defined( $pureftpd_conf->{'passiveportrange'}->{value} ) ) {
                $passiveports = $pureftpd_conf->{'passiveportrange'}->{value};
            }

            if ( defined( $pureftpd_conf->{'forcepassiveip'} ) && defined( $pureftpd_conf->{'forcepassiveip'}->{value} ) ) {
                $passiveip = $pureftpd_conf->{'forcepassiveip'}->{value};
            }
        }
        if ( $CPCONF{'ftpserver'} eq 'proftpd' ) {
            if ( defined( $proftpd_conf->{'passiveports'} ) && defined( $proftpd_conf->{'passiveports'}->{value} ) ) {
                $passiveports = $proftpd_conf->{'passiveports'}->{value};
            }

            if ( defined( $proftpd_conf->{'masqueradeaddress'} ) && defined( $proftpd_conf->{'masqueradeaddress'}->{value} ) ) {
                $passiveip = $proftpd_conf->{'masqueradeaddress'}->{value};
            }
        }
    }
    my $fwppactive = 0;
    if ($passiveports) {
        $passiveports =~ s/\s+/:/;
        my @fwcommand = timed_run( 10, '/sbin/iptables', '-nL' );
        foreach my $fwline (@fwcommand) {
            chomp($fwline);
            if ( $fwline =~ m/$passiveports/ and $fwline =~ m/ACCEPT/ ) {
                $fwppactive = 1;
                last;
            }
        }
    }

    my $passivetext = $passiveports ? "enabled - " . ( $fwppactive ? "allowed in iptables" : "not found in iptables" ) : "not enabled";

    if ( $passiveip ne "" && defined($external_ip_address) && $passiveip ne $external_ip_address && defined( $CPCONF{'ftpserver'} ) ) {
        if ( $CPCONF{'ftpserver'} eq 'proftpd' ) {
            $passivetext .= " - MasqueradeAddress ( $passiveip ) doesn't match license IP";
        }
        elsif ( $CPCONF{'ftpserver'} eq 'pure-ftpd' ) {
            $passivetext .= " - ForcePassiveIP ( $passiveip ) doesn't match license IP";
        }
    }

    if ( defined( $CPCONF{'ftpserver'} ) ) {
        print_normal("$CPCONF{ftpserver} ( Passive ports $passivetext )");
    }
    else {
        print_warning('missing ftpserver setting in cpanel.config');
    }
    return;
}

sub print_exim_info {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    return unless my $exim_localopts = get_exim_localopts_href();
    if ( defined $exim_localopts->{acl_delay_unknown_hosts} && $exim_localopts->{acl_delay_unknown_hosts} ) {
        my $info     = '20 second SMTP delay active (by default) for unknown hosts and spam, see DOC-6092.';
        my $disabled = '';
        if ( defined $exim_localopts->{acl_dont_delay_greylisting_trusted_hosts} && $exim_localopts->{acl_dont_delay_greylisting_trusted_hosts} ) {
            $disabled .= ' [Greylisting Trusted Hosts]';
        }
        if ( defined $exim_localopts->{acl_dont_delay_greylisting_common_mail_providers} && $exim_localopts->{acl_dont_delay_greylisting_common_mail_providers} ) {
            $disabled .= ' [Greylisting Common Mail Providers]';
        }
        $info .= ' Disabled for:' . $disabled if length $disabled;
        print_info('Exim: ');
        print_normal($info);
    }
}

sub check_for_custom_webtemplates {
    return unless i_am('cpanel');
    my $template_dir = '/var/cpanel/webtemplates';
    return unless -d $template_dir;

    my $found;
    find(
        sub {
            return unless /\.tmpl$/s;
            $found = 1;
        },
        $template_dir
    );
    return unless $found;

    print_info('Web templates: ');
    print_normal("found in ${template_dir} -- https://documentation.cpanel.net/display/ALD/Web+Template+Editor");
}

sub check_for_custom_restoremodules {
    return unless i_am('cpanel');
    my $restoremodule_dir = '/var/cpanel/perl/Whostmgr/Transfers/Systems';
    return unless -d $restoremodule_dir;

    my $found;
    find(
        sub {
            return unless /\.pm$/s;
            $found = 1;
        },
        $restoremodule_dir
    );
    return unless $found;

    print_info('Custom Restore Modules: ');
    print_normal("found in ${restoremodule_dir} -- can cause issues with restoration process -- check ticket #9858879");
}

sub check_for_custom_zonetemplates {
    return unless i_am('cpanel');
    my $template_dir = '/var/cpanel/zonetemplates';
    return unless -d $template_dir;

    my $is_empty = 0;

    opendir( my $fh, $template_dir ) or return;
    my @dirents = grep { !/^\.\.?/ } readdir $fh;
    closedir $fh;

    return if !@dirents;

    for my $file (@dirents) {
        if ( -z "${template_dir}/${file}" ) {
            $is_empty = 1;
            last;
        }
    }

    print_info('Zone templates: ');
    if ( $is_empty == 1 ) {
        print_red("found in $template_dir - some may be empty! See ticket 4897373");
    }
    else {
        print_normal("found in $template_dir");
    }
}

sub print_lsws_info {
    return unless i_am('cpanel');
    return unless my ( $lsws_full_version, $lsws_numeric_version ) = @{ get_lsws_version_aref() };

    print_info('LiteSpeed Web Server: ');
    print_normal("version [ $lsws_full_version ]");

    my %lshttpd_ports = ();

    my $ports = get_lsof_port_href();
    while ( my ( $portnum, $aref ) = each(%$ports) ) {
        for my $href (@$aref) {
            next if not $href->{USER} eq "root";
            next if not $href->{CMD} eq "litespeed";
            $lshttpd_ports{$portnum} = 1;
        }
    }

    if ( scalar keys(%lshttpd_ports) ) {
        print_info('LiteSpeed Web Server: ');
        print_normal( 'is listening on ports [ ' . join( " ", sort( keys(%lshttpd_ports) ) ) . ' ]' );
    }

    print_info('LiteSpeed Web Server: ');
    if ( $lsws_full_version =~ /Enterprise/ ) {
        print_normal('is supported, see http://cpanel.wiki/display/LS/LiteSpeed');
    }
    else {
        print_warning('non-Enterprise editions of LiteSpeed are NOT directly supported');
    }
    print_info('LiteSpeed Web Server: ');
    print_warning('whm-server-status is incompatible with LiteSpeed');
    get_kernel_pid_max();
}

sub check_for_lsws_update {
    return unless i_am('cpanel');
    return unless my ( $lsws_full_version, $lsws_numeric_version ) = @{ get_lsws_version_aref() };
    return if $lsws_numeric_version eq "unknown";
    return unless $lsws_full_version =~ /Enterprise/;

    my $reply = _http_get( Host => 'update.litespeedtech.com', Path => '/ws/latest.php', MultiHomed => 0, Timeout => 5 );
    return unless defined $reply;

    my $available_lsws_version;
    my @lsws_data = split /\n/, $reply;
    for (@lsws_data) {
        if (m{ \A LSWS=(\d+\.\d+\.\d+) \z }xms) {
            $available_lsws_version = $1;
            last;
        }
    }
    return unless $available_lsws_version;

    if ( version_compare( $lsws_numeric_version, '<', $available_lsws_version ) ) {
        print_info('LiteSpeed Web Server: ');
        print_warning("UPDATE AVAILABLE ($lsws_numeric_version -> $available_lsws_version)");
    }
}

##############################
#  END [INFO] CHECKS
##############################

##############################
#  BEGIN [WARN] CHECKS
##############################

sub check_for_license_error {
    my $cpanel_error_log   = '/usr/local/cpanel/logs/error_log';
    my $license_error_file = '/usr/local/cpanel/logs/license_error.display';

    # TECH-695
    # These can occur at any time so we need to handle it before bailing out when $license_error_file doesn't exist.
    # Look for lines like the following, discard anything BEFORE LAST "cpsrvd started" line indicating most recent cpsrvd restart, if any.
    # ==> cpsrvd 11.78.0.11 started
    # License Check Failed during startup of cpsrvd: License corrupted, please fetch a new license file.
    # License Check Failed: License corrupted, please fetch a new license file.
    if ( cpanel_version_is(qw(>= 11.78.0.11)) ) {
        my $start_match = qr{ \A ==> \s cpsrvd \s .+ \s started }xms;
        my $error_match = qr{ \A License \s Check \s Failed }xms;
        my $match_limit = 4;
        my $error_ar    = get_last_lines_from_file( $cpanel_error_log, qr{ (?: $start_match | $error_match ) }xms, $match_limit );
        tail_array_after_match( $error_ar, $start_match );
        if ( @{$error_ar} ) {
            my $indent = "\n         \\_ ";
            print_warn('License Error: ');
            print_warning( "found in $cpanel_error_log after last cpsrvd restart:$indent" . join $indent, @{$error_ar} );
        }
    }

    # Everything after this depends on something in $license_error_file

    return unless -f $license_error_file && -s _;

    my $license_error;
    if ( open( my $license_error_fh, '<', $license_error_file ) ) {
        while (<$license_error_fh>) {
            if (m{\AThe exact message was: (.+)\Z}ms) {
                $license_error = $1;
                chomp $license_error;
                last;
            }
        }
        close $license_error_fh;
    }

    return unless defined $license_error;

    if ( cpanel_version_is(qw( < 11.32.0.0 )) ) {
        print_warn('License Error: ');
        print_warning( '[ ' . $license_error . ' ]' );
        print_warning('         \_ Try updating to WHM 11.32 or later to resolve any license-related problems.');
        return;
    }

    # Everything after this is WHM 11.32+

    if (
        $license_error =~ m{ \A
         \QThe hostname must be a Fully Qualified Domain Name! (\E.+\)
       | \QAbort, Retry, Fail?\E
         \Z }xms
    ) {
        print_warn('License Error: ');
        print_warning( '[ ' . $license_error . ' ]' );
        print_warning('         \_ If this license error is not resolved after correctly setting the hostname AND ensuring that the hostname can be pinged from the local host ( ping `hostname` ), then fork ticket for license issue if not related to current issue, send "ESCALATE - License issue to Dev" response, and escalate ticket to "QA/Development".');
    }
    elsif (    # This one is CRIT because it requires escalation
        $license_error =~ m{ \A
         \QDoes not compute!\E
       | \QReturn without Gosub.\E
       | \QPrinting is not supported on this printer.\E
       | \QCannot issue a license to \E[^ ]+\Q without a \E(DISTRO|OSVER)\.
       \Z }xms
    ) {
        print_crit('License Error: ');
        print_critical( '[ ' . $license_error . ' ]' );
        print_critical('         \_ Fork ticket for license issue if not related to current issue, send "ESCALATE - License issue to Dev" response, and escalate ticket to "QA/Development".');
    }
    else {
        print_warn('License Error: ');
        print_warning( '[ ' . $license_error . ' ]' );
    }
}

sub check_for_license_creds {
    return unless i_am('cpanel');
    my $license_creds_file = '/var/cpanel/licenseid_credentials.json';
    my $license_creds_raw;
    if ( open( my $fh, '<', $license_creds_file ) ) {
        local $/ = undef;
        $license_creds_raw = <$fh>;
        close($fh);
    }
    my $license_creds_href = $license_creds_raw && get_json_href($license_creds_raw);
    if ( !$license_creds_href || !keys(%$license_creds_href) ) {
        print_warn('License Error: ');
        print_warning( "$license_creds_file" . ' is missing or empty. Is port 2083 inbound open? This can cause login issues or "Unauthorized" error in AutoSSL' );
    }
}

sub get_last_lines_from_file {

    # Returns array-ref containing the LAST $return_limit (number) lines that match $pattern which exist within $bytes_limit bytes of the end of $file.
    # Everything but $file is optional.
    # TODO: Read file in reverse similar to File::ReadBackwards (not using it right now because not a core module)
    my ( $file, $pattern, $return_limit, $bytes_limit ) = @_;
    return [] unless -e $file;
    return [] unless my $size = ( stat($file) )[7];
    $bytes_limit  = defined $bytes_limit  ? $bytes_limit + 0  : 5_000_000;    # Default is last 5MB of file
    $return_limit = defined $return_limit ? $return_limit + 0 : 1;            # Default is 1 line returned
    $pattern      = defined $pattern      ? $pattern          : qr{};         # Default is any match
    my @lines;

    if ( open my $fh, '<', $file ) {
        seek $fh, -$bytes_limit, 2;
        while (<$fh>) {
            if (m{$pattern}xms) {
                chomp;
                shift @lines if scalar @lines >= $return_limit;
                push @lines, $_;
            }
        }
        close $fh;
    }
    return \@lines;
}

sub tail_array_after_match {

    # Modifies $aref in place by searching backwards for first $pattern match and discarding the match and everything before it.
    my ( $aref, $pattern ) = @_;
    for ( my $i = $#{$aref}; $i >= 0; $i-- ) {
        next unless $aref->[$i] =~ $pattern;
        splice( @{$aref}, 0, $i + 1 );
        last;
    }
}

sub check_port_hash {
    my $ports = get_lsof_port_href();
    return if scalar keys(%$ports);
    print_warn('lsof: ');
    print_warning('Did not return a list of TCP ports in LISTEN state. Either lsof is broken or there are zero listening services. Some port-based checks will be skipped!');
}

sub check_selinux_status {
    my @selinux_status = split /\n/, timed_run( 0, 'sestatus' );

    return if !@selinux_status;

    for my $line (@selinux_status) {
        if ( $line =~ m{ \A SELinux \s status: \s+ ([^\s\n]+) }xms ) {
            return if $1 eq "disabled";
        }

        elsif ( $line =~ m{ \A Current \s mode: \s+ ([^\s\n]+) }xms ) {
            if ( $1 eq "permissive" ) {
                print_info('SELinux: ');
                print_normal('Permissive');
                return;
            }

            else {
                print_warn('SELinux: ');
                print_warning('is ENFORCING!');
                return;
            }
        }
    }
}

sub check_runlevel {
    my $runlevel;
    my $who_r = timed_run( 0, 'who', '-r' );

    # CentOS 5.7, 5.8:
    #         run-level 3  2012-01-25 10:38                   last=S
    if ( $who_r =~ m{ \A \s* run-level \s (\S+) }xms ) {
        $runlevel = $1;

        if ( $runlevel ne "3" ) {
            print_warn('Runlevel: ');
            print_warning("runlevel is not 3 (current runlevel: $runlevel)");
        }
    }
}

sub check_multiuser_target {
    return if os_version_is(qw( < 7 )) || i_am('amazon');
    chomp( my $target = timed_run( 0, 'systemctl', 'get-default' ) || return );
    chomp( my $active = timed_run( 0, 'systemctl', 'is-active', 'multi-user.target' ) || return );

    my $warn;
    $warn .= ' [current ' . $target . '] (default multi-user.target)' if ( $target ne 'multi-user.target' );
    $warn .= ' [current ' . $active . '] (default active)'            if ( $active ne 'active' );

    if ($warn) {
        print_warn('Boot Target:');
        print_warning( $warn . ' This can prevent tailwatchd or queueprocd from properly starting, see TECH-362/TECH-1024' );
    }
}

sub check_for_missing_root_cron {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    my $cron = '/var/spool/cron/root';
    return if -f $cron;

    print_warn('Missing cron: ');
    print_warning("root's cron file $cron is missing!");
}

sub check_for_missing_usr_bin_crontab {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    my $crontab = '/usr/bin/crontab';
    return if -f $crontab;

    print_warn('Missing crontab binary: ');
    print_warning( 'file ' . $crontab . ' is missing!  Seeing "warn [jail_safe_crontab] Cpanel::Wrap::send_cpwrapd_request error"?  This may be why.' );
}

sub check_if_upcp_is_running {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    if ( exists_process_cmd( qr{ cPanel \s Update \s \(upcp\) }xms, 'root' ) ) {
        print_warn('upcp check: ');
        print_warning('upcp is currently running');
    }
    elsif ( -e '/usr/local/cpanel/upgrade_in_progress.txt' ) {
        print_warn('upcp check: ');
        print_warning('/usr/local/cpanel/upgrade_in_progress.txt found, but upcp doesn\'t appear to be running. Last run failed? If Tweak Settings is not loading, this may be why.');
    }
}

sub check_valid_upcp {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    my $updatenow_static = '/scripts/updatenow.static';

    if ( !-f $updatenow_static ) {
        print_warn('Valid updatenow.static: ');
        print_warning("$updatenow_static does not exist as a file!");
    }
    else {
        my $update_now_text = '';
        if ( open( my $updatenow_fh, '<', $updatenow_static ) ) {
            local $/ = undef;
            $update_now_text = readline($updatenow_fh);
            close $updatenow_fh;
        }
        if ( $update_now_text !~ m/our \$VERSION_BUILD/s ) {
            print_warn('Valid updatenow.static: ');
            print_warning("No VERSION_BUILD info found in $updatenow_static, could be broken!");
        }
    }
}

sub check_cpupdate_conf {
    return unless my $cpupdate_conf = get_cpupdate_conf();
    ( $YUM_OR_DNF, $YUM_CONF ) = os_version_is(qw( >= 8)) ? ( 'dnf', '/etc/dnf/dnf.conf' ) : ( 'yum', '/etc/yum.conf' );
    return unless ( -f $YUM_CONF );
    my $_is_allowed = sub {
        my ($type) = @_;
        return 0 if ( defined $cpupdate_conf->{$type} and ( $cpupdate_conf->{$type} eq "never" or $cpupdate_conf->{$type} eq "manual" ) );
        return 1;
    };
    unless ( $_is_allowed->('UPDATES') ) {
        print_warn('/etc/cpupdate.conf: ');
        print_warning('UPDATES set to never or manual -- do not run /scripts/upcp without customer approval. Recommend enabling automatic updates if the issue would be resolved by an update.');
    }
    unless ( $_is_allowed->('RPMUP') ) {
        print_warn('/etc/cpupdate.conf: ');
        print_warning( 'RPMUP set to never or manual -- prevents automatic updates to EA4 and other ' . $YUM_OR_DNF . '-managed packages. Recommend enabling automatic updates if the issue would be resolved by an update.' );
    }
    unless ( $_is_allowed->('SARULESUP') ) {
        print_warn('/etc/cpupdate.conf: ');
        print_warning('SARULESUP set to never or manual -- prevents automatic updates of SpamAssassin rules. Recommend enabling automatic updates if the issue would be resolved by an update.');
    }
}

sub check_interface_lo {
    my $output = timed_run( 0, 'ip', 'addr', 'show', 'dev', 'lo' );
    $output ||= timed_run( 0, 'ifconfig', 'lo' );

    return check_loopback_connection() if $output =~ /UP.LOOPBACK|LOOPBACK.UP/;    # ip addr and ifconfig swap the LOOPBACK and UP keywords

    print_warn('Loopback Interface: ');
    print_warning('loopback interface is not up!');
}

sub check_loopback_connection {
    return if $OPT_SKIP_NETWORKING;
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    my @ports     = qw( 25 80 2086 );
    my $connected = 0;

    for my $port (@ports) {
        my $sock = IO::Socket::INET->new(
            PeerAddr => '127.0.0.1',
            PeerPort => $port,
            Proto    => 'tcp',
            Timeout  => '1',
        );

        if ($sock) {
            $connected = 1;
            close $sock;
            last;
        }
    }

    if ( !$connected ) {
        print_warn('Loopback connectivity: ');
        print_warning('could not connect to 127.0.0.1 on port 25, 80, or 2086');
    }
}

sub check_cpanelconfig_filetype {
    return unless -e $CPANEL_CONFIG_FILE;
    chomp( my $file = timed_run( 0, 'file', $CPANEL_CONFIG_FILE ) );
    if ( $file !~ m{ \A \Q$CPANEL_CONFIG_FILE\E: \s ASCII \s ( English \s)? text (, \s with \s very \s long \s lines)? \z }xms ) {
        print_warn("$CPANEL_CONFIG_FILE: ");
        print_warning("filetype is something other than 'ASCII text'! ($file)");
    }
}

sub check_cpanelsync_exclude_no_chmod {
    my $cpanelsync_exclude_no_chmod = '/etc/cpanelsync.no_chmod';
    return unless ( -f $cpanelsync_exclude_no_chmod && -s _ );
    print_warn('cpanelsync exclude: ');
    print_warning("$cpanelsync_exclude_no_chmod is not empty!");
}

sub check_cpanelsync_exclude {
    my $cpanelsync_exclude = '/etc/cpanelsync.exclude';
    return unless -f $cpanelsync_exclude;
    return unless -s $cpanelsync_exclude;
    my $rpmversions_file = '/usr/local/cpanel/etc/rpm.versions';

    print_warn('cpanelsync exclude: ');
    print_warning("$cpanelsync_exclude is not empty!");
    if ( open my $file_fh, '<', $cpanelsync_exclude ) {
        while (<$file_fh>) {
            chomp;
            if (m{ \A \s* $rpmversions_file \s* \z }xms) {
                print_warn('cpanelsync exclude: ');
                print_warning("$rpmversions_file found! This should NEVER be done!");
                last;
            }
        }
        close $file_fh;
    }
}

sub check_for_rawopts {
    return unless i_am('ea3') or ea3_downgrade_is_possible();
    my $rawopts_dir = '/var/cpanel/easy/apache/rawopts';
    return unless -d $rawopts_dir;
    my @dir_contents;

    opendir( my $dir_fh, $rawopts_dir );
    @dir_contents = grep { !/^\.\.?$/ } readdir $dir_fh;
    closedir $dir_fh;

    if (@dir_contents) {
        print_warn('EA3 Rawopts Detected: ');
        print_warning('check /var/cpanel/easy/apache/rawopts !');
    }
}

sub check_for_rawenv {
    return unless i_am('ea3') or ea3_downgrade_is_possible();
    my $rawenv_dir = '/var/cpanel/easy/apache/rawenv';
    return unless -d $rawenv_dir;
    my @dir_contents;

    opendir( my $dir_fh, $rawenv_dir );
    @dir_contents = grep { !/^\.\.?$/ } readdir $dir_fh;
    closedir $dir_fh;

    if (@dir_contents) {
        print_warn('EA3 Rawenv detected: ');
        print_warning('check /var/cpanel/easy/apache/rawenv !');
    }
}

sub check_for_custom_opt_mods {
    return unless i_am('ea3') or ea3_downgrade_is_possible();
    my $custom_opt_mods;
    my $dir = '/var/cpanel/easy/apache/custom_opt_mods';
    return unless -d $dir;

    my @custom_opt_mods;    # items in /var/cpanel/easy/apache/custom_opt_mods/
    find(
        sub {
            # ignore these, Attracta:
            #  /var/cpanel/easy/apache/custom_opt_mods/Cpanel/Easy/ModFastInclude.pm
            #  /var/cpanel/easy/apache/custom_opt_mods/Cpanel/Easy/ModFastInclude.pm.tar.gz

            my $file = $File::Find::name;
            if ( -f $file and $file !~ m{ /ModFastInclude\.pm(.*) }xms ) {
                $file =~ s#/var/cpanel/easy/apache/custom_opt_mods/##;
                push @custom_opt_mods, $file;
            }
        },
        $dir
    );

    if ( scalar @custom_opt_mods > 10 ) {
        print_warn("EA3 $dir: ");
        print_warning('many custom opt mods exist, check manually');
    }
    elsif (@custom_opt_mods) {
        for my $custom_opt_mod (@custom_opt_mods) {
            $custom_opt_mods .= "$custom_opt_mod ";
        }

        print_warn("EA3 $dir: ");
        print_warning($custom_opt_mods);
    }
}

sub check_for_local_templates {
    return unless i_am('cpanel');
    my @templatedirs = qw(
      /var/cpanel/templates/apache2_4
      /var/cpanel/templates/apache2_2
      /var/cpanel/templates/apache2_0
      /var/cpanel/templates/apache2
      /var/cpanel/templates/apache1_3
      /var/cpanel/templates/apache1
      /var/cpanel/templates/dovecot2.2
      /var/cpanel/templates/dovecot2.3
      /var/cpanel/templates/dovecotSNI
      );    # Order is somewhat important above for cosmetic reasons, due to symlinks
    my %templatedirs = ();

    for my $templatedir (@templatedirs) {    # Canonicalize symlinks so we only check a real path once, but store original name for printing.
        next if !-d $templatedir;
        $templatedirs{ abs_path($templatedir) } = $templatedir;
    }
    for my $templatedir ( sort( keys(%templatedirs) ) ) {
        my @dir_contents = ();
        if ( opendir( my $dir_fh, $templatedir ) ) {
            @dir_contents = readdir $dir_fh;
            closedir $dir_fh;
        }

        my $templates = undef;
        for my $template (@dir_contents) {
            if ( $template =~ m{ \.local \z }xms ) {
                $templates .= " $template";
            }
        }
        if ($templates) {
            print_warn( 'Custom templates (' . $templatedirs{$templatedir} . '): ' );
            print_warning($templates);
        }
    }
}

sub check_for_missing_account_suspensions_conf {
    return unless i_am('cpanel');
    my @templates;
    if ( i_am('ea4') ) {
        return unless -f '/etc/apache2/conf.d/includes/account_suspensions.conf';
        @templates = qw (
          /var/cpanel/templates/apache2_4/ea4_main.local
        );
    }
    elsif ( i_am('ea3') ) {
        return unless -f '/usr/local/apache/conf/includes/account_suspensions.conf';
        @templates = qw(
          /var/cpanel/templates/apache2_4/main.local
          /var/cpanel/templates/apache2_2/main.local
          /var/cpanel/templates/apache2_0/main.local
          /var/cpanel/templates/apache2/main.local
          /var/cpanel/templates/apache1_3/main.local
          /var/cpanel/templates/apache1/main.local
          );    # Order is somewhat important above for cosmetic reasons, due to symlinks
    }
    else { return; }

    my %templates = ();
    for my $template (@templates) {    # Canonicalize symlinks so we only check a real path once, but store original name for printing.
        next unless -f $template;
        $templates{ abs_path($template) }[0] = $template;
    }

    for my $template ( sort( keys(%templates) ) ) {
        $templates{$template}[1] = 0;
        if ( open my $template_fh, '<', $template ) {
            while (<$template_fh>) {
                if (m{ \A \s* Include .+ account_suspensions.conf }x) {
                    $templates{$template}[1] = 1;
                }
            }
            close $template_fh;
        }
    }

    for my $template ( keys(%templates) ) {
        if ( !$templates{$template}[1] ) {
            print_warn("Custom templates: ");
            print_warning( $templates{$template}[0] . " is missing include for account_suspensions.conf!\n\t\\_ Use predefined \"WEBSERVER - Suspensions Template Update\"" );
        }
    }
}

sub check_for_custom_apache_includes {
    return unless i_am('cpanel');
    my $include_dir = i_am('ea4') ? '/etc/apache2/conf.d/includes' : '/usr/local/apache/conf/includes';

    return if !-d $include_dir;

    my @includes = qw(
      post_virtualhost_1.conf
      post_virtualhost_2.conf
      post_virtualhost_global.conf
      pre_main_1.conf
      pre_main_2.conf
      pre_main_global.conf
      pre_virtualhost_1.conf
      pre_virtualhost_2.conf
      pre_virtualhost_global.conf
    );

    my $custom_includes;
    for my $include (@includes) {
        if ( -s "${include_dir}/${include}" ) {
            if ( $include eq 'pre_virtualhost_global.conf' ) {
                my $md5 = timed_run( 0, 'md5sum', $include_dir . 'pre_virtualhost_global.conf' );
                next if ( $md5 && $md5 =~ m{ \A 1693b9075fa54ede224bfeb8ad42a182 \s }xms );
            }
            $custom_includes .= ' [' . $include . ']';
        }
    }

    if ($custom_includes) {
        print_warn( 'Apache Includes [' . $include_dir . ']:' );
        print_warning($custom_includes);
    }
}

sub check_for_tomcatoptions {
    return unless i_am('cpanel');
    my $tomcat_options = '/var/cpanel/tomcat.options';
    if ( -f $tomcat_options and not -z $tomcat_options ) {
        my $md5 = timed_run( 0, 'md5sum', '/var/cpanel/tomcat.options' );
        return if ( $md5 && $md5 =~ m{ \A 0cb9b170cbb81795c2669f8ebf08d0dd \s }xms );    ## -Xss2m
        print_warn('Tomcat options: ');
        print_warning("$tomcat_options exists");
    }
}

sub check_for_sneaky_htaccess {
    return unless i_am('cpanel');
    ## this is lazy checking. ideally we'd check HOMEMATCH from wwwacct.conf and go from there.
    ## but then, nothing guarantees the current HOMEMATCH has always been the same, either.
    my @dirs = qw( / /var/www/html/ /home/ /home2/ /home3/ /home4/ /home5/ /home6/ /home7/ /home8/ /home9/ );
    my $htaccess;

    for my $dir (@dirs) {
        if ( -f $dir . '.htaccess' and not -z $dir . '.htaccess' ) {
            $htaccess .= $dir . '.htaccess ';
        }
    }

    if ($htaccess) {
        print_warn('Sneaky .htaccess file(s) found: ');
        print_warning($htaccess);
    }
}

sub check_ea4_paths_conf {
    return unless i_am('ea4');
    my $paths_conf = '/etc/cpanel/ea4/paths.conf';
    lstat($paths_conf);    # Can now use _ for file tests.

    if ( !-e _ ) {
        print_warn('EA4: ');
        print_warning('/etc/cpanel/ea4/paths.conf is missing!');
        return;
    }
    if ( !-f _ ) {
        print_warn('EA4: ');
        print_warning('/etc/cpanel/ea4/paths.conf is not a normal file!');
        return;
    }
    if ( -z _ ) {
        print_warn('EA4: ');
        print_warning('/etc/cpanel/ea4/paths.conf is empty!');
        return;
    }
    if ( !-T _ ) {
        print_warn('EA4: ');
        print_warning('/etc/cpanel/ea4/paths.conf does not appear to be an ASCII text file!');
        return;
    }

    my $unknown_count;
    my %conf;
    my %default_conf = (    # From ea-apache24-config-runtime-1.0-81.81.4.cpanel.noarch
        'bin_apachectl'        => '/usr/sbin/apachectl',
        'bin_httpd'            => '/usr/sbin/httpd',
        'bin_suexec'           => '/usr/sbin/suexec',
        'dir_base'             => '/etc/apache2',
        'dir_conf'             => '/etc/apache2/conf.d',
        'dir_conf_includes'    => '/etc/apache2/conf.d/includes',
        'dir_conf_userdata'    => '/etc/apache2/conf.d/userdata',
        'dir_docroot'          => '/var/www/html',
        'dir_domlogs'          => '/etc/apache2/logs/domlogs',
        'dir_logs'             => '/etc/apache2/logs',
        'dir_modules'          => '/etc/apache2/modules',
        'dir_run'              => '/run/apache2',
        'file_access_log'      => '/etc/apache2/logs/access_log',
        'file_conf'            => '/etc/apache2/conf/httpd.conf',
        'file_conf_mime_types' => '/etc/apache2/conf/mime.types',
        'file_conf_php_conf'   => '/etc/apache2/conf.d/php.conf',
        'file_conf_srm_conf'   => '/etc/apache2/conf.d/srm.conf',
        'file_error_log'       => '/etc/apache2/logs/error_log',
    );
    if ( os_version_is(qw( < 7.0 )) or i_am('amazon') ) {
        $default_conf{'dir_run'} = '/var/run/apache2';
    }
    if ( open my $conf_fh, '<', $paths_conf ) {
        while (<$conf_fh>) {
            next if /^(#|$)/;
            if (m{ \A \s* ([^=]+?) \s* = \s* ([^\$]*?) \Z }x) {    # Cpanel::Config::LoadConfig::loadConfig( $path, $conf, '\s*=\s*', undef, '^\s*' );
                $conf{$1} = $2;
            }
        }
        close $conf_fh;
        if ( !scalar keys %conf ) {
            print_warn('EA4: ');
            print_warning('/etc/cpanel/ea4/paths.conf does not appear to contain any valid configuration!');
            return;
        }
        foreach my $key ( sort keys %conf ) {
            if ( !exists $default_conf{$key} ) {                   # EA4 appears to ignore any unknown options, but count them.
                $unknown_count++;
                next;
            }
            if ( $default_conf{$key} ne $conf{$key} ) {
                print_warn('EA4: ');
                print_warning( '/etc/cpanel/ea4/paths.conf non-default setting: [ ' . $key . '=' . $conf{$key} . ' ]' );
            }
        }
        foreach my $key ( sort keys %default_conf ) {
            next if exists $conf{$key};
            print_warn('EA4: ');
            print_warning( '/etc/cpanel/ea4/paths.conf missing default setting: [ ' . $key . ' ]' );
        }
        if ($unknown_count) {
            print_warn('EA4: ');
            print_warning( '/etc/cpanel/ea4/paths.conf contains ' . $unknown_count . ' unknown configuration setting(s)!' );
        }
    }
}

sub check_ea4_missing_cl_repo {
    return unless i_am( 'cloudlinux', 'ea4' );
    my $clea4repo = '/etc/yum.repos.d/cloudlinux-ea4.repo';
    return if -f $clea4repo;

    print_warn('Missing Repo: ');
    print_warning( $clea4repo . ' - This can break EA4, see TECH-764.' );
}

sub check_apache_modules {
    return unless i_am('cpanel');
    my $installed_modules = get_apache_modules_href();
    return unless scalar keys %{$installed_modules};
    my $apache_version = get_apache_version();
    my ( $lsws_full_version, $lsws_numeric_version ) = @{ get_lsws_version_aref() };

    # Example: 'foo_module' => { help => [ 'Some help text.' ], check_missing => 1 }
    #      or: push @{ $check{'foo_module'}{'help'} }, 'More help text.';
    # Set check_missing => 1 to report missing instead of installed module
    my %check = (
        'evasive20_module' => { help => ['Can result in random 403s. Check /var/log/apache2/mod_evasive/ if relevant.'] },
        'evasive24_module' => { help => ['Can result in random 403s. Check /var/log/apache2/mod_evasive/ if relevant.'] },
        'headers_module'   => { help => ['May cause proxy subdomains to redirect infinitely, see CPANEL-12707.'], check_missing => 1 },
        'hive_module'      => { help => ['Third-party - 1H Hive.  Not supported.'] },
        'lua_module'       => { help => ['Experimental.  Potential security issues in shared hosting environments.'] },
        'rpaf_module'      => { help => ['May prevent mod_http2 from working -- see 8772327. May prevent .htaccess from denying access -- see 4422297.'] },
        'spdy_module'      => { help => ['May break proxy subdomains. See 4973361.'] },
    );
    my $add = sub {
        my ( $mod, $text, $check_missing ) = @_;
        push @{ $check{$mod}{'help'} }, $text;
        $check{$mod}{'check_missing'} = 1 if $check_missing;
    };

    $add->( 'http2_module',   'Causes segfaults in Apache 2.4.25, see EAL-3153.' )                                        if version_compare( $apache_version, qw ( == 2.4.25 ) );
    $add->( 'userdir_module', 'Does not work with passenger_module.' )                                                    if defined $installed_modules->{'passenger_module'};
    $add->( 'userdir_module', 'Does not work with ruid2_module.' )                                                        if defined $installed_modules->{'ruid2_module'};
    $add->( 'userdir_module', 'Does not work with mpm_itk_module.' )                                                      if defined $installed_modules->{'mpm_itk_module'};
    $add->( 'ruid2_module',   'Can cause file permission problems when using LiteSpeed Web Server (see ticket 5154193)' ) if $lsws_full_version;
    if ( i_am('ea4') ) {
        $add->( 'fcgid_module',   'Has many caveats, see https://documentation.cpanel.net/display/EA4/Apache+Module%3A+FCGId' );
        $add->( 'userdir_module', 'PHP scripts accessed by userdir will not be executed via PHP-FPM.' );
        my $ea4_php = get_installed_ea4_php_href();
        if ( defined($ea4_php) && defined( $ea4_php->{default} ) && defined( $ea4_php->{ $ea4_php->{default} }->{handler} ) && $ea4_php->{ $ea4_php->{default} }->{handler} eq "cgi" ) {
            $add->( 'userdir_module', 'Will not execute PHP scripts via CGI handler.' );
        }
    }
    if ( defined $installed_modules->{'security_module'} or defined $installed_modules->{'security2_module'} ) {
        $add->( 'mpm_itk_module', 'Incompatible with ModSecurity SecDataDir (collections). See case EA-4093.' );
        $add->( 'ruid2_module',   'Incompatible with ModSecurity SecDataDir (collections). See case EA-4093.' );
    }
    if ( defined $CPCONF{'jailapache'} && $CPCONF{'jailapache'} == 1 ) {
        $add->( 'ruid2_module', 'Enabled with Jail Apache Virtual Hosts tweak. This can break some Mailman URLs. See CPANEL-9501 and CPANEL-18127.' );
        my $ea3_php = get_ea3_php_conf_href();
        if ( defined $ea3_php and defined $ea3_php->{'php5handler'} and $ea3_php->{'php5handler'} eq 'suphp' ) {
            $add->( 'ruid2_module', 'Enabled with Jail Apache Virtual Hosts tweak and suPHP handler, these are NOT COMPATIBLE, see FB-70561, FB-105901.' );
        }
    }
    if ( i_am('cloudlinux') ) {
        $add->( 'mpm_itk_module', 'CloudLinux LVE memory limits not imposed on Apache processes, and not compatible with PHP Selector - https://docs.cloudlinux.com/index.html?compatiblity_matrix.html' );
        $add->( 'ruid2_module',   'CloudLinux LVE memory limits not imposed on Apache processes, and not compatible with PHP Selector - https://docs.cloudlinux.com/index.html?compatiblity_matrix.html' );
    }

    for my $module ( sort keys %check ) {
        my $help_text = join( "\n" . ' ' x ( length($module) + 23 ) . '\_ - ', @{ $check{$module}{'help'} } );
        if ( defined $check{$module}{'check_missing'} and not defined $installed_modules->{$module} ) {
            print_warn('Apache: ');
            print_warning( 'Missing ' . $module . ' - ' . $help_text );
        }
        if ( not defined $check{$module}{'check_missing'} and defined $installed_modules->{$module} ) {
            print_warn('Apache: ');
            print_warning( ' Loaded ' . $module . ' - ' . $help_text );
        }
    }
}

sub check_apache_niceness {
    return unless my $httpd_bin = find_httpd_bin();
    return unless my %procs     = grep_process_cmd( qr{ $httpd_bin \s+ \-k }xms, 'root' );
    my $apache_nice;
    my $apache_ionice;
    for my $pid ( sort keys %procs ) {
        $apache_nice   = $procs{$pid}->{'NICE'};
        $apache_ionice = timed_run( 0, 'ionice', '-p', $pid );
        chomp $apache_ionice;
        last;
    }
    my $cp20037_nice         = '18';
    my $cp20037_bw_ionice    = defined $CPCONF{'ionice_bandwidth_processing'} ? $CPCONF{'ionice_bandwidth_processing'} : '6';
    my $cp20037_log_ionice   = defined $CPCONF{'ionice_log_processing'}       ? $CPCONF{'ionice_log_processing'}       : '7';
    my $cp20037_ionice_regex = '\A best-effort: \s prio \s (?:' . $cp20037_bw_ionice . '|' . $cp20037_log_ionice . ') \Z';

    my $cp20037info = ' - See CPANEL-20037 for a possible cause.';    # Make the text conditional on build version after CPANEL-20037 is published

    if ($apache_nice) {                                               # Anything other than 0
        print_warn('Apache: ');
        print_warning( 'has unexpected nice value [ ' . $apache_nice . ' ] - May result in Apache performance issues' . ( $apache_nice eq $cp20037_nice ? $cp20037info : '' ) );
    }
    if ( $apache_ionice and not $apache_ionice =~ m{ \A (?:none|unknown): \s prio \s [04] \Z }xms ) {    # "none: prio 0", "unknown: prio 0", "none: prio 4", "unknown: prio 4" all acceptable
        print_warn('Apache: ');
        print_warning( 'has unexpected ionice value [ ' . $apache_ionice . ' ] - May result in Apache performance issues' . ( $apache_ionice =~ m{ $cp20037_ionice_regex }xms ? $cp20037info : '' ) );
    }
}

sub check_perl_sanity {
    return unless i_am('cpanel');
    my $usr_bin_perl       = '/usr/bin/perl';
    my $usr_local_bin_perl = '/usr/local/bin/perl';

    if ( !-e $usr_bin_perl ) {
        print_warn('perl: ');
        print_warning("$usr_bin_perl does not exist!");
    }

    if ( -l $usr_bin_perl and -l $usr_local_bin_perl ) {
        my $usr_bin_perl_link       = readlink $usr_bin_perl;
        my $usr_local_bin_perl_link = readlink $usr_local_bin_perl;
        if ( -l $usr_bin_perl_link and -l $usr_local_bin_perl_link ) {
            print_warn('perl: ');
            print_warning("$usr_bin_perl and $usr_local_bin_perl are both symlinks!");
        }
    }

    ## a symlink will test true for both -x AND -l
    if ( -x $usr_bin_perl and not -l $usr_bin_perl ) {
        if ( -x $usr_local_bin_perl and not -l $usr_local_bin_perl ) {
            print_warn('perl: ');
            print_warning("$usr_bin_perl and $usr_local_bin_perl are both binaries!");
        }
    }

    if ( -x $usr_bin_perl and not -l $usr_bin_perl ) {
        my $mode = ( stat($usr_bin_perl) )[2] & oct(7777);
        $mode = sprintf "%lo", $mode;
        if ( $mode != 755 ) {
            print_warn('Perl Permissions: ');
            print_warning("$usr_bin_perl is $mode");
        }
    }

    if ( -x $usr_local_bin_perl and not -l $usr_local_bin_perl ) {
        my $mode = ( stat($usr_local_bin_perl) )[2] & oct(7777);
        $mode = sprintf "%lo", $mode;
        if ( $mode != 755 ) {
            print_warn('Perl Permissions: ');
            print_warning("$usr_local_bin_perl is $mode");
        }
    }
}

sub check_for_non_default_permissions {
    my $timeout  = $OPT_TIMEOUT ? $OPT_TIMEOUT : 10;    # This only applies to the recursive loop.
    my $hostinfo = get_hostinfo_href();

    # Example: '/path' => { mode => ['0755','0555',...], user => 'root', group => 'root', perms_help => 'Additional info if mode/user/group incorrect', attr_check => [ 'IMMUTABLE' ], attr_recursive => 1, attr_help => 'Additional info if immutable/append-only/etc', symlink => '/path', symlink_no_absolute => 1, check_missing => 1 },
    # Attributes are always checked, mode is only checked if specified.
    # User is always checked if mode is specified, which defaults to 'root'.
    # A '*' can be used to specify any user or group is allowed.
    # Only symlink ownership can be verified, not its mode.
    # attr_recursive only works on directories, default is 0 (do not recurse).
    # attr_check is optional, default is to check all of IMMUTABLE, APPEND-ONLY, UNDELETABLE.
    # symlink_no_absolute defines whether the absolute target path of a symlink will be computed before comparing. Default behavior is to resolve the absolute target path.  Enabling this option allows you to compare a symlink at face-value.
    # check_missing causes a missing object to be reported
    # tidyoff
    my %check = (
        '/'                   => { mode => [ '0755', '0555' ], perms_help => '.ftpquota issues? see ticket 4429843', attr_help => 'This can break EA. See ticket 4929961' },
        '/bin/bash'           => { mode => ['0755'] },
        '/bin/df'             => { mode => ['0755'] },
        '/bin/gtar'           => { symlink => 'tar',    symlink_no_absolute => 1, perms_help => 'May prevent creating backups via cPanel UI if users can not use this.' },
        '/bin/gzip'           => { mode => ['755'],  perms_help          => 'May prevent creating backups via cPanel UI if users can not use this.' },
        '/bin/ln'             => { mode => [ '0755', '0555' ] },
        '/bin/rm'             => { mode => [ '0755', '0555' ], perms_help => 'File Manager unable to delete files?  This may be why.' },
        '/bin/tar'            => { mode => ['755'],  perms_help => 'May prevent creating backups via cPanel UI if users can not use this.' },
        '/dev'                => { mode => ['0755'], perms_help => 'Breaks many things if non-root users can\'t access this.' },
        '/dev/log'            => { mode => ['0666'], perms_help => 'CSF RESTRICT_SYSLOG can change this. See ticket 4875833. Non-root users may not be able to log to syslog, including user cron jobs to /var/log/cron.' },
        '/dev/null'           => { mode => ['0666'], perms_help => 'Breaks many things if non-root users can\'t write to this.' },
        '/dev/random'         => { mode => [ '0666', '0664', '0644', '0444' ], perms_help => 'Breaks many things if non-root users can\'t read this.' },
        '/dev/stderr'         => { symlink => '/proc/self/fd/2', symlink_no_absolute => 1, check_missing => 1 },
        '/dev/stdin'          => { symlink => '/proc/self/fd/0', symlink_no_absolute => 1, check_missing => 1 },
        '/dev/stdout'         => { symlink => '/proc/self/fd/1', symlink_no_absolute => 1, check_missing => 1 },
        '/dev/urandom'        => { mode => [ '0666', '0664', '0644', '0444' ], perms_help => 'Breaks many things if non-root users can\'t read this.' },
        '/etc'                => { mode => ['0755'] },
        '/etc/aliases'        => { mode => ['0644'] },
        '/etc/fstab'          => { mode => ['0644'], check_missing => 1, perms_help => 'Missing fstab can break /scripts/fixquotas (CPANEL-6082), and bad perms can break cPanel UI (CPANEL-11201).' },
        '/etc/group'          => { mode => ['0644'] },
        '/etc/hosts'          => { mode => ['0644'] },
        '/etc/localaliases'   => { mode => ['0644'] },
        '/etc/passwd'         => { mode => ['0644'] },
        '/etc/shadow'         => { mode => [ '0600', '0400', '0200', '0000' ] },
        '/etc/ssh/sshd_config'          => { mode => ['0600'], attr_check => [ 'IMMUTABLE' ] },
        '/root/.bash_history' => { mode => ['0600'], attr_check => [ 'IMMUTABLE', 'APPEND-ONLY' ], check_missing => 1 },
        '/opt'                => { mode => ['0755'] },
        '/proc'               => { mode => ['0555'], perms_help => 'Breaks quotas if we cannot read /proc/mounts, prevents setrtlimit(), is the server LXC/proxmox? See TECH-775' },
        '/root/.ssh/authorized_keys'    => { mode => ['0600'], attr_check => [ 'IMMUTABLE' ] },
        '/sbin/ifconfig'      => { mode => [ '0755', '0555' ] },
        '/tmp'                => { mode => ['1777'] },
        '/usr'                => { mode => ['0755'] },
        '/usr/bin'            => { mode => [ '0755', '0711', '0555' ] },
        '/usr/bin/crontab'    => { mode => [ '6755', '4755', '4711', '4555', '4511' ] },
        '/usr/bin/passwd'     => { mode => [ '6755', '4755', '4711', '4555', '4511' ] },
        '/usr/bin/screen'     => { mode => ['2755'], group => 'screen', perms_help => 'Screen doesn\'t work? Run "rpm --setugids screen && rpm --setperms screen" to fix.' },
        '/usr/local'          => { mode => ['0755'] },
        '/usr/local/bin'      => { mode => [ '0755', '0711', '0555' ] },
        '/usr/local/sbin'     => { mode => [ '0755', '0711', '0555' ] },
        '/usr/sbin'           => { mode => [ '0755', '0711', '0555' ] },
        '/usr/sbin/exim'      => { mode => ['4755'] },
        '/usr/share'          => { mode => ['0755'] },
        '/usr/share/zoneinfo' => { mode => ['0755'] },
        '/var'                => { mode => ['0755'] },
        '/var/lib'            => { mode => ['0755'] },
        '/var/lib/mysql'      => { mode => ['0751'], user => 'mysql', group => 'mysql' },
        '/var/lib/mysql/mysql.sock' => { mode => ['0777'], user => 'mysql', group => 'mysql' },
        '/var/log'            => { mode => [ '0755', '0751', '0711' ], perms_help => 'If non-root users cannot write to log files it can cause service failure' },
        '/bin/cagefs_enter.proxied' =>  { mode => ['4755'], perms_help => 'Issues with cPanel Terminal emulator on CloudLinux, see UPS-264' },

    );

    if ( os_version_is(qw( >= 8 ))) {
        $check{'/etc/nsswitch.conf'} = { symlink => '/etc/authselect/nsswitch.conf', check_missing => 1 };
        $check{'/etc/authselect/nsswitch.conf'} = { mode => ['0644'], check_missing => 1 };
    }
    else {
        $check{'/etc/nsswitch.conf'} = { mode => ['0644'], check_missing => 1 };
    }

    if ( i_am_one_of( 'cpanel', 'dnsonly' ) ) {
        %check = (
            %check,
            '/scripts'                                    => { symlink => '/usr/local/cpanel/scripts', check_missing => 1 },
            '/usr/local/cpanel'                           => { mode => ['0711'] },
            $CPANEL_LICENSE_FILE                          => { mode => ['0644'] },
            '/usr/local/cpanel/cpsanitycheck.so'          => { mode => ['0754'] },
            '/usr/local/cpanel/logs/cphulkd.log'          => { attr_check => [ 'IMMUTABLE', 'UNDELETABLE' ] },
            '/usr/local/cpanel/logs/cphulkd_errors.log'   => { attr_check => [ 'IMMUTABLE', 'UNDELETABLE' ] },
            '/usr/local/cpanel/logs/dnsadmin_log'         => { attr_check => [ 'IMMUTABLE', 'UNDELETABLE' ] },
            '/usr/local/cpanel/logs/error_log'            => { attr_check => [ 'IMMUTABLE', 'UNDELETABLE' ] },
            '/usr/local/cpanel/logs/queueprocd.log'       => { attr_check => [ 'IMMUTABLE', 'UNDELETABLE' ] },
            '/usr/local/cpanel/logs/tailwatchd_log'       => { attr_check => [ 'IMMUTABLE', 'UNDELETABLE' ] },
            '/var/cpanel/analytics/system_id'             => { attr_check => [ 'APPEND-ONLY', 'UNDELETABLE' ] },
            '/var/cpanel/config'                          => { mode => ['0755'] },
            $CPANEL_CONFIG_FILE                           => { mode => ['0644'] },
            '/var/cpanel/datastore'                       => { mode => [ '0755', '0711' ], perms_help => 'Users must be able to read some of the datastore contents for cPanel UI usage stats.' },
        );
    }

    if ( i_am('cpanel') ) {
        %check = (
            %check,
            '/bin/passwd'                                 => { symlink => '/usr/local/cpanel/bin/jail_safe_passwd' },
            '/etc/backupmxhosts'                          => { mode => ['0640'], group => 'mail' },
            '/etc/cpbackup.conf'                          => { mode => ['0644'] },
            '/etc/cpupdate.conf'                          => { mode => ['0644'], attr_check => [ 'IMMUTABLE' ], attr_help => 'Please tag the ticket to case TECH-853' },
            '/etc/dbowners'                               => { mode => ['0640'], group => 'mail' },
            '/etc/demodomains'                            => { mode => ['0640'], group => 'mail' },
            '/etc/demouids'                               => { mode => ['0640'], group => 'mail' },
            '/etc/demousers'                              => { mode => ['0640'], group => 'mail' },
            '/etc/domainusers'                            => { mode => ['0640'], group => 'mail' },
            '/etc/email_send_limits'                      => { mode => ['0640'], group => 'mail' },
            '/etc/exim.conf'                              => { mode => ['0644'] },
            '/etc/eximmailtrap'                           => { mode => ['0644'] },
            '/etc/eximrejects'                            => { mode => ['0644'] },
            '/etc/global_spamassassin_enable'             => { mode => ['0644'] },
            '/etc/greylist_common_mail_providers'         => { mode => ['0644'] },
            '/etc/greylist_trusted_netblocks'             => { mode => ['0640'], group => 'mail' },
            '/etc/localdomains'                           => { mode => ['0640'], group => 'mail', perms_help => 'Failing to properly create an email forwarder? See ticket 5234627.' },
            '/etc/mailbox_formats'                        => { mode => ['0640'], group => 'mail' },
            '/etc/mailhelo'                               => { mode => ['0640'], group => 'mail' },
            '/etc/mailips'                                => { mode => ['0640'], group => 'mail' },
            '/etc/neighbor_netblocks'                     => { mode => [ '0640', '0644' ], group => '*' },    # 644 root:root before first account created, 640 root:mail after.
            '/etc/outgoing_mail_hold_users'               => { mode => ['0640'], group => 'mail' },
            '/etc/outgoing_mail_suspended_users'          => { mode => ['0640'], group => 'mail' },
            '/etc/recent_authed_mail_ips'                 => { mode => ['0644'] },
            '/etc/recent_authed_mail_ips_users'           => { mode => ['0644'] },
            '/etc/recent_recipient_mail_server_ips'       => { mode => ['0640'], group => 'mail' },
            '/etc/remotedomains'                          => { mode => ['0644'], group => 'mail' },
            '/etc/secondarymx'                            => { mode => ['0640'], group => 'mail' },
            '/etc/senderverifybypasshosts'                => { mode => ['0640'], group => 'mail' },
            '/etc/skipsmtpcheckhosts'                     => { mode => ['0640'], group => 'mail' },
            '/etc/spammeripblocks'                        => { mode => ['0640'], group => 'mail' },
            '/etc/spammers'                               => { mode => ['0644'] },
            '/etc/stats.conf'                             => { mode => ['0644'] },
            '/etc/trueuserdomains'                        => { mode => ['0640'], group => 'mail' },
            '/etc/trusted_mail_users'                     => { mode => ['0640'], group => 'mail' },
            '/etc/trustedmailhosts'                       => { mode => ['0640'], group => 'mail' },
            '/etc/userdomains'                            => { mode => ['0640'], group => 'mail' },
            '/etc/valiases'                               => { mode => [ '0755', '0711' ] },
            '/etc/vdomainaliases'                         => { mode => ['0711'] },
            '/etc/vfilters'                               => { mode => [ '0755', '0711' ] },
            '/etc/webspam'                                => { mode => ['0644'] },
            '/home'                                       => { mode => [ '0755', '0711' ] },
            '/home1'                                      => { mode => [ '0755', '0711' ] },
            '/home2'                                      => { mode => [ '0755', '0711' ] },
            '/home3'                                      => { mode => [ '0755', '0711' ] },
            '/home4'                                      => { mode => [ '0755', '0711' ] },
            '/home5'                                      => { mode => [ '0755', '0711' ] },
            '/root/cpanel3-skel'                          => { mode => ['0755'] },
            '/usr/local/apache'                           => { mode => ['0755'], attr_recursive => 1 },
            '/usr/local/apache/conf'                      => { mode => ['0755'] },
            '/usr/local/cpanel/base/3rdparty/phpMyAdmin/index.php'                          => { mode => ['0644'] },
            '/usr/local/cpanel/base/3rdparty/phpPgAdmin/index.php'                          => { mode => ['0644'] },
            '/usr/local/cpanel/base/3rdparty/roundcube/index.php'                           => { mode => ['0644'] },
            '/usr/local/cpanel/base/3rdparty/roundcube/plugins/cpanellogin/cpanellogin.php' => { mode => ['0644'] },
            '/usr/local/cpanel/base/3rdparty/squirrelmail/index.php'                        => { mode => ['0644'] },
            '/usr/local/cpanel/base/horde/index.php'      => { mode => ['0644'] },
            '/usr/local/cpanel/bin/cpwrap'                => { mode => ['0755'] },
            '/usr/local/cpanel/bin/sendmail'              => { mode => ['0755'] },
            '/usr/local/cpanel/logs/cpdavd_error_log'     => { attr_check => [ 'IMMUTABLE',   'UNDELETABLE' ] },
            '/usr/local/cpanel/logs/cpdavd_error_log'     => { attr_check => [ 'IMMUTABLE',   'UNDELETABLE' ] },
            '/usr/local/cpanel/logs/spamd_error_log'      => { attr_check => [ 'IMMUTABLE',   'UNDELETABLE' ] },
            '/usr/local/cpanel/logs/stats_log'            => { attr_check => [ 'IMMUTABLE',   'UNDELETABLE' ] },
            '/usr/local/cpanel/php/cpanel.php'            => { mode => ['0644'] },
            '/var/cpanel'                                 => { mode => [ '0755', '0711', '0555' ], attr_recursive => 1 },
            '/var/cpanel/backups'                         => { mode => [ '0755', '0750' ] },
            '/var/cpanel/backups/config'                  => { mode => ['0644'] },
            '/var/cpanel/bandwidth.cache'                 => { mode => ['0711'] },
            '/var/cpanel/config/apache'                   => { mode => ['0755'] },
            '/var/cpanel/config/apache/port'              => { mode => ['0644'] },
            '/var/cpanel/config/email'                    => { mode => ['0755'] },
            '/var/cpanel/config/email/query_apache_for_nobody_senders' => { mode => ['0644'] },
            '/var/cpanel/config/email/trust_x_php_script' => { mode => ['0644'] },
            '/var/cpanel/domain_keys_private'             => { mode => ['0640'], group => 'wheel' },
            '/var/cpanel/email_send_limits'               => { mode => [ '0750', '0751' ] },    # 0750 before, 0751 after first account
            '/var/cpanel/features'                        => { mode => ['0755'] },
            '/var/cpanel/locale'                          => { mode => ['0755'] },
            '/var/cpanel/locale/themes'                   => { mode => ['0755'] },
            '/var/cpanel/resellers'                       => { mode => ['0644'] },
            '/var/cpanel/userhomes'                       => { mode => [ '0755', '0711' ] },
            '/var/cpanel/userhomes/cpanelroundcube'       => { mode => ['0711'], user => 'cpanelroundcube', group => 'cpanelroundcube' },
            '/var/cpanel/users'                           => { mode => ['0711'] },
            $CPANEL_CONFIG_FILE                           => { mode => ['0644'], user => 'root', group => '*', perms_help => 'Ensure this is readable by all users or it can cause cPanel UI issues.' },
            '/var/log/exim_mainlog'                       => { mode => ['0640'], user => 'mailnull', group => 'mail' },
            '/var/log/exim_paniclog'                      => { mode => ['0640'], user => 'mailnull', group => 'mail' },
            '/var/log/exim_rejectlog'                     => { mode => ['0640'], user => 'mailnull', group => 'mail' },
        );
    }
    # tidyon

    ## If EA4 is running we need to redefine some of the paths from %check as these paths have turned into symlinks.
    ## We also conditionally need to add the new paths that EA4 introduces.
    if ( i_am('ea4') ) {
        $check{'/etc'}                              = { mode    => ['0755'], perms_help => 'Users need execute permission for this directory to use cPanel MultiPHP. See CPANEL-905.' };
        $check{'/etc/apache2'}                      = { mode    => ['0755'] };
        $check{'/etc/apache2/conf'}                 = { mode    => ['0755'] };
        $check{'/etc/apache2/conf.d'}               = { mode    => ['0755'] };
        $check{'/etc/apache2/conf.modules.d'}       = { mode    => ['0755'] };
        $check{'/etc/apache2/logs'}                 = { symlink => '/var/log/apache2', check_missing => 1 };
        $check{'/etc/apache2/logs/domlogs'}         = { mode    => [ '0711', '0755' ], perms_help => 'If users can\'t access logs, stats won\'t process.  See ticket 5413079.' };
        $check{'/etc/apache2/logs/error_log'}       = { mode    => ['0644'], user => '*', perms_help => 'If users can\'t read error_log, cPanel Errors (Last 300) won\'t work.' };
        $check{'/etc/apache2/logs/suexec_log'}      = { mode    => ['0644'], user => '*', perms_help => 'If users can\'t read suexec_log, cPanel Errors (Last 300) won\'t work.' };
        $check{'/etc/apache2/run'}                  = { symlink => '../../var/run/apache2', symlink_no_absolute => 1, check_missing => 1 };
        $check{'/etc/cpanel'}                       = { mode    => [ '0755', '0751', '0711' ], perms_help => 'Users need execute permission for this directory to use cPanel MultiPHP.  See CPANEL-905.' };
        $check{'/etc/cpanel/ea4'}                   = { mode    => ['0755'], perms_help => 'Users need read/execute permission for this directory to use cPanel MultiPHP.  See CPANEL-905.' };
        $check{'/usr/local/apache/bin/apachectl'}   = { symlink => '/usr/sbin/apachectl',          check_missing => 1 };
        $check{'/usr/local/apache/bin/httpd'}       = { symlink => '/usr/sbin/httpd',              check_missing => 1 };
        $check{'/usr/local/apache/bin/suexec'}      = { symlink => '/usr/sbin/suexec',             check_missing => 1 };
        $check{'/usr/local/apache/conf/httpd.conf'} = { symlink => '/etc/apache2/conf/httpd.conf', check_missing => 1 };
        $check{'/usr/local/apache/conf/includes'}   = { symlink => '/etc/apache2/conf.d/includes', check_missing => 1 };
        $check{'/usr/local/apache/conf/mime.types'} = { symlink => '/etc/apache2/conf/mime.types', check_missing => 1 };
        $check{'/usr/local/apache/conf/php.conf'}   = { symlink => '/etc/apache2/conf.d/php.conf', check_missing => 1 };
        $check{'/usr/local/apache/domlogs'}         = { symlink => '/etc/apache2/logs/domlogs',    check_missing => 1 };
        $check{'/usr/local/apache/htdocs'}          = { symlink => '/var/www/html',                check_missing => 1 };
        $check{'/usr/local/apache/logs'}            = { symlink => '/etc/apache2/logs',            check_missing => 1 };
        $check{'/usr/local/apache/modules'}         = { symlink => '/etc/apache2/modules',         check_missing => 1 };
        $check{'/usr/sbin/httpd'}                   = { mode    => ['0755'] };
        $check{'/var/log/apache2'}                  = { mode    => ['0711'] };
        $check{'/var/www/html'}                     = { mode    => ['0755'] };

        if ( defined( $hostinfo->{'hardware'} ) && $hostinfo->{'hardware'} eq 'i386' ) {
            $check{'/etc/apache2/modules'} = { symlink => '/usr/lib/apache2/modules', check_missing => 1 };
        }
        else {
            $check{'/etc/apache2/modules'} = { symlink => '/usr/lib64/apache2/modules', check_missing => 1 };
        }
    }
    elsif ( i_am('ea3') ) {
        $check{'/usr/local/apache/bin/httpd'}      = { mode => ['0755'] };
        $check{'/usr/local/apache/domlogs'}        = { mode => ['0711'], perms_help => 'If users can\'t access logs, stats won\'t process.  See ticket 5413079.' };
        $check{'/usr/local/apache/htdocs'}         = { mode => ['0755'] };
        $check{'/usr/local/apache/logs/error_log'} = { mode => ['0644'], user => '*', perms_help => 'If users can\'t read error_log, cPanel Errors (Last 300) won\'t work.' };
    }

    if ( -l '/var/tmp' ) {
        $check{'/var/tmp'} = { symlink => '/tmp' };
    }
    else {
        $check{'/var/tmp'} = { mode => ['1777'] };
    }

    # Add defined homedir to the checklist
    my $wwwacctconf = '/etc/wwwacct.conf';
    if ( open my $wwwacctconf_fh, '<', $wwwacctconf ) {
        while (<$wwwacctconf_fh>) {
            if (m{ ^ HOMEDIR \s ([/\-_A-Za-z0-9]+) $ }x) {
                $check{$1} = { mode => [ '0755', '0711' ] };
                last;
            }
        }
        close $wwwacctconf_fh;
    }

    if ( os_version_is(qw( < 7 )) or i_am('amazon') ) {
        $check{'/bin'}     = { mode => [ '0755', '0711', '0555' ] };
        $check{'/sbin'}    = { mode => [ '0755', '0711', '0555' ] };
        $check{'/var/run'} = { mode => ['0755'], perms_help => 'Exim auth can fail if this is too restrictive. See ticket 7455551.' };
        if ( i_am('cpanel') ) {
            $check{'/bin/crontab'} = { symlink => '/usr/local/cpanel/bin/jail_safe_crontab' };
            $check{'/bin/passwd'}  = { symlink => '/usr/local/cpanel/bin/jail_safe_passwd' };
        }
    }
    if ( os_version_is(qw( >= 7 )) and not i_am('amazon') ) {    # https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/7/html/Migration_Planning_Guide/sect-Red_Hat_Enterprise_Linux-Migration_Planning_Guide-File_System_Layout.html
        $check{'/bin'}      = { symlink => '/usr/bin',   check_missing => 1 };
        $check{'/lib'}      = { symlink => '/usr/lib',   check_missing => 1 };
        $check{'/lib64'}    = { symlink => '/usr/lib64', check_missing => 1 };
        $check{'/run'}      = { mode    => ['0755'], perms_help => 'Exim auth can fail if this is too restrictive. See ticket 7455551.' };
        $check{'/sbin'}     = { symlink => '/usr/sbin', check_missing => 1 };
        $check{'/var/lock'} = { symlink => '/run/lock', check_missing => 1 };
        $check{'/var/run'}  = { symlink => '/run',      check_missing => 1 };
        if ( i_am('cpanel') ) {
            $check{'/usr/local/bin/crontab'} = { symlink => '/usr/local/cpanel/bin/jail_safe_crontab', check_missing => 1 };
            $check{'/usr/local/bin/passwd'}  = { symlink => '/usr/local/cpanel/bin/jail_safe_passwd',  check_missing => 1 };
        }
    }
    if ( defined $CPCONF{'skipawstats'} && $CPCONF{'skipawstats'} == 0 ) {
        $check{'/usr/local/cpanel/3rdparty/bin/awstats.pl'} = { mode => ['0755'] };
    }
    if ( defined $CPCONF{'skipwebalizer'} && $CPCONF{'skipwebalizer'} == 0 ) {
        $check{'/usr/local/cpanel/3rdparty/bin/webalizer_lang/english'} = { mode => ['0755'], user => 'bin', group => 'bin' };
    }
    if ( i_am('cloudlinux') ) {
        if ( os_version_is(qw( == 6 )) ) {
            $check{'/usr/bin/python'} = { mode => ['0755'], perms_help => 'If the Python binary is not executable by non-root users it can break CloudLinux functions in cPanel UI' };
        }
        if ( os_version_is(qw( == 7 )) ) {
            $check{'/usr/bin/python'}    = { symlink => '/usr/bin/python2',   check_missing => 1, perms_help => 'If the Python binary is not executable by non-root users it can break CloudLinux functions in cPanel UI' };
            $check{'/usr/bin/python2'}   = { symlink => '/usr/bin/python2.7', check_missing => 1, perms_help => 'If the Python binary is not executable by non-root users it can break CloudLinux functions in cPanel UI' };
            $check{'/usr/bin/python2.7'} = { mode    => ['0755'], perms_help => 'If the Python binary is not executable by non-root users it can break CloudLinux functions in cPanel UI' };
        }
        if ( os_version_is(qw( == 8 )) ) {
            $check{'/usr/bin/python2'}   = { symlink => '/usr/bin/python2.7', check_missing => 1, perms_help => 'If the Python binary is not executable by non-root users it can break CloudLinux functions in cPanel UI' };
            $check{'/usr/bin/python2.7'} = { mode    => ['0755'], perms_help => 'If the Python binary is not executable by non-root users it can break CloudLinux functions in cPanel UI' };
        }
    }

    if ( cpanel_version_is(qw( >= 11.77.0.0 )) ) {
        $check{'/etc/exim.conf.localopts'} = { mode => ['0644'], perms_help => 'If users can\'t read this file it can affect the Email Deliverability feature.' };
    }

    # Add configured backup path(s) to the checklist
    my $new_backup_conf = get_new_backup_conf_href();
    my $old_backup_conf = get_old_backup_conf_href();
    my @backupdirs;
    push @backupdirs, $new_backup_conf->{BACKUPDIR} if defined $new_backup_conf;
    push @backupdirs, $old_backup_conf->{BACKUPDIR} if defined $old_backup_conf;
    foreach my $backup_path (@backupdirs) {
        if ( defined $backup_path ) {
            my @backup_path_parts = split( '/', $backup_path );

            # the first element will be an empty string. We don't want to check the permissions of /.
            shift @backup_path_parts;
            my $backup_path_parent;
            foreach my $part (@backup_path_parts) {
                $backup_path_parent .= '/' . $part;
                $check{$backup_path_parent} = { mode => [ '0755', '0751', '0711' ], perms_help => "Ensure that all backup directories are traversable (+x) by all users. See CPANEL-4336." };
            }
        }
    }

    # Process the checklist
    for my $path ( sort keys %check ) {
        my ( $mode, $uid, $gid ) = ( lstat($path) )[ 2, 4, 5 ];    # _ can now be used in place of $path for -d, -e, -f, -l, etc...
        if ( exists $check{$path}->{check_missing} ) {
            my $symlink    = '';
            my $perms_help = '';
            if ( exists $check{$path}->{symlink} && !-l _ ) {
                $symlink = exists $check{$path}->{symlink} ? ' symlink to ' . $check{$path}->{symlink} : '';
            }
            if ( $symlink || !-e _ ) {
                $perms_help = exists $check{$path}->{perms_help} ? ' - ' . $check{$path}->{perms_help} : '';
                if ( -d _ ) {
                    $perms_help .= " - $path currently exists as a directory. An RPM that owns this path may fail to install.  See EA-7912.";
                }
                elsif ( -f _ ) {
                    $perms_help .= " - $path currently exists as a file.";
                }
                elsif ( -e _ ) {
                    $perms_help .= " - $path currently exists and is not a file or directory.";
                }
                print_warn('Missing: ');
                print_warning( $path . $symlink . $perms_help );
            }
        }
        if ( -l _ ) {
            my $linktarget = readlink($path);
            if ( !exists $check{$path}{'symlink_no_absolute'} || $check{$path}{'symlink_no_absolute'} == 0 ) {
                if ( !( $linktarget =~ m{ \A / }x ) ) {    # If a symlink has a relative prefix, try to prepend the base path and convert to absolute
                    my @basepath = split( m{/}, $path );
                    $linktarget = abs_path( join( '/', @basepath[ 0 .. $#basepath - 1 ] ) . '/' . $linktarget );
                }
                elsif ( $linktarget =~ m{ / .{1,2} / }x ) {    # If a symlink is relative, convert to absolute
                    $linktarget = defined( abs_path($linktarget) ) ? abs_path($linktarget) : $linktarget;
                }
            }
            if ( !exists $check{$path}{'symlink'} || $linktarget ne $check{$path}{'symlink'} ) {
                my $perms_help = exists $check{$path}->{perms_help} ? ' - ' . $check{$path}->{perms_help} : '';
                print_warn('Non-default symlink: ');
                print_warning( '[ ' . $path . ' -> ' . $linktarget . ' ] ( default -> ' . ( exists $check{$path}{'symlink'} ? $check{$path}{'symlink'} : 'no symlink' ) . ' )' . $perms_help );
            }
        }
        elsif ( -e _ ) {
            if ( exists $check{$path}->{mode} ) {
                $mode &= oct(7777);
                $mode = sprintf "%lo", $mode;
                my $user = getpwuid($uid);
                $user = $user || $uid;
                my $group = getgrgid($gid);
                $group = $group || $gid;
                my $checkuser  = defined( $check{$path}->{user} )   ? $check{$path}->{user}               : 'root';
                my $checkgroup = defined( $check{$path}->{group} )  ? $check{$path}->{group}              : '*';
                my $perms_help = exists $check{$path}->{perms_help} ? ' - ' . $check{$path}->{perms_help} : '';

                if ( !( $checkuser eq '*' || $user eq $checkuser ) || !( $checkgroup eq '*' || $group eq $checkgroup ) ) {
                    print_warn('Non-default Perms: ');
                    print_warning( $path . ' [owner ' . $user . ':' . $group . '] (default ' . $checkuser . ':' . $checkgroup . ')' . $perms_help );
                }
                my $default = 0;
                for my $checkmode ( @{ $check{$path}->{mode} } ) {
                    if ( $mode == $checkmode ) {
                        $default = 1;
                    }
                }
                if ( !$default == 1 ) {
                    print_warn('Non-default Perms: ');
                    print_warning( $path . ' [mode ' . sprintf( "%04d", $mode ) . '] (default ' . join( ' or ', @{ $check{$path}->{mode} } ) . ')' . $perms_help );
                }
            }
            my @attr_check             = exists $check{$path}->{attr_check}     ? @{ $check{$path}->{attr_check} }   : qw( APPEND-ONLY IMMUTABLE UNDELETABLE );
            my $attr_help              = exists $check{$path}->{attr_help}      ? ' - ' . $check{$path}->{attr_help} : '';
            my $is_recursive           = exists $check{$path}->{attr_recursive} ? $check{$path}->{attr_recursive}    : 0;
            my $recursive_report_limit = 15;
            my %recursive_report_count_by_path;
            my $test_ref = sub {
                my $linktarget;
                if ($is_recursive) {
                    return if substr( $_, -10 ) eq "quota.user";
                    $linktarget = -l $_ ? readlink($_) : "";
                    return                                     if exists $check{$_} && !exists $check{$_}->{attr_recursive};           # If we check a path directly then we don't need to check it in a recursive sweep.
                    return                                     if exists $check{$linktarget};                                          # Don't check a link target that we've explicitly checked.
                    $recursive_report_count_by_path{$path} = 0 if !defined( $recursive_report_count_by_path{$path} );
                    return                                     if $recursive_report_count_by_path{$path} >= $recursive_report_limit;
                }
                if ( ( -f $_ || -d _ ) && ( my %attr = get_attributes( $_, @attr_check ) ) ) {
                    my $attributes = join( ' & ', keys(%attr) );
                    my $linktext   = $linktarget ? " -> " . $linktarget : "";
                    print_warn('Non-default Perms: ');
                    print_warning( $_ . $linktext . ' [' . $attributes . '] ' . $attr_help );
                    $recursive_report_count_by_path{$path}++;
                    if ( $recursive_report_count_by_path{$path} >= $recursive_report_limit ) {
                        print_warn('Non-default Perms: ');
                        print_warning( 'recursive reporting limit reached for ' . $path . ' -- there may be more files like this!' );
                    }
                }
            };
            if ($is_recursive) {
                eval {
                    local $SIG{'ALRM'} = sub {
                        print_warn('Non-default Perms: ');
                        print_warning( 'recursive check of ' . $path . ' timed out after ' . $timeout . ' seconds.' );
                        die;
                    };
                    alarm $timeout;
                    find( { wanted => $test_ref, no_chdir => 1 }, $path );
                    alarm 0;
                };
            }
            else {
                local $_ = $path;
                &$test_ref();
            }
        }
    }
}

sub check_for_non_default_file_capabilities {

    $YUM_OR_DNF = os_version_is(qw( >= 8)) ? 'dnf' : 'yum';

    # Check for at least these capabilities, more is OK
    # Example: '/path' => { cap => ['cap_setgid','cap_setuid+ep'], help => 'Some help text' },
    # tidyoff
    my %check = (
    );
    # tidyon

    if ( i_am('ea4') and not i_am('cloudlinux') ) {
        $check{'/usr/sbin/suexec'} = { cap => [ 'cap_setgid', 'cap_setuid', '+ep' ], help => 'Will break cpanel/webmail redirects and other suexec usage, "' . $YUM_OR_DNF . ' reinstall ea-apache24" to fix' };
    }

    for my $path ( sort keys %check ) {
        next unless -e $path;
        my @result = get_fcap($path);
        next unless scalar @result;
        my @missing;
        foreach my $cap ( @{ $check{$path}->{cap} } ) {
            push @missing, $cap unless grep { /^\Q${cap}\E$/ } @result;
        }
        if ( scalar @missing ) {
            my $help = exists $check{$path}->{help} ? ' - ' . $check{$path}->{help} : '';
            print_warn('Non-default capabilities: ');
            print_warning( $path . ' is missing ' . join( ',', @missing ) . $help );
        }
    }
}

sub get_attributes {

    # @want is optional, not specifying anything returns all checked attributes
    my ( $path, @want ) = @_;
    open( my $fh, '<', $path ) or return;
    my %attributes = (
        'APPEND-ONLY' => 0x00000020,    # FS_APPEND_FL in linux/fs.h
        'IMMUTABLE'   => 0x00000010,    # FS_IMMUTABLE_FL in linux/fs.h
        'UNDELETABLE' => 0x00000002,    # FS_UNRM_FL in linux/fs.h
    );
    my $FS_IOC_GETFLAGS = 0x80086601;    # Tested on CentOS 6.7 and 7.2 using: strace -e trace=ioctl -e raw=ioctl lsattr -d /
    my $flags = pack 'i', 0;
    return unless defined ioctl( $fh, $FS_IOC_GETFLAGS, $flags );
    close $fh;
    $flags = unpack 'i', $flags;
    @want  = keys(%attributes) if !scalar @want;
    my %result;

    foreach my $attr (@want) {
        next unless exists $attributes{$attr};
        $result{$attr} = 1 if $flags & $attributes{$attr};
    }
    return %result;
}

sub get_fcap {
    my ($path) = @_;
    my $getcap = '/usr/sbin/getcap';
    return unless -x $getcap;
    return unless -e $path;
    my $output = timed_run( 0, $getcap, $path );
    my @result;
    if ( $output =~ m/^$path = ([^+]*)(\+.*)?$/ ) {
        @result = split( /,/, $1 );
        push @result, $2 if defined $2;
    }
    @result = ('none') unless scalar @result;
    return @result;
}

sub check_for_non_default_sysctl {
    my $sysctl = { map { split( /\s=\s/, $_, 2 ) } split( /\n/, timed_run( 0, 'sysctl', '-a' ) ) };

    my %check = (    # 'sysctl_key' => [ ['default1', 'default2', ...], 'Additional info' ]
        'fs.enforce_symlinksifowner'    => [ [ '0', '1' ], ' - Invalid setting - https://docs.cloudlinux.com/index.html?symlink_owner_match_protection.html' ],
        'fs.protected_hardlinks_create' => [ ['0'],        '' ],
        'fs.protected_symlinks_create'  => [ ['0'],        ' - Not recommended. Can prevent creation of access_log symlink in user homes, or switching to a custom cPanel style that is not owned by the "linksafe" group.' ],
        'kernel.user_ptrace'            => [ ['1'],        '' ],
        'net.ipv4.tcp_tw_recycle'       => [ ['0'],        ' - This should generally never be enabled. Clients behind NAT or Proxy can have problems connecting to this server.' ],
        'net.ipv4.tcp_tw_reuse'         => [ ['0'],        ' - This should generally never be enabled. Clients behind NAT or Proxy can have problems connecting to this server.' ],
        'vm.overcommit_memory'          => [ [ '0', '1' ], ' - Seeing "Out of memory" but there is free memory and no limits?  This might be why.' ],
    );
    if ( defined( $sysctl->{'fs.enforce_symlinksifowner'} && $sysctl->{'fs.enforce_symlinksifowner'} ) ) {
        $check{'fs.symlinkown_gid'} = [ [ '99', '65534' ], ' - Incorrect GID - https://docs.cloudlinux.com/index.html?symlink_owner_match_protection.html' ];
    }
    for my $key ( sort keys %check ) {
        if ( exists $sysctl->{$key} ) {
            my $default = 0;
            for my $checksysctl ( @{ $check{$key}[0] } ) {
                if ( $sysctl->{$key} eq $checksysctl ) {
                    $default = 1;
                }
            }
            if ( !$default == 1 ) {
                print_warn('Non-default sysctl: ');
                print_warning( "$key = $sysctl->{$key} (default: " . join( ' or ', @{ $check{$key}[0] } ) . ")$check{$key}[1]" );
            }
        }
    }
}

sub check_for_stale_lockfiles {
    my %check = (    # '/path' => [ 'type', 'Additional info' ]; # type is pid, fcntl, touch, etc., for future use.
        '/etc/digestshadow.lock' => [ 'fcntl', 'Can prevent modifying system digestshadow file, check if active with lsof and MOVE ONLY IF STALE.' ],
        '/etc/group.lock'        => [ 'fcntl', 'Can prevent modifying system group file, check if active with lsof and MOVE ONLY IF STALE.' ],
        '/etc/gshadow.lock'      => [ 'fcntl', 'Can prevent modifying system gshadow file, check if active with lsof and MOVE ONLY IF STALE.' ],
        '/etc/gtmp'              => [ 'touch', 'Can prevent modifying system group file, check if active with lsof and MOVE ONLY IF STALE.' ],
        '/etc/passwd.lock'       => [ 'fcntl', 'Can prevent modifying system passwd file, check if active with lsof and MOVE ONLY IF STALE.' ],
        '/etc/ptmp'              => [ 'touch', 'Can prevent modifying system passwd file, check if active with lsof and MOVE ONLY IF STALE. See ticket 5315853.' ],
        '/etc/shadow.lock'       => [ 'fcntl', 'Can prevent modifying system shadow file, check if active with lsof and MOVE ONLY IF STALE.' ]
    );

    for my $resource ( sort keys %check ) {
        if ( -e $resource ) {
            print_warn('Lockfile exists: ');
            print_warning( $resource . ' (' . $check{$resource}[1] . ')' );
        }
    }
}

sub check_var_cpanel_users {
    return unless i_am('cpanel');
    my $var_cpanel_users = '/var/cpanel/users';
    return unless -d $var_cpanel_users;
    return unless opendir( my $dir_fh, $var_cpanel_users );
    if ( -e "/var/cpanel/users/nobody" ) {
        print_warn('nobody user ');
        print_warning("was found within the /var/cpanel/users/ directory. - Causes many issues");
    }
    my @files = grep { !m/^(?:\.\.?|root|system|nobody|^cptkt\w{11})$/ } readdir $dir_fh;
    closedir $dir_fh;

    my $group_root_files;
    for my $file (@files) {
        next if ( $file !~ /^[a-z0-9]+$/ );
        my $gid = ( stat( '/var/cpanel/users/' . $file ) )[5];
        if ( $gid == 0 ) {
            $group_root_files .= " $file";
        }
    }

    if ($group_root_files) {
        print_warn('/v/c/users file(s) owned by group "root": ');
        print_warning($group_root_files);
    }

    # No need to continue if no users
    return unless scalar @files;

    my $userdatadomains = '/etc/userdatadomains';
    if ( !-e $userdatadomains ) {
        print_warn('Missing file: ');
        print_warning("$userdatadomains (new server with no accounts, perhaps)");
    }
    elsif ( -f $userdatadomains and -z $userdatadomains ) {
        print_warn('Empty file: ');
        print_warning("$userdatadomains (generate it with /scripts/updateuserdatacache --force)");
    }

    if ( license_file_is_solo() and scalar @files > 1 ) {
        print_crit('License: ');
        print_critical('Solo, too many accounts, please escalate this ticket to L3 for further review!');
    }
}

sub check_root_suspended {
    return unless i_am('cpanel');
    if ( -e '/var/cpanel/suspended/root' ) {
        print_warn('root suspended: ');
        print_warning('the root account is suspended! Unsuspend it to avoid problems.');
    }
}

sub check_limitsconf {
    my @limitsconf;

    if ( open my $limitsconf_fh, '<', '/etc/security/limits.conf' ) {
        while (<$limitsconf_fh>) {
            push @limitsconf, $_;
        }
        close $limitsconf_fh;
    }

    @limitsconf = grep { !/^(\s+|#)/ } @limitsconf;

    if (@limitsconf) {
        print_warn('/etc/security/limits.conf: ');
        print_warning('customizations found. DON\'T move/alter! Seeing "Unable to set uids"? See CronUnableToSetUID article, FB-76597.');
    }
}

sub check_themesconf {
    my $themesconf = '/var/cpanel/themes.conf';
    return unless ( -s $themesconf );
    print_warn('/var/cpanel/themes.conf: ');
    print_warning('limits the themes users can select in WHM/cPanel, see TECH-303');
}

sub check_disk_space {
    my @df = split /\n/, timed_run( 0, 'df' );
    for my $line (@df) {
        if ( $line =~ m{ (9[8-9]|100)% \s+ (.*) }xms ) {
            my ( $usage, $partition ) = ( $1, $2 );
            next if $partition =~ m{ / ( virtfs | ( dev | proc | optimumcache ) \Z ) }xms;
            print_warn('Disk space: ');
            print_warning( $usage . '% usage on ' . $partition );
        }
    }
}

sub check_disk_inodes {
    my @df_i = split /\n/, timed_run( 0, 'df', '-i' );
    for my $line (@df_i) {
        if ( $line =~ m{ (9[8-9]|100)% \s+ (.*) }xms ) {
            my ( $usage, $partition ) = ( $1, $2 );
            unless ( $line =~ m{ / (virtfs|dev|proc|optimumcache) \z }xms ) {
                print_warn('Disk inodes: ');
                print_warning("${usage}% inode usage on $partition");
            }
        }
    }
}

sub check_for_hooks_in_scripts_directory {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    if ( -f '/usr/local/cpanel/Cpanel/CustomEventHandler.pm' ) {
        print_warn('Hooks: ');
        print_warning('/usr/local/cpanel/Cpanel/CustomEventHandler.pm exists!');
    }

    my @hooks;
    if ( -d '/scripts' ) {
        opendir( my $scripts_fh, '/scripts' );
        @hooks = sort grep { /^(pre|post)/ } readdir $scripts_fh;
        closedir $scripts_fh;
    }

    # these exist by default
    @hooks = grep { !/postsuexecinstall/ && !/post_sync_cleanup/ } @hooks;

    # CloudLinux stuff
    @hooks = grep { !/\.l\.v\.e-manager\.bak/ } @hooks;

    # EA3 stuff checked elsewhere
    @hooks = grep { !/easyapache/ } @hooks;

    my @custom_hooks;
    my $comment_pattern = '\s*(?:\#.*)?';
    my $cl_post_pattern = '(?:\.(?:pl|py|sh))?(?:\s+"\$@")?';

    for my $hook (@hooks) {
        my $hook_file = '/scripts/' . $hook;
        next if -z $hook_file;

        if ( open my $hook_fd, '<', $hook_file ) {    ## no critic (BriefOpen)
            my $is_custom;
            my $first_line = readline $hook_fd;
            if ( $first_line and $first_line !~ m{ \A #!/bin/bash $comment_pattern \Z }xms ) {
                $is_custom = 1;
            }
            while (<$hook_fd>) {
                next if m{ \A $comment_pattern \Z }xms;

                if ( i_am('cloudlinux') ) {
                    next if m{ \A
                       (
                         \Q/usr/share/cagefs/cpanel/cagefs_\E $hook \Q_hook\E $cl_post_pattern
                       | \Q/usr/share/l.v.e-manager/cpanel/hooks/\E (lve|l\.v\.e-) \Qmanager_\E $hook \Q_hook\E $cl_post_pattern
                       | \Q/usr/share/lve/dbgovernor/cpanel/upgrade-mysql-disabler.sh\E
                       )
                       $comment_pattern
                       \Z }xms;
                    if ( $hook eq 'postrestoreacct' ) {
                        next if m{ \A
                        (
                          \Q/usr/share/cagefs/cpanel/cagefs_postwwwacct_hook.pl\E \s+ user \s+ \"\$1\"
                        ) $comment_pattern
                        \Z }xms;
                    }
                    if ( $hook eq 'postkillacct' ) {
                        next if m{ \A
                        (
                          \Q/usr/share/lve/dbgovernor/cpanel/upgrade-mysql-disabler.sh\E
                        ) $comment_pattern
                        \Z }xms;
                    }
                    if ( $hook eq 'postupcp' ) {
                        next if m{ \A
                        (
                          \Q/usr/share/cloudlinux-linksafe/cpanel/hooks/cloudlinux_linksafe_hook.sh\E
                        | \Q/usr/share/lve/dbgovernor/cpanel/upgrade-mysql-disabler.sh\E
                        ) $comment_pattern
                        \Z }xms;
                    }
                    if ( $hook eq 'prekillacct' ) {
                        next if m{ \A
                        (
                          \Q/usr/share/cagefs-plugins/hooks/terminate_cagefs_account\E $cl_post_pattern
                        ) $comment_pattern
                        \Z }xms;
                    }
                }
                push @custom_hooks, $hook_file;
                last;
            }
            close $hook_fd;
        }
    }

    if ( scalar @custom_hooks ) {
        print_warn('Custom Hooks: ');
        print_warning( join( ' ', @custom_hooks ) );
    }
}

sub check_for_huge_logs {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );

    # Default size is 2_100_000_000, with no additional help text.
    # Example: '/path/to/file' => { info_size => 500_000_000, warn_size => 1_000_000_000, help => 'If file is too hyooge, THERE IS MUCH FAIL' }
    my %logs = (
        '/usr/local/apache/logs/access_log'          => {},
        '/usr/local/apache/logs/error_log'           => {},
        '/usr/local/apache/logs/mod_jk.log'          => {},
        '/usr/local/apache/logs/modsec_audit.log'    => {},
        '/usr/local/apache/logs/modsec_debug.log'    => {},
        '/usr/local/apache/logs/suexec_log'          => {},
        '/usr/local/apache/logs/suphp_log'           => {},
        '/var/cpanel/secdatadir/ip.pag'              => { help => 'Apache using a lot of CPU for no good reason? Try moving aside and restarting Apache. See EA-4092.' },
        '/var/cpanel/secdatadir/default_SESSION.pag' => { help => 'Apache using a lot of CPU for no good reason? Try moving aside and restarting Apache. See UPS-134.' },
        '/var/named/data/named.run'                  => {},
        '/var/cpanel/backups/metadata.sqlite'        => { info_size => 1_000_000_000, warn_size => 10_000_000_000 },
        '/var/cpanel/eximstats_db.sqlite3'           => { warn_size => 5_000_000_000 },
    );
    for my $log ( keys(%logs) ) {
        if ( -e $log ) {
            my $size       = ( stat($log) )[7];
            my $info_size  = exists( $logs{$log}->{info_size} ) ? $logs{$log}->{info_size}    : undef;
            my $warn_size  = exists( $logs{$log}->{warn_size} ) ? $logs{$log}->{warn_size}    : 2_100_000_000;
            my $help       = exists( $logs{$log}->{help} )      ? ' - ' . $logs{$log}->{help} : '';
            my $print_size = sprintf( "%0.2fGB", $size / 1073741824 );
            if ( $size > $warn_size ) {
                print_warn('M-M-M-MONSTER LOG!: ');
                print_warning( $log . ' (' . $print_size . ')' . $help );
            }
            elsif ( $info_size and ( $size > $info_size ) ) {
                print_info('Large Log: ');
                print_normal( $log . ' (' . $print_size . ')' . $help );
            }
        }
    }
}

sub check_easy_skip_cpanelsync {
    if ( -e '/var/cpanel/easy_skip_cpanelsync' ) {
        print_warn('Touchfile: ');
        print_warning('/var/cpanel/easy_skip_cpanelsync exists! ');
    }
}

sub check_pkgacct_override {
    if ( -d '/var/cpanel/lib/Whostmgr' ) {
        print_warn('pkgacct override: ');
        print_warning(' /var/cpanel/lib/Whostmgr exists, override may exist');
    }
}

sub check_for_yunsuo {
    return unless exists_process_cmd( qr{ yunsuo_agent }xms, 'root' );
    print_warn('yunsuo_agent process found: ');
    print_warning('Can break account creation/termination. See tickets 11476733, 7012047, and 7525481');
}

sub check_for_gdm {
    return unless exists_process_cmd( qr{ gdm }xms, 'root' );
    print_warn('gdm Process: ');
    print_warning('is running');
}

sub check_for_redhat_firewall {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    if ( timed_run( 0, 'iptables', '-L', 'RH-Firewall-1-INPUT' ) ) {
        print_warn('Default Redhat Firewall Check: ');
        print_warning('RH-Firewall-1-INPUT table exists. /scripts/configure_rh_firewall_for_cpanel to open ports.');
    }
}

sub check_easyapache {
    return unless i_am('cpanel');
    my $ea_is_running_file       = '/usr/local/apache/AN_EASYAPACHE_BUILD_IS_CURRENTLY_RUNNING';
    my $apache_update_no_restart = '/var/cpanel/mgmt_queue/apache_update_no_restart';
    my $ea_is_running            = 0;

    if ( -e $ea_is_running_file ) {
        if ( exists_process_cmd( qr{ easyapache }xms, 'root' ) ) {
            $ea_is_running = 1;
            print_warn('EA3: ');
            print_warning('is running');
        }
        else {
            print_warn('EA3: ');
            print_warning("$ea_is_running_file exists, but 'easyapache' not found in process list");
        }
    }

    if ( -e $apache_update_no_restart and not $ea_is_running ) {    # The touchfile can exist outside of legitimate EA3 usage, move to generic touchfile check after EA3 is gone.
        print_warn('Apache: ');
        print_warning("$apache_update_no_restart exists and EA3 does not appear to be running! This will prevent Apache restarts in some situations.");
    }
}

sub check_for_missing_ea3_php {
    return unless my $php = get_ea3_php_conf_href();
    return if $php->{'php5version_valid'};
    print_warn('EA3 to EA4 Migration: ');
    print_warning('\'/usr/bin/php -n -v\' did not return an expected version.  This will prevent PHP from being detected during an EA3 to EA4 migration. See EA-8109.');
}

sub check_for_ea3_hooks {
    return unless i_am('ea3') or ea3_downgrade_is_possible();

    my $hooks;

    my @hooks = qw(
      /scripts/after_apache_make_install
      /scripts/after_httpd_restart_tests
      /scripts/before_apache_make
      /scripts/before_httpd_restart_tests
      /scripts/posteasyapache
      /scripts/preeasyapache
    );

    # default CloudLinux hooks that can be ignored
    my %hooks_ignore = qw(
      24214790021e1df53a0a6e3741ca74c3  /scripts/before_apache_make
      2af1cea5d3eea8d837b719131ec6d67e  /scripts/before_apache_make
      407df66f28c8822cd4f51fe56160f74e  /scripts/before_apache_make
      41ec2d3f35d8cd7cb01b60485fb3bdbb  /scripts/before_apache_make
      16d94b5426681a977e2beedd0ad871e9  /scripts/posteasyapache
      e5e13640299ec439fb4c7f79a054e42b  /scripts/posteasyapache
    );

    for my $hook (@hooks) {
        if ( -f $hook and not -z $hook ) {
            chomp( my $checksum = timed_run( 0, 'md5sum', $hook ) );
            $checksum =~ s/\s.*//g;
            next if exists $hooks_ignore{$checksum};
            $hooks .= " $hook";
        }
    }

    if ($hooks) {
        print_warn('EA3 hooks: ');
        print_warning($hooks);
    }
}

sub check_mounts {
    my @mounts = split /\n/, timed_run( 0, 'mount' );
    return unless scalar @mounts;
    my $old_backup_conf = get_old_backup_conf_href();
    my $new_backup_conf = get_new_backup_conf_href();
    my $has_nfs         = 0;
    my $old_backups_dir;
    my $old_backups_dir_nfs;
    my $new_backups_dir;
    my $new_backups_dir_nfs;

    if (    defined $old_backup_conf
        and defined $old_backup_conf->{'BACKUPDIR'}
        and defined $old_backup_conf->{'BACKUPENABLE'}
        and $old_backup_conf->{'BACKUPENABLE'} =~ /yes/i ) {
        $old_backups_dir = $old_backup_conf->{'BACKUPDIR'};
        $old_backups_dir .= '/' unless substr( $old_backups_dir, -1 ) eq '/';
    }
    if (    defined $new_backup_conf
        and defined $new_backup_conf->{'BACKUPDIR'}
        and defined $new_backup_conf->{'BACKUPENABLE'}
        and $new_backup_conf->{'BACKUPENABLE'} =~ /yes/i ) {
        $new_backups_dir = $new_backup_conf->{'BACKUPDIR'};
        $new_backups_dir .= '/' unless substr( $new_backups_dir, -1 ) eq '/';
    }

    for my $mount (@mounts) {
        my $nfs_mount_path;
        if ( $mount =~ m{ on \s ([^\s]+) \s type \s nfs[34]? \s }xms ) {
            $nfs_mount_path = $1;
            $nfs_mount_path .= '/' unless substr( $nfs_mount_path, -1 ) eq '/';
            $has_nfs = 1;
            if ( $old_backups_dir and index( $old_backups_dir, $nfs_mount_path ) == 0 ) {
                $old_backups_dir_nfs = 1;
            }
            if ( $new_backups_dir and index( $new_backups_dir, $nfs_mount_path ) == 0 ) {
                $new_backups_dir_nfs = 1;
            }
        }
        if ( $mount =~ m{ \s on \s (/home([^\s]?)) \s (:?.*) noexec }xms ) {
            my $noexec_partition = $1;
            print_warn('mounted noexec: ');
            print_warning($noexec_partition);
        }
        if ( i_am('cptech') and $mount =~ m{ \s \Q$CPANEL_LICENSE_FILE\E \s }xms ) {
            print_generic_hack_predef('LICENSE');
            print_critical('The following mount entry was found:');
            print_critical( "\t" . $mount );
            print_critical("\tL3: Both CGLS and C.PRO are known to do this. See ticket 7790559 for reference.");
            print_critical();
        }
    }

    if ($has_nfs) {
        print_warn('NFS: ');
        print_warning('filesystem(s) with NFS detected.');
    }
    if ($old_backups_dir_nfs) {
        print_warn('Backups: ');
        print_warning("$old_backups_dir is NFS (used by old backup system)");
    }
    if ($new_backups_dir_nfs) {
        print_warn('Backups: ');
        print_warning("$new_backups_dir is NFS (used by new backup system)");
    }
}

## compare external IP addr with local IP addrs, OR
## check if only internal IP addrs are bound to server (this is not as reliable,
## as NAT can still be used with external IP addrs of course)
sub check_for_unsupported_nat {
    return if -e '/var/cpanel/cpnat';
    my @local_ipaddrs = @{ get_local_ipaddrs_aref() };
    my @external_ipaddrs;
    my $external_ip_address = get_external_ip();

    if ( defined($external_ip_address) ) {
        if ( !grep { /$external_ip_address/ } @local_ipaddrs ) {
            print_warn('NAT: ');
            print_warning("external IP address $external_ip_address is not bound to server and /var/cpanel/cpnat does not exist");
        }
        return;
    }

    for my $ipaddr (@local_ipaddrs) {

        # Matches any local IP (127.0.0.0/8) or RFC-1918 IP (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16
        if ( $ipaddr !~ /(?:10\.|127\.|172\.(?:1[6-9]|2[0-9]|3[0-1])\.|192\.168\.)/ ) {
            push @external_ipaddrs, $ipaddr;
        }
    }
    if ( !@external_ipaddrs ) {
        print_warn('NAT: ');
        print_warning('no external IP addresses detected');
    }
}

sub check_for_oracle_linux {
    my $centos_5_oracle_release_file = '/etc/enterprise-release';
    my $centos_6_oracle_release_file = '/etc/oracle-release';

    if ( -f $centos_5_oracle_release_file ) {
        print_warn('Oracle Linux: ');
        print_warning("$centos_5_oracle_release_file detected!");
    }
    elsif ( -f $centos_6_oracle_release_file ) {
        print_warn('Oracle Linux: ');
        print_warning("$centos_6_oracle_release_file detected!");
    }
}

sub check_for_usr_local_cpanel_hooks {
    my $hooks;
    my $dir = '/usr/local/cpanel/hooks';
    return unless -d $dir;

    my @usr_local_cpanel_hooks;    # items in /usr/local/cpanel/hooks/
    find(
        sub {
            my $file = $File::Find::name;
            if ( -f $file and $file !~ m{ ( README | \.example ) \z }xms ) {
                $file =~ s#/usr/local/cpanel/hooks/##;
                push @usr_local_cpanel_hooks, $file;
            }
        },
        $dir
    );

    # default CloudLinux hooks that can be ignored
    my %hooks_ignore = qw(
      677da3bdd8fbd16d4b8917a9fe0f6f89    /usr/local/cpanel/hooks/addondomain/addaddondomain
      677da3bdd8fbd16d4b8917a9fe0f6f89    /usr/local/cpanel/hooks/addondomain/deladdondomain
      677da3bdd8fbd16d4b8917a9fe0f6f89    /usr/local/cpanel/hooks/subdomain/addsubdomain
      677da3bdd8fbd16d4b8917a9fe0f6f89    /usr/local/cpanel/hooks/subdomain/delsubdomain
      289b2b4c8b5293103def4557d3538060    /usr/local/cpanel/hooks/mysql/adduser
      289b2b4c8b5293103def4557d3538060    /usr/local/cpanel/hooks/mysql/deluser
    );

    for my $hook (@usr_local_cpanel_hooks) {
        my $tmp_hook = '/usr/local/cpanel/hooks/' . $hook;
        if ( -f $tmp_hook and not -z $tmp_hook ) {
            chomp( my $checksum = timed_run( 0, 'md5sum', $tmp_hook ) );
            $checksum =~ s/\s.*//g;
            next if exists $hooks_ignore{$checksum};
            $hooks .= "$hook ";
        }
    }

    if ($hooks) {
        print_warn("$dir: ");
        print_warning($hooks);
    }
}

sub check_for_sql_safe_mode {
    return unless i_am('ea3');
    return unless my $phpini = get_phpini_aref();
    if ( grep { m# \A (?:[ \t]+)? sql\.safe_mode \s* = \s* on #ixms } @$phpini ) {
        print_warn('/usr/local/lib/php.ini: ');
        print_warning('sql.safe_mode is enabled! This may break PHP SQL authentication.');
    }
}

sub get_mysql_full_version {
    return unless my $mysql_output = timed_run( 0, 'mysql', '-V' );
    chomp $mysql_output;
    return $mysql_output;
}

sub get_mysql_numeric_version {
    return unless my $version = get_mysql_full_version();
    my $numeric;
    if ( $version =~ m{Distrib \s* ([0-9A-Za-z.]+)}xms ) {
        $numeric = $1;
    }
    return $numeric;
}

sub get_mysql_datadir {
    my $datadir    = '/var/lib/mysql/';
    my $mysql_conf = get_mysql_conf_href();
    if ( defined $mysql_conf and defined $mysql_conf->{'mysqld'} and defined $mysql_conf->{'mysqld'}{'datadir'} ) {
        $datadir = $mysql_conf->{'mysqld'}{'datadir'}[1];
        if ( $datadir !~ m{ / \z }xms ) {
            $datadir .= '/';
        }
    }
    return $datadir;
}

sub check_mysqld_warnings_errors {
    foreach my $mysql_err ( grep { m{\[(?:err)}i } split( /\n/, timed_run_trap_stderr( 0, 'mysqld', '-u', 'mysql', '--help' ) ) ) {
        print_warn('MySQL config errors: ');
        print_warning($mysql_err);
    }
}

sub check_for_domain_forwarding {
    return unless i_am_one_of( 'ea4', 'ea3' );
    my $domainfwdip = '/var/cpanel/domainfwdip';

    if ( -f $domainfwdip and not -z $domainfwdip ) {
        print_warn('Domain Forwarding: ');
        print_warning("cat $domainfwdip to see what is being forwarded!");
    }
}

sub check_for_empty_apache_templates {
    return unless i_am_one_of( 'ea4', 'ea3' );
    return unless my $apache_version = get_apache_version();
    my $apache2_template_dir = '/var/cpanel/templates/apache2';
    if ( version_compare( $apache_version, qw( >= 2.4.0 ) ) ) {
        $apache2_template_dir = '/var/cpanel/templates/apache2_4';
    }

    my @dir_contents;
    my $empty_templates;

    if ( -d $apache2_template_dir ) {
        opendir( my $dir_fh, $apache2_template_dir );
        @dir_contents = grep { !/^\.\.?$/ } readdir $dir_fh;
        closedir $dir_fh;
    }

    if ( !@dir_contents ) {
        print_warn('Apache templates: ');
        print_warning("none found in $apache2_template_dir !");
    }
    else {
        for my $template (@dir_contents) {
            if ( -z "$apache2_template_dir/$template" ) {
                $empty_templates .= "$template ";
            }
        }
    }

    if ($empty_templates) {
        print_warn("Empty Apache templates in $apache2_template_dir (this can affect the ability to remove domains): ");
        print_warning("$empty_templates");
    }
}

sub check_for_empty_postgres_config {
    return if i_am('dnsonly');
    my $postgres_config = '/var/lib/pgsql/data/pg_hba.conf';
    if ( -f $postgres_config and -z $postgres_config ) {
        print_warn('Postgres config: ');
        print_warning("$postgres_config is empty (install via WHM >> Postgres Config)");
    }
}

sub check_for_empty_easyapache_profiles {
    return unless i_am('ea3');
    my $dir = '/var/cpanel/easy/apache/profile';
    return unless -d $dir;

    my @easyapache_templates;    # items in /var/cpanel/easy/apache/profile/
    find(
        sub {
            my $file = $File::Find::name;
            if ( -f $file and -z $file ) {
                $file =~ s#/var/cpanel/easy/apache/profile/##g;
                push @easyapache_templates, $file;
            }
        },
        $dir
    );
    if (@easyapache_templates) {
        print_warn("EA3 Empty template(s) in $dir: ");
        print_warning( join( ' ', @easyapache_templates ) );
    }
}

sub check_for_missing_timezone_from_phpini {
    return unless i_am('ea3');
    return unless my $phpini = get_phpini_aref();
    my $timezone;

    for my $line (@$phpini) {
        if ( $line =~ m{ \A date\.timezone (?:\s+)? = (?:\s+)? (?:["'])? ([^/"']+) / ([^/"']+) (?:["'])? (?:\s+)? \z }xms ) {
            $timezone = $1 . '/' . $2;
            last;
        }
    }

    if ($timezone) {
        my ( $tz1, $tz2 ) = split /\//, $timezone;
        my $path = '/usr/share/zoneinfo/' . $tz1 . '/' . $tz2;

        if ( !-f $path ) {
            print_warn("date.timezone from /usr/local/lib/php.ini: ");
            print_warning("$path not found!");
        }
    }
}

sub check_for_proc_mdstat_recovery {
    my $mdstat = '/proc/mdstat';

    my $recovery = 0;

    if ( open my $mdstat_fh, '<', $mdstat ) {
        while (<$mdstat_fh>) {
            if (/recovery/) {
                $recovery = 1;
                last;
            }
        }
        close $mdstat_fh;
    }

    if ( $recovery == 1 ) {
        print_warn('Software RAID recovery: ');
        print_warning("cat $mdstat to check the status");
    }
}

sub check_usr_local_cpanel_path_for_symlinks {
    my @dirs = qw(
      /usr
      /usr/local
      /usr/local/cpanel
    );

    for my $dir (@dirs) {
        if ( -l $dir ) {
            print_warn('Directory is a symlink: ');
            print_warning("$dir (this can cause Internal Server Errors for redirects like /cpanel, etc)");
        }
    }
}

sub check_for_additional_rpms {
    return unless my $rpms = get_rpm_href();

    my @additional_rpms = grep { /^(php-|kde-|psa-|clamav|clamd|rrdtool)|(http|apache|pear|sendmail)/ } keys( %{$rpms} );
    @additional_rpms = grep { !/httpd-tools|^cpanel-|alt-php|apache-tomcat-apis|^ea-|^libnghttp2/ } @additional_rpms;
    @additional_rpms = map  { get_printable_rpm_packages($_) } @additional_rpms;
    return unless @additional_rpms;
    print "\n";
    print_magenta('This is informational only. Unless these rpms directly relate to an issue, they can be ignored:');
    @additional_rpms = sort @additional_rpms;
    for my $rpm (@additional_rpms) {
        print_start('Additional RPM: ');
        print_warning($rpm);
    }
}

sub check_for_system_mem_below_required {
    my $meminfo = get_meminfo();

    # Calculations have a 64-96MB fudge factor for overhead
    return unless defined( $meminfo->{installed} ) && $meminfo->{installed} =~ /[0-9]+/;
    my $memtotal   = int( $meminfo->{installed} / 1024 );
    my $memmin     = 704;                                   # 768 - 64
    my $memmintext = "768MB";

    if ( os_version_is(qw( >= 7 )) ) {
        $memmin     = 928;                                  # 1024 - 96
        $memmintext = "1024MB";
    }

    if ( $memtotal < $memmin ) {
        print_warn('Memory: ');
        print_warning( "Server has less than ${memmintext} installed memory! [ " . format_meminfo( $meminfo->{installed} ) . " ]" );
    }

    #TECH-74 recommends >= 1024MB swap on systems with <= 1024MB RAM
    return if $memtotal > 928;
    my $swaptotal = 0;
    $swaptotal = int( $meminfo->{swapinstalled} / 1024 ) if defined( $meminfo->{swapinstalled} ) && $meminfo->{swapinstalled} =~ /[0-9]+/;
    my $swapmin     = 928;
    my $swapmintext = "1024MB";
    if ( $swaptotal < $swapmin ) {
        print_warn('Memory: ');
        print_warning( "Server has less than ${swapmintext} swap! [ " . format_meminfo( $meminfo->{swapinstalled} ) . " ]" );
    }
}

sub check_yum_conf {
    ( $YUM_OR_DNF, $YUM_CONF ) = os_version_is(qw( >= 8)) ? ( 'dnf', '/etc/dnf/dnf.conf' ) : ( 'yum', '/etc/yum.conf' );
    my $exclude_line_count = 0;
    my $exclude_kernel;
    my $exclude_wget;
    my $exclude_perl;
    my $distroverpkg_cloudlinux;
    my $plugins_enabled;    # Default is disabled, needs to be explicitly enabled.
    my $repo_gpgcheck_enabled;
    my $assumeyes_enabled;
    my $remove_leaf_only_enabled;
    my $exclude_ea;

    if ( !-e $YUM_CONF ) {
        print_warn( uc $YUM_OR_DNF . ': ' );
        print_warning( $YUM_CONF . ' is missing!' );
        return;
    }
    elsif ( -z $YUM_CONF ) {
        print_warn( uc $YUM_OR_DNF . ': ' );
        print_warning( $YUM_CONF . ' is empty!' );
        return;
    }

    if ( open my $file_fh, '<', $YUM_CONF ) {    ## no critic (BriefOpen)
        while (<$file_fh>) {
            if (m{ \A \s* exclude }xmsi) {
                $exclude_line_count++;
            }
            if (m{ \A  \s* exclude .* kernel }xmsi) {
                $exclude_kernel = 1;
            }
            if (m{ \A \s* exclude .* wget }xmsi) {
                $exclude_wget = 1;
            }
            if (m{ \A \s* exclude .* perl }xmsi) {
                $exclude_perl = 1;
            }
            if (m{ \A \s* exclude .* ea- }xmsi) {
                $exclude_ea = 1;
            }
            if (m{ \A \s* distroverpkg \s* = \s* cloudlinux-release }xmsi) {
                $distroverpkg_cloudlinux = 1;
            }
            if (m{ \A \s* plugins \s* = \s* 1 \s* \Z }xmsi) {
                $plugins_enabled = 1;
            }
            if (m{ \A \s* repo_gpgcheck \s* = \s* 1 \s* \Z }xmsi) {
                $repo_gpgcheck_enabled = 1;
            }
            if (m{ \A \s* assumeyes \s* = \s* 1 \s* \Z }xmsi) {
                $assumeyes_enabled = 1;
            }
            if (m{ \A \s* remove_leaf_only \s* = \s* 1 \s* \Z }xmsi) {
                $remove_leaf_only_enabled = 1;
            }
        }
        close $file_fh;
    }

    if ( $exclude_line_count > 1 ) {
        print_warn( uc $YUM_OR_DNF . ': ' );
        print_warning( $YUM_CONF . ' contains multiple "exclude" lines!' );
    }
    if ($exclude_kernel) {
        print_warn( uc $YUM_OR_DNF . ': ' );
        print_warning( $YUM_CONF . ' may be excluding kernel updates!' );
    }
    if ($exclude_wget) {
        print_warn( uc $YUM_OR_DNF . ': ' );
        print_warning( $YUM_CONF . ' may be excluding wget updates!' );
    }
    if ($exclude_perl) {
        print_warn( uc $YUM_OR_DNF . ': ' );
        print_warning( $YUM_CONF . ' may be excluding system perl updates!' );
    }
    if ($exclude_ea) {
        print_warn( uc $YUM_OR_DNF . ': ' );
        print_warning( $YUM_CONF . ' may be excluding ea-* updates!' );
    }
    if ($distroverpkg_cloudlinux) {
        print_warn( uc $YUM_OR_DNF . ': ' );
        print_warning( $YUM_CONF . ' has distroverpkg=cloudlinux-release set! This is known to cause issues with installing EA4, see ticket 7615739.' );
    }
    unless ($plugins_enabled) {
        print_warn( uc $YUM_OR_DNF . ': ' );
        print_warning( 'plugins=1 not found in ' . $YUM_CONF . '.  If plugins are disabled it can cause issues with RHEL/CloudLinux and EA4 updates.' );
    }
    if ($repo_gpgcheck_enabled) {
        print_warn( uc $YUM_OR_DNF . ': ' );
        print_critical( 'repo_gpgcheck=1 found in ' . $YUM_CONF . '.  GPG Armor not yet supported by RHEL, ' . $YUM_OR_DNF . ' update may fail with 404 error on repomd.xml.asc.' );
    }
    if ($assumeyes_enabled) {
        print_warn( uc $YUM_OR_DNF . ': ' );
        print_critical( 'assumeyes=1 found in ' . $YUM_CONF . '.  Be careful, ' . $YUM_OR_DNF . ' will NOT ask to proceed before performing an action.  Use "--assumeno" option to override.' );
    }
    if ($remove_leaf_only_enabled) {
        print_warn( uc $YUM_OR_DNF . ': ' );
        print_warning( 'remove_leaf_only=1 found in ' . $YUM_CONF . '.  This is known to cause issues with EA4 provisioning.' );
    }
}

sub check_yum_plugins {
    my $plugin_dir    = '/etc/yum/pluginconf.d/';
    my $univhooksfile = os_version_is(qw ( >= 8 )) ? "universal_hooks.conf" : "universal-hooks.conf";
    my %plugins       = (
        'ea4' => {
            $univhooksfile => {
                msg => 'This can break MultiPHP, see TECH-655',
            },
        },
        'cloudlinux' => {
            'rhnplugin.conf' => {
                msg => 'This breaks CloudLinux updates',
            },
        },
    );

    $YUM_OR_DNF = os_version_is(qw( >= 8)) ? 'dnf' : 'yum';
    if ( !-e $plugin_dir ) {
        print_warn( uc $YUM_OR_DNF . ': ' );
        print_warning( $plugin_dir . ' does not exist! This can break MultiPHP, see TECH-655' );
        return;
    }
    foreach my $type ( keys %plugins ) {
        next unless i_am($type);
        foreach my $plugin ( keys %{ $plugins{$type} } ) {
            if ( !-e $plugin_dir . $plugin || -z $plugin_dir . $plugin ) {
                print_warn( uc $YUM_OR_DNF . ': ' );
                print_warning( $plugin_dir . $plugin . ' is either empty or missing! ' . $plugins{$type}{$plugin}{'msg'} );
            }
            elsif ( open my $fh, '<', $plugin_dir . $plugin ) {
                while (<$fh>) {
                    $plugins{$type}{$plugin}{'enabled'} = 1 if (m/ \A \s* enabled \s* = \s* 1 \s* \Z /xmsi);
                }
                close($fh);
                if ( !$plugins{$type}{$plugin}{'enabled'} ) {
                    print_warn( uc $YUM_OR_DNF . ': ' );
                    print_warning( $plugin_dir . $plugin . ' is not enabled! ' . $plugins{$type}{$plugin}{'msg'} );
                }
            }
        }
    }
}

sub check_for_cpanel_files {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    my @files = qw(
      /usr/local/cpanel/cpanel
      /usr/local/cpanel/cpsrvd
    );

    for my $file (@files) {
        if ( !-e $file ) {
            print_warn('Critical file missing: ');
            print_warning("$file");
        }
    }
}

sub check_bash_history_for_certain_commands {
    my $bash_history     = '/root/.bash_history';
    my %history_commands = ();
    my $commands;

    if ( -l $bash_history ) {
        my $link = readlink $bash_history;
        print_warn("$bash_history: ");
        print_warning("is a symlink! Linked to $link");
    }
    elsif ( -f $bash_history ) {
        if ( open my $history_fh, '<', $bash_history ) {
            while (<$history_fh>) {
                if (/chattr/) {
                    $history_commands{'chattr'} = 1;
                }
                if (/chmod/) {
                    $history_commands{'chmod'} = 1;
                }
                if (/openssl(?:.*)\.tar/) {
                    $history_commands{'openssl*.tar'} = 1;
                }
            }
            close $history_fh;
        }
    }

    if (%history_commands) {
        while ( my ( $key, $value ) = each(%history_commands) ) {
            $commands .= "[$key] ";
        }

        print_warn("$bash_history commands found: ");
        print_warning($commands);
    }
}

sub get_cron_files {
    my @crons;
    my @cronlist = glob(
        q{
        /etc/cron.d/{.,}*
        /etc/cron.hourly/{.,}*
        /etc/cron.daily/{.,}*
        /etc/cron.weekly/{.,}*
        /etc/cron.monthly/{.,}*
        /etc/crontab
        /var/spool/cron/root
    }
    );

    my %cron_ignore = (    # These contain 'interesting' commands that should be ignored if they aren't going to cause problems
        '7440999604d3517ca235c7949f803ece' => '/etc/cron.daily/maldet',                  # Default
        'aede9174c0a1b0cf225165e204aa6fd8' => '/etc/cron.hourly/modsecparse.pl',         # Default
        '195ddc2ac97502c2a96cc758cb7c9097' => '/etc/cron.daily/freshclam',               # Default
        'c7a32553c9f6d3d16c07281cae8572e9' => '/etc/cron.daily/tmpwatch',                # Default
        '3ffb5926bb7533bb077e2ec37f767851' => '/etc/cron.daily/tmpwatch',                # modified to not remove symlinks
        'ff511360a325783a06f494a05b514689' => '/etc/cron.d/kill_orphaned_php-cron',      # standard CloudLinux script
        'f392372e36f56fb3e70e850a6bd1a550' => '/etc/cron.d/lvedbgovernor-utils-cron',    # standard CloudLinux script
    );

    for my $cron (@cronlist) {
        next if !( -f $cron || -l $cron );
        chomp( my $checksum = timed_run( 0, 'md5sum', $cron ) );
        $checksum =~ s/\s.*//g;
        next if ( $checksum && $cron_ignore{$checksum} );
        push @crons, $cron;
    }

    return \@crons;
}

sub check_roots_cron_for_certain_commands {
    my %missing = ();
    my %warning = ();
    return unless my $crons_aref = get_cron_files();

    for my $cron (@$crons_aref) {
        next
          if (
            $cron =~ m{ /(
            freshclam |
            kill_orphaned_php-cron |
            lvedbgovernor-utils-cron |
            makewhatis\.cron |
            maldet |
            man-db\.cron |
            modsecparse\.pl |
            rpm
            ) $ }x
          );    # Common false-positives to filter out entirely.
        if ( open my $cron_fh, '<', $cron ) {
            while (<$cron_fh>) {
                if (m{ \A [^#]+ /usr/local/cpanel/(bin/backup|scripts/upcp) \s }xms) {
                    $missing{$1} = 1;
                }
                if (m{ \A [^#]* (?:^|\s|\/)(rm|unlink|ch(?:mod|own|attr)|(?:p|s)?kill(?:all)?5?|tmpwatch|cpkeyclt)\s }x) {
                    $warning{$cron}{$1} = 1;
                }
            }
            close $cron_fh;
        }
    }

    if ( !exists $missing{'bin/backup'} ) {
        print_warn("cron: ");
        print_warning("cPanel backup cron is missing (/usr/local/cpanel/bin/backup)");
    }
    if ( !exists $missing{'scripts/upcp'} ) {
        print_warn("cron: ");
        print_warning("cPanel update cron is missing (/usr/local/cpanel/scripts/upcp)");
    }

    if (%warning) {
        for my $cron ( keys(%warning) ) {
            print_warn("cron: ");
            print_warning( $cron . " contains [ " . join( ' ', sort( keys( %{ $warning{$cron} } ) ) ) . " ]" );
        }
    }
}

sub check_for_missing_or_commented_customlog {
    return unless i_am_one_of( 'ea4', 'ea3' );
    return unless my $apache_version = get_apache_version();
    my $commented_templates;
    my $missing_customlog_templates;
    my $httpdconf = i_am('ea4') ? '/etc/apache2/conf/httpd.conf' : '/usr/local/apache/conf/httpd.conf';
    my $httpdconf_commented_customlog;
    my $httpdconf_customlog_exists;

    my $templates_dir = '/var/cpanel/templates/apache2';
    if ( version_compare( $apache_version, qw( >= 2.4.0 ) ) ) {
        $templates_dir = '/var/cpanel/templates/apache2_4';
    }

    my %templates = (
        'ea4_main.default'  => 0,
        'ea4_main.local'    => 0,
        'main.default'      => 0,
        'main.local'        => 0,
        'vhost.default'     => 0,
        'vhost.local'       => 0,
        'ssl_vhost.default' => 0,
        'ssl_vhost.local'   => 0,
    );

    for my $template ( keys %templates ) {
        my $template_full_path = $templates_dir . '/' . $template;
        if ( -f $template_full_path ) {
            if ( open my $template_fh, '<', $template_full_path ) {
                while (<$template_fh>) {
                    if (/#(?:\s+)?CustomLog\s/i) {
                        $commented_templates .= "$template_full_path ";
                        $templates{$template} = 1;
                        last;
                    }
                    elsif (/CustomLog\s/i) {
                        $templates{$template} = 1;
                    }
                }
                close $template_fh;
            }
        }
    }

    while ( my ( $template, $value ) = each(%templates) ) {
        if ( $value == 0 and -f "$templates_dir/$template" ) {
            $missing_customlog_templates .= "$templates_dir/$template ";
        }
    }

    if ( open my $httpdconf_fh, '<', $httpdconf ) {
        local $/ = undef;
        my $httpdconf_txt = readline($httpdconf_fh);
        close $httpdconf_fh;
        if ( $httpdconf_txt =~ m/\n[\t ]*#[\t ]*CustomLog\s/si ) {
            $httpdconf_commented_customlog = 1;
        }
        if ( $httpdconf_txt =~ m/\n[\t ]*CustomLog\s/si ) {
            $httpdconf_customlog_exists = 1;
        }
    }

    if ($httpdconf_commented_customlog) {
        $commented_templates .= ' httpd.conf';
    }
    elsif ( !$httpdconf_customlog_exists ) {
        $missing_customlog_templates .= ' httpd.conf';
    }

    if ($commented_templates) {
        print_warn('CustomLog commented out: ');
        print_warning($commented_templates);
    }

    if ($missing_customlog_templates) {
        print_warn('CustomLog entries missing: ');
        print_warning($missing_customlog_templates);
    }
}

sub check_for_cpsources_conf {
    my $cpsources_conf = '/etc/cpsources.conf';
    return unless -f $cpsources_conf and not -z $cpsources_conf;

    print_warn('/etc/cpsources.conf: ');
    print_warning('exists! This can affect upcp and EA.');

    my @cpsources_servers;
    if ( open( my $cpsources_conf_fh, '<', $cpsources_conf ) ) {
        while (<$cpsources_conf_fh>) {
            my ( $key, $value ) = split( '=', $_ );
            if ( $key eq 'HTTPUPDATE' ) {
                push @cpsources_servers, $value;
            }
        }
        close $cpsources_conf_fh;
    }

    foreach my $cpsources_server (@cpsources_servers) {
        chomp $cpsources_server;
        my ( $host, $port ) = split( ':', $cpsources_server );
        if ( !defined($port) ) {
            $port = 80;
        }
        my $tiers_data = _http_get( Host => $host, Port => $port, Path => '/cpanelsync/TIERS', WantHeaders => 1 );
        if ( !$tiers_data ) {
            print_warn('/etc/cpsources.conf: ');
            print_warning("Unresponsive mirror: http://$host:$port");
            next;
        }
        ( my $http_response ) = split( '\n', $tiers_data );
        ($http_response) = split( '\r', $http_response );
        if ( $http_response =~ /^HTTP\/(\d*?)\.(\d*?) (\d\d\d) (.*?)$/ ) {
            if ( $3 != 200 ) {
                print_warn('/etc/cpsources.conf: ');
                print_warning("Server $host:$port responded with [ $http_response ] when fetching TIERS file.");
            }
        }
        else {
            print_warn('/etc/cpsources.conf: ');
            print_warning("Server $host:$port responds with weird HTTP response [ $http_response ]");
        }
    }
}

sub check_for_apache_rlimits {
    return unless i_am_one_of( 'ea4', 'ea3' );
    my $httpdconf = i_am('ea4') ? '/etc/apache2/conf/httpd.conf' : '/usr/local/apache/conf/httpd.conf';
    my ( $rlimitmem, $rlimitcpu );
    my $output;

    if ( open my $httpdconf_fh, '<', $httpdconf ) {
        while (<$httpdconf_fh>) {
            if (m{\A \s* RLimitMEM \s+ (\d+)}xmsi) {
                $rlimitmem = $1;
            }
            if (m{\A \s* RLimitCPU \s+ (\d+)}xmsi) {
                $rlimitcpu = $1;
            }
            last if m{\A \s* <VirtualHost}xmsi;
        }
        close $httpdconf_fh;
    }

    if ($rlimitmem) {
        my $rlimitmem_converted = sprintf( '%.0f MB', $rlimitmem / 1024 / 1024 );
        $output = "RLimitMEM $rlimitmem [$rlimitmem_converted]";
    }

    if ($rlimitcpu) {
        $output .= " / RLimitCPU $rlimitcpu";
    }

    if ($output) {
        print_warn('Apache RLimits: ');
        print_warning($output);
    }
}

sub check_for_usr_local_lib_libz_so {
    return unless i_am('ea3') or ea3_downgrade_is_possible();
    if ( -f '/usr/local/lib/libz.so' ) {
        print_warn('/usr/local/lib/libz.so: ');
        print_warning('exists. This can prevent an EA3 build from completing');
    }
}

sub check_for_non_default_modsec_rules {
    return unless i_am_one_of( 'ea4', 'ea3' );
    my $modules = get_apache_modules_href();
    return unless defined $modules->{'security_module'} or defined $modules->{'security2_module'};

    my $conf_prefix       = i_am('ea4') ? '/etc/apache2/conf.d/' : '/usr/local/apache/conf/';
    my $modsec_prefix     = i_am('ea4') ? 'modsec/'              : '';
    my $modsec2_conf      = $conf_prefix . 'modsec2.conf';
    my $modsec2_user_conf = $conf_prefix . $modsec_prefix . 'modsec2.user.conf';
    my $modsec_rules_dir  = $conf_prefix . $modsec_prefix . 'modsec_rules';

    if ( my $modsec2_conf_size = -s $modsec2_conf ) {
        my $modsec2_conf_max_size = 1580;
        if ( $modsec2_conf_size > $modsec2_conf_max_size ) {
            print_warn('Mod Security: ');
            print_warning("$modsec2_conf is > $modsec2_conf_max_size bytes, may contain custom rules");
        }
    }

    if ( -s $modsec2_user_conf ) {
        print_warn('Mod Security: ');
        print_warning("$modsec2_user_conf is not empty, may contain custom rules");
    }

    if ( -d $modsec_rules_dir ) {
        print_warn('Mod Security: ');
        print_warning("$modsec_rules_dir exists, 3rd-party rules may be in use");
    }
}

sub check_etc_hosts_sanity {
    my $hosts = '/etc/hosts';
    my ( $localhost, $httpupdate, $localhost_not_127, $hostname_entry ) = ( 0, 0, 0, 0 );
    my $hostname = get_hostname();

    if ( !-f $hosts ) {
        print_warn("$hosts: ");
        print_warning('missing!');
        return;
    }

    if ( open my $hosts_fh, '<', $hosts ) {
        while ( my $line = <$hosts_fh> ) {
            chomp $line;

            next if ( $line =~ /^(\s+)?#/ );

            if ( $line =~ m{  127\.0\.0\.1 (.*) localhost }xms ) {
                $localhost = 1;
            }
            if ( ( $line =~ m{ \s localhost (\s|\z) }xmsi ) and ( $line !~ m{ 127\.0\.0\.1 | ::1 }xms ) ) {
                $localhost_not_127 = 1;
            }
            if ( $line =~ m{ httpupdate\.cpanel\.net }xmsi ) {
                $httpupdate = 1;
            }
            if ( $line =~ m{ $hostname }xmsi ) {
                $hostname_entry = 1;
            }
        }
        close $hosts_fh;
    }

    if ( !$localhost ) {
        print_warn("$hosts: ");
        print_warning('no entry for localhost, or commented out');
    }

    if ($httpupdate) {
        print_warn("$hosts: ");
        print_warning('contains an entry for httpupdate.cpanel.net');
    }

    if ($localhost_not_127) {
        print_warn("$hosts: ");
        print_warning('contains an entry for "localhost" that isn\'t 127.0.0.1! This can break EA3 or webmail logins');
    }

    if ( !$hostname_entry ) {
        print_warn("$hosts: ");
        print_warning("no entry found for the server's hostname! [$hostname] (Can break EA3 or Apache when mod_unique_id is enabled)");
    }
}

sub check_localhost_resolution {
    return if $OPT_SKIP_NETWORKING;

    # At this time we only require localhost to resolve to "127.0.0.1", but "::1" is accounted for
    my $print_check;
    my $found_127_forward;
    my @localhost = _resolve( 'localhost', 1 );
    foreach my $addr (@localhost) {
        unless ( $addr eq '127.0.0.1' or $addr eq '::1' ) {
            print_warn('Resolver: ');
            print_warning( 'returned unexpected address [ ' . $addr . ' ] (out of ' . scalar @localhost . ' total addresses) for "localhost"!' );
            $print_check = 1;
        }
        $found_127_forward = 1 if $addr eq '127.0.0.1';
    }
    unless ($found_127_forward) {
        print_warn('Resolver: ');
        print_warning('did not return expected "127.0.0.1" when resolving "localhost"!');
        $print_check = 1;
    }

    # Check 127.0.0.1 -> name (may be "localhost.domain") -> 127.0.0.1 match
    my $found_127_reverse;
    my @reverse = _resolve( '127.0.0.1', 1 );
    foreach my $host (@reverse) {
        unless ( $host =~ '^localhost' ) {
            print_warn('Resolver: ');
            print_warning( 'returned unexpected name [ ' . $host . ' ] (out of ' . scalar @reverse . ' total names) when resolving "127.0.0.1"!' );
            $print_check = 1;
        }
        my @forward = _resolve( $host, 1 );
        foreach my $addr (@forward) {
            unless ( $addr eq '127.0.0.1' or $addr eq '::1' ) {
                print_warn('Resolver: ');
                print_warning( '"127.0.0.1" resolved to [ ' . $host . ' ] (out of ' . scalar @forward . ' total addresses) which resolved to unexpected localhost address [ ' . $addr . ' ]!' );
                $print_check = 1;
            }
            $found_127_reverse = 1 if $addr eq '127.0.0.1';
        }
    }
    unless ($found_127_reverse) {
        print_warn('Resolver: ');
        print_warning('No full reverse path for "127.0.0.1" found! ( 127.0.0.1 -> localhost -> 127.0.0.1 )');
        $print_check = 1;
    }
    if ($print_check) {
        print_warn('Resolver: ');
        print_warning('Check /etc/{hosts,host.conf,nsswitch.conf,resolv.conf} for sanity.  Localhost resolution problems could cause errant behavior.');
    }
}

sub _resolve {
    my ( $addr, $report_errors, $timeout ) = @_;
    return if $OPT_SKIP_NETWORKING;
    $report_errors = ( defined $report_errors && $report_errors ne "0" ) ? 1        : 0;
    $timeout       = defined $timeout                                    ? $timeout : 3;
    my @results;

    local $SIG{'ALRM'} = sub {
        if ($report_errors) {
            print_warn('Resolver: ');
            print_warning( 'Timed out (' . $timeout . ' seconds) resolving "' . $addr . '"' );
        }
        return;
    };
    alarm $timeout;

    if (%SOCKET) {
        my $getnameinfo_flags = ( $addr =~ /^\d+\.\d+\.\d+\.\d+$/ ) ? $SOCKET{'NI_NAMEREQD'} : $SOCKET{'NI_NUMERICHOST'};    # If looking up IP address we require name resolution, otherwise we want the IP address returned.
        my ( $err, @socks ) = $SOCKET{'getaddrinfo'}->( $addr, "", { socktype => $SOCKET{'SOCK_RAW'} } );
        if ( $report_errors && $err ) {
            print_warn('Resolver: ');
            print_warning(qq{getaddrinfo() failed to resolve "$addr": $err});
        }
        foreach my $sock (@socks) {
            my ( $err, $result ) = $SOCKET{'getnameinfo'}->( $sock->{addr}, $getnameinfo_flags, $SOCKET{'NIx_NOSERV'} );
            if ( $report_errors && $err ) {
                print_warn('Resolver: ');
                print_warning(qq{getnameinfo() failed to resolve "$addr": $err});
                next;
            }
            push @results, $result;
        }
    }
    else {    # Fall back to older (deprecated) Socket functions
        my %h_errno = (
            '1' => 'HOST_NOT_FOUND',
            '2' => 'TRY_AGAIN',
            '3' => 'NO_RECOVERY',
            '4' => 'NO_DATA'
        );
        local $?;
        if ( $addr =~ /^\d+\.\d+\.\d+\.\d+$/ ) {
            my $packed = inet_aton($addr);
            return unless defined $packed;
            my $result = gethostbyaddr( $packed, AF_INET );
            my $error  = $? ? "h_errno " . ( exists $h_errno{$?} ? $h_errno{$?} : $? ) : 0;
            if ( $report_errors && $error ) {
                print_warn('Resolver: ');
                print_warning( 'gethostbyaddr() failed to resolve "' . $addr . '": ' . $error );
                return;
            }
            push @results, $result;
        }
        else {
            my $packed_result = gethostbyname($addr);
            my $error = $? ? "h_errno " . ( exists $h_errno{$?} ? $h_errno{$?} : $? ) : 0;
            if ( $report_errors && $error ) {
                print_warn('Resolver: ');
                print_warning( 'gethostbyname() failed to resolve "' . $addr . '": ' . $error );
            }
            if ( defined $packed_result ) {
                my $result = inet_ntoa($packed_result);
                push @results, $result;
            }
        }
    }
    alarm 0;
    return @results;
}

sub check_for_apache_listen_host_is_localhost {
    return unless i_am_one_of( 'ea4', 'ea3' );
    return if not defined $CPCONF{'apache_port'};

    my $apache_setting = $CPCONF{'apache_port'};
    $apache_setting =~ s/:.*//g;

    if ( $apache_setting eq '127.0.0.1' ) {
        print_warn('Apache listen host: ');
        print_warning('Apache may only be listening on 127.0.0.1');
    }
}

sub check_roundcube_mysql_pass_mismatch {
    return unless i_am('cpanel');
    return if ( defined $CPCONF{'roundcube_db'} and $CPCONF{'roundcube_db'} ne 'mysql' );

    my $roundcubepass;
    my $rc_mysql_pass;

    return unless open my $rc_pass_fh, '<', '/var/cpanel/roundcubepass';
    while (<$rc_pass_fh>) {
        chomp( $roundcubepass = $_ );
    }
    close $rc_pass_fh;

    return unless open my $db_inc_fh, '<', '/usr/local/cpanel/base/3rdparty/roundcube/config/db.inc.php';
    while (<$db_inc_fh>) {
        if (m{ \A \$rcmail_config\['db_dsnw'\] \s = \s 'mysql://roundcube:(.*)\@(?:.*)/roundcube';  }xms) {
            $rc_mysql_pass = $1;
        }
    }
    close $db_inc_fh;

    return if ( not $roundcubepass or not $rc_mysql_pass );

    if ( $roundcubepass ne $rc_mysql_pass ) {
        print_warn('RoundCube: ');
        print_warning('password mismatch [/var/cpanel/roundcubepass] [/usr/local/cpanel/base/3rdparty/roundcube/config/db.inc.php]');
    }
}

sub check_for_hooks_from_manage_hooks {
    return unless my $hooks_list = timed_run( 0, '/usr/local/cpanel/bin/manage_hooks', 'list' );
    my ( @hooks_tmp, @hooks );

    foreach ( split( /\n/, $hooks_list ) ) {
        if (/hook: (.*)/) {

            # tidyoff - Ignore default Attracta hooks
            next if ( $1 =~ m{ \A ( /usr/local/cpanel/3rdparty/attracta/scripts/pkgacct-restore | # Attracta
                                    /usr/local/cpanel/Cpanel/ThirdParty/Attracta/Hooks/pkgacct-restore | # Attracta
                                    CCSHooks::\S+ | # Calendar and Contacts Server
                                    NginxHooks::\S+) \z }xms ); # tidyon - ea-nginx
            push @hooks_tmp, "$1 ";
        }
    }

    for my $hook (@hooks_tmp) {
        if ( $hook =~ m/^\// ) {
            if ( -e $hook and not -z $hook ) {
                push @hooks, $hook;
            }
        }
        else {
            push @hooks, $hook;    # we don't check for the existence of all hooks since they could be anywhere in perl's @INC, I think?
        }
    }

    if ( scalar @hooks == 1 ) {
        print_warn('Hooks in /var/cpanel/hooks.yaml: ');
        print_warning(@hooks);
    }
    elsif ( scalar @hooks > 1 ) {
        print_warn("Hooks in /var/cpanel/hooks.yaml:\n");
        for my $hook (@hooks) {
            print_magenta("\t \\_ $hook");
        }
    }
}

sub check_mysql_config {
    return unless my $mysql_conf = get_mysql_conf_href();
    return unless defined $mysql_conf->{'mysqld'};
    my $meminfo = get_meminfo();

    # Example: 'optionwithoutdashesorunderscores' => { default => 'defaultvalue', check_missing => 1, orig_name => 'option_with_underscores-or-dashes', help => '- Help text' },
    # default, check_missing, and help are optional, but orig_name should be defined if check_missing is used so that its name can be properly printed
    # If check_missing exists, a warning is generated if the item is NOT in my.cnf and does not match the default (if given)

    my %mysqld_checks = (
        'datadir'             => { default => '/var/lib/mysql' },
        'innodbforcerecovery' => { default => '0', help => 'Makes all InnoDB databases read-only, will break MySQL upgrades.' },
        'logerror'            => { default => '/var/lib/mysql/' . get_hostname() . '.err' },
        'lowercasetablenames' => { default => '0', help => 'Will break space usage reporting of databases with mixed-case names. See CPANEL-8453.' },
        'oldpasswords'        => { default => '0', help => 'Not recommended, non-native passwords are incompatible with MySQL 5.6+ and some MySQL clients. See also CPANEL-10047, 10826, 10940, 10954, 11018, and 11019.' },
        'skipnameresolve'     => { help    => 'Seeing "Can\'t find any matching row"? That may be why.' },
        'skipnetworking'      => { help    => 'Webmail or other MySQL related items not functioning properly? That may be why.' },
        'sqlmode'             => { help    => 'Seeing "Field \'ssl_cipher\' doesn\'t have a default value"? That may be why.' }
    );

    if ( defined( $meminfo->{memtotal} ) ) {
        my $mem_mb = int( $meminfo->{memtotal} / 1024 );

        # Systems w/2GB RAM report around 1800MB total after overhead, so 1700 should be a good tipping point
        if ( $mem_mb < 1700 && version_compare( $CPCONF{'mysql-version'}, qw( >= 5.6 ) ) ) {
            $mysqld_checks{'performanceschema'} = { default => '0', check_missing => 1, orig_name => "performance_schema", help => 'performance_schema is enabled by default in MySQL 5.6+ and could use a significant amount of memory, recommend "performance_schema = 0" with less than 2GB RAM [detected ' . $mem_mb . 'MB].' };
        }
    }

    for my $check ( sort( keys(%mysqld_checks) ) ) {
        $mysqld_checks{$check}->{default}       = "" unless defined( $mysqld_checks{$check}->{default} );
        $mysqld_checks{$check}->{check_missing} = 0  unless defined( $mysqld_checks{$check}->{check_missing} );
        $mysqld_checks{$check}->{help}          = "" unless defined( $mysqld_checks{$check}->{help} );
        my $help = $mysqld_checks{$check}->{help} ? ' - ' . $mysqld_checks{$check}->{help} : '';
        if ( defined( $mysql_conf->{'mysqld'}{$check} ) && !( $mysql_conf->{'mysqld'}{$check}[1] eq $mysqld_checks{$check}->{default} ) ) {
            print_warn("MySQL $MYSQL_CONF_FILE: ");
            if ( $mysql_conf->{'mysqld'}{$check}[1] eq "enabled" ) {
                print_warning("[ $mysql_conf->{'mysqld'}{$check}[0] ] found $help");
            }
            else {
                print_warning("[ $mysql_conf->{'mysqld'}{$check}[0] = $mysql_conf->{'mysqld'}{$check}[1] ] $help");
            }
        }
        elsif ( $mysqld_checks{$check}->{check_missing} && !defined( $mysql_conf->{'mysqld'}{$check} ) ) {
            my $optname = defined( $mysqld_checks{$check}->{orig_name} ) ? $mysqld_checks{$check}->{orig_name} : $check;
            print_warn("MySQL $MYSQL_CONF_FILE: ");
            print_warning("[ $optname ] not found. $help");
        }
    }
}

sub check_mysql_datadir {
    return unless os_version_is(qw( >= 7 ));
    return unless my $mysql_numeric_version = get_mysql_numeric_version();
    return unless version_compare( $mysql_numeric_version, qw( >= 10.1.16 ) );
    my $datadir        = get_mysql_datadir();
    my $bad_path_regex = '^\/(home|usr|etc|boot)(\/|\s*$)';
    if ( defined $datadir and $datadir =~ m/$bad_path_regex/ ) {
        print_warn('MySQL: ');
        print_warning("$MYSQL_CONF_FILE datadir points to a systemd protected directory which can prevent MariaDB 10.1.16+ from starting. See CPANEL-15633.");
    }
    if ( -l '/var/lib/mysql' and readlink('/var/lib/mysql') =~ m/$bad_path_regex/ ) {
        print_warn('MySQL: ');
        print_warning('/var/lib/mysql symlink resolves to a systemd protected directory which can prevent MariaDB 10.1.16+ from starting. See CPANEL-15633.');
    }

}

sub check_for_extra_mysql_config_files {

    # It's silly how many locations mysqld looks for a configuration file.
    # These locations are reported by mysqld --help and by looking through /usr/bin/mysqld_safe code
    my @extra_locations = qw(
      /etc/mysql/my.cnf
      /usr/my.cnf
      /usr/etc/my.cnf
      /var/lib/mysql/my.cnf
    );
    my @found_locations = grep { -f $_ } @extra_locations;

    return if !@found_locations;
    print_warn('MySQL - extra my.cnf files found: ');
    print_warning( '[ ' . join( " ", @found_locations ) . ' ]' );
    print_warning("              \\_ These may replace or be merged with $MYSQL_CONF_FILE settings!");
}

sub check_cpanel_config {
    return unless keys(%CPCONF);

    # Example: 'exact_option_name' => { default => 'defaultvalue', check_missing => 1, help => '- Help text' },
    # default, check_missing, and help are optional
    # If check_missing exists, a warning is generated if the item is NOT in cpanel.config and does not match the default (if given) or is empty

    my %cpanel_checks = (
        'enablecompileroptimizations' => { default       => '0', help => 'Tweak setting "Enable optimizations for the C compiler" enabled. If Sandy Bridge CPU, problems MAY occur (see ticket 3355885)' },
        'ftpserver'                   => { check_missing => 1 },
        'mailserver'                  => { check_missing => 1 },
        'mysql-version'               => { check_missing => 1 },
        'nativessl'                   => { default       => '1', help => 'Native SSL support for WHM/cPanel services is disabled' },
        'pma_disableis'               => { default       => '0', help => 'Can cause various issues with phpMyAdmin displaying databases -- see CPANEL-16866, CPANEL-16867, CPANEL-18742.' },
        'root'                        => { default       => '/usr/local/cpanel', help => 'An invalid root setting can cause WHM to fail to start.  See ticket 5463121.' },
        'skiphttpauth'                => { default       => '1', help => 'HTTP auth enabled' },
        'skipparentcheck'             => { default       => '0', help => 'Allows other applications to run the cPanel and admin binaries' },
        'acls'                        => { default       => '0', help => 'Can cause issues with FileProtect -- see CPANEL-19578' },
    );

    if ( defined $CPCONF{'maxmem'} && $CPCONF{'maxmem'} < 512 ) {
        $cpanel_checks{'maxmem'} = { default => '512', help => '< 512M, phpmyadmin may fail' };
    }

    for my $check ( sort( keys(%cpanel_checks) ) ) {
        $cpanel_checks{$check}->{check_missing} = 0  unless defined( $cpanel_checks{$check}->{check_missing} );
        $cpanel_checks{$check}->{help}          = "" unless defined( $cpanel_checks{$check}->{help} );
        my $help = $cpanel_checks{$check}->{help} ? ' - ' . $cpanel_checks{$check}->{help} : '';
        if ( defined( $CPCONF{$check} ) && defined( $cpanel_checks{$check}->{default} ) && !( $CPCONF{$check} eq $cpanel_checks{$check}->{default} ) ) {
            print_warn('cpanel.config: ');
            print_warning("[ $check = $CPCONF{$check} ] $help");
        }
        elsif ( $cpanel_checks{$check}->{check_missing} && ( !defined( $CPCONF{$check} ) || ( defined( $CPCONF{$check} ) && $CPCONF{$check} eq "" ) ) ) {
            print_warn('cpanel.config: ');
            print_warning("[ $check ] not found or has empty value. $help");
        }
    }
}

sub check_for_low_ulimit_for_root {
    my $ulimit_m = timed_run( 0, 'echo `ulimit -m`' );
    my $ulimit_v = timed_run( 0, 'echo `ulimit -v`' );

    chomp( $ulimit_m, $ulimit_v );

    if ( $ulimit_m =~ /\d+/ ) {
        $ulimit_m = sprintf( '%.0f', $ulimit_m / 1024 );
    }
    if ( $ulimit_v =~ /\d+/ ) {
        $ulimit_v = sprintf( '%.0f', $ulimit_v / 1024 );
    }

    if ( $ulimit_m =~ /\d+/ and $ulimit_m <= 256 or $ulimit_v =~ /\d+/ and $ulimit_v <= 256 ) {
        if ( $ulimit_m =~ /\d+/ ) {
            $ulimit_m .= 'MB';
        }
        if ( $ulimit_v =~ /\d+/ ) {
            $ulimit_v .= 'MB';
        }

        print_warn('ulimit: ');
        print_warning("-m [ $ulimit_m ] -v [ $ulimit_v ] Low ulimits can cause EA3 or other commands to fail when run via the shell");
    }
}

sub check_for_fork_bomb_protection {
    if ( -f '/etc/profile.d/limits.sh' or -f '/etc/profile.d/limits.csh' ) {
        print_warn('Fork Bomb Protection: ');
        print_warning('enabled!');
    }
}

sub check_for_custom_exim_conf_local {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    my $exim_conf_local = '/etc/exim.conf.local';
    my $is_customized   = 0;

    if ( open my $file_fh, '<', $exim_conf_local ) {
        while ( my $line = <$file_fh> ) {
            chomp $line;
            if ( $line !~ m{ \A ( @ | \Z | chunking_advertise_hosts="" \Z ) }xms ) {
                $is_customized = 1;
                last;
            }
        }
        close $file_fh;
    }

    if ($is_customized) {
        print_warn('Exim: ');
        print_warning("$exim_conf_local contains customizations");
    }
}

sub check_for_maxclients_or_maxrequestworkers_reached {
    return unless i_am_one_of( 'ea4', 'ea3' );
    return unless my $apache_version = get_apache_version();
    my $log            = i_am('ea4') ? '/etc/apache2/logs/error_log' : '/usr/local/apache/logs/error_log';
    my $size           = ( stat($log) )[7];
    my $bytes_to_check = 20_971_520 / 2;                                                                     # 10M limit of logs to check, may need adjusting, depending how much time it adds to SSP
    my $seek_position  = 0;
    my $log_data;
    my @logs;
    my $limit_last_hit_date;

    return if !$size;

    if ( $size > $bytes_to_check ) {
        $seek_position = ( $size - $bytes_to_check );
    }

    if ( open my $file_fh, '<', $log ) {
        seek $file_fh, $seek_position, 0;
        read $file_fh, $log_data, $bytes_to_check;
        close $file_fh;
    }

    my $apache24 = version_compare( $apache_version, qw( >= 2.4.0 ) );
    if ( $log_data =~ m/(?:MaxClients|MaxRequestWorkers)/s ) {
        @logs = split /\n/, $log_data;
        undef $log_data;
        @logs = reverse @logs;
        for my $log_line (@logs) {
            if ( $apache24 and $log_line =~ m{ \A \[ (\S+ \s+ \S+ \s+ \S+ \s+ \S+ \s+ \S+ ) \] \s .* server \s reached \s MaxRequestWorkers }xms ) {

                # [Fri Feb 08 09:58:45.875187 2013] [mpm_prefork:error] [pid 23220] AH00161: server reached MaxRequestWorkers
                $limit_last_hit_date = $1;
                last;
            }
            elsif ( not $apache24 and $log_line =~ m{ \A \[ (\S+ \s+ \S+ \s+ \S+ \s+ \S+ \s+ \S+ ) \] \s+ \[error\] \s+ server \s+ reached \s+ MaxClients }xms ) {

                # [Wed Nov 14 05:55:04 2012] [error] server reached MaxClients setting, consider raising the MaxClients setting
                $limit_last_hit_date = $1;
                last;
            }
        }
    }

    return unless $limit_last_hit_date;

    if ($apache24) {
        print_warn('Apache MaxRequestWorkers: ');
    }
    else {
        print_warn('Apache MaxClients: ');
    }

    print_warning("limit last reached at $limit_last_hit_date");
}

sub check_for_non_default_umask {
    my $umask = timed_run( 0, 'echo `umask`' );
    chomp $umask;

    return if !$umask || $umask =~ /2$/;

    print_warn('umask: ');
    print_warning("Non-default value [$umask] (check FB-62683 if permissions error when running convert_roundcube_mysql2sqlite)");
}

sub check_for_multiple_imagemagick_installs {
    return unless i_am('cpanel');
    if ( -x '/usr/bin/convert' and not -l '/usr/bin/convert' ) {
        if ( -x '/usr/local/bin/convert' and not -l '/usr/local/bin/convert' ) {
            print_warn('ImageMagick: ');
            print_warning('multiple "convert" binaries found [/usr/bin/convert] [/usr/local/bin/convert]');
        }
    }
}

sub check_for_kernel_headers_rpm {
    if ( not -f '/usr/include/linux/limits.h' and i_am('ea3') ) {
        print_warn('Missing file: ');
        print_warning('/usr/include/linux/limits.h not found. This can cause problems with EA3. kernel-headers RPM missing/broken?');
    }
    return unless my $rpms = get_rpm_href();
    unless ( exists $rpms->{'kernel-headers'} ) {
        print_warn('kernel-headers RPM: ');
        print_warning('not found. This can cause problems with EA3 and compiling various WHM/cPanel wrapper binaries');
    }
}

sub check_for_broken_rpm {
    return unless my $rpms = get_rpm_href();
    $YUM_OR_DNF = os_version_is(qw( >= 8)) ? 'dnf' : 'yum';
    my %check = (    # 'rpm name' => { check => ['check1', 'check2', ...], help => 'Additional info' } # Check types should be ordered by dependency, first failure skips other checks for that RPM.
        'bind-chroot'        => { check => ['exists'], help => 'Not supported -- should be removed and excluded in yum.conf. See https://documentation.cpanel.net/display/ALD/Installation+Guide+-+Customize+Your+Installation#InstallationGuide-CustomizeYourInstallation-Excludepackages' },
        'bitninja'           => { check => ['exists'], help => 'May cause AutoSSL to fail.  See tickets 8352131, 10723019.' },
        'cpuspeed'           => { check => ['exists'], help => 'May cause ' . $YUM_OR_DNF . ' to crash when updating kernel. Send "cpuspeed detected" predefined response.' },
        'crypto-utils'       => { check => ['exists'], help => '"certwatch" cron job can send email notices regarding expiring certificates.  See ticket 7822329.' },
        'letsencrypt-cpanel' => { check => ['exists'], help => 'Third-party FleetSSL Let\'s Encrypt plugin -- no direct support provided, use "3RDP - FleetSSL/LetsEncrypt" predef if relevant.' },
    );

    if ( i_am('ea4') ) {
        $check{'httpd-tools'} = { check => ['exists'], help => 'Conflicts with ea-apache24-tools, will break EA4.' };
    }
    elsif ( i_am('ea3') ) {
        $check{'cpp'} = { check => ['verify-fail'], help => 'Missing or modified files, may cause EasyApache to fail, verify with "rpm -V cpp"' };
    }

    my %types = (
        'exists' => {
            help => 'RPM exists',
            run  => sub {           # Only care that the RPM exists.
                my $rpm = shift;
                return 1 if exists $rpms->{$rpm};
                return 0;
            }
        },
        'missing' => {
            help => 'Missing RPM',
            run  => sub {            # Only care that the RPM is missing.
                my $rpm = shift;
                return 0 if exists $rpms->{$rpm};
                return 1;
            }
        },
        'verify-fail' => {
            help => 'Verify Failed',
            run  => sub {              # Only performs check if the RPM exists.
                my $rpm = shift;
                return 0 unless exists $rpms->{$rpm};
                my $output = timed_run( 0, 'rpm', '-V', $rpm );
                return 0 if $output eq "";
                return 1 if $output =~ m{ \A missing }xms;
                return 1 if $output =~ m{ \A ..5 }xms;
                return 0;
            }
        }
    );

    for my $rpm ( sort keys %check ) {
        for my $type ( @{ $check{$rpm}{check} } ) {
            if ( $types{$type}{run}->($rpm) ) {
                my $additional_info = $check{$rpm}{help} ? ("( $check{$rpm}{help} )") : '';
                print_warn('RPM check: ');
                print_warning("$types{$type}{help} - [ $rpm ] $additional_info");
                last;
            }
        }
    }
}

sub check_for_ea4_mismatch {
    return unless i_am('ea4');
    return unless my $rpms = get_rpm_href();
    my ( $cp_count, $cl_count ) = ( 0, 0 );
    for my $name ( keys %{$rpms} ) {
        next unless index( $name, 'ea-' ) == 0;
        foreach my $rpm_ref ( @{ $rpms->{$name} } ) {
            $cp_count++ if index( $rpm_ref->{'release'}, '.cpanel' ) != -1;
            $cl_count++ if index( $rpm_ref->{'release'}, '.cloudlinux' ) != -1;
        }
    }
    if ( i_am('cloudlinux') ) {
        return unless $cp_count;
        print_warn('EA4 RPMs: ');
        print_warning(qq{Found $cp_count "ea-*.cpanel" RPMs on a CloudLinux system!  Using wrong EA4 repo?});
        return;
    }
    return unless $cl_count;
    print_warn('EA4 RPMs: ');
    print_warning(qq{Found $cl_count "ea-*.cloudlinux" RPMs on a non-CloudLinux system!  Using wrong EA4 repo?});
    if ( -e '/etc/yum.repos.d/imunify360-ea-php-hardened.repo' ) {
        print_warning('         \_ Imunify360 Hardened EA-PHP repo found. It is probably OK if all of the RPMs originated from this repo.');
    }
}

sub check_mysql_skip_grants {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    return unless exists_process_cmd( qr{ /mysqld }xms, 'mysql' );

    # Test with very unique password, if it works, 99% skip-grants is enabled
    my $fake_pass = 'KsA9tsRzaoa01dWl1dJBv9WtyA5Ahe7T2qrzVt7MPNa9IrdnEsRVI8raUUOtQ8Xls1SroikQy4gm1ohU';
    my $ver = timed_run( 0, 'mysql', '--user=root', '--password=' . $fake_pass, '-NBe', 'SELECT version()' );
    if ($ver) {
        print_crit('MySQL: ');
        print_critical('Root MySQL access was obtained with a random password, is \'skip-grant-tables\' enabled?');
        print_critical( "\t \\_ " . 'Verify \'skip-grant-tables\' is enabled and inform the client of the inherent risks of running MySQL without grants' );
    }
}

sub check_eximstats_size {
    return unless i_am('cpanel');
    return unless my $mysql_datadir = get_mysql_datadir();
    my $eximstats_dir = $mysql_datadir . 'eximstats/';
    return unless -d $eximstats_dir;
    my @dir_contents;
    my $size;

    opendir( my $dir_fh, $eximstats_dir );
    @dir_contents = grep { /(defers|failures|sends|smtp)\.(frm|MYI|MYD)$/ } readdir $dir_fh;
    closedir $dir_fh;

    for my $file (@dir_contents) {
        $file = $eximstats_dir . $file;
        $size += ( stat($file) )[7];
    }

    if ( $size && $size > 5_000_000_000 ) {
        $size = sprintf( "%0.2fGB", $size / 1073741824 );
        print_warn('eximstats db: ');
        print_warning($size);
    }
}

sub check_for_broken_mysql_tables {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    my %broken;
    my @schemas;
    my $sql_schemas;
    push @schemas, 'horde'     if cpanel_version_is(qw ( < 11.53.0.0 ));
    push @schemas, 'whmxfer'   if cpanel_version_is(qw ( < 11.57.0.0 ));
    push @schemas, 'modsec'    if cpanel_version_is(qw ( < 11.61.0.0 ));
    push @schemas, 'eximstats' if cpanel_version_is(qw ( < 11.63.0.0 ));
    push @schemas, 'cphulkd'   if cpanel_version_is(qw ( < 11.65.0.0 ));
    push @schemas, 'roundcube' if defined $CPCONF{roundcube_db} && $CPCONF{roundcube_db} eq 'mysql';

    return unless scalar @schemas;

    for my $schema ( sort @schemas ) {
        $sql_schemas .= ' OR ' if defined $sql_schemas && length $sql_schemas;
        $sql_schemas .= 'table_schema=\'' . $schema . '\'';
    }
    for ( split /\n/, timed_run( 0, 'mysql', '-NBe', 'SELECT table_schema,table_name,engine,row_format,create_time FROM information_schema.tables WHERE ( ' . $sql_schemas . ' ) AND engine IS NULL' ) ) {
        my @line = split /\t/;
        next unless scalar @line >= 2;
        $broken{ $line[0] }{ $line[1] } = 1;
    }
    for ( sort( keys(%broken) ) ) {
        print_warn('mysql: broken tables - ');
        my $tables = scalar keys( %{ $broken{$_} } ) > 4 ? 'More than 4!' : join( ' ', sort( keys( %{ $broken{$_} } ) ) );
        print_warning( $_ . ' [ ' . $tables . ' ]' );
    }
}

sub get_clock_skew {
    return if $OPT_SKIP_NETWORKING;
    my $rdatecmd = os_version_is(qw ( >= 8 )) ? "/usr/local/cpanel/bin/rdate" : "rdate";
    #my $rdateTimeout = os_version_is(qw ( >= 8 )) ? "" : qq("-t 3");
    my $rdateTimeout = os_version_is(qw ( >= 8 )) ? "" : qq(-t 3);
    ## last updated 2018-03-24
    ## we do this to avoid having to do the DNS lookup
    my @rdate_servers = qw(
        208.74.121.36
        208.74.121.43
        208.74.123.15
        208.74.123.23
    );

    my $localtime = time();
    my $rdate_time;
    my $clock_skew;

    my %months = qw( Jan 0   Feb 1   Mar 2   Apr 3   May 4   Jun 5 Jul 6   Aug 7   Sep 8   Oct 9   Nov 10  Dec 11);

    for ( 1 .. 2 ) {
        my $num = int rand scalar @rdate_servers;
        $rdate_time = timed_run( 10, $rdatecmd, '-p', $rdateTimeout, $rdate_servers[$num] );
        next if $rdate_time =~ /timeout/;
        last if $rdate_time;
    }

    return if !$rdate_time;

    $rdate_time =~ s/\A rdate: \s \[[^\]]+\] \s+//gxms;

    if ( $rdate_time =~ m{ \A \S+ \s+ (\S+) \s+ (\d+) \s+ (\d+):(\d+):(\d+) \s+ (\d+) }xms ) {
        my ( $mon, $mday, $hour, $min, $sec, $year ) = ( $1, $2, $3, $4, $5, $6 );
        $mon = $months{$mon};

        $rdate_time = timelocal( $sec, $min, $hour, $mday, $mon, $year );
    }
    return if ( $rdate_time !~ /\d{10,}/ );

    $clock_skew = ( $rdate_time - $localtime );
    $clock_skew = abs $clock_skew;                # convert negative numbers to positive
    return $clock_skew;
}

sub check_for_clock_skew {
    return unless my $clock_skew = get_clock_skew();
    my $max_skew = 120;
    if ( defined $CPCONF{'SecurityPolicy::TwoFactorAuth'} && $CPCONF{'SecurityPolicy::TwoFactorAuth'} == 1 ) {
        $max_skew = 25;
    }

    return if ( $clock_skew < $max_skew );

    if ( $clock_skew >= 31536000 ) {
        $clock_skew = sprintf '%d', ( $clock_skew / 31536000 );
        $clock_skew .= ' year(s)';
    }
    elsif ( $clock_skew >= 86400 ) {
        $clock_skew = sprintf '%d', ( $clock_skew / 86400 );
        $clock_skew .= ' day(s)';
    }
    elsif ( $clock_skew >= 3600 ) {
        $clock_skew = sprintf '%d', ( $clock_skew / 3600 );
        $clock_skew .= ' hour(s)';
    }
    elsif ( $clock_skew >= 60 ) {
        $clock_skew = sprintf '%d', ( $clock_skew / 60 );
        $clock_skew .= ' minute(s)';
    }
    else {
        $clock_skew = sprintf '%d', ($clock_skew);
        $clock_skew .= ' seconds';
    }

    print_warn('Clock skew: ');
    print_warning("server time may be off by ${clock_skew}. A very large difference may cause SSL/TLS connection errors, and more than about 30 seconds can cause 2FA failure.");
}

sub check_for_zlib_h {
    return unless i_am('ea3');
    if ( -f '/usr/local/include/zlib.h' ) {
        print_warn('/usr/local/include/zlib.h: ');
        print_warning('This file can cause EA3 to fail with libxml issues. You may need to mv it, run EA3 again');
    }
}

sub check_for_duplicate_rpms {
    return unless my $rpms = get_rpm_href();
    my %SEEN_RPMS;
    my %DUP_RPMS;
    foreach my $name ( keys %{$rpms} ) {
        foreach my $rpm_ref ( @{ $rpms->{$name} } ) {
            push @{ $SEEN_RPMS{ $name . '-' . $rpm_ref->{'arch'} } }, $rpm_ref->{'version'} . '-' . $rpm_ref->{'release'};
            if ( scalar @{ $SEEN_RPMS{ $name . '-' . $rpm_ref->{'arch'} } } > 1 ) {
                $DUP_RPMS{ $name . '-' . $rpm_ref->{'arch'} } = 1;
            }

        }
    }

    foreach my $dup_rpm ( sort keys %DUP_RPMS ) {
        next if ( $dup_rpm =~ m{^(?:gpg-pubkey|kernel)} );

        print_warn('DUPLICATE RPM: ');
        print_warning( "$dup_rpm has multiple versions: " . join( " ", @{ $SEEN_RPMS{$dup_rpm} } ) );

    }
}

sub check_for_percona_rpms {
    return unless my $rpms = get_rpm_href();

    for my $rpm ( keys %{$rpms} ) {
        next if $rpm =~ /^percona-toolkit/i;    # Can be used with MariaDB/MySQL
        if ( $rpm =~ /^Percona/i ) {
            print_warn("Percona RPMs found:\n");
            print_magenta("\t \\_ EA3 failing with \"Cannot find libmysqlclient\"? libmysqlclient.so missing? see FB-93349");
            print_magenta("\t \\_ If Exim is segfaulting after STARTTLS, this may be why. See ticket 3658929");
            print_magenta("\t \\_ If Apache with PHP DSO is segfaulting after restart, this may be why. See ticket 5525179");
            last;
        }
    }
}

sub check_for_wordpress_manager_rpms {
    return unless my $rpms = get_rpm_href();

    for my $rpm ( keys %{$rpms} ) {
        if ( $rpm =~ /^(?:wordpress-cpaddon|cpanel-wordpress-instance-manager-plugin)/i ) {
            print_warn("WordPress Manager (deprecated) is installed/enabled.\n");
            if ( ! i_am('wptk') ) {
                print_magenta("\t \\_ Send Customer The PREDEFS::WordPressManager Installed Use WordPress Toolkit Macro");
            }
            last;
        }
    }
}

sub check_imunify_config {
    return unless( -e '/usr/bin/imunify360-agent' );
    my %i360Conf;
    my $i360_config_file = "/etc/sysconfig/imunify360/imunify360.config";
    return unless( open( my $conf_fh, '<', $i360_config_file ));
    while (<$conf_fh>) {
        next unless ( m/\s(\S+):\s(\w+)/ );
        $i360Conf{$1} = $2;
    }
    close($conf_fh);
    my $raw = timed_run( 0, 'imunify360-agent', 'version', '--json' );
    my $i360versioninfo = get_json_href($raw);
    my $imunify_licenseType = $i360versioninfo->{license}->{license_type};
    return unless($imunify_licenseType);
    $imunify_licenseType =~ s/Plus/\+/;
    print_info("$imunify_licenseType is running: ");
    print_normal("Default Action: $i360Conf{'default_action'}" );
    my @trimFiles;
    my @quarantineFiles;
    @trimFiles = glob('/var/imunify360/cleanup_storage/*') unless( !-d '/var/imunify360/cleanup_storage');
    @quarantineFiles = glob('/home/.imunify.quarantined/*') unless( !-d '/home/.imunify.quarantined');
    print_normal("\t\\_ cleanup (trim) files found within /var/imunify360/cleanup_storage [ will be automatically removed within $i360Conf{'keep_original_files_days'} days ]") unless( scalar @trimFiles == 0 );
    print_normal("\t\\_ quarantined files found within /home/.imunify.quarantined") unless( scalar @quarantineFiles == 0 );
    if ( $imunify_licenseType eq "imunify360" or $imunify_licenseType eq "imunify360Trial") {
        print_normal("\t\\_ Issues with installing PECL extensions or running PHP's composer? Try setting Imunify360's Proactive Defense to log mode");
    }
}

sub check_for_etc_ubic_dir {
    if ( -e '/etc/ubic' ) {
        print_warn('/etc/ubic: ');
        print_warning('exists! This directory will cause ea-tomcat85 to fail when adding new users, see TECH-707');
    }
}

sub check_for_acls_cpconf {
    return unless i_am('cpanel');
    return unless defined $CPCONF{'acls'};
    if ( $CPCONF{'acls'} ) {
        print_warn('Permissions: ');
        print_warning('"acls" Tweak Setting enabled. This affects homedir, docroot, and htpasswd, see TECH-751');
    }
}

sub check_for_port_53_dnsmasq {
    return unless my $ports = get_lsof_port_href();
    return unless exists $ports->{'53'};
    for my $ref ( @{ $ports->{'53'} } ) {
        if ( $ref->{'CMD'} =~ m{ \A dnsmasq }xms ) {
            print_warn('Dnsmasq: ');
            print_warning('listening on port 53, known to cause issues with AutoSSL, see TECH-718');
            last;
        }
    }
}

sub check_for_port_21_ftp {
    return unless my $ports = get_lsof_port_href();
    return unless exists $ports->{'21'};

    my ( $pid, $comm ) = @{ $ports->{'21'}->[0] }{qw( PID CMD )};
    return unless my $exe = readlink "/proc/${pid}/exe";
    return unless my $cwd = readlink "/proc/${pid}/cwd";
    return if ( $exe eq '/usr/sbin/pure-ftpd' || $exe eq '/usr/sbin/proftpd' );

    print_warn('Port 21: ');
    print_warning("something other than Pure-FTPd or ProFTPd is running:");
    print_magenta("\t \\_ cmd     [$comm]");
    print_magenta("\t \\_ exe     [$exe]");
    print_magenta("\t \\_ cwd     [$cwd]");
}

sub check_ftpusers_file {
    my $ftpusers_file = '/etc/ftpusers';
    return unless ( -s $ftpusers_file );
    print_warn('/etc/ftpusers exists: ');
    print_warning('users in this file cannot access FTP');
}

# TODO: Once 84 is largely distrubuted, update this to use one of our modules
sub check_root_dns_resolvers {
    my $limit     = 3;
    my @resolvers = ( 'a' .. 'm' );
    my @failed;
    foreach my $dns (@resolvers) {
        last if scalar @failed >= $limit;
        chomp( my $ip = timed_run( 5, 'dig', '-4', '+short',  "$dns.root-servers.net", "\@$dns.root-servers.net" ) );
        push @failed, $dns unless ( $ip =~ /\d+\.\d+\.\d+\.\d+/ );
    }
    return unless @failed;
    print_warn('DNS: ');
    print_warning('Root DNS servers are unreachable [ ' . join(' ', @failed) . " ]. This can break AutoSSL. We stop after $limit failures, verify with this command:");
    print_warning("\t \\_ " . 'for i in {a..m}; do echo -n "$i.root-servers.net: "; dig -4 "$i".root-servers.net @"$i".root-servers.net +short;done');
}

sub check_cloudlinux_phphandler_file {
    return unless i_am('cloudlinux');
    my $phphandler_file = '/etc/container/php.handler';
    return unless not -z $phphandler_file;
    if (open my $fh, '<', $phphandler_file) {
        while(<$fh>) {
            next unless (m/x-httpd-ea-.*\/opt\/cloudlinux\/.*lsphp/);
            print_warn('CloudLinux: ');
            print_warning("/etc/container/php.handler exists. Having issues with downloading the PHP file? See TECH-792");
            last;
        }
        close($fh);
    }
}

sub check_auto_migrate_ea3_to_ea4 {
    my @files   = glob('/var/cpanel/auto_migrate_ea3_to_ea4*');
    return unless ( scalar @files );

    # Use the Schwartzian sort method because the versioning of our auto-migration script didn't use padding, and the versioning
    #   is more than a single digit's place, meaning we need to ensure that '.10' is greater than '.9'
    @files      = map { $_->[0] } sort { $a->[1] <=> $b->[1] } map { [$_, ( $_ =~ /v(\d+)$/ || '1' ) ] } @files;
    my $file    = pop @files;
    my $version = ( $file =~ /v(\d+)$/ ) ? $1 : 1;
    my ($correlation_id, $line, $result, $state, $macro);

    if ( open my $fh, '<', $file ) {
        $line = readline($fh);
        close $fh;
    }

    if (defined $line && $line =~ m/^(\S+) = (.*)$/) {
        $correlation_id = $1;
        $result         = $2;
        ($state,$macro) = $result eq '1' ? ('succeeded', 'PREDEFS::TS::EA3 to EA4 Migration') : $result eq '0' ? ('failed', 'PREDEFS::TS::EA3 to EA4 Migration Failed') : ($result, 'PREDEFS::TS::EA3 to EA4 Migration');
    } else {
        $correlation_id = 'UNKNOWN';
        $state          = 'failed';
        $macro          = 'PREDEFS::TS::EA3 to EA4 Migration Failed';
    }

    print_crit('EA4: ');
    print_critical("Auto upgrade v$version from EA3, ID:$correlation_id, $state, use macro '$macro'");
}

sub check_if_httpdconf_ipaddrs_exist {
    return unless i_am_one_of( 'ea4', 'ea3' );
    my $httpdconf = i_am('ea4') ? '/etc/apache2/conf/httpd.conf' : '/usr/local/apache/conf/httpd.conf';
    return unless -f $httpdconf;
    my @local_ipaddrs = @{ get_local_ipaddrs_aref() };
    my @vhost_ipaddrs;
    my @unbound_ipaddrs;

    if ( open my $httpdconf_fh, '<', $httpdconf ) {
        local $/ = undef;
        my $httpdconf_txt = readline($httpdconf_fh);
        close $httpdconf_fh;
        while ( $httpdconf_txt =~ m/<VirtualHost\s+(\d+\.\d+\.\d+\.\d+):(?:\d+)>/sig ) {
            push @vhost_ipaddrs, $1;
        }
    }

    # uniq IP addrs only
    @vhost_ipaddrs = do {
        my %seen;
        grep { !$seen{$_}++ } @vhost_ipaddrs;
    };

    for my $vhost_ipaddr (@vhost_ipaddrs) {
        my $is_bound = 0;
        for my $local_ipaddr (@local_ipaddrs) {
            if ( $vhost_ipaddr eq $local_ipaddr ) {
                $is_bound = 1;
                last;
            }
        }
        if ( $is_bound == 0 ) {
            push @unbound_ipaddrs, $vhost_ipaddr;
        }
    }

    return unless @unbound_ipaddrs;
    print_warn('Apache: ');
    print_warning('httpd.conf has VirtualHosts for these IP addrs, which aren\'t bound to the server:');

    for my $unbound_ipaddr (@unbound_ipaddrs) {
        print_magenta("\t \\_ $unbound_ipaddr");
    }
}

sub check_distcache_and_libapr {
    return unless i_am('ea3');
    return unless my $httpd_bin = find_httpd_bin();
    my $last_success_profile           = '/var/cpanel/easy/apache/profile/_last_success.yaml';
    my $has_distcache                  = 0;
    my $httpd_not_linked_to_system_apr = 0;

    if ( open my $profile_fh, '<', $last_success_profile ) {
        while (<$profile_fh>) {
            if (/Distcache:/) {
                $has_distcache = 1;
                last;
            }
        }
        close $profile_fh;
    }

    if ($has_distcache) {
        my @ldd = split /\n/, timed_run( 0, 'ldd', $httpd_bin );
        for my $line (@ldd) {
            if ( $line =~ m{ libapr(?:.*) \s+ => \s+ (\S+) }xms ) {
                if ( $1 !~ m{ \A /usr/local/apache/lib/libapr }xms ) {
                    $httpd_not_linked_to_system_apr = 1;
                    last;
                }
            }
        }
    }

    if ($httpd_not_linked_to_system_apr) {
        print_warn('Apache: ');
        print_warning('httpd linked to system APR, not APR in /usr/local/apache/lib/ (see EAL-2551)');
    }
}

sub check_for_custom_repos {
    my $yum_repos_dir = '/etc/yum.repos.d/';
    my $hit           = 0;
    my @dir_contents;
    my %custom_repos = (
        'PostgreSQL' => {
            msg   => '',
            regex => qr{ \A pgdg-(\d+)-centos\.repo }x,
        },
        'City-Fan' => {
            msg   => ' see TECH-716',
            regex => qr{ \A city-fan.org\.repo }x,
        },
        'Jperkster' => {
            msg   => ' see UPS-136',
            regex => qr {\A EA4-freetds.repo }x,
        },
    );

    return if !-d $yum_repos_dir;
    opendir( my $dir_fh, $yum_repos_dir );
    @dir_contents = grep { !/^\.\.?$/ } readdir $dir_fh;
    closedir $dir_fh;

    foreach my $repo (@dir_contents) {
        foreach my $key ( keys %custom_repos ) {
            if ( $repo =~ m/ ( $custom_repos{$key}{regex} ) /xms ) {
                $custom_repos{$key}{file} = $1;
                $hit++;
                print_warn( 'Custom repo(s) found in ' . $yum_repos_dir . ":\n" ) if ( $hit eq 1 );
                print_warning( "\t \\_ $key: " . $custom_repos{$key}{file} . $custom_repos{$key}{msg} );
            }
        }
    }
}

sub check_for_rpm_overrides {
    my $rpm_override_dir = '/var/cpanel/rpm.versions.d/';
    return unless -d $rpm_override_dir;

    my $expected_file_version = '2';
    my $local_versions        = $rpm_override_dir . 'local.versions';
    my $easy_versions         = $rpm_override_dir . 'easy.versions';
    my $cloud_versions        = $rpm_override_dir . 'cloudlinux.versions';
    my $md5_easy;
    my $md5_cloud;
    my $easy_is_default  = 0;
    my $cloud_is_default = 0;

    if ( -f $easy_versions ) {
        $md5_easy = timed_run( 0, 'md5sum', $easy_versions );
    }

    if ( -f $cloud_versions ) {
        $md5_cloud = timed_run( 0, 'md5sum', $cloud_versions );
    }

    ## these are checksums for default files. we ignore them to prevent needless output from SSP
    if (
        defined $md5_easy
        and $md5_easy =~ m{ \A (
                                  350e47b97efd4b75563837b6b3502d71
                                | 436a6863b1a1ec3bb7607066beca7356
                                | 5818611cb4c0bf4086806aced5669f25
                                | 600ff436e5939656a5645c6139cc0228
                                | 657d59cc9627d30a95d0b84ef4245185
                                | 89d631ef7c1d43475c20d7be7b7290ff
                                | d56abe76c47853eceb706f0855e642a7
                                | eed54b4202d0b2655f37a5c1edfa0853
                                ) \s }xms
    ) {
        $easy_is_default = 1;
    }

    if (
        defined $md5_cloud
        and $md5_cloud =~ m{ \A (
                                  23b40252dc77a775d00c87b89a64621a
                                | 956e6d177a790389572c7fcc8b33f8c5
                                | a33f5a902bbadebd7b3029e0337fde35
                                | cf40c6ac1543464a937b8c39c4ad1da6
                                ) \s }xms
    ) {
        $cloud_is_default = 1;
    }

    opendir( my $dir_fh, $rpm_override_dir );
    my @dir_contents = grep { !/^(?:\.\.?|local.versions)$/ } readdir $dir_fh;
    closedir $dir_fh;
    if ( $easy_is_default == 1 ) {
        @dir_contents = grep { $_ ne 'easy.versions' } @dir_contents;
    }
    if ( $cloud_is_default == 1 ) {
        @dir_contents = grep { $_ ne 'cloudlinux.versions' } @dir_contents;
    }
    if (@dir_contents) {
        print_warn( $rpm_override_dir . ': ' );
        print_warning( '[ ' . join( ' ', @dir_contents ) . ' ] may contain custom entries, manually verify. More info: http://go.cpanel.net/rpmversions' );
    }

    return unless -s $local_versions;
    return if !load_module_with_fallbacks(
        'needed_subs'  => [qw{LoadFile}],
        'modules'      => [qw{YAML::Syck}],
        'fail_warning' => 'can\'t check rpm.versions.d/local.versions for customization',
    );

    my $ignore = {
        'target_settings' => {
            'analog'       => '(?:un)?installed',
            'awstats'      => '(?:un)?installed',
            'clamav'       => '(?:un)?installed',
            'easy-icu'     => '(?:un)?installed',
            'easy-tomcat7' => '(?:un)?installed',
            'munin'        => '(?:un)?installed',
            'webalizer'    => '(?:un)?installed',
        },
    };
    if ( i_am('dnsonly') ) {
        $ignore->{'target_settings'}->{'composer'}            = 'uninstalled';
        $ignore->{'target_settings'}->{'cpanel-squirrelmail'} = 'uninstalled';
        $ignore->{'target_settings'}->{'mailman'}             = 'uninstalled';
        $ignore->{'target_settings'}->{'proftpd'}             = 'uninstalled';
        $ignore->{'target_settings'}->{'pure-ftpd'}           = 'uninstalled';
        $ignore->{'target_settings'}->{'roundcube'}           = 'uninstalled';
    }
    if ( cpanel_version_is(qw( >= 11.61.0.0 )) ) {
        $ignore->{'target_settings'}->{'perl522'} = 'uninstalled';    # Normal for WHM 62 upgrade process
    }
    if ( cpanel_version_is(qw( >= 11.69.0.0 )) ) {
        $ignore->{'target_settings'}->{'perl524'} = 'uninstalled';    # Normal for WHM 70 upgrade process
    }
    if ( cpanel_version_is(qw( >= 11.77.0.0 )) ) {
        $ignore->{'target_settings'}->{'perl526'} = 'uninstalled';    # Normal for WHM 78 upgrade process
    }

    my $ref;
    eval { $ref = YAML::Syck::LoadFile($local_versions); };
    if ( defined $ref ) {
        if ( not defined $ref->{'file_format'} or not defined $ref->{'file_format'}->{'version'} ) {
            print_warn( $local_versions . ': ' );
            print_warning('file version is missing!');
        }
        else {
            if ( $ref->{'file_format'}->{'version'} ne $expected_file_version ) {
                print_warn( $local_versions . ': ' );
                print_warning( 'file version is ' . $ref->{'file_format'}->{'version'} . ', expected version ' . $expected_file_version . '!' );
            }
        }
        if ( exists $ref->{'target_settings'} ) {
            foreach my $package ( sort keys %{ $ref->{'target_settings'} } ) {
                next if exists $ignore->{'target_settings'}->{$package} and $ref->{'target_settings'}->{$package} =~ m{ \A $ignore->{'target_settings'}->{$package} \Z }xms;
                print_warn( $local_versions . ': ' );
                print_warning( $package . ' is configured as ' . $ref->{'target_settings'}->{$package} );
            }
        }
        for my $section (qw( install_targets location_keys rpm_groups rpm_locations srpm_sub_packages srpm_versions url_templates )) {
            if ( exists $ref->{$section} and scalar keys %{ $ref->{$section} } ) {
                print_warn( $local_versions . ': ' );
                print_warning( 'Custom ' . $section . ' exists for: [ ' . join( ' ', sort keys %{ $ref->{$section} } ) . ' ]' );
            }
        }
    }
    else {
        print_warn( $local_versions . ': ' );
        print_warning('YAML not successfully imported, corrupted file?');
    }
}

sub check_var_cpanel_immutable_files {
    my $immutable_files = '/var/cpanel/immutable_files';

    if ( -e $immutable_files and not -z $immutable_files ) {
        print_warn('immutable files: ');
        print_warning("$immutable_files is not empty!");
    }
}

sub check_mylogincnf {
    my $logincnf = '/root/.mylogin.cnf';
    return unless ( -e $logincnf );
    print_warn('/root/.mylogin.cnf exists: ');
    print_warning('This file takes precedence over "/root/.my.cnf", see TECH-637');
}

sub check_for_noxsave_in_grub_conf {
    my $grub_conf = '/boot/grub/grub.conf';
    return if !-f $grub_conf;

    if ( open my $grub_fh, '<', $grub_conf ) {
        while (<$grub_fh>) {
            if (/noxsave/) {
                print_warn('noxsave: ');
                print_warning("found in ${grub_conf}. kernel panics? segfaults? see ticket 3689211");
                last;
            }
        }
        close $grub_fh;
    }
}

sub check_for_rpm_dist_ver_unknown {
    my $sysinfo_config = '/var/cpanel/sysinfo.config';
    return unless -f $sysinfo_config;

    if ( open my $file_fh, '<', $sysinfo_config ) {
        while (<$file_fh>) {
            if (/^rpm_dist_ver=unknown$/) {
                print_warn("${sysinfo_config}: ");
                print_warning("contains 'rpm_dist_ver=unknown'. Try running '/scripts/gensysinfo' to fix");
                last;
            }
        }
        close $file_fh;
    }
}

sub check_for_homeloader_php_extension {
    return unless i_am('ea3');
    return unless my $phpini = get_phpini_aref();
    return unless my $ea3_php = get_ea3_php_conf_href();
    return unless defined $ea3_php->{'php5handler'} and $ea3_php->{'php5handler'} eq 'dso';

    if ( grep { m# \A ([\s\t]+)? extension ([\s\t]+)? = ([\s\t]+)? ["']? homeloader\.so ['"]? #xms } @$phpini ) {
        print_warn('/usr/local/lib/php.ini: ');
        print_warning("homeloader.so extension found. This can cause errors. See FB-4471 and FB-63838");
    }
}

sub check_for_networkmanager {
    return unless -s '/etc/ips';
    return unless exists_process_cmd( qr{ NetworkManager }xms, 'root' );
    print_warn('NetworkManager: ');
    print_warning('is running, could disrupt ipaliases service - see "How to Disable Network Manager" documentation');
}

sub check_for_dhclient {
    return unless exists_process_cmd( qr{ dhclient }xms, 'root' );
    print_warn('dhclient: ');
    print_warning('found in the process list');
}

sub check_for_var_cpanel_roundcube_install {
    my $install = '/var/cpanel/roundcube/install';

    return unless ( -f $install and -x $install );
    print_warn('RoundCube: ');
    print_warning("$install exists. /u/l/c/b/update-roundcube won't fully run (by design - see the docs)");
}

sub check_for_missing_etc_localtime {
    stat('/etc/localtime');
    if ( not -f _ or not -s _ ) {
        print_warn('/etc/localtime: ');
        print_warning('Missing or empty! Can break many things including upcp and stats (see tickets 3811269, 7092613, 7124961)');
    }
}

sub check_for_perl_env_var {
    if ( exists( $ENV{'PERL5LIB'} ) ) {
        print_warn('PERL5LIB env var: ');
        print_warning('exists! This can break cPanel\'s perl or EA3. See FB-64265');
    }
    if ( exists( $ENV{'PERL_LOCAL_LIB_ROOT'} ) ) {
        print_warn('PERL_LOCAL_LIB_ROOT env var: ');
        print_warning('exists! This can break cPanel\'s perl or EA3.');
    }
    if ( exists( $ENV{'PERL_MB_OPT'} ) ) {
        print_warn('PERL_MB_OPT env var: ');
        print_warning('exists! This can break cPanel\'s perl or EA3.');
    }
    if ( exists( $ENV{'PERL_MM_OPT'} ) ) {
        print_warn('PERL_MM_OPT env var: ');
        print_warning('exists! This can break cPanel\'s perl or EA3.');
    }
    if ( grep { /\/perl5?\// } $ORIGINAL_PATH ) {
        print_warn('Custom perl in PATH env var: ');
        print_warning('exists! This can break EA3 or cause other unexpected system-perl behavior.');
    }
}

sub check_for_disabled_services {
    my %disabled_services;

    my %touchfiles = (
        '/etc/antirelayddisable'                                 => 'antirelayd',
        '/var/cpanel/ssl/disable_auto_hostname_certificate'      => '/var/cpanel/ssl/disable_auto_hostname_certificate',
        '/etc/rrdtooldisable'                                    => 'rrdtool',
        '/var/run/chkservd.suspend'                              => 'chksrvd',
        '/etc/checkyumdisable'                                   => 'checkyum',
        '/etc/clamddisable'                                      => 'clamd',
        '/etc/cppopdisable'                                      => 'courier',
        '/etc/pop3disable'                                       => 'courier',
        '/etc/popddisable'                                       => 'courier',
        '/etc/popdisable'                                        => 'courier',
        '/etc/cpanel-dovecot-solrdisable'                        => 'cpanel-dovecot-solr',
        '/etc/cpanellogddisable'                                 => 'cpanellogd',
        '/etc/cpdavddisable'                                     => 'cpdavd',
        '/etc/cpsrvdddisable'                                    => 'cpsrvd',
        '/etc/binddisable'                                       => 'dnsserver',
        '/etc/dnsdisable'                                        => 'dnsserver',
        '/etc/nameddisable'                                      => 'dnsserver',
        '/etc/eximdisable'                                       => 'exim',
        '/etc/exiscandisable'                                    => 'exiscan',
        '/etc/disablehackcheck'                                  => 'hackcheck',
        '/etc/ftpddisable'                                       => 'ftpd',
        '/etc/ftpserverdisable'                                  => 'ftpd',
        '/var/cpanel/ssl/disable_hostname_mismatch_check'        => '/var/cpanel/ssl/disable_hostname_mismatch_check',
        '/etc/apachedisable'                                     => 'httpd',
        '/etc/httpddisable'                                      => 'httpd',
        '/etc/httpdisable'                                       => 'httpd',
        '/etc/httpdisevil'                                       => 'httpd',
        '/etc/cpimapdisable'                                     => 'imapd',
        '/etc/imapddisable'                                      => 'imapd',
        '/etc/imapdisable'                                       => 'imapd',
        '/etc/ipaliasesdisable'                                  => 'ipaliases',
        '/etc/mailmandisable'                                    => 'mailman',
        '/etc/mydnsdisable'                                      => 'mydns',
        '/etc/mysqldisable'                                      => 'mysql',
        '/etc/nsddisable'                                        => 'nsd',
        '/etc/postgresdisable'                                   => 'postgresql',
        '/etc/postgresqldisable'                                 => 'postgresql',
        '/etc/postmasterdisable'                                 => 'postgresql',
        '/etc/proftpddisable'                                    => 'proftpd',
        '/etc/pureftpddisable'                                   => 'pureftpd',
        '/etc/pure-ftpddisable'                                  => 'pureftpd',
        '/etc/queueprocddisable'                                 => 'queueprocd',
        '/etc/rsyslogddisable'                                   => 'rsyslogd',
        '/etc/rsyslogdisable'                                    => 'rsyslogd',
        '/etc/securemysqldisable'                                => 'securemysql',
        '/var/cpanel/version/securetmp_disabled'                 => 'securetmp',
        '/var/cpanel/ssl/disable_service_certificate_management' => '/var/cpanel/ssl/disable_service_certificate_management',
        '/etc/spamddisable'                                      => 'spamd',
        '/etc/spamdisable'                                       => 'spamd',
        '/etc/sshddisable'                                       => 'sshd',
        '/etc/syslogddisable'                                    => 'syslogd',
        '/etc/syslogdisable'                                     => 'syslogd',
        '/etc/tailwatchddisable'                                 => 'tailwatchd',
        '/etc/tailwatchdisable'                                  => 'tailwatchd',
        '/etc/tomcatdisable'                                     => 'tomcat',
        '/var/cpanel/disabled/secureit'                          => 'secureit',
        '/var/cpanel/disabled/securetmp'                         => 'securetmp',
        '/var/cpanel/disabled/auto-restart-services'             => '/var/cpanel/disabled/auto-restart-services',
    );

    my %cpconfcheck = (
        'skipapnspush'                  => 'apnspush',
        'skipchkservd'                  => 'chkservd',
        'skipcpbandwd'                  => 'cpbandwd',
        'skipeximstats'                 => 'eximstats',
        'skipmailhealth'                => 'mailhealth',
        'skipmailman'                   => 'mailman',
        'skipmodseclog'                 => 'modseclog',
        'skiprecentauthedmailiptracker' => 'recentauthedmailiptracker',
        'skiptailwatchd'                => 'tailwatchd',
    );

    while ( my ( $touchfile, $service ) = each(%touchfiles) ) {
        if ( -e $touchfile ) {
            $disabled_services{$service} = 1;
        }
    }

    while ( my ( $skip, $service ) = each(%cpconfcheck) ) {
        if ( $CPCONF{$skip} ) {
            $disabled_services{$service} = 1;
        }
    }

    return if !%disabled_services;

    print_warn('Disabled services: ');
    print_warning( '[ ' . join( ' ', sort( keys(%disabled_services) ) ) . ' ]' );
}

sub get_license_info {
    return unless my $external_ip_address = get_external_ip();
    my $path = '/api/ipaddrs?ip=' . $external_ip_address;
    my $host = 'verify.cpanel.net';
    return _http_get( Host => $host, Path => $path, ReportTimeout => 1, SSL => 1, SSLLoadFailWarning => 'can\'t make HTTPS request to verify.cpanel.net' );
}

sub check_for_license_info {
    check_for_license_status_json();     ## 92+
    my %license;
    my $host                     = 'verify.cpanel.net';
    my $helper_url               = "https://" . $host . '/app/verify?ip=';
    my $file_is_solo             = license_file_is_solo();
    my $file_is_dnsonly          = license_file_is_dnsonly();
    my $dnsonly_touchfile        = '/var/cpanel/dnsonly';
    my $license_query            = get_license_info();
    my $license_href             = get_json_href($license_query);
    my $external_ip_address      = get_external_ip();
    my $external_license_address = get_external_license_ip();

    if ( defined($external_ip_address) ) {
        $helper_url .= $external_ip_address;
        foreach my $key ( @{ $license_href->{current} } ) {
            if ( defined( $key->{'status'} ) && $key->{'status'} eq 1 ) {
                ++$license{ $key->{'product'} };
                ++$license{ $key->{'product'} } if ( defined( $key->{'package'} ) && $key->{'package'} eq 'ONE TIME FEE' );
            }
        }
        if ( my $licenses = join " ", map { "[$_]" } sort keys %license ) {
            print_info('License: ');
            print_normal("$external_ip_address has $licenses");
        }
        else {
            print_info('License: ');
            print_normal("Not found on verify.cpanel.net, or request timed out -- verify at ${helper_url}");
        }
        foreach my $key ( sort keys %license ) {
            if ( $license{$key} > 1 ) {
                print_info('License: ');
                print_normal("[$key] may be a one-time license -- manually verify at [ ${helper_url} ]");
            }
        }
        if ( ( defined($external_license_address) ) && !( $external_ip_address eq $external_license_address ) ) {
            print_crit('License: ');
            print_critical("external IP detected via port 80 [ $external_ip_address ] does not match IP detected via port 2089 [ $external_license_address ] which can result in unexpected cPanel license update behavior.");
        }
    }
    if ($file_is_solo) {
        print_info('License: ');
        print_normal('cPanel Solo (limited to 1 account)');
    }

    if ( cpanel_version_is(qw ( >= 11.68.0.25 )) ) {
        if ( not $file_is_dnsonly and -e $dnsonly_touchfile ) {
            print_crit('License: ');
            print_critical("$dnsonly_touchfile exists but installed license file is not DNSONLY.  Try using '/usr/local/cpanel/cpkeyclt --force' to update license.");
        }
        if ( $file_is_dnsonly and not -e $dnsonly_touchfile ) {
            print_crit('License: ');
            print_critical("Installed license file is DNSONLY but $dnsonly_touchfile does not exist.  Try using '/usr/local/cpanel/cpkeyclt --force' to update license.");
        }
    }

    if ( i_am('cloudlinux') and not exists( $license{'CloudLinux'} ) ) {
        print_warn('CloudLinux: ');
        print_warning(qq{MAY NOT BE LICENSED! - verify at $helper_url - use "LICENSE - CloudLinux Not Licensed through cPanel" if relevant});
    }

    if ( i_am('kernelcare') and not exists( $license{'KernelCare'} ) ) {
        print_warn('KernelCare: ');
        print_warning(qq{MAY NOT BE LICENSED! - verify at $helper_url - use "LICENSE - Kernelcare Not Licensed through cPanel" if relevant});
    }
    if ( i_am('wptk') ) {
        my $wptk_version_file = '/usr/local/cpanel/3rdparty/wp-toolkit/build.json';
        return unless ( -e $wptk_version_file );
        return unless ( open( my $version_fh, '<', $wptk_version_file ) );
        my $raw;
        while (<$version_fh>) {
            $raw .= $_;
        }
        close($version_fh);
        my $wptk_version_info = get_json_href($raw);
        my $wptk_build        = ( $wptk_version_info->{buildNumber} ) ? $wptk_version_info->{buildNumber} : "build";
        my $wptk_version      = ( $wptk_version_info->{version} ) ? $wptk_version_info->{version} : "Unknown Version";
        my $wptk_deluxe       = ( $license{'WordPress Toolkit'} ) ? " [Deluxe]" : " [Lite]";
        print_info('WordPress-Toolkit version: ');
        print_normal( $wptk_version . "-" . $wptk_build . $wptk_deluxe );
    }
}

sub check_for_license_status_json() {
    return if ! -e "/var/cpanel/license.status.json";
    my $licstatus;
    if ( open my $file_fh, '<', "/var/cpanel/license.status.json" ) {
        while (<$file_fh>) {
            $licstatus = get_json_href($_);
        }
        close($file_fh);
    }
    return unless( defined $licstatus->{code} && $licstatus->{code} != 200 );
    print_warn('License: ');
    print_warning("Last cpkeyclt check failed: " . BOLD CYAN $licstatus->{message} );
    return;
}

# Yes, this actually happened...
sub check_for_cpbackup_exclude_everything {
    my $conf = '/etc/cpbackup-exclude.conf';
    return unless -f $conf;

    open my $conf_fh, '<', $conf or return;
    while (<$conf_fh>) {
        chomp;
        if (/^\*$/) {
            print_warn('Backups: ');
            print_warning("'*' exists by itself in $conf . This can cause 0 byte backups");
            last;
        }
        elsif (/\x00/) {
            print_warn('Backups: ');
            print_warning('/etc/cpbackup-exclude.conf contains NUL bytes - can result in homedir being skipped. See CPANEL-17065.');
            last;
        }
    }
    close $conf_fh;
}

sub check_for_usr_local_include_jpeglib_h {
    return unless i_am('ea3');
    my $jpeglib = '/usr/local/include/jpeglib.h';
    if ( -f $jpeglib ) {
        print_warn("$jpeglib: ");
        print_warning('Seeing "Wrong JPEG library version"? This file may be the cause. See ticket 4159697');
    }
}

sub check_for_bw_module_and_more_than_1024_vhosts {
    return unless i_am_one_of( 'ea4', 'ea3' );
    my $modules = get_apache_modules_href();
    return unless defined $modules->{'bw_module'};

    # Remove on EA3 deprecation
    my $httpdconf = i_am('ea4') ? '/etc/apache2/conf/httpd.conf' : '/usr/local/apache/conf/httpd.conf';
    return if !-f $httpdconf;

    my $num_vhosts = 0;

    open my $httpdconf_fh, '<', $httpdconf or return;
    while (<$httpdconf_fh>) {
        if (m{ \A (?:\s+)? <VirtualHost \s }xms) {
            $num_vhosts++;
        }
    }
    close $httpdconf_fh;

    if ( $num_vhosts and $num_vhosts > 1024 ) {
        print_warn('bw_module: ');
        print_warning("loaded, and httpd.conf has >1024 VirtualHosts ($num_vhosts). Apache failing to start? See EAL-2347");
    }
}

sub check_for_uppercase_chars_in_hostname {
    if ( get_hostname() =~ /[A-Z]/ ) {
        print_warn('Hostname: ');
        print_warning('contains UPPERCASE characters. Seeing incorrect info at cPanel >> Configure Email Client? See ticket 4231465');
    }
}

sub check_for_harmful_php_mode_600_cron {
    return unless i_am('cpanel');
    return unless -d '/etc/cron.daily';

    my @dir_contents;
    opendir( my $dir_fh, '/etc/cron.daily' ) or return;
    @dir_contents = grep { !/^\.\.?$/ } readdir $dir_fh;
    closedir $dir_fh;

    for my $file (@dir_contents) {
        $file = '/etc/cron.daily/' . $file;
        open my $file_fh, '<', $file or next;
        while (<$file_fh>) {
            if (/^mytmpfile=\/tmp\/php-mode-/) {
                print_warn('harmful cron: ');
                print_warning("${file}! Breaks webmail, phpMyAdmin, and more! See tickets 4225765, 4237465, 4099807, 4231469, 4231473. Vendor: http://whmscripts.net/misc/2013/apache-symlink-security-issue-fixpatch/");
                last;
            }
        }
        close $file_fh;
    }
}

sub check_for_bad_permissions_on_named_ca {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    my $namedca = '/var/named/named.ca';
    if ( !-e $namedca ) {
        print_warn("${namedca}: ");
        print_warning('missing. named may not start without it');
        return;
    }

    my ( $mode, $uid, $gid ) = ( stat('/var/named/named.ca') )[ 2, 4, 5 ];
    my $world_readable_bit = $mode & oct(4);
    my $user               = getpwuid($uid);
    my $group              = getgrgid($gid);

    if ( not( $user eq 'named' or $group eq 'named' ) and ( $world_readable_bit == 0 ) ) {
        print_warn("${namedca}: ");
        print_warning('may not be readable to the \'named\' user, causing named to not restart');
    }
}

sub check_for_jailshell_additional_mounts_trailing_slash {
    return unless i_am('cpanel');
    my $mounts_file = '/var/cpanel/jailshell-additional-mounts';
    return if ( !-f $mounts_file );

    if ( open my $file_fh, '<', $mounts_file ) {
        while (<$file_fh>) {
            chomp;
            if (m#/(?:[\s\t]+)?\z#) {
                print_warn("$mounts_file: ");
                print_warning('contains trailing slashes! Server may become unstable. See FB-71613');
                last;
            }
        }
        close $file_fh;
    }
}

sub check_for_allow_query_localhost {
    my $named_conf = '/etc/named.conf';
    return if !-f $named_conf;

    my $namedconf_contents;
    if ( open my $named_conf_fh, '<', $named_conf ) {
        local $/;
        $namedconf_contents = <$named_conf_fh>;
        close $named_conf_fh;
    }
    return unless $namedconf_contents;

    if ( $namedconf_contents =~ m#allow-query ([\s\t\r\n]+)? { ([\s\t]+)? ( localhost | 127\. )#xms ) {
        print_warn('named.conf: ');
        print_warning('allow-query is restricted to localhost. Remote DNS queries may not work');
    }
}

sub check_for_nocloudlinux_touchfile {
    if ( -e '/var/cpanel/nocloudlinux' && -e '/var/cpanel/disabled/cloudlinux' ) {
        print_warn('nocloudlinux: ');
        print_warning('\'/var/cpanel/nocloudlinux\' and \'/var/cpanel/disabled/cloudlinux\' exist! These block cPanel from installing/updating CloudLinux via:');
        print_warning( "\t \\_ " . '/usr/local/cpanel/scripts/cpanel_initial_install' );
        print_warning( "\t \\_ " . '/usr/local/cpanel/bin/cloudlinux_update' );
        print_warning( "\t \\_ " . '/usr/local/cpanel/bin/cloudlinux_system_install' );
        return;
    }
    elsif ( -e '/var/cpanel/nocloudlinux' ) {
        print_warn('nocloudlinux: ');
        print_warning('\'/var/cpanel/nocloudlinux\' exists! This blocks cPanel from installing/updating CloudLinux via:');
        print_warning( "\t \\_ " . '/usr/local/cpanel/scripts/cpanel_initial_install' );
        print_warning( "\t \\_ " . '/usr/local/cpanel/bin/cloudlinux_update' );
        print_warning( "\t \\_ " . '/usr/local/cpanel/bin/cloudlinux_system_install' );
        return;
    }
    elsif ( -e '/var/cpanel/disabled/cloudlinux' ) {
        print_warn('nocloudlinux: ');
        print_warning('\'/var/cpanel/disabled/cloudlinux\' exists! This blocks cPanel from installing/updating CloudLinux via:');
        print_warning( "\t \\_ " . '/usr/local/cpanel/scripts/cpanel_initial_install' );
        print_warning( "\t \\_ " . '/usr/local/cpanel/bin/cloudlinux_update' );
    }
}

sub check_for_stupid_touchfile {
    return if !-e '/etc/allowstupidstuff';
    print_warn('/etc/allowstupidstuff: ');
    print_warning('exists! Can allow usernames to be created that begin with digits.');
}

sub check_for_dev_sandbox {
    my @touchfiles = qw{
      /var/cpanel/align_memory_to_arch
      /var/cpanel/dev_sandbox
    };
    my @found = grep { -e $_ } @touchfiles;
    for my $touchfile (@found) {
        print_warn( $touchfile . ': ' );
        print_warning('Should NEVER exist on production servers.');
    }
}

sub check_for_jail_owner {
    return unless -e '/jail_owner';
    print_warn('/jail_owner: ');
    print_warning('exists! Should NEVER exist outside of jailshell.  Will cause Exim mail delivery issues.');
}

sub check_for_phphandler_and_opcode_caching_incompatibility {
    return unless i_am('ea3');
    return unless my $phpini = get_phpini_aref();
    return unless my $ea3_php = get_ea3_php_conf_href();
    return unless defined $ea3_php->{'php5handler'} and $ea3_php->{'php5handler'} eq 'suphp';

    my $message;

    if ( grep { m# \A ([\s\t]+)? extension ([\s\t]+)? = ([\s\t]+)? ["']? eaccelerator\.so ['"]? #xms } @$phpini ) {
        $message .= '[eAccelerator] ';
    }
    if ( grep { m# \A ([\s\t]+)? extension ([\s\t]+)? = ([\s\t]+)? ["']? xcache\.so ['"]? #xms } @$phpini ) {
        $message .= '[XCache] ';
    }
    if ( grep { m# \A ([\s\t]+)? extension ([\s\t]+)? = ([\s\t]+)? ["']? apc\.so ['"]? #xms } @$phpini ) {
        $message .= '[APC] ';
    }

    if ($message) {
        print_warn('PHP: ');
        print_warning("suPHP enabled, but the following installed opcode cachers are not suPHP compatible: $message");
    }
}

sub check_for_invalid_HOMEDIR {
    my $wwwacctconf = '/etc/wwwacct.conf';
    return unless -f $wwwacctconf;

    my $homedir;
    if ( open my $file_fh, '<', $wwwacctconf ) {
        while (<$file_fh>) {
            if (/\AHOMEDIR[\s\t]+([^\s]+)/) {
                $homedir = $1;
                last;
            }
        }
        close $file_fh;
    }

    if ( !$homedir ) {
        print_warn("$wwwacctconf: ");
        print_warning('HOMEDIR value not found!');
        return;
    }

    if ( !-d $homedir ) {
        print_warn("$wwwacctconf: ");
        print_warning("the directory that is specified as the HOMEDIR does not exist! ($homedir)");
    }
}

sub check_for_unsupported_options_in_phpini {    # FB-75397
    return unless i_am('ea3');
    return unless my $phpini = get_phpini_aref();
    return unless my $ea3_php = get_ea3_php_conf_href();
    return unless defined $ea3_php->{'php5version'};

    my ( undef, $php5minor ) = split /\./, $ea3_php->{'php5version'};
    return if ( !$php5minor || $php5minor <= 3 );

    my $unsupported_options;

    ## http://www.php.net/manual/en/migration54.ini.php
    ## apparently "safe_mode = off" won't trigger 75397, but "safe_mode = on" will.
    ## some items like "y2k_compliance = On" don't appear to trigger the issue
    if ( grep { m# \A (?:[\s\t]+)? register_globals (?:[\s\t]+)? = (?:[\s\t]+)? ["']? on ['"]? #ixms } @$phpini ) {
        $unsupported_options .= "[register_globals] ";
    }

    if ( grep { m# \A (?:[\s\t]+)? safe_mode (?:[\s\t]+)? = (?:[\s\t]+)? ["']? on ['"]? #ixms } @$phpini ) {
        $unsupported_options .= "[safe_mode] ";
    }

    if ($unsupported_options) {
        $unsupported_options =~ s/\s$//g;
        print_warn('/usr/local/lib/php.ini: ');
        print_warning("PHP $ea3_php->{'php5version'} does not support $unsupported_options, but found enabled in php.ini. See FB-75397");
    }
}

sub check_for_suphp_but_no_fileprotect {
    return unless i_am('ea3');
    return if -e '/var/cpanel/fileprotect';
    return unless my $ea3_php = get_ea3_php_conf_href();
    return unless defined $ea3_php->{'php5handler'} and $ea3_php->{'php5handler'} eq 'suphp';
    print_warn('suPHP: ');
    print_warning("enabled, but /var/cpanel/fileprotect not found. New users' public_htmls will be user:user");
}

sub check_if_hostname_missing_from_localdomains {
    return unless i_am('cpanel');
    my $hostname = get_hostname();
    if ( open my $localdomains_fh, '<', '/etc/localdomains' ) {
        while (<$localdomains_fh>) {
            return if (/^${hostname}$/);
        }
        close $localdomains_fh;
    }
    print_warn('Hostname: ');
    print_warning('not found in /etc/localdomains. This can cause "lowest numbered MX record points to local host"');
}

sub check_for_eximstats_newline {
    return unless i_am('cpanel');
    return unless cpanel_version_is(qw( < 11.63.0.0 ));
    my $eximstatspass = '/var/cpanel/eximstatspass';
    if ( !-e $eximstatspass ) {
        print_warn("$eximstatspass: ");
        print_warning('missing!');
        return;
    }
    if ( open my $eximstatspass_fh, '<', $eximstatspass ) {
        while (<$eximstatspass_fh>) {
            if (/\n/) {
                print_warn("$eximstatspass: ");
                print_warning('contains a newline. Breaks Mail Delivery Reports / eximstats');
                last;
            }
        }
        close $eximstatspass_fh;
    }
}

sub check_for_processes_killed_by_lfd {
    my $log = '/var/log/lfd.log';
    return if !-e $log;

    my $size = ( stat($log) )[7];
    return if !$size;

    my $bytes_to_check = 20_971_520 / 2;    # 10M
    my $seek_position  = 0;
    my $log_data;
    my $count = 0;
    my @killed_by_lfd;

    if ( $size > $bytes_to_check ) {
        $seek_position = ( $size - $bytes_to_check );
    }

    open my $file_fh, '<', $log or return;
    seek $file_fh, $seek_position, 0;
    read $file_fh, $log_data, $bytes_to_check;
    close $file_fh;

    if ( $log_data =~ /\sKill:1\s/ ) {
        my @logs = split /\n/, $log_data;
        undef $log_data;
        @logs = reverse @logs;

        for my $line (@logs) {
            if ( $line =~ /(.*?)[ \t]+\*User Processing\*[ \t]+(.*?)[ \t]+CMD:(.*)/ ) {
                my $header   = $1;
                my %keypairs = map { ( split( m{:}, $_, 2 ) )[ 0, 1 ] } split( m{[ \t]+}, $2 );
                my $cmd      = $3;
                next if $keypairs{"Kill"} == "0";
                push @killed_by_lfd, "[$header] " . join( " ", ( map { "[$_: $keypairs{$_}]" } sort keys %keypairs ) ) . " [cmd: $cmd]\n";
                $count++;
            }
            last if $count >= 20;
        }
    }

    if (@killed_by_lfd) {
        chomp @killed_by_lfd;
        print_warn("Last 20 processes killed by 3rd party software \"LFD\" (\"grep Kill:1 /var/log/lfd.log\"):\n");
        for my $killed_process (@killed_by_lfd) {
            print_magenta("\t \\_ $killed_process");
        }
    }
}

sub check_for_processes_killed_by_oom {
    my $log = "/var/log/messages";
    return unless -e $log;

    my $size = ( stat($log) )[7];
    return unless defined $size;

    my $seek_position  = 0;
    my $bytes_to_check = 10485760;
    if ( $size > $bytes_to_check ) {
        $seek_position = ( $size - $bytes_to_check );
    }

    my $log_data;
    my $count         = 0;
    my @killed_by_oom = ();

    open my $file_fh, '<', $log or return;
    seek $file_fh, $seek_position, 0;
    read $file_fh, $log_data, $bytes_to_check;
    close $file_fh;

    my $prefix_pattern = '^([a-zA-Z]{3} +\d{1,2} +\d{2}:\d{2}:\d{2}) (?:.*?) kernel:(?: \[ *\d*\.\d{6}\])?';

    if ( $log_data =~ /[Kk]illed process/ ) {
        my @logs = split /\n/, $log_data;
        undef $log_data;
        @logs = reverse @logs;

        for my $line (@logs) {

            # CloudLinux 6 LVE OOM, Virtuozzo + CentOS 6 OOM
            if ( $line =~ /$prefix_pattern Out of memory in UB (\d*): OOM killed process (\d*) \(([^)]+)\) score \d* vm:(\d*)kB, rss:(\d*)kB, swap:(\d*)kB/o ) {
                push @killed_by_oom, "[$1] [pid: $3] [cmd: $4] [LVE ID: $2] [vm: $5 kB] [rss: $6 kB] [swap: $7 kB]";
                $count++;
                last if $count >= 10;
            }

            if ( $line =~ /$prefix_pattern Out of memory: OOM killed process (\d*?) \(([^)]+)\) score \d* vm:(\d*)kB, rss:(\d*)kB, swap:(\d*)kB/o ) {
                push @killed_by_oom, "[$1] [pid: $2] [cmd: $3] [vm: $4 kB] [rss: $5 kB] [swap: $6 kB]";
                $count++;
                last if $count >= 10;

            }

            # CloudLinux 7
            # Feb  6 08:52:43 lin02 kernel: Killed process 22541 (php) in VE "0" total-vm:357304kB, anon-rss:77080kB, file-rss:11116kB
            if ( $line =~ /$prefix_pattern Killed process (\d*) \(([^)]+)\) in VE "(\d*)" total-vm:(\d*)kB, anon-rss:(\d*)kB, file-rss:(\d*)kB/o ) {
                push @killed_by_oom, "[$1] [pid: $2] [cmd: $3] [LVE ID: $4] [total-vm: $5 kB] [anon-rss: $6 kB] [file-rss: $7 kB]";
                $count++;
                last if $count >= 10;
            }

            # CentOS 6/7
            # Oct  8 12:51:37 whm-76-c7 kernel: Killed process 3172 (a.out) total-vm:571097852kB, anon-rss:547696kB, file-rss:4kB, shmem-rss:0kB
            # Oct  3 18:33:48 petersdevbox kernel: OOM killed process 14862 (spamd child) total-vm:246308kB, anon-rss:120088kB, file-rss:1168kB
            if ( $line =~ /$prefix_pattern (?:OOM\s)?[Kk]illed process (\d+)(?:, UID \d+,)? \(([^)]+)\) total-vm:(\d+)kB, anon-rss:(\d+)kB, file-rss:(\d+)kB/o ) {
                push @killed_by_oom, "[$1] [pid: $2] [cmd: $3] [total-vm: $4 kB] [anon-rss: $5 kB] [file-rss: $6 kB]";
                $count++;
                last if $count >= 10;
            }
        }

        if (@killed_by_oom) {
            print_warn("Last $count processes killed by Linux Out of memory killer (grep -i \"killed process\" /var/log/messages):\n");
            for my $killed_process (@killed_by_oom) {
                print_magenta("\t \\_ $killed_process");
            }
        }
    }
}

sub check_for_processes_killed_by_prm {
    my @logfiles = qw( /usr/local/prm/prm_log /usr/local/prm/log_prm /usr/local/prm/logs/prm.log );
    for my $log (@logfiles) {
        next if -l $log;
        next if !-f $log;
        my $size = ( stat($log) )[7];
        next if !$size;

        my $bytes_to_check = 10485760;    # 10M
        my $seek_position  = 0;
        my $log_data;
        my $count         = 0;
        my @killed_by_prm = ();

        if ( $size > $bytes_to_check ) {
            $seek_position = ( $size - $bytes_to_check );
        }

        open my $file_fh, '<', $log or return;
        seek $file_fh, $seek_position, 0;
        read $file_fh, $log_data, $bytes_to_check;
        close $file_fh;

        if ( $log_data =~ /\sKILLED\s/ ) {
            my @logs = split /\n/, $log_data;
            undef $log_data;
            @logs = reverse @logs;

            for my $line (@logs) {
                if ( $line =~ /(\S+\s\d+\s\S+)\s(?:\S+)\s(?:\S+) proc pid:(?:\d+) \{user:(\S+) cmd:(\S+)\}.+(MAX_.+) KILLED/ ) {
                    push @killed_by_prm, "[$1] [user: $2] [cmd: $3] [$4]";
                    $count++;
                }

                last if $count >= 10;
            }
        }
        if (@killed_by_prm) {
            print_warn( "Last 10 processes killed by 3rd party software \"PRM\" (\"grep KILLED " . $log . "\"):\n" );
            for my $killed_process (@killed_by_prm) {
                print_magenta("\"\t \\_ $killed_process\"");
            }
        }
    }
}

sub check_for_broken_userdatadomains {
    return unless i_am('cpanel');
    return unless -f '/etc/userdatadomains';

    open my $userdatadomains_fh, '<', '/etc/userdatadomains' or return;
    while (<$userdatadomains_fh>) {
        if (/^:/) {
            print_warn('/etc/userdatadomains: ');
            print_warning('contains a line that begins with ":". Check the following for accuracy (see 4416539 for examples):');
            print_magenta("\t \\_ /etc/userdatadomains");
            print_magenta("\t \\_ /var/cpanel/users/USER (check the ^DNS= lines)");
            print_magenta("\t \\_ /var/cpanel/userdata/USER/main (check for things like '')");
            print_magenta("\t \\_ /var/cpanel/userdata/USER/DOMAIN (check serveralias line)");
            print_magenta("\t \\_ /var/cpanel/userdata/USER/cache (userdatadomains uses this)");
            print_magenta("\t \\_ /usr/local/apache/conf/httpd.conf (may need rebuilding after fixing userdata)");
            last;
        }
    }
    close $userdatadomains_fh;
}

sub check_ssl_db_perms {
    return unless i_am('cpanel');
    my $ssldb = '/var/cpanel/ssl/installed/ssl.db';
    return unless -e $ssldb;

    my ( $uid, $gid ) = ( stat($ssldb) )[ 4, 5 ];
    if ( $uid != 0 or $gid != 0 ) {
        print_warn("$ssldb: ");
        print_warning('not owned by the root user and/or group. This can prevent pkgacct from completing. See ticket 4422237');
    }
}

sub check_for_stray_index_php {
    return unless i_am('cpanel');
    my $indexphp = '/usr/local/cpanel/base/index.php';
    if ( -e $indexphp ) {
        print_warn("$indexphp: ");
        print_warning("exists! Errors when logging into cPanel? See ticket 4421775");
    }
}

sub check_for_port_80_not_apache {
    return unless i_am_one_of( 'ea4', 'ea3' );
    return unless my $ports = get_lsof_port_href();
    return unless exists $ports->{'80'};

    my ( $pid, $comm ) = @{ $ports->{'80'}->[0] }{qw( PID CMD )};
    my $exe = readlink "/proc/${pid}/exe" or return;
    my $cwd = readlink "/proc/${pid}/cwd" or return;
    if ( $exe ne '/usr/local/apache/bin/httpd' && $exe ne '/usr/sbin/httpd' ) {

        open my $file_fh, '<', "/proc/${pid}/cmdline" or return;
        my $cmdline = readline $file_fh;
        close $file_fh;

        return if !$cmdline;
        $cmdline =~ s/\0/ /g;
        $cmdline =~ s/(\s+)$//g;

        my $ipcs = timed_run( 0, 'ipcs', '-m' );

        print_warn('Port 80: ');

        print_warning("something other than Apache is running:");
        print_magenta("\t \\_ cmd     [$comm]");
        print_magenta("\t \\_ exe     [$exe]");
        print_magenta("\t \\_ cmdline [$cmdline]");
        print_magenta("\t \\_ cwd     [$cwd]");
        print_magenta($ipcs) if ( $ipcs =~ /nobody/ );
    }
}

sub check_for_missing_groups {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    my @groups = qw(
      cpanel
      cpaneleximfilter
      cpaneleximscanner
      cpanellogin
      cpanelphpmyadmin
      cpanelphppgadmin
      cpanelroundcube
      cpses
      mail
      mailman
      mailnull
      mailtrap
      mysql
      named
      nobody
      root
      sshd
      wheel
    );

    my $missing_groups;

    for my $group (@groups) {
        my $gid = getgrnam($group);
        next if ( defined $gid and $gid =~ /^\d+$/ );
        $missing_groups .= "[$group] ";
    }

    if ($missing_groups) {
        print_warn('Missing groups: ');
        print_warning($missing_groups);
    }
}

sub check_for_noquotafs {
    my $noquotafs = '/var/cpanel/noquotafs';
    return if not -f $noquotafs or -z $noquotafs;

    print_warn("$noquotafs: ");
    print_warning('exists. quota issues? See https://documentation.cpanel.net/display/ALD/The+Quota+File+Systems+Configuration+File');
}

sub check_for_roundcube_overlay {
    my $rcdir = '/var/cpanel/roundcube';
    return unless -d $rcdir;

    opendir( my $dh, $rcdir );
    my @contents = grep { !/^\.\.?$/ } readdir $dh;
    close $dh;

    if ( grep { /^overlay/ } @contents ) {
        print_warn('Roundcube overlay: ');
        print_warning("found an overlay file in $rcdir . Login issues? See ticket 4542785.");
    }
}

sub check_for_hostname_park_zoneexists {
    return unless i_am('cpanel');
    my $hostname = get_hostname();
    if ( -f "/var/named/$hostname.db" and ( not defined $CPCONF{'allowparkonothers'} or $CPCONF{'allowparkonothers'} != 1 ) ) {
        print_warn('Parking: ');
        print_warning('since zone of hostname exists, "Allow cPanel users to create subdomains across accounts" must be ON to park on the hostname');
    }
}

sub check_for_pgpass_colon_in_password_field {
    my $pgpass = '/root/.pgpass';
    return if !-f $pgpass;

    if ( open my $fh, '<', $pgpass ) {
        while (<$fh>) {
            if (/^\*:\*:\*:postgres:(.*)/) {
                if ( $1 =~ /:/ ) {
                    print_warn("$pgpass: ");
                    print_warning('password field contains a colon. Seeing Postgres auth issues? See FB-89093');
                    last;
                }
            }
        }
        close $fh;
    }
}

sub check_for_dirs_that_break_ea {
    return unless i_am('ea3') or ea3_downgrade_is_possible();
    my @dirs = qw(
      /usr/local/cpanel/cgi-sys/php5
      /var/cpanel/conf/apache/wrappers/php5
    );

    for my $dir (@dirs) {
        if ( -d $dir ) {
            print_warn("$dir: ");
            print_warning('is a directory! This can cause EA3 issues. See ticket 4537779.');
        }
    }
}

sub check_for_extra_uid_0_user {
    my @uid_0_users;
    open my $file_fh, '<', '/etc/passwd' or die $!;
    while (<$file_fh>) {
        if (/\A([^:]+):[^:]+:0:/) {
            next if $1 eq 'root';
            push @uid_0_users, $1;
            last if scalar @uid_0_users >= 5;
        }
    }
    close $file_fh;

    if (@uid_0_users) {
        my $info;
        print_warn('non-root UID 0 user(s) found:');
        for my $user (@uid_0_users) {
            $info .= ' [' . $user . ']';
            next unless ( scalar @uid_0_users < 6 );
        }
        $info .= ' (limit of 5, there may be more!)' if scalar @uid_0_users >= 5;
        $info .= ' -- and nscd is running! This can break things. See CPANEL-2360.' if exists_process_cmd(qr{bin/nscd (?:\s|$)}xms);    # Don't specify root user here, it may not match.
        print_warning($info);
        my @suspectusers = grep { /cpanellogin/ } @uid_0_users;
        if ( scalar( @suspectusers > 0 )) {
            print_generic_hack_predef('Potential Root Compromise');
            print_critical('User with UID 0 found known to be associated with root compromises!');
            for my $suspectuser (@suspectusers) {
                print_critical( "\t\\_ $suspectuser" );
            }
        }
    }
}

sub check_sudoers_files {
    my @sudoersfiles = glob(q{/etc/sudoers.d/*});
    push @sudoersfiles, "/etc/sudoers" unless( !-e "/etc/sudoers" );
    my $showHeader=0;
    foreach my $sudoerfile (@sudoersfiles) {
        chomp($sudoerfile);
        next if ( $sudoerfile eq '/etc/sudoers.d/48-wp-toolkit' || $sudoerfile eq '/etc/sudoers.d/90-cloud-init-users' );
        next if ( $sudoerfile =~ m{/etc/sudoers.d/ticket[0-9]} );
        open( my $fh, '<', $sudoerfile );
        my @sudoers = <$fh>;
        close( $fh );
        foreach my $sudoerline (@sudoers) {
            chomp($sudoerline);
            next if ( $sudoerline =~ m/^(#|$|root|Defaults|%wheel)/ );
            next unless ( $sudoerline =~ m/ALL$/ );
            print_warn( "Found non-root users with insecure privileges in a sudoer file.\n" ) unless($showHeader == 1);
            $showHeader = 1;
            if ( $sudoerline =~ m/ALL, !root/ ) {
                print_warning( "\t\\_ $sudoerfile: $sudoerline has !root - might be susceptible to CVE-2019-14287" );
            }
            else {
                print_warning( "\t\\_ $sudoerfile: $sudoerline" );
            }
        }
    }
}

sub check_for_easyparams_attributes {
    return unless i_am_one_of( 'ea4', 'ea3' );    # Check when using EA4 until it is no longer possible to revert to EA3.
    my $easyparams = '/scripts/easyparams';
    return if !-e $easyparams;

    my $attributes = timed_run( 0, 'lsattr', $easyparams );
    if ( $attributes =~ m/^-[-]*(?:a|i)/ ) {
        print_warn("$easyparams: ");
        print_warning('is immutable or append only. This should never be done, and can break EasyApache!');
    }
}

sub check_for_allow_update_in_named_conf {
    my $namedconf = '/etc/named.conf';
    return if !-e $namedconf;

    if ( open my $fh, '<', $namedconf ) {
        while (<$fh>) {
            if (/allow-update/i) {
                print_warn('named.conf: ');
                print_warning('allow-update found. This can possibly prevent rndc from reloading. See ticket 4717591');
                last;
            }
            last if /^\s*view /i;
        }
        close $fh;
    }
}

sub check_for_modsec2_stage_files {
    return unless i_am_one_of( 'ea4', 'ea3' );
    my $file = i_am('ea4') ? '/etc/apache2/conf/modsec2.user.conf.STAGE' : '/usr/local/apache/conf/modsec2.user.conf.STAGE';
    return unless -e $file;
    print_warn('Mod Security: ');
    print_warning('/usr/local/apache/conf/modsec2.user.conf.STAGE exists -- This may prevent editing of rules in WHM ModSecurity Tools.');
}

sub check_for_cron_allow {
    return unless i_am('cpanel');
    my $cron_allow = "/etc/cron.allow";
    my $cron_deny  = "/etc/cron.deny";

    if ( -s $cron_deny ) {

        # By default /etc/cron.deny can contain the "nobody" user.  Checking the contents can be expensive, size will do.
        my $cron_deny_size     = ( stat($cron_deny) )[7];
        my $cron_deny_max_size = 8;
        if ( $cron_deny_size > $cron_deny_max_size ) {
            print_warn('crontab: ');
            print_warning( $cron_deny . ' is > ' . $cron_deny_max_size . ' bytes, may contain users other than "nobody".  A user listed here cannot see or edit cron jobs in the cPanel UI.' );
        }
    }

    if ( -e $cron_allow ) {
        print_warn('crontab: ');
        print_warning('/etc/cron.allow exists. Any user NOT listed cannot see or edit cron jobs in the cPanel UI.');
    }
}

# TODO: Add cptech only warn option?
sub check_cves_vulnerable_versions {
    my $changelog = get_rpm_vulnerable();
    # Loop through each package
    foreach my $pkg ( keys %{ $changelog } ) {
        my @cves;
        # Bail if we did not get a package version(i.e. not installed or corrupt rpmdb)
        next unless ( defined( $changelog->{$pkg}->{version} ) );
        # Loop through each CVE
        foreach my $cve ( grep { /^CVE/ } keys %{ $changelog->{$pkg} } ) {
            # Skip if changelog contains the CVE(i.e. seen = 1)
            next if $changelog->{$pkg}->{$cve}->{seen};
            # Skip unless the OS is vulnerable to this CVE, if "os_vuln" exists
            if ( exists( $changelog->{$pkg}->{$cve}->{os_vuln} ) ) {
                next unless $changelog->{$pkg}->{$cve}->{os_vuln};
            }
            # If the anonymous sub "vuln" returns true, the package is vulnerable
            if ( exists( $changelog->{$pkg}->{$cve}->{vuln} ) ) {
                next unless( $changelog->{$pkg}->{$cve}->{vuln}->( $changelog->{$pkg}->{version} ) );
            }
            push @cves, $cve;
        }
        next unless @cves;
        print_critical();
        print_crit("$pkg:\n");
        foreach my $cve ( @cves ) {
            my $msg = exists( $changelog->{$pkg}->{$cve}->{warn} ) ? $changelog->{$pkg}->{$cve}->{warn} : "notify client https://nvd.nist.gov/vuln/detail/$cve";
            print_critical("\t \\_ Vulnerable to $cve, $msg");
        }
        print_critical('The following check was used: rpm -q --changelog ' . $pkg . ' | grep \'CVE-XXXX-XXXX\'');
        print_critical('!! VERIFY THE CHECK USING THE COMMAND ABOVE BEFORE SENDING THE PREMADE !!');
        print_critical('This check does NOT take corrupt RPM dbs into account, and CAN report false-positive results if corrupt');
        print_critical();
    }
}

sub openssl_version_to_numbers {
    my ($ver) = @_;
    return unless ( $ver =~ /(\d+)\.(\d+)\.(\d+)([a-z])([a-z]?)/ );
    my ( $maj, $min, $patch ) = ( $1, $2, $3 );

    # If we map the alphas into a number and sum the values the version will be compatible with version_compare() and save us a lot of trouble, i.e. h=8, m=13, and za=27
    my %al2num = map { ( "a" .. "z" )[ $_ - 1 ] => $_ } ( 1 .. 26 );    # Isn't there a better way to do this?
    my $sub = 0;
    if ($4) { $sub += $al2num{ lc($4) } }
    if ($5) { $sub += $al2num{ lc($5) } }
    $ver = join( '.', $maj, $min, $patch, $sub );
    return $ver;
}

# The "regex" key provides the search term for the package's changelog
# The "warn" key provides the warning for the end-user, should the package be found vulnerable
# The "seen" key indicates the "regex" key was seen in the packages changelog(i.e. not vulnerable)
# The "os_vuln" key indicates whether the OS is vulnerable to the CVE. This key should only be used to skip if not vulnerable, not a confirmation the packge is vulnerable
# The "vuln" key indicates if the package version is vulnerable
sub get_rpm_vulnerable {
    my %changelog_queries = (
        'openssl' => {
            'CVE-2014-0160' => {
                regex   => qr{ CVE-2014-0160 },
                warn    => 'send premade: SECURITY - OpenSSL Heartbleed Vulnerability - Discovery',
                seen    => 0,
                os_vuln => sub {
                    return 0 if ( os_version_is(qw( >= 6.6 )) );    # If >= 6.6, not vulnerable
                    return 0 if ( os_version_is(qw( <= 5 )) );      # If <= 5, not vulnerable
                    return 1;                                       # Else, vulnerable
                },
                vuln => sub {
                    my ($ver) = @_;
                    $ver = openssl_version_to_numbers($ver);
                    return 1 if ( version_compare( $ver, qw( >= 1.0.1.1 ) ) && version_compare( $ver, qw( <= 1.0.1.6 ) ) );    # If >= 1.0.1.a and <= 1.0.1.f, vulnerable
                    return 0;                                                                                                  # Else, not vulnerable
                },
            },
            'CVE-2014-0224' => {
                regex   => qr{ CVE-2014-0224 },
                warn    => 'send premade: SECURITY - OpenSSL advisory 2014-06-05 - Discovery',
                seen    => 0,
                os_vuln => sub {
                    return ( os_version_is(qw( <= 6 )) ? 1 : 0 );
                },
                vuln => sub {
                    my ($ver) = @_;
                    $ver = openssl_version_to_numbers($ver);
                    return 0 if ( version_compare( $ver, qw( >= 1.0.1.8 ) ) );                                                 # If >= 1.0.1h, not vulnerable
                    return 0 if ( version_compare( $ver, qw( < 1.0.1.0 ) ) && version_compare( $ver, qw( >= 1.0.0.13 ) ) );    # If < 1.0.1 and >= 1.0.0m, not vulnerable
                    return 0 if ( version_compare( $ver, qw( < 0.9.9.0 ) ) && version_compare( $ver, qw( >= 0.9.8.27 ) ) );    # If < 0.9.9 and >= 0.9.8za, not vulnerable
                    return 1;                                                                                                  # Else, vulnerable
                }
            },
        },
        'dovecot' => {
            'CVE-2019-11500' => {
                regex => qr{CVE-2019-11500},
                warn  => 'send premade: SECURITY - Dovecot/Pigeonhole - CVE-2019-11500',
                seen  => 0,
                vuln  => sub {
                    my ($ver) = @_;
                    return 0 if ( version_compare( $ver, qw( >= 2.3.7.2 ) ) );                                                 # If >= 2.3.7.2, not vulnerable
                    return 1 if ( version_compare( $ver, qw( < 2.2.36.4 ) ) );                                                 # If < 2.2.36.4, vulnerable
                    return 1 if ( version_compare( $ver, qw( >= 2.3.0.0 ) ) && version_compare( $ver, qw( < 2.3.7.2 ) ) );     # If >= 2.3.0.0 and < 2.3.7.2, vulnerable
                    return 0;                                                                                                  # Else, not vulnerable
                },
            },
            'CVE-2019-7524' => {
                regex => qr{ CVE-2019-7524},
                warn  => 'send premade: SECURITY - Dovecot CVE-2019-7524',
                seen  => 0,
                vuln  => sub {
                    my ($ver) = @_;
                    return 0 if ( version_compare( $ver, qw( >= 2.3.5.1 ) ) );                                                 # If >= 2.3.5.1, not vulnerable
                    return 1 if ( version_compare( $ver, qw( < 2.2.36.3 ) ) );                                                 # If < 2.2.36.3, vulnerable
                    return 1 if ( version_compare( $ver, qw( >= 2.3.0.0 ) ) && version_compare( $ver, qw( < 2.3.5.1 ) ) );     # If >= 2.3.0.0 and < 2.3.5.1, vulnerable
                    return 0;                                                                                                  # Else, not vulnerable
                },
            },
        },
        'ea-apache24' => {
            'CVE-2019-0211' => {
                regex => qr{CVE-2019-0211:},
                warn  => 'send premade: SECURITY - CVE-2019-0211 (Apache 2.4.38)',
                seen  => 0,
                vuln  => sub {
                    my ($ver) = @_;
                    return 1 if ( version_compare( $ver, qw( >= 2.4.17 ) ) && version_compare( $ver, qw( <= 2.4.38 ) ) );       # If >= 2.4.17 and <= 2.4.38, vulnerable
                    return 0;                                                                                                   # Else, not vulnerable
                },
            },
        },
        'sudo' => {
            'CVE-2021-3156' => {
                regex => qr{CVE-2021-3156},
                warn  => 'send premade: SECURITY::SUDO::CVE-2021-3156',
                seen  => 0,
                vuln  => sub {
                    my ($ver) = @_;
                    return 0 if (version_compare($ver, qw( >= 1.8.24 )));
                    return 1;
                },
            },
        },
        'exim' => {
            'CVE-2019-16928' => {
                regex => qr{CVE-2019-16928|Fix buffer overflow in string_vformat},
                warn  => 'send premade: SECURITY - EXIM CVE-2019-16928',
                seen  => 0,
                vuln  => sub {
                    my ($ver) = @_;
                    return 1 if ( cpanel_version_is( qw( < 11.78.0.0 ) ) );                                               # We didn't patch older than 78
                    return 1 if ( cpanel_version_is( qw( < 11.82.0.0 ) ) && version_compare( $ver, qw( < 4.92.6 ) ) );    # cP78 received patch in 4.92.6
                    return 1 if ( cpanel_version_is( qw( > 11.82.0.0 ) ) && version_compare( $ver, qw( < 4.92.4 ) ) );    # cP82 received patch in 4.92.4
                    return 0;
                }
            },
            'CVE-2019-15846' => {
                regex => qr{CVE-2019-15846},
                warn  => 'send premade: SECURITY - Exim CVE-2019-15846',
                seen  => 0,
                vuln  => sub {
                    my ($ver) = @_;
                    return 0 if ( version_compare( $ver, qw( >= 4.92.3 ) ) );    # If >= 4.92.3, not vulnerable
                    return 1;                                                    # Else, vulnerable
                },
            },
            'CVE-2019-10149' => {
                regex => qr{CVE-2019-10149},
                warn  => 'send premade: SECURITY - EXIM - CVE-2019-10149',
                seen  => 0,
                vuln  => sub {
                    my ($ver) = @_;
                    return 0 if ( version_compare( $ver, qw( > 4.91 ) ) );       # If > 4.91, not vulnerable
                    return 0 if ( version_compare( $ver, qw( < 4.87 ) ) );       # If <= 4.87, not vulnerable
                    return 1;                                                    # Else, vulnerable
                },
            },
            'CVE-2018-6789' => {
                regex => qr{CVE-2018-6789},
                warn  => 'send premade: SECURITY - EXIM CVE-2018-6789 - DISCOVERY',
                seen  => 0,
                vuln  => sub {
                    my ($ver) = @_;
                    return 0 if ( version_compare( $ver, qw( >= 4.90.1 ) ) );    # If >= 4.90.1, not vulnerable
                    return 1;                                                    # Else, vulnerable
                },
            },
        },
    );
    my $field_separator = '#^#';
    my $query_format    = '\'' . $field_separator . '%{NAME}' . $field_separator . '%{VERSION}' . $field_separator . '%{RELEASE}\\n\'';
    my $changelog       = timed_run( 0, 'rpm --queryformat ' . $query_format . ' -q ' . join( ' ', keys %changelog_queries ) . ' --changelog' );
    my $package;
    open( my $fh, '<', \$changelog );
    while (<$fh>) {
        chomp;
        if ( $_ =~ m/ \A \Q$field_separator\E ([\w-]+) \Q$field_separator\E ([\w\.]+) \Q$field_separator\E (\d+)\. /xms ) {
            $package = $1;
            $changelog_queries{$package}{version} = $2 . '_' . $3;
        }
        if ($package) {
            foreach my $cve ( grep { /^CVE/ } keys %{ $changelog_queries{$package} } ) {
                $changelog_queries{$package}{$cve}{seen} = 1 if ( $_ =~ m/$changelog_queries{$package}{$cve}{regex}/ms );
            }
        }
    }
    close($fh);
    return \%changelog_queries;
}

sub check_for_bash_secadv_20140924 {
    chomp( my $bash_output = timed_run( 0, 'env x=\'() { :;}; echo vulnerable\' bash -c ""' ) );
    return if !( $bash_output =~ m{ vulnerable }xms );

    print_critical();
    print_crit('Installed \'bash\' shell is vulnerable to remote code injection.  Verify by running the following at a shell prompt, it should return "vulnerable":');
    print_critical();
    print_critical('    env x=\'() { :;}; echo vulnerable\' bash -c ""');
    print_critical('Send customer this premade: "SECURITY - Bash advisory 2014-09-24 - Discovery"');
    print_critical();
}

sub get_exim_version_build_date {
    my $eximbin = '/usr/sbin/exim';
    return unless -x $eximbin;
    return unless my $exim_out = timed_run( 0, $eximbin, '--version' );

    my $ver;
    my $build_date;
    if ( $exim_out =~ m{ version \s ([^\s]+) (?:\s \#\d+)? \s built \s ([^\s]+) }xms ) {
        $ver        = $1;
        $build_date = $2;
    }
    return unless $ver;
    return ( $ver, $build_date );
}

sub check_for_broken_mysqldump {
    return if i_am('dnsonly');
    return unless -f '/usr/bin/mysql';
    my $md = '/usr/bin/mysqldump';

    if ( !-x $md ) {
        print_warn("$md: ");
        print_warning('not found or not executable!');
        return;
    }

    local $?;
    timed_run( 0, $md );
    my $exit_status = $? >> 8;
    if ( $exit_status != 1 ) {    # Running with no options is error 1
        print_warn("$md: ");
        print_warning('may be broken (exit status != 1).');
    }
}

sub check_exim_log_sanity {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    my @logs = qw( /var/log/exim_mainlog /var/log/exim_paniclog /var/log/exim_rejectlog );

    for my $log (@logs) {
        if ( !-f $log ) {
            print_warn("$log: ");
            print_warning('is missing!');
        }
        else {
            my $uid  = ( stat($log) )[4];
            my $user = getpwuid($uid);
            if ( $user ne 'mailnull' ) {
                print_warn("$log: ");
                print_warning('is not owned by "mailnull"');
            }
        }
    }
}

sub check_exim_localopts {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    return unless my $exim_localopts = get_exim_localopts_href();

    # Example: 'exact_option_name' => { default => 'defaultvalue', check_missing => 1, help => '- Help text' },
    # default, check_missing, and help are optional
    # If check_missing exists, a warning is generated if the item is NOT in exim.conf.localopts and does not match the default (if given) or is empty

    my %checks = (
        'allowweakciphers' => { default => '0', help => '"Allow weak SSL/TLS ciphers" enabled, this removes tls_require_ciphers from exim.conf.  It is better to disable this and customize "SSL/TLS Cipher Suite List" instead.' },
    );

    for my $check ( sort( keys(%checks) ) ) {
        $checks{$check}->{check_missing} = 0  unless defined( $checks{$check}->{check_missing} );
        $checks{$check}->{help}          = '' unless defined( $checks{$check}->{help} );
        my $help = $checks{$check}->{help} ? ' - ' . $checks{$check}->{help} : '';
        if ( defined( $exim_localopts->{$check} ) && defined( $checks{$check}->{default} ) && !( $exim_localopts->{$check} eq $checks{$check}->{default} ) ) {
            print_warn('/etc/exim.conf.localopts: ');
            print_warning("[ $check = $exim_localopts->{$check} ] $help");
        }
        elsif ( $checks{$check}->{check_missing} && ( !defined( $exim_localopts->{$check} ) || ( defined( $exim_localopts->{$check} ) && $exim_localopts->{$check} eq '' ) ) ) {
            print_warn('/etc/exim.conf.localopts: ');
            print_warning("[ $check ] not found or has empty value. $help");
        }
    }
}

sub check_for_readonly_filesystems {
    open my $fh, '<', '/proc/mounts' or return;
    my @read_only_fs = ();
    while (<$fh>) {
        if ( my @fs = split(' ') ) {
            next if grep { m{ / (virtfs|cagefs-skeleton|machine-id) / }x } $fs[1];
            next if "/sys/fs/cgroup" eq $fs[1] and os_version_is(qw( >= 7 ));
            if ( grep { m{ (^|,) ro (,|$) }x } $fs[3] ) {
                push( @read_only_fs, $fs[1] );
            }
        }
    }
    if ( scalar @read_only_fs ) {
        print_warn('Read-only filesystems: ');
        print_warning( join( " ", @read_only_fs ) );
    }
    close($fh);
}

sub check_for_unsupported_php {
    return unless i_am('ea3');
    return unless my $ea3_php = get_ea3_php_conf_href();
    return unless defined $ea3_php->{'phpversion'};
    return if $ea3_php->{'phpversion'} >= 6;
    return if $ea3_php->{'phpversion'} == 4 and not defined $ea3_php->{'php4version'};
    return if $ea3_php->{'phpversion'} == 5 and not defined $ea3_php->{'php5version'};

    my $min_php5 = '5.3.0';
    return if $ea3_php->{'phpversion'} == 5 and not version_compare( $ea3_php->{'php5version'}, '<', $min_php5 );

    print_critical();
    print_crit('!! RUNNING A VERSION OF PHP THAT IS NO LONGER SUPPORTED BY EASYAPACHE !! ');
    print_critical();
    print_critical('Do not run EasyApache without confirmation that this will replace PHP with a supported PHP version!');
    if ( $ea3_php->{'phpversion'} == 4 ) {
        print_critical( 'PHP4: ' . $ea3_php->{'php4version'} );
    }
    if ( $ea3_php->{'phpversion'} == 5 and version_compare( $ea3_php->{'php5version'}, '<', $min_php5 ) ) {
        print_critical( 'PHP5: ' . $ea3_php->{'php5version'} );
    }
    print_critical();
}

sub check_for_cl_unsupported_memory_limits {
    return unless i_am('cloudlinux');
    return unless os_version_is(qw( < 6 ));
    return unless i_am_one_of( 'ea4', 'ea3' );
    my ( $lsws_full_version, $lsws_numeric_version ) = @{ get_lsws_version_aref() };

    # See ticket # 5557825 - Several memory limits are not imposed: URL: https://helpdesk.cloudlinux.com/index.php?/Knowledgebase/Article/View/69/3/memory-limits-are-not-working
    # Per Igor:
    # CL 5 - doesn't support memory limits for: mod_php, mod_ruid2, MPM ITK & LiteSpeed.
    # CL 6 - doesn't support memory limits for: mod_php & mod_ruid2
    if ($lsws_full_version) {
        print_warn('LiteSpeed: ');
        print_warning('in use on CL5 or earlier. CloudLinux memory limits not imposed on Apache processes.');
    }
    if ( i_am('ea3') ) {
        my $ea3_php = get_ea3_php_conf_href();
        if ( defined $ea3_php and defined $ea3_php->{'php5handler'} and $ea3_php->{'php5handler'} eq 'dso' ) {
            print_warn('PHP DSO: ');
            print_warning('in use on CL 5 or earlier. CloudLinux memory limits not imposed on Apache processes.');
        }
    }
}

sub check_for_eblockers {
    return unless -e '/var/cpanel/update_blocks.config';
    open my $blocker_fh, '<', '/var/cpanel/update_blocks.config' or return;
    print_warn("WHM Update Blocker found:\n");
    while (<$blocker_fh>) {
        chomp;
        next if /^$/;
        print_magenta("\t \\_ $_");
    }
    close $blocker_fh;
}

sub check_for_frontpage_rpms {
    return unless i_am('cpanel');
    return unless -e '/usr/local/frontpage/version5.0/bin/owsadm.exe';
    return unless my $rpms = get_rpm_href();
    if ( !grep { m{frontpage} } keys %{$rpms} ) {
        print_warn('FrontPage: ');
        print_warning('RPM not installed, but /usr/local/frontpage/version5.0/bin/owsadm.exe exists -- will prevent upgrade to 11.46');
    }
}

sub check_for_php_selector_incompatibilities {
    return unless i_am( 'cloudlinux', 'ea3' );
    return unless my $ea3_php = get_ea3_php_conf_href();
    return unless defined $ea3_php->{'php5handler'} and $ea3_php->{'php5handler'} eq 'dso';
    print_warn('DSO PHP handler: ');
    print_warning('is enabled but not compatible with PHP Selector - http://docs.cloudlinux.com/index.html?php_selector.html');
}

sub check_cloudlinux_sanity {
    return unless i_am('cloudlinux');
    if ( !-x "/usr/sbin/lvectl" ) {
        print_warn('CloudLinux: ');
        print_warning('missing or non-executable /usr/sbin/lvectl - check/repair lve-utils RPM');
    }
    if ( !-x '/usr/bin/selectorctl' ) {
        print_warn('CloudLinux: ');
        print_warning('missing or non-executable /usr/bin/selectorctl - check/repair lvemanager RPM - This will incorrectly cause cPanel EA4 to be used.');
    }
    if ( !-x '/usr/sbin/lveinfo' ) {
        print_warn('CloudLinux: ');
        print_warning('missing or non-executable /usr/sbin/lveinfo - check/repair lve-stats RPM - This will incorrectly cause cPanel EA4 to be used.');
    }
    if ( !-x '/usr/sbin/spacewalk-channel' ) {
        print_warn('CloudLinux: ');
        print_warning('missing or non-executable /usr/sbin/spacewalk-channel - check/repair rhn-setup RPM');
    }
    if ( !-x '/usr/bin/python' && os_version_is(qw ( < 8 )) ) {
        print_warn('CloudLinux: ');
        print_warning('missing or non-executable /usr/bin/python - check/repair python RPM');
    }
    if ( -e '/var/.cagefs' ) {
        print_warn('CloudLinux: ');
        print_warning('/var/.cagefs - EXISTS! Should never exist outside of CageFS, will break many things.');
    }
    if ( -x '/usr/sbin/cagefsctl' ) {
        my $cagefsctl_help = timed_run_trap_stderr( 0, '/usr/sbin/cagefsctl', '--help' );
        if ( defined $cagefsctl_help and $cagefsctl_help =~ m{ --sanity-check }xms ) {
            local $?;
            my $cagefsctl_sanity_check = timed_run( 0, '/usr/sbin/cagefsctl', '--sanity-check' );
            my $cagefsctl_sanity_check_status = $? >> 8;
            if ($cagefsctl_sanity_check_status) {
                print_warn('CloudLinux: ');
                print_warning('[ /usr/sbin/cagefsctl --sanity-check ] returned non-zero exit status, run it for more information.');
            }
        }
    }
    my $cldetect          = '/usr/bin/cldetect';
    my $check_license_opt = '--check-license';
    if ( -x $cldetect ) {
        my $cldetect_help = timed_run_trap_stderr( 0, $cldetect, '--help' );
        if ( defined $cldetect_help and $cldetect_help =~ m{ $check_license_opt }xms ) {
            local $?;
            timed_run( 0, $cldetect, $check_license_opt, '-q' );    # The -q is necessary for it to use exit status
            my $cldetect_check_license_status = $? >> 8;
            if ($cldetect_check_license_status) {
                print_warn('CloudLinux: ');
                print_warning("[ $cldetect $check_license_opt ] returned an error, run it for more information. An invalid license can prevent CL repos appearing in Yum.");
            }
        }
    }
}

sub check_for_UMBREON_rootkit {
    my $dir = '/usr/local/__UMBREON__';
    if ( chdir $dir ) {
        print_generic_hack_predef('UMBREON ROOTKIT');
        print_critical('The following directory was found:');
        print_critical( "\t" . $dir );
        print_critical("\tL3: use \"L3 - Jynx2 Predef [L3 Only]\"");
        print_critical();
    }
}

sub check_for_libms_rootkit {
    my $dir = '/lib/udev/x.modules';
    if ( chdir $dir ) {
        print_generic_hack_predef('LIBMS ROOTKIT');
        print_critical('The following directory was found:');
        print_critical( "\t" . $dir );
        print_critical("\tL3: see ticket 7488621\"");
        print_critical();
    }
}

sub check_for_jynx2_rootkit {
    my $dir = '/usr/bin64';
    if ( chdir $dir ) {
        my @found_jynx2_files = ();
        my @jynx2_files       = qw( 3.so 4.so );
        for (@jynx2_files) {
            my $file = $dir . "/" . $_;
            if ( -e $file ) {
                push( @found_jynx2_files, $file );
            }
        }
        return if ( ( scalar @found_jynx2_files ) == 0 );
        print_generic_hack_predef('Jynx 2 ROOTKIT');
        print_critical('The following files were found:');
        print_critical( "\t" . join( " ", @found_jynx2_files ) );
        print_critical("\tL3: use \"L3 - Jynx2 Predef [L3 Only]\"");
        print_critical();
    }
}

sub check_for_fritzfrog_botnet {
    my $maliciousPortListening = 0;
    my $maliciousProcessFound = 0;
    my $ports = get_lsof_port_href();
    if ( defined( $ports->{'1234'} )) {
        $maliciousPortListening = 1;
    }
    my $info;
    my @MaliciousProcs = qw( nginx ifconfig libexec php-fpm );
    foreach my $proc ( @MaliciousProcs ) {
        chomp (my $PidOf = timed_run( 0, "pidof $proc" ) );
        next unless ($PidOf);
        my $isDeleted = timed_run( 0, "ls -l /proc/$PidOf/exe" );
        next unless( $isDeleted =~ m/deleted/ );
        $info .= ' [ ' . $proc . '/' . $PidOf . ' ]';
        $maliciousProcessFound++;
    }
    if ( $maliciousPortListening and $maliciousProcessFound > 0 ) {
        print_generic_hack_predef('FritzFrog Botnet');
        print_critical('Suspicious process(es) running ' . CYAN $info . MAGENTA ' and listening on port 1234.');
        print_critical();
    }
}

sub check_for_bg_botnet {

    # /bin/ps, /bin/netstat, and /usr/sbin/lsof have also been found to be modified
    # This one is causing some rare false-positives:
    # /root/aa
    my @bg_files = qw(
      /boot/pro
      /boot/proh
      /etc/atdd
      /etc/atddd
      /etc/cupsdd
      /etc/cupsddd
      /etc/dsfrefr
      /etc/fdsfsfvff
      /etc/ferwfrre
      /etc/gdmorpen
      /etc/gfhddsfew
      /etc/gfhjrtfyhuf
      /etc/ksapd
      /etc/ksapdd
      /etc/kysapd
      /etc/kysapdd
      /etc/rewgtf3er4t
      /etc/sdmfdsfhjfe
      /etc/sfewfesfs
      /etc/sfewfesfsh
      /etc/sksapd
      /etc/sksapdd
      /etc/skysapd
      /etc/skysapdd
      /etc/smarvtd
      /etc/whitptabil
      /etc/xfsdx
      /etc/xfsdxd
      /etc/rc.d/init.d/DbSecuritySpt
      /etc/rc.d/init.d/selinux
      /usr/bin/.sshd
      /usr/bin/bsd-port/getty
      /usr/bin/bsd-port/knerl
      /usr/bin/pojie
      /usr/lib/libamplify.so
      /var/.lug.txt
      /lost+found/mimipenguin-master/kautomount--pid-file-var-run-au
    );

    # Because tmp, we must check that these are owned by root. Leaving "/tmp/notify.file" out of this list due to potential false-positives.
    my @root_bg_files = qw(
      /tmp/bill.lock
      /tmp/gates.lock
      /tmp/moni.lock
      /tmp/fdsfsfvff
      /tmp/gdmorpen
      /tmp/gfhjrtfyhuf
      /tmp/rewgtf3er4t
      /tmp/sfewfesfs
      /tmp/smarvtd
      /tmp/whitptabil
    );

    my @found_bg_files = grep { -e $_ } @bg_files;
    for my $file (@root_bg_files) {
        if ( -e $file && ( stat $file )[4] eq 0 ) {
            push( @found_bg_files, $file );
        }
    }
    return unless ( scalar @found_bg_files );
    print_generic_hack_predef('BG BOTNET/Elknot');
    print_critical('The following files were found:');
    print_critical( "\t" . join( " ", @found_bg_files ) );
    print_critical("\tL3: use \"L3 - BG Botnet Predef [L3 Only]\", see TECH-863");
    print_critical();
}

sub check_for_dragnet {
    if ( open my $fh, '<', '/proc/self/maps' ) {
        while (<$fh>) {
            if (m{ (\s|\/) libc\.so\.0 (\s|$) }x) {
                print_generic_hack_predef('DRAGNET ROOTKIT');
                print_critical("\t\\_ 'libc.so.0' found in process maps");
                print_critical();
                last;
            }
        }
        close($fh);
    }
}

sub check_for_xor_ddos {
    my @libs = qw(
      /lib/libgcc.so
      /lib/libgcc.so.bak
      /lib/libgcc4.4.so
      /lib/libgcc4.so
      /lib/libudev.so
    );

    my @matched;
    for my $lib (@libs) {
        next if -l $lib;
        push @matched, $lib if -f $lib;
    }

    if (@matched) {
        print_generic_hack_predef('Linux/XOR.DDoS');
        print_critical('The following file(s) were found:');
        print_critical( "\t" . join( "\n\t", @matched ) );
        print_critical();
    }
}

sub check_for_shellbot {
    my @libs = qw(
      /lib/libgrubd.so
    );

    my @matched;
    for my $lib (@libs) {
        next if -l $lib;
        push @matched, $lib if -f $lib;
    }

    if (@matched) {
        print_generic_hack_predef('ShellBot');
        print_critical('The following file(s) were found:');
        print_critical( "\t" . join( "\n\t", @matched ) );
        print_critical("\tL3: use \"L3 - ShellBot Predef [L3 Only]\"");
        print_critical();
    }
}

sub check_updatelog {
    my $log = '/var/cpanel/updatelogs/last';
    return unless -e $log;

    my $size = ( stat($log) )[7];
    return if !$size;
    my $bytes_to_read = 10485760;    # 10M
    $bytes_to_read = $size if $bytes_to_read > $size;

    my $log_data;
    open my $file_fh, '<', $log or return;
    read $file_fh, $log_data, $bytes_to_read;
    close $file_fh;

    foreach ( split( "\n", $log_data ) ) {
        if (m{ installing .+ needs .+ on .+ filesystem }x) {
            print_warn("WHM update: ");
            print_warning( $log . ' contains errors which indicate that RPM installation had insufficient free disk space available' );
            return;
        }
    }
}

sub check_for_saltstack {
    return unless exists_process_cmd( qr{ salt-minion }xms, 'root' );
    print_warn('SaltStack Minion: ');
    print_warning('the salt-minion process is running. Seeing files being reverted, this may be why');
}

sub check_for_puppet_agent {
    return unless exists_process_cmd( qr{ puppet }xms, 'root' );
    print_warn('Puppet Agent: ');
    print_warning('the puppet process is running. Seeing files being reverted, this may be why');
}

sub get_kernel_modules_aref {
    my $lsmod = timed_run( 0, 'lsmod' );
    my @kernel_modules;
    for my $line ( split( /\n/, $lsmod ) ) {
        next if ( $line =~ m/\A Module \s+/x );
        push @kernel_modules, $1 if ( $line =~ m/\A ([^\s]+) \s/x );
    }
    return \@kernel_modules;
}

sub check_use_apache_md5_for_htaccess {
    return unless ( defined $CPCONF{'use_apache_md5_for_htaccess'} );
    return if ( $CPCONF{'use_apache_md5_for_htaccess'} == 1 );
    print_warn('Tweak Setting: ');
    print_warning('use_apache_md5_for_htaccess is disabled, htpasswd limits passwords to a length of 8 characters');
}

sub missing_open_opath_flag {
    return if os_version_is(qw( < 7 ));
    my $eloop       = 'Too many levels of symbolic links';
    my $SYS_open    = 2;            # See syscall.ph or h2ph
    my $SYS_close   = 3;
    my $sym         = '/proc/self';
    my $flags       = 0x2e0000;     # O_RDONLY|O_NOFOLLOW|O_NOATIME|O_CLOEXEC|O_PATH
    my $buf         = pack q/i/, 0;
    local $!        = 0;
    my $fd;

    eval { $fd = syscall($SYS_open, $sym, $flags, $buf); };
    unless ($!) {
        syscall($SYS_close, $fd) if ($fd);
        return;
    }
    ( $! =~ m/\Q$eloop\E/ ) ? return 1 : return;
}

##############################
#  END [WARN] CHECKS
##############################

##############################
#  BEGIN [3RDP] CHECKS
##############################

sub check_smtp_processes {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    my $ports = get_lsof_port_href();
    return unless scalar keys(%$ports);
    if ( !defined( $ports->{'25'} ) && !-f '/etc/eximdisable' ) {
        print_warn('Exim: ');
        print_warning('not disabled and does not appear to be up -- nothing listening on port 25');
    }

    return if !defined( $ports->{'25'} );

    my $procs = get_process_pid_href();
    for my $href ( @{ $ports->{'25'} } ) {
        my $pid = $href->{PID};
        next unless defined $procs->{$pid};
        my $cmd = $procs->{$pid}->{ARGS};
        if ( $href->{PROTO} eq "TCP" && !( $cmd =~ m{ \A /usr/sbin/exim \b }xms ) ) {
            print_3rdp('SMTP: ');
            print_3rdp2( 'a process other than exim is listening on port 25 [' . $href->{IPV} . ' ' . $href->{IP} . '] [USER: ' . $href->{USER} . '] [CMD: ' . $cmd . '] [PID: ' . $pid . ']' );
        }
        if ( $cmd =~ m{ assp\.pl }xms ) {
            print_3rdp('ASSP: ');
            print_3rdp2( 'assp.pl is listening on port 25 [PID: ' . $pid . ']' );
            print_3rdp2('         \_  SNI is not supported unless enabled in the plugin: http://www.grscripts.com/howtofaq.html -> "Does ASSP Support SNI?"');
        }
    }
}

sub check_for_varnish {
    return unless i_am_one_of( 'ea4', 'ea3' );
    return unless my $ports = get_lsof_port_href();
    return unless exists $ports->{'80'};
    for my $ref ( @{ $ports->{'80'} } ) {
        if ( $ref->{'CMD'} =~ m{ \A varnish }xms ) {
            print_3rdp('Varnish: ');
            print_3rdp2('varnish is listening on port 80, known to break proxy subdomains. See "RareIssues" wiki article');
            last;
        }
    }
}

sub check_for_nginx {
    return unless i_am_one_of( 'ea4', 'ea3' );
    my %procs = grep_process_cmd( qr{ nginx: \s master }xms, 'root' );
    return unless grep { $procs{$_}->{ARGS} !~ /imunify360/ } keys %procs;    # Ignore imunify360 nginx process
    my $rpms = get_rpm_href();
    if ( exists $rpms->{'ea-nginx'} ) {         # We support ea-nginx installs
        print_info('cPanel supported ea-nginx is running');
        print "\n";
    }
    else {
        print_3rdp('nginx: ');
        print_3rdp2('is running');
    }
}

sub check_for_mailscanner {
    return unless i_am_one_of( 'ea4', 'ea3' );
    my $running = exists_process_cmd( qr{ MailScanner }xms, 'mailnull' ) ? 1 : 0;

    if ($running) {
        print_3rdp('MailScanner: ');
        print_3rdp2('is running');
    } else {
        my $bin     = '/usr/mailscanner/usr/sbin/MailScanner';
        return unless ( -e $bin );
        print_3rdp('MailScanner: ');
        if ( -s '/etc/exim_outgoing.conf' ) {
            print_warning('not running but /etc/exim_outgoing.conf file exists');
        }
        my $version = timed_run_trap_stderr( 0, $bin, '--version');
        return unless ( $version =~ m/failed--compilation/ );
        print_warning('is installed but failed to compile, an update may be required to start the service, see UPS-210');
    }
}

sub check_for_apf {
    my $chkconfig_apf = timed_run( 0, 'chkconfig', '--list', 'apf' );
    if ($chkconfig_apf) {
        if ( $chkconfig_apf =~ /3:on/ ) {
            print_3rdp('APF: ');
            print_3rdp2('installed, may be enabled.');
        }
    }
}

sub check_for_csf {
    return unless -f '/etc/csf/csf.conf';
    print_3rdp('CSF: ');
    my $lfd = exists_process_cmd( qr{ lfd }xms, 'root' ) ? 'is' : 'is not';
    print_3rdp2( 'installed, LFD ' . $lfd . ' running' );
}

sub check_for_prm {
    if ( -e '/usr/local/prm' ) {
        print_3rdp('PRM: ');
        print_3rdp2('PRM exists at /usr/local/prm');
    }
}

sub check_for_les {
    if ( -e '/usr/local/sbin/les' ) {
        print_3rdp('LES: ');
        print_3rdp2('Linux Environment Security is installed at /usr/local/sbin/les');
    }
}

sub check_for_1h {
    return unless -d '/usr/local/1h';
    my $guardian = exists_process_cmd( qr{ Guardian }xms, 'root' ) ? 'running' : 'not running';
    print_3rdp('1H Software: ');
    print_3rdp2("/usr/local/1h exists. Guardian process: [ $guardian ]");
}

sub check_for_webmin {
    return unless my $ports = get_lsof_port_href();
    return unless exists( $ports->{'10000'} );
    for my $ref ( @{ $ports->{'10000'} } ) {
        if ( index( $ref->{'CMD'}, 'miniserv.pl' ) == 0 ) {
            print_3rdp('Webmin: ');
            print_3rdp2('Webmin is running and is listening on port 10000');
            last;
        }
    }
}

sub check_for_symantec {
    return unless exists_process_cmd( qr{ symantec_antivirus }xms, 'root' );
    print_3rdp('Symantec: ');
    print_3rdp2('found symantec_antivirus in process list');
}

sub check_for_haproxy {
    return unless exists_process_cmd( qr{ haproxy }xms, 'haproxy' );
    print_3rdp('HAProxy: ');
    print_3rdp2('found haproxy in process list');
}

sub check_for_newrelic {
    return unless exists_process_cmd(qr{ newrelic-daemon }xms);
    print_3rdp('newrelic-daemon: ');
    print_3rdp2('found in process list. Caused server stability issues in 4396009');
}

sub check_for_multilevel_reseller {
    return unless i_am('cpanel');
    my @ml_plugins = qw/ zamfoo whmreseller whmphp whmamp /;
    my $cgi_root   = '/usr/local/cpanel/whostmgr/docroot/cgi';
    foreach my $plugin (@ml_plugins) {
        if ( -d "$cgi_root/$plugin" ) {
            print_3rdp( uc($plugin) . ' ' );
            print_3rdp2('is installed. Multi-level reseller setups are not supported');

        }
    }
}

sub check_for_cpremote {
    return unless i_am('cpanel');
    return unless -e '/var/spool/cron/root';

    open my $file_fh, '<', '/var/spool/cron/root' or return;
    while (<$file_fh>) {
        if (m#/scripts/cpremotebackup#) {
            print_3rdp('cpremote: ');
            print_3rdp2('installed. third party backup software (cron job found for root)');
            last;
        }
    }
    close $file_fh;
}

sub check_for_whmxtra {
    return unless i_am('cpanel');
    my $ionsh = '/usr/local/cpanel/whostmgr/docroot/themes/x/xtra/functions/ion.sh';
    return if !-f $ionsh;

    print_3rdp('WHMXtra: ');
    print_3rdp2("$ionsh exists. 'cPanel PHP loader' Tweak Settings or php.ini settings reverted? See 4622167, 4628203");
}

sub check_for_usr_local_mis {
    return unless i_am('ea3');
    my $dir = '/usr/local/mis';
    return if !-d $dir;

    print_3rdp("$dir: ");
    print_3rdp2('found! This can prevent EA3 from completing. See ticket 4822059');
}

sub check_for_opt_gsi_tools {
    return unless i_am('cpanel');
    my $dir = '/opt/gsi-tools';
    return if !-d $dir;

    print_3rdp("$dir: ");
    print_3rdp2('found! These admin scripts have been known to automatically lock user accounts. See ticket 6122361.');
}

sub check_for_pyxsoft_antimalware {
    return unless i_am('cpanel');
    my $dir = '/usr/share/ilabs_antimalware';
    return if !-d $dir;

    print_3rdp("$dir: ");
    print_3rdp2('found! This has been known to cause failed uploads, segfaults on PHP-FPM, and 500 ISE. See TECH-174');
}

sub check_for_magicspam {
    return unless i_am('cpanel');
    my $dir = '/etc/magicspam';
    return if !-d $dir;

    my $running = exists_process_cmd(qr{ magicspam }xms) ? 'and' : 'but not';
    print_3rdp('MagicSpam: ');
    print_3rdp2( 'installed ' . "$running" . ' running. See TECH-725' );
}

##############################
#  END [3RDP] CHECKS
##############################

sub build_libkeyutils_file_list {
    my @dirs = qw( /lib /lib/tls /lib64 /lib64/tls );
    my @libkeyutils_files;

    for my $dir (@dirs) {
        next unless -e $dir;
        opendir( my $dir_fh, $dir );
        while ( my $file = readdir($dir_fh) ) {
            if ( $file =~ /^libkeyutils\.so\.(?:[\.\d]+)?$/ ) {
                push @libkeyutils_files, "$dir/$file\n";
            }
        }
        closedir $dir_fh;
    }

    chomp @libkeyutils_files;
    return \@libkeyutils_files;
}

## BEGIN malware checks

sub print_generic_hack_predef {
    my $name = shift;
    print_critical();
    print_crit( '!! [ ' . $name . ' ] !! ' );
    print_critical('Escalate this ticket to L3 using "ESCALATE - Hacked Server Response for L1/L2->L3 ( All Analysts )"');
    print_critical("\t-- If this appears to be the direct cause of services being down, please escalate the ticket to Emergency status.");
    print_critical("\tL1/L2: LOG OUT NOW.  Do not execute any other commands unless given explicit directions by an L3 analyst or Supervisor.");
}

sub print_ebury_cdorked_predef {
    my $name = shift;
    print_generic_hack_predef($name);
    print_critical("\tL3: use \"L3 - eBury / CDorked [L3 only]\"");
}

sub check_for_cdorked_A {
    return unless my $httpd_bin = find_httpd_bin();
    return unless -f $httpd_bin;
    my $max_bin_size = 10_485_760;    # avoid slurping too much mem
    return if ( ( stat($httpd_bin) )[7] > $max_bin_size );

    my @apache_bins = ();
    push @apache_bins, $httpd_bin;
    my %procs = grep_process_cmd( qr{ $httpd_bin }xms, 'root' );
    for my $pid ( keys %procs ) {
        my $proc_pid_exe = "/proc/" . $pid . "/exe";
        if ( -l $proc_pid_exe && readlink($proc_pid_exe) =~ m{ \(deleted\) }xms ) {
            next if ( ( stat($proc_pid_exe) )[7] > $max_bin_size );
            push @apache_bins, $proc_pid_exe;
        }
    }

    for my $check_bin (@apache_bins) {
        my $httpd;
        if ( open my $fh, '<', $check_bin ) {
            local $/;
            $httpd = <$fh>;
            close $fh;
        }

        next if !$httpd;

        if ( $httpd =~ /(open_tty|hangout|ptsname|Qkkbal)/ ) {
            my $signature = $check_bin . ": \"" . $1 . "\"";
            print_ebury_cdorked_predef('CDORKED');
            print_critical("\tString found in $signature (see ticket 4482347)");
            print_critical();
            last;
        }
    }
}

sub check_for_cdorked_B {
    my @files = ( '/usr/sbin/arpd ', '/usr/sbin/tunelp ', '/usr/bin/s2p ' );
    my $cdorked_files;

    for my $file (@files) {
        if ( -e $file ) {
            $cdorked_files .= "[$file] ";
        }
    }

    if ($cdorked_files) {
        print_ebury_cdorked_predef('CDORKED');
        print_critical("\tThe following files were found (note the spaces at the end of the files):");
        print_critical("\t$cdorked_files");
        print_critical();
    }
}

sub check_for_libkeyutils_symbols {
    local $ENV{'LD_DEBUG'} = 'symbols';
    my $output = timed_run_trap_stderr( 0, '/bin/true' );
    return unless $output;
    if ( $output =~ m{ /lib(keyutils|ns[25]|pw[35]|s[bl]r)\. }xms ) {
        print_ebury_cdorked_predef('EBURY');
        print_critical('Ebury libs were found in symbol table.');
        print_critical('To confirm: LD_DEBUG=symbols /bin/true 2>&1 | egrep \'/lib(keyutils|ns[25]|pw[35]|s[bl]r)\\.\'');
        print_critical();
    }
}

sub check_for_libkeyutils_filenames {
    my ( $bad_libs, @list );
    my @dirs  = qw( /lib /lib64 );
    my @files = qw(
      libhdx.so
      libkeyutils.so.1.9
      libkeyutils-1.2.so.0
      libkeyutils-1.2.so.2
      libkeyutils.so.1.3.0
      libkeyutils.so.1.3.2
      libns2.so
      libns5.so
      libpw3.so
      libpw5.so
      libsbr.so
      libslr.so
      libtsq.so
      libtsr.so
      tls/libkeyutils.so.1
      tls/libkeyutils.so.1.5
    );

    for my $dir (@dirs) {
        next if !-e $dir;
        for my $file (@files) {
            if ( -f "${dir}/${file}" ) {
                $bad_libs .= "\t${dir}/${file}\n";
            }
            else {
                push @list, "${dir}/${file}";
            }
        }
    }

    if ( my $bad_libs_aref = eximbin_stat( \@list ) ) {
        foreach (@$bad_libs_aref) {
            $bad_libs .= "\t$_ evading discovery via LD_PRELOAD\n";
        }
    }

    if ($bad_libs) {
        print_ebury_cdorked_predef('EBURY');
        print_critical('The following file(s) were found:');
        print_critical($bad_libs);
        print_critical();
    }
}

sub eximbin_stat {
    my ($aref) = @_;
    my @exists;
    my $eximbin = qw( /usr/sbin/exim );
    my $statbin = qw( /usr/bin/stat );

    foreach my $file (@$aref) {
        push @exists, $file if ( grep { /File: \'$file\'/ } timed_run( 0, $eximbin, '-be', '\'${run {' . $statbin . ' ' . $file . '}}\'' ) );
    }
    return \@exists;
}

sub check_sha1_sigs_libkeyutils {
    my $libkeyutils_files_ref = build_libkeyutils_file_list();

    # p67 http://www.welivesecurity.com/wp-content/uploads/2014/03/operation_windigo.pdf
    # https://www.welivesecurity.com/2017/10/30/windigo-ebury-update-2/
    my @checksums = qw(
      09c8af3be4327c83d4a7124a678bbc81e12a1de4
      17c40a5858a960afd19cc02e07d3a5e47b2ab97a
      1a9aff1c382a3b139b33eeccae954c2d65b64b90
      1d3aafce8cd33cf51b70558f33ec93c431a982ef
      267d010201c9ff53f8dc3fb0a48145dc49f9de1e
      27ed035556abeeb98bc305930403a977b3cc2909
      2e571993e30742ee04500fbe4a40ee1b14fa64d7
      2f382e31f9ef3d418d31653ee124c0831b6c2273
      2fc132440bafdbc72f4d4e8dcb2563cc0a6e096b
      39ec9e03edb25f1c316822605fe4df7a7b1ad94a
      3c5ec2ab2c34ab57cba69bb2dee70c980f26b1bf
      44b340e90edba5b9f8cf7c2c01cb4d45dd25189e
      471ee431030332dd636b8af24a428556ee72df37
      58f185c3fe9ce0fb7cac9e433fb881effad31421
      5c796dc566647dd0db74d5934e768f4dfafec0e5
      5d3ec6c11c6b5e241df1cc19aa16d50652d6fac0
      615c6b022b0fac1ff55c25b0b16eb734aed02734
      7248e6eada8c70e7a468c0b6df2b50cf8c562bc9
      74aa801c89d07fa5a9692f8b41cb8dd07e77e407
      7adb38bf14e6bf0d5b24fa3f3c9abed78c061ad1
      899b860ef9d23095edb6b941866ea841d64d1b26
      8daad0a043237c5e3c760133754528b97efad459
      8f75993437c7983ac35759fe9c5245295d411d35
      9bb6a2157c6a3df16c8d2ad107f957153cba4236
      9e2af0910676ec2d92a1cad1ab89029bc036f599
      a559ee8c2662ee8f3c73428eaf07d4359958cae1
      a7b8d06e2c0124e6a0f9021c911b36166a8b62c5
      adfcd3e591330b8d84ab2ab1f7814d36e7b7e89f
      b58725399531d38ca11d8651213b4483130c98e2
      b8508fc2090ddee19a19659ea794f60f0c2c23ff
      bbce62fb1fc8bbed9b40cfb998822c266b95d148
      bf1466936e3bd882b47210c12bf06cb63f7624c0
      d4eeada3d10e76a5755c6913267135a925e195c6
      d552cbadee27423772a37c59cb830703b757f35e
      e14da493d70ea4dd43e772117a61f9dbcff2c41c
      e2a204636bda486c43d7929880eba6cb8e9de068
      e8d392ae654f62c6d44c00da517f6f4f33fe7fed
      e8d3c369a231552081b14076cf3eaa8901e6a1cd
      eb352686d1050b4ab289fe8f5b78f39e9c85fb55
      f1ada064941f77929c49c8d773cbad9c15eba322
    );

    for my $lib (@$libkeyutils_files_ref) {
        next unless my $checksum = timed_run( 0, 'sha1sum', "$lib" );
        chomp $checksum;
        $checksum =~ s/\s.*//g;
        if ( grep { /$checksum/ } @checksums ) {
            my $trojaned_lib = "$lib\n\tSHA-1 checksum: $checksum";
            print_ebury_cdorked_predef('EBURY');
            print_critical('The following file(s) were found:');
            print_critical( "\t" . $trojaned_lib );
            print_critical("\tReference: http://www.welivesecurity.com/2014/02/21/an-in-depth-analysis-of-linuxebury/ and https://www.welivesecurity.com/2017/10/30/windigo-ebury-update-2/");
            print_critical();
            last;
        }
    }
}

sub check_sha1_sigs_httpd {
    return unless my $httpd_bin = find_httpd_bin();
    return unless my $sha1sum = timed_run( 0, 'sha1sum', $httpd_bin );
    if ( $sha1sum =~ m{ \A (\S+) \s }xms ) {
        $sha1sum = $1;
    }

    my @sigs = qw(
      0004b44d110ad9bc48864da3aea9d80edfceed3f
      03592b8147e2c84233da47f6e957acd192b3796a
      0eb1108a9d2c9fe1af4f031c84e30dcb43610302
      10c6ce8ee3e5a7cb5eccf3dffd8f580e4fb49089
      149cf77d2c6db226e172390a9b80bc949149e1dc
      1972616a731c9e8a3dbda8ece1072bd16c44aa35
      24e3ebc0c5a28ba433dfa69c169a8dd90e05c429
      4f40bb464526964ba49ed3a3b2b2b74491ea89a4
      5b87807b4a1796cfb1843df03b3dca7b17995d20
      62c4b65e0c4f52c744b498b555c20f0e76363147
      78c63e9111a6701a8308ad7db193c6abb17c65c4
      858c612fe020fd5089a05a3ec24a6577cbeaf7eb
      9018377c0190392cc95631170efb7d688c4fd393
      a51b1835abee79959e1f8e9293a9dcd8d8e18977
      a53a30f8cdf116de1b41224763c243dae16417e4
      ac96adbe1b4e73c95c28d87fa46dcf55d4f8eea2
      dd7846b3ec2e88083cae353c02c559e79124a745
      ddb9a74cd91217cfcf8d4ecb77ae2ae11b707cd7
      ee679661829405d4a57dbea7f39efeb526681a7f
      fc39009542c62a93d472c32891b3811a4900628a
      fdf91a8c0ff72c9d02467881b7f3c44a8a3c707a
    );

    for my $sig (@sigs) {
        if ( $sha1sum eq $sig ) {
            print_ebury_cdorked_predef('CDORKED');
            print_critical( "\t" . $httpd_bin . " has a SHA-1 signature of " . $sha1sum );
            print_critical("\tReference: p67-68 from http://www.welivesecurity.com/wp-content/uploads/2014/03/operation_windigo.pdf");
            print_critical();
            last;
        }
    }
}

sub check_sha1_sigs_named {
    my $named = '/usr/sbin/named';
    return if !-e $named;
    return unless my $sha1sum = timed_run( 0, 'sha1sum', $named );
    if ( $sha1sum =~ m{ \A (\S+) \s }xms ) {
        $sha1sum = $1;
    }

    my @sigs = qw(
      42123cbf9d51fb3dea312290920b57bd5646cefb
      ebc45dd1723178f50b6d6f1abfb0b5a728c01968
    );

    for my $sig (@sigs) {
        if ( $sha1sum eq $sig ) {
            print_ebury_cdorked_predef('CDORKED');
            print_critical( "\t" . $named . " has a SHA-1 signature of " . $sha1sum );
            print_critical("\tReference: p67-68 from http://www.welivesecurity.com/wp-content/uploads/2014/03/operation_windigo.pdf");
            last;
        }
    }
}

sub check_sha1_sigs_ssh {
    my $ssh = '/usr/bin/ssh';
    return if !-e $ssh;
    return unless my $sha1sum = timed_run( 0, 'sha1sum', $ssh );
    if ( $sha1sum =~ m{ \A (\S+) \s }xms ) {
        $sha1sum = $1;
    }

    my @sigs = qw(
      c4c28d0372aee7001c44a1659097c948df91985d
      fa6707c7ef12ce9b0f7152ca300ebb2bc026ce0b
    );

    for my $sig (@sigs) {
        if ( $sha1sum eq $sig ) {
            print_ebury_cdorked_predef('EBURY');
            print_critical( "\t" . $ssh . " has a SHA-1 signature of " . $sha1sum );
            print_critical("\tReference: p67-68 from http://www.welivesecurity.com/wp-content/uploads/2014/03/operation_windigo.pdf");
            print_critical();
            last;
        }
    }
}

sub check_sha1_sigs_ssh_add {
    my $ssh_add = '/usr/bin/ssh-add';
    return if !-e $ssh_add;
    return unless my $sha1sum = timed_run( 0, 'sha1sum', $ssh_add );
    if ( $sha1sum =~ m{ \A (\S+) \s }xms ) {
        $sha1sum = $1;
    }

    my @sigs = qw(
      575bb6e681b5f1e1b774fee0fa5c4fe538308814
    );

    for my $sig (@sigs) {
        if ( $sha1sum eq $sig ) {
            print_ebury_cdorked_predef('EBURY');
            print_critical("\t$ssh_add has a SHA-1 signature of $sha1sum\n");
            print_critical("\tReference: p67-68 from http://www.welivesecurity.com/wp-content/uploads/2014/03/operation_windigo.pdf\n");
            print_critical();
            last;
        }
    }
}

sub check_sha1_sigs_sshd {
    my $sshd = '/usr/sbin/sshd';
    return if !-e $sshd;
    return unless my $sha1sum = timed_run( 0, 'sha1sum', $sshd );
    if ( $sha1sum =~ m{ \A (\S+) \s }xms ) {
        $sha1sum = $1;
    }

    my @sigs = qw(
      0daa51519797cefedd52864be0da7fa1a93ca30b
      4d12f98fd49e58e0635c6adce292cc56a31da2a2
      7314eadbdf18da424c4d8510afcc9fe5fcb56b39
      98cdbf1e0d202f5948552cebaa9f0315b7a3731d
    );

    for my $sig (@sigs) {
        if ( $sha1sum eq $sig ) {
            print_ebury_cdorked_predef('EBURY');
            print_critical( "\t" . $sshd . " has a SHA-1 signature of " . $sha1sum );
            print_critical("\tReference: p67-68 from http://www.welivesecurity.com/wp-content/uploads/2014/03/operation_windigo.pdf");
            print_critical();
            last;
        }
    }
}

sub check_for_ebury_ssh_G {
    my $ssh = '/usr/bin/ssh';
    return if !-e $ssh;
    return if !-f _;
    return if !-x _;
    return if -z _;

    my $ssh_version = timed_run_trap_stderr( 0, $ssh, '-V' );
    return if $ssh_version !~ m{ \A OpenSSH_5 }xms;

    my $ssh_G = timed_run_trap_stderr( 0, $ssh, '-G' );
    if ( $ssh_G !~ /illegal|unknown/ ) {
        print_ebury_cdorked_predef('EBURY');
        print_critical( "\t'" . $ssh . " -G' did not return either 'illegal' or 'unknown'" );
        print_critical("\tReference: http://www.welivesecurity.com/2014/02/21/an-in-depth-analysis-of-linuxebury/");
        print_critical();
    }
}

sub check_for_ebury_ssh_shmem {
    my $ipcs_ref = get_ipcs_href();

    # As far as we know, sshd sholudn't be using shared memory at all, so any usage is a strong sign of ebury.
    return if !defined( $ipcs_ref->{root}{mp} );

    for my $href ( @{ $ipcs_ref->{root}{mp} } ) {
        my $shmid = $href->{shmid};
        my $cpid  = $href->{cpid};
        my $procs = get_process_pid_href();
        if ( defined $procs->{$cpid} && $procs->{$cpid}->{ARGS} =~ m{ \A /usr/sbin/sshd \b }xms ) {
            print_ebury_cdorked_predef('EBURY');
            print_critical("\tShared memory segment created by sshd process exists:");
            print_critical( "\t\tsshd PID: " . $cpid );
            print_critical( "\t\tshmid: " . $shmid );
            print_critical("\tReference: http://www.welivesecurity.com/2014/02/21/an-in-depth-analysis-of-linuxebury/");
            print_critical( timed_run( 0, "echo --- ps -p ${cpid} uww ---;ps -p ${cpid} uww; echo --- ipcs -m -i ${shmid} ---; ipcs -m -i ${shmid}; echo ---" ) );
            last;
        }
    }
}

sub check_for_ebury_root_file {
    my $file = '/home/ ./root';

    if ( -e $file ) {
        print_ebury_cdorked_predef('EBURY');
        print_critical( "\tFound file: " . $file );
        print_critical("\tReference: p24 from http://www.welivesecurity.com/wp-content/uploads/2014/03/operation_windigo.pdf");
        print_critical();
    }
}

sub check_for_ncom_filenames {
    my @bad_libs;
    my @dirs  = qw( /lib /lib64 );
    my @files = qw(
      libnano.so.4
      libncom.so.4.0.1
      libselinux.so.4
    );

    for my $dir (@dirs) {
        next if !-e $dir;
        for my $file (@files) {
            my $fullpath = $dir . "/" . $file;
            stat $fullpath;
            if ( -f _ and not -z _ ) {
                push @bad_libs, $fullpath;
            }
        }
    }

    if (@bad_libs) {
        print_generic_hack_predef('NCOM ROOTKIT');
        print_critical('The following files were found:');
        print_critical( "\t" . join( " ", @bad_libs ) );
        print_critical("\tL3: use \"L3 - Ncom Rootkit Predef [L3 Only]\"");
        print_critical("\tL3: check /etc/ld.so.preload");
        print_critical();
    }
}

sub check_passwd_hiddenwasp_dirtycow {
    my @uid_0_users;
    my ( $firefart, $hiddenwasp );
    open my $fh, '<', '/etc/passwd' or die $!;
    while (<$fh>) {
        if (/\A([^:]+):[^:]+:0:0:([^:]+):/) {
            push @uid_0_users, $1 if $1 ne 'root';
            ++$firefart if ( $1 eq 'firefart' || $2 eq 'pwned' );
            ++$hiddenwasp if $1 eq 'sftp';
            last if scalar @uid_0_users >= 5;
        }
    }
    close $fh;

    if ($firefart) {
        print_generic_hack_predef('FireFart / Dirty COW');
        print_critical('The root user GECOS field in the passwd database is "pwned", which is a typical indication of the FireFart Dirty COW exploit tool.');
        print_critical("\tL3: use \"L3 - FireFart / Dirty COW Exploit [L3 Only]\"");
        print_critical();
    }
    if ($hiddenwasp) {
        print_generic_hack_predef('HiddenWasp');
        print_critical('Root SFTP user detected - possible HiddenWasp');
        print_critical("\tL3: see TECH-755");
        print_critical();
    }
}

sub check_for_malicious_root_cron {
    my %warning = ();
    return unless my $crons_aref = get_cron_files();

    for my $cron (@$crons_aref) {
        if ( open my $cron_fh, '<', $cron ) {
            while (<$cron_fh>) {
                if (m{ \A [^#]* (tor2web|\.onion\.|pastebin) }x) {
                    $warning{$cron}{$1} = 1;
                }
            }
            close $cron_fh;
        }
    }

    if (%warning) {
        print_crit("Potentially malicious root cronjob: ");
        print_critical('Escalate to L3 to before proceeding');
        for my $cron ( keys(%warning) ) {
            print_critical( "\t \\_ " . $cron . " contains [ " . join( ' ', sort( keys( %{ $warning{$cron} } ) ) ) . " ]" );
        }
    }
}

sub check_for_spoofed_kernel_modules {
    my @dirs = qw(
      /usr/local/sbin
      /usr/local/bin
      /sbin
      /bin
      /usr/sbin
      /usr/bin
    );

    my @files;
    foreach my $dir (@dirs) {
        ( -d $dir && !-l $dir ) ? push @files, glob("$dir/[*") : next;
    }

    my (@spoofs) = grep { !/(\/usr\/)?\/bin\/\[$/m } @files;
    if (@spoofs) {
        print_crit('Suspicious file found: ');
        print_critical('Escalate to L3 to confirm their validity before proceeding(see TECH-762)');
        print_critical( join "\n", map { "\t \\_ $_" } @spoofs );
    }
}

sub check_for_bad_perms_auth_keys {
    my $checkmode = '0600';
    my $path      = '/root/.ssh/authorized_keys';
    return unless my ( $uid, $gid ) = ( lstat($path) )[ 4, 5 ];
    my $user = getpwuid($uid);
    $user = $user || $uid;
    my $group = getgrgid($gid);
    $group = $group || $gid;

    if ( !( $user eq 'root' ) || !( $group eq 'root' ) ) {
        print_generic_hack_predef('Potential Root Compromise');
        print_critical( 'Non-default Perms: ' . $path . ' [owner ' . $user . ':' . $group . '] (default root:root)' );
    }
}

sub check_for_strange_setuid_binaries {
    my @paths = qw( /etc/.ip /var/log/.log );
    return unless my @bad_paths = grep { -e $_ } @paths;
    print_generic_hack_predef('Potential Root Compromise');
    print_critical('The following files were found (see TECH-791):');
    print_critical( "\t [run the file command against these and if ASCII, no need to escalate. Tag UPS-293]" );
    print_critical( "\t" . join( "\n\t", @bad_paths ) );
}

sub check_for_cpro {
    return unless i_am('cptech');
    my @paths = qw(
      /etc/customcspips
      /opt/cpanel/csp/
      /usr/bin/clnupdate
      /usr/bin/checkstatus
      /usr/bin/cspdaemon
      /usr/bin/cpd
      /usr/bin/HostCartCP
      /usr/bin/HostCart_cpanel
      /usr/bin/HSLicenseCP
      /usr/bin/RcCpanel
      /usr/bin/rccpcontrol
      /usr/bin/RcLicense_cpanel
      /usr/bin/update_clnv2
      /usr/bin/update_clnv2.lock
      /usr/bin/update_cloudv2
      /usr/bin/update_cloudv2.lock
      /usr/bin/update_cpanelv2
      /usr/bin/update_cpanelv2.lock
      /usr/bin/update_cpanelv3
      /usr/bin/update_cpanelv3.lock
      /usr/bin/update_jetbackup
      /usr/bin/update_jetbackup.lock
      /usr/bin/update_lswsv2
      /usr/bin/update_lswsv2.lock
      /usr/bin/yasin
      /usr/bin/gblicensecp
      /usr/bin/gblicensecp_update
      /usr/bin/gblicensels_run
      /usr/bin/gblicensewr
      /usr/local/cpanel/bin/cspconnector
      /usr/local/cpanel/bin/sendmail.txt
      /usr/local/cpanel/bin/update_cpanelv2
      /usr/local/cpanel/bin/update_cpanelv2.lock
      /usr/local/cpanel/bin/way2/
      /usr/local/cpanel/license/update_cpanelv2
      /usr/local/cpanel/whostmgr/cgi/check.php
      /var/log/updatecp/
      /var/run/cspdaemon.lock
      /YasITCSP/
    );

    if ( my @bad_paths = grep { -e $_ } @paths ) {
        print_generic_hack_predef('C.PRO');
        print_critical('The following files or directories were found:');
        print_critical( "\t" . join( " ", @bad_paths ) );
        print_critical("\tL3: See ticket 7790559 for reference.");
        print_critical();
    }
    if ( my %procs = grep_process_cmd( '^(cspdaemon|checkstatus)$', 'root' ) ) {
        print_generic_hack_predef('C.PRO');
        print_critical('The following suspicious running processes were found (please verify):');
        print_critical( "\t\\_ " . join( "\n\t\\_ ", map { my ( $u, $c, $a ) = @{ $procs{$_} }{ 'USER', 'COMM', 'ARGS' }; "[pid: $_] [user: $u] [cmd: $c] [args: $a]" } keys %procs ) );
        print_critical("\tL3: See TECH-693 for reference.");
        print_critical();
    }
}

sub check_for_fkcplisc {
    return unless i_am('cptech');
    my @paths = qw(
      /bin/cpk
      /bin/cplicense
      /bin/cplicense-32bit
      /bin/cplicense-64bit
      /bin/jonior-license
      /sbin/jonrebo
      /srv/license.php
      /usr/local/cpanel/cpkeyclt.license
      /usr/local/cpl2016
      /usr/local/fkcplisc
      /usr/local/sectools
    );

    my @bad_paths = grep { -e $_ } @paths;
    if (@bad_paths) {
        print_generic_hack_predef('FKCPLISC');
        print_critical('The following files or directories were found:');
        print_critical( "\t" . join( " ", @bad_paths ) );
        print_critical("\tL3: See TECH-269 and TECH-318 for reference.");
        print_critical();
    }
}

sub check_for_yoncu {
    return unless i_am('cptech');

    my $bad_paths_found    = 0;
    my $bad_cpkeyclt_found = 0;

    my @paths = qw(
      /usr/bin/chattrx
      /usr/local/cpanel/cpkeyclt.yedek
    );

    my @bad_paths = grep { -e $_ } @paths;
    if (@bad_paths) {
        $bad_paths_found = 1;
    }

    if ( -s '/usr/local/cpanel/cpkeyclt' && -T _ ) {
        $bad_cpkeyclt_found = 1;
    }

    if ( $bad_paths_found || $bad_cpkeyclt_found ) {
        print_generic_hack_predef('YONCU');
        if ($bad_paths_found) {
            print_critical('The following files or directories were found:');
            print_critical( "\t" . join( " ", @bad_paths ) );
        }
        if ($bad_cpkeyclt_found) {
            print_critical("\t/usr/local/cpanel/cpkeyclt was found to be a text file");
        }
        print_critical("\tL3: See TECH-657 for reference.");
        print_critical();
    }

    return;
}

sub check_for_cgls {
    return unless i_am('cptech');
    my @paths = qw(
      /etc/cron.d/cgls
      /usr/local/cgls
    );

    my @bad_paths = grep { -e $_ } @paths;
    if (@bad_paths) {
        print_generic_hack_predef('CGLS');
        print_critical('The following files or directories were found:');
        print_critical( "\t" . join( " ", @bad_paths ) );
        print_critical("\tL3: See TECH-333 for reference.");
        print_critical();
    }
}

sub check_for_ctls {
    return unless i_am('cptech');
    my @paths = qw(
      /usr/local/ctls
      /usr/local/ctls/ctls
    );

    my @bad_paths = grep { -e $_ } @paths;
    if (@bad_paths) {
        print_generic_hack_predef('CTLS');
        print_critical('The following files or directories were found:');
        print_critical( "\t" . join( " ", @bad_paths ) );
        print_critical("\tL3: See TECH-642 for reference.");
        print_critical();
    }
}

sub check_for_xbash {
    return unless my $mysql_datadir = get_mysql_datadir();
    return unless -d $mysql_datadir;
    opendir( my $dh, $mysql_datadir );
    my ($hit) = grep { /PLEASE_READ/i } readdir $dh;
    closedir $dh;

    if ($hit) {
        print_generic_hack_predef('Xbash');
        print_critical("\tThe following files or directories were found:");
        print_critical( "\t\t" . $mysql_datadir . $hit );
    }
}

sub check_for_missing_ps_cmd {
    return if ( -s '/bin/ps' or -s '/usr/bin/ps' );
    print_generic_hack_predef('Potential Root Compromise');
    print_critical("\t\\_ The ps command is missing!");
}

sub check_changepasswd_modules {
    my $dir = '/usr/local/cpanel/Cpanel/ChangePasswd/';
    return unless ( -d $dir );
    return unless opendir( my $dh, $dir );
    my @dir_contents = grep { /\.pm\Z/ } readdir $dh;
    close $dh;
    return unless @dir_contents;
    my @suspicious;
    foreach my $module (@dir_contents) {
        next if ( $module eq 'DigestAuth.pm' );
        push @suspicious, $module if ( -s $dir . $module );
    }

    if (@suspicious) {
        print_generic_hack_predef('Cpanel::ChangePasswd');
        print_critical("\tFound custom ChangePasswd module(s) [ " . join( ' ', @suspicious ) . ' ]' );
        print_critical("\tThese modules can intercept all system password changes, see TECH-893");
    }
}

## END malware checks

sub get_rpm_href {
    my $timeout = $OPT_TIMEOUT ? $OPT_TIMEOUT : 25;
    return unless my $list = timed_run( $timeout, 'rpm', '-qa', '--queryformat', q{%{NAME}\t%{VERSION}\t%{RELEASE}\t%{ARCH}\t%{INSTALLTIME}\n} );
    my %rpms;
    for my $line ( split( /\n/, $list ) ) {
        my ( $name, $version, $release, $arch, $installtime ) = split( /\t/, $line );
        push @{ $rpms{$name} }, {
            'version'     => defined $version     ? $version     : '',
            'release'     => defined $release     ? $release     : '',
            'arch'        => defined $arch        ? $arch        : '',
            'installtime' => defined $installtime ? $installtime : '',
        };
    }
    return \%rpms;
}

sub get_printable_rpm_packages {
    my ($name) = @_;
    return unless my $rpms = get_rpm_href();
    return unless exists $rpms->{$name};
    my @list;
    for my $ref ( @{ $rpms->{$name} } ) {
        push @list, $name . "-" . $ref->{version} . "-" . $ref->{release} . "." . $ref->{arch};
    }
    @list = sort @list;
    return @list;
}

sub get_flat_conf_href {

    # Import an unstructured, commented config file delimited by whitespace, equal, or colon after a single keyword.
    # Hash keys are stored lowercase if $lower_key is true (only use when configuration keywords are case-insensitive!)
    # The original keyword case is stored in array position 0, and the value is stored in array position 1.
    # Should work for sshd_config with $lowercase=1, cpanel.config and php.ini with $lowercase=0, etc.
    # This does not work for configuration keywords that can be used multiple times, only the last keyword found will be stored.
    my ( $file, $lower_key ) = @_;
    return unless defined $file;
    return unless -e $file;
    my %conf;
    open my $fh, '<', $file or return;
    while ( my $line = <$fh> ) {
        next if $line =~ m{ \A \s* ( [;#] | \s* \Z ) }xms;    # Ignore comments prefixed by hash or semicolon, or empty lines
        my ( $key, $value ) = split( /\s*[\s=:]\s*/, $line, 2 );    # Split on whitespace, =, or :.  Ignore additional whitespace.
        next unless defined $key;
        $key =~ s{ \A \s+ | \s+ \Z }{}gxms;                         # Strip leading/trailing whitespace
        $key =~ s{ \A ['"] | ['"] \Z }{}gxms;                       # Strip leading/trailing quotes
        if ( defined $value ) {
            $value =~ s{ \A \s+ | \s+ \Z }{}gxms;
            $value =~ s{ \A ['"] | ['"] \Z }{}gxms;
        }
        $conf{ $lower_key ? lc($key) : $key } = [ $key, $value ];
    }
    close $fh;
    return \%conf;
}

sub check_sshd_config {
    return unless i_am_one_of( 'cpanel', 'dnsonly' );
    my $sshd_config = '/etc/ssh/sshd_config';
    return unless my $conf = get_flat_conf_href( $sshd_config, 1 );

    # Example: 'lowercaseoption' => { default => 'lowercasedefaultvalue', check_missing => 1, orig_name => 'MixedCaseName', help => 'Help text' },
    # default, check_missing, and help are optional, but orig_name should be defined if check_missing is used so that the mixed-case name can be printed
    # If check_missing exists, a warning is generated if the item is NOT in sshd_conf and does not match the default (if given)
    my %checks;
    my $add = sub {
        my ( $name, $new_ref ) = @_;
        $checks{$name}{'help'} = [] unless defined $checks{$name}{'help'};
        for my $key ( keys %{$new_ref} ) {
            if ( $key eq 'help' ) {
                push @{ $checks{$name}{'help'} }, @{ $new_ref->{'help'} };
                next;
            }
            $checks{$name}{$key} = $new_ref->{$key};
        }
    };

    if ( not -e '/etc/cphulkddisable' ) {
        $add->( 'usedns', { default => 'no',  check_missing => 1, orig_name => 'UseDNS', help => ['Must be explicitly set to "no" for cPHulk to work correctly for SSH logins'] } );
        $add->( 'usepam', { default => 'yes', check_missing => 1, orig_name => 'UsePAM', help => ['Must be explicitly set to "yes" for cPHulk to work with SSH logins'] } );
    }
    if ( i_am('cloudlinux') ) {
        $add->( 'usepam', { default => 'yes', check_missing => 1, orig_name => 'UsePAM', help => ['Must be explicitly set to "yes" for CloudLinux LVE limits or CageFS to apply to SSH sessions'] } );
    }

    my $indent = 34;
    for my $lc_key ( sort( keys(%checks) ) ) {
        my $check         = $checks{$lc_key};
        my $lc_default    = defined $check->{default} ? $check->{default} : '';
        my $check_missing = defined $check->{check_missing} ? 1 : 0;
        my $missing_name  = defined $check->{orig_name} ? $check->{orig_name} : $lc_key;
        my $help          = scalar @{ $check->{help} } ? '- ' : '';
        my ( $conf_key, $conf_value ) = defined $conf->{$lc_key} ? @{ $conf->{$lc_key} } : undef;
        my $lc_conf_value = defined $conf_value ? lc($conf_value) : undef;
        if ( defined $lc_conf_value and $lc_conf_value ne $lc_default ) {
            $help .= join( "\n" . ' ' x ( $indent + length($conf_key) + length($conf_value) ) . '\_ - ', @{ $check->{help} } );
            print_warn( $sshd_config . ': ' );
            print_warning("[ $conf_key $conf_value ] $help");
        }
        elsif ( $check_missing and not defined $conf_value ) {
            $help .= join( "\n" . ' ' x ( $indent + length($missing_name) + 9 ) . '\_ - ', @{ $check->{help} } );
            print_warn( $sshd_config . ': ' );
            print_warning("[ $missing_name ] not found $help");
        }
    }
}

sub check_pure_ftpd_conf_for_upload_script_and_dead {
    return unless i_am('cpanel');
    return unless defined( $CPCONF{'ftpserver'} ) && $CPCONF{'ftpserver'} eq 'pure-ftpd';
    return unless my $pureftpd_conf = get_pureftpd_conf_href();
    if ( defined( $pureftpd_conf->{'calluploadscript'} ) && $pureftpd_conf->{'calluploadscript'}->{value} eq 'yes' ) {
        if ( !-e '/var/run/pure-ftpd.upload.pipe' ) {
            print_warn("$PURE_FTPD_CONF_FILE: ");
            print_warning("CallUploadScript set to yes, /var/run/pure-ftpd.upload.pipe is missing [might be broken ConfigServer's cxs ( http://configserver.com/cp/cxs.html ) or Imunify360]");
        }
        else {
            unless ( grep_process_cmd( 'pure-uploadscri', 'root' ) ) {
                print_warn("$PURE_FTPD_CONF_FILE: ");
                print_warning("CallUploadScript set to yes, but pure-uploadscript does not appear to be running. [might be broken ConfigServer's cxs ( http://configserver.com/cp/cxs.html ) or Imunify360]");
            }
        }
    }
}

sub get_myip {
    my ($port) = @_;
    $port = defined $port ? $port : '80';

    # myip.cpanel.net supports HTTP ports 80, 2089 and HTTPS port 443.
    my $ip;
    my $reply = _http_get( Host => 'myip.cpanel.net', Path => '/v1.0/', ReportTimeout => 1, Port => $port );
    if ( defined($reply) && $reply =~ m{ ^ \s* ([0-9]+.[0-9]+.[0-9]+.[0-9]+) \s* $ }xms ) {
        $ip = $1;
        chomp $ip;
    }
    return $ip;
}

sub get_external_ip {
    return get_myip();
}

sub get_external_license_ip {
    return get_myip('2089');
}

sub _http_get {

    # SSL connections have external dependencies that are not part of core Perl 5.6.
    my (%opts_in) = @_;
    return if $OPT_SKIP_NETWORKING;
    my %opts = (
        Agent               => "SSP/${VERSION}",
        HostReconnectDelay  => 0.25,
        MaxReply            => 50_000,
        MultiHomed          => 1,
        Path                => '/',
        Proto               => 'tcp',
        ReportTimeout       => 0,
        ReportTimeoutHeader => '',
        SSL                 => 0,
        SSLLoadFailWarning  => undef,
        SSLVerifyHost       => 1,
        Tries               => 1,
        WantHeaders         => 0,
        %opts_in
    );
    my $try = 0;
    my $reply;
    my $scheme;
    die unless defined $opts{Host};
    if ( $opts{SSL} ) {
        $scheme = 'https://';
        return unless load_module_with_fallbacks(
            'needed_subs'  => [qw(start_SSL import)],
            'modules'      => [qw(IO::Socket::SSL)],
            'fail_warning' => $opts{SSLLoadFailWarning},
        );
        $opts{Port}    = 443 if not defined $opts{Port};
        $opts{Timeout} = 3   if not defined $opts{Timeout};
    }
    else {
        $scheme = 'http://';
        $opts{Port}    = 80 if not defined $opts{Port};
        $opts{Timeout} = 2  if not defined $opts{Timeout};
    }
    for ( 1 .. $opts{Tries} ) {
        local $@;
        $try++;
        eval {
            my $sock;
            local $SIG{'ALRM'} = sub {
                close $sock if defined $sock;
                die "alarm\n";
            };
            my $now = time();
            if ( defined $HTTP_GET_HOST_CACHE->{ $opts{Host} } and $now - 1 - $HTTP_GET_HOST_CACHE->{ $opts{Host} } <= $opts{HostReconnectDelay} ) {
                select( undef, undef, undef, $opts{HostReconnectDelay} );    ## no critic (ProhibitSleepViaSelect) # old perl compat
            }
            $HTTP_GET_HOST_CACHE->{ $opts{Host} } = $now;
            alarm $opts{Timeout} + ( $opts{MultiHomed} ? $opts{Timeout} + 1 : 0 );    # MultiHomed = double timeout + 1 to give it a chance to work
            $sock = IO::Socket::INET->new(
                MultiHomed => $opts{MultiHomed},
                PeerAddr   => $opts{Host},
                PeerPort   => $opts{Port},
                Proto      => $opts{Proto},
                Timeout    => $opts{Timeout},
            );
            if ( $opts{SSL} ) {
                my $ssl_verify_opt;
                ## no critic (StringyEval) # The SSL_VERIFY_* constants may not be loaded at compile time.
                eval q(
                    IO::Socket::SSL->import(qw(SSL_VERIFY_PEER SSL_VERIFY_NONE));
                    $ssl_verify_opt = $opts{SSLVerifyHost} ? IO::Socket::SSL::SSL_VERIFY_PEER : IO::Socket::SSL::SSL_VERIFY_NONE;
                );
                ## use critic
                return unless defined $ssl_verify_opt;
                return unless IO::Socket::SSL->start_SSL( $sock, SSL_hostname => $opts{Host}, SSL_verify_mode => $ssl_verify_opt );
            }
            my $header_host = ( $opts{Port} != 80 and $opts{Port} != 443 ) ? $opts{Host} . ":" . $opts{Port} : $opts{Host};
            if ($sock) {
                print $sock "GET " . $opts{Path} . " HTTP/1.0\r\nUser-Agent: " . $opts{Agent} . "\r\nHost: " . $header_host . "\r\nAccept: */*\r\n\r\n";
                read $sock, $reply, $opts{MaxReply};
                close $sock;
            }
            alarm 0;
        };
        if ( $@ eq "alarm\n" && $opts{ReportTimeout} ) {
            print_warn( $opts{ReportTimeoutHeader} . 'Request for ' . $scheme . $opts{Host} . ':' . $opts{Port} . $opts{Path} . ' timed out' );
            print_warning( ( $opts{Tries} > 1 ) ? ': attempt ' . $try . ' of ' . $opts{Tries} : '' );
            print RESET;
            next;
        }
        if ( defined $reply && length $reply ) {
            $reply =~ s/^.*?(\r\n){2}//s unless defined $opts{WantHeaders} && $opts{WantHeaders};
            return $reply;
        }
    }
    return;
}

sub _month_to_num {

    # Convert "Jan" to 0 for timegm(), etc.
    local ($_) = @_;
    return 0  if /^Jan/i;
    return 1  if /^Feb/i;
    return 2  if /^Mar/i;
    return 3  if /^Apr/i;
    return 4  if /^May/i;
    return 5  if /^Jun/i;
    return 6  if /^Jul/i;
    return 7  if /^Aug/i;
    return 8  if /^Sep/i;
    return 9  if /^Oct/i;
    return 10 if /^Nov/i;
    return 11 if /^Dec/i;
}

sub print_bug_report {
    my $mysql_full_version = get_mysql_full_version();
    my $hostinfo           = get_hostinfo_href();
    my $cpuinfo            = get_cpuinfo_href();
    my $version            = _get_run_var('cpanel_original_version');
    my $os                 = _get_run_var('os_release');
    my $kernel             = $hostinfo->{'kernel'} ? $hostinfo->{'kernel'} : 'Unknown';
    my $arch               = $hostinfo->{'hardware'} ? $hostinfo->{'hardware'} : 'Unknown';
    my $environment        = $hostinfo->{'environment'} ? $hostinfo->{'environment'} : 'Unknown';
    my $cpu                = $cpuinfo->{'model'} ? $cpuinfo->{'model'} : 'Unknown';
    my $cores              = $cpuinfo->{'numcores'} ? $cpuinfo->{'numcores'} : 'Unknown';
    my $ticket             = ( defined $ENV{HISTFILE} and $ENV{HISTFILE} =~ /ticket.(\d+)$/ ) ? $1 : '';
    print <<END_OF_TEMPLATE;
==== BEGIN COPY BELOW ====
{panel:title=(i) Details}

{panel}

----

{panel:title=(+) Step-by-Step Instructions}

{panel}

----

{panel:title=(?) Suggested Resolution / Expected Behavior}

{panel}

----

{panel:title=(/) Known work around(s)}

{panel}
==== END COPY ABOVE ====

Useful Information:

Ticket Number: $ticket

!!! THE INFORMATION BELOW SHOULD BE PASTED IN THE ENVIRONMENT SECTION OF YOUR CASE !!!

cPanel Version: $version
OS: $os
Kernel: $kernel
Arch: $arch
Environment: $environment
CPU: $cpu w/ $cores core(s)
MySQL: $mysql_full_version

END_OF_TEMPLATE
}

sub print_doc_bug_report {
    my $version = _get_run_var('cpanel_original_version');
    my $ticket  = "";
    if ( defined $ENV{HISTFILE} and $ENV{HISTFILE} =~ /ticket.(\d+)$/ ) {
        $ticket = $1;
    }
    print <<END_OF_TEMPLATE;
==== BEGIN COPY BELOW ====
{panel:title=(i) Document and ticket URLS}

* Link to the document that requires modification.
* Link to the support ticket that contains the relevant information.

{panel}

----

{panel:title=(+) Details}

Provide relevant information for any documentation changes.
If this is a request for new documentation, provide an outline with the technical details required to complete the document.

{panel}

----

{panel:title=(?) Steps to reproduce}

Provide any technical steps required by the writer for the changes needed.

{panel}

----

{panel:title=(/) Additional Information}


Any other useful information for the tech writer to complete the changes.

{panel}
==== END COPY ABOVE ====

Useful Information:

Ticket Number: $ticket
cPanel Version: $version

END_OF_TEMPLATE
}

sub print_profiling_data {
    if ( defined $MEMOIZE_CACHE{'PROFILING'} ) {
        my $combined_elapsed;
        my $combined_precache;
        print "\n\nMemoize cache profile:\n";
        for my $func ( sort keys %MEMOIZE_CACHE ) {
            next if $func eq 'PRECACHE';
            next if $func eq 'PROFILING';
            my $precached = $MEMOIZE_CACHE{$func}->{'PRECACHED'} ? ' [precached]' : '';
            print "  ${func}${precached} elapsed:\n";
            for my $key ( sort keys %{ $MEMOIZE_CACHE{$func}->{'PROFILE'} } ) {
                $combined_elapsed += $MEMOIZE_CACHE{$func}->{'PROFILE'}->{$key}->{'elapsed'};
                $combined_precache += $MEMOIZE_CACHE{$func}->{'PROFILE'}->{$key}->{'elapsed'} if $precached;
                printf "    %.3f s [%s]\n", $MEMOIZE_CACHE{$func}->{'PROFILE'}->{$key}->{'elapsed'}, $key;
            }
        }
        printf "\nCombined memoized functions elapsed time: %.3f s\n", $combined_elapsed;
        if ( defined $MEMOIZE_CACHE{'PRECACHE'} and defined $MEMOIZE_CACHE{'PRECACHE'}->{'PROFILE'} ) {
            my $precache_wall = $MEMOIZE_CACHE{'PRECACHE'}->{'PROFILE'}->{'elapsed'};
            printf "Precache combined time elapsed: %.3f s\n", $combined_precache;
            printf "Precache wall time elapsed: %.3f s\n",     $precache_wall;
            printf "Precache combined minus wall (wall time saved by precaching): %.3f s\n", ( $combined_precache - $precache_wall );
        }
        print "\n";
        my @times = times;
        my @labels = ( 'User', 'System', 'User (all children)', 'System (all children)' );
        print "$labels[$_]: $times[$_] s\n" for 0 .. $#times;
    }
}

sub get_tiers_json_href {
    my $raw = _http_get( Host => 'httpupdate.cpanel.net', Path => '/cpanelsync/TIERS.json' );
    return unless $raw;
    my $href = get_json_href($raw);
    return if not exists $href->{'tiers'};
    return $href;
}

sub get_tier_info_for_version_href {
    my ($version) = @_;
    return unless my $tiers_ref = get_tiers_json_href();
    my ( $parent_ver, $major_ver ) = split( /\./, $version, 3 );
    my @found_tiers = ();
    return unless defined $parent_ver and $parent_ver =~ /^\d+$/;
    return unless defined $major_ver  and $major_ver =~ /^\d+$/;
    $major_ver++ if $major_ver % 2;    # Bump odd dev versions
    my $full_ver = join( '.', $parent_ver, $major_ver );

    if ( exists $tiers_ref->{'tiers'}->{$full_ver} ) {
        return $tiers_ref->{'tiers'}->{$full_ver}->[0] unless (scalar @{ $tiers_ref->{'tiers'}->{$full_ver} } gt 1);
        foreach my $tier ( @{ $tiers_ref->{'tiers'}->{$full_ver} } ) {
            return $tier if exists $tier->{'is_main'} && $tier->{'is_main'};
        }
    }

    return;
}

sub _populate_run_state {              ## no critic (RequireArgUnpacking)
    return unless _init_run_state();
    _set_run_type('kernelcare') if -x '/usr/bin/kcarectl';
    _set_run_type('wptk') if -x '/usr/local/bin/wp-toolkit';
    _set_run_type('jetbackup') if -x '/usr/bin/jetbackup';

    if ( -d '/usr/local/cpanel' ) {
        if ( -e '/var/cpanel/dnsonly' or license_file_is_dnsonly() ) {
            _set_run_type('dnsonly');
        }
        else {
            _set_run_type('cpanel');
            _set_run_type('solo') if license_file_is_solo();
            if ( -f '/etc/cpanel/ea4/is_ea4' ) {
                _set_run_type('ea4');
            }
            elsif ( -d '/usr/local/apache' ) {
                _set_run_type('ea3');
            }
        }
        my ( $cp_numeric_version, $cp_original_version ) = get_cpanel_version();
        _set_run_var( 'cpanel_numeric_version',  $cp_numeric_version );
        _set_run_var( 'cpanel_original_version', $cp_original_version );
    }

    if ( exists $ENV{'PACHA_AUTOFIXER'} ) {
        _set_run_type('cptech');
    }
    elsif ( defined $ENV{'HISTFILE'} and index( $ENV{'HISTFILE'}, 'cpanel_ticket' ) != -1 ) {
        _set_run_type('cptech');
    }
    else {
        foreach ( @ENV{ 'SSH_CLIENT', 'SSH_CONNECTION' } ) {
            next unless defined $_;
            next unless m{\A (184\.94\.197\.[2-6]|208\.74\.123\.98)}xms;
            _set_run_type('cptech');
            last;
        }
    }

    # Loosely based on info found in Cpanel::Sys::OS and Cpanel::Sys::GetOS.  First release file wins.
    foreach my $test_release_file ( 'CentOS-release', 'redhat-release', 'system-release' ) {
        if ( open my $fh, '<', '/etc/' . $test_release_file ) {
            my $release = readline $fh;
            close $fh;
            chomp $release;
            $release =~ s/^\s+|\s+$//;

            if ( length $release >= 4 ) { _set_run_var( 'os_release', $release ); }

            if ( $test_release_file eq 'system-release' ) { _set_run_type('amazon'); _set_run_var( 'os_ises', 1 ) }

            if    ( $release =~ m/(?:Amazon)/i ) { _set_run_type('amazon');     _set_run_var( 'os_ises', 1 ); }
            elsif ( $release =~ /CloudLinux/i )  { _set_run_type('cloudlinux'); _set_run_var( 'os_ises', 2 ); }
            elsif ( $release =~ m/(?:Corporate|Advanced\sServer|Enterprise)/i ) { _set_run_var( 'os_ises', 1 ); }
            elsif ( $release =~ /CentOS/i ) { _set_run_var( 'os_ises', 2 ); }

            if    ( $release =~ /(\d+\.\d+)/ ) { _set_run_var( 'os_version', $1 ); }
            elsif ( $release =~ /(\d+)/ )      { _set_run_var( 'os_version', $1 ); }

            last;
        }
    }
}

sub _init_run_state {
    return if defined $RUN_STATE;
    $RUN_STATE = {
        STATE => 0,
        type  => {
            cpanel     => 1 << 0,
            solo       => 1 << 1,
            dnsonly    => 1 << 2,
            ea3        => 1 << 3,
            ea4        => 1 << 4,
            cloudlinux => 1 << 5,
            kernelcare => 1 << 6,
            amazon     => 1 << 7,
            cptech     => 1 << 8,
            wptk       => 1 << 9,
        },
        var => {
            cpanel_numeric_version  => "UNKNOWN",
            cpanel_original_version => "UNKNOWN",
            os_release              => "UNKNOWN",
            os_version              => "UNKNOWN",
            os_ises                 => 0,
        },
    };
    return 1;
}

sub _set_run_type {
    my ($type) = @_;
    print STDERR "SSP DEBUG - Runtime type ${type} doesn't exist\n" and return unless exists $RUN_STATE->{type}->{$type};
    return $RUN_STATE->{STATE} |= $RUN_STATE->{type}->{$type};
}

sub _set_run_var {
    my ( $key, $value ) = @_;
    print STDERR "SSP DEBUG - Runtime var ${key} doesn't exist\n" and return unless exists $RUN_STATE->{var}->{$key};
    return $RUN_STATE->{var}->{$key} = $value;
}

sub _get_run_var {
    my ($key) = @_;
    print STDERR "SSP DEBUG - Runtime var ${key} doesn't exist\n" and return unless exists $RUN_STATE->{var}->{$key};
    return $RUN_STATE->{var}->{$key};
}

sub _simulate_run_state {    ## no critic (RequireArgUnpacking)
    my ($value) = @_;
    _init_run_state();
    _set_run_type($value);
}

sub _simulate_run_var {      ## no critic (RequireArgUnpacking)
    my ( $key, $value ) = @_;
    _init_run_state();
    _set_run_var( $key, $value );
}

sub i_am {                   ## no critic (RequireArgUnpacking)
                             # All These Things are True
    my $want = 0;
    grep { return 0 unless exists $RUN_STATE->{type}->{$_}; $want |= $RUN_STATE->{type}->{$_} } @_;
    return $want == ( $want & $RUN_STATE->{STATE} );
}

sub i_am_only {              ## no critic (RequireArgUnpacking)
                             # Only These Things are True
    my $want = 0;
    grep { return 0 unless exists $RUN_STATE->{type}->{$_}; $want |= $RUN_STATE->{type}->{$_} } @_;
    return $want == $RUN_STATE->{STATE};
}

sub i_am_one_of {            ## no critic (RequireArgUnpacking)
                             # At Least One Of These Things are True
    return scalar grep { exists $RUN_STATE->{type}->{$_} and $RUN_STATE->{type}->{$_} & $RUN_STATE->{STATE} } @_;
}

sub cpanel_version_is {
    my ( $mode, $ver ) = @_;
    return version_compare( $RUN_STATE->{var}->{cpanel_numeric_version}, $mode, $ver );
}

sub os_version_is {
    my ( $mode, $ver ) = @_;
    return version_compare( $RUN_STATE->{var}->{os_version}, $mode, $ver );
}

sub ea3_downgrade_is_possible {
    return 1 if cpanel_version_is(qw( < 11.77.0.0 )) and i_am('ea4');
    return 0;
}

sub get_hostname {

    # Return hostname() as-is on versions prior to 70.
    if ( cpanel_version_is(qw( < 11.69.0.0 )) ) {
        return hostname();
    }
    my $h = timed_run( 10, 'hostname', '-f' );
    chomp $h if defined $h;
    if ( not length($h) ) {    # Fall back to Sys::Hostname
        $h = hostname();
    }
    return $h;
}

# SUB load_module_with_fallbacks(
#   'modules'      => [ 'module1', 'module2', ... ],
#   'needed_subs'  => [ 'do_needful', ... ],
#   'fallback'     => sub { *do_needful = sub { ... }; return; },
#   'fail_warning' => "Oops, something went wrong, you may want to do something about this",
#   'fail_fatal'   => 1,
# );
#
# Input is HASH of options:
#   'modules'      => ARRAYREF of SCALAR strings corresponding to module names to attempt to import. These are attempted first.
#   'needed_subs'  => ARRAYREF of SCALAR strings corresponding to subroutine names you need defined from the module(s).
#   'fallback'     => CODEREF which defines the needed subs manually. Only used if all modules passed in above fail to load. Optional.
#   'fail_warning' => SCALAR string that will convey a message to the user if the module(s) fail to load. Optional.
#   'fail_fatal'   => BOOL whether you want to die if you fail to load the needed subs/modules via all available methods. Optional.
#
# Returns the module/namespace that loaded correctly, throws if all available attempts at finding the desired needed_subs subs fail and fail_fatal is passed.
sub load_module_with_fallbacks {
    my %opts = @_;
    my $namespace_loaded;
    foreach my $module2try ( @{ $opts{'modules'} } ) {

        # Don't 'require' it if we already have it.
        my $inc_entry = join( "/", split( "::", $module2try ) ) . ".pm";
        if ( !$INC{$module2try} ) {
            local $@;
            next if !eval "require $module2try; 1";    ## no critic (StringyEval)
        }

        # Check if the imported modules 'can' do the job
        next if ( scalar( grep { $module2try->can($_) } @{ $opts{'needed_subs'} } ) != scalar( @{ $opts{'needed_subs'} } ) );

        # Ok, we're good to go!
        $namespace_loaded = $module2try;
        last;
    }

    # Fallback to coderef, but don't do sanity checking on this, as it is presumed the caller "knows what they are doing" if passing a coderef.
    if ( !$namespace_loaded ) {
        if ( !$opts{'fallback'} || ref $opts{'fallback'} != 'CODE' ) {
            print_warn( 'Missing Perl Module(s): ' . join( ', ', @{ $opts{'modules'} } ) . ' -- ' . $opts{'fail_warning'} . " -- Try using /usr/local/cpanel/3rdparty/bin/perl?\n" ) if $opts{'fail_warning'};
            die "Stopping here." if $opts{'fail_fatal'};
        }
        else {
            $opts{'fallback'}->();

            # call like main::subroutine instead of Name::Space::subroutine
            $namespace_loaded = 'main';
        }
    }
    return $namespace_loaded;
}

# Memoize

sub _memoize {    ## no critic (RequireArgUnpacking)
    for my $func (@_) {
        if ( defined &{$func} ) {
            my $func_ref = \&{$func};
            $MEMOIZE_CACHE{$func} = {};
            my $cache_sub = sub {
                my $key = ( wantarray() ? 'L' : 'S' ) . '^' . join( ' ', @_ );
                if ( not exists $MEMOIZE_CACHE{$func}->{$key} ) {
                    if ( defined $MEMOIZE_CACHE{'PROFILING'} ) {
                        $MEMOIZE_CACHE{$func}->{'PROFILE'}->{$key}->{'start'} = [ Time::HiRes::gettimeofday() ];
                    }
                    $MEMOIZE_CACHE{$func}->{$key} = $func_ref->(@_);
                    if ( defined $MEMOIZE_CACHE{'PROFILING'} ) {
                        $MEMOIZE_CACHE{$func}->{'PROFILE'}->{$key}->{'elapsed'} = Time::HiRes::tv_interval( $MEMOIZE_CACHE{$func}->{'PROFILE'}->{$key}->{'start'} );
                    }
                }
                print STDOUT $MEMOIZE_CACHE{$func}->{'STDOUT'} if defined $MEMOIZE_CACHE{$func}->{'STDOUT'};
                delete $MEMOIZE_CACHE{$func}->{'STDOUT'}       if defined $MEMOIZE_CACHE{$func}->{'STDOUT'};
                print STDERR $MEMOIZE_CACHE{$func}->{'STDERR'} if defined $MEMOIZE_CACHE{$func}->{'STDERR'};
                delete $MEMOIZE_CACHE{$func}->{'STDERR'}       if defined $MEMOIZE_CACHE{$func}->{'STDERR'};
                return $MEMOIZE_CACHE{$func}->{$key};
            };
            no strict 'refs';          ## no critic (ProhibitNoStrict)
            no warnings 'redefine';    ## no critic (ProhibitNoWarnings)
            *{$func} = $cache_sub;
        }
        else {
            print STDERR "SSP DEBUG - Tried to memoize function that does not exist: $func\n";
        }
    }
}

sub _memoize_parallel_populate_cache {    ## no critic (RequireArgUnpacking)
    return if !load_module_with_fallbacks(
        'needed_subs'  => [qw{Purity Terse Indent Dump new}],
        'modules'      => [qw{Data::Dumper}],
        'fail_warning' => 'SSP will take longer to run',
    );
    return if defined $MEMOIZE_CACHE{'PRECACHE'} and $MEMOIZE_CACHE{'PRECACHE'}->{'disabled'};

    print_start("Gathering some information, this may take a few moments...\n");

    if ( defined $MEMOIZE_CACHE{'PROFILING'} ) {
        $MEMOIZE_CACHE{'PRECACHE'}->{'PROFILE'}->{'start'} = [ Time::HiRes::gettimeofday() ];
    }
    for my $func (@_) {

        # Subs prefixed with _serialize_ can be used to serialize multiple memoized functions
        if ( exists $MEMOIZE_CACHE{$func} or ( index( $func, '_serialize_' ) == 0 and exists &{$func} ) ) {
            new_async_call( $func, \&$func );
        }
        else {
            print STDERR "SSP DEBUG - Tried to populate memoize cache that does not exist: $func\n";
        }
    }
    event_loop();
    for my $func (@_) {
        if ( defined &$func ) {
            async_call_result($func);
        }
    }
    if ( defined $MEMOIZE_CACHE{'PROFILING'} ) {
        $MEMOIZE_CACHE{'PRECACHE'}->{'PROFILE'}->{'elapsed'} = Time::HiRes::tv_interval( $MEMOIZE_CACHE{'PRECACHE'}->{'PROFILE'}->{'start'} );
    }
}

sub _merge_into_memoize_cache {
    my ( $new_ref, $job_name, $stdout_sref, $stderr_sref ) = @_;
    for my $key ( keys %{$new_ref} ) {
        next unless scalar keys %{ $new_ref->{$key} };
        $MEMOIZE_CACHE{$key} = $new_ref->{$key};
    }
    $MEMOIZE_CACHE{$job_name}->{'PRECACHED'} = 1;
    $MEMOIZE_CACHE{$job_name}->{'STDOUT'}    = $$stdout_sref if defined $$stdout_sref;
    $MEMOIZE_CACHE{$job_name}->{'STDERR'}    = $$stderr_sref if defined $$stderr_sref;
}

{    # Async jobs
    my $fds;
    my %jobs;
    my %results;

    sub new_async_call {
        my ( $job_name, $coderef, @args ) = @_;

        defined $fds or $fds = '';

        # Trying to use more than one pipe here can result in deadlocks
        # If we eventually do non-blocking reads and writes then more than one pipe could be used
        pipe my ( $dumper_r, $dumper_w );

        my $pid = fork;

        die "Unable to fork: $!" unless defined $pid;

        if ( $pid == 0 ) {
            close $dumper_r;
            my $child_stdout;
            my $child_stderr;

            close STDOUT;
            close STDERR;
            open STDIN,  '<', '/dev/null';
            open STDOUT, '>', \$child_stdout;
            open STDERR, '>', \$child_stderr;
            $coderef->(@args);

            my $dumper = Data::Dumper->new( [ \%MEMOIZE_CACHE, $child_stdout, $child_stderr ], [qw(*child_memoize_cache child_stdout child_stderr)] );
            print $dumper_w $dumper->Purity(1)->Terse(0)->Indent(0)->Dump;
            exit;                   ## no critic (NoExitsFromSubroutines)
        }
        close $dumper_w;

        $jobs{$job_name} = {
            pipes => {
                dumper => {
                    buf => '',
                    fh  => $dumper_r,
                },
            },
            pid => $pid,
        };
        vec( $fds, fileno($dumper_r), 1 ) = 1;
    }

    sub event_loop {
        while (%jobs) {
            last unless select my $ready = $fds, undef, undef, undef;

            foreach my $job_name ( keys %jobs ) {
                foreach my $pipe ( keys %{ $jobs{$job_name}{pipes} } ) {
                    my $buffer = \$jobs{$job_name}{pipes}{$pipe}{buf};
                    my $fh     = $jobs{$job_name}{pipes}{$pipe}{fh};
                    my $fileno = fileno($fh);
                    if ( vec( $ready, $fileno, 1 ) ) {
                        if ( read $fh, my $buf, 4096 ) {
                            ${$buffer} .= $buf;
                        }
                        else {
                            $results{$job_name}{$pipe} = $buffer;

                            vec( $fds, $fileno, 1 ) = 0;
                            close $fh;
                            delete $jobs{$job_name}{pipes}{$pipe};
                        }
                    }
                }
                delete $jobs{$job_name} unless scalar keys %{ $jobs{$job_name}{pipes} };
            }
        }
    }

    sub async_call_result {
        my ($job_name) = @_;
        my %child_memoize_cache;
        my $child_stdout;
        my $child_stderr;
        defined $results{$job_name}{dumper} && defined ${ $results{$job_name}{dumper} } && eval ${ $results{$job_name}{dumper} };    ## no critic (StringyEval)
        _merge_into_memoize_cache( \%child_memoize_cache, $job_name, \$child_stdout, \$child_stderr );
    }
}

sub get_kernel_pid_max {
    my $sysctl = (split(/\s+/, timed_run( 0, 'sysctl', 'kernel.pid_max' ) ) )[2];
    if ( $sysctl <= 32768) {
        print_warn('kernel.pid_max may be too low: ' . YELLOW $sysctl);
        print_normal(' - LiteSpeed might SIGKILL anything, including MySQL. See:UPS-90');
    }
}

sub check_for_unbound_stub_resolver {
    my $unboundstub = '/var/cpanel/dns_unbound_resolve_mode';
    return unless( -l $unboundstub );
    my $link = readlink $unboundstub;
    if ($link eq 'stub') {
        print_info("stub touchfile exists - See TECH-1002\n");
    }
}

sub check_for_public_resolvers {
    return if ( ! -e "/etc/resolv.conf" );
    my $query_refused = timed_run( 4, 'dig', '+short', 'TXT', '2.0.0.127.multi.uribl.com' );
    return unless( $query_refused =~ m/Query Refused/ );
    print_info( 'DNSBL Queries Rate Limited: ' );
    print_normal( '/etc/resolv.conf might contain public DNS resolvers. Seeing issues with RBLs not working?' );
    print_normal( "\t \\_ See: https://support.cpanel.net/hc/en-us/articles/360053079473" );
}

sub check_hostname_resolution {
    my $lcHost = shift;
    my $external_ip_address = get_external_ip();
    my $hostname_ip = timed_run( 4, 'dig', "$lcHost", '+short' );
    return 1 if ( $hostname_ip eq $external_ip_address );
    return 0;
}

1;