esg-gitstrap

#!/usr/bin/env perl
# -*-mode:cperl-*-
#
# esg-gitstrap
#
# This script clones all of the ESG git repositories and symlinks the
# relevant portions into standard locations.  It is intended to
# replace esg-bootstrap for developers.
#
# Since it will create files in /usr/local, it must be run with root
# privileges.
#
# Since this script will be run on a bare-bones system with nothing
# but (and maybe not even) all of the requisite system packages
# installed, this script should never use any Perl module outside of
# the Perl core.
#
# If you are reading this because you want to update this script, you
# probably want to start at the "Global Variables" section below.
#
#
### Perl Critic Notes ###
#
## no critic qw(ErrorHandling::RequireCarping)
#
# This is a standalone script, not a module.  Nobody should be calling
# our subroutines from outside this script, and again we need to avoid
# using external modules like Carp.
#
## no critic qw(InputOutput::ProhibitBacktickOperators)
#
# We want error dumps on STDERR when things fail.  Debugging data
# should be as simple to capture as possible.
#
## no critic qw(Subroutines::RequireArgUnpacking)
#
# Argument arrays are being passed and used verbatim, unmodified in
# debugging subroutines.  Perl Critic doesn't allow this to be
# overridden on a per-subroutine basis, so we have to just disable it
# globally.
#
## no critic qw(ValuesAndExpressions::ProhibitConstantPragma)
#
# We are using constants for exit values.  These will never be
# interpolated, and a bootstrap script needs to not use any modules
# outside of core, so ReadOnly is unavailable.
#
## no critic qw(ValuesAndExpressions::ProhibitLeadingZeros)
#
# We are using octal in a few places for filesystem permissions that
# Perl Critic doesn't automatically recognize.
#

use strict; use warnings;
use English '-no_match_vars';

use Cwd qw(chdir cwd);
use File::Path qw(make_path);
use Getopt::Long qw(:config bundling);


###################
### Exit Values ###
###################

use constant EXIT_SUCCESS      => 0;
use constant EXIT_INTERNALFAIL => 1;
use constant EXIT_EXTERNALFAIL => 2;


#######################
### Option Handling ###
#######################

my %opt = (
    'all'          => 0,
    'debug'        => 0,
    'prefix'       => '/usr/local',
    'bindir'       => 'bin',
    'configdir'    => 'esgf/config_defaults',
    'installerdir' => 'esgf/installer',
    'srcdir'       => 'src/esgf',
    'branch'       => 'devel',
    'force-branch' => 0,
    'os'           => detect_os(),
   );

GetOptions(
    \%opt,
    'all',
    'debug|v',
    'prefix=s',
    'bindir=s',
    'configdir=s',
    'installerdir=s',
    'srcdir=s',
    'branch=s',
    'force-branch',
    'os=s',
   );


########################
### Global Variables ###
########################

local $ENV{"PATH"} = "/usr/local/bin:/usr/bin:/bin";
my $prefix = $opt{'prefix'};
my $binpath = "$opt{'prefix'}/$opt{'bindir'}";
my $configpath = "$opt{'prefix'}/$opt{'configdir'}";
my $installerpath = "$opt{'prefix'}/$opt{'installerdir'}";
my $srcpath = "$opt{'prefix'}/$opt{'srcdir'}";

# System package prerequisites for a full build
my %packages = (
    'RedHat' => [ "autoconf", "automake",
                  "bison",
                  "curl",
                  "file",
                  "flex",
                  "gcc", "gcc-c++",
                  "gettext-devel",
                  "git",
                  "libtool",
                  "libuuid-devel",
                  "libxml2", "libxml2-devel",
                  "libxslt", "libxslt-devel",
                  "lsof",
                  "make",
                  "openssl-devel",
                  "pam-devel",
                  "pax",
                  "postgresql",
                  "readline-devel",
                  "tk-devel",
                  "wget",
                  "zlib-devel",
                 ],
    'SuSE' => [ ],
    'Debian' => [ ],
   );

my %repoinfo = (
    'esgf-installer' => {
        'url' => 'git://github.com/ESGF/esgf-installer.git',
        # Alternate installer path if you want to help with experimentation
        #'url' => 'git://github.com/AZed/esgf-installer.git',
        'branch' => {
            'stable'     => 'master',
            'devel'      => 'devel',
            'experiment' => 'experiment',
        },
        # esg-installarg MUST NOT be included in this list -- the
        # install script will break if it is a symlink, and it will be
        # downloaded by the first esg-node --set-type command
        'binlinks' => [ "esg-autoinstall",
                        "esg-functions",
                        "esg-gitstrap",
                        "esg-init",
                        "esg-node",
                        "esg-purge.sh",
                        "filters/esg-access-logging-filter",
                        "filters/esg-security-las-ip-filter",
                        "filters/esg-security-tokenless-filters",
                        "globus/esg-globus",
                        "product-server/esg-product-server",
                      ],
        'configlinks' => [],
        'installerlinks' => [],
    },
    'config' => {
        'url' => 'https://github.com/ESGF/config.git',
        'branch' => {
            'stable'     => 'master',
            'devel'      => 'devel',
            'experiment' => 'devel',
        },
        'binlinks'       => [],
        'configlinks'    => [],
        'installerlinks' => [],
    },
    'esgf-dashboard' => {
        'url' => 'git://github.com/ESGF/esgf-dashboard.git',
        'branch' => {
            'stable'     => 'master',
            'devel'      => 'devel',
            'experiment' => 'devel',
        },
        'binlinks' => ["bin/esg-dashboard"],
        'configlinks' => [],
        'installerlinks' => [],
    },
    'esgf-drslib' => {
        'url' => 'git://github.com/ESGF/esgf-drslib.git',
        'branch' => {
            'stable'     => 'master',
            'devel'      => 'master',
            'experiment' => 'master',
        },
        'binlinks' => [],
        'configlinks' => [],
        'installerlinks' => [],
    },
    'esgf-idp' => {
        'url' => 'git://github.com/ESGF/esgf-idp.git',
        'branch' => {
            'stable' => 'master',
            'devel' => 'devel',
            'experiment' => 'devel',
        },
        'binlinks' => ["bin/esg-idp"],
        'configlinks' => [],
        'installerlinks' => [],
    },
    'esgf-node-manager' => {
        'url' => 'git://github.com/ESGF/esgf-node-manager.git',
        'branch' => {
            'stable' => 'master',
            'devel'  => 'devel',
            'experiment' => 'devel',
        },
        'binlinks' => ["bin/esg-node-manager","bin/esgf-sh","bin/esgf-spotcheck"],
        'configlinks' => [],
        'installerlinks' => [],
    },
    'esg-orp' => {
        'url' => 'git://github.com/ESGF/esg-orp.git',
        'branch' => {
            'stable' => 'master',
            'devel' => 'devel',
            'experiment' => 'devel',
        },
        'binlinks' => ["bin/esg-orp"],
        'configlinks' => [],
        'installerlinks' => [],
    },
    'esg-search' => {
        'url' => 'git://github.com/ESGF/esg-search.git',
        'branch' => {
            'stable' => 'master',
            'devel' => 'devel',
            'experiment' => 'devel',
        },
        'binlinks' => ["bin/esg-search","bin/esgf-crawl"],
        'configlinks' => [],
        'installerlinks' => [],
    },
    'esgf-security' => {
        'url' => 'git://github.com/ESGF/esgf-security.git',
        'branch' => {
            'stable' => 'master',
            'devel' => 'devel',
            'experiment' => 'devel',
        },
        'binlinks' => ["bin/esg-security","bin/esgf-user-migrate"],
        'configlinks' => [],
        'installerlinks' => [],
    },
    'esgf-web-fe' => {
        'url' => 'git://github.com/ESGF/esgf-web-fe.git',
        'branch' => {
            'stable' => 'master',
            'devel' => 'devel',
            'experiment' => 'devel',
        },
        'binlinks' => ["bin/esg-web-fe"],
        'configlinks' => [],
        'installerlinks' => [],
    },
   );


####################
### Main Routine ###
####################

debug_print("Main routine begins");
umask 022;

install_packages();

my @repolist;
if($opt{'all'}) {
    @repolist = keys %repoinfo;
}
else {
    @repolist = ( 'esgf-installer', 'config' );
}

foreach my $repo (@repolist) {
    my $repobranch = $repoinfo{$repo}->{'branch'}->{$opt{'branch'}};

    debug_print("now handling ${repo}");
    git_clone($repo);
    if($opt{'force-branch'}) {
        unless($repobranch) {
            print {*STDERR} "ERROR: no branch defined for repository ${repo} and branch type $opt{'branch'}!\n";
            exit EXIT_INTERNALFAIL;
        }
        git_checkout($repo,$repobranch);
    }
    git_pull($repo);
    setup_links($repo);
}

exit EXIT_SUCCESS;


########################
### Main Subroutines ###
########################

#
# sub install_packages
#
# Installs all of the prerequisite system packages if they are not
# already installed.
#
# Arguments: none
# Returns: 1 (true) on success, dies on failure
#
sub install_packages {
    if($opt{'os'} eq 'RedHat') {
        foreach my $package (@{$packages{'RedHat'}}) {
            my $version = rpm_installed_version($package);
            if(!$version) {
                print "Package '${package}' not found, installing...";
                yum_install($package);
                print " done.\n";
            }
        }
    }
    else
    {
        print {*STDERR} "WARNING: package prerequisites are not handled for OS '$opt{'os'}'!\n";
        print {*STDERR} "Skipping package validation -- you must verify this yourself.\n";
    }
    return 1;
}


#
# sub git_checkout($repo,$revision)
#
# Checks out a git repository to a specified revision (or branch or tag)
#
# Arguments:
#   $repo: the repository name (this must be a key of the %repoinfo global hash)
#   $revision: the branch name or specific revision to check out
#
# Returns:
#   1 (true) on success, dies on failure
#
sub git_checkout {
    my ($repo,$revision) = @_;
    my $repopath = "${srcpath}/${repo}";
    my @systemcmd;

    debug_print("checking out ${repo} at ${revision}");

    if(!$repo) {
        print {*STDERR} "git_checkout called with no repository specified!\n";
        exit EXIT_INTERNALFAIL;
    }

    if(!$revision) {
        print {*STDERR} "git_checkout called with no revision specified!\n";
        exit EXIT_INTERNALFAIL;
    }

    if(! -d "${repopath}") {
        print {*STDERR} "Cannot check out ${repo} at ${revision}!\n";
        print {*STDERR} "The expected path '${repopath}' does not exist!\n";
        exit EXIT_INTERNALFAIL;
    }

    if(! -d "${repopath}/.git") {
        print {*STDERR} "Cannot check out ${repo} at ${revision}!\n";
        print {*STDERR} "${repopath} is not a git repository!\n";
        exit EXIT_INTERNALFAIL;
    }
    chdir($repopath)
      or die("ERROR: unable to change to directory ${repopath}!");
    @systemcmd = ('git','checkout',$revision);
    system(@systemcmd);
    system_result($CHILD_ERROR,$OS_ERROR,@systemcmd);
    return 1;
}


#
# sub git_clone($repo)
#
# Clones a git repository into the source path, if a repository does not already exist there.
#
# Arguments:
#   $repo: the repository name (this must be a key of the %repoinfo global hash)
#
# Returns:
#   1 (true) on success, dies on failure
#
sub git_clone {
    my $repo = shift;
    my $repopath = "${srcpath}/${repo}";
    my @systemcmd;

    debug_print("cloning ${repo}");
    umask 022;

    if (-e ${repopath}) {
        if (-d ${repopath}) {
            if (-d "${repopath}/.git") {
                # Target directory is already a git repository.  We
                # assume it is set up in a desired fashion and return
                # silently.
                debug_print("${repo} already present, skipping");
                return;
            }
            # Target directory exists, but is not a git repository.
            print {*STDERR} "The destination directory for ${repo}",
              " (${repopath}) exists, but is not a repository!\n";
            print {*STDERR} "Please check your filesystem.\n";
            exit EXIT_INTERNALFAIL;
        }
        # The target name exists, but isn't even a directory
        print {*STDERR} "The destination name for ${repo}",
          " (${repopath}) exists, but is not a directory!\n";
        print {*STDERR} "Please check your filesystem.\n";
        exit EXIT_INTERNALFAIL;
    }
    make_path($srcpath, { mode => 2755, });
    chdir($srcpath)
      or die("ERROR: unable to change to directory ${srcpath}!");
    @systemcmd = ('git','clone',$repoinfo{$repo}->{'url'});
    system(@systemcmd);
    system_result($CHILD_ERROR,$OS_ERROR,@systemcmd);

    # On a fresh clone, we should also check out to the expected branch
    unless($repoinfo{$repo}->{'branch'}->{$opt{'branch'}}) {
        print {*STDERR} "FATAL (git_clone): No branch is defined for repository ${repo}, branch type $opt{'branch'}!\n";
        exit EXIT_INTERNALFAIL;
    }
    git_checkout($repo,$repoinfo{$repo}->{'branch'}->{$opt{'branch'}});
    return 1;
}


#
# sub git_pull($repo)
#
# Pulls a repository up to the most recent revision for the branch it
# is in
#
# If you want to change branches, or switch to a specific revision,
# use git_checkout($repo) instead.
#
# Arguments:
#   $repo: the repository name to handle
#          (must be a key of the %repoinfo global hash)
#
# Returns:
#   1 (true) on success, dies on failure
#
sub git_pull {
    my ($repo) = @_;
    my $repopath = "${srcpath}/${repo}";
    my @systemcmd;

    debug_print("pulling latest revision in repository ${repo}");
    umask 022;

    if(! -d "${repopath}") {
        print {*STDERR} "Cannot pull in ${repo}!\n";
        print {*STDERR} "The expected path '${repopath}' does not exist!\n";
        exit EXIT_INTERNALFAIL;
    }
    if(! -d "${repopath}/.git") {
        print {*STDERR} "Cannot pull in ${repo}!\n";
        print {*STDERR} "${repopath} is not a git repository!\n";
        exit EXIT_INTERNALFAIL;
    }
    chdir($repopath)
      or die("ERROR: unable to change to directory ${srcpath}!");
    @systemcmd = ('git','pull','-q');
    system(@systemcmd);
    system_result($CHILD_ERROR,$OS_ERROR,@systemcmd);
    return 1;
}


#
# sub setup_links($repo)
#
# Sets up the symlinks for a repository
#
# Arguments:
#   $repo: the repository name to handle
#          (must be a key of the %repoinfo global hash)
# Returns:
#   1 (true) on success, dies on failure
#
sub setup_links {
    my $repo = shift;

    debug_print("setting up links for ${repo}");
    umask 022;

    # Links that go into the bin path
    create_repo_symlinks($repo,'binlinks',$binpath);

    # Links that go into the default configuration file directory
    create_repo_symlinks($repo,'configlinks',$configpath);

    # Links that go into the installer source directory
    create_repo_symlinks($repo,'installerlinks',$installerpath);

    return 1;
}


##########################
### Helper Subroutines ###
##########################

#
# sub create_repo_symlinks($repo,$linktype,$linkpath)
#
# Helper function for setup_links() that actually creates the symlinks.
#
# Arguments:
#   $repo: the repository to set up
#          This must be a top-level entry in the %repoinfo hash.
#   $linktype: the type of link being handled
#              This must be a second-level array entry in the %repoinfo hash.
#   $linkpath: the path into which the links will be installed
#              This path will be created with mode 0755 if it does not exist.
#
# Returns:
#   1 (true) on success, dies on failure
#
sub create_repo_symlinks {
    my ($repo,$linktype,$linkpath) = @_;
    my $repopath = "${srcpath}/${repo}";

    # Argument sanity checks
    unless(ref($repoinfo{$repo}) eq 'HASH') {
        print {*STDERR} "Unable to find repository info for ${repo}!\n";
        exit EXIT_INTERNALFAIL;
    }
    unless(ref($repoinfo{$repo}->{$linktype}) eq 'ARRAY') {
        print {*STDERR} "'${linktype}' is not a valid link type for repository ${repo}!\n";
        exit EXIT_INTERNALFAIL;
    }
    unless($linkpath) {
        print {*STDERR} "create_repo_symlinks called without a link path for ${repo}->${linktype}!\n";
        exit EXIT_INTERNALFAIL;
    }

    umask 022;

    make_path($linkpath, { mode => 0755, });
    chdir($linkpath)
      or die("ERROR: unable to change to directory ${linkpath}!");

    foreach my $srcfile (@{$repoinfo{$repo}->{$linktype}}) {
        my $destfile;
        (undef,$destfile) = $srcfile =~ m|(.*/)?([-.\w]+)|x;
        debug_print("handling ${linktype} symlink for ${srcfile} into ${linkpath}/${destfile}");
        # We don't care if the unlink fails (file might not exist, and
        # a failed unlink will cause the next symlink to fail, and
        # that is tested)
        unlink("${linkpath}/${destfile}");
        symlink("${repopath}/${srcfile}","${linkpath}/${destfile}")
          or die("Unable to create symlink from ${repopath}/${srcfile} into ${linkpath}/${destfile}!");

        # Anything placed in the binlinks area should be executable
        if($linktype eq 'binlinks') {
            chmod(0755,"${repopath}/${srcfile}")
              or die("Unable to set executable bit on binlink ${repopath}/${srcfile}!");
        }
    }
    return 1;
}


#
# sub debug_print(@text)
#
# Prints a string to STDERR if $opt{'debug'} is true
# The text will be prefixed with "DEBUG: " and ended with a newline.
#
# Arguments:
#   @text is passed to the print statement unmodified
#
# Returns 1 (true) on a successful print and undefined if
# $opt{'debug'} is not set.
# I'm not sure how it could fail.
#
sub debug_print {
    return unless $opt{'debug'};
    print {*STDERR} "DEBUG: ",@_,"\n";
    return 1;
}


#
# sub detect_os()
#
# Attempts to detect the existing OS version
#
# This is called by the option parser, and thus sets the default, but
# the result can be overridden on the command line.
#
# This is a very simplistic detection, and easy to spoof.  RedHat and
# CentOS are both treated as RedHat.
#
# Arguments: none
#
# Returns:
#   One of the following strings, depending on what was detected:
#     Debian
#     RedHat
#     SuSE
#     Unknown
#
sub detect_os {
    if(-e "/etc/redhat-release" or -e "/etc/centos-release") {
        return "RedHat";
    }
    elsif(-e "/etc/debian_version") {
        return "Debian";
    }
    elsif(-e "/etc/SuSE-release") {
        return "SuSE";
    }
    return "Unknown";
}


#
# sub rpm_installed_version($package)
#
# Finds the version of an installed RPM package
#
# Arguments:
#   $package: package name to check
#
# Returns:
#   A string containing the RPM version if found, or undef otherwise
#
# Dies if the 'rpm' command fails
#
sub rpm_installed_version {
    my ($package) = @_;
    my $systemcmd;
    my $version;

    $systemcmd = "rpm -q --queryformat %{VERSION} ${package}";
    $version = `${systemcmd}`;
    # This command will return with error 1 if the package is not
    # found, which is an expected case, so we're only going to handle
    # a failure to execute, and not use system_result()
    if($CHILD_ERROR == -1) {
        print {*STDERR} "ERROR: failed to execute: '${systemcmd}'\n";
        print {*STDERR} "  Error message: ${OS_ERROR}\n";
        exit EXIT_EXTERNALFAIL;
    }
    if($version =~ m/not installed/) {
        $version = undef;
    }
    return $version;
}


#
# sub system_result($CHILD_ERROR,$OS_ERROR,@systemcmd)
#
# Result handler for system calls
#
# Arguments:
#   $CHILD_ERROR: return value from previous system command (aka $?)
#   $OS_ERROR: error value from previous system command (aka $!)
#   @systemcmd: array passed to the system call
#
# Returns:
#   1 (true) if the return value was 0 (success)
#   exits EXIT_EXTERNALFAIL after an error message on any error
#
sub system_result {
    my $retval = shift;
    my $error = shift;
    my $cmdtext = join(' ',@_);

    return 1 if ($retval == 0);

    if ($retval == -1) {
        print {*STDERR} "ERROR: failed to execute: '${cmdtext}'\n";
        print {*STDERR} "  Error message: ${error}\n";
        exit EXIT_EXTERNALFAIL;
    }
    elsif ($retval & 127) {
        printf {*STDERR} "ERROR: '${cmdtext}' died with signal %d, %s coredump\n",
          ($retval & 127), ($retval & 128) ? 'with' : 'without';
        exit EXIT_EXTERNALFAIL;
    }
    else {
        printf "ERROR: '${cmdtext}' exited with value %d\n", $retval >> 8;
        print  "  CWD = ",cwd(),"\n";
        exit EXIT_EXTERNALFAIL;
    }
}


#
# sub yum_install($package)
#
# Installs a package via 'yum'
# This is only expected to work on RedHat/CentOS
#
# Arguments:
#   $package: the package name to install
#
# Returns:
#   1 (true) on success, dies on failure.
#
sub yum_install {
    my ($package) = @_;

    my @systemcmd = ('yum','install','-q','-y',$package);
    system(@systemcmd);
    system_result($CHILD_ERROR,$OS_ERROR,@systemcmd);
    return 1;
}