add: files from nitrc.org

ncanda-data-integration/LICENSE

@@ -0,0 +1,12 @@
+Copyright (c) 2015, SRI International
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ncanda-data-integration/scripts/crond/back-nightly

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+##
+##  Copyright 2015 SRI International
+##  License: https://ncanda.sri.com/software-license.txt
+##
+##  $Revision: 2114 $
+##  $LastChangedBy: dj0330 $
+##  $LastChangedDate: 2015-08-07 09:42:11 -0700 (Fri, 07 Aug 2015) $
+##
+
+# When generating a release do not update pipeline
+# exit 0
+[ -r $HOME/.bashrc ] && . $HOME/.bashrc
+
+# Import some useful functions
+. $(dirname $0)/crontools.sh
+
+# Import data from XNAT into REDCap and feed image analysis pipeline
+catch_output_email [email protected],[email protected] "NCANDA Pipeline Feeder Messages (import_mr_sessions)" ${HOME}/scripts/redcap/import_mr_sessions --max-days-after-visit 120 --pipeline-root-dir /fs/ncanda-share/pipeline/cases --run-pipeline-script /fs/ncanda-share/pipeline/scripts/bin/ncanda_all_pipelines
+
+# Export NP/clinical/dempgraphics data into image analysis pipeline directories
+#   This needs to come AFTER "import_mr_sessions", because otherwise we cannot get ages-at-MRI from REDCap for the export.
+catch_output_email [email protected],[email protected] "NCANDA Pipeline REDCap Exporter Messages (export_measures)" ${HOME}/scripts/redcap/export_measures --datadict-dir /fs/ncanda-share/pipeline/datadict --locked_form_report /fs/ncanda-share/pipeline/cases 
+
+# Update CSV summary files for the working pipeline
+catch_output_email [email protected],[email protected] "NCANDA Pipeline Summaries (update_csv_summaries)" /fs/ncanda-share/pipeline/scripts/utils/update_csv_summaries /fs/ncanda-share/pipeline/cases/ /fs/ncanda-share/pipeline/summaries/
+
+# Remove pyxnat cache directory to conserve space
+catch_output_email [email protected],[email protected] "NCANDA Drop pyXNAT Cache Directory" rm -rf /tmp/RestAPI@localhost_8080.xnat /tmp/cache

ncanda-data-integration/scripts/crond/crontools.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+##
+##  Copyright 2015 SRI International
+##  License: https://ncanda.sri.com/software-license.txt
+##
+##  $Revision: 2114 $
+##  $LastChangedBy: dj0330 $
+##  $LastChangedDate: 2015-08-07 09:42:11 -0700 (Fri, 07 Aug 2015) $
+##
+
+# Run a command, and send its output (stdout and stderr) to a given email address, but only if there is output
+catch_output_email()
+{
+    local mailto="$1"
+    local subject="$2"
+
+    shift 2
+    local cmd="$*"
+
+    local tmpfile=$(mktemp)
+
+    eval ${cmd} &> ${tmpfile}
+    if [ -s ${tmpfile} ]; then
+        eval "mailx -r [email protected] -s \"${subject}\" ${mailto} < ${tmpfile}"
+        eval "python ${HOME}/scripts/crond/post_github_issues.py --org ncanda --repo ncanda-datacore --title \"${subject}\" --body ${tmpfile}"
+    fi
+
+    rm -f ${tmpfile}
+}
+

ncanda-data-integration/scripts/crond/front-hourly

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+##
+##  Copyright 2015 SRI International
+##  License: https://ncanda.sri.com/software-license.txt
+##
+##  $Revision: 2114 $
+##  $LastChangedBy: dj0330 $
+##  $LastChangedDate: 2015-08-07 09:42:11 -0700 (Fri, 07 Aug 2015) $
+##
+
+[ -r $HOME/.bashrc ] && . $HOME/.bashrc
+
+# Import some useful functions
+. $(dirname $0)/crontools.sh
+
+# Run QA on fBIRN and ADNI phantom scans
+catch_output_email [email protected],[email protected] "NCANDA XNAT: Phantom QA Messages (phantom_qa)" ${HOME}/scripts/xnat/phantom_qa
+
+# Import data from the sites' data capture laptops into REDCap and reconcile imported with longitudinal data
+catch_output_email [email protected],[email protected] "NCANDA: Laptop Data Import Stage 1 (harvester)" ${HOME}/scripts/import/laptops/harvester ${HOME}/laptops/ncanda ${HOME}/laptops/imported
+
+# At midnight PST, run full update; otherwise, add only previously-missing data
+update_args=""
+hour=$(date +%H)
+if [ ${hour} -ne 0 ]; then
+    update_args+=" --missing-only"
+fi
+
+catch_output_email [email protected],[email protected] "NCANDA: Laptop Data Import Stage 2 (update_visit_date)" ${HOME}/scripts/import/laptops/update_visit_data --max-days-after-visit 120 ${update_args}
+
+# REDCap updates
+update_args=""
+hour=$(date +%H)
+if [ ${hour} -eq 0 ]; then
+    update_args+="--update-all"
+fi
+catch_output_email [email protected],[email protected] "NCANDA REDCap: Update Scores (update_summary_scores)" ${HOME}/scripts/redcap/update_summary_scores ${update_args}
+
+catch_output_email [email protected],[email protected] "NCANDA REDCap: Update Form Status (update_bulk_forms)" ${HOME}/scripts/redcap/update_bulk_forms

ncanda-data-integration/scripts/crond/front-nightly

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+##
+##  Copyright 2015 SRI International
+##  License: https://ncanda.sri.com/software-license.txt
+##
+##  $Revision: 2114 $
+##  $LastChangedBy: dj0330 $
+##  $LastChangedDate: 2015-08-07 09:42:11 -0700 (Fri, 07 Aug 2015) $
+##
+
+[ -r $HOME/.bashrc ] && . $HOME/.bashrc
+
+# Import some useful functions
+. $(dirname $0)/crontools.sh
+
+######################################
+# XNAT / Imaging Related
+######################################
+
+# Check MR session names etc. in XNAT
+catch_output_email [email protected],[email protected] "NCANDA XNAT: Check Object Names Messgae (check_object_names)" ${HOME}/scripts/xnat/check_object_names --send-mail --zip-root /fs/storage/share/burn2dvd
+
+# Check for new or updated sessions and
+catch_output_email [email protected],[email protected] "NCANDA XNAT: Check New Sessions Message (check_new_sessions)" ${HOME}/scripts/xnat/check_new_sessions --send-mail-to [email protected],[email protected],[email protected],[email protected]
+
+# Check whether any MR sessions are missing corresponding phantom scans
+catch_output_email [email protected],[email protected] "NCANDA XNAT: Phantom Scan Messages (check_phantom_scans)" ${HOME}/scripts/xnat/check_phantom_scans --check-all
+
+# Run fMRI QA on subjects ## Currently disabled because it isn't looked at but takes a long time to run
+##catch_output_email [email protected] "NCANDA XNAT: Subject fMRI QA Messages" ${HOME}/scripts/xnat/fmri_qa_subjects
+
+######################################
+# REDCap / NP / Clinical Data Related
+######################################
+
+# Import data from UPenn into REDCap
+catch_output_email [email protected],[email protected] "NCANDA REDCap: WebCNP Import Messages (cnp2redcap)" ${HOME}/scripts/import/webcnp/cnp2redcap --last-3-months
+
+# Check whether subject birth dates and gender match checksum digit, plus whether all subjects on study arms appear in main arm also
+catch_output_email [email protected],[email protected] "NCANDA REDCap: Subject ID Checks (check_subject_ids)" ${HOME}/scripts/redcap/check_subject_ids
+
+# Check (and update, if necessary) drinking exception status
+catch_output_email [email protected],[email protected] "NCANDA REDCap: Undeclared Drinking Exceptions (check_exceptions)" ${HOME}/scripts/redcap/check_exceptions --update-fn

ncanda-data-integration/scripts/crond/post_github_issues.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python
+
+##
+##  Copyright 2015 SRI International
+##  License: https://ncanda.sri.com/software-license.txt
+##
+##  $Revision: 2114 $
+##  $LastChangedBy: dj0330 $
+##  $LastChangedDate: 2015-08-07 09:42:11 -0700 (Fri, 07 Aug 2015) $
+##
+"""
+Post GitHub Issues
+------------------
+
+Take the stdout and stderr passed to the catch_output_email and create an issue
+on GitHub that uses a tag corresponding to the script any error was detected with.
+
+Example Usage:
+
+python post_github_issues.py -o ncanda -r ncanda-datacore \
+                             -t "NCANDA: Laptop Data Import Stage 2 (update_visit_date)" \
+                             -b /tmp/test.txt -v
+"""
+__author__ = 'Nolan Nichols <https://orcid.org/0000-0003-1099-3328>'
+
+import os
+import sys
+import hashlib
+import ConfigParser
+
+import github
+from github.GithubException import UnknownObjectException
+
+
+def create_connection(cfg, verbose=None):
+    """
+    Get a connection to github api
+
+    :param cfg: path to configuration file
+    :return: github.MainClass.Github
+    """
+    if verbose:
+        print "Parsing config: {0}".format(cfg)
+    # Get the redcap mysql configuration
+    config = ConfigParser.RawConfigParser()
+    config_path = os.path.expanduser(cfg)
+    config.read(config_path)
+
+    user = config.get('github', 'user')
+    passwd = config.get('github', 'password')
+
+    g = github.Github(user, passwd)
+    if verbose:
+        print "Connected to GitHub..."
+    return g
+
+
+def get_label(repo, title, verbose=None):
+    """
+    Get a label object to tag the issue
+
+    :param repo: github.Repository
+    :param title: str
+    :return: github.Label
+    """
+    if verbose:
+        print "Checking for label..."
+    label = None
+    label_text = None
+    try:
+        label_start = 1 + title.index('(')
+        label_end = title.index(')')
+        label_text = title[label_start:label_end]
+    except ValueError, e:
+        print "Warning: This tile has no embeded label. {0}".format(e)
+    if label_text:
+        try:
+            label = [repo.get_label(label_text)]
+            if verbose:
+                print "Found label: {0}".format(label)
+        except UnknownObjectException, e:
+            print "Error: The label '{0}' does not exist on Github. {1}".format(label_text, e)
+    return label
+
+
+def is_open_issue(repo, subject, verbose=None):
+    """
+    Verify if issue already exists
+    :param repo: github.Repository
+    :param subject: str
+    :return: bool
+    """
+    if verbose:
+        print "Checking for open issue: {0}".format(subject)
+    for issue in repo.get_issues():
+        if issue.title == subject and issue.state == 'open':
+            if verbose:
+                print "Issue already exists... See: {0}".format(issue.url)
+            return True
+    if verbose:
+        print "Issue does not already exist... Creating.".format(subject)
+    return False
+
+
+def create_issues(repo, title, body, verbose=None):
+    """
+    Create a GitHub issue for the provided repository with a label
+
+    :param repo: github.Repository
+    :param title: str
+    :param body: str
+    :return: None
+    """
+    label = get_label(repo, title)
+    if not label:
+        raise NotImplementedError("A label embedded in parentheses is currently required. "
+                                  "For example 'Title of Error (title_tag).' You provided:"
+                                  "{0}".format(title))
+    # get stdout written to file
+    with open(body) as fi:
+        subject_base = title[0:title.index(' (')]
+        issues = fi.readlines()
+        fi.close()
+
+    # Handle multiline error messages.
+    if 'Traceback' in ''.join(issues):
+        if verbose:
+            print "Issue is a Traceback..."
+        issues = [''.join(issues)]
+    for issue in issues:
+        # Create a unique title.
+        sha1 = hashlib.sha1(issue).hexdigest()[0:6]
+        subject = subject_base + ": {0}".format(sha1)
+        if is_open_issue(repo, subject, verbose=verbose):
+            continue
+        else:
+            github_issue = repo.create_issue(subject, body=issue, labels=label)
+            if verbose:
+                print "Created issue... See: {0}".format(github_issue.url)
+    return None
+
+
+def main(args=None):
+    if args.verbose:
+        print "Initializing..."
+    g = create_connection(args.config, verbose=args.verbose)
+    organization = g.get_organization(args.org)
+    repo = organization.get_repo(args.repo)
+    create_issues(repo, args.title, args.body, verbose=args.verbose)
+    if args.verbose:
+        print "Finished!"
+
+if __name__ == "__main__":
+    import argparse
+
+    formatter = argparse.RawDescriptionHelpFormatter
+    default = 'default: %(default)s'
+    parser = argparse.ArgumentParser(prog="post_github_issues.py",
+                                     description=__doc__,
+                                     formatter_class=formatter)
+    parser.add_argument("-c", "--config", dest="config",
+                        default=os.path.expanduser('~/.server_config/github.cfg'),
+                        help="GitHub authentication info.".format(default))
+    parser.add_argument("-o", "--org", dest="org", required=True,
+                        help="GitHub organization.")
+    parser.add_argument("-r", "--repo", dest="repo", required=True,
+                        help="GitHub repo.")
+    parser.add_argument("-t", "--title", dest="title", required=True,
+                        help="GitHub issue title with label in parentheses.")
+    parser.add_argument("-b", "--body", dest="body", required=True,
+                        help="GitHub issue body.")
+    parser.add_argument("-v", "--verbose", dest="verbose", action='store_true',
+                        help="Turn on verbose.")
+    argv = parser.parse_args()
+    sys.exit(main(args=argv))