Merge branch 'hotfix-1.31.8'

gemma-cli/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>gemma</artifactId>
         <groupId>gemma</groupId>
-        <version>1.31.7</version>
+        <version>1.31.8</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>gemma-cli</artifactId>

gemma-cli/src/main/java/ubic/gemma/core/apps/AffyDataFromCelCli.java

@@ -165,10 +165,20 @@ protected void doWork() throws Exception {
                     addSuccessObject( thawedEe );
                 } else if ( asd.isMerged( Collections.singleton( ad.getId() ) ).get( ad.getId() ) ) {
                     ad = asd.thawLite( ad );
-                    if ( GeoPlatform.isAffyPlatform( ad.getMergees().iterator().next().getShortName() ) ) {
+                    ArrayDesign mergee = ad.getMergees().iterator().next();
+
+                    // handle one level of sub-mergees....
+                    if ( asd.isMerged( Collections.singleton( mergee.getId() ) ).get( mergee.getId() ) ) {
+                        mergee = asd.thawLite( mergee );
+                        mergee = asd.thawLite( mergee.getMergees().iterator().next() );
+                    }
+
+                    if ( GeoPlatform.isAffyPlatform( mergee.getShortName() ) ) {
                         AbstractCLI.log.info( ad + " looks like Affy array made from merger of other platforms" );
                         serv.reprocessAffyDataFromCel( thawedEe );
                         addSuccessObject( thawedEe );
+                    } else {
+                        addErrorObject( ee, ad + " is not recognized as an Affymetrix platform. If this is a mistake, the Gemma configuration needs to be updated." );
                     }
                 } else {
 

gemma-cli/src/main/java/ubic/gemma/core/apps/ArrayDesignSequenceManipulatingCli.java

@@ -265,7 +265,7 @@ private boolean needToAutoRun( ArrayDesign arrayDesign, Class<? extends ArrayDes
         AuditEvent lastEventOfCurrentType = eventsOfCurrentType.get( eventsOfCurrentType.size() - 1 );
         assert lastEventOfCurrentType != null;
 
-        if ( lastEventOfCurrentType.getEventType().getClass().isAssignableFrom( eventClass ) ) {
+        if ( lastEventOfCurrentType.getEventType() != null && eventClass.isAssignableFrom( lastEventOfCurrentType.getEventType().getClass() ) ) {
             // then definitely don't run it. The last event was the same as the one we're trying to renew.
             AbstractCLI.log.debug( "Last event on " + arrayDesign + " was also a " + eventClass + ", skipping." );
             return false;

gemma-cli/src/main/java/ubic/gemma/core/apps/BlacklistCli.java

@@ -28,14 +28,14 @@
 import ubic.gemma.core.loader.expression.geo.service.GeoBrowser;
 import ubic.gemma.core.util.AbstractAuthenticatedCLI;
 import ubic.gemma.core.util.AbstractCLI;
-import ubic.gemma.model.common.description.DatabaseEntry;
-import ubic.gemma.model.common.description.ExternalDatabase;
 import ubic.gemma.model.blacklist.BlacklistedEntity;
-import ubic.gemma.model.blacklist.BlacklistedPlatform;
 import ubic.gemma.model.blacklist.BlacklistedExperiment;
+import ubic.gemma.model.blacklist.BlacklistedPlatform;
+import ubic.gemma.model.common.description.DatabaseEntry;
+import ubic.gemma.model.common.description.ExternalDatabase;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
-import ubic.gemma.persistence.service.common.description.ExternalDatabaseService;
 import ubic.gemma.persistence.service.blacklist.BlacklistedEntityService;
+import ubic.gemma.persistence.service.common.description.ExternalDatabaseService;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
 
 import java.io.BufferedReader;
@@ -54,12 +54,20 @@
 public class BlacklistCli extends AbstractAuthenticatedCLI {
 
     private static final int MAX_RETRIES = 3;
-    String fileName = null;
+
+    // accessions of file containing accessions to blacklist
+    private String accession = null;
+    private String fileName = null;
+
+    // reason (if adding to blacklist)
+    private String reason = null;
+
+    // remove from blacklist
     private boolean remove = false;
+
+    // proactive mode
     private boolean proactive = false;
     private Collection<String> platformsToScreen;
-    private String reason = null;
-    private String accession = null;
 
     @Override
     public CommandGroup getCommandGroup() {
@@ -124,7 +132,7 @@ protected void doWork() throws Exception {
                 return;
             }
 
-            if (reason.isEmpty()) {
+            if ( reason.isEmpty() ) {
                 throw new IllegalArgumentException( "A reason for blacklisting must be provided for " + accession );
             }
 
@@ -356,46 +364,38 @@ private int fetchAndBlacklist( ExternalDatabase geo, GeoBrowser gbs, Blacklisted
 
     @Override
     protected void processOptions( CommandLine commandLine ) {
-
         if ( commandLine.hasOption( "accession" ) ) {
-            if ( !commandLine.hasOption( "reason" ) && !commandLine.hasOption( "undo" ) ) {
-                throw new IllegalArgumentException( "Must provide a reason for blacklisting (unless using -sundo)" );
+            if ( commandLine.hasOption( "pp" ) || commandLine.hasOption( "file" ) ) {
+                throw new IllegalArgumentException( "The -accession option cannot be combined with -pp or -file" );
             }
-
-            if ( commandLine.hasOption( "file" ) ) {
-                throw new IllegalArgumentException( "The accession option cannot be combined with the file option" );
-
+            if ( !commandLine.hasOption( "undo" ) && StringUtils.isBlank( commandLine.getOptionValue( "reason" ) ) ) {
+                throw new IllegalArgumentException( "Must provide a reason for blacklisting (unless using -undo)" );
             }
-
             this.accession = commandLine.getOptionValue( "accession" );
             this.reason = commandLine.getOptionValue( "reason" );
-            return;
-        }
-
-
-        if ( commandLine.hasOption( "pp" ) ) {
-            if ( this.remove || this.fileName != null ) {
+            this.remove = commandLine.hasOption( "undo" );
+        } else if ( commandLine.hasOption( "pp" ) ) {
+            if ( commandLine.hasOption( "accession" ) || commandLine.hasOption( "file" ) ) {
                 throw new IllegalArgumentException( "The pp option cannot be combined with others" );
             }
+            if ( commandLine.hasOption( "undo" ) ) {
+                throw new IllegalArgumentException( "The -pp option cannot be compiled with -undo." );
+            }
             this.proactive = true;
-
             if ( commandLine.hasOption( "a" ) ) {
                 this.platformsToScreen = Arrays.asList( StringUtils.split( commandLine.getOptionValue( "a" ) ) );
             }
-
-            return;
-        }
-
-        if ( commandLine.hasOption( "file" ) ) {
+        } else if ( commandLine.hasOption( "file" ) ) {
+            if ( commandLine.hasOption( "accession" ) || commandLine.hasOption( "pp" ) ) {
+                throw new IllegalArgumentException( "The -file option cannot be combined with -pp or -accession." );
+            }
+            if ( commandLine.hasOption( "reason" ) ) {
+                throw new IllegalArgumentException( "The -file option cannot be combined with -reason, use the second column to specify blacklisting reason." );
+            }
             this.fileName = commandLine.getOptionValue( "file" );
+            this.remove = commandLine.hasOption( "undo" );
         } else {
-            throw new IllegalArgumentException( "Must provide an input file" );
+            throw new IllegalArgumentException( "Must provide one of -accession, -pp or -file option." );
         }
-
-        if ( commandLine.hasOption( "undo" ) ) {
-            this.remove = true;
-        }
-
     }
-
 }

gemma-cli/src/main/java/ubic/gemma/core/apps/ExpressionExperimentPlatformSwitchCli.java

@@ -77,7 +77,7 @@ public String getShortDesc() {
     protected void buildOptions( Options options ) {
         super.buildOptions( options );
         Option arrayDesignOption = Option.builder( "a" ).hasArg().argName( "Array design" ).desc(
-                        "Array design name (or short name) - no need to specifiy if the platforms used by the EE are merged" )
+                        "Array design short name to be switched to - no need to specify if the platforms used by the EE are merged" )
                 .longOpt( "array" ).build();
         options.addOption( arrayDesignOption );
         this.addForceOption( options );
@@ -115,7 +115,6 @@ private void processExperiment( ExpressionExperiment ee ) {
 
     /**
      * @param  name               of the array design to find.
-     * @param  arrayDesignService the arrayDesignService to use for the AD retrieval
      * @return an array design, if found. Bails otherwise with an error exit code
      */
     private ArrayDesign locateArrayDesign( String name ) {

gemma-cli/src/main/java/ubic/gemma/core/apps/ExternalDatabaseOverviewCli.java

@@ -69,7 +69,10 @@ private String summarize( AuditTrail auditTrail ) {
     }
 
     private String summarize( AuditEvent auditEvent ) {
-        String s = String.format( "%s: %s by %s", auditEvent.getDate(), auditEvent.getAction(), auditEvent.getPerformer().getUserName() );
+        String s = String.format( "%s: %s", auditEvent.getDate(), auditEvent.getAction() );
+        if ( auditEvent.getPerformer() != null ) {
+            s += " by " + auditEvent.getPerformer().getUserName();
+        }
         if ( auditEvent.getNote() != null ) {
             s += "\n\t" + auditEvent.getNote();
         }

gemma-cli/src/main/java/ubic/gemma/core/apps/GemmaCLI.java

@@ -19,7 +19,6 @@
 package ubic.gemma.core.apps;
 
 import lombok.Value;
-import lombok.extern.apachecommons.CommonsLog;
 import org.apache.commons.cli.*;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
@@ -50,10 +49,16 @@
  *
  * @author paul
  */
-@SuppressWarnings({ "unused", "WeakerAccess" }) // Possible external use
-@CommonsLog
 public class GemmaCLI {
 
+    static {
+        // set this as early as possible, otherwise logging might be broken
+        if ( System.getProperty( "gemma.log.dir" ) == null ) {
+            System.err.println( "The 'gemma.log.dir' system property is not set, will default to the current folder." );
+            System.setProperty( "gemma.log.dir", "." );
+        }
+    }
+
     private static final String
             HELP_OPTION = "h",
             HELP_ALL_OPTION = "ha",
@@ -184,7 +189,8 @@ public static void main( String[] args ) {
             try {
                 cliInstance = ( CLI ) beanClass.newInstance();
             } catch ( InstantiationException | IllegalAccessException e2 ) {
-                log.warn( String.format( "Failed to create %s using reflection, will have to create a complete bean to extract its metadata.", beanClass.getName() ), e2 );
+                System.err.printf( "Failed to create %s using reflection, will have to create a complete bean to extract its metadata.%n", beanClass.getName() );
+                e2.printStackTrace( System.err );
                 cliInstance = ctx.getBean( beanName, CLI.class );
             }
             String commandName = cliInstance.getCommandName();
@@ -291,7 +297,7 @@ public static void main( String[] args ) {
     static String getOptStringForLogging( Object[] argsToPass ) {
         Matcher matcher = PASSWORD_IN_CLI_MATCHER.matcher( StringUtils.join( argsToPass, " " ) );
         if ( matcher.matches() ) {
-            log.warn( "It seems that you still supply the -p/--password argument through the CLI. This feature has been removed for security purposes in Gemma 1.29." );
+            System.err.println( "It seems that you still supply the -p/--password argument through the CLI. This feature has been removed for security purposes in Gemma 1.29." );
         }
         return matcher.replaceAll( "$1 XXXXXX" );
     }

gemma-core/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>gemma</artifactId>
         <groupId>gemma</groupId>
-        <version>1.31.7</version>
+        <version>1.31.8</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>gemma-core</artifactId>

gemma-core/src/main/java/ubic/gemma/core/analysis/expression/coexpression/links/LinkAnalysisServiceImpl.java

@@ -59,6 +59,7 @@
 import ubic.gemma.persistence.service.common.auditAndSecurity.AuditTrailService;
 import ubic.gemma.persistence.service.expression.bioAssayData.ProcessedExpressionDataVectorService;
 import ubic.gemma.persistence.service.expression.designElement.CompositeSequenceService;
+import ubic.gemma.core.analysis.preprocess.batcheffects.ExpressionExperimentBatchInformationService;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
 
 import java.io.IOException;
@@ -102,6 +103,9 @@ public class LinkAnalysisServiceImpl implements LinkAnalysisService {
     @Autowired
     private ProcessedExpressionDataVectorService processedExpressionDataVectorService;
 
+    @Autowired
+    private ExpressionExperimentBatchInformationService expressionExperimentBatchInformationService;
+
     @Override
     public LinkAnalysis process( ExpressionExperiment ee, FilterConfig filterConfig,
             LinkAnalysisConfig linkAnalysisConfig ) {
@@ -441,9 +445,9 @@ private void qcCheck( LinkAnalysisConfig config, ExpressionExperiment ee ) throw
         }
 
         if ( config.isCheckForBatchEffect() ) {
-            BatchEffectDetails batchEffect = eeService.getBatchEffectDetails( ee );
+            BatchEffectDetails batchEffect = expressionExperimentBatchInformationService.getBatchEffectDetails( ee );
 
-            if ( batchEffect.getDataWasBatchCorrected() ) {
+            if ( batchEffect.dataWasBatchCorrected() ) {
                 LinkAnalysisServiceImpl.log.info( "Data are batch-corrected" );
                 return;
             }

gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/PreprocessingException.java

@@ -19,7 +19,7 @@
 
 /**
  * Allows us to catch preprocessing errors and handle them correctly.
- *
+ * <p>
  * The main kind of preprocessing exceptions are {@link ubic.gemma.core.analysis.preprocess.filter.FilteringException}
  * and {@link ubic.gemma.core.analysis.preprocess.batcheffects.BatchInfoPopulationException}.
  *
@@ -33,6 +33,10 @@ public PreprocessingException( ExpressionExperiment ee, String message ) {
         super( String.format( "Failed to pre-process %s: %s", ee.getShortName(), message ) );
     }
 
+    public PreprocessingException( ExpressionExperiment ee, String message, Throwable cause ) {
+        super( String.format( "Failed to pre-process %s: %s", ee.getShortName(), message ), cause );
+    }
+
     public PreprocessingException( ExpressionExperiment ee, Throwable cause ) {
         super( String.format( "Failed to pre-process %s: %s", ee.getShortName(), ExceptionUtils.getRootCauseMessage( cause ) ), cause );
     }

gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/VectorMergingServiceImpl.java

@@ -231,6 +231,7 @@ public void mergeVectors( ExpressionExperiment ee ) {
         // replace raw vectors with
         expressionExperimentService.removeAllRawDataVectors( ee );
         ee.getRawExpressionDataVectors().addAll( newVectors );
+        ee.getQuantitationTypes().addAll( qt2Vec.keySet() );
 
         // remove processed vectors
         expressionExperimentService.removeProcessedDataVectors( ee );