diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/coexpression/links/LinkAnalysis.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/coexpression/links/LinkAnalysis.java index b8d526dcde..65aea8eba7 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/coexpression/links/LinkAnalysis.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/coexpression/links/LinkAnalysis.java @@ -169,7 +169,7 @@ public void setExpressionExperiment( ExpressionExperiment expressionExperiment ) */ public Set getGenesTested() { Set genes = new HashSet<>(); - for ( CompositeSequence cs : dataMatrix.getRowNames() ) { + for ( CompositeSequence cs : dataMatrix.getDesignElements() ) { Set geneClusters = this.probeToGeneMap.get( cs ); if ( geneClusters == null ) { if ( numWarnings <= LinkAnalysis.MAX_WARNINGS ) { diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java index 67c7c2350d..9993f293d1 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/expression/diff/LinearModelAnalyzer.java @@ -706,7 +706,7 @@ private DifferentialExpressionAnalysis doAnalysis( BioAssaySet bioAssaySet, throw new FilteringRelatedAnalysisException( config, filterResult, e ); } - DoubleMatrix bareFilteredDataMatrix = expressionData.getMatrix(); + DoubleMatrix bareFilteredDataMatrix = expressionData.asDoubleMatrix(); DoubleMatrix1D librarySizes = getLibrarySizes( config, expressionData ); diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/TwoChannelMissingValuesImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/TwoChannelMissingValuesImpl.java index e6972c7050..842220effd 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/TwoChannelMissingValuesImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/TwoChannelMissingValuesImpl.java @@ -27,7 +27,6 @@ import org.springframework.transaction.annotation.Transactional; import ubic.basecode.math.distribution.Histogram; import ubic.gemma.core.datastructure.matrix.ExpressionDataDoubleMatrix; -import ubic.gemma.core.datastructure.matrix.ExpressionDataMatrixRowElement; import ubic.gemma.core.datastructure.matrix.TwoChannelExpressionDataMatrixBuilder; import ubic.gemma.model.common.auditAndSecurity.eventType.MissingValueAnalysisEvent; import ubic.gemma.model.common.quantitationtype.*; @@ -46,6 +45,7 @@ import javax.annotation.Nullable; import java.util.Collection; import java.util.HashSet; +import java.util.Objects; /** * Computes a missing value matrix for ratiometric data sets. @@ -239,14 +239,14 @@ private Collection computeMissingValues( ExpressionExpe Double signalThreshold = Double.NaN; if ( bkgChannelA == null && bkgChannelB == null ) { - signalThreshold = this.computeSignalThreshold( preferred, signalChannelA, signalChannelB, baseChannel ); + signalThreshold = this.computeSignalThreshold( baseChannel, signalChannelA, signalChannelB ); } QuantitationType present = this.getMissingDataQuantitationType( signalToNoiseThreshold, signalThreshold ); source.getQuantitationTypes().add( present ); - for ( ExpressionDataMatrixRowElement element : baseChannel.getRowElements() ) { + for ( CompositeSequence designElement : baseChannel.getDesignElements() ) { count = this.examineVector( source, preferred, signalChannelA, signalChannelB, bkgChannelA, bkgChannelB, signalToNoiseThreshold, extraMissingValueIndicators, results, count, baseChannel, - signalThreshold, present, element ); + signalThreshold, present, designElement ); } TwoChannelMissingValuesImpl.log.info( "Finished: " + count + " vectors examined for missing values" ); @@ -268,16 +268,14 @@ private int examineVector( ExpressionExperiment source, ExpressionDataDoubleMatr double signalToNoiseThreshold, @Nullable Collection extraMissingValueIndicators, Collection results, int count, ExpressionDataDoubleMatrix baseChannel, Double signalThreshold, QuantitationType present, - ExpressionDataMatrixRowElement element ) { - CompositeSequence designElement = element.getDesignElement(); - + CompositeSequence designElement ) { RawExpressionDataVector vect = RawExpressionDataVector.Factory.newInstance(); vect.setQuantitationType( present ); vect.setExpressionExperiment( source ); vect.setDesignElement( designElement ); vect.setBioAssayDimension( baseChannel.getBioAssayDimension( designElement ) ); - int numCols = preferred.columns( designElement ); + int numCols = vect.getBioAssayDimension().getBioAssays().size(); Boolean[] detectionCalls = new Boolean[numCols]; double[] prefRow = preferred.getRowAsDoubles( designElement ); @@ -357,17 +355,14 @@ private boolean checkMissingValue( @Nullable Collection extraMissingValu /** * Determine a threshold based on the data. */ - private Double computeSignalThreshold( ExpressionDataDoubleMatrix preferred, - @Nullable ExpressionDataDoubleMatrix signalChannelA, @Nullable ExpressionDataDoubleMatrix signalChannelB, - ExpressionDataDoubleMatrix baseChannel ) { + private Double computeSignalThreshold( ExpressionDataDoubleMatrix baseChannel, + @Nullable ExpressionDataDoubleMatrix signalChannelA, @Nullable ExpressionDataDoubleMatrix signalChannelB ) { double min = Double.MAX_VALUE; double max = Double.MIN_VALUE; - for ( ExpressionDataMatrixRowElement element : baseChannel.getRowElements() ) { - CompositeSequence designElement = element.getDesignElement(); - - int numCols = preferred.columns( designElement ); + for ( CompositeSequence designElement : baseChannel.getDesignElements() ) { + int numCols = Objects.requireNonNull( baseChannel.getBioAssayDimension( designElement ) ).getBioAssays().size(); for ( int col = 0; col < numCols; col++ ) { double[] signalA = null; @@ -397,10 +392,8 @@ private Double computeSignalThreshold( ExpressionDataDoubleMatrix preferred, } Histogram h = new Histogram( "range", 100, min, max ); - for ( ExpressionDataMatrixRowElement element : baseChannel.getRowElements() ) { - CompositeSequence designElement = element.getDesignElement(); - - int numCols = preferred.columns( designElement ); + for ( CompositeSequence designElement : baseChannel.getDesignElements() ) { + int numCols = Objects.requireNonNull( baseChannel.getBioAssayDimension( designElement ) ).getBioAssays().size(); for ( int col = 0; col < numCols; col++ ) { double[] signalA = null; diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java index a7db7f1135..66252642ec 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/batcheffects/ExpressionExperimentBatchCorrectionServiceImpl.java @@ -223,18 +223,19 @@ private ExpressionDataDoubleMatrix restoreOutliers( ExpressionDataDoubleMatrix o Iterate over the rows and columns of the original matrix and copy the values from the corrected matrix. If the column is an outlier in the original matrix, just skip it. */ - for ( int i = 0; i < originalDataMatrix.rows(); i++ ) { + DoubleMatrix dmatrix = originalDataMatrix.asDoubleMatrix(); + for ( int i = 0; i < dmatrix.rows(); i++ ) { int skip = 0; - for ( int j = 0; j < originalDataMatrix.columns(); j++ ) { + for ( int j = 0; j < dmatrix.columns(); j++ ) { if ( outlierColumns.contains( j ) ) { skip++; continue; // leave it alone; normally this will be an NaN. } - originalDataMatrix.set( i, j, correctedMatrix.getAsDouble( i, j - skip ) ); + dmatrix.set( i, j, correctedMatrix.getAsDouble( i, j - skip ) ); } } - return originalDataMatrix; + return originalDataMatrix.withMatrix( dmatrix ); } /** @@ -312,7 +313,7 @@ private ExpressionDataDoubleMatrix doComBat( ExpressionExperiment ee, Expression originalDataMatrix, ObjectMatrix design ) { ObjectMatrix designU = this.convertFactorValuesToStrings( design ); - DoubleMatrix matrix = originalDataMatrix.getMatrix(); + DoubleMatrix matrix = originalDataMatrix.asDoubleMatrix(); designU = this.orderMatrix( matrix, designU ); diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/convert/QuantitationTypeConversionUtils.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/convert/QuantitationTypeConversionUtils.java index 1e63a2cd41..b47ffa9408 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/convert/QuantitationTypeConversionUtils.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/convert/QuantitationTypeConversionUtils.java @@ -101,7 +101,7 @@ public static ExpressionDataDoubleMatrix ensureLog2Scale( ExpressionDataDoubleMa } StandardQuantitationType type = quantitationType.getType(); - DoubleMatrix transformedMatrix = dmatrix.getMatrix().copy(); + DoubleMatrix transformedMatrix = dmatrix.asDoubleMatrix(); switch ( quantitationType.getScale() ) { case LOG2: log.warn( String.format( "Data was detected on a log2-scale, but the quantitation type indicate %s. No transformation is necessary.", diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/detect/QuantitationTypeDetectionUtils.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/detect/QuantitationTypeDetectionUtils.java index 7cb0683596..d948a9a592 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/detect/QuantitationTypeDetectionUtils.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/detect/QuantitationTypeDetectionUtils.java @@ -107,7 +107,7 @@ public static QuantitationType inferQuantitationType( ExpressionDataMatrix ex private static InferredQuantitationType infer( ExpressionDataMatrix expressionData, @Nullable QuantitationType qt ) { Object matrix; if ( expressionData instanceof ExpressionDataDoubleMatrix ) { - matrix = new DenseDoubleMatrix2D( ( ( ExpressionDataDoubleMatrix ) expressionData ).getMatrix().asArray() ); + matrix = new DenseDoubleMatrix2D( ( ( ExpressionDataDoubleMatrix ) expressionData ).asDoubleMatrix().asArray() ); } else if ( expressionData instanceof SingleCellExpressionDataDoubleMatrix ) { matrix = ( ( SingleCellExpressionDataDoubleMatrix ) expressionData ).getMatrix(); } else { @@ -453,6 +453,7 @@ private static boolean isPercent100( DoubleMatrix2D matrix, double maximum ) { /** * Check if any of the rows of a given matrix are normalized. + * * @see #isZScore(DoubleMatrix1D) */ private static boolean isZScore( Object matrix ) { @@ -535,10 +536,11 @@ private static boolean isClose( double a, double b ) { /** * Detect suspicious values for a given quantitation type. + * * @throws SuspiciousValuesForQuantitationException if there are any suspicious values */ public static void detectSuspiciousValues( ExpressionDataDoubleMatrix a, QuantitationType qt ) throws SuspiciousValuesForQuantitationException { - DoubleMatrix2D matrix = new DenseDoubleMatrix2D( a.getMatrix().asArray() ); + DoubleMatrix2D matrix = new DenseDoubleMatrix2D( a.getMatrixAsDoubles() ); List flaggingResults = new ArrayList<>(); diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/ExpressionDataFilterUtils.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/ExpressionDataFilterUtils.java index 850d7a6a25..e06e49c54c 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/ExpressionDataFilterUtils.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/ExpressionDataFilterUtils.java @@ -16,7 +16,7 @@ public static Set getSamplesWithData( ExpressionDataDoubleMatrix da Set samplesWithData = new HashSet<>( dataMatrix.columns() ); for ( int j = 0; j < dataMatrix.columns(); j++ ) { for ( int i = 0; i < dataMatrix.rows(); i++ ) { - if ( !Double.isNaN( dataMatrix.getMatrix().get( i, j ) ) ) { + if ( !Double.isNaN( dataMatrix.getAsDouble( i, j ) ) ) { samplesWithData.add( dataMatrix.getBioMaterialForColumn( j ) ); break; } @@ -35,7 +35,7 @@ public static int countSamplesWithData( ExpressionDataDoubleMatrix dataMatrix ) int samplesWithData = 0; for ( int j = 0; j < dataMatrix.columns(); j++ ) { for ( int i = 0; i < dataMatrix.rows(); i++ ) { - if ( !Double.isNaN( dataMatrix.getMatrix().get( i, j ) ) ) { + if ( !Double.isNaN( dataMatrix.getAsDouble( i, j ) ) ) { samplesWithData++; break; } diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/ExpressionExperimentFilter.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/ExpressionExperimentFilter.java index de3c3c3a07..868af5e4e3 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/ExpressionExperimentFilter.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/ExpressionExperimentFilter.java @@ -29,10 +29,7 @@ import ubic.gemma.model.expression.designElement.CompositeSequence; import javax.annotation.Nullable; -import java.util.Collection; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; /** * Default filter used for various analyses of expression experiments. @@ -155,7 +152,11 @@ public ExpressionDataDoubleMatrix filter( ExpressionDataDoubleMatrix dataMatrix, // Filtering lowly expressed genes. if ( config.getLowExpressionCut() > 0.0 ) { ExpressionExperimentFilter.log.debug( "Filtering for low or too high expression" ); - Map ranks = dataMatrix.getRanksByMean(); + Map ranks = new HashMap<>(); + double[] rbm = dataMatrix.getRanksByMean(); + for ( int i = 0; i < dataMatrix.rows(); i++ ) { + ranks.put( dataMatrix.getDesignElementForRow( i ), rbm[i] ); + } dataMatrix = this.filterLowExpression( dataMatrix, ranks ); result.setLowExpressionFilterApplied( true ); result.setAfterLowExpressionFilter( dataMatrix.rows() ); diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/MinimumCellsFilter.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/MinimumCellsFilter.java index aef6aa6e15..4c845f5677 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/MinimumCellsFilter.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/MinimumCellsFilter.java @@ -126,7 +126,7 @@ else if ( dataMatrix.getQuantitationType().isPreferred( RawExpressionDataVector. if ( allowSlicingColumns ) { return dataMatrix.sliceColumns( keptSamples ); } else { - DoubleMatrix maskedMatrix = dataMatrix.getMatrix().copy(); + DoubleMatrix maskedMatrix = dataMatrix.asDoubleMatrix(); for ( int j = 0; j < dataMatrix.columns(); j++ ) { BioMaterial sample = dataMatrix.getBioMaterialForColumn( j ); if ( !keptSamples.contains( sample ) ) { diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/OutliersFilter.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/OutliersFilter.java index 08739dcc21..0675b66c54 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/OutliersFilter.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/OutliersFilter.java @@ -40,7 +40,7 @@ public ExpressionDataDoubleMatrix filter( ExpressionDataDoubleMatrix dataMatrix } log.info( "There are " + outlierAssays.size() + " outlier assays; masking them out..." ); - DoubleMatrix maskedMatrix = dataMatrix.getMatrix().copy(); + DoubleMatrix maskedMatrix = dataMatrix.asDoubleMatrix(); Set dimensionWithOutliers = dataMatrix.getBioAssayDimensions().stream() .filter( bad -> CollectionUtils.containsAny( bad.getBioAssays(), outlierAssays ) ) diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/RepetitiveValuesFilter.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/RepetitiveValuesFilter.java index e970903d9f..4e621e9c71 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/RepetitiveValuesFilter.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/filter/RepetitiveValuesFilter.java @@ -1,9 +1,8 @@ package ubic.gemma.core.analysis.preprocess.filter; +import cern.colt.matrix.DoubleMatrix2D; import lombok.extern.apachecommons.CommonsLog; import org.springframework.util.Assert; -import ubic.basecode.dataStructure.matrix.DenseDoubleMatrix; -import ubic.basecode.dataStructure.matrix.DoubleMatrix; import ubic.gemma.core.analysis.preprocess.convert.QuantitationTypeConversionException; import ubic.gemma.core.datastructure.matrix.ExpressionDataDoubleMatrix; import ubic.gemma.core.util.MatrixStats; @@ -13,7 +12,6 @@ import ubic.gemma.model.common.quantitationtype.StandardQuantitationType; import ubic.gemma.model.expression.bioAssay.BioAssay; import ubic.gemma.model.expression.bioAssayData.BioAssayDimension; -import ubic.gemma.model.expression.biomaterial.BioMaterial; import ubic.gemma.model.expression.designElement.CompositeSequence; import java.util.ArrayList; @@ -191,15 +189,15 @@ private Optional getLibrarySize( BioAssayDimension bioAssayDimension ) { private ExpressionDataDoubleMatrix filterLog2cpm( ExpressionDataDoubleMatrix dmatrix, long[] librarySize ) throws NoDesignElementsException { - DoubleMatrix unnormalizedMatrix = dmatrix.getMatrix().copy(); + double[][] unnormalizedMatrix = dmatrix.getMatrixAsDoubles(); double[] log2LibrarySize = new double[librarySize.length]; for ( int j = 0; j < librarySize.length; j++ ) { log2LibrarySize[j] = Math.log( librarySize[j] + 1.0 ) / Math.log( 2 ); } // undo the log2cpm transformation, but keep values in the log2 scale - for ( int i = 0; i < unnormalizedMatrix.rows(); i++ ) { - for ( int j = 0; j < unnormalizedMatrix.columns(); j++ ) { - unnormalizedMatrix.set( i, j, unnormalizedMatrix.get( i, j ) + log2LibrarySize[j] ); + for ( int i = 0; i < unnormalizedMatrix.length; i++ ) { + for ( int j = 0; j < unnormalizedMatrix[i].length; j++ ) { + unnormalizedMatrix[i][j] = unnormalizedMatrix[i][j] + log2LibrarySize[j]; } } Map unnormalizedQts = dmatrix.getQuantitationTypes().stream() @@ -239,9 +237,7 @@ private ExpressionDataDoubleMatrix filterDistinctValuesByRanks( ExpressionDataDo } private ExpressionDataDoubleMatrix rank( ExpressionDataDoubleMatrix dmatrix ) { - DenseDoubleMatrix rankMatrix = new DenseDoubleMatrix<>( MatrixStats.ranksByColumn( dmatrix.getMatrix() ).toArray() ); - rankMatrix.setRowNames( dmatrix.getMatrix().getRowNames() ); - rankMatrix.setColumnNames( dmatrix.getMatrix().getColNames() ); + DoubleMatrix2D rankMatrix = MatrixStats.ranksByColumn( dmatrix.asDoubleMatrix() ); Map rankQts = dmatrix.getQuantitationTypes().stream() .collect( Collectors.toMap( qt -> qt, qt -> { qt = QuantitationType.Factory.newInstance( qt ); diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/ExpressionDataSVD.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/ExpressionDataSVD.java index 81eb57a0e6..859119d70b 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/ExpressionDataSVD.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/preprocess/svd/ExpressionDataSVD.java @@ -25,7 +25,6 @@ import cern.colt.matrix.impl.DenseDoubleMatrix2D; import cern.colt.matrix.linalg.Algebra; import org.apache.commons.lang3.StringUtils; -import ubic.basecode.dataStructure.matrix.DenseDoubleMatrix; import ubic.basecode.dataStructure.matrix.DoubleMatrix; import ubic.basecode.math.DescriptiveWithMissing; import ubic.basecode.math.MatrixStats; @@ -122,7 +121,7 @@ public ExpressionDataSVD( ExpressionDataDoubleMatrix expressionData, boolean nor // } this.normalized = normalizeMatrix; - DoubleMatrix matrix = this.expressionData.getMatrix(); + DoubleMatrix matrix = this.expressionData.asDoubleMatrix(); assert matrix.getRowNames().size() > 0; assert matrix.getColNames().size() > 0; @@ -158,17 +157,13 @@ public ExpressionDataDoubleMatrix equalize() { DoubleMatrix2D v = new DenseDoubleMatrix2D( rawV ); Algebra a = new Algebra(); - DoubleMatrix reconstructed = new DenseDoubleMatrix<>( - a.mult( a.mult( u, s ), a.transpose( v ) ).toArray() ); - - reconstructed.setRowNames( this.expressionData.getMatrix().getRowNames() ); - reconstructed.setColumnNames( this.expressionData.getMatrix().getColNames() ); + DoubleMatrix2D reconstructed = a.mult( a.mult( u, s ), a.transpose( v ) ); // re-mask the missing values. for ( int i = 0; i < reconstructed.rows(); i++ ) { for ( int j = 0; j < reconstructed.columns(); j++ ) { if ( Double.isNaN( this.missingValueInfo.get( i, j ) ) ) { - reconstructed.set( i, j, Double.NaN ); + reconstructed.setQuick( i, j, Double.NaN ); } } } @@ -286,11 +281,7 @@ public ExpressionDataDoubleMatrix removeHighestComponents( int numComponentsToRe DoubleMatrix2D v = new DenseDoubleMatrix2D( rawV ); Algebra a = new Algebra(); - DoubleMatrix reconstructed = new DenseDoubleMatrix<>( - a.mult( a.mult( u, s ), a.transpose( v ) ).toArray() ); - - reconstructed.setRowNames( this.expressionData.getMatrix().getRowNames() ); - reconstructed.setColumnNames( this.expressionData.getMatrix().getColNames() ); + DoubleMatrix2D reconstructed = a.mult( a.mult( u, s ), a.transpose( v ) ); // re-mask the missing values. for ( int i = 0; i < reconstructed.rows(); i++ ) { @@ -322,19 +313,14 @@ public ExpressionDataDoubleMatrix uMatrixAsExpressionData() { DoubleMatrix rawUMatrix = svd.getU(); - DoubleMatrix result = new DenseDoubleMatrix<>( rawUMatrix.rows(), - rawUMatrix.columns() ); + DenseDoubleMatrix2D result = new DenseDoubleMatrix2D( rawUMatrix.rows(), rawUMatrix.columns() ); // take the absolute value of the U matrix. for ( int i = 0; i < rawUMatrix.rows(); i++ ) { for ( int j = 0; j < rawUMatrix.columns(); j++ ) { - result.set( i, j, Math.abs( rawUMatrix.get( i, j ) ) ); + result.setQuick( i, j, Math.abs( rawUMatrix.get( i, j ) ) ); } } - List colNames = svd.getV().getColNames(); - - result.setColumnNames( colNames ); - result.setRowNames( rawUMatrix.getRowNames() ); // use that as the 'expression data' return this.expressionData.withMatrix( result ); diff --git a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixServiceImpl.java index 2e5d2e817d..75e0d40768 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixServiceImpl.java @@ -231,7 +231,7 @@ private ExpressionDataDoubleMatrix logTransform( ExpressionDataDoubleMatrix data if ( alreadyLogged ) { return datamatrix; } - DoubleMatrix matrix = datamatrix.getMatrix().copy(); + DoubleMatrix matrix = datamatrix.asDoubleMatrix(); // this is a log2 MatrixStats.logTransform( matrix ); Map qts = datamatrix.getQuantitationTypes().stream() diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/AbstractMultiAssayExpressionDataMatrix.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/AbstractMultiAssayExpressionDataMatrix.java index fd11d4bd2d..9202b2e5d1 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/AbstractMultiAssayExpressionDataMatrix.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/AbstractMultiAssayExpressionDataMatrix.java @@ -71,6 +71,7 @@ abstract public class AbstractMultiAssayExpressionDataMatrix extends Abstract private Map columnAssayMap; @Nullable + @Deprecated private List rowElements = null; /** @@ -164,16 +165,6 @@ public int columns() { return columnBioMaterials.size(); } - @Override - public int columns( CompositeSequence el ) { - BioAssayDimension dimension = rowElementBioAssayDimensionMap.get( el ); - return ( int ) dimension.getBioAssays().stream() - .map( BioAssay::getSampleUsed ) - .distinct() // in case a BioMaterial is used for more than one BioAssay in this dimension - .filter( columnBioMaterialMap::containsKey ) - .count(); - } - @Override public Collection getBioAssaysForColumn( int index ) { if ( index < 0 || index >= columns() ) { @@ -241,6 +232,7 @@ public CompositeSequence getDesignElementForRow( int index ) { return this.rowDesignElements.get( index ); } + @Nullable @Override public T get( CompositeSequence designElement, BioAssay bioAssay ) { Integer index = this.rowElementMap.get( designElement ); @@ -254,6 +246,7 @@ public T get( CompositeSequence designElement, BioAssay bioAssay ) { return get( index, j ); } + @Nullable @Override public ExpressionExperiment getExpressionExperiment() { return this.expressionExperiment; @@ -279,6 +272,7 @@ public QuantitationType getQuantitationType( CompositeSequence designElement ) { } @Override + @Deprecated public List getRowElements() { if ( this.rowElements == null ) { int rows = rows(); @@ -308,6 +302,7 @@ public int[] getRowIndices( CompositeSequence designElement ) { } @Override + @Deprecated public ExpressionDataMatrixRowElement getRowElement( int index ) { if ( rowElements != null ) { return rowElements.get( index ); diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataDoubleMatrix.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataDoubleMatrix.java index b3da6bfc26..d1d2200a31 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataDoubleMatrix.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataDoubleMatrix.java @@ -68,6 +68,26 @@ public double[] getColumnAsDoubles( int column ) { return matrix.viewColumn( column ).toArray(); } + @Override + public double getAsDouble( CompositeSequence designElement, BioMaterial bioMaterial ) { + int row = getRowIndex( designElement ); + int column = getColumnIndex( bioMaterial ); + if ( row == -1 || column == -1 ) { + return Double.NaN; + } + return matrix.get( row, column ); + } + + @Override + public double getAsDouble( CompositeSequence designElement, BioAssay bioAssay ) { + int row = getRowIndex( designElement ); + int column = getColumnIndex( bioAssay ); + if ( row == -1 || column == -1 ) { + return Double.NaN; + } + return matrix.get( row, column ); + } + @Override public double[] getColumnAsDoubles( BioAssay bioAssay ) { int column = getColumnIndex( bioAssay ); @@ -123,7 +143,7 @@ public double getAsDouble( int row, int column ) { } @Override - public Double[][] getRawMatrix() { + public Double[][] getMatrix() { Double[][] result = new Double[rows()][]; for ( int i = 0; i < rows(); i++ ) { result[i] = getRow( i ); @@ -142,7 +162,7 @@ public BulkExpressionDataDoubleMatrix sliceColumns( List bioMateria } @Override - public double[][] getRawMatrixAsDoubles() { + public double[][] getMatrixAsDoubles() { return matrix.toArray(); } diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataIntMatrix.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataIntMatrix.java index 153c8d5f31..115ac34561 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataIntMatrix.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataIntMatrix.java @@ -46,7 +46,7 @@ public boolean hasMissingValues() { } @Override - public Integer[][] getRawMatrix() { + public Integer[][] getMatrix() { Integer[][] result = new Integer[matrix.length][]; for ( int i = 0; i < matrix.length; i++ ) { result[i] = ArrayUtils.toObject( matrix[i] ); @@ -65,7 +65,7 @@ public BulkExpressionDataIntMatrix sliceColumns( List bioMaterials, } @Override - public int[][] getRawMatrixAsInts() { + public int[][] getMatrixAsInts() { return matrix; } diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataMatrix.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataMatrix.java index d6909948c4..67e40e9d5a 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataMatrix.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataMatrix.java @@ -91,7 +91,7 @@ static BulkExpressionDataMatrix getMatrix( Collection getBioMaterials(); diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataPrimitiveDoubleMatrix.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataPrimitiveDoubleMatrix.java index 87b0b6a7f4..679a43711f 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataPrimitiveDoubleMatrix.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataPrimitiveDoubleMatrix.java @@ -1,22 +1,47 @@ package ubic.gemma.core.datastructure.matrix; import ubic.gemma.model.expression.bioAssay.BioAssay; +import ubic.gemma.model.expression.biomaterial.BioMaterial; +import ubic.gemma.model.expression.designElement.CompositeSequence; + +import javax.annotation.Nonnull; /** * Interface for bulk expression data matrices that can be efficiently accessed as a primitive double matrix. + * * @author poirigui */ public interface BulkExpressionDataPrimitiveDoubleMatrix extends BulkExpressionDataMatrix, ExpressionDataPrimitiveDoubleMatrix { + @Nonnull + @Override + Double get( int row, int column ); + + /** + * Retrieve the value for a given design element and biomaterial without boxing. + * + * @return the value as a primitive double, or {@link Double#NaN} if not found + */ + double getAsDouble( CompositeSequence designElement, BioMaterial bioMaterial ); + + /** + * Retrieve the value for a given design element and bioassay without boxing. + * + * @return the value as a primitive double, or {@link Double#NaN} if not found + */ + double getAsDouble( CompositeSequence designElement, BioAssay bioAssay ); + /** * Retrieve the given column without boxing. + * * @see #getColumn(BioAssay) */ double[] getColumnAsDoubles( BioAssay bioAssay ); /** * Obtain the raw matrix as a double array. - * @see #getRawMatrix() + * + * @see #getMatrix() */ - double[][] getRawMatrixAsDoubles(); + double[][] getMatrixAsDoubles(); } diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataPrimitiveIntMatrix.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataPrimitiveIntMatrix.java index 1f3dbabb1d..b9a70cd3f5 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataPrimitiveIntMatrix.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataPrimitiveIntMatrix.java @@ -6,12 +6,14 @@ /** * Interface for bulk expression data matrices that can be efficiently accessed as a primitive int matrix. + * * @author poirigui */ public interface BulkExpressionDataPrimitiveIntMatrix extends BulkExpressionDataMatrix, ExpressionDataPrimitiveIntMatrix { /** * Retrieve the given column without boxing. + * * @see #getColumn(int) */ @Nullable @@ -19,7 +21,8 @@ public interface BulkExpressionDataPrimitiveIntMatrix extends BulkExpressionData /** * Obtain the raw matrix as a int array. - * @see #getRawMatrix() + * + * @see #getMatrix() */ - int[][] getRawMatrixAsInts(); + int[][] getMatrixAsInts(); } diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/EmptyBulkExpressionDataMatrix.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/EmptyBulkExpressionDataMatrix.java index e7cc4aed65..a07ae1a063 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/EmptyBulkExpressionDataMatrix.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/EmptyBulkExpressionDataMatrix.java @@ -29,7 +29,7 @@ public boolean hasMissingValues() { } @Override - public Object[][] getRawMatrix() { + public Object[][] getMatrix() { return EMPTY_MATRIX; } diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/EmptyExpressionMatrix.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/EmptyExpressionMatrix.java index 2b88d32252..9ac1960fef 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/EmptyExpressionMatrix.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/EmptyExpressionMatrix.java @@ -64,7 +64,7 @@ public Object[] getColumn( int column ) { } @Override - public Object[][] getRawMatrix() { + public Object[][] getMatrix() { return EMPTY_MATRIX; } diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataBooleanMatrix.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataBooleanMatrix.java index 932231d94a..faa83b9a47 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataBooleanMatrix.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataBooleanMatrix.java @@ -18,9 +18,6 @@ */ package ubic.gemma.core.datastructure.matrix; -import org.apache.commons.lang3.ArrayUtils; -import ubic.basecode.dataStructure.matrix.AbstractMatrix; -import ubic.basecode.dataStructure.matrix.ObjectMatrixImpl; import ubic.gemma.model.common.quantitationtype.PrimitiveType; import ubic.gemma.model.common.quantitationtype.QuantitationType; import ubic.gemma.model.expression.bioAssay.BioAssay; @@ -31,7 +28,8 @@ import ubic.gemma.model.expression.designElement.CompositeSequence; import ubic.gemma.model.expression.experiment.ExpressionExperiment; -import java.util.*; +import java.util.Collection; +import java.util.List; /** * Matrix of booleans mapped from an ExpressionExperiment. @@ -40,52 +38,42 @@ */ public class ExpressionDataBooleanMatrix extends AbstractMultiAssayExpressionDataMatrix { - private final ObjectMatrixImpl matrix; + private final Boolean[][] matrix; + + private final boolean hasMissingValues; public ExpressionDataBooleanMatrix( ExpressionExperiment ee, Collection vectors ) { super( ee ); - for ( BulkExpressionDataVector dedv : vectors ) { - if ( !dedv.getQuantitationType().getRepresentation().equals( PrimitiveType.BOOLEAN ) ) { - throw new IllegalStateException( "Cannot convert non-boolean quantitation types into boolean matrix" ); - } - } - this.matrix = vectorsToMatrix( selectVectors( vectors ) ); + this.matrix = createMatrix( selectVectors( vectors ) ); + this.hasMissingValues = checkMissingValues( this.matrix ); } - public ExpressionDataBooleanMatrix( ExpressionExperiment ee, Collection vectors, - List qtypes ) { + public ExpressionDataBooleanMatrix( ExpressionExperiment ee, Collection vectors, List qtypes ) { super( ee ); - this.matrix = vectorsToMatrix( selectVectors( vectors, qtypes ) ); - } - - @Override - public int columns() { - return matrix.columns(); + this.matrix = createMatrix( selectVectors( vectors, qtypes ) ); + this.hasMissingValues = checkMissingValues( this.matrix ); } @Override public Boolean get( int row, int column ) { - return matrix.get( row, column ); + return matrix[row][column]; } @Override public Boolean[] getColumn( int column ) { - Boolean[] res = new Boolean[matrix.rows()]; + if ( column < 0 || column >= columns() ) { + throw new IndexOutOfBoundsException( "Column index " + column + " is out of bounds." ); + } + Boolean[] res = new Boolean[rows()]; for ( int i = 0; i < res.length; i++ ) { - res[i] = this.matrix.get( i, column ); + res[i] = this.matrix[i][column]; } return res; } @Override - public Boolean[][] getRawMatrix() { - Boolean[][] dMatrix = new Boolean[matrix.rows()][matrix.columns()]; - for ( int i = 0; i < dMatrix.length; i++ ) { - for ( int j = 0; j < dMatrix[i].length; j++ ) { - dMatrix[i][j] = matrix.get( i, j ); - } - } - return dMatrix; + public Boolean[][] getMatrix() { + return matrix; } @Override @@ -100,11 +88,7 @@ public ExpressionDataBooleanMatrix sliceColumns( List bioMaterials, @Override public Boolean[] getRow( int index ) { - Boolean[] row = new Boolean[matrix.columns()]; - for ( int j = 0; j < row.length; j++ ) { - row[j] = this.matrix.get( index, j ); - } - return row; + return matrix[index]; } @Override @@ -114,77 +98,31 @@ public ExpressionDataBooleanMatrix sliceRows( List designElem @Override public boolean hasMissingValues() { - for ( int i = 0; i < matrix.rows(); i++ ) { - for ( int j = 0; j < matrix.columns(); j++ ) { - if ( matrix.get( i, j ) == null ) - return true; - } - } - return false; + return hasMissingValues; } @Override protected String format( int row, int column ) { - Boolean val = matrix.get( row, column ); + Boolean val = matrix[row][column]; return val != null ? String.valueOf( val ) : ""; } - private ObjectMatrixImpl vectorsToMatrix( Collection vectors ) { - if ( vectors.isEmpty() ) { - throw new IllegalArgumentException(); - } - return this.createMatrix( vectors ); - } - /** * Fill in the data */ - private ObjectMatrixImpl createMatrix( - Collection vectors ) { - ObjectMatrixImpl mat = new ObjectMatrixImpl<>( vectors.size(), columns() ); - - // initialize the matrix to false - for ( int i = 0; i < mat.rows(); i++ ) { - for ( int j = 0; j < mat.columns(); j++ ) { - mat.set( i, j, Boolean.FALSE ); + private Boolean[][] createMatrix( List vectors ) { + // gaps in the matrix will be null + Boolean[][] mat = new Boolean[rows()][columns()]; + for ( int i = 0; i < vectors.size(); i++ ) { + BulkExpressionDataVector vector = vectors.get( i ); + boolean[] vec = getVals( vector ); + List bioAssays = vector.getBioAssayDimension().getBioAssays(); + for ( int j = 0; j < bioAssays.size(); j++ ) { + BioAssay ba = bioAssays.get( j ); + int column = getColumnIndex( ba ); + mat[i][column] = vec[j]; } } - for ( int j = 0; j < mat.columns(); j++ ) { - mat.addColumnName( j ); - } - - Map rowNames = new TreeMap<>(); - - for ( BulkExpressionDataVector vector : vectors ) { - BioAssayDimension dimension = vector.getBioAssayDimension(); - - CompositeSequence designElement = vector.getDesignElement(); - - int rowIndex = getRowIndex( designElement ); - assert rowIndex != -1; - - rowNames.put( rowIndex, designElement ); - - boolean[] vals = this.getVals( vector ); - - Collection bioAssays = dimension.getBioAssays(); - - if ( bioAssays.size() != vals.length ) { - throw new IllegalStateException( - "Expected " + vals.length + " bioassays at design element " + designElement + ", got " - + bioAssays.size() ); - } - - Iterator it = bioAssays.iterator(); - this.setMatBioAssayValues( mat, rowIndex, ArrayUtils.toObject( vals ), bioAssays, it ); - } - - for ( int i = 0; i < mat.rows(); i++ ) { - mat.addRowName( rowNames.get( i ) ); - } - - assert mat.getRowNames().size() == mat.rows(); - return mat; } @@ -192,12 +130,11 @@ private ObjectMatrixImpl createMatrix( * Note that if we have trouble interpreting the data, it gets left as false. */ private boolean[] getVals( DesignElementDataVector vector ) { - boolean[] vals = null; if ( vector.getQuantitationType().getRepresentation().equals( PrimitiveType.BOOLEAN ) ) { - vals = vector.getDataAsBooleans(); + return vector.getDataAsBooleans(); } else if ( vector.getQuantitationType().getRepresentation().equals( PrimitiveType.CHAR ) ) { char[] charVals = vector.getDataAsChars(); - vals = new boolean[charVals.length]; + boolean[] vals = new boolean[charVals.length]; int j = 0; for ( char c : charVals ) { switch ( c ) { @@ -216,9 +153,10 @@ private boolean[] getVals( DesignElementDataVector vector ) { } j++; } + return vals; } else if ( vector.getQuantitationType().getRepresentation().equals( PrimitiveType.STRING ) ) { String[] fields = vector.getDataAsTabbedStrings(); - vals = new boolean[fields.length]; + boolean[] vals = new boolean[fields.length]; int j = 0; for ( String c : fields ) { switch ( c ) { @@ -237,17 +175,20 @@ private boolean[] getVals( DesignElementDataVector vector ) { } j++; } + return vals; + } else { + throw new IllegalArgumentException( "Unsupported representation: " + vector.getQuantitationType().getRepresentation() ); } - return vals; } - private void setMatBioAssayValues( AbstractMatrix mat, Integer rowIndex, V[] vals, - Collection bioAssays, Iterator it ) { - for ( int j = 0; j < bioAssays.size(); j++ ) { - BioAssay bioAssay = it.next(); - int column = getColumnIndex( bioAssay ); - assert column != -1; - mat.set( rowIndex, column, vals[j] ); + private static boolean checkMissingValues( Boolean[][] matrix ) { + for ( Boolean[] vector : matrix ) { + for ( Boolean b : vector ) { + if ( b == null ) { + return true; + } + } } + return false; } } diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataDoubleMatrix.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataDoubleMatrix.java index 4b12dc8541..a868e00120 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataDoubleMatrix.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataDoubleMatrix.java @@ -19,17 +19,17 @@ package ubic.gemma.core.datastructure.matrix; import cern.colt.list.DoubleArrayList; -import org.apache.commons.collections4.CollectionUtils; +import cern.colt.matrix.DoubleMatrix2D; +import cern.colt.matrix.impl.DenseDoubleMatrix2D; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.util.Assert; -import ubic.basecode.dataStructure.matrix.AbstractMatrix; +import org.springframework.util.CollectionUtils; import ubic.basecode.dataStructure.matrix.DenseDoubleMatrix; import ubic.basecode.dataStructure.matrix.DoubleMatrix; import ubic.basecode.math.DescriptiveWithMissing; import ubic.basecode.math.Rank; -import ubic.gemma.model.common.quantitationtype.PrimitiveType; import ubic.gemma.model.common.quantitationtype.QuantitationType; import ubic.gemma.model.expression.bioAssay.BioAssay; import ubic.gemma.model.expression.bioAssayData.BioAssayDimension; @@ -38,6 +38,7 @@ import ubic.gemma.model.expression.designElement.CompositeSequence; import ubic.gemma.model.expression.experiment.ExpressionExperiment; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.*; import java.util.stream.Collectors; @@ -56,13 +57,12 @@ public class ExpressionDataDoubleMatrix extends AbstractMultiAssayExpressionData private static final Log log = LogFactory.getLog( ExpressionDataDoubleMatrix.class.getName() ); - private final DoubleMatrix matrix; + private final double[][] matrix; /** * Indicate if {@link #matrix} contains missing values. */ - @Nullable - private Boolean hasMissingValues = null; + private final boolean hasMissingValues; /** * Indicate how many cells were used to compute each value in the matrix. @@ -76,54 +76,40 @@ public class ExpressionDataDoubleMatrix extends AbstractMultiAssayExpressionData * Indicate the rank (by mean signal intensity) of each row in the matrix. */ @Nullable - private Map ranks = null; + private double[] ranks = null; public ExpressionDataDoubleMatrix( @Nullable ExpressionExperiment ee, Collection vectors ) { super( ee ); - for ( BulkExpressionDataVector dedv : vectors ) { - if ( !dedv.getQuantitationType().getRepresentation().equals( PrimitiveType.DOUBLE ) ) { - throw new IllegalStateException( - "Cannot convert non-double quantitation types into double matrix:" + dedv - .getQuantitationType() ); - } - } List selectedVectors = selectVectors( vectors ); if ( selectedVectors.isEmpty() ) { throw new IllegalArgumentException( "No vectors!" ); } this.matrix = this.createMatrix( selectedVectors ); + this.hasMissingValues = computeHasMissingValues( this.matrix ); this.numberOfCells = getNumberOfCellsFromVectors( selectedVectors ); } public ExpressionDataDoubleMatrix( ExpressionExperiment ee, Collection dataVectors, Collection quantitationTypes ) { super( ee ); - for ( QuantitationType qt : quantitationTypes ) { - if ( !qt.getRepresentation().equals( PrimitiveType.DOUBLE ) ) { - throw new IllegalStateException( - "Cannot convert non-double quantitation types into double matrix: " + qt ); - } - } List selectedVectors = selectVectors( dataVectors, quantitationTypes ); if ( selectedVectors.isEmpty() ) { throw new IllegalArgumentException( "No vectors!" ); } this.matrix = this.createMatrix( selectedVectors ); + this.hasMissingValues = computeHasMissingValues( this.matrix ); this.numberOfCells = getNumberOfCellsFromVectors( selectedVectors ); } public ExpressionDataDoubleMatrix( ExpressionExperiment ee, Collection dataVectors, QuantitationType quantitationType ) { super( ee ); - if ( !quantitationType.getRepresentation().equals( PrimitiveType.DOUBLE ) ) { - throw new IllegalStateException( - "Cannot convert non-double quantitation types into double matrix: " + quantitationType ); - } List selectedVectors = selectVectors( dataVectors, quantitationType ); if ( selectedVectors.isEmpty() ) { throw new IllegalArgumentException( "No vectors!" ); } this.matrix = this.createMatrix( selectedVectors ); + this.hasMissingValues = computeHasMissingValues( this.matrix ); this.numberOfCells = getNumberOfCellsFromVectors( selectedVectors ); } @@ -168,7 +154,8 @@ public ExpressionDataDoubleMatrix( ExpressionExperiment ee, DoubleMatrix dataMatrix ) { + private ExpressionDataDoubleMatrix( ExpressionDataDoubleMatrix sourceMatrix, double[][] dataMatrix ) { super( sourceMatrix ); - Assert.isTrue( dataMatrix.getColNames().equals( sourceMatrix.getBioMaterials() ) - && dataMatrix.getRowNames().equals( sourceMatrix.getDesignElements() ), - "The rows and columns of the new matrix correspond to the original matrix." ); + Assert.isTrue( dataMatrix.length == rows() && dataMatrix[0].length == columns(), + "The new matrix has the same dimensions as the original matrix." ); this.matrix = dataMatrix; + this.hasMissingValues = computeHasMissingValues( dataMatrix ); this.numberOfCells = sourceMatrix.numberOfCells; } @@ -200,13 +186,12 @@ private ExpressionDataDoubleMatrix( ExpressionDataDoubleMatrix sourceMatrix, * @param quantitationTypes quantitation type(s) used by dataMatrix */ private ExpressionDataDoubleMatrix( ExpressionDataDoubleMatrix sourceMatrix, - DoubleMatrix dataMatrix, Map quantitationTypes ) { + double[][] dataMatrix, Map quantitationTypes ) { super( sourceMatrix.getExpressionExperiment() ); - Assert.isTrue( dataMatrix.getColNames().equals( sourceMatrix.getBioMaterials() ) - && dataMatrix.getRowNames().equals( sourceMatrix.getDesignElements() ), + Assert.isTrue( dataMatrix.length == sourceMatrix.rows() && dataMatrix[0].length == sourceMatrix.columns(), "The rows and columns of the new matrix correspond to the original matrix." ); - for ( int i = 0; i < dataMatrix.rows(); i++ ) { - CompositeSequence element = dataMatrix.getRowName( i ); + for ( int i = 0; i < sourceMatrix.rows(); i++ ) { + CompositeSequence element = sourceMatrix.getDesignElementForRow( i ); QuantitationType qt = sourceMatrix.getQuantitationType( element ); if ( qt == null ) { throw new IllegalArgumentException( "The source matrix does not have a quantitation type for " + element + "." ); @@ -225,6 +210,7 @@ private ExpressionDataDoubleMatrix( ExpressionDataDoubleMatrix sourceMatrix, setUpColumnElements(); this.matrix = dataMatrix; + this.hasMissingValues = computeHasMissingValues( dataMatrix ); this.numberOfCells = sourceMatrix.numberOfCells; } @@ -236,8 +222,7 @@ private ExpressionDataDoubleMatrix( ExpressionDataDoubleMatrix sourceMatrix, */ private ExpressionDataDoubleMatrix( ExpressionDataDoubleMatrix sourceMatrix, List rowsToUse ) { super( sourceMatrix.getExpressionExperiment() ); - this.matrix = new DenseDoubleMatrix<>( rowsToUse.size(), sourceMatrix.columns() ); - this.matrix.setColumnNames( sourceMatrix.matrix.getColNames() ); + this.matrix = new double[rowsToUse.size()][sourceMatrix.columns()]; int[][] numberOfCells = null; for ( int i = 0; i < rowsToUse.size(); i++ ) { CompositeSequence element = rowsToUse.get( i ); @@ -248,12 +233,7 @@ private ExpressionDataDoubleMatrix( ExpressionDataDoubleMatrix sourceMatrix, Lis throw new IllegalArgumentException( "Source matrix does not have a row for " + element + "." ); } - this.matrix.addRowName( element ); - - for ( int j = 0; j < rowVals.length; j++ ) { - Double val = rowVals[j]; - this.matrix.set( i, j, val ); - } + System.arraycopy( rowVals, 0, this.matrix[i], 0, rowVals.length ); int[] noc = sourceMatrix.getNumberOfCellsForRow( i ); if ( noc != null ) { if ( numberOfCells == null ) { @@ -274,15 +254,17 @@ private ExpressionDataDoubleMatrix( ExpressionDataDoubleMatrix sourceMatrix, Lis setUpColumnElements(); } + this.hasMissingValues = sourceMatrix.hasMissingValues && computeHasMissingValues( this.matrix ); this.numberOfCells = numberOfCells; } private ExpressionDataDoubleMatrix( ExpressionDataDoubleMatrix sourceMatrix, List columnsToUse, boolean dummy ) { super( sourceMatrix.getExpressionExperiment() ); - this.matrix = new DenseDoubleMatrix<>( sourceMatrix.rows(), columnsToUse.size() ); - this.matrix.setRowNames( sourceMatrix.matrix.getRowNames() ); - this.matrix.setColumnNames( columnsToUse ); + this.matrix = new double[sourceMatrix.rows()][columnsToUse.size()]; + for ( double[] row : matrix ) { + Arrays.fill( row, Double.NaN ); + } int k = 0; int[] originalBioMaterialIndices = new int[columnsToUse.size()]; @@ -314,8 +296,7 @@ private ExpressionDataDoubleMatrix( ExpressionDataDoubleMatrix sourceMatrix, Lis } ); addToRowMaps( designElement, qt, slicedDim ); for ( int j = 0; j < originalBioMaterialIndices.length; j++ ) { - Double val = sourceRow[originalBioMaterialIndices[j]]; - this.matrix.set( i, j, val ); + this.matrix[i][j] = sourceRow[originalBioMaterialIndices[j]]; if ( sourceNumberOfCells != null ) { if ( numberOfCells == null ) { numberOfCells = new int[sourceMatrix.rows()][]; @@ -330,6 +311,7 @@ private ExpressionDataDoubleMatrix( ExpressionDataDoubleMatrix sourceMatrix, Lis setUpColumnElements(); + this.hasMissingValues = sourceMatrix.hasMissingValues && computeHasMissingValues( this.matrix ); this.numberOfCells = numberOfCells; } @@ -345,9 +327,7 @@ private ExpressionDataDoubleMatrix( ExpressionDataDoubleMatrix sourceMatrix, Lis BioAssayDimension reorderedDim ) { super( sourceMatrix.getExpressionExperiment() ); - this.matrix = new DenseDoubleMatrix<>( sourceMatrix.rows(), columnsToUse.size() ); - this.matrix.setRowNames( sourceMatrix.matrix.getRowNames() ); - this.matrix.setColumnNames( columnsToUse ); + this.matrix = new double[sourceMatrix.rows()][columnsToUse.size()]; int k = 0; int[] originalBioMaterialIndices = new int[columnsToUse.size()]; @@ -363,8 +343,7 @@ private ExpressionDataDoubleMatrix( ExpressionDataDoubleMatrix sourceMatrix, Lis int[] sourceNumberOfCells = sourceMatrix.getNumberOfCellsForRow( i ); addToRowMaps( designElement, requireNonNull( sourceMatrix.getQuantitationType( designElement ) ), reorderedDim ); for ( int j = 0; j < originalBioMaterialIndices.length; j++ ) { - Double val = sourceRow[originalBioMaterialIndices[j]]; - this.matrix.set( i, j, val ); + this.matrix[i][j] = sourceRow[originalBioMaterialIndices[j]]; if ( sourceNumberOfCells != null ) { if ( numberOfCells == null ) { numberOfCells = new int[sourceMatrix.rows()][]; @@ -379,6 +358,7 @@ private ExpressionDataDoubleMatrix( ExpressionDataDoubleMatrix sourceMatrix, Lis setUpColumnElements(); + this.hasMissingValues = sourceMatrix.hasMissingValues && computeHasMissingValues( this.matrix ); this.numberOfCells = numberOfCells; } @@ -386,18 +366,38 @@ public ExpressionDataDoubleMatrix( ExpressionDataDoubleMatrix sourceMatrix, @Nul super( sourceMatrix ); Assert.isTrue( numberOfCells == null || ( numberOfCells.length == sourceMatrix.rows() && Arrays.stream( numberOfCells ).allMatch( row -> row.length == sourceMatrix.columns() ) ) ); this.matrix = sourceMatrix.matrix; + this.hasMissingValues = sourceMatrix.hasMissingValues; this.ranks = sourceMatrix.ranks; this.numberOfCells = numberOfCells; } + @Nonnull @Override public Double get( int row, int column ) { - return matrix.get( row, column ); + return matrix[row][column]; } @Override public double getAsDouble( int row, int column ) { - return matrix.get( row, column ); + return matrix[row][column]; + } + + public double getAsDouble( CompositeSequence designElement, BioMaterial bioMaterial ) { + int row = getRowIndex( designElement ); + int column = getColumnIndex( bioMaterial ); + if ( row == -1 || column == -1 ) { + return Double.NaN; + } + return matrix[row][column]; + } + + public double getAsDouble( CompositeSequence designElement, BioAssay bioAssay ) { + int row = getRowIndex( designElement ); + int column = getColumnIndex( bioAssay ); + if ( row == -1 || column == -1 ) { + return Double.NaN; + } + return matrix[row][column]; } @Override @@ -410,14 +410,27 @@ public double[] getColumnAsDoubles( BioAssay bioAssay ) { } @Override - public Double[] getColumn( int index ) { - // FIXME: DoubleMatrix.getColObj is not efficient - return ArrayUtils.toObject( matrix.getColumn( index ) ); + public Double[] getColumn( int column ) { + if ( column < 0 || column >= columns() ) { + throw new IndexOutOfBoundsException( "Column index out of bounds: " + column ); + } + Double[] col = new Double[rows()]; + for ( int i = 0; i < col.length; i++ ) { + col[i] = matrix[i][column]; + } + return col; } @Override - public double[] getColumnAsDoubles( int index ) { - return this.matrix.getColumn( index ); + public double[] getColumnAsDoubles( int column ) { + if ( column < 0 || column >= columns() ) { + throw new IndexOutOfBoundsException( "Column index out of bounds: " + column ); + } + double[] col = new double[rows()]; + for ( int i = 0; i < col.length; i++ ) { + col[i] = matrix[i][column]; + } + return col; } @Override @@ -431,11 +444,10 @@ public ExpressionDataDoubleMatrix sliceColumns( List bioMaterials, } @Override - public Double[][] getRawMatrix() { - Double[][] dMatrix = new Double[matrix.rows()][matrix.columns()]; - for ( int i = 0; i < matrix.rows(); i++ ) { - // FIXME: getRowObj() is not efficient - dMatrix[i] = ArrayUtils.toObject( matrix.getRow( i ) ); + public Double[][] getMatrix() { + Double[][] dMatrix = new Double[matrix.length][]; + for ( int i = 0; i < matrix.length; i++ ) { + dMatrix[i] = ArrayUtils.toObject( matrix[i] ); } return dMatrix; } @@ -443,10 +455,10 @@ public Double[][] getRawMatrix() { /** * Obtain the raw matrix without boxing. * - * @see #getRawMatrix() + * @see #getMatrix() */ - public double[][] getRawMatrixAsDoubles() { - return matrix.getRawMatrix(); + public double[][] getMatrixAsDoubles() { + return matrix; } @Override @@ -460,8 +472,7 @@ public double[] getRowAsDoubles( CompositeSequence designElement ) { @Override public Double[] getRow( int index ) { - // FIXME: DoubleMatrix.getRowObj is not efficient - return ArrayUtils.toObject( matrix.getRow( index ) ); + return ArrayUtils.toObject( matrix[index] ); } @Override @@ -472,55 +483,46 @@ public ExpressionDataDoubleMatrix sliceRows( List designEleme @Override public double[] getRowAsDoubles( int index ) { - return matrix.getRow( index ); + return matrix[index]; } @Override public boolean hasMissingValues() { - if ( hasMissingValues == null ) { - hasMissingValues = computeHasMissingValues( matrix ); - } return hasMissingValues; } - private boolean computeHasMissingValues( DoubleMatrix matrix ) { - int rows = matrix.rows(); - int cols = matrix.columns(); - for ( int i = 0; i < rows; i++ ) { - for ( int j = 0; j < cols; j++ ) { - if ( Double.isNaN( matrix.get( i, j ) ) ) - return true; - } - } - return false; - } - /** - * @deprecated modifying the matrix directly is not recommended, make a copy instead. + * Convert this matrix to a {@link DoubleMatrix} object. + *

+ * This always returns a copy. */ - @Deprecated - public void set( int row, int column, @Nullable Double value ) { - if ( value == null || value.isNaN() ) { - matrix.set( row, column, Double.NaN ); - hasMissingValues = true; - } else { - matrix.set( row, column, value ); - // invalidate missing values cache - hasMissingValues = null; - } - // invalidate ranks - this.ranks = null; + public DoubleMatrix asDoubleMatrix() { + DenseDoubleMatrix mat = new DenseDoubleMatrix<>( matrix ); + mat.setRowNames( getDesignElements() ); + mat.setColumnNames( getBioMaterials() ); + return mat; } - public DoubleMatrix getMatrix() { - return matrix; + /** + * Convert this matrix to a Colt {@link DoubleMatrix2D} object. + *

+ * This always returns a copy. + */ + public DoubleMatrix2D asDoubleMatrix2D() { + return new DenseDoubleMatrix2D( matrix ); } /** * Create a copy of this matrix with the given data matrix. */ public ExpressionDataDoubleMatrix withMatrix( DoubleMatrix matrix ) { - return new ExpressionDataDoubleMatrix( this, matrix ); + Assert.isTrue( getDesignElements().equals( matrix.getRowNames() ) && getBioMaterials().equals( matrix.getColNames() ), + "New matrix must have matching design elements and biomaterials." ); + return new ExpressionDataDoubleMatrix( this, matrix.getRawMatrix() ); + } + + public ExpressionDataDoubleMatrix withMatrix( DoubleMatrix2D matrix ) { + return new ExpressionDataDoubleMatrix( this, matrix.toArray() ); } /** @@ -530,6 +532,16 @@ public ExpressionDataDoubleMatrix withMatrix( DoubleMatrix matrix, Map quantitationTypes ) { + Assert.isTrue( getDesignElements().equals( matrix.getRowNames() ) && getBioMaterials().equals( matrix.getColNames() ), + "New matrix must have matching design elements and biomaterials." ); + return withMatrix( matrix.getRawMatrix(), quantitationTypes ); + } + + public ExpressionDataDoubleMatrix withMatrix( DoubleMatrix2D matrix, Map quantitationTypes ) { + return new ExpressionDataDoubleMatrix( this, matrix.toArray(), quantitationTypes ); + } + + public ExpressionDataDoubleMatrix withMatrix( double[][] matrix, Map quantitationTypes ) { return new ExpressionDataDoubleMatrix( this, matrix, quantitationTypes ); } @@ -596,40 +608,31 @@ public ExpressionDataDoubleMatrix withNumberOfCells( @Nullable int[][] numberOfC /** * @return The expression level ranks (based on mean signal intensity in the vectors) */ - public Map getRanksByMean() { + public double[] getRanksByMean() { if ( this.ranks == null ) { this.ranks = computeRanksByMean( this.matrix ); } return this.ranks; } - private Map computeRanksByMean( DoubleMatrix matrix ) { - DoubleArrayList means = new DoubleArrayList( matrix.rows() ); - for ( int i = 0; i < matrix.rows(); i++ ) { - means.add( DescriptiveWithMissing.mean( matrix.getRowArrayList( i ) ) ); + private double[] computeRanksByMean( double[][] matrix ) { + int numRows = rows(); + DoubleArrayList means = new DoubleArrayList( numRows ); + for ( int i = 0; i < numRows; i++ ) { + means.add( DescriptiveWithMissing.mean( new DoubleArrayList( matrix[i] ) ) ); } DoubleArrayList ranks = Rank.rankTransform( means ); - Map rankMap = new HashMap<>( matrix.rows() ); - for ( int i = 0; i < matrix.rows(); i++ ) { - rankMap.put( matrix.getRowName( i ), ranks.get( i ) / matrix.rows() ); + double[] ranksAsDoubles = ranks.elements(); + // convert to [0, 1] range + for ( int i = 0; i < ranksAsDoubles.length; i++ ) { + ranksAsDoubles[i] = ranksAsDoubles[i] / numRows; } - return rankMap; - } - - public List getRowNames() { - return matrix.getRowNames(); - } - - @Deprecated - public void set( CompositeSequence designElement, BioAssay bioAssay, Double value ) { - int row = this.getRowIndex( designElement ); - int column = this.getColumnIndex( bioAssay ); - set( row, column, value ); + return ranksAsDoubles; } @Override protected String format( int row, int column ) { - return format( matrix.get( row, column ) ); + return format( matrix[row][column] ); } /** @@ -637,68 +640,22 @@ protected String format( int row, int column ) { * * @return DoubleMatrixNamed */ - private DoubleMatrix createMatrix( - Collection vectors ) { - - int numRows = rows(); - int numColumns = columns(); - - DoubleMatrix mat = new DenseDoubleMatrix<>( numRows, numColumns ); - - for ( int j = 0; j < mat.columns(); j++ ) { - mat.addColumnName( this.getBioMaterialForColumn( j ) ); + private double[][] createMatrix( List vectors ) { + double[][] mat = new double[rows()][columns()]; + for ( double[] doubles : mat ) { + Arrays.fill( doubles, Double.NaN ); } - - // initialize the matrix to -Infinity; this marks values that are not yet initialized. - for ( int i = 0; i < mat.rows(); i++ ) { - for ( int j = 0; j < mat.columns(); j++ ) { - mat.set( i, j, Double.NEGATIVE_INFINITY ); - } - } - - Map rowNames = new TreeMap<>(); - for ( BulkExpressionDataVector vector : vectors ) { + for ( int i = 0; i < vectors.size(); i++ ) { + BulkExpressionDataVector vector = vectors.get( i ); BioAssayDimension dimension = vector.getBioAssayDimension(); - - CompositeSequence designElement = vector.getDesignElement(); - assert designElement != null : "No design element for " + vector; - - int rowIndex = getRowIndex( designElement ); - assert rowIndex != -1; - - rowNames.put( rowIndex, designElement ); - double[] vals = vector.getDataAsDoubles(); - - Collection bioAssays = dimension.getBioAssays(); - if ( bioAssays.size() != vals.length ) - throw new IllegalStateException( - "Mismatch: " + vals.length + " values in vector ( " + vector.getData().length + " bytes) for " - + designElement + " got " + bioAssays.size() + " bioassays in the bioAssayDimension" ); - - Iterator it = bioAssays.iterator(); - - this.setMatBioAssayValues( mat, rowIndex, ArrayUtils.toObject( vals ), bioAssays, it ); - } - - /* - * Note: these row names aren't that important unless we use the bare matrix. - */ - for ( int i = 0; i < mat.rows(); i++ ) { - mat.addRowName( rowNames.get( i ) ); - } - assert mat.getRowNames().size() == mat.rows(); - - // fill in remaining missing values. - for ( int i = 0; i < mat.rows(); i++ ) { - for ( int j = 0; j < mat.columns(); j++ ) { - if ( mat.get( i, j ) == Double.NEGATIVE_INFINITY ) { - // log.debug( "Missing value at " + i + " " + j ); - mat.set( i, j, Double.NaN ); - } + List bioAssays = dimension.getBioAssays(); + for ( int j = 0; j < bioAssays.size(); j++ ) { + int column = getColumnIndex( bioAssays.get( j ) ); + mat[i][column] = vals[j]; } } - ExpressionDataDoubleMatrix.log.debug( "Created a " + mat.rows() + " x " + mat.columns() + " matrix" ); + ExpressionDataDoubleMatrix.log.debug( "Created a " + rows() + " x " + columns() + " double matrix." ); return mat; } @@ -717,13 +674,13 @@ private static int[][] getNumberOfCellsFromVectors( List void setMatBioAssayValues( AbstractMatrix mat, Integer rowIndex, V[] vals, - Collection bioAssays, Iterator it ) { - for ( int j = 0; j < bioAssays.size(); j++ ) { - BioAssay bioAssay = it.next(); - int column = getColumnIndex( bioAssay ); - assert column != -1; - mat.set( rowIndex, column, vals[j] ); + private static boolean computeHasMissingValues( double[][] matrix ) { + for ( double[] doubles : matrix ) { + for ( double d : doubles ) { + if ( Double.isNaN( d ) ) + return true; + } } + return false; } } diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataIntegerMatrix.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataIntegerMatrix.java index 079a374141..9ddbf0a777 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataIntegerMatrix.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataIntegerMatrix.java @@ -1,65 +1,55 @@ package ubic.gemma.core.datastructure.matrix; -import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import ubic.basecode.dataStructure.matrix.AbstractMatrix; -import ubic.basecode.dataStructure.matrix.IntegerMatrix; -import ubic.gemma.model.common.quantitationtype.PrimitiveType; import ubic.gemma.model.expression.bioAssay.BioAssay; import ubic.gemma.model.expression.bioAssayData.BioAssayDimension; import ubic.gemma.model.expression.bioAssayData.BulkExpressionDataVector; -import ubic.gemma.model.expression.bioAssayData.DesignElementDataVector; import ubic.gemma.model.expression.biomaterial.BioMaterial; import ubic.gemma.model.expression.designElement.CompositeSequence; import ubic.gemma.model.expression.experiment.ExpressionExperiment; -import java.util.*; +import java.util.Collection; +import java.util.List; /** * Warning, not fully tested. * * @author pavlidis */ -@SuppressWarnings("unused") // Possible external use public class ExpressionDataIntegerMatrix extends AbstractMultiAssayExpressionDataMatrix { private static final Log log = LogFactory.getLog( ExpressionDataIntegerMatrix.class.getName() ); - private final IntegerMatrix matrix; + private final Integer[][] matrix; + private final boolean hasMissingValues; public ExpressionDataIntegerMatrix( ExpressionExperiment ee, Collection vectors ) { super( ee ); - for ( DesignElementDataVector dedv : vectors ) { - if ( !dedv.getQuantitationType().getRepresentation().equals( PrimitiveType.INT ) ) { - throw new IllegalStateException( "Cannot convert non-integer quantitation types into int matrix" ); - } - } this.matrix = this.createMatrix( selectVectors( vectors ) ); - } - - @Override - public int columns() { - return matrix.columns(); + this.hasMissingValues = checkMissingValues( this.matrix ); } @Override public Integer get( int row, int column ) { - return matrix.get( row, column ); + return matrix[row][column]; } @Override - public Integer[] getColumn( int index ) { - return this.matrix.getColumn( index ); + public Integer[] getColumn( int column ) { + if ( column < 0 || column >= this.columns() ) { + throw new IndexOutOfBoundsException( "Column index " + column + " is out of bounds. Matrix has " + this.columns() + " columns." ); + } + Integer[] result = new Integer[rows()]; + for ( int i = 0; i < result.length; i++ ) { + result[i] = matrix[i][column]; + } + return result; } @Override - public Integer[][] getRawMatrix() { - Integer[][] res = new Integer[this.rows()][]; - for ( int i = 0; i < this.rows(); i++ ) { - res[i] = this.matrix.getRow( i ); - } - return res; + public Integer[][] getMatrix() { + return matrix; } @Override @@ -74,7 +64,7 @@ public ExpressionDataIntegerMatrix sliceColumns( List bioMaterials, @Override public Integer[] getRow( int index ) { - return this.matrix.getRow( index ); + return this.matrix[index]; } @Override @@ -84,24 +74,12 @@ public ExpressionDataMatrix sliceRows( List designEl @Override public boolean hasMissingValues() { - for ( int i = 0; i < matrix.rows(); i++ ) { - for ( int j = 0; j < matrix.columns(); j++ ) { - // only null values count as missing since there's no NAN for Integers. - if ( matrix.get( i, j ) == null ) - return true; - } - } - return false; - } - - public Integer get( CompositeSequence designElement, BioMaterial bioMaterial ) { - return this.matrix.get( matrix.getRowIndexByName( designElement ), - matrix.getColIndexByName( this.getColumnIndex( bioMaterial ) ) ); + return hasMissingValues; } @Override protected String format( int row, int column ) { - return String.valueOf( matrix.get( row, column ) ); + return matrix[row][column] != null ? String.valueOf( matrix[row][column] ) : ""; } /** @@ -109,63 +87,39 @@ protected String format( int row, int column ) { * * @return DoubleMatrixNamed */ - private IntegerMatrix createMatrix( - List vectors ) { - + private Integer[][] createMatrix( List vectors ) { int numRows = rows(); int numCols = columns(); - IntegerMatrix mat = new IntegerMatrix<>( numRows, numCols ); - - for ( int j = 0; j < mat.columns(); j++ ) { - mat.addColumnName( j ); - } - - // initialize the matrix to 0 - for ( int i = 0; i < mat.rows(); i++ ) { - for ( int j = 0; j < mat.columns(); j++ ) { - mat.set( i, j, 0 ); - } - } - - Map rowNames = new TreeMap<>(); - for ( BulkExpressionDataVector vector : vectors ) { - - CompositeSequence designElement = vector.getDesignElement(); - assert designElement != null : "No design element for " + vector; - - int rowIndex = this.getRowIndex( designElement ); - assert rowIndex != -1; - - rowNames.put( rowIndex, designElement ); + // missing values will be filled with NaNs + Integer[][] mat = new Integer[numRows][numCols]; + for ( int i = 0; i < vectors.size(); i++ ) { + BulkExpressionDataVector vector = vectors.get( i ); int[] vals = vector.getDataAsInts(); - BioAssayDimension dimension = vector.getBioAssayDimension(); - Collection bioAssays = dimension.getBioAssays(); + List bioAssays = dimension.getBioAssays(); assert bioAssays.size() == vals.length : "Expected " + vals.length + " got " + bioAssays.size(); - - Iterator it = bioAssays.iterator(); - - this.setMatBioAssayValues( mat, rowIndex, ArrayUtils.toObject( vals ), bioAssays, it ); - - } - - for ( int i = 0; i < mat.rows(); i++ ) { - mat.addRowName( rowNames.get( i ) ); + for ( int j = 0; j < bioAssays.size(); j++ ) { + BioAssay bioAssay = bioAssays.get( j ); + int column = getColumnIndex( bioAssay ); + assert column != -1; + mat[i][column] = vals[j]; + } } - ExpressionDataIntegerMatrix.log.debug( "Created a " + mat.rows() + " x " + mat.columns() + " matrix" ); + ExpressionDataIntegerMatrix.log.debug( "Created a " + rows() + " x " + columns() + " matrix" ); return mat; } - private void setMatBioAssayValues( AbstractMatrix mat, Integer rowIndex, V[] vals, - Collection bioAssays, Iterator it ) { - for ( int j = 0; j < bioAssays.size(); j++ ) { - BioAssay bioAssay = it.next(); - int column = getColumnIndex( bioAssay ); - assert column != -1; - mat.set( rowIndex, column, vals[j] ); + private boolean checkMissingValues( Integer[][] mat ) { + for ( Integer[] vec : mat ) { + for ( Integer val : vec ) { + if ( val == null ) { + return true; + } + } } + return false; } } diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataMatrix.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataMatrix.java index f6ac244a34..28935940a2 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataMatrix.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataMatrix.java @@ -136,5 +136,6 @@ public interface ExpressionDataMatrix { * * @throws IndexOutOfBoundsException if either the row or column is outside the matrix bounds */ + @Nullable T get( int row, int column ); } diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataMatrixRowElement.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataMatrixRowElement.java index 87134773c7..4513732ad7 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataMatrixRowElement.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataMatrixRowElement.java @@ -26,6 +26,7 @@ * * @author pavlidis */ +@Deprecated public class ExpressionDataMatrixRowElement implements Comparable { private final CompositeSequence designElement; diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataPrimitiveDoubleMatrix.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataPrimitiveDoubleMatrix.java index d744a6ff8a..16856ad17e 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataPrimitiveDoubleMatrix.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataPrimitiveDoubleMatrix.java @@ -8,24 +8,28 @@ * Interface for matrices that can provide unboxed doubles. *

* Use the methods in this interface to avoid the overhead of boxing and unboxing {@link Double}. + * * @author poirigui */ public interface ExpressionDataPrimitiveDoubleMatrix extends ExpressionDataMatrix { /** * Retrieve the value at the given row and column without boxing. + * * @see #get(int, int) */ double getAsDouble( int row, int column ); /** * Retrieve a row without boxing. + * * @see #getRow(int) */ double[] getRowAsDoubles( int index ); /** * Retrieve the row for the given design element without boxing. + * * @see #getRow(CompositeSequence) */ @Nullable @@ -33,6 +37,7 @@ public interface ExpressionDataPrimitiveDoubleMatrix extends ExpressionDataMatri /** * Retrieve the given column without boxing. + * * @see #getColumn(int) */ double[] getColumnAsDoubles( int column ); diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataPrimitiveIntMatrix.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataPrimitiveIntMatrix.java index 391a04c70c..fd52dda5e0 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataPrimitiveIntMatrix.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/ExpressionDataPrimitiveIntMatrix.java @@ -8,18 +8,21 @@ public interface ExpressionDataPrimitiveIntMatrix extends ExpressionDataMatrix { private static final Log log = LogFactory.getLog( ExpressionDataStringMatrix.class.getName() ); - private final StringMatrix matrix; + + private final String[][] matrix; + private final boolean hasMissingValues; public ExpressionDataStringMatrix( ExpressionExperiment ee, Collection vectors ) { super( ee ); - this.matrix = vectorsToMatrix( selectVectors( vectors ) ); + List vectors1 = selectVectors( vectors ); + if ( vectors1.isEmpty() ) { + throw new IllegalArgumentException( "No vectors!" ); + } + this.matrix = this.createMatrix( vectors1 ); + this.hasMissingValues = checkMissingValues( this.matrix ); } + @Nullable @Override public String get( int row, int column ) { - return matrix.get( row, column ); + return matrix[row][column]; } @Override public String[] getColumn( int index ) { - return this.matrix.getColumn( index ); + if ( index < 0 || index >= columns() ) { + throw new IndexOutOfBoundsException( "Column index " + index + " is out of bounds." ); + } + String[] col = new String[rows()]; + for ( int i = 0; i < col.length; i++ ) { + col[i] = matrix[i][index]; + } + return col; } @Override - public String[][] getRawMatrix() { - String[][] res = new String[this.rows()][]; - for ( int i = 0; i < this.rows(); i++ ) { - res[i] = this.matrix.getRow( i ); - } - return res; + public String[][] getMatrix() { + return matrix; } @Override @@ -76,7 +86,7 @@ public ExpressionDataStringMatrix sliceColumns( List bioMaterials, @Override public String[] getRow( int index ) { - return matrix.getRow( index ); + return matrix[index]; } @Override @@ -86,79 +96,47 @@ public ExpressionDataMatrix sliceRows( List designEle @Override public boolean hasMissingValues() { - for ( int i = 0; i < matrix.rows(); i++ ) { - for ( int j = 0; j < matrix.columns(); j++ ) { - // Note that blank strings don't count as missing. - if ( matrix.get( i, j ) == null ) - return true; - } - } - return false; + return hasMissingValues; } @Override protected String format( int row, int column ) { - return matrix.get( row, column ); + return matrix[row][column]; } - private StringMatrix vectorsToMatrix( Collection vectors ) { - if ( vectors.isEmpty() ) { - throw new IllegalArgumentException( "No vectors!" ); - } - - return this.createMatrix( vectors ); - } - - private StringMatrix createMatrix( Collection vectors ) { - + private String[][] createMatrix( List vectors ) { int numRows = rows(); int numCols = columns(); - StringMatrix mat = new StringMatrix<>( numRows, numCols ); - - for ( int j = 0; j < mat.columns(); j++ ) { - mat.addColumnName( j ); - } - - // initialize the matrix to ""; - for ( int i = 0; i < mat.rows(); i++ ) { - for ( int j = 0; j < mat.columns(); j++ ) { - mat.set( i, j, "" ); - } - } - - for ( BulkExpressionDataVector vector : vectors ) { + // missing values will be filled with nulls + String[][] mat = new String[numRows][numCols]; + for ( int i = 0; i < vectors.size(); i++ ) { + BulkExpressionDataVector vector = vectors.get( i ); CompositeSequence designElement = vector.getDesignElement(); - assert designElement != null : "No designelement for " + vector; - int rowIndex = getRowIndex( designElement ); assert rowIndex != -1; - - mat.addRowName( rowIndex ); - String[] vals = vector.getDataAsStrings(); - - BioAssayDimension dimension = vector.getBioAssayDimension(); - Collection bioAssays = dimension.getBioAssays(); - assert bioAssays.size() == vals.length : "Expected " + vals.length + " got " + bioAssays.size(); - - Iterator it = bioAssays.iterator(); - - for ( int j = 0; j < bioAssays.size(); j++ ) { - - BioAssay bioAssay = it.next(); + for ( int j = 0; j < vector.getBioAssayDimension().getBioAssays().size(); j++ ) { + BioAssay bioAssay = vector.getBioAssayDimension().getBioAssays().get( j ); int column = getColumnIndex( bioAssay ); - - assert column != -1; - - mat.setByKeys( rowIndex, column, vals[j] ); + mat[i][column] = vals[j]; } - } - ExpressionDataStringMatrix.log.debug( "Created a " + mat.rows() + " x " + mat.columns() + " matrix" ); + ExpressionDataStringMatrix.log.debug( "Created a " + rows() + " x " + columns() + " string matrix." ); return mat; } + + private static boolean checkMissingValues( String[][] matrix ) { + for ( String[] strings : matrix ) { + for ( String string : strings ) { + if ( string == null || string.isEmpty() ) { + return true; + } + } + } + return false; + } } diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/MaskedExpressionDataMatrix.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/MaskedExpressionDataMatrix.java index 547e44c6c5..c974836775 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/MaskedExpressionDataMatrix.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/MaskedExpressionDataMatrix.java @@ -19,8 +19,9 @@ public class MaskedExpressionDataMatrix extends AbstractExpressionDataMatrix< /** * Mask individual elements of the given matrix. - * @see MatrixMask#maskElements(int, int, boolean[][]) + * * @param maskedValue value to use for masked elements, usually {@code null} + * @see MatrixMask#maskElements(int, int, boolean[][]) */ public static MaskedExpressionDataMatrix maskElements( ExpressionDataMatrix matrix, boolean[][] mask, @Nullable T maskedValue ) { return new MaskedExpressionDataMatrix<>( matrix, MatrixMask.maskElements( matrix.rows(), matrix.columns(), mask ), maskedValue ); @@ -31,6 +32,7 @@ public static MaskedExpressionDataMatrix maskElements( ExpressionDataMatr *

* This uses a sparse representation internally, which is more efficient for large matrices with few masked * elements. + * * @see MatrixMask#maskElements(int, int, int[], int[]) */ public static MaskedExpressionDataMatrix maskElements( ExpressionDataMatrix matrix, int[] i, int[] j, @Nullable T maskedValue ) { @@ -39,6 +41,7 @@ public static MaskedExpressionDataMatrix maskElements( ExpressionDataMatr /** * Mask whole rows of the given matrix. + * * @see MatrixMask#maskRows(int, int, boolean[]) */ public static MaskedExpressionDataMatrix maskRows( ExpressionDataMatrix matrix, boolean[] rowMask, @Nullable T maskedValue ) { @@ -47,6 +50,7 @@ public static MaskedExpressionDataMatrix maskRows( ExpressionDataMatrix MaskedExpressionDataMatrix maskColumns( ExpressionDataMatrix matrix, boolean[] columnMask, @Nullable T maskedValue ) { diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/MatrixMask.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/MatrixMask.java index 8f166e04ce..d68a89916b 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/MatrixMask.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/MatrixMask.java @@ -135,10 +135,7 @@ public boolean isMasked( int row, int column ) { if ( mask != null && mask[row][column] ^ inverted ) { return true; } - if ( sparseMask != null && ( Arrays.binarySearch( sparseMask, row * cols + column ) >= 0 ) ^ inverted ) { - return true; - } - return false; + return sparseMask != null && ( Arrays.binarySearch( sparseMask, row * cols + column ) >= 0 ) ^ inverted; } /** diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/MultiAssayBulkExpressionDataMatrix.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/MultiAssayBulkExpressionDataMatrix.java index e0580731df..4ff660159d 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/MultiAssayBulkExpressionDataMatrix.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/MultiAssayBulkExpressionDataMatrix.java @@ -109,15 +109,6 @@ static MultiAssayBulkExpressionDataMatrix getMatrix( Collection populateMissingValueInfo( Collection dmatrix = matrix.asDoubleMatrix(); + int rows = dmatrix.rows(); + int columns = dmatrix.columns(); double log2 = Math.log( LOGARITHM_BASE ); - for ( ExpressionDataMatrixRowElement el : matrix.getRowElements() ) { - CompositeSequence del = el.getDesignElement(); - for ( int i = 0; i < columns; i++ ) { - BioAssay bm = matrix.getBioAssayForColumn( i ); - double valA = matrix.get( del, bm ); + for ( int i = 0; i < rows; i++ ) { + for ( int j = 0; j < columns; j++ ) { + double valA = matrix.getAsDouble( i, j ); if ( valA <= 0 ) { - matrix.set( del, bm, Double.NaN ); + dmatrix.set( i, j, Double.NaN ); } else { - matrix.set( del, bm, Math.log( valA ) / log2 ); + dmatrix.set( i, j, Math.log( valA ) / log2 ); } } } + return matrix.withMatrix( dmatrix ); } /** @@ -782,28 +790,30 @@ public static void logTransformMatrix( ExpressionDataDoubleMatrix matrix ) { * @param b matrix b * @throws IllegalArgumentException if the matrices are not column-conformant. */ - private void subtractMatrices( ExpressionDataDoubleMatrix a, ExpressionDataDoubleMatrix b ) { + private ExpressionDataDoubleMatrix subtractMatrices( ExpressionDataDoubleMatrix a, ExpressionDataDoubleMatrix b ) { // checkConformant( a, b ); if ( a.columns() != b.columns() ) throw new IllegalArgumentException( "Unequal column counts: " + a.columns() + " != " + b.columns() ); + DoubleMatrix2D dmatrix = a.asDoubleMatrix2D(); int columns = a.columns(); - for ( ExpressionDataMatrixRowElement el : a.getRowElements() ) { - int rowNum = el.getIndex(); - CompositeSequence del = el.getDesignElement(); - - if ( b.getRowAsDoubles( del ) == null ) { + List designElements = a.getDesignElements(); + for ( int i = 0; i < designElements.size(); i++ ) { + CompositeSequence del = designElements.get( i ); + if ( b.getRowIndex( del ) == -1 ) { log.warn( "Matrix 'b' is missing a row for " + del + ", it will not be subtracted" ); continue; } - - for ( int i = 0; i < columns; i++ ) { + for ( int j = 0; j < columns; j++ ) { BioAssay assay = a.getBioAssayForColumn( i ); - double valA = a.get( del, assay ); - double valB = b.get( del, assay ); - a.set( rowNum, i, valA - valB ); + double valA = a.get( i, j ); + Double valB = b.get( del, assay ); + if ( valB != null ) { + dmatrix.set( i, j, valA - valB ); + } } } + return a.withMatrix( dmatrix ); } @@ -819,25 +829,29 @@ private void subtractMatrices( ExpressionDataDoubleMatrix a, ExpressionDataDoubl * @param b matrix b * @throws IllegalArgumentException if the matrices are not column-conformant. */ - public void addMatrices( ExpressionDataDoubleMatrix a, ExpressionDataDoubleMatrix b ) { + public ExpressionDataDoubleMatrix addMatrices( ExpressionDataDoubleMatrix a, ExpressionDataDoubleMatrix b ) { // checkConformant( a, b ); if ( a.columns() != b.columns() ) throw new IllegalArgumentException( "Unequal column counts: " + a.columns() + " != " + b.columns() ); + DoubleMatrix2D dmatrix = new DenseDoubleMatrix2D( a.getMatrixAsDoubles() ); int columns = a.columns(); - for ( ExpressionDataMatrixRowElement el : a.getRowElements() ) { - CompositeSequence del = el.getDesignElement(); - - if ( b.getRowAsDoubles( del ) == null ) { + List designElements = a.getDesignElements(); + for ( int i = 0; i < designElements.size(); i++ ) { + CompositeSequence del = designElements.get( i ); + if ( b.getRowIndex( del ) == -1 ) { log.warn( "Matrix 'b' is missing a row for " + del + ", this row will not be added" ); continue; } - for ( int i = 0; i < columns; i++ ) { - BioAssay bm = a.getBioAssayForColumn( i ); - double valA = a.get( del, bm ); - double valB = b.get( del, bm ); - a.set( del, bm, valA + valB ); + for ( int j = 0; j < columns; j++ ) { + BioAssay bm = a.getBioAssayForColumn( j ); + double valA = a.get( i, j ); + Double valB = b.get( del, bm ); + if ( valB != null ) { + dmatrix.set( i, j, valA + valB ); + } } } + return a.withMatrix( dmatrix ); } /** @@ -847,18 +861,18 @@ public void addMatrices( ExpressionDataDoubleMatrix a, ExpressionDataDoubleMatri * @param dividend dividend * @throws IllegalArgumentException if dividend == 0. */ - public void scalarDivideMatrix( ExpressionDataDoubleMatrix matrix, double dividend ) { + public ExpressionDataDoubleMatrix scalarDivideMatrix( ExpressionDataDoubleMatrix matrix, double dividend ) { if ( dividend == 0 ) throw new IllegalArgumentException( "Can't divide by zero" ); + DoubleMatrix2D dmatrix = matrix.asDoubleMatrix2D(); int columns = matrix.columns(); - for ( ExpressionDataMatrixRowElement el : matrix.getRowElements() ) { - CompositeSequence del = el.getDesignElement(); - for ( int i = 0; i < columns; i++ ) { - BioAssay bm = matrix.getBioAssayForColumn( i ); - double valA = matrix.get( del, bm ); - matrix.set( del, bm, valA / dividend ); - + List designElements = matrix.getDesignElements(); + for ( int i = 0; i < designElements.size(); i++ ) { + for ( int j = 0; j < columns; j++ ) { + double valA = matrix.getAsDouble( i, j ); + dmatrix.set( i, j, valA / dividend ); } } + return matrix.withMatrix( dmatrix ); } /** diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/ExperimentalDesignWriter.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/ExperimentalDesignWriter.java index 9073a4198f..9ef3917120 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/ExperimentalDesignWriter.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/ExperimentalDesignWriter.java @@ -92,6 +92,7 @@ public ExperimentalDesignWriter( EntityUrlBuilder entityUrlBuilder, BuildInfo bu /** * Write the experimental design of the given {@link ExpressionExperiment} to the given {@link Writer}. + * * @see #write(ExpressionExperiment, Collection, boolean, Writer) */ public void write( ExpressionExperiment ee, Writer writer ) throws IOException { @@ -101,6 +102,7 @@ public void write( ExpressionExperiment ee, Writer writer ) throws IOException { /** * Write the experimental design of the given {@link ExpressionExperiment} to the given {@link Writer} for a given * collection of assays. + * * @param bioAssays assays to write, the order is defined by the order of their corresponding biomaterials * as per {@link BioMaterial#COMPARATOR}. * @param writeBaseHeader whether to write the base header (experiment URL, build info, etc.), see diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/ExpressionDataMatrixWriter.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/ExpressionDataMatrixWriter.java index 2002973887..abc3c688a2 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/ExpressionDataMatrixWriter.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/ExpressionDataMatrixWriter.java @@ -14,6 +14,7 @@ /** * Base interface for writing {@link ExpressionDataMatrix}. + * * @author poirigui */ public interface ExpressionDataMatrixWriter, VT extends DataVector> { @@ -34,16 +35,18 @@ public interface ExpressionDataMatrixWriter, V /** * Write the matrix to the given writer. + * * @param vectorType the type of vectors to write. This helps the writer determine how the {@link QuantitationType} - * of the matrix should be interpreted. + * of the matrix should be interpreted. * @return the number of vectors written * @throws UnsupportedOperationException if the matrix cannot be written to a text output (i.e. if this is a binary - * format). + * format). */ int write( T matrix, Class vectorType, Writer writer ) throws IOException, UnsupportedOperationException; /** * Write the matrix to the given output stream. + * * @return the number of vectors written */ default int write( T matrix, Class vectorType, OutputStream stream ) throws IOException { diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/ExpressionDataWriterUtils.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/ExpressionDataWriterUtils.java index 1b63f0db2b..5a03f30ef4 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/ExpressionDataWriterUtils.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/ExpressionDataWriterUtils.java @@ -45,6 +45,7 @@ /** * Utilities for writing expression data files. + * * @author keshav */ @SuppressWarnings({ "unused", "WeakerAccess" }) // Possible external use @@ -128,6 +129,7 @@ public static String formatQuantitationType( QuantitationType quantitationType, /** * Construct a sample name in case there is only one BioAssay attached to the corresponding BioMaterial. + * * @see #constructSampleName(BioMaterial, Collection, boolean, boolean, char) */ public static String constructSampleName( BioMaterial bm, BioAssay ba, boolean useIds, boolean useRawColumnNames ) { @@ -146,6 +148,7 @@ public static String constructAssaysName( Collection bas, boolean useI /** * Construct a BioAssay column name prefixed by the {@link BioMaterial} from which it originates. + * * @param bioMaterial the biomaterial * @param bioAssays the bioassay(s) associated to the biomaterial * @param useIds use biomaterial and bioassay IDs instead of names (or short names) @@ -186,6 +189,7 @@ private static String getBioAssayName( BioAssay ba, boolean useIds ) { /** * Construct an ExperimentalFactor column name. + * * @see StringUtil#makeNames(String) */ public static String constructExperimentalFactorName( ExperimentalFactor ef ) { @@ -194,6 +198,7 @@ public static String constructExperimentalFactorName( ExperimentalFactor ef ) { /** * Construct an ExperimentalFactor column names for a list of factors. + * * @see StringUtil#makeNames(String[], boolean) */ public static String[] constructExperimentalFactorNames( List factors ) { diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/JsonMatrixWriter.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/JsonMatrixWriter.java index bea2a5ee6b..f54b44779c 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/JsonMatrixWriter.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/JsonMatrixWriter.java @@ -11,6 +11,7 @@ /** * Writes a {@link BulkExpressionDataMatrix} to a JSON format. + * * @author paul */ public class JsonMatrixWriter implements BulkExpressionDataMatrixWriter { diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/MatrixWriter.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/MatrixWriter.java index 22955b39bd..4a605e6bda 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/MatrixWriter.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/MatrixWriter.java @@ -50,6 +50,7 @@ /** * Writes {@link BulkExpressionDataMatrix} to a tabular format. + * * @author pavlidis */ @Setter @@ -69,6 +70,7 @@ public class MatrixWriter implements BulkExpressionDataMatrixWriter { private boolean useBioAssayIds = false; /** * Do not make the column names R-friendly. + * * @see ubic.basecode.util.StringUtil#makeNames(String) */ private boolean useRawColumnNames = false; diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/MexMatrixBundler.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/MexMatrixBundler.java index fc41ec0940..e7df6b99a6 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/MexMatrixBundler.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/MexMatrixBundler.java @@ -17,6 +17,7 @@ /** * Bundles a directory containing MEX files into a TAR archive. + * * @author poirigui * @see MexMatrixWriter */ diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/MexMatrixWriter.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/MexMatrixWriter.java index de3a6812f7..137e4b3554 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/MexMatrixWriter.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/MexMatrixWriter.java @@ -47,6 +47,7 @@ * The data is written as a TAR archive containing the following entries for each bioassay: {@code {bioAssayName}/barcodes.tsv}, * {@code {bioAssayName}/features.tsv}, {@code {bioAssayName}/matrix.mtx}. If using the directory output, individual * files are compressed and will have a {@code .gz} extension. + * * @author poirigui */ @CommonsLog diff --git a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/TabularMatrixWriter.java b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/TabularMatrixWriter.java index 8fe3fc7e8d..bbd85270e9 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/TabularMatrixWriter.java +++ b/gemma-core/src/main/java/ubic/gemma/core/datastructure/matrix/io/TabularMatrixWriter.java @@ -45,6 +45,7 @@ * - sample_name * - cell_id * - value + * * @author poirigui */ @Setter diff --git a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/DataUpdaterImpl.java b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/DataUpdaterImpl.java index 667b56190a..4c854c401f 100644 --- a/gemma-core/src/main/java/ubic/gemma/core/loader/expression/DataUpdaterImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/core/loader/expression/DataUpdaterImpl.java @@ -317,13 +317,13 @@ public void log2cpmFromCounts( ExpressionExperiment ee, QuantitationType qt ) { ee = experimentService.thawLite( ee ); // so updated QT is attached. QuantitationType log2cpmQt = this.makelog2cpmQt(); - DoubleMatrix1D librarySize = MatrixStats.colSums( countMatrix.getMatrix() ); + DoubleMatrix1D librarySize = MatrixStats.colSums( countMatrix.asDoubleMatrix() ); /* FIXME: filter out rows from the count matrix that have all zero counts */ DoubleMatrix log2cpmMatrix = MatrixStats - .convertToLog2Cpm( countMatrix.getMatrix(), librarySize ); + .convertToLog2Cpm( countMatrix.asDoubleMatrix(), librarySize ); ExpressionDataDoubleMatrix log2cpmEEMatrix = new ExpressionDataDoubleMatrix( ee, log2cpmMatrix, log2cpmQt ); diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/sampleCoexpression/SampleCoexpressionAnalysisServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/sampleCoexpression/SampleCoexpressionAnalysisServiceImpl.java index a1981aadac..eda2d453ff 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/sampleCoexpression/SampleCoexpressionAnalysisServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/analysis/expression/sampleCoexpression/SampleCoexpressionAnalysisServiceImpl.java @@ -276,7 +276,7 @@ private SampleCoexpressionMatrix getMatrix( ExpressionExperiment ee, boolean reg private DoubleMatrix dataToDoubleMat( ExpressionDataDoubleMatrix matrix ) { - DoubleMatrix transposeR = matrix.getMatrix().transpose(); + DoubleMatrix transposeR = matrix.asDoubleMatrix().transpose(); DoubleMatrix transpose = new DenseDoubleMatrix<>( transposeR.getRawMatrix() ); transpose.setColumnNames( transposeR.getColNames() ); @@ -402,22 +402,20 @@ private ExpressionDataDoubleMatrix regressionResiduals( ExpressionDataDoubleMatr DesignMatrix properDesignMatrix = new DesignMatrix( designMatrix, true ); ExpressionDataDoubleMatrix dmatrix = matrix.sliceColumns( samplesUsed, bad ); - DoubleMatrix namedMatrix = dmatrix.getMatrix(); + DoubleMatrix namedMatrix = dmatrix.asDoubleMatrix(); DoubleMatrix sNamedMatrix = DiffExAnalyzerUtils.makeDataMatrix( designMatrix, namedMatrix ); LeastSquaresFit fit = new LeastSquaresFit( properDesignMatrix, sNamedMatrix ); DoubleMatrix2D residuals = fit.getResiduals(); - DoubleMatrix f = new DenseDoubleMatrix<>( residuals.toArray() ); - f.setRowNames( dmatrix.getMatrix().getRowNames() ); - f.setColumnNames( dmatrix.getMatrix().getColNames() ); + DoubleMatrix2D f = residuals.copy(); DoubleArrayList rowmeans = MatrixRowStats.means( sNamedMatrix ); for ( int i = 0; i < f.rows(); i++ ) { double rowmean = rowmeans.get( i ); for ( int j = 0; j < f.columns(); j++ ) { - f.set( i, j, f.get( i, j ) + rowmean ); + f.setQuick( i, j, f.getQuick( i, j ) + rowmean ); } } return dmatrix.withMatrix( f ); diff --git a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorHelperServiceImpl.java b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorHelperServiceImpl.java index 0dae3fa5ca..73d032d956 100644 --- a/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorHelperServiceImpl.java +++ b/gemma-core/src/main/java/ubic/gemma/persistence/service/expression/bioAssayData/ProcessedExpressionDataVectorHelperServiceImpl.java @@ -27,11 +27,11 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.Assert; +import ubic.basecode.dataStructure.matrix.DoubleMatrix; import ubic.basecode.math.DescriptiveWithMissing; import ubic.basecode.math.Rank; import ubic.gemma.core.datastructure.matrix.ExpressionDataBooleanMatrix; import ubic.gemma.core.datastructure.matrix.ExpressionDataDoubleMatrix; -import ubic.gemma.core.datastructure.matrix.ExpressionDataMatrixRowElement; import ubic.gemma.core.datastructure.matrix.TwoChannelExpressionDataMatrixBuilder; import ubic.gemma.model.common.quantitationtype.QuantitationType; import ubic.gemma.model.expression.arrayDesign.ArrayDesign; @@ -237,7 +237,7 @@ private ExpressionDataDoubleMatrix loadIntensities( ExpressionExperiment ee, } ProcessedExpressionDataVectorHelperServiceImpl.log.info( "Masking ..." ); - this.maskMissingValues( intensities, missingValues ); + intensities = this.maskMissingValues( intensities, missingValues ); } else { ProcessedExpressionDataVectorHelperServiceImpl.log @@ -306,22 +306,19 @@ private DoubleArrayList getRanks( ExpressionDataDoubleMatrix intensities, ProcessedExpressionDataVectorHelperServiceImpl.log.debug( "Getting ranks" ); DoubleArrayList result = new DoubleArrayList( intensities.rows() ); - for ( ExpressionDataMatrixRowElement de : intensities.getRowElements() ) { - double[] rowObj = intensities.getRowAsDoubles( de.getDesignElement() ); - double valueForRank = Double.MIN_VALUE; - if ( rowObj != null ) { - DoubleArrayList row = new DoubleArrayList( rowObj ); - switch ( method ) { - case max: - valueForRank = DescriptiveWithMissing.max( row ); - break; - case mean: - valueForRank = DescriptiveWithMissing.mean( row ); - break; - default: - throw new UnsupportedOperationException(); - } - + for ( int i = 0; i < intensities.rows(); i++ ) { + double[] rowObj = intensities.getRowAsDoubles( i ); + double valueForRank; + DoubleArrayList row = new DoubleArrayList( rowObj ); + switch ( method ) { + case max: + valueForRank = DescriptiveWithMissing.max( row ); + break; + case mean: + valueForRank = DescriptiveWithMissing.mean( row ); + break; + default: + throw new UnsupportedOperationException(); } result.add( valueForRank ); } @@ -335,8 +332,8 @@ private DoubleArrayList getRanks( ExpressionDataDoubleMatrix intensities, * * @param inMatrix The matrix to be masked */ - private void maskMissingValues( ExpressionDataDoubleMatrix inMatrix, ExpressionDataBooleanMatrix missingValues ) { - maskMatrix( inMatrix, missingValues ); + private ExpressionDataDoubleMatrix maskMissingValues( ExpressionDataDoubleMatrix inMatrix, ExpressionDataBooleanMatrix missingValues ) { + return maskMatrix( inMatrix, missingValues ); } /** @@ -348,25 +345,30 @@ private void maskMissingValues( ExpressionDataDoubleMatrix inMatrix, ExpressionD * @param matrix matrix * @param mask if null, masking is not attempted. */ - private void maskMatrix( ExpressionDataDoubleMatrix matrix, @Nullable ExpressionDataBooleanMatrix mask ) { - if ( mask == null ) return; + private ExpressionDataDoubleMatrix maskMatrix( ExpressionDataDoubleMatrix matrix, @Nullable ExpressionDataBooleanMatrix mask ) { + if ( mask == null ) return matrix; + DoubleMatrix dmatrix = matrix.asDoubleMatrix(); // checkConformant( a, b ); - if ( matrix.columns() != mask.columns() ) + if ( dmatrix.columns() != mask.columns() ) throw new IllegalArgumentException( "Unequal column counts: " + matrix.columns() + " != " + mask.columns() ); - int columns = matrix.columns(); - for ( ExpressionDataMatrixRowElement el : matrix.getRowElements() ) { - CompositeSequence del = el.getDesignElement(); - if ( mask.getRow( del ) == null ) { + int rows = dmatrix.rows(); + int columns = dmatrix.columns(); + for ( int i = 0; i < rows; i++ ) { + CompositeSequence del = matrix.getDesignElementForRow( i ); + if ( mask.getRowIndex( del ) == -1 ) { log.warn( "Mask Matrix is missing a row for " + del ); continue; } - for ( int i = 0; i < columns; i++ ) { - BioAssay bm = matrix.getBioAssayForColumn( i ); - boolean present = mask.get( del, bm ); - if ( !present ) { - matrix.set( del, bm, Double.NaN ); + for ( int j = 0; j < columns; j++ ) { + BioAssay bm = matrix.getBioAssayForColumn( j ); + // if mask says it's missing, set to NaN + // if mask is not defined for a particular element, ignore + Boolean present = mask.get( del, bm ); + if ( present != null && !present ) { + dmatrix.set( i, j, Double.NaN ); } } } + return matrix.withMatrix( dmatrix ); } } diff --git a/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/ExpressionDataSVDTest.java b/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/ExpressionDataSVDTest.java index 040c7593ae..9e938dd1aa 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/ExpressionDataSVDTest.java +++ b/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/ExpressionDataSVDTest.java @@ -173,7 +173,7 @@ public void testGetU() { public void testMatrixReconstruct() { ExpressionDataDoubleMatrix svdNormalize = svd.removeHighestComponents( 0 ); assertNotNull( svdNormalize ); - RegressionTesting.closeEnough( testData.getMatrix(), svdNormalize.getMatrix(), 0.001 ); + RegressionTesting.closeEnough( testData.asDoubleMatrix(), svdNormalize.asDoubleMatrix(), 0.001 ); } /* diff --git a/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/TwoChannelExpressionDataDoubleMatrixTest.java b/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/TwoChannelExpressionDataDoubleMatrixTest.java index f024bf4419..bf45f49226 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/TwoChannelExpressionDataDoubleMatrixTest.java +++ b/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/TwoChannelExpressionDataDoubleMatrixTest.java @@ -185,7 +185,7 @@ public void testConstructExpressionDataDoubleMatrix() { log.debug( aRow ); } - double[][] dMatrix = expressionDataDoubleMatrix.getRawMatrixAsDoubles(); + double[][] dMatrix = expressionDataDoubleMatrix.getMatrixAsDoubles(); assertEquals( 200, dMatrix.length ); assertEquals( 59, dMatrix[0].length ); diff --git a/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/TwoChannelMissingValuesTest.java b/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/TwoChannelMissingValuesTest.java index 50dde4271e..fa5086f3b3 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/TwoChannelMissingValuesTest.java +++ b/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/TwoChannelMissingValuesTest.java @@ -186,11 +186,11 @@ public void testMissingValueGSE11017() throws Exception { ExpressionDataBooleanMatrix missingValues = builder.getMissingValueData(); assertTrue( missingValues.getQuantitationTypes().iterator().next().getDescription() .contains( "signal threshold" ) ); - Boolean[][] mm = missingValues.getRawMatrix(); + Boolean[][] mm = missingValues.getMatrix(); boolean hasPresent = false; for ( Boolean[] aMm : mm ) { for ( Boolean anAMm : aMm ) { - if ( anAMm ) { + if ( anAMm != null && anAMm ) { hasPresent = true; break; } diff --git a/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/convert/QuantitationTypeConversionUtilsTest.java b/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/convert/QuantitationTypeConversionUtilsTest.java index 595892fddc..cfe6ee9f47 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/convert/QuantitationTypeConversionUtilsTest.java +++ b/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/convert/QuantitationTypeConversionUtilsTest.java @@ -2,6 +2,7 @@ import org.junit.Before; import org.junit.Test; +import ubic.basecode.dataStructure.matrix.DoubleMatrix; import ubic.gemma.core.analysis.preprocess.detect.InferredQuantitationMismatchException; import ubic.gemma.core.datastructure.matrix.ExpressionDataDoubleMatrix; import ubic.gemma.model.common.quantitationtype.*; @@ -45,7 +46,9 @@ public void setUp() { bad.setBioAssays( Collections.singletonList( ba ) ); ev.setBioAssayDimension( bad ); matrix = new ExpressionDataDoubleMatrix( null, Collections.singleton( ev ), Collections.singleton( qt ) ); - matrix.set( 0, 0, 4.0 ); + DoubleMatrix dmatrix = matrix.asDoubleMatrix(); + dmatrix.set( 0, 0, 4.0 ); + matrix = matrix.withMatrix( dmatrix ); RandomExpressionDataMatrixUtils.setSeed( 123L ); } @@ -62,7 +65,7 @@ public void testLinearConversion() throws QuantitationTypeConversionException { assertThat( ensureLog2Scale( matrix ) ) .satisfies( this::basicBasicLog2MatrixChecks ) .satisfies( m -> { - assertThat( m.getMatrix().get( 0, 0 ) ).isEqualTo( Math.log( 4.0 ) / Math.log( 2 ) ); + assertThat( m.asDoubleMatrix().get( 0, 0 ) ).isEqualTo( Math.log( 4.0 ) / Math.log( 2 ) ); } ); } @@ -72,7 +75,7 @@ public void testLog10Conversion() throws QuantitationTypeConversionException { assertThat( ensureLog2Scale( matrix ) ) .satisfies( this::basicBasicLog2MatrixChecks ) .satisfies( m -> { - assertThat( m.getMatrix().get( 0, 0 ) ).isEqualTo( 4.0 * Math.log( 10 ) / Math.log( 2 ), within( 1e-10 ) ); + assertThat( m.asDoubleMatrix().get( 0, 0 ) ).isEqualTo( 4.0 * Math.log( 10 ) / Math.log( 2 ), within( 1e-10 ) ); } ); } @@ -83,7 +86,7 @@ public void testCountConversion() throws QuantitationTypeConversionException { assertThat( ensureLog2Scale( matrix ) ) .satisfies( this::basicBasicLog2MatrixChecks ) .satisfies( m -> { - assertThat( m.getMatrix().get( 0, 0 ) ) + assertThat( m.asDoubleMatrix().get( 0, 0 ) ) .isEqualTo( Math.log( 1e6 * ( 4 + 0.5 ) / ( 4 + 1 ) ) / Math.log( 2 ), within( 1e-10 ) ); } ); } diff --git a/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/filter/RepetitiveValuesFilterTest.java b/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/filter/RepetitiveValuesFilterTest.java index 7782f6bdbb..34e5b0883a 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/filter/RepetitiveValuesFilterTest.java +++ b/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/filter/RepetitiveValuesFilterTest.java @@ -45,7 +45,7 @@ public void testFilterLog2cpmData() throws FilteringException { // fill a row with zeroes CompositeSequence deToDrop = countMatrix.getDesignElementForRow( 5 ); - DoubleMatrix cm = countMatrix.getMatrix().copy(); + DoubleMatrix cm = countMatrix.asDoubleMatrix(); for ( int j = 0; j < cm.columns(); j++ ) { cm.set( 5, j, 0.0 ); } @@ -61,7 +61,7 @@ public void testFilterLog2cpmData() throws FilteringException { } ); // calculate library sizes - DoubleMatrix1D librarySize = MatrixStats.colSums( new DenseDoubleMatrix<>( countMatrix.getRawMatrixAsDoubles() ) ); + DoubleMatrix1D librarySize = MatrixStats.colSums( new DenseDoubleMatrix<>( countMatrix.getMatrixAsDoubles() ) ); for ( int i = 0; i < librarySize.size(); i++ ) { countMatrix.getBioAssayDimension().getBioAssays().get( i ) .setSequenceReadCount( Math.round( librarySize.get( i ) ) ); @@ -73,7 +73,7 @@ public void testFilterLog2cpmData() throws FilteringException { log2cpmQt.setType( StandardQuantitationType.AMOUNT ); log2cpmQt.setScale( ScaleType.LOG2 ); log2cpmQt.setRepresentation( PrimitiveType.DOUBLE ); - DoubleMatrix log2cpmM = countMatrix.getMatrix().copy(); + DoubleMatrix log2cpmM = countMatrix.asDoubleMatrix(); for ( int i = 0; i < log2cpmM.rows(); i++ ) { for ( int j = 0; j < log2cpmM.columns(); j++ ) { @@ -113,7 +113,7 @@ public void testQuantileNormalizedData() throws FilteringException { // fill a row with zeroes CompositeSequence deToDrop = countMatrix.getDesignElementForRow( 5 ); - DoubleMatrix cm = countMatrix.getMatrix().copy(); + DoubleMatrix cm = countMatrix.asDoubleMatrix(); for ( int j = 0; j < cm.columns(); j++ ) { cm.set( 5, j, 0.0 ); } @@ -129,7 +129,7 @@ public void testQuantileNormalizedData() throws FilteringException { } ); // calculate library sizes - DoubleMatrix1D librarySize = MatrixStats.colSums( new DenseDoubleMatrix<>( countMatrix.getRawMatrixAsDoubles() ) ); + DoubleMatrix1D librarySize = MatrixStats.colSums( new DenseDoubleMatrix<>( countMatrix.getMatrixAsDoubles() ) ); for ( int i = 0; i < librarySize.size(); i++ ) { countMatrix.getBioAssayDimension().getBioAssays().get( i ) .setSequenceReadCount( Math.round( librarySize.get( i ) ) ); @@ -141,7 +141,7 @@ public void testQuantileNormalizedData() throws FilteringException { log2cpmQt.setType( StandardQuantitationType.AMOUNT ); log2cpmQt.setScale( ScaleType.LOG2 ); log2cpmQt.setRepresentation( PrimitiveType.DOUBLE ); - DoubleMatrix log2cpmM = countMatrix.getMatrix().copy(); + DoubleMatrix log2cpmM = countMatrix.asDoubleMatrix(); for ( int i = 0; i < log2cpmM.rows(); i++ ) { for ( int j = 0; j < log2cpmM.columns(); j++ ) { @@ -152,7 +152,7 @@ public void testQuantileNormalizedData() throws FilteringException { ExpressionDataDoubleMatrix log2cpmMatrix = new ExpressionDataDoubleMatrix( ee, log2cpmM, log2cpmQt ); DoubleMatrix normalizedLog2cpm = new QuantileNormalizer() - .normalize( log2cpmMatrix.getMatrix() ); + .normalize( log2cpmMatrix.asDoubleMatrix() ); QuantitationType normalizedQt = QuantitationType.Factory.newInstance( log2cpmQt ); normalizedQt.setIsNormalized( true ); @@ -189,7 +189,7 @@ public void testRepetitiveValuesWithNonZeroVariance() throws FilteringException // only fill half the row with zeroes CompositeSequence deToDrop = countMatrix.getDesignElementForRow( 5 ); - DoubleMatrix cm = countMatrix.getMatrix().copy(); + DoubleMatrix cm = countMatrix.asDoubleMatrix(); for ( int j = 0; j < 10; j++ ) { if ( j < 7 ) { cm.set( 5, j, 0.0 ); @@ -233,7 +233,7 @@ public void testRepetitiveValuesWithCountData() throws FilteringException { // only fill half the row with zeroes CompositeSequence deToDrop = countMatrix.getDesignElementForRow( 5 ); - DoubleMatrix cm = countMatrix.getMatrix().copy(); + DoubleMatrix cm = countMatrix.asDoubleMatrix(); for ( int j = 0; j < 10; j++ ) { if ( j < 8 ) { cm.set( 5, j, 0.0 ); @@ -277,7 +277,7 @@ public void testFilterWithNonLogScaleData() throws FilteringException { // only fill half the row with zeroes CompositeSequence deToDrop = countMatrix.getDesignElementForRow( 5 ); - DoubleMatrix cm = countMatrix.getMatrix().copy(); + DoubleMatrix cm = countMatrix.asDoubleMatrix(); for ( int j = 0; j < 10; j++ ) { if ( j < 8 ) { cm.set( 5, j, 0.0 ); diff --git a/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/filter/RowLevelFilterTest.java b/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/filter/RowLevelFilterTest.java index 1b55fb5152..07425be84b 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/filter/RowLevelFilterTest.java +++ b/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/filter/RowLevelFilterTest.java @@ -1,7 +1,9 @@ package ubic.gemma.core.analysis.preprocess.filter; +import cern.colt.matrix.DoubleMatrix2D; import org.junit.Before; import org.junit.Test; +import ubic.basecode.dataStructure.matrix.DoubleMatrix; import ubic.basecode.math.Constants; import ubic.gemma.core.datastructure.matrix.ExpressionDataDoubleMatrix; import ubic.gemma.model.expression.arrayDesign.ArrayDesign; @@ -54,9 +56,11 @@ public void testFilterWithLowCut() { @Test public void testFilterWithLowCutForDroppingZeros() { + DoubleMatrix dmatrix = matrix.asDoubleMatrix(); for ( int i = 0; i < 8; i++ ) { - matrix.set( 0, i, 0.0 ); + dmatrix.set( 0, i, 0.0 ); } + matrix = matrix.withMatrix( dmatrix ); RowLevelFilter filter = new RowLevelFilter( RowLevelFilter.Method.VAR ); filter.setLowCut( Constants.SMALLISH ); ExpressionDataDoubleMatrix filteredMatrix = filter.filter( matrix ); @@ -165,9 +169,11 @@ public void testFilterWithCustomValues() { @Test public void testFilterWithMissingValues() { + DoubleMatrix dmatrix = matrix.asDoubleMatrix(); for ( int j = 0; j < 8; j++ ) { - matrix.set( 0, j, Double.NaN ); + dmatrix.set( 0, j, Double.NaN ); } + matrix = matrix.withMatrix( dmatrix ); RowLevelFilter filter = new RowLevelFilter( RowLevelFilter.Method.VAR ); ExpressionDataDoubleMatrix filteredMatrix = filter.filter( matrix ); @@ -197,11 +203,13 @@ public void testFilterEmptyMatrix() { @Test public void testFilterMatrixFilledWithNaNs() { + DoubleMatrix2D dmatrix = matrix.asDoubleMatrix2D(); for ( int i = 0; i < 100; i++ ) { for ( int j = 0; j < 8; j++ ) { - matrix.set( i, j, Double.NaN ); + dmatrix.set( i, j, Double.NaN ); } } + matrix = matrix.withMatrix( dmatrix ); RowLevelFilter filter = new RowLevelFilter( RowLevelFilter.Method.MEAN ); filter.filter( matrix.sliceRows( Collections.emptyList() ) ); filter = new RowLevelFilter( RowLevelFilter.Method.MEAN ); @@ -213,11 +221,13 @@ public void testFilterMatrixFilledWithNaNs() { @Test public void testFilterMatrixFilledWithZeroes() { + DoubleMatrix2D dmatrix = matrix.asDoubleMatrix2D(); for ( int i = 0; i < 100; i++ ) { for ( int j = 0; j < 8; j++ ) { - matrix.set( i, j, 0.0 ); + dmatrix.set( i, j, 0.0 ); } } + matrix = matrix.withMatrix( dmatrix ); RowLevelFilter filter = new RowLevelFilter( RowLevelFilter.Method.MEAN ); filter.setLowCut( Constants.SMALLISH ); diff --git a/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/filter/ZeroVarianceFilterTest.java b/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/filter/ZeroVarianceFilterTest.java index 2c0d40661a..8b8a8f1a93 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/filter/ZeroVarianceFilterTest.java +++ b/gemma-core/src/test/java/ubic/gemma/core/analysis/preprocess/filter/ZeroVarianceFilterTest.java @@ -1,5 +1,6 @@ package ubic.gemma.core.analysis.preprocess.filter; +import cern.colt.matrix.DoubleMatrix2D; import org.apache.commons.math3.distribution.NormalDistribution; import org.junit.Test; import ubic.basecode.math.Constants; @@ -35,7 +36,7 @@ public void test() { } QuantitationType qt = QuantitationType.Factory.newInstance(); qt.setGeneralType( GeneralType.QUANTITATIVE ); - qt.setType( StandardQuantitationType.COUNT ); + qt.setType( StandardQuantitationType.AMOUNT ); qt.setScale( ScaleType.LOG2 ); qt.setRepresentation( PrimitiveType.DOUBLE ); ExpressionDataDoubleMatrix matrix = new ExpressionDataDoubleMatrix( ee, randomBulkVectors( ee, ad, qt, RawExpressionDataVector.class ) ); @@ -45,9 +46,11 @@ public void test() { .isEqualTo( 10 ); // fill a row with zeroes + DoubleMatrix2D dmatrix = matrix.asDoubleMatrix2D(); for ( int j = 0; j < 10; j++ ) { - matrix.getMatrix().set( 5, j, 10.0 ); + dmatrix.set( 5, j, 10.0 ); } + matrix = matrix.withMatrix( dmatrix ); filteredMatrix = new ZeroVarianceFilter().filter( matrix ); assertThat( filteredMatrix.rows() ) @@ -55,9 +58,11 @@ public void test() { // fill low-variance noise NormalDistribution dist = new NormalDistribution( 0, Math.sqrt( 0.1 * Constants.SMALLISH ) ); + dmatrix = matrix.asDoubleMatrix2D(); for ( int j = 0; j < 10; j++ ) { - matrix.getMatrix().set( 5, j, dist.sample() ); + dmatrix.set( 5, j, dist.sample() ); } + matrix = matrix.withMatrix( dmatrix ); filteredMatrix = new ZeroVarianceFilter().filter( matrix ); assertThat( filteredMatrix.rows() ) @@ -65,9 +70,11 @@ public void test() { // fill noise above the detection threshold dist = new NormalDistribution( 0, Math.sqrt( 10 * Constants.SMALLISH ) ); + dmatrix = matrix.asDoubleMatrix2D(); for ( int j = 0; j < 10; j++ ) { - matrix.getMatrix().set( 5, j, dist.sample() ); + dmatrix.set( 5, j, dist.sample() ); } + matrix = matrix.withMatrix( dmatrix ); filteredMatrix = new ZeroVarianceFilter().filter( matrix ); assertThat( filteredMatrix.rows() ) diff --git a/gemma-core/src/test/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataMatrixTest.java b/gemma-core/src/test/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataMatrixTest.java index e3a3cde795..a1ce4a7470 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataMatrixTest.java +++ b/gemma-core/src/test/java/ubic/gemma/core/datastructure/matrix/BulkExpressionDataMatrixTest.java @@ -29,7 +29,7 @@ public void testEmptyMatrix() { assertThrows( IndexOutOfBoundsException.class, () -> matrix.get( 0, 0 ) ); assertThrows( IndexOutOfBoundsException.class, () -> matrix.getRow( 0 ) ); assertEquals( 0, matrix.getColumn( 0 ).length ); - assertEquals( 0, matrix.getRawMatrix().length ); + assertEquals( 0, matrix.getMatrix().length ); assertEquals( -1, matrix.getRowIndex( new CompositeSequence() ) ); Assertions.assertThat( BulkExpressionDataMatrixUtils.toVectors( matrix, RawExpressionDataVector.class ) ) .isEmpty(); diff --git a/gemma-core/src/test/java/ubic/gemma/core/loader/expression/DataUpdaterTest.java b/gemma-core/src/test/java/ubic/gemma/core/loader/expression/DataUpdaterTest.java index 4f6acaf278..50239b9cae 100644 --- a/gemma-core/src/test/java/ubic/gemma/core/loader/expression/DataUpdaterTest.java +++ b/gemma-core/src/test/java/ubic/gemma/core/loader/expression/DataUpdaterTest.java @@ -159,7 +159,7 @@ public void testAddData() throws Exception { assertNotNull( data.getBestBioAssayDimension() ); assertEquals( rawMatrix.columns(), data.getBioAssayDimension().getBioAssays().size() ); - assertEquals( probes.size(), data.getMatrix().rows() ); + assertEquals( probes.size(), data.rows() ); /* * Replace it. diff --git a/gemma-core/src/test/java/ubic/gemma/persistence/service/expression/bioAssayData/RandomExpressionDataMatrixUtils.java b/gemma-core/src/test/java/ubic/gemma/persistence/service/expression/bioAssayData/RandomExpressionDataMatrixUtils.java index 8b443de71f..f6c231a53f 100644 --- a/gemma-core/src/test/java/ubic/gemma/persistence/service/expression/bioAssayData/RandomExpressionDataMatrixUtils.java +++ b/gemma-core/src/test/java/ubic/gemma/persistence/service/expression/bioAssayData/RandomExpressionDataMatrixUtils.java @@ -63,17 +63,19 @@ public static ExpressionDataDoubleMatrix randomCountMatrix( ExpressionExperiment if ( scaleType == ScaleType.COUNT || scaleType == ScaleType.LINEAR ) { return matrix; } + DoubleMatrix dmatrix = matrix.asDoubleMatrix(); for ( int i = 0; i < matrix.rows(); i++ ) { for ( int j = 0; j < matrix.columns(); j++ ) { double val = matrix.getAsDouble( i, j ); try { - matrix.set( i, j, convertData( new double[] { val }, StandardQuantitationType.COUNT, ScaleType.COUNT, scaleType )[0] ); + double value = convertData( new double[] { val }, StandardQuantitationType.COUNT, ScaleType.COUNT, scaleType )[0]; + dmatrix.set( i, j, value ); } catch ( UnsupportedQuantitationScaleConversionException e ) { throw new RuntimeException( e ); } } } - return matrix; + return matrix.withMatrix( dmatrix ); } public static ExpressionDataDoubleMatrix randomLinearMatrix( ExpressionExperiment ee ) { @@ -224,8 +226,8 @@ public static ExpressionDataDoubleMatrix randomLog2cpmMatrix( ExpressionExperime log2cpmQt.setScale( ScaleType.LOG2 ); log2cpmQt.setRepresentation( PrimitiveType.DOUBLE ); ExpressionDataDoubleMatrix dm = randomCountMatrix( ee ); - DoubleMatrix1D librarySize = MatrixStats.colSums( dm.getMatrix() ); - DoubleMatrix log2cpmData = MatrixStats.convertToLog2Cpm( dm.getMatrix(), librarySize ); + DoubleMatrix1D librarySize = MatrixStats.colSums( dm.asDoubleMatrix() ); + DoubleMatrix log2cpmData = MatrixStats.convertToLog2Cpm( dm.asDoubleMatrix(), librarySize ); return dm.withMatrix( log2cpmData, Collections.singletonMap( dm.getQuantitationType(), log2cpmQt ) ); }