Examples with GeoDomainObjectGeneratorLocal ubic.gemma.core.loader.expression.geo.GeoDomainObjectGeneratorLocal used on opensource projects

Example 6 with GeoDomainObjectGeneratorLocal

use of ubic.gemma.core.loader.expression.geo.GeoDomainObjectGeneratorLocal in project Gemma by PavlidisLab.

the class ExpressionDataDoubleMatrixTest method testMatrixConversion.

@Test
public void testMatrixConversion() throws Exception {
    try {
        geoService.setGeoDomainObjectGenerator(new GeoDomainObjectGeneratorLocal(this.getTestFileBasePath("")));
        Collection<?> results = geoService.fetchAndLoad("GSE8294", false, true, false);
        newee = (ExpressionExperiment) results.iterator().next();
    } catch (AlreadyExistsInSystemException e) {
        newee = (ExpressionExperiment) ((List<?>) e.getData()).iterator().next();
    }
    newee = expressionExperimentService.thaw(newee);
    // make sure we really thawRawAndProcessed them, so we can get the design element sequences.
    Collection<RawExpressionDataVector> vectors = newee.getRawExpressionDataVectors();
    rawExpressionDataVectorService.thaw(vectors);
    ExpressionDataMatrixBuilder builder = new ExpressionDataMatrixBuilder(vectors);
    ExpressionDataDoubleMatrix matrix = builder.getPreferredData();
    assertTrue(!Double.isNaN(matrix.get(10, 0)));
    assertEquals(66, matrix.rows());
    assertEquals(9, matrix.columns());
    /*
         * Additional tests for files and outlier marking.
         */
    processedDataVectorService.computeProcessedExpressionData(newee);
    File f1 = expressionDataFileService.writeOrLocateDataFile(expressionExperimentService.load(newee.getId()), true, true);
    assertNotNull(f1);
    assertTrue(f1.exists());
    expressionDataFileService.deleteAllFiles(newee);
    assertTrue(!f1.exists());
    /*
         * outlier removal.
         */
    BioAssay tba = newee.getBioAssays().iterator().next();
    Collection<BioAssay> ol = new HashSet<>();
    ol.add(tba);
    sampleRemoveService.markAsMissing(ol);
    assertTrue(tba.getIsOutlier());
    newee = expressionExperimentService.thaw(newee);
    Collection<ProcessedExpressionDataVector> vecs = newee.getProcessedExpressionDataVectors();
    this.processedDataVectorService.thaw(vecs);
    assertTrue(!vecs.isEmpty());
    ExpressionDataMatrixBuilder matrixBuilder = new ExpressionDataMatrixBuilder(vecs);
    ExpressionDataDoubleMatrix data = matrixBuilder.getProcessedData();
    assertNotNull(data);
    assertTrue(Double.isNaN(data.getColumn(tba)[10]));
    sampleRemoveService.unmarkAsMissing(ol);
    newee = expressionExperimentService.load(newee.getId());
    newee = expressionExperimentService.thaw(newee);
    vecs = newee.getProcessedExpressionDataVectors();
    this.processedDataVectorService.thaw(vecs);
    assertTrue(!vecs.isEmpty());
    matrixBuilder = new ExpressionDataMatrixBuilder(vecs);
    data = matrixBuilder.getProcessedData();
    assertTrue(!tba.getIsOutlier());
    assertTrue(!Double.isNaN(data.getColumn(tba)[10]));
}

Example 7 with GeoDomainObjectGeneratorLocal

use of ubic.gemma.core.loader.expression.geo.GeoDomainObjectGeneratorLocal in project Gemma by PavlidisLab.

the class DataUpdaterTest method testAddData.

@Test
public void testAddData() throws Exception {
    /*
         * Load a regular data set that has no data. Platform is (basically) irrelevant.
         */
    geoService.setGeoDomainObjectGenerator(new GeoDomainObjectGeneratorLocal(this.getTestFileBasePath()));
    ExpressionExperiment ee;
    try {
        // RNA-seq data.
        Collection<?> results = geoService.fetchAndLoad("GSE37646", false, true, false);
        ee = (ExpressionExperiment) results.iterator().next();
    } catch (AlreadyExistsInSystemException e) {
        // log.warn( "Test skipped because GSE37646 was not removed from the system prior to test" );
        ee = (ExpressionExperiment) ((List<?>) e.getData()).get(0);
    }
    ee = experimentService.thawLite(ee);
    List<BioAssay> bioAssays = new ArrayList<>(ee.getBioAssays());
    assertEquals(31, bioAssays.size());
    List<BioMaterial> bms = new ArrayList<>();
    for (BioAssay ba : bioAssays) {
        bms.add(ba.getSampleUsed());
    }
    targetArrayDesign = this.getTestPersistentArrayDesign(100, true);
    DoubleMatrix<CompositeSequence, BioMaterial> rawMatrix = new DenseDoubleMatrix<>(targetArrayDesign.getCompositeSequences().size(), bms.size());
    /*
         * make up some fake data on another platform, and match it to those samples
         */
    for (int i = 0; i < rawMatrix.rows(); i++) {
        for (int j = 0; j < rawMatrix.columns(); j++) {
            rawMatrix.set(i, j, (i + 1) * (j + 1) * Math.random() / 100.0);
        }
    }
    List<CompositeSequence> probes = new ArrayList<>(targetArrayDesign.getCompositeSequences());
    rawMatrix.setRowNames(probes);
    rawMatrix.setColumnNames(bms);
    QuantitationType qt = this.makeQt(true);
    ExpressionDataDoubleMatrix data = new ExpressionDataDoubleMatrix(ee, qt, rawMatrix);
    assertNotNull(data.getBestBioAssayDimension());
    assertEquals(rawMatrix.columns(), data.getBestBioAssayDimension().getBioAssays().size());
    assertEquals(probes.size(), data.getMatrix().rows());
    /*
         * Replace it.
         */
    ee = dataUpdater.replaceData(ee, targetArrayDesign, data);
    for (BioAssay ba : ee.getBioAssays()) {
        assertEquals(targetArrayDesign, ba.getArrayDesignUsed());
    }
    ee = experimentService.thaw(ee);
    for (BioAssay ba : ee.getBioAssays()) {
        assertEquals(targetArrayDesign, ba.getArrayDesignUsed());
    }
    assertEquals(100, ee.getRawExpressionDataVectors().size());
    for (RawExpressionDataVector v : ee.getRawExpressionDataVectors()) {
        assertTrue(v.getQuantitationType().getIsPreferred());
    }
    assertEquals(100, ee.getProcessedExpressionDataVectors().size());
    Collection<DoubleVectorValueObject> processedDataArrays = dataVectorService.getProcessedDataArrays(ee);
    for (DoubleVectorValueObject v : processedDataArrays) {
        assertEquals(31, v.getBioAssays().size());
    }
    /*
         * Test adding data (non-preferred)
         */
    qt = this.makeQt(false);
    ExpressionDataDoubleMatrix moreData = new ExpressionDataDoubleMatrix(ee, qt, rawMatrix);
    ee = dataUpdater.addData(ee, targetArrayDesign, moreData);
    ee = experimentService.thaw(ee);
    try {
        // add preferred data twice.
        dataUpdater.addData(ee, targetArrayDesign, data);
        fail("Should have gotten an exception");
    } catch (IllegalArgumentException e) {
    // okay.
    }
    dataUpdater.deleteData(ee, qt);
}

Example 8 with GeoDomainObjectGeneratorLocal

use of ubic.gemma.core.loader.expression.geo.GeoDomainObjectGeneratorLocal in project Gemma by PavlidisLab.

the class ArrayDesignSequenceProcessorTest method testFetchAndLoadWithIdentifiers.

@Test
public void testFetchAndLoadWithIdentifiers() throws Exception {
    String fastacmdExe = Settings.getString(SimpleFastaCmd.FASTA_CMD_ENV_VAR);
    if (fastacmdExe == null) {
        log.warn("No fastacmd executable is configured, skipping test");
        return;
    }
    File fi = new File(fastacmdExe);
    if (!fi.canRead()) {
        log.warn(fastacmdExe + " not found, skipping test");
        return;
    }
    GeoService geoService = this.getBean(GeoService.class);
    geoService.setGeoDomainObjectGenerator(new GeoDomainObjectGeneratorLocal(this.getTestFileBasePath()));
    @SuppressWarnings("unchecked") final Collection<ArrayDesign> ads = (Collection<ArrayDesign>) geoService.fetchAndLoad("GPL226", true, true, false, true, true);
    result = ads.iterator().next();
    result = arrayDesignService.thaw(result);
    // have to specify taxon as this has two taxons in it
    try (InputStream f = this.getClass().getResourceAsStream("/data/loader/expression/arrayDesign/identifierTest.txt")) {
        Collection<BioSequence> res = app.processArrayDesign(result, f, new String[] { "testblastdb", "testblastdbPartTwo" }, FileTools.resourceToPath("/data/loader/genome/blast"), taxon, true);
        assertNotNull(res);
        for (BioSequence sequence : res) {
            assertNotNull(sequence.getSequence());
        }
        for (CompositeSequence cs : result.getCompositeSequences()) {
            assert cs.getBiologicalCharacteristic() != null;
        }
    }
}

Example 9 with GeoDomainObjectGeneratorLocal

use of ubic.gemma.core.loader.expression.geo.GeoDomainObjectGeneratorLocal in project Gemma by PavlidisLab.

the class GeoPlatformServiceTest method testFetchAndLoadGPL101Short.

/*
     * Test method for 'ubic.gemma.core.loader.expression.geo.GeoPlatformService.fetchAndLoad(String)'
     */
@Test
public void testFetchAndLoadGPL101Short() throws Exception {
    geoService.setGeoDomainObjectGenerator(new GeoDomainObjectGeneratorLocal(this.getTestFileBasePath("platform")));
    geoService.fetchAndLoad("GPL101", true, true, false);
}

Example 10 with GeoDomainObjectGeneratorLocal

use of ubic.gemma.core.loader.expression.geo.GeoDomainObjectGeneratorLocal in project Gemma by PavlidisLab.

the class GeoDatasetServiceTest method testFetchAndLoadGSE13657.

/*
     * Left out quantitation types due to bug in how quantitation types were cached during persisting, if the QTs didn't
     * have descriptions.
     */
@Test
public void testFetchAndLoadGSE13657() throws Exception {
    try {
        geoService.setGeoDomainObjectGenerator(new GeoDomainObjectGeneratorLocal(this.getTestFileBasePath()));
        Collection<?> results = geoService.fetchAndLoad("GSE13657", false, true, false);
        ee = (ExpressionExperiment) results.iterator().next();
    } catch (AlreadyExistsInSystemException e) {
        log.info("Test skipped because GSE13657 was already loaded - clean the DB before running the test");
        return;
    }
    ee = this.eeService.thawLite(ee);
    aclTestUtils.checkEEAcls(ee);
    Collection<QuantitationType> qts = eeService.getQuantitationTypes(ee);
    assertEquals(13, qts.size());
    // make sure we got characteristics and treatments for both channels.
    for (BioAssay ba : ee.getBioAssays()) {
        BioMaterial bm = ba.getSampleUsed();
        assertNotNull(bm);
        log.info(bm + " " + bm.getDescription());
        assertEquals(9, bm.getCharacteristics().size());
    }
}