Examples with RawDataFetcher ubic.gemma.core.loader.expression.geo.fetcher.RawDataFetcher used on opensource projects

Example 1 with RawDataFetcher

use of ubic.gemma.core.loader.expression.geo.fetcher.RawDataFetcher in project Gemma by PavlidisLab.

the class DataUpdater method addAffyExonArrayData.

/**
 * Replaces any existing "preferred" data. Must be a single-platform study
 *
 * @param ee ee
 * @param ad ad
 */
// Possible external use
@SuppressWarnings({ "unused", "WeakerAccess" })
public void addAffyExonArrayData(ExpressionExperiment ee, ArrayDesign ad) {
    RawDataFetcher f = new RawDataFetcher();
    Collection<LocalFile> files = f.fetch(ee.getAccession().getAccession());
    if (files.isEmpty()) {
        throw new RuntimeException("Data was apparently not available");
    }
    ad = arrayDesignService.thaw(ad);
    ee = experimentService.thawLite(ee);
    Taxon primaryTaxon = ad.getPrimaryTaxon();
    ArrayDesign targetPlatform = this.prepareTargetPlatformForExonArrays(primaryTaxon);
    assert !targetPlatform.getCompositeSequences().isEmpty();
    AffyPowerToolsProbesetSummarize apt = new AffyPowerToolsProbesetSummarize();
    Collection<RawExpressionDataVector> vectors = apt.processExonArrayData(ee, targetPlatform, files);
    if (vectors.isEmpty()) {
        throw new IllegalStateException("No vectors were returned for " + ee);
    }
    ee = experimentService.replaceRawVectors(ee, vectors);
    if (!targetPlatform.equals(ad)) {
        AuditEventType eventType = ExpressionExperimentPlatformSwitchEvent.Factory.newInstance();
        auditTrailService.addUpdateEvent(ee, eventType, "Switched in course of updating vectors using AffyPowerTools (from " + ad.getShortName() + " to " + targetPlatform.getShortName() + ")");
    }
    this.audit(ee, "Data vector computation from CEL files using AffyPowerTools for " + targetPlatform, true);
    this.postprocess(ee);
}

Example 2 with RawDataFetcher

use of ubic.gemma.core.loader.expression.geo.fetcher.RawDataFetcher in project Gemma by PavlidisLab.

the class BatchInfoPopulationServiceImpl method needToRun.

/**
 * @param ee ee
 * @return true if it needs processing
 */
private boolean needToRun(ExpressionExperiment ee) {
    ExpressionExperimentValueObject eevo = expressionExperimentService.loadValueObject(ee);
    assert eevo != null;
    if (StringUtils.isBlank(eevo.getAccession())) {
        BatchInfoPopulationServiceImpl.log.info(ee + " lacks an external accession to use for fetching, will not attempt to fetch raw data files.");
        return false;
    }
    if (eevo.getTechnologyType().equals("NONE")) {
        BatchInfoPopulationServiceImpl.log.info(ee + " has technology type 'NONE', will not attempt to fetch raw data files");
        return false;
    }
    AuditEvent e = auditEventService.getLastEvent(ee, BatchInformationFetchingEvent.class);
    if (e == null)
        return true;
    if (FailedBatchInformationFetchingEvent.class.isAssignableFrom(e.getClass()))
        // worth trying
        return true;
    // on occasions the files appear or were missed the first time ...? GSE20842
    if (FailedBatchInformationMissingEvent.class.isAssignableFrom(e.getClass())) {
        RawDataFetcher fetcher = new RawDataFetcher();
        return fetcher.checkForFile(ee.getAccession().getAccession());
    }
    // already did it.
    return false;
}

Example 3 with RawDataFetcher

use of ubic.gemma.core.loader.expression.geo.fetcher.RawDataFetcher in project Gemma by PavlidisLab.

the class BatchInfoPopulationServiceImpl method fetchRawDataFiles.

/**
 * Currently only supports GEO
 *
 * @param ee ee
 * @return local file
 */
private Collection<LocalFile> fetchRawDataFiles(ExpressionExperiment ee) {
    RawDataFetcher fetcher = new RawDataFetcher();
    DatabaseEntry accession = ee.getAccession();
    if (accession == null) {
        BatchInfoPopulationServiceImpl.log.warn("No accession for " + ee.getShortName());
        return new HashSet<>();
    }
    return fetcher.fetch(accession.getAccession());
}

Example 4 with RawDataFetcher

use of ubic.gemma.core.loader.expression.geo.fetcher.RawDataFetcher in project Gemma by PavlidisLab.

the class DataUpdater method reprocessAffyThreePrimeArrayData.

/**
 * @param ee ee
 * @return This replaces the existing raw data with the CEL file data. CEL file(s) must be found by configuration
 */
// Possible external use
@SuppressWarnings("UnusedReturnValue")
public ExpressionExperiment reprocessAffyThreePrimeArrayData(ExpressionExperiment ee) {
    Collection<ArrayDesign> arrayDesignsUsed = this.experimentService.getArrayDesignsUsed(ee);
    ee = experimentService.thawLite(ee);
    RawDataFetcher f = new RawDataFetcher();
    Collection<LocalFile> files = f.fetch(ee.getAccession().getAccession());
    if (files.isEmpty()) {
        throw new RuntimeException("Data was apparently not available");
    }
    Collection<RawExpressionDataVector> vectors = new HashSet<>();
    // Use the same QT for each one
    QuantitationType qt = AffyPowerToolsProbesetSummarize.makeAffyQuantitationType();
    qt = quantitationTypeService.create(qt);
    for (ArrayDesign ad : arrayDesignsUsed) {
        DataUpdater.log.info("Processing data for " + ad);
        String cdfFileName = this.findCdf(ad).getAbsolutePath();
        ad = arrayDesignService.thaw(ad);
        AffyPowerToolsProbesetSummarize apt = new AffyPowerToolsProbesetSummarize(qt);
        vectors.addAll(apt.processThreeprimeArrayData(ee, cdfFileName, ad, files));
    }
    if (vectors.isEmpty()) {
        throw new IllegalStateException("No vectors were returned for " + ee);
    }
    ee = experimentService.replaceRawVectors(ee, vectors);
    this.audit(ee, "Data vector computation from CEL files using AffyPowerTools for " + StringUtils.join(arrayDesignsUsed, "; "), true);
    if (arrayDesignsUsed.size() == 1) {
        this.postprocess(ee);
    } else {
        DataUpdater.log.warn("Skipping postprocessing for mult-platform experiment");
    }
    return ee;
}