Examples with ExpressionExperiment ubic.gemma.model.expression.experiment.ExpressionExperiment used on opensource projects

Example 36 with ExpressionExperiment

use of ubic.gemma.model.expression.experiment.ExpressionExperiment in project Gemma by PavlidisLab.

the class ExpressionExperimentReportServiceImpl method populateEventInformation.

/**
 * Fills in event and security information from the database. This will only retrieve the latest event (if any).
 * This is rather slow so should be avoided if the information isn't needed.
 */
@Override
public void populateEventInformation(Collection<ExpressionExperimentDetailsValueObject> vos) {
    StopWatch timer = new StopWatch();
    timer.start();
    Collection<Long> ids = EntityUtils.getIds(vos);
    // do this ahead to avoid round trips - this also filters...
    Collection<ExpressionExperiment> ees = expressionExperimentService.load(ids);
    if (ees.size() == 0) {
        return;
    }
    Map<Long, ExpressionExperiment> eeMap = EntityUtils.getIdMap(ees);
    Map<Long, Date> lastArrayDesignUpdates = expressionExperimentService.getLastArrayDesignUpdate(ees);
    Collection<Class<? extends AuditEventType>> typesToGet = Arrays.asList(eventTypes);
    Map<Class<? extends AuditEventType>, Map<Auditable, AuditEvent>> events = this.getEvents(ees, typesToGet);
    Map<Auditable, AuditEvent> linkAnalysisEvents = events.get(LinkAnalysisEvent.class);
    Map<Auditable, AuditEvent> missingValueAnalysisEvents = events.get(MissingValueAnalysisEvent.class);
    Map<Auditable, AuditEvent> rankComputationEvents = events.get(ProcessedVectorComputationEvent.class);
    Map<Auditable, AuditEvent> differentialAnalysisEvents = events.get(DifferentialExpressionAnalysisEvent.class);
    Map<Auditable, AuditEvent> batchFetchEvents = events.get(BatchInformationFetchingEvent.class);
    Map<Auditable, AuditEvent> pcaAnalysisEvents = events.get(PCAAnalysisEvent.class);
    Map<Long, Collection<AuditEvent>> sampleRemovalEvents = this.getSampleRemovalEvents(ees);
    /*
         * add in the last events of interest for all eeVos This step is remarkably slow.
         */
    for (ExpressionExperimentDetailsValueObject eeVo : vos) {
        Long id = eeVo.getId();
        ExpressionExperiment ee = eeMap.get(id);
        if (linkAnalysisEvents.containsKey(ee)) {
            AuditEvent event = linkAnalysisEvents.get(ee);
            if (event != null) {
                Date date = event.getDate();
                eeVo.setDateLinkAnalysis(date);
                eeVo.setLinkAnalysisEventType(event.getEventType().getClass().getSimpleName());
            }
        }
        if (missingValueAnalysisEvents.containsKey(ee)) {
            AuditEvent event = missingValueAnalysisEvents.get(ee);
            if (event != null) {
                Date date = event.getDate();
                eeVo.setDateMissingValueAnalysis(date);
                eeVo.setMissingValueAnalysisEventType(event.getEventType().getClass().getSimpleName());
            }
        }
        if (rankComputationEvents.containsKey(ee)) {
            AuditEvent event = rankComputationEvents.get(ee);
            if (event != null) {
                Date date = event.getDate();
                eeVo.setDateProcessedDataVectorComputation(date);
                eeVo.setProcessedDataVectorComputationEventType(event.getEventType().getClass().getSimpleName());
            }
        }
        if (differentialAnalysisEvents.containsKey(ee)) {
            AuditEvent event = differentialAnalysisEvents.get(ee);
            if (event != null) {
                Date date = event.getDate();
                eeVo.setDateDifferentialAnalysis(date);
            }
        }
        if (pcaAnalysisEvents.containsKey(ee)) {
            AuditEvent event = pcaAnalysisEvents.get(ee);
            if (event != null) {
                Date date = event.getDate();
                eeVo.setDatePcaAnalysis(date);
                eeVo.setPcaAnalysisEventType(event.getEventType().getClass().getSimpleName());
            }
        }
        if (batchFetchEvents.containsKey(ee)) {
            AuditEvent event = batchFetchEvents.get(ee);
            if (event != null) {
                Date date = event.getDate();
                eeVo.setDateBatchFetch(date);
                eeVo.setBatchFetchEventType(event.getEventType().getClass().getSimpleName());
            }
        }
        if (lastArrayDesignUpdates.containsKey(id)) {
            Date date = lastArrayDesignUpdates.get(id);
            eeVo.setDateArrayDesignLastUpdated(date);
        }
        if (sampleRemovalEvents.containsKey(id)) {
            Collection<AuditEvent> removalEvents = sampleRemovalEvents.get(id);
            // we find we are getting lazy-load exceptions from this guy.
            eeVo.auditEvents2SampleRemovedFlags(removalEvents);
        }
    }
    if (timer.getTime() > 1000)
        log.info("Retrieving audit events took " + timer.getTime() + "ms");
}

Example 37 with ExpressionExperiment

use of ubic.gemma.model.expression.experiment.ExpressionExperiment in project Gemma by PavlidisLab.

the class ExpressionExperimentReportServiceImpl method getSampleRemovalEvents.

private Map<Long, Collection<AuditEvent>> getSampleRemovalEvents(Collection<ExpressionExperiment> ees) {
    Map<Long, Collection<AuditEvent>> result = new HashMap<>();
    Map<ExpressionExperiment, Collection<AuditEvent>> rawr = expressionExperimentService.getSampleRemovalEvents(ees);
    for (ExpressionExperiment e : rawr.keySet()) {
        result.put(e.getId(), rawr.get(e));
    }
    return result;
}

Example 38 with ExpressionExperiment

use of ubic.gemma.model.expression.experiment.ExpressionExperiment in project Gemma by PavlidisLab.

the class OutlierFlaggingServiceImpl method markAsMissing.

@Override
public void markAsMissing(Collection<BioAssay> bioAssays) {
    if (bioAssays == null || bioAssays.size() == 0)
        return;
    boolean hasNewOutliers = false;
    /*
         * FIXME: if there are two (or more) platforms, make sure we flag all bioassays that use the same biomaterial.
         * However, we are intending to turn all multiplatform datasets into single platform ones
         */
    for (BioAssay ba : bioAssays) {
        if (ba.getIsOutlier()) {
            continue;
        }
        hasNewOutliers = true;
        ba.setIsOutlier(true);
        bioAssayService.update(ba);
        this.audit(ba, "Sample " + ba.getName() + " marked as missing data.");
    }
    if (!hasNewOutliers) {
        System.out.println("No new outliers.");
        return;
    }
    ExpressionExperiment expExp = expressionExperimentService.findByBioAssay(bioAssays.iterator().next());
    auditTrailService.addUpdateEvent(expExp, SampleRemovalEvent.Factory.newInstance(), bioAssays.size() + " flagged as outliers", StringUtils.join(bioAssays, ","));
    try {
        preprocessorService.process(expExp);
    } catch (PreprocessingException e) {
        OutlierFlaggingServiceImpl.log.error("Error during postprocessing, make sure additional steps are completed", e);
    }
}

Example 39 with ExpressionExperiment

use of ubic.gemma.model.expression.experiment.ExpressionExperiment in project Gemma by PavlidisLab.

the class AffyDataFromCelCli method doWork.

@Override
protected Exception doWork(String[] args) {
    Exception e = super.processCommandLine(args);
    if (e != null)
        return e;
    DataUpdater serv = this.getBean(DataUpdater.class);
    // This can be done for multiple experiments under some conditions; we get this one just  to test for some multi-platform situations
    Collection<ArrayDesign> arrayDesignsUsed = this.eeService.getArrayDesignsUsed(this.expressionExperiments.iterator().next());
    if (StringUtils.isNotBlank(aptFile)) {
        if (this.expressionExperiments.size() > 1) {
            throw new IllegalArgumentException("Can't use " + AffyDataFromCelCli.APT_FILE_OPT + " unless you are doing just one experiment");
        }
        ExpressionExperiment thawedEe = (ExpressionExperiment) this.expressionExperiments.iterator().next();
        thawedEe = this.eeService.thawLite(thawedEe);
        if (arrayDesignsUsed.size() > 1) {
            throw new IllegalArgumentException("Cannot use " + AffyDataFromCelCli.APT_FILE_OPT + " for experiment that uses multiple platforms");
        }
        ArrayDesign ad = arrayDesignsUsed.iterator().next();
        try {
            AbstractCLI.log.info("Loading data from " + aptFile);
            if ((ad.getTechnologyType().equals(TechnologyType.ONECOLOR) && GeoPlatform.isAffymetrixExonArray(ad.getShortName())) || ad.getName().toLowerCase().contains("exon")) {
                serv.addAffyExonArrayData(thawedEe, aptFile);
            } else if (ad.getTechnologyType().equals(TechnologyType.ONECOLOR) && ad.getName().toLowerCase().contains("affy")) {
                serv.addAffyData(thawedEe, aptFile);
            } else {
                throw new IllegalArgumentException("Option " + AffyDataFromCelCli.APT_FILE_OPT + " only valid if you are using an exon array.");
            }
        } catch (Exception exception) {
            return exception;
        }
        return null;
    }
    if (StringUtils.isNotBlank(cdfFile)) {
        if (arrayDesignsUsed.size() > 1) {
            throw new IllegalArgumentException("Cannot use " + AffyDataFromCelCli.CDF_FILE_OPT + " for experiment that uses multiple platforms");
        }
    }
    for (BioAssaySet ee : this.expressionExperiments) {
        try {
            Collection<ArrayDesign> adsUsed = this.eeService.getArrayDesignsUsed(ee);
            /*
                 * if the audit trail already has a DataReplacedEvent, skip it, unless --force. Ignore this for
                 * multiplatform studies (at our peril)
                 */
            if (adsUsed.size() == 1 && !force && this.checkForAlreadyDone(ee)) {
                AbstractCLI.log.warn(ee + ": Already has been recomputed from raw data, skipping (use 'force' to override')");
                this.errorObjects.add(ee + ": Already has been computed from raw data");
                continue;
            }
            ExpressionExperiment thawedEe = (ExpressionExperiment) ee;
            thawedEe = this.eeService.thawLite(thawedEe);
            ArrayDesign ad = adsUsed.iterator().next();
            /*
                 * Even if there are multiple platforms, we assume they are all the same type. If not, that's your
                 * problem :) (seriously, we could check...)
                 */
            if ((ad.getTechnologyType().equals(TechnologyType.ONECOLOR) && GeoPlatform.isAffymetrixExonArray(ad.getShortName())) || ad.getName().toLowerCase().contains("exon")) {
                AbstractCLI.log.info(thawedEe + " looks like affy exon array");
                serv.addAffyExonArrayData(thawedEe);
                this.successObjects.add(thawedEe.toString());
                AbstractCLI.log.info("Successfully processed: " + thawedEe);
            } else if (ad.getTechnologyType().equals(TechnologyType.ONECOLOR) && ad.getName().toLowerCase().contains("affy")) {
                AbstractCLI.log.info(thawedEe + " looks like a affy 3-prime array");
                serv.reprocessAffyThreePrimeArrayData(thawedEe);
                this.successObjects.add(thawedEe.toString());
                AbstractCLI.log.info("Successfully processed: " + thawedEe);
            } else {
                AbstractCLI.log.warn(ee + ": This CLI can only deal with Affymetrix platforms (exon or 3' probe designs)");
                this.errorObjects.add(ee + ": This CLI can only deal with Affymetrix platforms (exon or 3' probe designs)");
            }
        } catch (Exception exception) {
            AbstractCLI.log.error(exception, exception);
            this.errorObjects.add(ee + " " + exception.getLocalizedMessage());
        }
    }
    super.summarizeProcessing();
    return null;
}

Example 40 with ExpressionExperiment

use of ubic.gemma.model.expression.experiment.ExpressionExperiment in project Gemma by PavlidisLab.

the class ProcessedExpressionDataVectorCreateHelperServiceImpl method reorderByDesign.

@Override
@Transactional
public void reorderByDesign(Long eeId) {
    ExpressionExperiment ee = expressionExperimentDao.load(eeId);
    if (ee.getExperimentalDesign().getExperimentalFactors().size() == 0) {
        ProcessedExpressionDataVectorCreateHelperServiceImpl.log.info(ee.getShortName() + " does not have a populated experimental design, skipping");
        return;
    }
    Collection<ProcessedExpressionDataVector> processedDataVectors = ee.getProcessedExpressionDataVectors();
    if (processedDataVectors.size() == 0) {
        ProcessedExpressionDataVectorCreateHelperServiceImpl.log.info(ee.getShortName() + " does not have processed data");
        return;
    }
    Collection<BioAssayDimension> dims = this.eeService.getBioAssayDimensions(ee);
    if (dims.size() > 1) {
        this.checkAllBioAssayDimensionsMatch(dims);
    }
    BioAssayDimension bioassaydim = dims.iterator().next();
    List<BioMaterial> start = new ArrayList<>();
    for (BioAssay ba : bioassaydim.getBioAssays()) {
        start.add(ba.getSampleUsed());
    }
    /*
         * Get the ordering we want.
         */
    List<BioMaterial> orderByExperimentalDesign = ExpressionDataMatrixColumnSort.orderByExperimentalDesign(start, ee.getExperimentalDesign().getExperimentalFactors());
    /*
         * Map of biomaterials to the new order index.
         */
    final Map<BioMaterial, Integer> ordering = new HashMap<>();
    int i = 0;
    for (BioMaterial bioMaterial : orderByExperimentalDesign) {
        ordering.put(bioMaterial, i);
        i++;
    }
    /*
         * Map of the original order to new order of bioassays.
         */
    Map<Integer, Integer> indexes = new HashMap<>();
    Map<BioAssayDimension, BioAssayDimension> old2new = new HashMap<>();
    for (BioAssayDimension bioAssayDimension : dims) {
        Collection<BioAssay> bioAssays = bioAssayDimension.getBioAssays();
        assert bioAssays != null;
        /*
             * Initialize the new bioassay list.
             */
        List<BioAssay> resorted = new ArrayList<>(bioAssays.size());
        for (int m = 0; m < bioAssays.size(); m++) {
            resorted.add(null);
        }
        for (int oldIndex = 0; oldIndex < bioAssays.size(); oldIndex++) {
            BioAssay bioAssay = ((List<BioAssay>) bioAssays).get(oldIndex);
            BioMaterial sam1 = bioAssay.getSampleUsed();
            if (ordering.containsKey(sam1)) {
                Integer newIndex = ordering.get(sam1);
                resorted.set(newIndex, bioAssay);
                /*
                     * Should be the same for all dimensions....
                     */
                assert !indexes.containsKey(oldIndex) || indexes.get(oldIndex).equals(newIndex);
                indexes.put(oldIndex, newIndex);
            } else {
                throw new IllegalStateException();
            }
        }
        BioAssayDimension newBioAssayDimension = BioAssayDimension.Factory.newInstance();
        newBioAssayDimension.setBioAssays(resorted);
        newBioAssayDimension.setName("Processed data of ee " + ee.getShortName() + " ordered by design");
        newBioAssayDimension.setDescription("Data was reordered based on the experimental design.");
        newBioAssayDimension = bioAssayDimensionService.create(newBioAssayDimension);
        old2new.put(bioAssayDimension, newBioAssayDimension);
    }
    ByteArrayConverter converter = new ByteArrayConverter();
    for (ProcessedExpressionDataVector v : processedDataVectors) {
        BioAssayDimension revisedBioAssayDimension = old2new.get(v.getBioAssayDimension());
        assert revisedBioAssayDimension != null;
        double[] data = converter.byteArrayToDoubles(v.getData());
        /*
             * Put the data in the order of the bioAssayDimension.
             */
        Double[] resortedData = new Double[data.length];
        for (int k = 0; k < data.length; k++) {
            resortedData[k] = data[indexes.get(k)];
        }
        v.setData(converter.toBytes(resortedData));
        v.setBioAssayDimension(revisedBioAssayDimension);
    }
    ProcessedExpressionDataVectorCreateHelperServiceImpl.log.info("Updating bioassay ordering of " + processedDataVectors.size() + " vectors");
    this.auditTrailService.addUpdateEvent(ee, "Reordered the data vectors by experimental design");
}