Examples with ExternalDatabase ubic.gemma.model.common.description.ExternalDatabase used on opensource projects

Example 16 with ExternalDatabase

use of ubic.gemma.model.common.description.ExternalDatabase in project Gemma by PavlidisLab.

the class GeoDomainObjectGenerator method getProjectedAccessions.

/**
 * Determine the set of external accession values that will be generated during parsing. This can be used to
 * pre-empty time-consuming fetch and download of data we already have.
 *
 * @param geoAccession geo accession
 * @return database entries
 */
public Collection<DatabaseEntry> getProjectedAccessions(String geoAccession) {
    ExternalDatabase ed = ExternalDatabase.Factory.newInstance();
    ed.setName("GEO");
    Collection<DatabaseEntry> accessions = new HashSet<>();
    // DatabaseEntry
    StringBuilder seriesAccession = new StringBuilder("");
    if (geoAccession.startsWith("GSE")) {
        seriesAccession = new StringBuilder(geoAccession);
    } else if (geoAccession.startsWith("GPL")) {
        GeoDomainObjectGenerator.log.warn("Determining if the data already exist for a GPL (" + geoAccession + ") is not implemented.");
        return null;
    } else if (geoAccession.startsWith("GDS")) {
        Collection<String> seriesAccessions = DatasetCombiner.findGSEforGDS(geoAccession);
        if (seriesAccessions == null || seriesAccessions.size() == 0) {
            throw new InvalidAccessionException("There is no series (GSE) for the accession " + geoAccession);
        }
        for (String string : seriesAccessions) {
            seriesAccession.append(string).append(",");
        }
        seriesAccession = new StringBuilder(StringUtils.removeEnd(seriesAccession.toString(), ","));
    } else {
        if (StringUtils.isBlank(geoAccession)) {
            throw new InvalidAccessionException("GEO accession must not be blank. Enter a  GSE, GDS or GPL");
        }
        throw new InvalidAccessionException("'" + geoAccession + "' is not understood by Gemma; must be a GSE, GDS or GPL. Did you choose the right source database?");
    }
    DatabaseEntry de = DatabaseEntry.Factory.newInstance(ed);
    de.setAccession(seriesAccession.toString());
    accessions.add(de);
    return accessions;
}

Example 17 with ExternalDatabase

use of ubic.gemma.model.common.description.ExternalDatabase in project Gemma by PavlidisLab.

the class ExternalDatabaseUtils method getGenbank.

/**
 * @return a transient instance of the Genbank database reference.
 */
private static ExternalDatabase getGenbank() {
    ExternalDatabase genBank = ExternalDatabase.Factory.newInstance();
    genBank.setType(DatabaseType.SEQUENCE);
    genBank.setName("Genbank");
    return genBank;
}

Example 18 with ExternalDatabase

use of ubic.gemma.model.common.description.ExternalDatabase in project Gemma by PavlidisLab.

the class ShellDelegatingBlat method blatQuery.

@Override
public Collection<BlatResult> blatQuery(BioSequence b, Taxon taxon, boolean sensitive) throws IOException {
    assert seqDir != null;
    // write the sequence to a temporary file.
    String seqName = b.getName().replaceAll(" ", "_");
    File querySequenceFile = File.createTempFile(seqName, ".fa");
    try (BufferedWriter out = new BufferedWriter(new FileWriter(querySequenceFile))) {
        String trimmed = SequenceManipulation.stripPolyAorT(b.getSequence(), ShellDelegatingBlat.POLY_AT_THRESHOLD);
        out.write(">" + seqName + "\n" + trimmed);
        ShellDelegatingBlat.log.info("Wrote sequence to " + querySequenceFile.getPath());
    }
    String outputPath = this.getTmpPslFilePath(seqName);
    Collection<BlatResult> results = this.gfClient(querySequenceFile, outputPath, this.choosePortForQuery(taxon, sensitive));
    ExternalDatabase searchedDatabase = ShellDelegatingBlat.getSearchedGenome(taxon);
    for (BlatResult result : results) {
        result.setSearchedDatabase(searchedDatabase);
    }
    this.cleanUpTmpFiles(querySequenceFile, outputPath);
    return results;
}

Example 19 with ExternalDatabase

use of ubic.gemma.model.common.description.ExternalDatabase in project Gemma by PavlidisLab.

the class ShellDelegatingBlat method blatQuery.

@Override
public Map<BioSequence, Collection<BlatResult>> blatQuery(Collection<BioSequence> sequences, boolean sensitive, Taxon taxon) throws IOException {
    Map<BioSequence, Collection<BlatResult>> results = new HashMap<>();
    File querySequenceFile = File.createTempFile("sequences-for-blat", ".fa");
    int count = SequenceWriter.writeSequencesToFile(sequences, querySequenceFile);
    if (count == 0) {
        EntityUtils.deleteFile(querySequenceFile);
        throw new IllegalArgumentException("No sequences!");
    }
    String outputPath = this.getTmpPslFilePath("blat-output");
    Integer port = this.choosePortForQuery(taxon, sensitive);
    if (port == null) {
        throw new IllegalStateException("Could not locate port for BLAT with settings taxon=" + taxon + ", sensitive=" + sensitive + ", check your configuration.");
    }
    Collection<BlatResult> rawResults = this.gfClient(querySequenceFile, outputPath, port);
    ShellDelegatingBlat.log.info("Got " + rawResults.size() + " raw blat results");
    ExternalDatabase searchedDatabase = ShellDelegatingBlat.getSearchedGenome(taxon);
    for (BlatResult blatResult : rawResults) {
        blatResult.setSearchedDatabase(searchedDatabase);
        BioSequence query = blatResult.getQuerySequence();
        if (!results.containsKey(query)) {
            results.put(query, new HashSet<BlatResult>());
        }
        results.get(query).add(blatResult);
    }
    EntityUtils.deleteFile(querySequenceFile);
    return results;
}

Example 20 with ExternalDatabase

use of ubic.gemma.model.common.description.ExternalDatabase in project Gemma by PavlidisLab.

the class GenericGenelistDesignGenerator method doWork.

@Override
protected Exception doWork(String[] args) {
    Exception exception = super.processCommandLine(args);
    if (exception != null) {
        return exception;
    }
    ExternalDatabase genbank = externalDatabaseService.findByName("Genbank");
    ExternalDatabase ensembl = externalDatabaseService.findByName("Ensembl");
    assert genbank != null;
    assert ensembl != null;
    /*
         * Create the stub array design for the organism. The name and etc. are generated automatically. If the design
         * exists, we update it.
         */
    String shortName = this.generateShortName();
    ArrayDesign arrayDesign = ArrayDesign.Factory.newInstance();
    arrayDesign.setShortName(shortName);
    // common name
    arrayDesign.setPrimaryTaxon(taxon);
    String nameExt = useNCBIIds ? ", indexed by NCBI IDs" : useEnsemblIds ? ", indexed by Ensembl IDs" : "";
    arrayDesign.setName("Generic platform for " + taxon.getScientificName() + nameExt);
    arrayDesign.setDescription("Created by Gemma");
    // this is key
    arrayDesign.setTechnologyType(TechnologyType.NONE);
    if (arrayDesignService.find(arrayDesign) != null) {
        AbstractCLI.log.info("Platform for " + taxon + " already exists, will update");
        arrayDesign = arrayDesignService.find(arrayDesign);
        arrayDesignService.deleteGeneProductAssociations(arrayDesign);
        arrayDesign = arrayDesignService.load(arrayDesign.getId());
    } else {
        AbstractCLI.log.info("Creating new 'generic' platform");
        arrayDesign = arrayDesignService.create(arrayDesign);
    }
    arrayDesign = arrayDesignService.thaw(arrayDesign);
    // temporary: making sure we set it, as it is new.
    arrayDesign.setTechnologyType(TechnologyType.NONE);
    /*
         * Load up the genes for the organism.
         */
    Collection<Gene> knownGenes = geneService.loadAll(taxon);
    AbstractCLI.log.info("Taxon has " + knownGenes.size() + " genes");
    // this would be good for cases where the identifier we are using has changed.
    Map<Gene, CompositeSequence> existingGeneMap = new HashMap<>();
    if (!useNCBIIds && !useEnsemblIds) {
        // only using this for symbol changes.
        existingGeneMap = this.getExistingGeneMap(arrayDesign);
    }
    Map<String, CompositeSequence> existingSymbolMap = this.getExistingProbeNameMap(arrayDesign);
    int count = 0;
    int numWithNoTranscript = 0;
    // int hasGeneAlready = 0;
    // int numNewGenes = 0;
    int numNewElements = 0;
    int numUpdatedElements = 0;
    for (Gene gene : knownGenes) {
        gene = geneService.thaw(gene);
        Collection<GeneProduct> products = gene.getProducts();
        if (products.isEmpty()) {
            numWithNoTranscript++;
            AbstractCLI.log.debug("No transcript for " + gene);
            continue;
        }
        count++;
        CompositeSequence csForGene = null;
        if (useNCBIIds) {
            if (gene.getNcbiGeneId() == null) {
                AbstractCLI.log.debug("No NCBI ID for " + gene + ", skipping");
                continue;
            }
            if (existingSymbolMap.containsKey(gene.getNcbiGeneId().toString())) {
                csForGene = existingSymbolMap.get(gene.getNcbiGeneId().toString());
            }
        } else if (useEnsemblIds) {
            if (gene.getEnsemblId() == null) {
                AbstractCLI.log.debug("No Ensembl ID for " + gene + ", skipping");
                continue;
            }
            if (existingSymbolMap.containsKey(gene.getEnsemblId())) {
                csForGene = existingSymbolMap.get(gene.getEnsemblId());
            }
        } else {
            /*
                 * detect when the symbol has changed
                 */
            if (existingSymbolMap.containsKey(gene.getOfficialSymbol())) {
                csForGene = existingSymbolMap.get(gene.getOfficialSymbol());
            } else if (existingGeneMap.containsKey(gene)) {
                csForGene = existingGeneMap.get(gene);
                AbstractCLI.log.debug("Gene symbol has changed for: " + gene + "? Current element has name=" + csForGene.getName());
                csForGene.setName(gene.getOfficialSymbol());
            }
        }
        assert csForGene == null || csForGene.getId() != null : "Null id for " + csForGene;
        /*
             * We arbitrarily link the "probe" to one of the gene's RNA transcripts. We could consider other strategies
             * to pick the representative, but it generally doesn't matter.
             */
        for (GeneProduct geneProduct : products) {
            if (!GeneProductType.RNA.equals(geneProduct.getType())) {
                continue;
            }
            /*
                 * Name is usually the genbank or ensembl accession
                 */
            String name = geneProduct.getName();
            BioSequence bioSequence = BioSequence.Factory.newInstance();
            Collection<DatabaseEntry> accessions = geneProduct.getAccessions();
            bioSequence.setName(name);
            bioSequence.setTaxon(taxon);
            bioSequence.setPolymerType(PolymerType.RNA);
            bioSequence.setType(SequenceType.mRNA);
            BioSequence existing = null;
            if (accessions.isEmpty()) {
                // this should not be hit.
                AbstractCLI.log.warn("No accession for " + name);
                DatabaseEntry de = DatabaseEntry.Factory.newInstance();
                de.setAccession(name);
                if (name.startsWith("ENS") && name.length() > 10) {
                    de.setExternalDatabase(ensembl);
                } else {
                    if (name.matches("^[A-Z]{1,2}(_?)[0-9]+(\\.[0-9]+)?$")) {
                        de.setExternalDatabase(genbank);
                    } else {
                        AbstractCLI.log.info("Name doesn't look like genbank or ensembl, skipping: " + name);
                        continue;
                    }
                }
                bioSequence.setSequenceDatabaseEntry(de);
            } else {
                bioSequence.setSequenceDatabaseEntry(accessions.iterator().next());
                existing = bioSequenceService.findByAccession(accessions.iterator().next());
            // FIXME It is possible that this sequence will have been aligned to the genome, which is a bit
            // confusing. So it will map to a gene. Worse case: it maps to more than one gene ...
            }
            if (existing == null) {
                bioSequence = (BioSequence) this.getPersisterHelper().persist(bioSequence);
            } else {
                bioSequence = existing;
            }
            assert bioSequence != null && bioSequence.getId() != null;
            if (bioSequence.getSequenceDatabaseEntry() == null) {
                AbstractCLI.log.info("No DB entry for " + bioSequence + "(" + gene + "), will look for a better sequence to use ...");
                continue;
            }
            if (csForGene == null) {
                if (AbstractCLI.log.isDebugEnabled())
                    AbstractCLI.log.debug("New element " + " with " + bioSequence + " for " + gene);
                csForGene = CompositeSequence.Factory.newInstance();
                if (useNCBIIds) {
                    if (gene.getNcbiGeneId() == null) {
                        continue;
                    }
                    csForGene.setName(gene.getNcbiGeneId().toString());
                } else if (useEnsemblIds) {
                    if (gene.getEnsemblId() == null) {
                        continue;
                    }
                    csForGene.setName(gene.getEnsemblId());
                } else {
                    csForGene.setName(gene.getOfficialSymbol());
                }
                csForGene.setArrayDesign(arrayDesign);
                csForGene.setBiologicalCharacteristic(bioSequence);
                csForGene.setDescription("Generic expression element for " + gene);
                csForGene = compositeSequenceService.create(csForGene);
                assert csForGene.getId() != null : "No id for " + csForGene + " for " + gene;
                arrayDesign.getCompositeSequences().add(csForGene);
                numNewElements++;
            } else {
                if (AbstractCLI.log.isDebugEnabled())
                    AbstractCLI.log.debug("Updating existing element: " + csForGene + " with " + bioSequence + " for " + gene);
                csForGene.setArrayDesign(arrayDesign);
                csForGene.setBiologicalCharacteristic(bioSequence);
                csForGene.setDescription("Generic expression element for " + gene);
                assert csForGene.getId() != null : "No id for " + csForGene + " for " + gene;
                compositeSequenceService.update(csForGene);
                // making sure ...
                csForGene = compositeSequenceService.load(csForGene.getId());
                assert csForGene.getId() != null;
                arrayDesign.getCompositeSequences().add(csForGene);
                numUpdatedElements++;
            }
            assert bioSequence.getId() != null;
            assert geneProduct.getId() != null;
            assert csForGene.getBiologicalCharacteristic() != null && csForGene.getBiologicalCharacteristic().getId() != null;
            AnnotationAssociation aa = AnnotationAssociation.Factory.newInstance();
            aa.setGeneProduct(geneProduct);
            aa.setBioSequence(bioSequence);
            annotationAssociationService.create(aa);
            break;
        }
        if (count % 100 == 0)
            AbstractCLI.log.info(count + " genes processed; " + numNewElements + " new elements; " + numUpdatedElements + " updated elements; " + numWithNoTranscript + " genes had no transcript and were skipped.");
    }
    // is this necessary? causes an error sometimes.
    // arrayDesignService.update( arrayDesign );
    AbstractCLI.log.info("Array design has " + arrayDesignService.numCompositeSequenceWithGenes(arrayDesign) + " 'probes' associated with genes.");
    arrayDesignReportService.generateArrayDesignReport(arrayDesign.getId());
    auditTrailService.addUpdateEvent(arrayDesign, AnnotationBasedGeneMappingEvent.Factory.newInstance(), count + " genes processed; " + numNewElements + " new elements; " + numUpdatedElements + " updated elements; " + numWithNoTranscript + " genes had no transcript and were skipped.");
    arrayDesignAnnotationService.deleteExistingFiles(arrayDesign);
    AbstractCLI.log.info("Don't forget to update the annotation files");
    return null;
}