Examples with DatabaseManager massbank.db.DatabaseManager used on opensource projects

Example 1 with DatabaseManager

use of massbank.db.DatabaseManager in project MassBank-web by MassBank.

the class Validator method main.

public static void main(String[] arguments) {
    // load version and print
    final Properties properties = new Properties();
    try {
        properties.load(ClassLoader.getSystemClassLoader().getResourceAsStream("project.properties"));
    } catch (IOException e) {
        e.printStackTrace();
        System.exit(1);
    }
    System.out.println("Validator version: " + properties.getProperty("version"));
    // parse command line
    Options options = new Options();
    options.addOption(null, "db", false, "also read record from database and compare with original Record; Developer Feature!");
    options.addOption(null, "legacy", false, "less strict mode for legacy records with minor problems.");
    options.addOption(null, "online", false, "also do online checks, like PubChem CID check.");
    CommandLine cmd = null;
    try {
        cmd = new DefaultParser().parse(options, arguments);
    } catch (ParseException e) {
        // oops, something went wrong
        System.err.println("Parsing command line failed. Reason: " + e.getMessage());
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp("Validator [OPTIONS] <FILE|DIR> [<FILE|DIR> ...]", options);
        System.exit(1);
    }
    if (cmd.getArgList().size() == 0) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp("Validator [OPTIONS] <FILE|DIR> [<FILE|DIR> ...]", options);
        System.exit(1);
    }
    if (cmd.hasOption("legacy"))
        System.out.println("Validation mode: legacy");
    // find all files in arguments and all *.txt files in directories and subdirectories
    // specified in arguments
    List<File> recordfiles = new ArrayList<>();
    for (String argument : cmd.getArgList()) {
        File argumentf = new File(argument);
        if (argumentf.isFile() && FilenameUtils.getExtension(argument).equals("txt")) {
            recordfiles.add(argumentf);
        } else if (argumentf.isDirectory()) {
            recordfiles.addAll(FileUtils.listFiles(argumentf, new String[] { "txt" }, true));
        } else {
            logger.warn("Argument " + argument + " could not be processed.");
        }
    }
    if (recordfiles.size() == 0) {
        logger.error("No files found for validation.");
        System.exit(1);
    }
    // validate all files
    logger.trace("Validating " + recordfiles.size() + " files");
    AtomicBoolean haserror = new AtomicBoolean(false);
    AtomicBoolean doDatbase = new AtomicBoolean(cmd.hasOption("db"));
    AtomicBoolean legacyMode = new AtomicBoolean(cmd.hasOption("legacy"));
    AtomicBoolean onlineMode = new AtomicBoolean(cmd.hasOption("online"));
    List<String> accessions = recordfiles.parallelStream().map(filename -> {
        String recordString;
        String accession = null;
        logger.info("Working on " + filename + ".");
        try {
            recordString = FileUtils.readFileToString(filename, StandardCharsets.UTF_8);
            if (hasNonStandardChars(recordString)) {
                logger.warn("Check " + filename + ".");
            }
            ;
            // basic validation
            Set<String> config = new HashSet<String>();
            if (legacyMode.get())
                config.add("legacy");
            if (onlineMode.get())
                config.add("online");
            Record record = validate(recordString, "", config);
            if (record == null) {
                logger.error("Error in \'" + filename + "\'.");
                haserror.set(true);
            } else // additional tests
            {
                logger.trace("validation passed for " + filename);
                // compare ACCESSION with filename
                accession = record.ACCESSION();
                if (!accession.equals(FilenameUtils.getBaseName(filename.toString()))) {
                    logger.error("Error in \'" + filename.getName().toString() + "\'.");
                    logger.error("ACCESSION \'" + record.ACCESSION() + "\' does not match filename \'" + filename.getName().toString() + "\'");
                    haserror.set(true);
                }
                // validate correct serialization: String <-> (String -> Record class -> String)
                String recordStringFromRecord = record.toString();
                int position = StringUtils.indexOfDifference(new String[] { recordString, recordStringFromRecord });
                if (position != -1) {
                    logger.error("Error in \'" + filename + "\'.");
                    logger.error("File content differs from generated record string.\nThis might be a code problem. Please Report!");
                    String[] tokens = recordStringFromRecord.split("\\n");
                    int line = 0, col = 0, offset = 0;
                    for (String token : tokens) {
                        offset = offset + token.length() + 1;
                        if (position < offset) {
                            col = position - (offset - (token.length() + 1));
                            logger.error("Error in line " + (line + 1) + ".");
                            logger.error(tokens[line]);
                            StringBuilder error_at = new StringBuilder(StringUtils.repeat(" ", col));
                            error_at.append('^');
                            logger.error(error_at);
                            haserror.set(true);
                            break;
                        }
                        line++;
                    }
                }
                // validate correct serialization with db: String <-> (db -> Record class -> String)
                if (doDatbase.get()) {
                    Record recordDatabase = null;
                    try {
                        DatabaseManager dbMan = new DatabaseManager("MassBank");
                        recordDatabase = dbMan.getAccessionData(record.ACCESSION());
                        dbMan.closeConnection();
                    } catch (SQLException | ConfigurationException e) {
                        e.printStackTrace();
                        System.exit(1);
                    }
                    if (recordDatabase == null) {
                        String errormsg = "retrieval of '" + record.ACCESSION() + "' from database failed";
                        logger.error(errormsg);
                        System.exit(1);
                    }
                    String recordStringFromDB = recordDatabase.toString();
                    position = StringUtils.indexOfDifference(new String[] { recordString, recordStringFromDB });
                    if (position != -1) {
                        logger.error("Error in \'" + filename + "\'.");
                        logger.error("File content differs from generated record string from database content.\nThis might be a code problem. Please Report!");
                        String[] tokens = recordStringFromDB.split("\\n");
                        int line = 0, col = 0, offset = 0;
                        for (String token : tokens) {
                            offset = offset + token.length() + 1;
                            if (position < offset) {
                                col = position - (offset - (token.length() + 1));
                                logger.error("Error in line " + (line + 1) + ".");
                                logger.error(tokens[line]);
                                StringBuilder error_at = new StringBuilder(StringUtils.repeat(" ", col));
                                error_at.append('^');
                                logger.error(error_at);
                                haserror.set(true);
                                break;
                            }
                            line++;
                        }
                    }
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
            System.exit(1);
        }
        return accession;
    }).filter(Objects::nonNull).collect(Collectors.toList());
    // check duplicates
    Set<String> duplicates = new LinkedHashSet<String>();
    Set<String> uniques = new HashSet<String>();
    for (String c : accessions) {
        // System.out.println(c);
        if (!uniques.add(c)) {
            duplicates.add(c);
        }
    }
    if (duplicates.size() > 0) {
        logger.error("There are duplicates in all accessions:");
        logger.error(duplicates.toString());
        haserror.set(true);
    }
    // return 1 if there were errors
    if (haserror.get())
        System.exit(1);
    else
        System.exit(0);
}

Example 2 with DatabaseManager

use of massbank.db.DatabaseManager in project MassBank-web by MassBank.

the class RefreshDatabase method main.

public static void main(String[] args) throws FileNotFoundException, SQLException, ConfigurationException, IOException {
    // load version and print
    final Properties properties = new Properties();
    try {
        properties.load(ClassLoader.getSystemClassLoader().getResourceAsStream("project.properties"));
    } catch (IOException e) {
        e.printStackTrace();
        System.exit(1);
    }
    System.out.println("RefreshDatabase version: " + properties.getProperty("version"));
    logger.trace("Creating a new database \"" + Config.get().tmpdbName() + "\" and initialize a MassBank database scheme.");
    DatabaseManager.init_db(Config.get().tmpdbName());
    logger.trace("Creating a DatabaseManager for \"" + Config.get().tmpdbName() + "\".");
    final DatabaseManager db = new DatabaseManager(Config.get().tmpdbName());
    logger.trace("Get version of data source.");
    String version = FileUtils.readFileToString(new File(Config.get().DataRootPath() + "/VERSION"), StandardCharsets.UTF_8);
    logger.info("Opening DataRootPath \"" + Config.get().DataRootPath() + "\" and iterate over content.");
    File dataRootPath = new File(Config.get().DataRootPath());
    List<File> recordfiles = new ArrayList<>();
    for (String file : dataRootPath.list(DirectoryFileFilter.INSTANCE)) {
        if (file.equals(".scripts"))
            continue;
        if (file.equals(".figure"))
            continue;
        recordfiles.addAll(FileUtils.listFiles(new File(dataRootPath, file), new String[] { "txt" }, true));
    }
    AtomicInteger index = new AtomicInteger(0);
    int chunkSize = 5000;
    Stream<List<File>> chunkedRecordfiles = recordfiles.stream().collect(Collectors.groupingBy(x -> index.getAndIncrement() / chunkSize)).entrySet().stream().map(Map.Entry::getValue);
    AtomicInteger processed = new AtomicInteger(1);
    int numRecordFiles = recordfiles.size();
    chunkedRecordfiles.forEach(chunk -> {
        chunk.parallelStream().map(filename -> {
            Record record = null;
            logger.info("Validating \"" + filename + "\".");
            String contributor = filename.getParentFile().getName();
            try {
                String recordAsString = FileUtils.readFileToString(filename, StandardCharsets.UTF_8);
                Set<String> config = new HashSet<String>();
                config.add("legacy");
                record = Validator.validate(recordAsString, contributor, config);
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
            if (record == null) {
                logger.error("Error reading/validating record \"" + filename.toString() + "\".");
            }
            return record;
        }).filter(Objects::nonNull).forEachOrdered((r) -> {
            db.persistAccessionFile(r);
            System.out.print("Processed: " + processed.getAndIncrement() + "/" + numRecordFiles + "\r");
        });
    });
    logger.trace("Setting Timestamp in database");
    PreparedStatement stmnt = db.getConnection().prepareStatement("INSERT INTO LAST_UPDATE (TIME,VERSION) VALUES (CURRENT_TIMESTAMP,?);");
    stmnt.setString(1, version);
    stmnt.executeUpdate();
    db.getConnection().commit();
    db.closeConnection();
    logger.trace("Moving new database to MassBank database.");
    DatabaseManager.move_temp_db_to_main_massbank();
}

Example 3 with DatabaseManager

use of massbank.db.DatabaseManager in project MassBank-web by MassBank.

the class RecordApiServiceImpl method recordIdGet.

@Override
public Response recordIdGet(String id, SecurityContext securityContext) throws NotFoundException {
    Record record = null;
    // load record from database
    DatabaseManager dbMan;
    try {
        dbMan = new DatabaseManager("MassBank");
        record = dbMan.getAccessionData(id);
        dbMan.closeConnection();
    } catch (SQLException | ConfigurationException e) {
        return Response.status(Status.INTERNAL_SERVER_ERROR).entity("Database connection error").build();
    }
    if (record == null) {
        return Response.status(Status.NOT_FOUND).entity("Record \"" + id + "\" not found").build();
    }
    String recordstring = record.toString();
    return Response.ok().entity(recordstring).build();
}

Example 4 with DatabaseManager

use of massbank.db.DatabaseManager in project MassBank-web by MassBank.

the class RecordDisplay method doGet.

// @id  https://massbank.eu/MassBank/RecordDisplay.jsp?id=WA001202&dsn=Waters
// measurementTechnique LC-ESI-Q
// "biologicalRole": [
// {
// "@type": "DefinedTerm",
// "@id": "http://ncicb.nci.nih.gov/xml/owl/EVS/Thesaurus.owl#C66892",
// "inDefinedTermSet":
// {
// "@type":"DefinedTermSet",
// "@id":"http://data.bioontology.org/ontologies/NCIT/submissions/69/download?apikey=8b5b7825-538d-40e0-9e9e-5ab9274a9aeb",
// "name": "National Cancer Institute Thesaurus"
// },
// "termCode": "C66892",
// "name": "natural product",
// "url": "http://bioportal.bioontology.org/ontologies/NCIT?p=classes&conceptid=http%3A%2F%2Fncicb.nci.nih.gov%2Fxml%2Fowl%2FEVS%2FThesaurus.owl%23C66892"
// }
// ]
// }
// case "AUTHORS":
// //sb.append(tag + ": " + value + "\n");
// String[] authorTokens	= value.split(", ");
// 
// // check which tokens are authors
// int lastAuthorIdx	= -1;
// for(int i = 0; i < authorTokens.length; i++){
// // check if string is author like 'Akimoto AV'
// boolean isAuthor	= authorTokens[i].matches("\\w+ \\w+");
// if(isAuthor){
// // 2nd word is initials?
// String[] initials	= authorTokens[i].split(" ")[1].split("");
// for(int j = 0; j < initials.length; j++)
// if(!Character.isUpperCase(initials[j].toCharArray()[0])){
// isAuthor	= false;
// break;
// }
// }
// if(isAuthor){
// lastAuthorIdx	= i;
// }
// }
// 
// // create affiliation
// int numberOfAuthors	= lastAuthorIdx + 1;
// String affiliation	= String.join(", ", Arrays.copyOfRange(authorTokens, numberOfAuthors, authorTokens.length));
// 
// // create authors
// String[] authors	= new String[numberOfAuthors];
// for(int i = 0; i < numberOfAuthors; i++)
// authors[i]	=
// "<span property=\"schema:author\" typeof=\"schema:Person\">" +
// "<span property=\"schema:name\">" + authorTokens[i] + "</span>" +
// //((affiliation.length() > 0) ?"<span property=\"schema:affiliation\" style=\"visibility:hidden\">" + affiliation + "</span>" : "") +
// ((affiliation.length() > 0) ?"<span property=\"schema:affiliation\" style=\"display:none\">" + affiliation + "</span>" : "") +
// "</span>";
// 
// // paste
// sb.append(tag + ": ");
// sb.append(String.join(", ", authors));
// if(affiliation.length() > 0)
// sb.append(", " + affiliation);
// sb.append("\n");
// break;
// case "CH$CDK_DEPICT_SMILES":
// case "CH$CDK_DEPICT_GENERIC_SMILES":
// case "CH$CDK_DEPICT_STRUCTURE_SMILES":
// ClickablePreviewImageData clickablePreviewImageData2	= StructureToSvgStringGenerator.createClickablePreviewImage(
// tag, null, value,
// tmpFileFolder, tmpUrlFolder,
// 80, 200, 436
// );
// if(clickablePreviewImageData2 != null)
// sb.append(tag + ": " + clickablePreviewImageData2.getMediumClickablePreviewLink("CH$CDK_DEPICT_SMILES", value));
// else
// sb.append(tag + ": " + value + "\n");
// break;
// // AC$INSTRUMENT
// case "AC$INSTRUMENT_TYPE":
// // TODO property="schema:measurementTechnique"
// sb.append(tag + ": <span property=\"schema:measurementTechnique\">" + value + "</span>\n");
// instrumentType	= value;
// break;
// 
// if(recordTitle == null)
// recordTitle	= "NA";
// 
// String shortName	= recordTitle.split(";")[0].trim();
// 
// ClickablePreviewImageData clickablePreviewImageData	= StructureToSvgStringGenerator.createClickablePreviewImage(
// accession, inchi, smiles, tmpFileFolder, tmpUrlFolder,
// 80, 200, 436
// );
// String svgMedium	= null;
// if(clickablePreviewImageData != null)
// svgMedium	= clickablePreviewImageData.getMediumClickableImage();
// 
// // meta data
// String[] compoundClasses	= compoundClass.split("; ");
// String compoundClass2	= compoundClasses[0];
// if(compoundClass2.equals("NA") && compoundClasses.length > 1)	compoundClass2	= compoundClasses[1];
// 
// String description	=
// "This MassBank Record with Accession " + accession +
// " contains the " + msType + " mass spectrum" +
// " of '" + name + "'" +
// (inchiKey			!= null		? " with the InChIKey '" + inchiKey + "'"					: "") +
// (!compoundClass2.equals("N/A")	? " with the compound class '" + compoundClass2 + "'"	: "") +
// "." +
// " The mass spectrum was acquired on a " + instrumentType +
// //" with " + (ionization != null	? ionization											: "") +
// " with " + ionMode + " ionisation" +
// (fragmentation		!= null		? " using " + fragmentation + " fragmentation"			: "") +
// (collisionEnergy	!= null		? " with the collision energy '" + collisionEnergy + "'": "") +
// (collisionEnergy	!= null		? " at a resolution of " + resolution					: "") +
// (splash				!= null		? " and has the SPLASH '" + splash + "'"				: "") +
// ".";
// 
// // record
// String recordString	= sb.toString();
@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    // preprocess request
    Record record = null;
    try {
        // get parameters
        String accession = null;
        Enumeration<String> names = request.getParameterNames();
        while (names.hasMoreElements()) {
            String key = (String) names.nextElement();
            String val = (String) request.getParameter(key);
            switch(key) {
                case "id":
                    accession = val;
                    break;
                default:
                    logger.warn("unused argument " + key + "=" + val);
            }
        }
        // error handling
        if ("".equals(accession))
            accession = null;
        if (accession == null) {
            String errormsg = "missing argument 'id'.";
            logger.error(errormsg);
            String redirectUrl = "/NoRecordPage" + "?error=" + errormsg;
            response.sendRedirect(request.getContextPath() + redirectUrl);
            return;
        }
        // load record for display
        DatabaseManager dbMan = new DatabaseManager("MassBank");
        record = dbMan.getAccessionData(accession);
        dbMan.closeConnection();
        if (record == null) {
            String errormsg = "retrieval of '" + accession + "' from database failed";
            logger.error(errormsg);
            String redirectUrl = "/NoRecordPage" + "?id=" + accession + "&error=" + errormsg;
            response.sendRedirect(request.getContextPath() + redirectUrl);
            return;
        }
        if (record.DEPRECATED()) {
            logger.trace("Show deprecated record " + accession + ".");
            String shortname = "DEPRECATED RECORD " + accession;
            request.setAttribute("short_name", shortname);
            request.setAttribute("description", shortname);
            request.setAttribute("accession", accession);
            request.setAttribute("isDeprecated", true);
            request.setAttribute("record_title", accession + " has been deprecated.");
            request.setAttribute("recordstring", "<pre>\nACCESSION: " + accession + "\nDEPRECATED: " + record.DEPRECATED_CONTENT() + "\n<pre>");
            request.setAttribute("author", "MassBank");
        } else {
            logger.trace("Show record " + accession + ".");
            String shortname = record.RECORD_TITLE().get(0) + " Mass Spectrum";
            request.setAttribute("short_name", shortname);
            // find InChIKey in CH_LINK
            String inchikey = record.CH_LINK().get("INCHIKEY");
            String description = "This MassBank Record with Accession " + accession + " contains the " + record.AC_MASS_SPECTROMETRY_MS_TYPE() + " mass spectrum" + " of '" + record.RECORD_TITLE().get(0) + "'" + (inchikey != null ? " with the InChIKey '" + inchikey + "'" : "") + // (splash				!= null		? " and has the SPLASH '" + splash + "'"				: "") +
            ".";
            request.setAttribute("description", description);
            String keywords = accession + ", " + shortname + ", " + (inchikey != null ? inchikey + ", " : "") + "mass spectrum, MassBank record, mass spectrometry, mass spectral library";
            request.setAttribute("keywords", keywords);
            String author = record.AUTHORS();
            request.setAttribute("author", author);
            String recordstring = record.createRecordString();
            String structureddata = record.createStructuredData();
            IAtomContainer mol = record.CH_SMILES_obj();
            String svg = new DepictionGenerator().withAtomColors().withMolTitle().withTitleColor(Color.black).depict(mol).toSvgStr(Depiction.UNITS_PX);
            // adjust svg to fit nicely in RecordDisplay page
            DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
            factory.setNamespaceAware(false);
            factory.setValidating(false);
            factory.setFeature("http://xml.org/sax/features/namespaces", false);
            factory.setFeature("http://xml.org/sax/features/validation", false);
            factory.setFeature("http://apache.org/xml/features/nonvalidating/load-dtd-grammar", false);
            factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
            Document svgDoc = factory.newDocumentBuilder().parse(new InputSource(new StringReader(svg)));
            Element svgNode = (Element) svgDoc.getElementsByTagName("svg").item(0);
            NamedNodeMap attr = svgNode.getAttributes();
            attr.getNamedItem("width").setTextContent("100%");
            attr.getNamedItem("height").setTextContent("200px");
            svgNode.setAttribute("preserveAspectRatio", "xMinYMin meet");
            StringWriter writer = new StringWriter();
            TransformerFactory.newInstance().newTransformer().transform(new DOMSource(svgDoc), new StreamResult(writer));
            svg = writer.getBuffer().toString();
            request.setAttribute("peaklist", record.createPeakListData());
            request.setAttribute("accession", accession);
            request.setAttribute("record_title", record.RECORD_TITLE1());
            request.setAttribute("recordstring", recordstring);
            request.setAttribute("structureddata", structureddata);
            request.setAttribute("svg", svg);
        }
        request.getRequestDispatcher("/RecordDisplay.jsp").forward(request, response);
    } catch (Exception e) {
        throw new ServletException("Cannot load record", e);
    }
}

Example 5 with DatabaseManager

use of massbank.db.DatabaseManager in project MassBank-web by MassBank.

the class SiteMapServlet method init.

public void init() throws ServletException {
    logger.trace("ServletContext.TEMPDIR: " + getServletContext().getAttribute(ServletContext.TEMPDIR));
    File tmpdir = (File) getServletContext().getAttribute(ServletContext.TEMPDIR);
    // remove old index
    for (File file : tmpdir.listFiles()) {
        if (file.getName().matches("sitemap.*\\.xml$")) {
            logger.trace("Remove old sitemap: " + file.toString());
            file.delete();
        }
    }
    try {
        // create sitemap generator
        String sitemapbaseurl = Config.get().SitemapBaseURL();
        if (!sitemapbaseurl.endsWith("/"))
            sitemapbaseurl = sitemapbaseurl + "/";
        WebSitemapGenerator wsg = new WebSitemapGenerator(sitemapbaseurl, tmpdir);
        // add static content
        wsg.addUrl(sitemapbaseurl);
        wsg.addUrl(sitemapbaseurl + "Index");
        wsg.addUrl(sitemapbaseurl + "Search");
        wsg.addUrl(sitemapbaseurl + "RecordIndex");
        // add dynamic content
        DatabaseManager databaseManager = new DatabaseManager("MassBank");
        PreparedStatement stmnt = databaseManager.getConnection().prepareStatement("SELECT ACCESSION FROM RECORD");
        ResultSet res = stmnt.executeQuery();
        while (res.next()) {
            wsg.addUrl(sitemapbaseurl + "RecordDisplay?id=" + res.getString(1));
        }
        databaseManager.closeConnection();
        // write new sitemaps
        List<File> sitemaps = wsg.write();
        logger.trace("File(s) written:\n" + sitemaps);
        // write sitemap index
        SitemapIndexGenerator sig = new SitemapIndexGenerator(sitemapbaseurl, new File(tmpdir, "sitemapindex.xml"));
        for (File sitemap : sitemaps) {
            sig.addUrl(sitemapbaseurl + "sitemap/" + sitemap.getName());
        }
        sig.write();
        // get the current database timestamp
        timestamp = new DatabaseTimestamp();
    } catch (ConfigurationException | MalformedURLException | SQLException e) {
        logger.error(e.getMessage());
    }
}