use of massbank.db.DatabaseManager in project MassBank-web by MassBank.
the class Validator method main.
public static void main(String[] arguments) {
// load version and print
final Properties properties = new Properties();
try {
properties.load(ClassLoader.getSystemClassLoader().getResourceAsStream("project.properties"));
} catch (IOException e) {
e.printStackTrace();
System.exit(1);
}
System.out.println("Validator version: " + properties.getProperty("version"));
// parse command line
Options options = new Options();
options.addOption(null, "db", false, "also read record from database and compare with original Record; Developer Feature!");
options.addOption(null, "legacy", false, "less strict mode for legacy records with minor problems.");
options.addOption(null, "online", false, "also do online checks, like PubChem CID check.");
CommandLine cmd = null;
try {
cmd = new DefaultParser().parse(options, arguments);
} catch (ParseException e) {
// oops, something went wrong
System.err.println("Parsing command line failed. Reason: " + e.getMessage());
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("Validator [OPTIONS] <FILE|DIR> [<FILE|DIR> ...]", options);
System.exit(1);
}
if (cmd.getArgList().size() == 0) {
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("Validator [OPTIONS] <FILE|DIR> [<FILE|DIR> ...]", options);
System.exit(1);
}
if (cmd.hasOption("legacy"))
System.out.println("Validation mode: legacy");
// find all files in arguments and all *.txt files in directories and subdirectories
// specified in arguments
List<File> recordfiles = new ArrayList<>();
for (String argument : cmd.getArgList()) {
File argumentf = new File(argument);
if (argumentf.isFile() && FilenameUtils.getExtension(argument).equals("txt")) {
recordfiles.add(argumentf);
} else if (argumentf.isDirectory()) {
recordfiles.addAll(FileUtils.listFiles(argumentf, new String[] { "txt" }, true));
} else {
logger.warn("Argument " + argument + " could not be processed.");
}
}
if (recordfiles.size() == 0) {
logger.error("No files found for validation.");
System.exit(1);
}
// validate all files
logger.trace("Validating " + recordfiles.size() + " files");
AtomicBoolean haserror = new AtomicBoolean(false);
AtomicBoolean doDatbase = new AtomicBoolean(cmd.hasOption("db"));
AtomicBoolean legacyMode = new AtomicBoolean(cmd.hasOption("legacy"));
AtomicBoolean onlineMode = new AtomicBoolean(cmd.hasOption("online"));
List<String> accessions = recordfiles.parallelStream().map(filename -> {
String recordString;
String accession = null;
logger.info("Working on " + filename + ".");
try {
recordString = FileUtils.readFileToString(filename, StandardCharsets.UTF_8);
if (hasNonStandardChars(recordString)) {
logger.warn("Check " + filename + ".");
}
;
// basic validation
Set<String> config = new HashSet<String>();
if (legacyMode.get())
config.add("legacy");
if (onlineMode.get())
config.add("online");
Record record = validate(recordString, "", config);
if (record == null) {
logger.error("Error in \'" + filename + "\'.");
haserror.set(true);
} else // additional tests
{
logger.trace("validation passed for " + filename);
// compare ACCESSION with filename
accession = record.ACCESSION();
if (!accession.equals(FilenameUtils.getBaseName(filename.toString()))) {
logger.error("Error in \'" + filename.getName().toString() + "\'.");
logger.error("ACCESSION \'" + record.ACCESSION() + "\' does not match filename \'" + filename.getName().toString() + "\'");
haserror.set(true);
}
// validate correct serialization: String <-> (String -> Record class -> String)
String recordStringFromRecord = record.toString();
int position = StringUtils.indexOfDifference(new String[] { recordString, recordStringFromRecord });
if (position != -1) {
logger.error("Error in \'" + filename + "\'.");
logger.error("File content differs from generated record string.\nThis might be a code problem. Please Report!");
String[] tokens = recordStringFromRecord.split("\\n");
int line = 0, col = 0, offset = 0;
for (String token : tokens) {
offset = offset + token.length() + 1;
if (position < offset) {
col = position - (offset - (token.length() + 1));
logger.error("Error in line " + (line + 1) + ".");
logger.error(tokens[line]);
StringBuilder error_at = new StringBuilder(StringUtils.repeat(" ", col));
error_at.append('^');
logger.error(error_at);
haserror.set(true);
break;
}
line++;
}
}
// validate correct serialization with db: String <-> (db -> Record class -> String)
if (doDatbase.get()) {
Record recordDatabase = null;
try {
DatabaseManager dbMan = new DatabaseManager("MassBank");
recordDatabase = dbMan.getAccessionData(record.ACCESSION());
dbMan.closeConnection();
} catch (SQLException | ConfigurationException e) {
e.printStackTrace();
System.exit(1);
}
if (recordDatabase == null) {
String errormsg = "retrieval of '" + record.ACCESSION() + "' from database failed";
logger.error(errormsg);
System.exit(1);
}
String recordStringFromDB = recordDatabase.toString();
position = StringUtils.indexOfDifference(new String[] { recordString, recordStringFromDB });
if (position != -1) {
logger.error("Error in \'" + filename + "\'.");
logger.error("File content differs from generated record string from database content.\nThis might be a code problem. Please Report!");
String[] tokens = recordStringFromDB.split("\\n");
int line = 0, col = 0, offset = 0;
for (String token : tokens) {
offset = offset + token.length() + 1;
if (position < offset) {
col = position - (offset - (token.length() + 1));
logger.error("Error in line " + (line + 1) + ".");
logger.error(tokens[line]);
StringBuilder error_at = new StringBuilder(StringUtils.repeat(" ", col));
error_at.append('^');
logger.error(error_at);
haserror.set(true);
break;
}
line++;
}
}
}
}
} catch (IOException e) {
e.printStackTrace();
System.exit(1);
}
return accession;
}).filter(Objects::nonNull).collect(Collectors.toList());
// check duplicates
Set<String> duplicates = new LinkedHashSet<String>();
Set<String> uniques = new HashSet<String>();
for (String c : accessions) {
// System.out.println(c);
if (!uniques.add(c)) {
duplicates.add(c);
}
}
if (duplicates.size() > 0) {
logger.error("There are duplicates in all accessions:");
logger.error(duplicates.toString());
haserror.set(true);
}
// return 1 if there were errors
if (haserror.get())
System.exit(1);
else
System.exit(0);
}
use of massbank.db.DatabaseManager in project MassBank-web by MassBank.
the class RefreshDatabase method main.
public static void main(String[] args) throws FileNotFoundException, SQLException, ConfigurationException, IOException {
// load version and print
final Properties properties = new Properties();
try {
properties.load(ClassLoader.getSystemClassLoader().getResourceAsStream("project.properties"));
} catch (IOException e) {
e.printStackTrace();
System.exit(1);
}
System.out.println("RefreshDatabase version: " + properties.getProperty("version"));
logger.trace("Creating a new database \"" + Config.get().tmpdbName() + "\" and initialize a MassBank database scheme.");
DatabaseManager.init_db(Config.get().tmpdbName());
logger.trace("Creating a DatabaseManager for \"" + Config.get().tmpdbName() + "\".");
final DatabaseManager db = new DatabaseManager(Config.get().tmpdbName());
logger.trace("Get version of data source.");
String version = FileUtils.readFileToString(new File(Config.get().DataRootPath() + "/VERSION"), StandardCharsets.UTF_8);
logger.info("Opening DataRootPath \"" + Config.get().DataRootPath() + "\" and iterate over content.");
File dataRootPath = new File(Config.get().DataRootPath());
List<File> recordfiles = new ArrayList<>();
for (String file : dataRootPath.list(DirectoryFileFilter.INSTANCE)) {
if (file.equals(".scripts"))
continue;
if (file.equals(".figure"))
continue;
recordfiles.addAll(FileUtils.listFiles(new File(dataRootPath, file), new String[] { "txt" }, true));
}
AtomicInteger index = new AtomicInteger(0);
int chunkSize = 5000;
Stream<List<File>> chunkedRecordfiles = recordfiles.stream().collect(Collectors.groupingBy(x -> index.getAndIncrement() / chunkSize)).entrySet().stream().map(Map.Entry::getValue);
AtomicInteger processed = new AtomicInteger(1);
int numRecordFiles = recordfiles.size();
chunkedRecordfiles.forEach(chunk -> {
chunk.parallelStream().map(filename -> {
Record record = null;
logger.info("Validating \"" + filename + "\".");
String contributor = filename.getParentFile().getName();
try {
String recordAsString = FileUtils.readFileToString(filename, StandardCharsets.UTF_8);
Set<String> config = new HashSet<String>();
config.add("legacy");
record = Validator.validate(recordAsString, contributor, config);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
if (record == null) {
logger.error("Error reading/validating record \"" + filename.toString() + "\".");
}
return record;
}).filter(Objects::nonNull).forEachOrdered((r) -> {
db.persistAccessionFile(r);
System.out.print("Processed: " + processed.getAndIncrement() + "/" + numRecordFiles + "\r");
});
});
logger.trace("Setting Timestamp in database");
PreparedStatement stmnt = db.getConnection().prepareStatement("INSERT INTO LAST_UPDATE (TIME,VERSION) VALUES (CURRENT_TIMESTAMP,?);");
stmnt.setString(1, version);
stmnt.executeUpdate();
db.getConnection().commit();
db.closeConnection();
logger.trace("Moving new database to MassBank database.");
DatabaseManager.move_temp_db_to_main_massbank();
}
use of massbank.db.DatabaseManager in project MassBank-web by MassBank.
the class RecordApiServiceImpl method recordIdGet.
@Override
public Response recordIdGet(String id, SecurityContext securityContext) throws NotFoundException {
Record record = null;
// load record from database
DatabaseManager dbMan;
try {
dbMan = new DatabaseManager("MassBank");
record = dbMan.getAccessionData(id);
dbMan.closeConnection();
} catch (SQLException | ConfigurationException e) {
return Response.status(Status.INTERNAL_SERVER_ERROR).entity("Database connection error").build();
}
if (record == null) {
return Response.status(Status.NOT_FOUND).entity("Record \"" + id + "\" not found").build();
}
String recordstring = record.toString();
return Response.ok().entity(recordstring).build();
}
use of massbank.db.DatabaseManager in project MassBank-web by MassBank.
the class RecordDisplay method doGet.
// @id https://massbank.eu/MassBank/RecordDisplay.jsp?id=WA001202&dsn=Waters
// measurementTechnique LC-ESI-Q
// "biologicalRole": [
// {
// "@type": "DefinedTerm",
// "@id": "http://ncicb.nci.nih.gov/xml/owl/EVS/Thesaurus.owl#C66892",
// "inDefinedTermSet":
// {
// "@type":"DefinedTermSet",
// "@id":"http://data.bioontology.org/ontologies/NCIT/submissions/69/download?apikey=8b5b7825-538d-40e0-9e9e-5ab9274a9aeb",
// "name": "National Cancer Institute Thesaurus"
// },
// "termCode": "C66892",
// "name": "natural product",
// "url": "http://bioportal.bioontology.org/ontologies/NCIT?p=classes&conceptid=http%3A%2F%2Fncicb.nci.nih.gov%2Fxml%2Fowl%2FEVS%2FThesaurus.owl%23C66892"
// }
// ]
// }
// case "AUTHORS":
// //sb.append(tag + ": " + value + "\n");
// String[] authorTokens = value.split(", ");
//
// // check which tokens are authors
// int lastAuthorIdx = -1;
// for(int i = 0; i < authorTokens.length; i++){
// // check if string is author like 'Akimoto AV'
// boolean isAuthor = authorTokens[i].matches("\\w+ \\w+");
// if(isAuthor){
// // 2nd word is initials?
// String[] initials = authorTokens[i].split(" ")[1].split("");
// for(int j = 0; j < initials.length; j++)
// if(!Character.isUpperCase(initials[j].toCharArray()[0])){
// isAuthor = false;
// break;
// }
// }
// if(isAuthor){
// lastAuthorIdx = i;
// }
// }
//
// // create affiliation
// int numberOfAuthors = lastAuthorIdx + 1;
// String affiliation = String.join(", ", Arrays.copyOfRange(authorTokens, numberOfAuthors, authorTokens.length));
//
// // create authors
// String[] authors = new String[numberOfAuthors];
// for(int i = 0; i < numberOfAuthors; i++)
// authors[i] =
// "<span property=\"schema:author\" typeof=\"schema:Person\">" +
// "<span property=\"schema:name\">" + authorTokens[i] + "</span>" +
// //((affiliation.length() > 0) ?"<span property=\"schema:affiliation\" style=\"visibility:hidden\">" + affiliation + "</span>" : "") +
// ((affiliation.length() > 0) ?"<span property=\"schema:affiliation\" style=\"display:none\">" + affiliation + "</span>" : "") +
// "</span>";
//
// // paste
// sb.append(tag + ": ");
// sb.append(String.join(", ", authors));
// if(affiliation.length() > 0)
// sb.append(", " + affiliation);
// sb.append("\n");
// break;
// case "CH$CDK_DEPICT_SMILES":
// case "CH$CDK_DEPICT_GENERIC_SMILES":
// case "CH$CDK_DEPICT_STRUCTURE_SMILES":
// ClickablePreviewImageData clickablePreviewImageData2 = StructureToSvgStringGenerator.createClickablePreviewImage(
// tag, null, value,
// tmpFileFolder, tmpUrlFolder,
// 80, 200, 436
// );
// if(clickablePreviewImageData2 != null)
// sb.append(tag + ": " + clickablePreviewImageData2.getMediumClickablePreviewLink("CH$CDK_DEPICT_SMILES", value));
// else
// sb.append(tag + ": " + value + "\n");
// break;
// // AC$INSTRUMENT
// case "AC$INSTRUMENT_TYPE":
// // TODO property="schema:measurementTechnique"
// sb.append(tag + ": <span property=\"schema:measurementTechnique\">" + value + "</span>\n");
// instrumentType = value;
// break;
//
// if(recordTitle == null)
// recordTitle = "NA";
//
// String shortName = recordTitle.split(";")[0].trim();
//
// ClickablePreviewImageData clickablePreviewImageData = StructureToSvgStringGenerator.createClickablePreviewImage(
// accession, inchi, smiles, tmpFileFolder, tmpUrlFolder,
// 80, 200, 436
// );
// String svgMedium = null;
// if(clickablePreviewImageData != null)
// svgMedium = clickablePreviewImageData.getMediumClickableImage();
//
// // meta data
// String[] compoundClasses = compoundClass.split("; ");
// String compoundClass2 = compoundClasses[0];
// if(compoundClass2.equals("NA") && compoundClasses.length > 1) compoundClass2 = compoundClasses[1];
//
// String description =
// "This MassBank Record with Accession " + accession +
// " contains the " + msType + " mass spectrum" +
// " of '" + name + "'" +
// (inchiKey != null ? " with the InChIKey '" + inchiKey + "'" : "") +
// (!compoundClass2.equals("N/A") ? " with the compound class '" + compoundClass2 + "'" : "") +
// "." +
// " The mass spectrum was acquired on a " + instrumentType +
// //" with " + (ionization != null ? ionization : "") +
// " with " + ionMode + " ionisation" +
// (fragmentation != null ? " using " + fragmentation + " fragmentation" : "") +
// (collisionEnergy != null ? " with the collision energy '" + collisionEnergy + "'": "") +
// (collisionEnergy != null ? " at a resolution of " + resolution : "") +
// (splash != null ? " and has the SPLASH '" + splash + "'" : "") +
// ".";
//
// // record
// String recordString = sb.toString();
@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
// preprocess request
Record record = null;
try {
// get parameters
String accession = null;
Enumeration<String> names = request.getParameterNames();
while (names.hasMoreElements()) {
String key = (String) names.nextElement();
String val = (String) request.getParameter(key);
switch(key) {
case "id":
accession = val;
break;
default:
logger.warn("unused argument " + key + "=" + val);
}
}
// error handling
if ("".equals(accession))
accession = null;
if (accession == null) {
String errormsg = "missing argument 'id'.";
logger.error(errormsg);
String redirectUrl = "/NoRecordPage" + "?error=" + errormsg;
response.sendRedirect(request.getContextPath() + redirectUrl);
return;
}
// load record for display
DatabaseManager dbMan = new DatabaseManager("MassBank");
record = dbMan.getAccessionData(accession);
dbMan.closeConnection();
if (record == null) {
String errormsg = "retrieval of '" + accession + "' from database failed";
logger.error(errormsg);
String redirectUrl = "/NoRecordPage" + "?id=" + accession + "&error=" + errormsg;
response.sendRedirect(request.getContextPath() + redirectUrl);
return;
}
if (record.DEPRECATED()) {
logger.trace("Show deprecated record " + accession + ".");
String shortname = "DEPRECATED RECORD " + accession;
request.setAttribute("short_name", shortname);
request.setAttribute("description", shortname);
request.setAttribute("accession", accession);
request.setAttribute("isDeprecated", true);
request.setAttribute("record_title", accession + " has been deprecated.");
request.setAttribute("recordstring", "<pre>\nACCESSION: " + accession + "\nDEPRECATED: " + record.DEPRECATED_CONTENT() + "\n<pre>");
request.setAttribute("author", "MassBank");
} else {
logger.trace("Show record " + accession + ".");
String shortname = record.RECORD_TITLE().get(0) + " Mass Spectrum";
request.setAttribute("short_name", shortname);
// find InChIKey in CH_LINK
String inchikey = record.CH_LINK().get("INCHIKEY");
String description = "This MassBank Record with Accession " + accession + " contains the " + record.AC_MASS_SPECTROMETRY_MS_TYPE() + " mass spectrum" + " of '" + record.RECORD_TITLE().get(0) + "'" + (inchikey != null ? " with the InChIKey '" + inchikey + "'" : "") + // (splash != null ? " and has the SPLASH '" + splash + "'" : "") +
".";
request.setAttribute("description", description);
String keywords = accession + ", " + shortname + ", " + (inchikey != null ? inchikey + ", " : "") + "mass spectrum, MassBank record, mass spectrometry, mass spectral library";
request.setAttribute("keywords", keywords);
String author = record.AUTHORS();
request.setAttribute("author", author);
String recordstring = record.createRecordString();
String structureddata = record.createStructuredData();
IAtomContainer mol = record.CH_SMILES_obj();
String svg = new DepictionGenerator().withAtomColors().withMolTitle().withTitleColor(Color.black).depict(mol).toSvgStr(Depiction.UNITS_PX);
// adjust svg to fit nicely in RecordDisplay page
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setNamespaceAware(false);
factory.setValidating(false);
factory.setFeature("http://xml.org/sax/features/namespaces", false);
factory.setFeature("http://xml.org/sax/features/validation", false);
factory.setFeature("http://apache.org/xml/features/nonvalidating/load-dtd-grammar", false);
factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
Document svgDoc = factory.newDocumentBuilder().parse(new InputSource(new StringReader(svg)));
Element svgNode = (Element) svgDoc.getElementsByTagName("svg").item(0);
NamedNodeMap attr = svgNode.getAttributes();
attr.getNamedItem("width").setTextContent("100%");
attr.getNamedItem("height").setTextContent("200px");
svgNode.setAttribute("preserveAspectRatio", "xMinYMin meet");
StringWriter writer = new StringWriter();
TransformerFactory.newInstance().newTransformer().transform(new DOMSource(svgDoc), new StreamResult(writer));
svg = writer.getBuffer().toString();
request.setAttribute("peaklist", record.createPeakListData());
request.setAttribute("accession", accession);
request.setAttribute("record_title", record.RECORD_TITLE1());
request.setAttribute("recordstring", recordstring);
request.setAttribute("structureddata", structureddata);
request.setAttribute("svg", svg);
}
request.getRequestDispatcher("/RecordDisplay.jsp").forward(request, response);
} catch (Exception e) {
throw new ServletException("Cannot load record", e);
}
}
use of massbank.db.DatabaseManager in project MassBank-web by MassBank.
the class SiteMapServlet method init.
public void init() throws ServletException {
logger.trace("ServletContext.TEMPDIR: " + getServletContext().getAttribute(ServletContext.TEMPDIR));
File tmpdir = (File) getServletContext().getAttribute(ServletContext.TEMPDIR);
// remove old index
for (File file : tmpdir.listFiles()) {
if (file.getName().matches("sitemap.*\\.xml$")) {
logger.trace("Remove old sitemap: " + file.toString());
file.delete();
}
}
try {
// create sitemap generator
String sitemapbaseurl = Config.get().SitemapBaseURL();
if (!sitemapbaseurl.endsWith("/"))
sitemapbaseurl = sitemapbaseurl + "/";
WebSitemapGenerator wsg = new WebSitemapGenerator(sitemapbaseurl, tmpdir);
// add static content
wsg.addUrl(sitemapbaseurl);
wsg.addUrl(sitemapbaseurl + "Index");
wsg.addUrl(sitemapbaseurl + "Search");
wsg.addUrl(sitemapbaseurl + "RecordIndex");
// add dynamic content
DatabaseManager databaseManager = new DatabaseManager("MassBank");
PreparedStatement stmnt = databaseManager.getConnection().prepareStatement("SELECT ACCESSION FROM RECORD");
ResultSet res = stmnt.executeQuery();
while (res.next()) {
wsg.addUrl(sitemapbaseurl + "RecordDisplay?id=" + res.getString(1));
}
databaseManager.closeConnection();
// write new sitemaps
List<File> sitemaps = wsg.write();
logger.trace("File(s) written:\n" + sitemaps);
// write sitemap index
SitemapIndexGenerator sig = new SitemapIndexGenerator(sitemapbaseurl, new File(tmpdir, "sitemapindex.xml"));
for (File sitemap : sitemaps) {
sig.addUrl(sitemapbaseurl + "sitemap/" + sitemap.getName());
}
sig.write();
// get the current database timestamp
timestamp = new DatabaseTimestamp();
} catch (ConfigurationException | MalformedURLException | SQLException e) {
logger.error(e.getMessage());
}
}
Aggregations