Example 1 with SimulationComparisonSummary
use of cbit.vcell.solver.test.SimulationComparisonSummary in project vcell by virtualcell.
the class TestingFrameworkWindowManager method generateTestCriteriaReport.
/**
* Insert the method's description here.
* Creation date: (8/18/2003 5:36:47 PM)
*/
private String generateTestCriteriaReport(TestCaseNew testCase, TestCriteriaNew testCriteria, Simulation testSim, TFGenerateReport.VCDocumentAndSimInfo userSelectedRefSimInfo) /*,VCDocument refDoc,VCDocument testDocument*/
{
if (testSim.getScanCount() != 1) {
throw new RuntimeException("paramater scan is not supported in Math Testing Framework");
}
SimulationSymbolTable simSymbolTable = new SimulationSymbolTable(testSim, 0);
String simReportStatus = null;
String simReportStatusMessage = null;
StringBuffer reportTCBuffer = new StringBuffer();
VariableComparisonSummary[] failVarSummaries = null;
VariableComparisonSummary[] allVarSummaries = null;
double absErr = 0;
double relErr = 0;
if (testCriteria != null) {
absErr = testCriteria.getMaxAbsError().doubleValue();
relErr = testCriteria.getMaxRelError().doubleValue();
}
try {
VCDocument testDoc = null;
if (testCase instanceof TestCaseNewMathModel) {
MathModelInfo mmInfo = ((TestCaseNewMathModel) testCase).getMathModelInfo();
MathModel mathModel = getRequestManager().getDocumentManager().getMathModel(mmInfo);
testDoc = mathModel;
} else if (testCase instanceof TestCaseNewBioModel) {
TestCaseNewBioModel bioTestCase = (TestCaseNewBioModel) testCase;
// bioTestCase.
BioModelInfo bmInfo = bioTestCase.getBioModelInfo();
BioModel bioModel = getRequestManager().getDocumentManager().getBioModel(bmInfo);
testDoc = bioModel;
}
TFGenerateReport.VCDocumentAndSimInfo refVCDocumentAndSimInfo = null;
if (userSelectedRefSimInfo == null) {
SimulationInfo refSimInfo = testCriteria.getRegressionSimInfo();
if (refSimInfo != null) {
VCDocument refDoc = null;
if (testCriteria instanceof TestCriteriaNewMathModel) {
MathModelInfo mmInfo = ((TestCriteriaNewMathModel) testCriteria).getRegressionMathModelInfo();
MathModel mathModel = getRequestManager().getDocumentManager().getMathModel(mmInfo);
refDoc = mathModel;
} else if (testCriteria instanceof TestCriteriaNewBioModel) {
BioModelInfo bmInfo = ((TestCriteriaNewBioModel) testCriteria).getRegressionBioModelInfo();
BioModel bioModel = getRequestManager().getDocumentManager().getBioModel(bmInfo);
refDoc = bioModel;
}
refVCDocumentAndSimInfo = new TFGenerateReport.VCDocumentAndSimInfo(refSimInfo, refDoc);
}
reportTCBuffer.append("\t\t" + testSim.getName() + (refVCDocumentAndSimInfo != null ? " (Using TestCrit RegrRefSim)" : "") + " : " + "\n");
} else {
refVCDocumentAndSimInfo = userSelectedRefSimInfo;
reportTCBuffer.append("\t\t" + testSim.getName() + " (Using UserDefined RegrRefSim '" + userSelectedRefSimInfo.getSimInfo().getAuthoritativeVCSimulationIdentifier() + "') : " + "\n");
}
if (testCase.getType().equals(TestCaseNew.REGRESSION) && refVCDocumentAndSimInfo == null) {
reportTCBuffer.append("\t\t\tNo reference SimInfo, SimInfoKey=" + testCriteria.getSimInfo().getVersion().getName() + ". Cannot perform Regression Test!\n");
simReportStatus = TestCriteriaNew.TCRIT_STATUS_NOREFREGR;
} else {
VCDataIdentifier vcdID = new VCSimulationDataIdentifier(testSim.getSimulationInfo().getAuthoritativeVCSimulationIdentifier(), 0);
DataManager simDataManager = getRequestManager().getDataManager(null, vcdID, testSim.isSpatial());
double[] timeArray = null;
// can be histogram, so there won't be time array
try {
timeArray = simDataManager.getDataSetTimes();
} catch (Exception ex) {
ex.printStackTrace(System.out);
}
NonspatialStochSimOptions stochOpt = testSim.getSolverTaskDescription().getStochOpt();
if ((stochOpt == null || stochOpt.getNumOfTrials() == 1) && (timeArray == null || timeArray.length == 0)) {
reportTCBuffer.append("\t\t\tNO DATA : Simulation not run yet.\n");
simReportStatus = TestCriteriaNew.TCRIT_STATUS_NODATA;
} else {
// SPATIAL simulation
if (testSim.getMathDescription().isSpatial()) {
PDEDataManager pdeDataManager = (PDEDataManager) simDataManager;
// Get EXACT solution if test case type is EXACT, Compare with numerical
if (testCase.getType().equals(TestCaseNew.EXACT) || testCase.getType().equals(TestCaseNew.EXACT_STEADY)) {
SimulationComparisonSummary simCompSummary = MathTestingUtilities.comparePDEResultsWithExact(simSymbolTable, pdeDataManager, testCase.getType(), testCriteria.getMaxAbsError(), testCriteria.getMaxRelError());
// Failed var summaries
failVarSummaries = simCompSummary.getFailingVariableComparisonSummaries(absErr, relErr);
allVarSummaries = simCompSummary.getVariableComparisonSummaries();
if (failVarSummaries.length > 0) {
simReportStatus = TestCriteriaNew.TCRIT_STATUS_FAILEDVARS;
// Failed simulation
reportTCBuffer.append("\t\tTolerance test FAILED \n");
reportTCBuffer.append("\t\tFailed Variables : \n");
for (int m = 0; m < failVarSummaries.length; m++) {
reportTCBuffer.append("\t\t\t" + failVarSummaries[m].toShortString() + "\n");
}
} else {
simReportStatus = TestCriteriaNew.TCRIT_STATUS_PASSED;
reportTCBuffer.append("\t\tTolerance test PASSED \n");
}
reportTCBuffer.append("\t\tPassed Variables : \n");
// Check if varSummary exists in failed summaries list. If not, simulation passed.
for (int m = 0; m < allVarSummaries.length; m++) {
if (!BeanUtils.arrayContains(failVarSummaries, allVarSummaries[m])) {
reportTCBuffer.append("\t\t\t" + allVarSummaries[m].toShortString() + "\n");
}
}
// Get CONSTRUCTED solution if test case type is CONSTRUCTED, Compare with numerical
} else if (testCase.getType().equals(TestCaseNew.CONSTRUCTED)) {
SimulationComparisonSummary simCompSummary = MathTestingUtilities.comparePDEResultsWithExact(simSymbolTable, pdeDataManager, testCase.getType(), testCriteria.getMaxAbsError(), testCriteria.getMaxRelError());
// Failed var summaries
failVarSummaries = simCompSummary.getFailingVariableComparisonSummaries(absErr, relErr);
allVarSummaries = simCompSummary.getVariableComparisonSummaries();
if (failVarSummaries.length > 0) {
simReportStatus = TestCriteriaNew.TCRIT_STATUS_FAILEDVARS;
// Failed simulation
reportTCBuffer.append("\t\tTolerance test FAILED \n");
reportTCBuffer.append("\t\tFailed Variables : \n");
for (int m = 0; m < failVarSummaries.length; m++) {
reportTCBuffer.append("\t\t\t" + failVarSummaries[m].toShortString() + "\n");
}
} else {
simReportStatus = TestCriteriaNew.TCRIT_STATUS_PASSED;
reportTCBuffer.append("\t\tTolerance test PASSED \n");
}
reportTCBuffer.append("\t\tPassed Variables : \n");
// Check if varSummary exists in failed summaries list. If not, simulation passed.
for (int m = 0; m < allVarSummaries.length; m++) {
if (!BeanUtils.arrayContains(failVarSummaries, allVarSummaries[m])) {
reportTCBuffer.append("\t\t\t" + allVarSummaries[m].toShortString() + "\n");
}
}
} else if (testCase.getType().equals(TestCaseNew.REGRESSION)) {
Simulation refSim = ((ClientDocumentManager) getRequestManager().getDocumentManager()).getSimulation(refVCDocumentAndSimInfo.getSimInfo());
VCDataIdentifier refVcdID = new VCSimulationDataIdentifier(refVCDocumentAndSimInfo.getSimInfo().getAuthoritativeVCSimulationIdentifier(), 0);
PDEDataManager refDataManager = (PDEDataManager) getRequestManager().getDataManager(null, refVcdID, refSim.isSpatial());
if (refSim.getScanCount() != 1) {
throw new RuntimeException("paramater scan is not supported in Math Testing Framework");
}
SimulationSymbolTable refSimSymbolTable = new SimulationSymbolTable(refSim, 0);
String[] varsToCompare = getVariableNamesToCompare(simSymbolTable, refSimSymbolTable);
SimulationComparisonSummary simCompSummary = MathTestingUtilities.comparePDEResults(simSymbolTable, pdeDataManager, refSimSymbolTable, refDataManager, varsToCompare, testCriteria.getMaxAbsError(), testCriteria.getMaxRelError(), refVCDocumentAndSimInfo.getVCDocument(), getDataInfoProvider(refVCDocumentAndSimInfo.getVCDocument(), refDataManager.getPDEDataContext(), refSim.getName()), testDoc, getDataInfoProvider(testDoc, pdeDataManager.getPDEDataContext(), testSim.getName()));
// Failed var summaries
failVarSummaries = simCompSummary.getFailingVariableComparisonSummaries(absErr, relErr);
allVarSummaries = simCompSummary.getVariableComparisonSummaries();
if (failVarSummaries.length > 0) {
simReportStatus = TestCriteriaNew.TCRIT_STATUS_FAILEDVARS;
// Failed simulation
reportTCBuffer.append("\t\tTolerance test FAILED \n");
reportTCBuffer.append("\t\tFailed Variables : \n");
for (int m = 0; m < failVarSummaries.length; m++) {
reportTCBuffer.append("\t\t\t" + failVarSummaries[m].toShortString() + "\n");
}
} else {
simReportStatus = TestCriteriaNew.TCRIT_STATUS_PASSED;
reportTCBuffer.append("\t\tTolerance test PASSED \n");
}
reportTCBuffer.append("\t\tPassed Variables : \n");
// Check if varSummary exists in failed summaries list. If not, simulation passed.
for (int m = 0; m < allVarSummaries.length; m++) {
if (!BeanUtils.arrayContains(failVarSummaries, allVarSummaries[m])) {
reportTCBuffer.append("\t\t\t" + allVarSummaries[m].toShortString() + "\n");
}
}
}
} else {
// NON-SPATIAL CASE
ODEDataManager odeDataManager = (ODEDataManager) simDataManager;
ODESolverResultSet numericalResultSet = odeDataManager.getODESolverResultSet();
// Get EXACT result set if test case type is EXACT, Compare with numerical
if (testCase.getType().equals(TestCaseNew.EXACT) || testCase.getType().equals(TestCaseNew.EXACT_STEADY)) {
ODESolverResultSet exactResultSet = MathTestingUtilities.getExactResultSet(testSim.getMathDescription(), timeArray, testSim.getSolverTaskDescription().getSensitivityParameter());
String[] varsToCompare = getVariableNamesToCompare(simSymbolTable, simSymbolTable);
SimulationComparisonSummary simCompSummary_exact = MathTestingUtilities.compareResultSets(numericalResultSet, exactResultSet, varsToCompare, testCase.getType(), testCriteria.getMaxAbsError(), testCriteria.getMaxRelError());
// Get all the variable comparison summaries and the failed ones to print out report for EXACT solution comparison.
failVarSummaries = simCompSummary_exact.getFailingVariableComparisonSummaries(absErr, relErr);
allVarSummaries = simCompSummary_exact.getVariableComparisonSummaries();
if (failVarSummaries.length > 0) {
simReportStatus = TestCriteriaNew.TCRIT_STATUS_FAILEDVARS;
// Failed simulation
reportTCBuffer.append("\t\tTolerance test FAILED \n");
reportTCBuffer.append("\t\tFailed Variables : \n");
for (int m = 0; m < failVarSummaries.length; m++) {
reportTCBuffer.append("\t\t\t" + failVarSummaries[m].toShortString() + "\n");
}
} else {
simReportStatus = TestCriteriaNew.TCRIT_STATUS_PASSED;
reportTCBuffer.append("\t\tTolerance test PASSED \n");
}
reportTCBuffer.append("\t\tPassed Variables : \n");
// Check if varSummary exists in failed summaries list. If not, simulation passed.
for (int m = 0; m < allVarSummaries.length; m++) {
if (!BeanUtils.arrayContains(failVarSummaries, allVarSummaries[m])) {
reportTCBuffer.append("\t\t\t" + allVarSummaries[m].toShortString() + "\n");
}
}
// Get CONSTRUCTED result set if test case type is CONSTRUCTED , compare with numerical
} else if (testCase.getType().equals(TestCaseNew.CONSTRUCTED)) {
ODESolverResultSet constructedResultSet = MathTestingUtilities.getConstructedResultSet(testSim.getMathDescription(), timeArray);
String[] varsToCompare = getVariableNamesToCompare(simSymbolTable, simSymbolTable);
SimulationComparisonSummary simCompSummary_constr = MathTestingUtilities.compareResultSets(numericalResultSet, constructedResultSet, varsToCompare, testCase.getType(), testCriteria.getMaxAbsError(), testCriteria.getMaxRelError());
// Get all the variable comparison summaries and the failed ones to print out report for CONSTRUCTED solution comparison.
failVarSummaries = simCompSummary_constr.getFailingVariableComparisonSummaries(absErr, relErr);
allVarSummaries = simCompSummary_constr.getVariableComparisonSummaries();
if (failVarSummaries.length > 0) {
simReportStatus = TestCriteriaNew.TCRIT_STATUS_FAILEDVARS;
// Failed simulation
reportTCBuffer.append("\t\tTolerance test FAILED \n");
reportTCBuffer.append("\t\tFailed Variables : \n");
for (int m = 0; m < failVarSummaries.length; m++) {
reportTCBuffer.append("\t\t\t" + failVarSummaries[m].toShortString() + "\n");
}
} else {
simReportStatus = TestCriteriaNew.TCRIT_STATUS_PASSED;
reportTCBuffer.append("\t\tTolerance test PASSED \n");
}
reportTCBuffer.append("\t\tPassed Variables : \n");
// Check if varSummary exists in failed summaries list. If not, simulation passed.
for (int m = 0; m < allVarSummaries.length; m++) {
if (!BeanUtils.arrayContains(failVarSummaries, allVarSummaries[m])) {
reportTCBuffer.append("\t\t\t" + allVarSummaries[m].toShortString() + "\n");
}
}
} else if (testCase.getType().equals(TestCaseNew.REGRESSION)) {
Simulation refSim = ((ClientDocumentManager) getRequestManager().getDocumentManager()).getSimulation(testCriteria.getRegressionSimInfo());
if (refSim.getScanCount() != 1) {
throw new RuntimeException("paramater scan is not supported in Math Testing Framework");
}
SimulationSymbolTable refSimSymbolTable = new SimulationSymbolTable(refSim, 0);
String[] varsToTest = getVariableNamesToCompare(simSymbolTable, refSimSymbolTable);
VCDataIdentifier refVcdID = new VCSimulationDataIdentifier(refVCDocumentAndSimInfo.getSimInfo().getAuthoritativeVCSimulationIdentifier(), 0);
ODEDataManager refDataManager = (ODEDataManager) getRequestManager().getDataManager(null, refVcdID, refSim.isSpatial());
ODESolverResultSet referenceResultSet = refDataManager.getODESolverResultSet();
SimulationComparisonSummary simCompSummary_regr = null;
int interpolationOrder = 1;
SolverTaskDescription solverTaskDescription = refSim.getSolverTaskDescription();
if (solverTaskDescription.getOutputTimeSpec().isDefault() && ((DefaultOutputTimeSpec) solverTaskDescription.getOutputTimeSpec()).getKeepEvery() == 1) {
SolverDescription solverDescription = solverTaskDescription.getSolverDescription();
if ((!solverDescription.supportsAll(SolverDescription.DiscontinutiesFeatures)) || !refSim.getMathDescription().hasDiscontinuities()) {
interpolationOrder = solverDescription.getTimeOrder();
}
}
simCompSummary_regr = MathTestingUtilities.compareUnEqualResultSets(numericalResultSet, referenceResultSet, varsToTest, testCriteria.getMaxAbsError(), testCriteria.getMaxRelError(), interpolationOrder);
// Get all the variable comparison summaries and the failed ones to print out report for CONSTRUCTED solution comparison.
failVarSummaries = simCompSummary_regr.getFailingVariableComparisonSummaries(absErr, relErr);
allVarSummaries = simCompSummary_regr.getVariableComparisonSummaries();
if (failVarSummaries.length > 0) {
simReportStatus = TestCriteriaNew.TCRIT_STATUS_FAILEDVARS;
// Failed simulation
reportTCBuffer.append("\t\tTolerance test FAILED \n");
reportTCBuffer.append("\t\tFailed Variables : \n");
for (int m = 0; m < failVarSummaries.length; m++) {
reportTCBuffer.append("\t\t\t" + failVarSummaries[m].toShortString() + "\n");
}
} else {
simReportStatus = TestCriteriaNew.TCRIT_STATUS_PASSED;
reportTCBuffer.append("\t\tTolerance test PASSED \n");
}
reportTCBuffer.append("\t\tPassed Variables : \n");
// Check if varSummary exists in failed summaries list. If not, simulation passed.
for (int m = 0; m < allVarSummaries.length; m++) {
if (!BeanUtils.arrayContains(failVarSummaries, allVarSummaries[m])) {
reportTCBuffer.append("\t\t\t" + allVarSummaries[m].toShortString() + "\n");
}
}
}
}
}
}
} catch (Throwable e) {
simReportStatus = TestCriteriaNew.TCRIT_STATUS_RPERROR;
simReportStatusMessage = e.getClass().getName() + " " + e.getMessage();
reportTCBuffer.append("\t\t" + simReportStatusMessage + "\n");
e.printStackTrace(System.out);
}
if (userSelectedRefSimInfo == null) {
try {
// Remove any test results already present for testCriteria
RemoveTestResultsOP removeResultsOP = new RemoveTestResultsOP(new BigDecimal[] { testCriteria.getTCritKey() });
// testResultsOPsVector.add(removeResultsOP);
getRequestManager().getDocumentManager().doTestSuiteOP(removeResultsOP);
// Create new AddTestREsultsOP object for the current simulation./testCriteria.
if (allVarSummaries != null) {
AddTestResultsOP testResultsOP = new AddTestResultsOP(testCriteria.getTCritKey(), allVarSummaries);
// testResultsOPsVector.add(testResultsOP);
// Write the testResults for simulation/TestCriteria into the database ...
getRequestManager().getDocumentManager().doTestSuiteOP(testResultsOP);
}
// Update report status
updateTCritStatus(testCriteria, simReportStatus, simReportStatusMessage);
} catch (Throwable e) {
reportTCBuffer.append("\t\tUpdate DB Results failed. " + e.getClass().getName() + " " + e.getMessage() + "\n");
try {
getRequestManager().getDocumentManager().doTestSuiteOP(new EditTestCriteriaOPReportStatus(testCriteria.getTCritKey(), TestCriteriaNew.TCRIT_STATUS_RPERROR, e.getClass().getName() + " " + e.getMessage()));
} catch (Throwable e2) {
// Nothing more can be done
}
}
}
return reportTCBuffer.toString();
}
Also used :
MathModel(cbit.vcell.mathmodel.MathModel)
AddTestCasesOPMathModel(cbit.vcell.numericstest.AddTestCasesOPMathModel)
TestCaseNewMathModel(cbit.vcell.numericstest.TestCaseNewMathModel)
TestCriteriaNewMathModel(cbit.vcell.numericstest.TestCriteriaNewMathModel)
EditTestCriteriaOPMathModel(cbit.vcell.numericstest.EditTestCriteriaOPMathModel)
AddTestCriteriaOPMathModel(cbit.vcell.numericstest.AddTestCriteriaOPMathModel)
SolverDescription(cbit.vcell.solver.SolverDescription)
TestCriteriaNewBioModel(cbit.vcell.numericstest.TestCriteriaNewBioModel)
NonspatialStochSimOptions(cbit.vcell.solver.NonspatialStochSimOptions)
ClientDocumentManager(cbit.vcell.clientdb.ClientDocumentManager)
TFGenerateReport(cbit.vcell.client.task.TFGenerateReport)
TestCaseNewBioModel(cbit.vcell.numericstest.TestCaseNewBioModel)
RemoveTestResultsOP(cbit.vcell.numericstest.RemoveTestResultsOP)
TestCriteriaNewMathModel(cbit.vcell.numericstest.TestCriteriaNewMathModel)
AddTestResultsOP(cbit.vcell.numericstest.AddTestResultsOP)
SimulationComparisonSummary(cbit.vcell.solver.test.SimulationComparisonSummary)
ODESolverResultSet(cbit.vcell.solver.ode.ODESolverResultSet)
VariableComparisonSummary(cbit.vcell.solver.test.VariableComparisonSummary)
SolverTaskDescription(cbit.vcell.solver.SolverTaskDescription)
EditTestCriteriaOPReportStatus(cbit.vcell.numericstest.EditTestCriteriaOPReportStatus)
VCDocument(org.vcell.util.document.VCDocument)
SimulationSymbolTable(cbit.vcell.solver.SimulationSymbolTable)
BioModelInfo(org.vcell.util.document.BioModelInfo)
PDEDataManager(cbit.vcell.simdata.PDEDataManager)
ODEDataManager(cbit.vcell.simdata.ODEDataManager)
DataManager(cbit.vcell.simdata.DataManager)
MathModelInfo(org.vcell.util.document.MathModelInfo)
VCSimulationDataIdentifier(cbit.vcell.solver.VCSimulationDataIdentifier)
ObjectNotFoundException(org.vcell.util.ObjectNotFoundException)
DataAccessException(org.vcell.util.DataAccessException)
UserCancelException(org.vcell.util.UserCancelException)
Simulation(cbit.vcell.solver.Simulation)
PDEDataManager(cbit.vcell.simdata.PDEDataManager)
TestCaseNewBioModel(cbit.vcell.numericstest.TestCaseNewBioModel)
TestCriteriaNewBioModel(cbit.vcell.numericstest.TestCriteriaNewBioModel)
AddTestCasesOPBioModel(cbit.vcell.numericstest.AddTestCasesOPBioModel)
BioModel(cbit.vcell.biomodel.BioModel)
EditTestCriteriaOPBioModel(cbit.vcell.numericstest.EditTestCriteriaOPBioModel)
AddTestCriteriaOPBioModel(cbit.vcell.numericstest.AddTestCriteriaOPBioModel)
TestCaseNewMathModel(cbit.vcell.numericstest.TestCaseNewMathModel)
ODEDataManager(cbit.vcell.simdata.ODEDataManager)
VCDataIdentifier(org.vcell.util.document.VCDataIdentifier)
DefaultOutputTimeSpec(cbit.vcell.solver.DefaultOutputTimeSpec)
SimulationInfo(cbit.vcell.solver.SimulationInfo)
Example 2 with SimulationComparisonSummary
use of cbit.vcell.solver.test.SimulationComparisonSummary in project vcell by virtualcell.
the class BiomodelsDB_TestSuite method main.
public static void main(String[] args) {
try {
Logging.init();
/*sanity check -- we currently only have a copasi_java dll for 32-bit, so let's make sure
* we're running on the right JVM. (Note we can run this on 64 bit machine, just have to install
* a 32 bit JVM...)
*/
OperatingSystemInfo osi = OperatingSystemInfo.getInstance();
if (!osi.isWindows() || osi.is64bit()) {
System.err.println("run on 32 bit JVM");
System.exit(99);
}
// following are set in command line processing
SortedSet<BiomodelsNetEntry> modelIDs = new TreeSet<BiomodelsNetEntry>();
BioModelsWebServices service = null;
File outDir = null;
boolean isDetailed;
{
// scope for command line processing
Options commandOptions = new Options();
Option help = new Option("help", false, "show help");
commandOptions.addOption(help);
Option detailed = new Option("detailed", false, "record detailed information");
commandOptions.addOption(detailed);
Option output = new Option("output", true, "output directory");
output.setRequired(true);
commandOptions.addOption(output);
OptionGroup primary = new OptionGroup();
LOption min = new LOption("min", true, "minimum number of model to import", true);
min.setType(Number.class);
LOption only = new LOption("only", true, "run only this model", false);
only.setType(Number.class);
LOption include = new LOption("include", true, "run on models in specified file", false);
include.setType(String.class);
LOption exclude = new LOption("exclude", true, "exclude models in specified file", true);
exclude.setType(String.class);
primary.addOption(min).addOption(only).addOption(include).addOption(exclude);
for (Object obj : primary.getOptions()) {
// CLI old, non-generic API
commandOptions.addOption((Option) obj);
}
CommandLine cmdLine = null;
try {
CommandLineParser parser = new DefaultParser();
cmdLine = parser.parse(commandOptions, args);
} catch (ParseException e1) {
e1.printStackTrace();
HelpFormatter hf = new HelpFormatter();
hf.printHelp("BiomodelsDB_TestSuite", commandOptions);
System.exit(2);
}
Option[] present = cmdLine.getOptions();
Set<Option> optionSet = new HashSet<Option>(Arrays.asList(present));
if (optionSet.contains(help)) {
HelpFormatter hf = new HelpFormatter();
hf.printHelp("BiomodelsDB_TestSuite", commandOptions);
System.exit(0);
}
// placeholder, avoid messing with nulls
LOption primaryOpt = new LOption("", false, "", true);
@SuppressWarnings("unchecked") Collection<? extends Option> priOpts = primary.getOptions();
priOpts.retainAll(optionSet);
if (!priOpts.isEmpty()) {
assert (priOpts.size() == 1);
primaryOpt = (LOption) priOpts.iterator().next();
}
outDir = new File(cmdLine.getOptionValue(output.getOpt()));
if (!outDir.exists()) {
outDir.mkdirs();
}
isDetailed = optionSet.contains(detailed);
BioModelsWebServicesServiceLocator locator = new BioModelsWebServicesServiceLocator();
service = locator.getBioModelsWebServices();
if (primaryOpt.downloads) {
String[] modelStrings = service.getAllCuratedModelsId();
for (String s : modelStrings) {
modelIDs.add(new BiomodelsNetEntry(s));
}
}
if (primaryOpt == only) {
Long only1 = (Long) cmdLine.getParsedOptionValue(only.getOpt());
modelIDs.add(new BiomodelsNetEntry(only1.intValue()));
} else if (primaryOpt == min) {
Long lv = (Long) cmdLine.getParsedOptionValue(min.getOpt());
int minimumModel = lv.intValue();
for (int m = 0; m < minimumModel; m++) {
modelIDs.remove(new BiomodelsNetEntry(m));
}
} else if (primaryOpt == include) {
FileBaseFilter fbf = new FileBaseFilter(cmdLine.getOptionValue(include.getOpt()));
modelIDs.addAll(fbf.models);
} else if (primaryOpt == exclude) {
FileBaseFilter fbf = new FileBaseFilter(cmdLine.getOptionValue(exclude.getOpt()));
modelIDs.removeAll(fbf.models);
}
}
// end command line processing
WriterFlusher flusher = new WriterFlusher(10);
PrintWriter detailWriter;
PrintWriter bngWriter;
PrintWriter sbmlWriter;
SBMLExceptionSorter sbmlExceptions;
if (isDetailed) {
detailWriter = new PrintWriter(new File(outDir, "compareDetail.txt"));
bngWriter = new PrintWriter(new File(outDir, "bngErrors.txt"));
sbmlWriter = new PrintWriter(new File(outDir, "sbmlErrors.txt"));
sbmlExceptions = new LiveSorter();
flusher.add(detailWriter);
flusher.add(bngWriter);
flusher.add(sbmlWriter);
} else {
detailWriter = new PrintWriter(new NullWriter());
bngWriter = new PrintWriter(new NullWriter());
sbmlWriter = new PrintWriter(new NullWriter());
sbmlExceptions = new NullSorter();
}
PropertyLoader.loadProperties();
/**
* example properties
*
* vcell.COPASI.executable = "C:\\Program Files\\COPASI\\bin\\CopasiSE.exe"
* vcell.mathSBML.directory = "c:\\developer\\eclipse\\workspace\\mathsbml\\"
* vcell.mathematica.kernel.executable = "C:\\Program Files\\Wolfram Research\\Mathematica\\7.0\\MathKernel.exe"
*/
ResourceUtil.setNativeLibraryDirectory();
PrintWriter printWriter = new PrintWriter(new FileWriter(new File(outDir, "summary.log"), true));
flusher.add(printWriter);
listModels(printWriter, modelIDs);
// SBMLSolver copasiSBMLSolver = new CopasiSBMLSolver();
SBMLSolver copasiSBMLSolver = new SBMLSolver() {
@Override
public File solve(String filePrefix, File outDir, String sbmlText, SimSpec testSpec) throws IOException, SolverException, SbmlException {
throw new RuntimeException("COPASI SOLVER REMOVED, REIMPLEMENT");
}
@Override
public String getResultsFileColumnDelimiter() {
throw new RuntimeException("COPASI SOLVER REMOVED, REIMPLEMENT");
}
};
try {
printWriter.println(" | *BIOMODEL ID* | *BioModel name* | *PASS* | *Rel Error (VC/COP)(VC/MSBML)(COP/MSBML)* | *Exception* | ");
removeToxic(modelIDs, printWriter);
for (BiomodelsNetEntry modelID : modelIDs) {
String modelName = service.getModelNameById(modelID.toString());
String modelSBML = service.getModelById(modelID.toString());
Element bioModelInfo = new Element(BioModelsNetPanel.BIOMODELINFO_ELEMENT_NAME);
bioModelInfo.setAttribute(BioModelsNetPanel.ID_ATTRIBUTE_NAME, modelID.toString());
bioModelInfo.setAttribute(BioModelsNetPanel.SUPPORTED_ATTRIBUTE_NAME, "false");
bioModelInfo.setAttribute("vcell_ran", "false");
bioModelInfo.setAttribute("COPASI_ran", "false");
bioModelInfo.setAttribute("mSBML_ran", "false");
bioModelInfo.setAttribute(BioModelsNetPanel.MODELNAME_ATTRIBUTE_NAME, modelName);
boolean bUseUTF8 = true;
File sbmlFile = new File(outDir, modelID + ".sbml");
XmlUtil.writeXMLStringToFile(modelSBML, sbmlFile.getAbsolutePath(), bUseUTF8);
PrintStream saved_sysout = System.out;
PrintStream saved_syserr = System.err;
PrintStream new_sysout = null;
PrintStream new_syserr = null;
try {
String filePrefix = modelID.toString();
String sbmlText = modelSBML;
File logFile = new File(outDir, filePrefix + ".log");
new_sysout = new PrintStream(logFile);
new_syserr = new_sysout;
System.setOut(new_sysout);
System.setErr(new_syserr);
StringBuffer combinedErrorBuffer = new StringBuffer();
SimSpec simSpec = SimSpec.fromSBML(modelSBML);
String[] varsToTest = simSpec.getVarsList();
printWriter.print("ModelId: " + modelID + ": ");
// if a model crashes out the libSBML dll, we will terminate abruptly. Flush
// summary log before that happens
printWriter.flush();
try {
//
// get VCell solution with an embedded "ROUND TRIP" (time and species concentrations)
//
ODESolverResultSet vcellResults_RT = null;
try {
VCellSBMLSolver vcellSBMLSolver_RT = new VCellSBMLSolver();
vcellSBMLSolver_RT.setRoundTrip(false);
// TODO try with round-trip later.
String columnDelimiter = vcellSBMLSolver_RT.getResultsFileColumnDelimiter();
File resultFile = vcellSBMLSolver_RT.solve(filePrefix, outDir, sbmlFile.getAbsolutePath(), simSpec);
vcellResults_RT = readResultFile(resultFile, columnDelimiter);
bioModelInfo.setAttribute("vcell_ran", "true");
} catch (BNGException e) {
bngWriter.println(modelID + " " + e.getMessage());
throw e;
} catch (SBMLImportException e) {
ModelException me = new ModelException(modelID, e);
write(sbmlWriter, me);
sbmlExceptions.add(me);
throw e;
} catch (Exception e) {
printWriter.println("vcell solve(roundtrip=true) failed");
e.printStackTrace(printWriter);
System.out.println("vcell solve(roundtrip=true) failed");
e.printStackTrace(System.out);
combinedErrorBuffer.append(" *VCELL_RT* _" + e.getMessage() + "_ ");
Element vcellSolverReport = new Element("SolverReport");
vcellSolverReport.setAttribute("solverName", "vcell");
vcellSolverReport.setAttribute("errorMessage", e.getMessage());
bioModelInfo.addContent(vcellSolverReport);
bioModelInfo.setAttribute("vcell_ran", "false");
}
//
// get COPASI solution (time and species concentrations)
//
ODESolverResultSet copasiResults = null;
try {
String columnDelimiter = copasiSBMLSolver.getResultsFileColumnDelimiter();
File resultFile = copasiSBMLSolver.solve(filePrefix, outDir, sbmlText, simSpec);
copasiResults = readResultFile(resultFile, columnDelimiter);
bioModelInfo.setAttribute("COPASI_ran", "true");
} catch (Exception e) {
printWriter.println("Copasi solve() failed");
e.printStackTrace(printWriter);
System.out.println("Copasi solve() failed");
e.printStackTrace(System.out);
combinedErrorBuffer.append(" *COPASI* _" + e.getMessage() + "_ ");
Element copasiSolverReport = new Element("SolverReport");
copasiSolverReport.setAttribute("solverName", "COPASI");
copasiSolverReport.setAttribute("errorMessage", e.getMessage());
bioModelInfo.addContent(copasiSolverReport);
bioModelInfo.setAttribute("COPASI_ran", "false");
}
//
// get mSBML solution (time and species concentrations)
//
/*
ODESolverResultSet mSBMLResults = null;
if (idInt != 246 && idInt != 250 && idInt != 285 && idInt != 301) {
try {
MathSBMLSolver mSBMLSolver = new MathSBMLSolver();
String columnDelimiter = mSBMLSolver.getResultsFileColumnDelimiter();
org.sbml.libsbml.SBMLDocument sbmlDocument = new SBMLReader().readSBML(sbmlFile.getAbsolutePath());
long level = sbmlDocument.getLevel();
long version = sbmlDocument.getVersion();
String mathsbmlFilePrefix = filePrefix;
if (level!=2 || (level==2 && version>3)){
// sbmlDocument.setConsistencyChecksForConversion(libsbmlConstants.LIBSBML_CAT_MODELING_PRACTICE, false);
long numErrors = sbmlDocument.checkL2v3Compatibility();
if (numErrors==0){
boolean bConversionWorked = sbmlDocument.setLevelAndVersion(2, 3, false);
SBMLDocument doc = new SBMLDocument(sbmlDocument);
doc.setLevelAndVersion(2,3,false);
if (bConversionWorked){
mathsbmlFilePrefix = filePrefix+"_L2V3";
long newVersion = doc.getVersion();
SBMLWriter sbmlWriter = new SBMLWriter();
sbmlText = sbmlWriter.writeToString(doc);
try {
XmlUtil.writeXMLStringToFile(sbmlText, mathsbmlFilePrefix+".sbml", true);
} catch (IOException e1) {
e1.printStackTrace(System.out);
}
}else{
throw new RuntimeException("couldn't convert SBML from L"+level+"V"+version+" to L2V3");
}
}else{
throw new RuntimeException("couldn't convert SBML from L"+level+"V"+version+" to L2V3, not compatible: "+sbmlDocument.getError(0));
}
}
File resultFile = mSBMLSolver.solve(mathsbmlFilePrefix, outDir, sbmlText, simSpec);
mSBMLResults = readResultFile(resultFile, columnDelimiter);
bioModelInfo.setAttribute("mSBML_ran","true");
}catch (Exception e){
printWriter.println("mSBML solve() failed");
e.printStackTrace(printWriter);
System.out.println("mSBML solve() failed");
e.printStackTrace(System.out);
combinedErrorBuffer.append(" *mSBML* _"+e.getMessage()+"_ ");
Element mSBMLSolverReport = new Element("SolverReport");
mSBMLSolverReport.setAttribute("solverName","mSBML");
mSBMLSolverReport.setAttribute("errorMessage",e.getMessage());
bioModelInfo.addContent(mSBMLSolverReport);
bioModelInfo.setAttribute("mSBML_ran","false");
}
}
*/
//
// compare results from COPASI and VCELL_RT
//
Boolean bCOPASI_VCELL_matched = null;
if (copasiResults != null && vcellResults_RT != null) {
try {
SimulationComparisonSummary summary = MathTestingUtilities.compareResultSets(copasiResults, vcellResults_RT, varsToTest, TestCaseNew.REGRESSION, 1e-5, 1e-5);
double maxRelError = summary.getMaxRelativeError();
bioModelInfo.setAttribute("COPASI_VCELL_maxRelErr", Double.toString(maxRelError));
if (maxRelError < 1) {
bCOPASI_VCELL_matched = true;
} else {
detailWriter.println(modelID + " " + modelName);
bCOPASI_VCELL_matched = false;
Element solverComparison = new Element("SolverComparison");
solverComparison.setAttribute("solver1", "vcell");
solverComparison.setAttribute("solver2", "COPASI");
VariableComparisonSummary[] failedVCSummaries = summary.getFailingVariableComparisonSummaries(1e-5, 1e-5);
for (VariableComparisonSummary vcSummary : failedVCSummaries) {
Element failedVariableSummary = getVariableSummary(vcSummary);
solverComparison.addContent(failedVariableSummary);
String ss = vcSummary.toShortString();
detailWriter.print('\t');
detailWriter.println(ss);
System.out.println(ss);
}
detailWriter.println();
bioModelInfo.addContent(solverComparison);
}
} catch (Exception e) {
printWriter.println("COMPARE VC/COPASI failed");
e.printStackTrace(printWriter);
System.out.println("COMPARE VC/COPASI failed");
e.printStackTrace(System.out);
combinedErrorBuffer.append(" *COMPARE VC/COPASI* _" + e.getMessage() + "_ ");
Element solverComparison = new Element("SolverComparison");
solverComparison.setAttribute("solver1", "vcell");
solverComparison.setAttribute("solver2", "COPASI");
solverComparison.setAttribute("error", e.getMessage());
bioModelInfo.addContent(solverComparison);
}
}
/*
Boolean bmSBML_VCELL_matched = null;
if (mSBMLResults!=null && vcellResults_RT!=null){
try {
SimulationComparisonSummary summary = MathTestingUtilities.compareUnEqualResultSets(mSBMLResults, vcellResults_RT, varsToTest, 1e-5, 1e-5, 1);
double maxRelError = summary.getMaxRelativeError();
bioModelInfo.setAttribute("mSBML_VCELL_maxRelErr", Double.toString(maxRelError));
if (maxRelError<1){
bmSBML_VCELL_matched = true;
}else{
bmSBML_VCELL_matched = false;
Element solverComparison = new Element("SolverComparison");
solverComparison.setAttribute("solver1","vcell");
solverComparison.setAttribute("solver2","mSBML");
VariableComparisonSummary[] failedVCSummaries = summary.getFailingVariableComparisonSummaries(1e-5, 1e-5);
for (VariableComparisonSummary vcSummary : failedVCSummaries){
Element failedVariableSummary = getVariableSummary(vcSummary);
solverComparison.addContent(failedVariableSummary);
System.out.println(vcSummary.toShortString());
}
bioModelInfo.addContent(solverComparison);
}
} catch (Exception e) {
printWriter.println("COMPARE VCRT/mSBML failed");
e.printStackTrace(printWriter);
System.out.println("COMPARE VCRT/mSBML failed");
e.printStackTrace(System.out);
combinedErrorBuffer.append(" *COMPARE VCRT/mSBML* _"+e.getMessage()+"_ ");
Element solverComparison = new Element("SolverComparison");
solverComparison.setAttribute("solver1","vcell");
solverComparison.setAttribute("solver2","mSBML");
solverComparison.setAttribute("error",e.getMessage());
bioModelInfo.addContent(solverComparison);
}
}
//
// compare results from COPASI and mSBML
//
Boolean bCOPASI_mSBML_matched = null;
if (copasiResults!=null && mSBMLResults!=null){
try {
SimulationComparisonSummary summary = MathTestingUtilities.compareUnEqualResultSets(copasiResults, mSBMLResults, varsToTest, 1e-5, 1e-5, 1);
double maxRelError = summary.getMaxRelativeError();
bioModelInfo.setAttribute("COPASI_mSBML_maxRelErr", Double.toString(maxRelError));
if (maxRelError<1){
bCOPASI_mSBML_matched = true;
}else{
bCOPASI_mSBML_matched = false;
Element solverComparison = new Element("SolverComparison");
solverComparison.setAttribute("solver1","COPASI");
solverComparison.setAttribute("solver2","mSBML");
solverComparison.setAttribute("result","failed");
VariableComparisonSummary[] vcSummaries = summary.getVariableComparisonSummaries();
for (VariableComparisonSummary vcSummary : vcSummaries){
Element failedVariableSummary = getVariableSummary(vcSummary);
solverComparison.addContent(failedVariableSummary);
System.out.println(vcSummary.toShortString());
}
bioModelInfo.addContent(solverComparison);
}
} catch (Exception e) {
printWriter.println("COMPARE COPASI/mSBML failed");
e.printStackTrace(printWriter);
System.out.println("COMPARE COPASI/mSBML failed");
e.printStackTrace(System.out);
combinedErrorBuffer.append(" *COMPARE COPASI/mSBML* _"+e.getMessage()+"_ ");
Element solverComparison = new Element("SolverComparison");
solverComparison.setAttribute("solver1","COPASI");
solverComparison.setAttribute("solver2","mSBML");
solverComparison.setAttribute("error",e.getMessage());
}
}
*/
if ((bCOPASI_VCELL_matched != null && bCOPASI_VCELL_matched.booleanValue())) {
// || /*(bmSBML_VCELL_matched!=null && bmSBML_VCELL_matched.booleanValue()) */ )
bioModelInfo.setAttribute(BioModelsNetPanel.SUPPORTED_ATTRIBUTE_NAME, "true");
printWriter.println("PASS");
} else {
bioModelInfo.setAttribute(BioModelsNetPanel.SUPPORTED_ATTRIBUTE_NAME, "false");
printWriter.println("FAIL");
}
} catch (Exception e) {
e.printStackTrace(printWriter);
combinedErrorBuffer.append(" *UNKNOWN* _" + e.getMessage() + "_ ");
bioModelInfo.setAttribute(BioModelsNetPanel.SUPPORTED_ATTRIBUTE_NAME, "false");
bioModelInfo.setAttribute("exception", e.getMessage());
}
printWriter.flush();
// write for each model just in case files get corrupted (it happened).
write(bioModelInfo, new File(outDir, modelID + "_report.xml"));
} finally {
if (new_sysout != null) {
new_sysout.close();
new_sysout = null;
}
if (new_syserr != null) {
new_syserr.close();
new_syserr = null;
}
System.setOut(saved_sysout);
System.setOut(saved_syserr);
}
}
// this writes out the SBML import exceptions grouped by type
if (!sbmlExceptions.isEmpty()) {
Map<Category, Collection<ModelException>> map = sbmlExceptions.getMap();
try (PrintWriter pw = new PrintWriter(new File(outDir, "sbmlSorted.txt"))) {
// SBMLImportException.Category
for (Category c : Category.values()) {
Collection<ModelException> meCollection = map.get(c);
for (ModelException me : meCollection) {
write(pw, me);
}
}
}
}
} finally {
printWriter.close();
detailWriter.close();
bngWriter.close();
}
} catch (Throwable e) {
e.printStackTrace(System.out);
e.printStackTrace(System.err);
}
System.exit(0);
}
Also used :
BNGException(cbit.vcell.server.bionetgen.BNGException)
HelpFormatter(org.apache.commons.cli.HelpFormatter)
TreeSet(java.util.TreeSet)
SBMLSolver(org.vcell.sbml.SBMLSolver)
VariableComparisonSummary(cbit.vcell.solver.test.VariableComparisonSummary)
BioModelsWebServicesServiceLocator(uk.ac.ebi.www.biomodels_main.services.BioModelsWebServices.BioModelsWebServicesServiceLocator)
Collection(java.util.Collection)
Option(org.apache.commons.cli.Option)
File(java.io.File)
Options(org.apache.commons.cli.Options)
SortedSet(java.util.SortedSet)
Set(java.util.Set)
ODESolverResultSet(cbit.vcell.solver.ode.ODESolverResultSet)
TreeSet(java.util.TreeSet)
HashSet(java.util.HashSet)
Category(org.vcell.sbml.vcell.SBMLImportException.Category)
OperatingSystemInfo(cbit.vcell.resource.OperatingSystemInfo)
FileWriter(java.io.FileWriter)
Element(org.jdom.Element)
NullWriter(org.apache.commons.io.output.NullWriter)
OptionGroup(org.apache.commons.cli.OptionGroup)
ODESolverResultSet(cbit.vcell.solver.ode.ODESolverResultSet)
SimulationComparisonSummary(cbit.vcell.solver.test.SimulationComparisonSummary)
CommandLineParser(org.apache.commons.cli.CommandLineParser)
DefaultParser(org.apache.commons.cli.DefaultParser)
PrintWriter(java.io.PrintWriter)
PrintStream(java.io.PrintStream)
SBMLImportException(org.vcell.sbml.vcell.SBMLImportException)
SimSpec(org.vcell.sbml.SimSpec)
SbmlException(org.vcell.sbml.SbmlException)
FileNotFoundException(java.io.FileNotFoundException)
BNGException(cbit.vcell.server.bionetgen.BNGException)
SBMLImportException(org.vcell.sbml.vcell.SBMLImportException)
ParseException(org.apache.commons.cli.ParseException)
SolverException(cbit.vcell.solver.SolverException)
IOException(java.io.IOException)
BioModelsWebServices(uk.ac.ebi.www.biomodels_main.services.BioModelsWebServices.BioModelsWebServices)
CommandLine(org.apache.commons.cli.CommandLine)
ParseException(org.apache.commons.cli.ParseException)
Aggregations
ODESolverResultSet (cbit.vcell.solver.ode.ODESolverResultSet)2
SimulationComparisonSummary (cbit.vcell.solver.test.SimulationComparisonSummary)2
VariableComparisonSummary (cbit.vcell.solver.test.VariableComparisonSummary)2
BioModel (cbit.vcell.biomodel.BioModel)1
TFGenerateReport (cbit.vcell.client.task.TFGenerateReport)1
ClientDocumentManager (cbit.vcell.clientdb.ClientDocumentManager)1
MathModel (cbit.vcell.mathmodel.MathModel)1
AddTestCasesOPBioModel (cbit.vcell.numericstest.AddTestCasesOPBioModel)1
AddTestCasesOPMathModel (cbit.vcell.numericstest.AddTestCasesOPMathModel)1
AddTestCriteriaOPBioModel (cbit.vcell.numericstest.AddTestCriteriaOPBioModel)1
AddTestCriteriaOPMathModel (cbit.vcell.numericstest.AddTestCriteriaOPMathModel)1
AddTestResultsOP (cbit.vcell.numericstest.AddTestResultsOP)1
EditTestCriteriaOPBioModel (cbit.vcell.numericstest.EditTestCriteriaOPBioModel)1
EditTestCriteriaOPMathModel (cbit.vcell.numericstest.EditTestCriteriaOPMathModel)1
EditTestCriteriaOPReportStatus (cbit.vcell.numericstest.EditTestCriteriaOPReportStatus)1
RemoveTestResultsOP (cbit.vcell.numericstest.RemoveTestResultsOP)1
TestCaseNewBioModel (cbit.vcell.numericstest.TestCaseNewBioModel)1
TestCaseNewMathModel (cbit.vcell.numericstest.TestCaseNewMathModel)1
TestCriteriaNewBioModel (cbit.vcell.numericstest.TestCriteriaNewBioModel)1
TestCriteriaNewMathModel (cbit.vcell.numericstest.TestCriteriaNewMathModel)1
