Examples with VCSimulationDataIdentifier cbit.vcell.solver.VCSimulationDataIdentifier used on opensource projects

Example 16 with VCSimulationDataIdentifier

use of cbit.vcell.solver.VCSimulationDataIdentifier in project vcell by virtualcell.

the class RunRefSimulationFastOp method runRefSimulation.

private RowColumnResultSet runRefSimulation(ROI cellROI, ROI[] imageDataROIs, UShortImage psf, FloatImage initRefConc, double experimentalRecoveryTime, LocalWorkspace localWorkspace, ClientTaskStatusSupport progressListener) throws Exception {
    User owner = LocalWorkspace.getDefaultOwner();
    KeyValue simKey = LocalWorkspace.createNewKeyValue();
    // 
    // save first image from normalized time series as the initial concentration field data
    // 
    ExternalDataInfo initialConcentrationExtData = createNewExternalDataInfo(localWorkspace, INITCONC_EXTDATA_NAME);
    Extent extent = initRefConc.getExtent();
    Origin origin = initRefConc.getOrigin();
    ISize isize = new ISize(initRefConc.getNumX(), initRefConc.getNumY(), initRefConc.getNumZ());
    saveExternalData(initRefConc, INITCONC_EXTDATA_VARNAME, initialConcentrationExtData.getExternalDataIdentifier(), localWorkspace);
    FieldFunctionArguments initConditionFFA = new FieldFunctionArguments(INITCONC_EXTDATA_NAME, INITCONC_EXTDATA_VARNAME, new Expression(0.0), VariableType.VOLUME);
    // 
    // save ROIs as a multivariate field data
    // 
    ExternalDataInfo roiExtData = createNewExternalDataInfo(localWorkspace, ROI_EXTDATA_NAME);
    saveROIsAsExternalData(imageDataROIs, localWorkspace, roiExtData.getExternalDataIdentifier());
    ArrayList<FieldFunctionArguments> roiFFAs = new ArrayList<FieldFunctionArguments>();
    for (ROI roi : imageDataROIs) {
        roiFFAs.add(new FieldFunctionArguments(ROI_EXTDATA_NAME, ROI_MASK_NAME_PREFIX + roi.getROIName(), new Expression(0.0), VariableType.VOLUME));
    }
    // 
    // save PSF as a field data
    // 
    ExternalDataInfo psfExtData = createNewExternalDataInfo(localWorkspace, PSF_EXTDATA_NAME);
    savePsfAsExternalData(psf, PSF_EXTDATA_VARNAME, psfExtData.getExternalDataIdentifier(), localWorkspace);
    FieldFunctionArguments psfFFA = new FieldFunctionArguments(PSF_EXTDATA_NAME, PSF_EXTDATA_VARNAME, new Expression(0.0), VariableType.VOLUME);
    TimeBounds timeBounds = getEstimatedRefTimeBound(experimentalRecoveryTime);
    double timeStepVal = REFERENCE_DIFF_DELTAT;
    Expression chirpedDiffusionRate = new Expression(REFERENCE_DIFF_RATE_COEFFICIENT + "*(t+" + REFERENCE_DIFF_DELTAT + ")");
    BioModel bioModel = createRefSimBioModel(simKey, owner, origin, extent, cellROI, timeStepVal, timeBounds, VAR_NAME, new Expression(initConditionFFA.infix()), psfFFA, chirpedDiffusionRate);
    if (progressListener != null) {
        progressListener.setMessage("Running Reference Simulation...");
    }
    // run simulation
    Simulation simulation = bioModel.getSimulation(0);
    ROIDataGenerator roiDataGenerator = getROIDataGenerator(localWorkspace, imageDataROIs);
    simulation.getMathDescription().getPostProcessingBlock().addDataGenerator(roiDataGenerator);
    runFVSolverStandalone(new File(localWorkspace.getDefaultSimDataDirectory()), simulation, initialConcentrationExtData.getExternalDataIdentifier(), roiExtData.getExternalDataIdentifier(), psfExtData.getExternalDataIdentifier(), progressListener, true);
    KeyValue referenceSimKeyValue = simulation.getVersion().getVersionKey();
    VCSimulationIdentifier vcSimID = new VCSimulationIdentifier(referenceSimKeyValue, LocalWorkspace.getDefaultOwner());
    VCSimulationDataIdentifier vcSimDataID = new VCSimulationDataIdentifier(vcSimID, 0);
    File hdf5File = new File(localWorkspace.getDefaultSimDataDirectory(), vcSimDataID.getID() + SimDataConstants.DATA_PROCESSING_OUTPUT_EXTENSION_HDF5);
    // get post processing info (time points, variable sizes)
    DataOperation.DataProcessingOutputInfoOP dataOperationInfo = new DataOperation.DataProcessingOutputInfoOP(null, /*no vcDataIdentifier OK*/
    false, null);
    DataOperationResults.DataProcessingOutputInfo dataProcessingOutputInfo = (DataOperationResults.DataProcessingOutputInfo) DataSetControllerImpl.getDataProcessingOutput(dataOperationInfo, hdf5File);
    // get post processing data
    DataOperation.DataProcessingOutputDataValuesOP dataOperationDataValues = new DataOperation.DataProcessingOutputDataValuesOP(null, /*no vcDataIdentifier OK*/
    ROI_EXTDATA_NAME, TimePointHelper.createAllTimeTimePointHelper(), DataIndexHelper.createSliceDataIndexHelper(0), null, null);
    DataOperationResults.DataProcessingOutputDataValues dataProcessingOutputDataValues = (DataOperationResults.DataProcessingOutputDataValues) DataSetControllerImpl.getDataProcessingOutput(dataOperationDataValues, hdf5File);
    // 
    // delete the simulation files
    // 
    // 
    // remove reference simulation files and field data files
    // 
    File userDir = new File(localWorkspace.getDefaultSimDataDirectory());
    File[] oldSimFilesToDelete = getSimulationFileNames(userDir, referenceSimKeyValue);
    for (int i = 0; oldSimFilesToDelete != null && i < oldSimFilesToDelete.length; i++) {
        oldSimFilesToDelete[i].delete();
    }
    deleteCanonicalExternalData(localWorkspace, initialConcentrationExtData.getExternalDataIdentifier());
    deleteCanonicalExternalData(localWorkspace, roiExtData.getExternalDataIdentifier());
    deleteCanonicalExternalData(localWorkspace, roiExtData.getExternalDataIdentifier());
    // get ref sim time points (distorted by time dilation acceleration)
    double[] rawRefDataTimePoints = dataProcessingOutputInfo.getVariableTimePoints();
    // get shifted time points
    double[] correctedRefDataTimePoints = shiftTimeForBaseDiffRate(rawRefDataTimePoints);
    double[][] refData = dataProcessingOutputDataValues.getDataValues();
    // 
    // for rowColumnResultSet with { "t", "roi1", .... , "roiN" } for reference data
    // 
    int numROIs = imageDataROIs.length;
    String[] columnNames = new String[numROIs + 1];
    columnNames[0] = "t";
    for (int i = 0; i < numROIs; i++) {
        columnNames[i + 1] = imageDataROIs[i].getROIName();
    }
    RowColumnResultSet reducedData = new RowColumnResultSet(columnNames);
    for (int i = 0; i < correctedRefDataTimePoints.length; i++) {
        double[] row = new double[numROIs + 1];
        row[0] = correctedRefDataTimePoints[i];
        double[] roiData = refData[i];
        for (int j = 0; j < numROIs; j++) {
            // roiData[0] is the average over the cell .. postbleach this shouldn't change for pure diffusion
            row[j + 1] = roiData[j + 1];
        }
        reducedData.addRow(row);
    }
    return reducedData;
}

Example 17 with VCSimulationDataIdentifier

use of cbit.vcell.solver.VCSimulationDataIdentifier in project vcell by virtualcell.

the class RunSimulation2DOp method runRefSimulation.

public ImageTimeSeries<FloatImage> runRefSimulation(LocalWorkspace localWorkspace, Simulation simulation, String varName, ClientTaskStatusSupport progressListener) throws Exception {
    User owner = LocalWorkspace.getDefaultOwner();
    KeyValue simKey = LocalWorkspace.createNewKeyValue();
    runFVSolverStandalone(new File(localWorkspace.getDefaultSimDataDirectory()), simulation, progressListener);
    Extent extent = simulation.getMathDescription().getGeometry().getExtent();
    Origin origin = simulation.getMathDescription().getGeometry().getOrigin();
    VCDataIdentifier vcDataIdentifier = new VCSimulationDataIdentifier(new VCSimulationIdentifier(simKey, owner), 0);
    CartesianMesh mesh = localWorkspace.getDataSetControllerImpl().getMesh(vcDataIdentifier);
    ISize isize = new ISize(mesh.getSizeX(), mesh.getSizeY(), mesh.getSizeZ());
    double[] dataTimes = localWorkspace.getDataSetControllerImpl().getDataSetTimes(vcDataIdentifier);
    FloatImage[] solutionImages = new FloatImage[dataTimes.length];
    for (int i = 0; i < dataTimes.length; i++) {
        SimDataBlock simDataBlock = localWorkspace.getDataSetControllerImpl().getSimDataBlock(null, vcDataIdentifier, varName, dataTimes[i]);
        double[] doubleData = simDataBlock.getData();
        float[] floatPixels = new float[doubleData.length];
        for (int j = 0; j < doubleData.length; j++) {
            floatPixels[j] = (float) doubleData[j];
        }
        solutionImages[i] = new FloatImage(floatPixels, origin, extent, isize.getX(), isize.getY(), isize.getZ());
    }
    ImageTimeSeries<FloatImage> solution = new ImageTimeSeries<FloatImage>(FloatImage.class, solutionImages, dataTimes, 1);
    return solution;
}

Example 18 with VCSimulationDataIdentifier

use of cbit.vcell.solver.VCSimulationDataIdentifier in project vcell by virtualcell.

the class RunRefSimulationOp method runRefSimulation.

private static ImageTimeSeries<FloatImage> runRefSimulation(LocalWorkspace localWorkspace, ROI cellROI, double timeStepVal, TimeBounds timeBounds, FloatImage initRefConc, double baseDiffusionRate, ClientTaskStatusSupport progressListener) throws Exception {
    String INITCONC_EXTDATA_NAME = "initConc";
    String INITCONC_EXTDATA_VARNAME = "conc";
    String VAR_NAME = "species";
    User owner = LocalWorkspace.getDefaultOwner();
    KeyValue simKey = LocalWorkspace.createNewKeyValue();
    ExternalDataInfo initialConcentrationExtData = createNewExternalDataInfo(localWorkspace, INITCONC_EXTDATA_NAME);
    Extent extent = initRefConc.getExtent();
    Origin origin = initRefConc.getOrigin();
    ISize isize = new ISize(initRefConc.getNumX(), initRefConc.getNumY(), initRefConc.getNumZ());
    saveExternalData(initRefConc, INITCONC_EXTDATA_VARNAME, initialConcentrationExtData.getExternalDataIdentifier(), localWorkspace);
    FieldFunctionArguments initConditionFFA = new FieldFunctionArguments(INITCONC_EXTDATA_NAME, INITCONC_EXTDATA_VARNAME, new Expression(0.0), VariableType.VOLUME);
    BioModel bioModel = createRefSimBioModel(simKey, owner, origin, extent, cellROI, timeStepVal, timeBounds, VAR_NAME, new Expression(initConditionFFA.infix()), baseDiffusionRate);
    if (progressListener != null) {
        progressListener.setMessage("Running Reference Simulation...");
    }
    // run simulation
    runFVSolverStandalone(new File(localWorkspace.getDefaultSimDataDirectory()), bioModel.getSimulation(0), initialConcentrationExtData.getExternalDataIdentifier(), progressListener, true);
    VCDataIdentifier vcDataIdentifier = new VCSimulationDataIdentifier(new VCSimulationIdentifier(simKey, owner), 0);
    double[] dataTimes = localWorkspace.getDataSetControllerImpl().getDataSetTimes(vcDataIdentifier);
    FloatImage[] solutionImages = new FloatImage[dataTimes.length];
    for (int i = 0; i < dataTimes.length; i++) {
        SimDataBlock simDataBlock = localWorkspace.getDataSetControllerImpl().getSimDataBlock(null, vcDataIdentifier, VAR_NAME, dataTimes[i]);
        double[] doubleData = simDataBlock.getData();
        float[] floatPixels = new float[doubleData.length];
        for (int j = 0; j < doubleData.length; j++) {
            floatPixels[j] = (float) doubleData[j];
        }
        solutionImages[i] = new FloatImage(floatPixels, origin, extent, isize.getX(), isize.getY(), isize.getZ());
    }
    ImageTimeSeries<FloatImage> solution = new ImageTimeSeries<FloatImage>(FloatImage.class, solutionImages, dataTimes, 1);
    return solution;
}

Example 19 with VCSimulationDataIdentifier

use of cbit.vcell.solver.VCSimulationDataIdentifier in project vcell by virtualcell.

the class NagiosVCellMonitor method checkVCell.

private CheckResults checkVCell(VCELL_CHECK_LEVEL checkLevel, String rmiHostName, int rmiPort, String rmiBootstrapStubName, String vcellNagiosPassword, int criticalTimeout, int monitorPort) throws Exception {
    SimulationStatusPersistent lastSimStatus = null;
    String vcellVersion = null;
    TreeMap<VCELL_CHECK_LEVEL, Long> levelTimesMillisec = new TreeMap<NagiosVCellMonitor.VCELL_CHECK_LEVEL, Long>();
    long startTime = System.currentTimeMillis();
    VCellConnection vcellConnection = null;
    try {
        if (rmiHostName == null || rmiPort == -1) {
            throw new UnexpectedTestStateException("Host name/ip and rmiPort required for testing, rmihostname=" + rmiHostName + " rmiport=" + rmiPort);
        }
        String rmiUrl = "//" + rmiHostName + ":" + rmiPort + "/" + rmiBootstrapStubName;
        VCellBootstrap vcellBootstrap = null;
        try {
            vcellBootstrap = (VCellBootstrap) Naming.lookup(rmiUrl);
        } catch (Exception e) {
            throw new UnexpectedTestStateException("Error during bootstrap lookup, " + e.getClass().getSimpleName() + " " + e.getMessage());
        }
        vcellVersion = vcellBootstrap.getVCellSoftwareVersion();
        levelTimesMillisec.put(VCELL_CHECK_LEVEL.RMI_ONLY_0, System.currentTimeMillis() - startTime);
        if (checkLevel.ordinal() >= VCELL_CHECK_LEVEL.CONNECT_1.ordinal()) {
            if (vcellNagiosPassword == null) {
                throw new UnexpectedTestStateException("vcellNagios Password required for " + VCELL_CHECK_LEVEL.CONNECT_1.toString() + " and above");
            }
            UserLoginInfo userLoginInfo = new UserLoginInfo(VCELL_NAGIOS_USER, new DigestedPassword(vcellNagiosPassword));
            vcellConnection = vcellBootstrap.getVCellConnection(userLoginInfo);
            levelTimesMillisec.put(VCELL_CHECK_LEVEL.CONNECT_1, System.currentTimeMillis() - startTime - levelTimesMillisec.get(VCELL_CHECK_LEVEL.RMI_ONLY_0));
            if (checkLevel.ordinal() >= VCELL_CHECK_LEVEL.INFOS_2.ordinal()) {
                VCInfoContainer vcInfoContainer = vcellConnection.getUserMetaDbServer().getVCInfoContainer();
                levelTimesMillisec.put(VCELL_CHECK_LEVEL.INFOS_2, System.currentTimeMillis() - startTime - levelTimesMillisec.get(VCELL_CHECK_LEVEL.CONNECT_1));
                if (checkLevel.ordinal() >= VCELL_CHECK_LEVEL.LOAD_3.ordinal()) {
                    KeyValue bioModelKey = null;
                    final String testModelName = "Solver Suite 5.1 (BETA only ode)";
                    for (BioModelInfo bioModelInfo : vcInfoContainer.getBioModelInfos()) {
                        if (userLoginInfo.getUserName().equals(bioModelInfo.getVersion().getOwner().getName()) && bioModelInfo.getVersion().getName().equals(testModelName)) {
                            bioModelKey = bioModelInfo.getVersion().getVersionKey();
                            break;
                        }
                    }
                    BigString bioModelXML = vcellConnection.getUserMetaDbServer().getBioModelXML(bioModelKey);
                    BioModel bioModel = XmlHelper.XMLToBioModel(new XMLSource(bioModelXML.toString()));
                    bioModel.refreshDependencies();
                    levelTimesMillisec.put(VCELL_CHECK_LEVEL.LOAD_3, System.currentTimeMillis() - startTime - levelTimesMillisec.get(VCELL_CHECK_LEVEL.INFOS_2));
                    if (checkLevel.ordinal() >= VCELL_CHECK_LEVEL.DATA_4.ordinal()) {
                        final String testSimContextName = "non-spatial ODE";
                        SimulationContext simulationContext = bioModel.getSimulationContext(testSimContextName);
                        final String testSimName = "Copy of combined ida/cvode";
                        Simulation simulation = simulationContext.getSimulation(testSimName);
                        if (simulation == null) {
                            throw new UnexpectedTestStateException("Couldn't find sim '" + testSimName + "' for " + checkLevel.toString());
                        }
                        VCSimulationDataIdentifier vcSimulationDataIdentifier = new VCSimulationDataIdentifier(simulation.getSimulationInfo().getAuthoritativeVCSimulationIdentifier(), 0);
                        ArrayList<AnnotatedFunction> outputFunctionsList = simulationContext.getOutputFunctionContext().getOutputFunctionsList();
                        OutputContext outputContext = new OutputContext(outputFunctionsList.toArray(new AnnotatedFunction[outputFunctionsList.size()]));
                        double[] times = vcellConnection.getDataSetController().getDataSetTimes(vcSimulationDataIdentifier);
                        ODESimData odeSimData = vcellConnection.getDataSetController().getODEData(vcSimulationDataIdentifier);
                        levelTimesMillisec.put(VCELL_CHECK_LEVEL.DATA_4, System.currentTimeMillis() - startTime - levelTimesMillisec.get(VCELL_CHECK_LEVEL.LOAD_3));
                        if (checkLevel.ordinal() >= VCELL_CHECK_LEVEL.RUN_5.ordinal()) {
                            KeyValue copy1Key = null;
                            KeyValue copy2Key = null;
                            VCSimulationIdentifier testRunSimID = null;
                            try {
                                if (simulationContext.getSimulations().length != 1) {
                                    throw new UnexpectedTestStateException("Expecting only 1 sim to be copied for " + checkLevel.toString());
                                }
                                SimulationStatusPersistent simulationStatus = vcellConnection.getUserMetaDbServer().getSimulationStatus(simulation.getVersion().getVersionKey());
                                if (!simulationStatus.isCompleted()) {
                                    throw new UnexpectedTestStateException("Expecting completed sim to copy for " + checkLevel.toString());
                                }
                                String copyModelName = testModelName + "_" + rmiHostName + "_rmi" + rmiPort + "_siteprt" + monitorPort;
                                boolean bForceCleanup = true;
                                while (true) {
                                    boolean bMessy = false;
                                    for (BioModelInfo bioModelInfo : vcInfoContainer.getBioModelInfos()) {
                                        if (userLoginInfo.getUserName().equals(bioModelInfo.getVersion().getOwner().getName()) && bioModelInfo.getVersion().getName().equals(copyModelName)) {
                                            bMessy = true;
                                            if (bForceCleanup) {
                                                try {
                                                    vcellConnection.getUserMetaDbServer().deleteBioModel(bioModelInfo.getVersion().getVersionKey());
                                                } catch (Exception e) {
                                                    e.printStackTrace();
                                                }
                                            } else {
                                                throw new MessyTestEnvironmentException("Messy test environment, not expecting " + copyModelName + " and couldn't cleanup");
                                            }
                                        }
                                    }
                                    if (!bMessy) {
                                        break;
                                    }
                                    // get new vcInfoContainer without cleaned-up model
                                    vcInfoContainer = vcellConnection.getUserMetaDbServer().getVCInfoContainer();
                                    bForceCleanup = false;
                                }
                                BigString copyBioModelXMLStr = vcellConnection.getUserMetaDbServer().saveBioModelAs(bioModelXML, copyModelName, null);
                                BioModel copyBioModel = XmlHelper.XMLToBioModel(new XMLSource(copyBioModelXMLStr.toString()));
                                copy1Key = copyBioModel.getVersion().getVersionKey();
                                copyBioModel.refreshDependencies();
                                Simulation copySim = copyBioModel.getSimulationContext(testSimContextName).copySimulation(copyBioModel.getSimulationContext(testSimContextName).getSimulation(testSimName));
                                final String copyTestSimName = "test";
                                copySim.setName(copyTestSimName);
                                copyBioModel.refreshDependencies();
                                copyBioModelXMLStr = new BigString(XmlHelper.bioModelToXML(copyBioModel));
                                copyBioModelXMLStr = vcellConnection.getUserMetaDbServer().saveBioModel(copyBioModelXMLStr, null);
                                copyBioModel = XmlHelper.XMLToBioModel(new XMLSource(copyBioModelXMLStr.toString()));
                                copy2Key = copyBioModel.getVersion().getVersionKey();
                                copyBioModel.refreshDependencies();
                                Simulation newSimulation = copyBioModel.getSimulationContext(testSimContextName).getSimulation(copyTestSimName);
                                simulationStatus = vcellConnection.getUserMetaDbServer().getSimulationStatus(newSimulation.getVersion().getVersionKey());
                                if (simulationStatus != null && !simulationStatus.isNeverRan()) {
                                    throw new UnexpectedTestStateException("Expecting new sim to have 'never ran' status for " + checkLevel.toString());
                                }
                                testRunSimID = new VCSimulationIdentifier(newSimulation.getVersion().getVersionKey(), copyBioModel.getVersion().getOwner());
                                vcellConnection.getSimulationController().startSimulation(testRunSimID, 1);
                                lastSimStatus = simulationStatus;
                                MessageEvent[] messageEvents = null;
                                while (simulationStatus == null || (!simulationStatus.isStopped() && !simulationStatus.isCompleted() && !simulationStatus.isFailed())) {
                                    Thread.sleep(200);
                                    if (((System.currentTimeMillis() - startTime) / 1000) > criticalTimeout) {
                                        vcellConnection.getSimulationController().stopSimulation(testRunSimID);
                                        vcellConnection.getMessageEvents();
                                        break;
                                    }
                                    simulationStatus = vcellConnection.getUserMetaDbServer().getSimulationStatus(newSimulation.getVersion().getVersionKey());
                                    if (simulationStatus != null && !simulationStatus.toString().equals((lastSimStatus == null ? null : lastSimStatus.toString()))) {
                                        lastSimStatus = simulationStatus;
                                    }
                                    if (simulationStatus != null && simulationStatus.isFailed()) {
                                        throw new Exception("time " + ((System.currentTimeMillis() - startTime) / 1000) + ", Sim execution failed key:" + testRunSimID.getSimulationKey() + " sim " + newSimulation.getName() + " model " + copyBioModel.getVersion().getName() + " messg " + simulationStatus.getFailedMessage());
                                    }
                                    messageEvents = vcellConnection.getMessageEvents();
                                }
                            } finally {
                                try {
                                    if (copy1Key != null) {
                                        vcellConnection.getUserMetaDbServer().deleteBioModel(copy1Key);
                                    }
                                } catch (Exception e) {
                                    e.printStackTrace();
                                }
                                try {
                                    if (copy2Key != null) {
                                        vcellConnection.getUserMetaDbServer().deleteBioModel(copy2Key);
                                    }
                                } catch (Exception e) {
                                    e.printStackTrace();
                                }
                                if (testRunSimID != null) {
                                    deleteSimData(testRunSimID);
                                }
                            }
                            levelTimesMillisec.put(VCELL_CHECK_LEVEL.RUN_5, System.currentTimeMillis() - startTime - levelTimesMillisec.get(VCELL_CHECK_LEVEL.DATA_4));
                        }
                    }
                }
            }
        }
        return new CheckResults(vcellVersion, levelTimesMillisec, lastSimStatus, System.currentTimeMillis() - startTime, null);
    } catch (Exception e) {
        return new CheckResults(vcellVersion, levelTimesMillisec, lastSimStatus, System.currentTimeMillis() - startTime, e);
    } finally {
        vcellConnection = null;
    }
}

Example 20 with VCSimulationDataIdentifier

use of cbit.vcell.solver.VCSimulationDataIdentifier in project vcell by virtualcell.

the class TestMissingSimData method checkDataExists.

// private static void runSimsNew(String connectURL,String dbSchemaUser, String dbPassword) throws Exception{
// 
// //		VCellBootstrap vCellBootstrap = getVCellBootstrap("rmi-beta.cam.uchc.edu", 40105, "VCellBootstrapServer", 12, false);
// //		VCellBootstrap vCellBootstrap = getVCellBootstrap("rmi-alpha.cam.uchc.edu", 40106, "VCellBootstrapServer", 12, false);
// VCellBootstrap vCellBootstrap = getVCellBootstrap("rmi-alpha.cam.uchc.edu", 40111, "VCellBootstrapServer", 12, false);//Test2
// 
// if(true){
// Hashtable<KeyValue, UserLoginInfo> keyUserLoginInfo = doQuery(connectURL, dbSchemaUser, dbPassword);
// Enumeration<KeyValue> keys = keyUserLoginInfo.keys();
// while(keys.hasMoreElements()){
// KeyValue simKey = keys.nextElement();
// UserLoginInfo userLoginInfo = keyUserLoginInfo.get(simKey);
// VCSimulationIdentifier vcSimulationIdentifier = new VCSimulationIdentifier(simKey, userLoginInfo.getUser());
// acquireThread(vcSimulationIdentifier, vCellBootstrap, userLoginInfo, connectURL, dbSchemaUser, dbPassword);
// }
// return;
// }
// 
// 
// String itemSelectSQL = " select vc_userinfo.userid,vc_userinfo.id userkey,vc_userinfo.digestpw,missingdata.simjobsimref,vc_softwareversion.softwareversion ";
// 
// String sqlPart =
// " from missingdata,vc_simulation,vc_userinfo,vc_softwareversion "+
// " where "+
// " (vc_simulation.id in (select simref from vc_biomodelsim)) and " +
// //				" (vc_simulation.id in (select simref from vc_mathmodelsim)) and " +
// " vc_userinfo.userid='fgao5' and "+
// " vc_userinfo.id = vc_simulation.ownerref and "+
// " missingdata.simjobsimref = vc_simulation.id and "+
// //				" missingdata.dataexists not like 'readable%' and "+
// " (missingdata.dataexists = 'false') "+
// " and missingdata.notes is not null and " +
// " (missingdata.notes like '%Compiled_solvers_no_longer%' or missingdata.notes like '%Connection_refused%')" +
// //				" and missingdata.simjobsimref = 34080002 " +
// //				" or missingdata.dataexists like 'error - %') "+
// //				" and (missingdata.notes is null ) "+
// //				" or missingdata.notes not like 'reran OK%')" +
// " and vc_simulation.parentsimref is null and "+
// " (softwareversion is null or regexp_substr(softwareversion,'^((release)|(rel)|(alpha)|(beta))_version_([[:digit:]]+\\.?)+_build_([[:digit:]]+\\.?)+',1,1,'i') is not null) and "+
// " vc_softwareversion.versionableref (+) = vc_simulation.id " +
// //				" and rownum = 1 ";
// //				" and vc_simulation.id=39116536"
// " order by vc_userinfo.userid";
// 
// //		(mdt.dataexists = 'false' or mdt.dataexists like 'error - %') and
// //		(mdt.notes is null or mdt.notes not like 'reran OK%') and
// 
// //Create hash of sims and userlogininfo
// Hashtable<KeyValue, UserLoginInfo> simToUserLoginInfoHash = new Hashtable<>();
// Statement updateStatement = con.createStatement();
// Statement queryStatement = con.createStatement();
// //Get Count
// ResultSet rset = queryStatement.executeQuery("select count(*) "+sqlPart);
// rset.next();
// int totalCount = rset.getInt(1);
// rset.close();
// //Get sims
// rset = queryStatement.executeQuery(itemSelectSQL+sqlPart);
// UserLoginInfo userLoginInfo = null;
// VCellConnection vcellConnection = null;
// int currentCount = 1;
// while(rset.next()){
// KeyValue simJobSimRef = new KeyValue(rset.getString("simjobsimref"));
// try{
// String softwareVersion = rset.getString("softwareversion");
// String userid = rset.getString("userid");
// System.out.println("-----");
// System.out.println("-----running "+currentCount+" of "+totalCount+"   user="+userid+" simjobsimref="+simJobSimRef);
// currentCount+= 1;
// System.out.println("-----");
// 
// if(!rset.wasNull() && softwareVersion != null){
// StringTokenizer st = new StringTokenizer(softwareVersion, "_");
// st.nextToken();//site name
// st.nextToken();//'Version' literal string
// String majorVersion = st.nextToken();//major version number
// if(majorVersion.equals("5.4")){
// throw new Exception("Alpha-5.4 sims are not being re-run using Beta-5.3 code");
// }
// }
// 
// if(userid.toLowerCase().equals("vcelltestaccount")
// //					 || userid.toLowerCase().equals("anu")
// //					 || userid.toLowerCase().equals("fgao")
// //					 || userid.toLowerCase().equals("liye")
// //					 || userid.toLowerCase().equals("schaff")
// //					 || userid.toLowerCase().equals("ignovak")
// //					 || userid.toLowerCase().equals("jditlev")
// //					 || userid.toLowerCase().equals("sensation")
// ){
// continue;
// }
// String userkey = rset.getString("userkey");
// if(userLoginInfo == null || !userLoginInfo.getUserName().equals(userid)){
// userLoginInfo = new UserLoginInfo(userid,DigestedPassword.createAlreadyDigested(rset.getString("digestpw")));
// userLoginInfo.setUser(new User(userid, new KeyValue(userkey)));
// vcellConnection = null;
// }
// VCSimulationIdentifier vcSimulationIdentifier = new VCSimulationIdentifier(simJobSimRef, userLoginInfo.getUser());
// 
// 
// //
// //
// //
// if(true){
// acquireThread(vcSimulationIdentifier, vCellBootstrap, userLoginInfo, connectURL, dbSchemaUser, dbPassword);
// continue;
// }
// 
// try{
// if(vcellConnection != null){
// vcellConnection.getMessageEvents();
// }
// }catch(Exception e){
// e.printStackTrace();
// //assume disconnected
// vcellConnection = null;
// }
// if(vcellConnection == null){
// vcellConnection = vCellBootstrap.getVCellConnection(userLoginInfo);
// vcellConnection.getMessageEvents();
// }
// 
// SimulationStatusPersistent initSimulationStatus = vcellConnection.getUserMetaDbServer().getSimulationStatus(vcSimulationIdentifier.getSimulationKey());
// 
// System.out.println("initial status="+initSimulationStatus);
// //				if(!initSimulationStatus.isCompleted() || !initSimulationStatus.getHasData()){
// //					continue;
// //				}
// 
// 
// BigString simXML = vcellConnection.getUserMetaDbServer().getSimulationXML(simJobSimRef);
// Simulation sim = XmlHelper.XMLToSim(simXML.toString());
// //				SolverDescription solverDescription = sim.getSolverTaskDescription().getSolverDescription();
// //				if(solverDescription.equals(SolverDescription.StochGibson)  || solverDescription.equals(SolverDescription.FiniteVolume)){
// //					//These 2 solvers give too much trouble so skip
// //					System.out.println("--skipping solver");
// ////					notCompletedSimIDs.add(simIDAndJobID.simID.toString());
// //					return;
// //				}
// 
// 
// //				if(!sim.isSpatial()){
// //					continue;
// //				}
// //				if(sim.getSolverTaskDescription().isSerialParameterScan()/* || sim.getSolverTaskDescription().getExpectedNumTimePoints() > 20*/){
// //					continue;
// //				}
// 
// int scanCount = sim.getScanCount();
// //				if(true){return;}
// SimulationStatusPersistent simulationStatus = null;
// SimulationInfo simulationInfo = sim.getSimulationInfo();
// if(!simulationInfo.getAuthoritativeVCSimulationIdentifier().getSimulationKey().equals(vcSimulationIdentifier.getSimulationKey())){
// throw new Exception("Unexpected authoritative and sim id are not the same");
// }
// vcellConnection.getSimulationController().startSimulation(vcSimulationIdentifier, scanCount);
// long startTime = System.currentTimeMillis();
// while(simulationStatus == null || simulationStatus.isStopped() || simulationStatus.isCompleted() || simulationStatus.isFailed()){
// Thread.sleep(2000);
// simulationStatus = vcellConnection.getUserMetaDbServer().getSimulationStatus(vcSimulationIdentifier.getSimulationKey());
// if(simulationStatus.isFailed() && !initSimulationStatus.isFailed()){
// break;
// }
// MessageEvent[] messageEvents = vcellConnection.getMessageEvents();
// if((System.currentTimeMillis()-startTime) > 30000){
// throw new Exception("-----Sim finished too fast or took too long to start, status= "+simulationStatus);
// }
// System.out.println(simulationStatus);
// }
// SimulationStatusPersistent lastSimStatus = simulationStatus;
// while(!simulationStatus.isStopped() && !simulationStatus.isCompleted() && !simulationStatus.isFailed()){
// for(int i = 0;i<3;i++){
// MessageEvent[] messageEvents = vcellConnection.getMessageEvents();
// Thread.sleep(1000);
// }
// simulationStatus = vcellConnection.getUserMetaDbServer().getSimulationStatus(vcSimulationIdentifier.getSimulationKey());
// if(!simulationStatus.toString().equals(lastSimStatus.toString())){
// lastSimStatus = simulationStatus;
// System.out.println("running status="+simulationStatus);
// }
// //					MessageEvent[] messageEvents = vcellConnection.getMessageEvents();
// //					for (int i = 0; messageEvents != null && i < messageEvents.length; i++) {
// //						System.out.println(messageEvents[i]);
// //					}
// }
// 
// if(!lastSimStatus.isCompleted()){
// throw new Exception("Unexpected run status: "+lastSimStatus.toString());
// }
// 
// 
// String updatestr = "update missingdata set notes='"+DB_NOTES_SIM_RUNOK_CODE+"' where simjobsimref="+simJobSimRef;
// System.out.println(updatestr);
// updateStatement.executeUpdate(updatestr);
// con.commit();
// }catch(Exception e){
// e.printStackTrace();
// String errString = TokenMangler.fixTokenStrict(DB_NOTES_SIM_ERROR_CODE+e.getClass().getSimpleName()+" "+e.getMessage());
// if(errString.length() > 256){
// errString = errString.substring(0, 256);
// }
// String updatestr = "update missingdata set notes='"+errString+"' where simjobsimref="+simJobSimRef;
// System.out.println(updatestr);
// updateStatement.executeUpdate(updatestr);
// con.commit();
// }
// }
// 
// }
private static void checkDataExists(Connection con, boolean bExistOnly) throws SQLException {
    AmplistorCredential amplistorCredential = AmplistorUtilsTest.getAmplistorCredential();
    Statement queryStatement = con.createStatement();
    Statement updateStatement = con.createStatement();
    // Hashtable<KeyValue, Exception> errorHash = new Hashtable<>();
    String sql = "select missingdata.*,parentsimref " + " from missingdata,vc_simulation,vc_userinfo" + " where vc_simulation.id = missingdata.simjobsimref and" + " vc_simulation.parentsimref is null and " + " vc_simulation.ownerref=vc_userinfo.id and " + " missingdata.notes is not null and " + " (missingdata.notes like '%exceeded_maximum%' " + " or missingdata.notes ='recheck dataexists' )" + // " vc_userinfo.userid='schaff' "+
    " order by missingdata.userid";
    ResultSet rset = queryStatement.executeQuery(sql);
    while (rset.next()) {
        // if(!rset.getString("dataexists").equals("tbd")){
        // continue;
        // }
        KeyValue simJobSimRef = null;
        KeyValue parentsimref = null;
        User user = null;
        try {
            simJobSimRef = new KeyValue(rset.getString("simjobsimref"));
            parentsimref = (rset.getString("parentsimref") == null ? null : new KeyValue(rset.getString("parentsimref")));
            user = new User(rset.getString("userid"), new KeyValue(rset.getString("userinfoid")));
            int jobIndex = rset.getInt("jobindex");
            File primaryDataDir = new File("\\\\cfs02\\ifs\\raid\\vcell\\users\\" + user.getName());
            String updatestr = null;
            File filePathNamePrime = new File(primaryDataDir, SimulationData.createCanonicalSimLogFileName(simJobSimRef, 0, false));
            if (filePathNamePrime.exists()) {
                updatestr = "fileNewPrime";
            } else if (new File(primaryDataDir, SimulationData.createCanonicalSimLogFileName(simJobSimRef, 0, true)).exists()) {
                updatestr = "fileOldPrime";
            } else if (parentsimref != null && new File(primaryDataDir, SimulationData.createCanonicalSimLogFileName(parentsimref, 0, false)).exists()) {
                updatestr = "fileNewParent";
            } else if (parentsimref != null && new File(primaryDataDir, SimulationData.createCanonicalSimLogFileName(parentsimref, 0, true)).exists()) {
                updatestr = "fileOldParent";
            } else if (AmplistorUtils.bFileExists(new URL(AmplistorUtils.DEFAULT_AMPLI_SERVICE_VCELL_URL + user.getName() + "/" + filePathNamePrime.getName()), amplistorCredential)) {
                updatestr = "ampliNewPrime";
            } else if (AmplistorUtils.bFileExists(new URL(AmplistorUtils.DEFAULT_AMPLI_SERVICE_VCELL_URL + user.getName() + "/" + SimulationData.createCanonicalSimLogFileName(simJobSimRef, 0, true)), amplistorCredential)) {
                updatestr = "ampliOldPrime";
            } else if (parentsimref != null && AmplistorUtils.bFileExists(new URL(AmplistorUtils.DEFAULT_AMPLI_SERVICE_VCELL_URL + user.getName() + "/" + SimulationData.createCanonicalSimLogFileName(parentsimref, 0, false)), amplistorCredential)) {
                updatestr = "ampliNewParent";
            } else if (parentsimref != null && AmplistorUtils.bFileExists(new URL(AmplistorUtils.DEFAULT_AMPLI_SERVICE_VCELL_URL + user.getName() + "/" + SimulationData.createCanonicalSimLogFileName(parentsimref, 0, true)), amplistorCredential)) {
                updatestr = "ampliOldParent";
            } else {
                updatestr = "false";
            }
            if (bExistOnly || updatestr.equals("false")) {
                updateStatement.executeUpdate("update missingdata set dataexists='" + updatestr + "' where simjobsimref=" + simJobSimRef.toString());
                con.commit();
                continue;
            }
            // Log file exists, now check if the data can really be read
            VCSimulationIdentifier vcSimulationIdentifier = new VCSimulationIdentifier(simJobSimRef, user);
            VCSimulationDataIdentifier vcSimulationDataIdentifier = new VCSimulationDataIdentifier(vcSimulationIdentifier, jobIndex);
            // Try to read log,times and simdata to see if this data is well formed
            SimDataAmplistorInfo simDataAmplistorInfo = AmplistorUtils.getSimDataAmplistorInfoFromPropertyLoader();
            SimulationData simData = new SimulationData(vcSimulationDataIdentifier, primaryDataDir, null, simDataAmplistorInfo);
            double[] dataTimes = simData.getDataTimes();
            DataIdentifier[] dataIdentifiers = simData.getVarAndFunctionDataIdentifiers(null);
            DataIdentifier readDataIdentifier = null;
            for (DataIdentifier dataIdentifier : dataIdentifiers) {
                if (!dataIdentifier.isFunction()) {
                    if (simData.getIsODEData()) {
                        ODEDataBlock odeDataBlock = simData.getODEDataBlock();
                        odeDataBlock.getODESimData().getRow(dataTimes.length - 1);
                    } else {
                        simData.getSimDataBlock(null, dataIdentifier.getName(), dataTimes[dataTimes.length - 1]);
                    }
                    readDataIdentifier = dataIdentifier;
                    break;
                }
            }
            System.out.println(BeanUtils.forceStringSize("user= " + user.getName(), 20, " ", false) + " simref= " + BeanUtils.forceStringSize(simJobSimRef.toString(), 14, " ", false) + " numTimes= " + BeanUtils.forceStringSize(dataTimes.length + "", 8, " ", true) + " readDataID= " + BeanUtils.forceStringSize(readDataIdentifier.getName() + "", 20, " ", true));
            updatestr = "update missingdata set dataexists='readable' where simjobsimref=" + simJobSimRef.toString();
        // String updatestr = "update missingdata set dataexists='true' where"+
        // " userinfoid="+user.getID().toString()+
        // " and simjobsimref="+simJobSimRef.toString()+
        // " and maxtaskid="+rset.getString("maxtaskid")+
        // " and jobindex="+jobIndex;
        // updateStatement.executeUpdate(updatestr);
        // con.commit();
        } catch (Exception e) {
            if (simJobSimRef == null) {
                e.printStackTrace();
            // throw new SQLException("Error querying",e);
            } else {
                String errString = e.getClass().getSimpleName() + " " + e.getMessage();
                if (errString.length() > 512) {
                    errString = errString.substring(0, 512);
                }
                String updatestr = "update missingdata set dataexists='error - " + TokenMangler.fixTokenStrict(errString) + "' where simjobsimref=" + simJobSimRef.toString();
                // updateStatement.executeUpdate(updatestr);
                // con.commit();
                System.out.println(BeanUtils.forceStringSize("user= " + (user == null ? "unavailable" : user.getName()), 20, " ", false) + " simref= " + BeanUtils.forceStringSize(simJobSimRef.toString(), 14, " ", false) + " parentsimref= " + BeanUtils.forceStringSize((parentsimref == null ? "NULL" : parentsimref.toString()), 14, " ", false) + " failed= " + e.getMessage());
            // errorHash.put(simJobSimRef,e);
            }
        }
    }
    rset.close();
// return errorHash;
}