Examples with Cachetable cbit.vcell.simdata.Cachetable used on opensource projects

Example 1 with Cachetable

use of cbit.vcell.simdata.Cachetable in project vcell by virtualcell.

the class SimulationServiceImpl method getDataSetController.

private static DataSetControllerImpl getDataSetController(SimulationServiceContext simServiceContext) {
    try {
        OutputContext outputContext = new OutputContext(new AnnotatedFunction[0]);
        Cachetable cacheTable = new Cachetable(10000, 1000000L);
        DataSetControllerImpl datasetController = new DataSetControllerImpl(cacheTable, simServiceContext.localSimDataDir.getParentFile(), null);
        simServiceContext.times = datasetController.getDataSetTimes(simServiceContext.vcDataIdentifier);
        simServiceContext.dataIdentifiers = datasetController.getDataIdentifiers(outputContext, simServiceContext.vcDataIdentifier);
        return datasetController;
    } catch (IOException | DataAccessException e1) {
        e1.printStackTrace();
        throw new RuntimeException("failed to read dataset: " + e1.getMessage());
    }
}

Example 2 with Cachetable

use of cbit.vcell.simdata.Cachetable in project vcell by virtualcell.

the class VCellServices method main.

/**
 * Starts the application.
 * @param args an array of command-line arguments
 */
public static void main(java.lang.String[] args) {
    OperatingSystemInfo.getInstance();
    if (args.length != 3 && args.length != 0) {
        System.out.println("Missing arguments: " + VCellServices.class.getName() + " [sshHost sshUser sshKeyFile] ");
        System.exit(1);
    }
    try {
        PropertyLoader.loadProperties(REQUIRED_SERVICE_PROPERTIES);
        ResourceUtil.setNativeLibraryDirectory();
        new LibraryLoaderThread(false).start();
        PythonSupport.verifyInstallation(new PythonPackage[] { PythonPackage.VTK, PythonPackage.THRIFT });
        HtcProxy htcProxy = SlurmProxy.creatCommandService(args);
        int serviceOrdinal = 0;
        VCMongoMessage.serviceStartup(ServiceName.dispatch, new Integer(serviceOrdinal), args);
        // //
        // // JMX registration
        // //
        // MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
        // mbs.registerMBean(new VCellServiceMXBeanImpl(), new ObjectName(VCellServiceMXBean.jmxObjectName));
        ServiceInstanceStatus serviceInstanceStatus = new ServiceInstanceStatus(VCellServerID.getSystemServerID(), ServiceType.MASTER, serviceOrdinal, ManageUtils.getHostName(), new Date(), true);
        ConnectionFactory conFactory = DatabaseService.getInstance().createConnectionFactory();
        KeyFactory keyFactory = conFactory.getKeyFactory();
        DatabaseServerImpl databaseServerImpl = new DatabaseServerImpl(conFactory, keyFactory);
        AdminDBTopLevel adminDbTopLevel = new AdminDBTopLevel(conFactory);
        SimulationDatabase simulationDatabase = new SimulationDatabaseDirect(adminDbTopLevel, databaseServerImpl, true);
        String cacheSize = PropertyLoader.getRequiredProperty(PropertyLoader.simdataCacheSizeProperty);
        long maxMemSize = Long.parseLong(cacheSize);
        Cachetable cacheTable = new Cachetable(MessageConstants.MINUTE_IN_MS * 20, maxMemSize);
        DataSetControllerImpl dataSetControllerImpl = new DataSetControllerImpl(cacheTable, new File(PropertyLoader.getRequiredProperty(PropertyLoader.primarySimDataDirInternalProperty)), new File(PropertyLoader.getProperty(PropertyLoader.secondarySimDataDirInternalProperty, PropertyLoader.getRequiredProperty(PropertyLoader.primarySimDataDirInternalProperty))));
        ExportServiceImpl exportServiceImpl = new ExportServiceImpl();
        // add dataJobListener
        DataServerImpl dataServerImpl = new DataServerImpl(dataSetControllerImpl, exportServiceImpl);
        VCMessagingService vcMessagingService_int = new VCMessagingServiceActiveMQ();
        String jmshost_int = PropertyLoader.getRequiredProperty(PropertyLoader.jmsIntHostInternal);
        int jmsport_int = Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.jmsIntPortInternal));
        vcMessagingService_int.setConfiguration(new ServerMessagingDelegate(), jmshost_int, jmsport_int);
        VCMessagingService vcMessagingService_sim = new VCMessagingServiceActiveMQ();
        String jmshost_sim = PropertyLoader.getRequiredProperty(PropertyLoader.jmsSimHostInternal);
        int jmsport_sim = Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.jmsSimPortInternal));
        vcMessagingService_sim.setConfiguration(new ServerMessagingDelegate(), jmshost_sim, jmsport_sim);
        VCellServices vcellServices = new VCellServices(htcProxy, vcMessagingService_int, vcMessagingService_sim, serviceInstanceStatus, databaseServerImpl, dataServerImpl, simulationDatabase);
        dataSetControllerImpl.addDataJobListener(vcellServices);
        exportServiceImpl.addExportListener(vcellServices);
        vcellServices.init();
    } catch (Throwable e) {
        e.printStackTrace(System.out);
    }
}

Example 3 with Cachetable

use of cbit.vcell.simdata.Cachetable in project vcell by virtualcell.

the class SimDataServer method main.

/**
 * Starts the application.
 * @param args an array of command-line arguments
 */
public static void main(java.lang.String[] args) {
    OperatingSystemInfo.getInstance();
    if (args.length != 1) {
        System.out.println("Missing arguments: " + SimDataServer.class.getName() + " (CombinedData | ExportDataOnly | SimDataOnly)");
        System.exit(1);
    }
    try {
        PropertyLoader.loadProperties(REQUIRED_SERVICE_PROPERTIES);
        ResourceUtil.setNativeLibraryDirectory();
        new LibraryLoaderThread(false).start();
        PythonSupport.verifyInstallation(new PythonPackage[] { PythonPackage.VTK, PythonPackage.THRIFT });
        int serviceOrdinal = 99;
        SimDataServiceType simDataServiceType = SimDataServiceType.valueOf(args[0]);
        if (simDataServiceType == null) {
            throw new RuntimeException("expecting argument (CombinedData | ExportDataOnly | SimDataOnly)");
        }
        final ServiceInstanceStatus serviceInstanceStatus;
        final ServiceName serviceName;
        switch(simDataServiceType) {
            case ExportDataOnly:
                {
                    serviceInstanceStatus = new ServiceInstanceStatus(VCellServerID.getSystemServerID(), ServiceType.DATAEXPORT, serviceOrdinal, ManageUtils.getHostName(), new Date(), true);
                    serviceName = ServiceName.export;
                    break;
                }
            case SimDataOnly:
                {
                    serviceInstanceStatus = new ServiceInstanceStatus(VCellServerID.getSystemServerID(), ServiceType.DATA, serviceOrdinal, ManageUtils.getHostName(), new Date(), true);
                    serviceName = ServiceName.simData;
                    break;
                }
            case CombinedData:
                {
                    serviceInstanceStatus = new ServiceInstanceStatus(VCellServerID.getSystemServerID(), ServiceType.COMBINEDDATA, serviceOrdinal, ManageUtils.getHostName(), new Date(), true);
                    serviceName = ServiceName.combinedData;
                    break;
                }
            default:
                {
                    throw new RuntimeException("unexpected SimDataServiceType " + simDataServiceType);
                }
        }
        VCMongoMessage.serviceStartup(serviceName, new Integer(serviceOrdinal), args);
        // //
        // // JMX registration
        // //
        // MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
        // mbs.registerMBean(new VCellServiceMXBeanImpl(), new ObjectName(VCellServiceMXBean.jmxObjectName));
        String cacheSize = PropertyLoader.getRequiredProperty(PropertyLoader.simdataCacheSizeProperty);
        long maxMemSize = Long.parseLong(cacheSize);
        Cachetable cacheTable = new Cachetable(MessageConstants.MINUTE_IN_MS * 20, maxMemSize);
        DataSetControllerImpl dataSetControllerImpl = new DataSetControllerImpl(cacheTable, new File(PropertyLoader.getRequiredProperty(PropertyLoader.primarySimDataDirInternalProperty)), new File(PropertyLoader.getProperty(PropertyLoader.secondarySimDataDirInternalProperty, PropertyLoader.getRequiredProperty(PropertyLoader.primarySimDataDirInternalProperty))));
        ExportServiceImpl exportServiceImpl = new ExportServiceImpl();
        DataServerImpl dataServerImpl = new DataServerImpl(dataSetControllerImpl, exportServiceImpl);
        VCMessagingService vcMessagingService = VCellServiceHelper.getInstance().loadService(VCMessagingService.class);
        String jmshost = PropertyLoader.getRequiredProperty(PropertyLoader.jmsIntHostInternal);
        int jmsport = Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.jmsIntPortInternal));
        vcMessagingService.setConfiguration(new ServerMessagingDelegate(), jmshost, jmsport);
        SimDataServer simDataServer = new SimDataServer(serviceInstanceStatus, dataServerImpl, vcMessagingService, simDataServiceType, false);
        // add dataJobListener
        dataSetControllerImpl.addDataJobListener(simDataServer);
        // add export listener
        exportServiceImpl.addExportListener(simDataServer);
        simDataServer.init();
    } catch (Throwable e) {
        lg.error("uncaught exception initializing SimDataServer: " + e.getLocalizedMessage(), e);
        System.exit(1);
    }
}

Example 4 with Cachetable

use of cbit.vcell.simdata.Cachetable in project vcell by virtualcell.

the class IonItems method saveVCMLRemoveSimsWithNoData.

private static void saveVCMLRemoveSimsWithNoData(Connection con, File dirToSaveVCML, File usersDir, long startSimKey, long endSimKey) {
    // select id from vc_biomodel where versionflag=3 and privacy=0;
    String bmsql = "SELECT " + BioModelTable.table.id.getQualifiedColName() + " modelkey" + "," + UserTable.table.id.getQualifiedColName() + " userkey" + "," + UserTable.table.userid.getQualifiedColName() + " userid" + " FROM " + BioModelTable.table.getTableName() + "," + UserTable.table.getTableName() + " WHERE " + BioModelTable.table.versionFlag.getQualifiedColName() + "=" + VersionFlag.Published.getIntValue() + " AND " + BioModelTable.table.privacy.getQualifiedColName() + "=" + GroupAccess.GROUPACCESS_ALL.intValue() + " AND " + BioModelTable.table.ownerRef.getQualifiedColName() + "=" + UserTable.table.id.getQualifiedColName() + " AND " + BioModelTable.table.id.getQualifiedColName() + " IN " + "(SELECT " + PublicationModelLinkTable.table.bioModelRef.getQualifiedColName() + " FROM " + PublicationModelLinkTable.table.getTableName() + " WHERE " + PublicationModelLinkTable.table.bioModelRef.getQualifiedColName() + " IS NOT NULL)" + " ORDER BY modelKey";
    String mmsql = "SELECT " + MathModelTable.table.id.getQualifiedColName() + " modelkey" + "," + UserTable.table.id.getQualifiedColName() + " userkey" + "," + UserTable.table.userid.getQualifiedColName() + " userid" + " FROM " + MathModelTable.table.getTableName() + "," + UserTable.table.getTableName() + " WHERE " + MathModelTable.table.versionFlag.getQualifiedColName() + "=" + VersionFlag.Published.getIntValue() + " AND " + MathModelTable.table.privacy.getQualifiedColName() + "=" + GroupAccess.GROUPACCESS_ALL.intValue() + " AND " + MathModelTable.table.ownerRef.getQualifiedColName() + "=" + UserTable.table.id.getQualifiedColName() + " AND " + MathModelTable.table.id.getQualifiedColName() + " IN " + // vc_publicationmodellink.biomodelref
    "(SELECT " + PublicationModelLinkTable.table.mathModelRef.getQualifiedColName() + " FROM " + PublicationModelLinkTable.table.getTableName() + " WHERE " + PublicationModelLinkTable.table.mathModelRef.getQualifiedColName() + " IS NOT NULL)" + " ORDER BY modelKey";
    Statement stmt = null;
    try {
        stmt = con.createStatement();
        // ArrayList<VCDocument> successDocs = new ArrayList<VCDocument>();
        DatabaseServerImpl databaseServerImpl = new DatabaseServerImpl(connFac, null);
        // ServerDocumentManager serverDocManager = new ServerDocumentManager(databaseServerImpl);
        // QueryHashtable qht = new QueryHashtable();
        DataSetControllerImpl dsci = new DataSetControllerImpl(new Cachetable(Cachetable.minute * 3, 30000000L), usersDir, null);
        ArrayList<Object[]> modeKeyAndUser = new ArrayList<Object[]>();
        for (int i = 0; i < 2; i++) {
            String sql = (i == 0 ? bmsql : mmsql);
            ResultSet rset = stmt.executeQuery(sql);
            while (rset.next()) {
                BigDecimal modelKeyBigDecimal = rset.getBigDecimal("modelkey");
                KeyValue modelKey = new KeyValue(modelKeyBigDecimal);
                if (startSimKey == 0 || (modelKeyBigDecimal.longValue() >= startSimKey && modelKeyBigDecimal.longValue() <= endSimKey)) {
                    User user = new User(rset.getString("userid"), new KeyValue(rset.getBigDecimal("userkey")));
                    modeKeyAndUser.add(new Object[] { (i == 0 ? VCDocumentType.BIOMODEL_DOC : VCDocumentType.MATHMODEL_DOC), modelKey, user });
                }
            }
            rset.close();
        }
        stmt.close();
        VCDocument vcDoc = null;
        for (int ii = 0; ii < modeKeyAndUser.size(); ii++) {
            try {
                VCDocumentType vcDocType = (VCDocumentType) modeKeyAndUser.get(ii)[0];
                KeyValue modelKey = (KeyValue) modeKeyAndUser.get(ii)[1];
                User user = (User) modeKeyAndUser.get(ii)[2];
                // -----NOTE: (privacy, versionflag) may not match between xml and database (because privacy and versionflag can be set without saving model)
                // some models regenerated from xml will not have same privacy and versionflag matched in query from database (the database values are correct, ignore the xml values)
                BigString modelXML = (vcDocType == VCDocumentType.BIOMODEL_DOC ? databaseServerImpl.getBioModelXML(user, modelKey) : databaseServerImpl.getMathModelXML(user, modelKey));
                Simulation[] sims = null;
                vcDoc = null;
                if (vcDocType == VCDocumentType.BIOMODEL_DOC) {
                    BioModel bm = cbit.vcell.xml.XmlHelper.XMLToBioModel(new XMLSource(modelXML.toString()));
                    bm.refreshDependencies();
                    sims = bm.getSimulations().clone();
                    vcDoc = bm;
                } else {
                    MathModel mm = cbit.vcell.xml.XmlHelper.XMLToMathModel(new XMLSource(modelXML.toString()));
                    mm.refreshDependencies();
                    sims = mm.getSimulations().clone();
                    vcDoc = mm;
                }
                // if(true) {
                // System.out.println(modelKey+" "+user+" "+vcDoc);
                // continue;
                // }
                boolean bRemovedSims = false;
                for (int j = 0; sims != null && j < sims.length; j++) {
                    try {
                        // public FoundSimDataInfo(long simID, int lastTaskID, int jobIndex, String userid) {
                        File foundLog = findSimLogFile(usersDir, new FoundSimDataInfo(Long.parseLong(sims[j].getKey().toString()), -1, 0, user.getName()), true, vcDocType);
                        if (foundLog == null) {
                            throw new Exception("Logfile not found for sim " + sims[j].getKey().toString());
                        } else {
                            VCSimulationIdentifier vcsi = new VCSimulationIdentifier(sims[j].getKey(), user);
                            VCSimulationDataIdentifier vcdID = new VCSimulationDataIdentifier(vcsi, 0);
                            OutputFunctionContext outputFunctionContext = sims[j].getSimulationOwner().getOutputFunctionContext();
                            OutputContext outputContext = new OutputContext(outputFunctionContext.getOutputFunctionsList().toArray(new AnnotatedFunction[0]));
                            DataIdentifier[] dataIdentifiers = dsci.getDataIdentifiers(outputContext, vcdID);
                            double[] dataSetTimes = dsci.getDataSetTimes(vcdID);
                            if (sims[j].getMathDescription().getGeometry().getDimension() == 0) {
                                ODEDataBlock odeDataBlock = dsci.getODEDataBlock(vcdID);
                            } else {
                                // for(int k=0;k<dataIdentifiers.length;k++){
                                SimDataBlock simDataBlock = dsci.getSimDataBlock(outputContext, vcdID, dataIdentifiers[0].getName(), dataSetTimes[dataSetTimes.length - 1]);
                            // }
                            }
                        }
                    } catch (Exception e) {
                        System.out.println("-----SIMERROR " + vcDoc.getVersion().getVersionKey() + "\n-----Sim=" + sims[j].getVersion());
                        e.printStackTrace();
                        if (e.getMessage().contains("HDF5")) {
                            return;
                        }
                        bRemovedSims = true;
                        if (vcDoc.getDocumentType() == VCDocument.VCDocumentType.BIOMODEL_DOC) {
                            ((BioModel) vcDoc).removeSimulation(sims[j]);
                        } else {
                            ((MathModel) vcDoc).removeSimulation(sims[j]);
                        }
                    }
                }
                String docXml = null;
                if (vcDoc.getDocumentType() == VCDocument.VCDocumentType.BIOMODEL_DOC) {
                    docXml = XmlHelper.bioModelToXML(((BioModel) vcDoc));
                } else {
                    docXml = XmlHelper.mathModelToXML(((MathModel) vcDoc));
                }
                PrintWriter pw = new PrintWriter(new File(dirToSaveVCML, vcDoc.getVersion().getVersionKey().toString() + "_" + (bRemovedSims) + ".xml"));
                pw.write(docXml);
                pw.close();
            // }
            } catch (Exception e) {
                System.out.println("----------GENERALERROR " + vcDoc.getVersion().getVersionKey());
                e.printStackTrace();
                if (vcDoc != null) {
                    PrintWriter pw = new PrintWriter(new File(dirToSaveVCML, vcDoc.getVersion().getVersionKey().toString() + "_exception.xml"));
                    pw.write("");
                    pw.close();
                }
            }
        }
    // for(VCDocument successDoc:successDocs) {
    // if(successDoc.getDocumentType() == VCDocument.VCDocumentType.BIOMODEL_DOC) {
    // ((BioModel)vcDoc).removeSimulation(sims[i]);
    // }else {
    // ((MathModel)vcDoc).removeSimulation(sims[i]);
    // }
    // 
    // }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        if (stmt != null) {
            try {
                stmt.close();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
}

Example 5 with Cachetable

use of cbit.vcell.simdata.Cachetable in project vcell by virtualcell.

the class LocalWorkspace method getDataSetControllerImpl.

/**
 * Method getDataSetControllerImpl.
 * @return DataSetControllerImpl
 * @throws FileNotFoundException
 */
public DataSetControllerImpl getDataSetControllerImpl() throws FileNotFoundException {
    if (dataSetControllerImpl == null) {
        File rootDir = new File(getDefaultWorkspaceDirectory());
        dataSetControllerImpl = new DataSetControllerImpl(new Cachetable(10000, 1000000L), rootDir, rootDir);
    }
    return dataSetControllerImpl;
}