Examples with VCPooledQueueConsumer cbit.vcell.message.VCPooledQueueConsumer used on opensource projects

Example 1 with VCPooledQueueConsumer

use of cbit.vcell.message.VCPooledQueueConsumer in project vcell by virtualcell.

the class HtcSimulationWorker method initQueueConsumer.

private void initQueueConsumer() {
    this.sharedMessageProducer = vcMessagingService.createProducerSession();
    QueueListener queueListener = new QueueListener() {

        @Override
        public void onQueueMessage(VCMessage vcMessage, VCMessageSession session) throws RollbackException {
            SimulationTask simTask = null;
            try {
                SimulationTaskMessage simTaskMessage = new SimulationTaskMessage(vcMessage);
                simTask = simTaskMessage.getSimulationTask();
                if (lg.isInfoEnabled()) {
                    lg.info("onQueueMessage() run simulation key=" + simTask.getSimKey() + ", job=" + simTask.getSimulationJobID() + ", task=" + simTask.getTaskID() + " for user " + simTask.getUserName());
                }
                PostProcessingChores rd = choresFor(simTask);
                HtcProxy clonedHtcProxy = htcProxy.cloneThreadsafe();
                if (lg.isInfoEnabled()) {
                    lg.info("onQueueMessage() submit job: simulation key=" + simTask.getSimKey() + ", job=" + simTask.getSimulationJobID() + ", task=" + simTask.getTaskID() + " for user " + simTask.getUserName());
                }
                HtcJobID pbsId = submit2PBS(simTask, clonedHtcProxy, rd);
                if (lg.isInfoEnabled()) {
                    lg.info("onQueueMessage() sending 'accepted' message for job: simulation key=" + simTask.getSimKey() + ", job=" + simTask.getSimulationJobID() + ", task=" + simTask.getTaskID() + " for user " + simTask.getUserName());
                }
                synchronized (sharedMessageProducer) {
                    WorkerEventMessage.sendAccepted(sharedMessageProducer, HtcSimulationWorker.class.getName(), simTask, ManageUtils.getHostName(), pbsId);
                }
                if (lg.isInfoEnabled()) {
                    lg.info("onQueueMessage() sent 'accepted' message for job: simulation key=" + simTask.getSimKey() + ", job=" + simTask.getSimulationJobID() + ", task=" + simTask.getTaskID() + " for user " + simTask.getUserName());
                }
            } catch (Exception e) {
                lg.error(e.getMessage(), e);
                if (simTask != null) {
                    try {
                        lg.error("failed to process simTask request: " + e.getMessage() + " for simulation key=" + simTask.getSimKey() + ", job=" + simTask.getSimulationJobID() + ", task=" + simTask.getTaskID() + " for user " + simTask.getUserName(), e);
                        synchronized (sharedMessageProducer) {
                            WorkerEventMessage.sendFailed(sharedMessageProducer, HtcSimulationWorker.class.getName(), simTask, ManageUtils.getHostName(), SimulationMessage.jobFailed(e.getMessage()));
                        }
                        lg.error("sent 'failed' message for simulation key=" + simTask.getSimKey() + ", job=" + simTask.getSimulationJobID() + ", task=" + simTask.getTaskID() + " for user " + simTask.getUserName(), e);
                    } catch (VCMessagingException e1) {
                        lg.error(e1.getMessage(), e);
                    }
                } else {
                    lg.error("failed to process simTask request: " + e.getMessage(), e);
                }
            }
        }
    };
    int numHtcworkerThreads = Integer.parseInt(PropertyLoader.getProperty(PropertyLoader.htcworkerThreadsProperty, "5"));
    this.pooledQueueConsumer = new VCPooledQueueConsumer(queueListener, numHtcworkerThreads, sharedMessageProducer);
    this.pooledQueueConsumer.initThreadPool();
    VCellQueue queue = VCellQueue.SimJobQueue;
    VCMessageSelector selector = vcMessagingService.createSelector(getJobSelector());
    String threadName = "SimJob Queue Consumer";
    queueConsumer = new VCQueueConsumer(queue, pooledQueueConsumer, selector, threadName, MessageConstants.PREFETCH_LIMIT_SIM_JOB_HTC);
    vcMessagingService.addMessageConsumer(queueConsumer);
}

Example 2 with VCPooledQueueConsumer

use of cbit.vcell.message.VCPooledQueueConsumer in project vcell by virtualcell.

the class DatabaseServer method init.

public void init() throws Exception {
    int numDatabaseThreads = Integer.parseInt(PropertyLoader.getProperty(PropertyLoader.databaseThreadsProperty, "5"));
    this.sharedProducerSession = vcMessagingService_int.createProducerSession();
    rpcMessageHandler = new VCRpcMessageHandler(databaseServerImpl, VCellQueue.DbRequestQueue);
    this.pooledQueueConsumer = new VCPooledQueueConsumer(rpcMessageHandler, numDatabaseThreads, sharedProducerSession);
    this.pooledQueueConsumer.initThreadPool();
    rpcConsumer = new VCQueueConsumer(VCellQueue.DbRequestQueue, this.pooledQueueConsumer, null, "Database RPC Server Thread", MessageConstants.PREFETCH_LIMIT_DB_REQUEST);
    vcMessagingService_int.addMessageConsumer(rpcConsumer);
    this.databaseCleanupThread = new DatabaseCleanupThread(databaseServerImpl);
    this.databaseCleanupThread.start();
}

Example 3 with VCPooledQueueConsumer

use of cbit.vcell.message.VCPooledQueueConsumer in project vcell by virtualcell.

the class HtcSimulationWorker method initQueueConsumer.

// ------------------------------Job Monitor Section END
private void initQueueConsumer() {
    startJobMonitor();
    this.messageProducer_sim = vcMessagingService_sim.createProducerSession();
    this.messageProducer_int = vcMessagingService_int.createProducerSession();
    QueueListener queueListener = new QueueListener() {

        @Override
        public void onQueueMessage(VCMessage vcMessage, VCMessageSession session) throws RollbackException {
            SimulationTask simTask = null;
            try {
                SimulationTaskMessage simTaskMessage = new SimulationTaskMessage(vcMessage);
                simTask = simTaskMessage.getSimulationTask();
                if (lg.isInfoEnabled()) {
                    lg.info("onQueueMessage() run simulation key=" + simTask.getSimKey() + ", job=" + simTask.getSimulationJobID() + ", task=" + simTask.getTaskID() + " for user " + simTask.getUserName());
                }
                PostProcessingChores rd = choresFor(simTask);
                HtcProxy clonedHtcProxy = htcProxy.cloneThreadsafe();
                if (lg.isInfoEnabled()) {
                    lg.info("onQueueMessage() submit job: simulation key=" + simTask.getSimKey() + ", job=" + simTask.getSimulationJobID() + ", task=" + simTask.getTaskID() + " for user " + simTask.getUserName());
                }
                HtcJobID pbsId = submit2PBS(simTask, clonedHtcProxy, rd);
                addMonitorJob(pbsId.getJobNumber(), simTask, false);
                if (lg.isInfoEnabled()) {
                    lg.info("onQueueMessage() sending 'accepted' message for job: simulation key=" + simTask.getSimKey() + ", job=" + simTask.getSimulationJobID() + ", task=" + simTask.getTaskID() + " for user " + simTask.getUserName());
                }
                synchronized (messageProducer_sim) {
                    WorkerEventMessage.sendAccepted(messageProducer_sim, HtcSimulationWorker.class.getName(), simTask, ManageUtils.getHostName(), pbsId);
                    WorkerEventMessage.sendStarting(messageProducer_sim, HtcSimulationWorker.class.getName(), simTask, ManageUtils.getHostName(), SimulationMessage.MESSAGE_WORKEREVENT_STARTING);
                    WorkerEventMessage.sendProgress(messageProducer_sim, HtcSimulationWorker.class.getName(), simTask, ManageUtils.getHostName(), 0, 0, SimulationMessage.MESSAGE_JOB_RUNNING_UNKNOWN);
                }
                if (lg.isInfoEnabled()) {
                    lg.info("onQueueMessage() sent 'accepted' message for job: simulation key=" + simTask.getSimKey() + ", job=" + simTask.getSimulationJobID() + ", task=" + simTask.getTaskID() + " for user " + simTask.getUserName());
                }
            } catch (Exception e) {
                lg.error(e.getMessage(), e);
                if (simTask != null) {
                    try {
                        lg.error("failed to process simTask request: " + e.getMessage() + " for simulation key=" + simTask.getSimKey() + ", job=" + simTask.getSimulationJobID() + ", task=" + simTask.getTaskID() + " for user " + simTask.getUserName(), e);
                        synchronized (messageProducer_sim) {
                            WorkerEventMessage.sendFailed(messageProducer_sim, HtcSimulationWorker.class.getName(), simTask, ManageUtils.getHostName(), SimulationMessage.jobFailed(e.getMessage()));
                        }
                        lg.error("sent 'failed' message for simulation key=" + simTask.getSimKey() + ", job=" + simTask.getSimulationJobID() + ", task=" + simTask.getTaskID() + " for user " + simTask.getUserName(), e);
                    } catch (VCMessagingException e1) {
                        lg.error(e1.getMessage(), e);
                    }
                } else {
                    lg.error("failed to process simTask request: " + e.getMessage(), e);
                }
            }
        }
    };
    int numHtcworkerThreads = Integer.parseInt(PropertyLoader.getProperty(PropertyLoader.htcworkerThreadsProperty, "5"));
    this.pooledQueueConsumer_int = new VCPooledQueueConsumer(queueListener, numHtcworkerThreads, messageProducer_int);
    this.pooledQueueConsumer_int.initThreadPool();
    VCellQueue queue = VCellQueue.SimJobQueue;
    VCMessageSelector selector = vcMessagingService_int.createSelector(getJobSelector());
    String threadName = "SimJob Queue Consumer";
    queueConsumer = new VCQueueConsumer(queue, pooledQueueConsumer_int, selector, threadName, MessageConstants.PREFETCH_LIMIT_SIM_JOB_HTC);
    vcMessagingService_int.addMessageConsumer(queueConsumer);
}

Example 4 with VCPooledQueueConsumer

use of cbit.vcell.message.VCPooledQueueConsumer in project vcell by virtualcell.

the class SimDataServer method init.

public void init() throws Exception {
    String dataRequestFilter = "(" + VCMessagingConstants.MESSAGE_TYPE_PROPERTY + "='" + VCMessagingConstants.MESSAGE_TYPE_RPC_SERVICE_VALUE + "') " + " AND (" + VCMessagingConstants.SERVICE_TYPE_PROPERTY + "='" + ServiceType.DATA.getName() + "')";
    String exportOnlyFilter = "(" + ServiceType.DATAEXPORT.getName() + " is NOT NULL)";
    String dataOnlyFilter = "(" + ServiceType.DATAEXPORT.getName() + " is NULL)";
    VCMessageSelector selector;
    ServiceType serviceType = serviceInstanceStatus.getType();
    int numThreads;
    switch(simDataServiceType) {
        case CombinedData:
            {
                selector = vcMessagingService_int.createSelector(dataRequestFilter);
                int exportThreads = Integer.parseInt(PropertyLoader.getProperty(PropertyLoader.exportdataThreadsProperty, "3"));
                int simdataThreads = Integer.parseInt(PropertyLoader.getProperty(PropertyLoader.simdataThreadsProperty, "5"));
                numThreads = exportThreads + simdataThreads;
                break;
            }
        case ExportDataOnly:
            {
                selector = vcMessagingService_int.createSelector(dataRequestFilter + " AND " + exportOnlyFilter);
                numThreads = Integer.parseInt(PropertyLoader.getProperty(PropertyLoader.exportdataThreadsProperty, "3"));
                break;
            }
        case SimDataOnly:
            {
                selector = vcMessagingService_int.createSelector(dataRequestFilter + " AND " + dataOnlyFilter);
                numThreads = Integer.parseInt(PropertyLoader.getProperty(PropertyLoader.simdataThreadsProperty, "5"));
                break;
            }
        default:
            {
                throw new RuntimeException("expecting either Service type of " + ServiceType.DATA + " or " + ServiceType.DATAEXPORT);
            }
    }
    this.sharedProducerSession = vcMessagingService_int.createProducerSession();
    rpcMessageHandler = new VCRpcMessageHandler(dataServerImpl, VCellQueue.DataRequestQueue);
    this.pooledQueueConsumer = new VCPooledQueueConsumer(rpcMessageHandler, numThreads, sharedProducerSession);
    this.pooledQueueConsumer.initThreadPool();
    rpcConsumer = new VCQueueConsumer(VCellQueue.DataRequestQueue, pooledQueueConsumer, selector, serviceType.getName() + " RPC Server Thread", MessageConstants.PREFETCH_LIMIT_DATA_REQUEST);
    vcMessagingService_int.addMessageConsumer(rpcConsumer);
}

Example 5 with VCPooledQueueConsumer

use of cbit.vcell.message.VCPooledQueueConsumer in project vcell by virtualcell.

the class TestRPC method main.

public static void main(String[] args) throws Exception {
    try {
        PropertyLoader.loadProperties();
        VCMessagingService messagingService = VCellServiceHelper.getInstance().loadService(VCMessagingService.class);
        String jmshost = PropertyLoader.getRequiredProperty(PropertyLoader.jmsIntHostInternal);
        int jmsport = Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.jmsIntPortInternal));
        messagingService.setConfiguration(new SimpleMessagingDelegate(), jmshost, jmsport);
        // reading message and computing sum
        // create N comsumers
        MyRpcServer myRpcServer = new MyRpcServer();
        VCRpcMessageHandler rpcMessageHandler = new VCRpcMessageHandler(myRpcServer, VCellTestQueue.JimQueue);
        VCPooledQueueConsumer pooledQueueConsumer = new VCPooledQueueConsumer(rpcMessageHandler, NUM_THREADS, messagingService.createProducerSession());
        pooledQueueConsumer.initThreadPool();
        VCQueueConsumer rpcConsumer = new VCQueueConsumer(VCellTestQueue.JimQueue, pooledQueueConsumer, null, "Queue[" + VCellTestQueue.JimQueue.getName() + "] ==== RPC Consumer Master Thread ", 1000);
        // VCRpcMessageHandler rpcMessageHandler = new VCRpcMessageHandler(myRpcServer, VCellQueue.JimQueue, log);
        // VCQueueConsumer rpcConsumer = new VCQueueConsumer(VCellQueue.JimQueue, rpcMessageHandler, null, "Queue["+VCellQueue.JimQueue.getName()+"] ==== RPC Consumer Master Thread ", 1000);
        messagingService.addMessageConsumer(rpcConsumer);
        for (int i = 0; i < NUM_MESSAGES; i++) {
            // creating one messageProducer session
            final VCMessageSession messageSession = messagingService.createProducerSession();
            class MyTask implements Runnable {

                int msgNum;

                MyTask(int msgNum) {
                    this.msgNum = msgNum;
                }

                public void run() {
                    try {
                        // 
                        // create simple RPC request for service "Testing_Service"
                        // 
                        User user = new User("schaff", new KeyValue("17"));
                        Integer n1 = new Integer(msgNum);
                        Integer n2 = new Integer(msgNum + 1);
                        VCRpcRequest rpcRequest = new VCRpcRequest(user, RpcServiceType.TESTING_SERVICE, "add", new Object[] { n1, n2 });
                        // 
                        // send request and block for response (or timeout).
                        // RPC invocations don't need commits.
                        // 
                        Object returnValue = messageSession.sendRpcMessage(VCellTestQueue.JimQueue, rpcRequest, true, 20000, null, null, null);
                        // 
                        if (returnValue instanceof Integer) {
                            System.out.println("add(" + n1 + "," + n2 + ") ===> " + returnValue);
                        } else {
                            System.out.println("unexpected return value of " + returnValue);
                        }
                    } catch (VCMessagingInvocationTargetException e) {
                        e.printStackTrace(System.out);
                        System.out.println("the rpc service threw an exception");
                        e.getTargetException().printStackTrace(System.out);
                    } catch (VCMessagingException e) {
                        e.printStackTrace();
                    }
                }
            }
            ;
            new Thread(new MyTask(i)).start();
        }
        Thread.sleep(2000);
        System.out.println("main program calling closeAll()");
        pooledQueueConsumer.shutdownAndAwaitTermination();
        messagingService.close();
        System.out.println("main program exiting");
    } catch (Exception e) {
        e.printStackTrace(System.out);
    }
}