Examples with ProcessQueue pipelite.runner.process.ProcessQueue used on opensource projects

Example 1 with ProcessQueue

use of pipelite.runner.process.ProcessQueue in project pipelite by enasequence.

the class PipelineRunnerProcessQueueTest method test.

private void test(Pipeline pipeline) {
    ProcessEntityCreator processEntityCreator = new ProcessEntityCreator(pipeline, pipeliteServices.process());
    ProcessQueue processQueue = spy(new ProcessQueue(pipeliteConfiguration, pipeliteServices, pipeline));
    // Queue should be filled
    assertThat(processQueue.isRefreshQueue()).isTrue();
    // Queue should be empty
    assertThat(processQueue.getProcessQueueSize()).isZero();
    boolean lockProcess = true;
    PipelineRunner pipelineRunner = PipelineRunnerFactory.create(pipeliteConfiguration, pipeliteServices, pipeliteMetrics, pipeline, processEntityCreator, (pipeline1) -> processQueue, (pipelineName1, process1) -> new ProcessRunner(pipeliteConfiguration, pipeliteServices, pipeliteMetrics, pipelineName1, process1, lockProcess));
    // Wait for the processes to execute
    while (!pipelineRunner.isIdle()) {
        Time.wait(Duration.ofSeconds(1));
        pipelineRunner.runOneIteration();
    }
    // Refresh is called two times. Once before creating processes when
    // the process queue is created empty and immediately after
    // creating processing.
    verify(processQueue, times(2)).refreshQueue();
    assertThat(pipeliteMetrics.pipeline(pipeline.pipelineName()).stage().getSuccessCount()).isEqualTo(PROCESS_CNT);
    assertThat(processQueue.isRefreshQueue()).isFalse();
    assertThat(processQueue.getProcessQueueSize()).isZero();
}

Example 2 with ProcessQueue

use of pipelite.runner.process.ProcessQueue in project pipelite by enasequence.

the class PipelineRunner method refreshQueue.

private void refreshQueue() {
    if (processQueue.get() == null || processQueue.get().isRefreshQueue()) {
        if (activeRefreshQueue.compareAndSet(false, true)) {
            pipeliteServices.executor().refreshQueue().execute(() -> {
                internalErrorHandler.execute(() -> {
                    ProcessQueue p = processQueue.get();
                    boolean firstRefresh = p == null;
                    if (firstRefresh) {
                        // Create process queue.
                        // The process queue creation is intentionally deferred to
                        // happen here so that we will only assign the process queue
                        // after it has been refreshed once and processes have been
                        // created if needed.
                        p = processQueueFactory.create(pipeline);
                    }
                    // Refresh process queue.
                    p.refreshQueue();
                    if (p.getProcessQueueSize() == 0) {
                        // The process queue is empty.
                        // Create new process entities.
                        int createdProcessCount = processEntityCreator.create(p.getProcessQueueMaxSize());
                        replenishTime = ZonedDateTime.now();
                        if (createdProcessCount > 0) {
                            // Refresh process queue.
                            p.refreshQueue();
                        }
                    }
                    if (firstRefresh) {
                        processQueue.set(p);
                    }
                });
                activeRefreshQueue.set(false);
            });
        }
    }
}

Example 3 with ProcessQueue

use of pipelite.runner.process.ProcessQueue in project pipelite by enasequence.

the class PipelineRunnerFactory method create.

public static PipelineRunner create(PipeliteConfiguration pipeliteConfiguration, PipeliteServices pipeliteServices, PipeliteMetrics pipeliteMetrics, String pipelineName) {
    // Get registered pipeline.
    RegisteredPipelineService registeredPipelineService = pipeliteServices.registeredPipeline();
    Pipeline pipeline = registeredPipelineService.getRegisteredPipeline(pipelineName, Pipeline.class);
    if (pipeline == null) {
        throw new PipeliteException("Missing pipeline: " + pipelineName);
    }
    // Get process creator.
    ProcessEntityCreator processEntityCreator = new ProcessEntityCreator(pipeline, pipeliteServices.process());
    ProcessQueueFactory processQueueFactory = (pipeline1) -> new ProcessQueue(pipeliteConfiguration, pipeliteServices, pipeline1);
    boolean lockProcess = true;
    ProcessRunnerFactory processRunnerFactory = (pipelineName1, process1) -> new ProcessRunner(pipeliteConfiguration, pipeliteServices, pipeliteMetrics, pipelineName1, process1, lockProcess);
    return create(pipeliteConfiguration, pipeliteServices, pipeliteMetrics, pipeline, processEntityCreator, processQueueFactory, processRunnerFactory);
}