Examples with IncomingMessageEnvelope org.apache.samza.system.IncomingMessageEnvelope used on opensource projects

Example 1 with IncomingMessageEnvelope

use of org.apache.samza.system.IncomingMessageEnvelope in project samza by apache.

the class AsyncRunLoop method chooseEnvelope.

/**
   * Chooses an envelope from messageChooser without updating it. This enables flow control
   * on the SSP level, meaning the task will not get further messages for the SSP if it cannot
   * process it. The chooser is updated only after the callback to process is invoked, then the task
   * is able to process more messages. This flow control does not block. so in case of empty message chooser,
   * it will return null immediately without blocking, and the chooser will not poll the underlying system
   * consumer since there are still messages in the SystemConsumers buffer.
   */
private IncomingMessageEnvelope chooseEnvelope() {
    IncomingMessageEnvelope envelope = consumerMultiplexer.choose(false);
    if (envelope != null) {
        log.trace("Choose envelope ssp {} offset {} for processing", envelope.getSystemStreamPartition(), envelope.getOffset());
        containerMetrics.envelopes().inc();
    } else {
        log.trace("No envelope is available");
        containerMetrics.nullEnvelopes().inc();
    }
    return envelope;
}

Example 2 with IncomingMessageEnvelope

use of org.apache.samza.system.IncomingMessageEnvelope in project samza by apache.

the class AsyncRunLoop method run.

/**
   * The run loop chooses messages from the SystemConsumers, and run the ready tasks asynchronously.
   * Window and commit are run in a thread pool, and they are mutual exclusive with task process.
   * The loop thread will block if all tasks are busy, and resume if any task finishes.
   */
@Override
public void run() {
    try {
        for (AsyncTaskWorker taskWorker : taskWorkers) {
            taskWorker.init();
        }
        long prevNs = clock.nanoTime();
        while (!shutdownNow) {
            if (throwable != null) {
                log.error("Caught throwable and stopping run loop", throwable);
                throw new SamzaException(throwable);
            }
            long startNs = clock.nanoTime();
            IncomingMessageEnvelope envelope = chooseEnvelope();
            long chooseNs = clock.nanoTime();
            containerMetrics.chooseNs().update(chooseNs - startNs);
            runTasks(envelope);
            long blockNs = clock.nanoTime();
            blockIfBusy(envelope);
            long currentNs = clock.nanoTime();
            long activeNs = blockNs - chooseNs;
            long totalNs = currentNs - prevNs;
            prevNs = currentNs;
            containerMetrics.blockNs().update(currentNs - blockNs);
            if (totalNs != 0) {
                // totalNs is not 0 if timer metrics are enabled
                containerMetrics.utilization().set(((double) activeNs) / totalNs);
            }
        }
    } finally {
        workerTimer.shutdown();
        callbackExecutor.shutdown();
        if (callbackTimer != null)
            callbackTimer.shutdown();
    }
}

Example 3 with IncomingMessageEnvelope

use of org.apache.samza.system.IncomingMessageEnvelope in project samza by apache.

the class CoordinatorStreamSystemConsumer method bootstrap.

/**
   * Read all messages from the earliest offset, all the way to the latest.
   * Currently, this method only pays attention to config messages.
   */
public void bootstrap() {
    synchronized (bootstrapLock) {
        // Make a copy so readers aren't affected while we modify the set.
        final LinkedHashSet<CoordinatorStreamMessage> bootstrappedMessages = new LinkedHashSet<>(bootstrappedStreamSet);
        log.info("Bootstrapping configuration from coordinator stream.");
        SystemStreamPartitionIterator iterator = new SystemStreamPartitionIterator(systemConsumer, coordinatorSystemStreamPartition);
        try {
            while (iterator.hasNext()) {
                IncomingMessageEnvelope envelope = iterator.next();
                Object[] keyArray = keySerde.fromBytes((byte[]) envelope.getKey()).toArray();
                Map<String, Object> valueMap = null;
                if (envelope.getMessage() != null) {
                    valueMap = messageSerde.fromBytes((byte[]) envelope.getMessage());
                }
                CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, valueMap);
                log.debug("Received coordinator stream message: {}", coordinatorStreamMessage);
                // Remove any existing entry. Set.add() does not add if the element already exists.
                if (bootstrappedMessages.remove(coordinatorStreamMessage)) {
                    log.debug("Removed duplicate message: {}", coordinatorStreamMessage);
                }
                bootstrappedMessages.add(coordinatorStreamMessage);
                if (SetConfig.TYPE.equals(coordinatorStreamMessage.getType())) {
                    String configKey = coordinatorStreamMessage.getKey();
                    if (coordinatorStreamMessage.isDelete()) {
                        configMap.remove(configKey);
                    } else {
                        String configValue = new SetConfig(coordinatorStreamMessage).getConfigValue();
                        configMap.put(configKey, configValue);
                    }
                }
            }
            bootstrappedStreamSet = Collections.unmodifiableSet(bootstrappedMessages);
            log.debug("Bootstrapped configuration: {}", configMap);
            isBootstrapped = true;
        } catch (Exception e) {
            throw new SamzaException(e);
        }
    }
}

Example 4 with IncomingMessageEnvelope

use of org.apache.samza.system.IncomingMessageEnvelope in project samza by apache.

the class TestAsyncRunLoop method testCommitBehaviourWhenAsyncCommitIsEnabled.

//@Test
public void testCommitBehaviourWhenAsyncCommitIsEnabled() throws InterruptedException {
    int maxMessagesInFlight = 3;
    TestTask task0 = new TestTask(true, true, false, null, maxMessagesInFlight);
    task0.setCommitRequest(TaskCoordinator.RequestScope.CURRENT_TASK);
    TestTask task1 = new TestTask(true, false, true, null);
    task1.setCommitRequest(TaskCoordinator.RequestScope.CURRENT_TASK);
    IncomingMessageEnvelope firstMsg = new IncomingMessageEnvelope(ssp0, "0", "key0", "value0");
    IncomingMessageEnvelope secondMsg = new IncomingMessageEnvelope(ssp0, "1", "key1", "value1");
    IncomingMessageEnvelope thirdMsg = new IncomingMessageEnvelope(ssp0, "2", "key0", "value0");
    final CountDownLatch firstMsgCompletionLatch = new CountDownLatch(1);
    final CountDownLatch secondMsgCompletionLatch = new CountDownLatch(1);
    task0.callbackHandler = callback -> {
        IncomingMessageEnvelope envelope = ((TaskCallbackImpl) callback).envelope;
        try {
            if (envelope.equals(firstMsg)) {
                firstMsgCompletionLatch.await();
            } else if (envelope.equals(secondMsg)) {
                firstMsgCompletionLatch.countDown();
                secondMsgCompletionLatch.await();
            } else if (envelope.equals(thirdMsg)) {
                secondMsgCompletionLatch.countDown();
                verify(offsetManager).update(taskName0, firstMsg.getSystemStreamPartition(), firstMsg.getOffset());
                verify(offsetManager, atLeastOnce()).checkpoint(taskName0);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    };
    Map<TaskName, TaskInstance> tasks = new HashMap<>();
    tasks.put(taskName0, createTaskInstance(task0, taskName0, ssp0));
    tasks.put(taskName1, createTaskInstance(task1, taskName1, ssp1));
    when(consumerMultiplexer.choose(false)).thenReturn(firstMsg).thenReturn(secondMsg).thenReturn(thirdMsg).thenReturn(envelope1).thenReturn(null);
    AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, containerMetrics, () -> 0L, true);
    runLoop.run();
    firstMsgCompletionLatch.await();
    secondMsgCompletionLatch.await();
    assertEquals(3, task0.processed);
    assertEquals(3, task0.committed);
    assertEquals(1, task1.processed);
    assertEquals(0, task1.committed);
}

Example 5 with IncomingMessageEnvelope

use of org.apache.samza.system.IncomingMessageEnvelope in project samza by apache.

the class TestAsyncRunLoop method buildOutofOrderCallback.

private TestCode buildOutofOrderCallback(final TestTask task) {
    final CountDownLatch latch = new CountDownLatch(1);
    return new TestCode() {

        @Override
        public void run(TaskCallback callback) {
            IncomingMessageEnvelope envelope = ((TaskCallbackImpl) callback).envelope;
            if (envelope == envelope0) {
                // process first message will wait till the second one is processed
                try {
                    latch.await();
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            } else {
                // second envelope complete first
                assertEquals(0, task.completed.get());
                latch.countDown();
            }
        }
    };
}