Examples with WatermarkMessage org.apache.samza.system.WatermarkMessage used on opensource projects

Example 1 with WatermarkMessage

use of org.apache.samza.system.WatermarkMessage in project samza by apache.

the class StreamOperatorTask method processAsync.

/**
 * Passes the incoming message envelopes along to the {@link InputOperatorImpl} node
 * for the input {@link SystemStream}. It is non-blocking and dispatches the message to the container thread
 * pool. The thread pool size is configured through job.container.thread.pool.size. In the absence of the config,
 * the task executes the DAG on the run loop thread.
 * <p>
 * From then on, each {@link org.apache.samza.operators.impl.OperatorImpl} propagates its transformed output to
 * its chained {@link org.apache.samza.operators.impl.OperatorImpl}s itself.
 *
 * @param ime incoming message envelope to process
 * @param collector the collector to send messages with
 * @param coordinator the coordinator to request commits or shutdown
 * @param callback the task callback handle
 */
@Override
public final void processAsync(IncomingMessageEnvelope ime, MessageCollector collector, TaskCoordinator coordinator, TaskCallback callback) {
    Runnable processRunnable = () -> {
        try {
            SystemStream systemStream = ime.getSystemStreamPartition().getSystemStream();
            InputOperatorImpl inputOpImpl = operatorImplGraph.getInputOperator(systemStream);
            if (inputOpImpl != null) {
                CompletionStage<Void> processFuture;
                MessageType messageType = MessageType.of(ime.getMessage());
                switch(messageType) {
                    case USER_MESSAGE:
                        processFuture = inputOpImpl.onMessageAsync(ime, collector, coordinator);
                        break;
                    case END_OF_STREAM:
                        EndOfStreamMessage eosMessage = (EndOfStreamMessage) ime.getMessage();
                        processFuture = inputOpImpl.aggregateEndOfStream(eosMessage, ime.getSystemStreamPartition(), collector, coordinator);
                        break;
                    case WATERMARK:
                        WatermarkMessage watermarkMessage = (WatermarkMessage) ime.getMessage();
                        processFuture = inputOpImpl.aggregateWatermark(watermarkMessage, ime.getSystemStreamPartition(), collector, coordinator);
                        break;
                    default:
                        processFuture = failedFuture(new SamzaException("Unknown message type " + messageType + " encountered."));
                        break;
                }
                processFuture.whenComplete((val, ex) -> {
                    if (ex != null) {
                        callback.failure(ex);
                    } else {
                        callback.complete();
                    }
                });
            } else {
                // If InputOperator is not found in the operator graph for a given SystemStream, throw an exception else the
                // job will timeout due to async task callback timeout (TaskCallbackTimeoutException)
                final String errMessage = String.format("InputOperator not found in OperatorGraph for %s. The available input" + " operators are: %s. Please check SystemStream configuration for the `SystemConsumer` and/or task.inputs" + " task configuration.", systemStream, operatorImplGraph.getAllInputOperators());
                LOG.error(errMessage);
                callback.failure(new SamzaException(errMessage));
            }
        } catch (Exception e) {
            LOG.error("Failed to process the incoming message due to ", e);
            callback.failure(e);
        }
    };
    if (taskThreadPool != null) {
        LOG.debug("Processing message using thread pool.");
        taskThreadPool.submit(processRunnable);
    } else {
        LOG.debug("Processing message on the run loop thread.");
        processRunnable.run();
    }
}

Example 2 with WatermarkMessage

use of org.apache.samza.system.WatermarkMessage in project samza by apache.

the class TestIntermediateMessageSerde method testWatermarkMessageSerde.

@Test
public void testWatermarkMessageSerde() {
    IntermediateMessageSerde imserde = new IntermediateMessageSerde(new ObjectSerde());
    String taskName = "task-1";
    WatermarkMessage watermark = new WatermarkMessage(System.currentTimeMillis(), taskName);
    byte[] bytes = imserde.toBytes(watermark);
    WatermarkMessage de = (WatermarkMessage) imserde.fromBytes(bytes);
    assertEquals(MessageType.of(de), MessageType.WATERMARK);
    assertEquals(de.getTaskName(), taskName);
    assertTrue(de.getTimestamp() > 0);
}

Example 3 with WatermarkMessage

use of org.apache.samza.system.WatermarkMessage in project samza by apache.

the class IntermediateMessageSerde method toBytes.

@Override
public byte[] toBytes(Object object) {
    final byte[] data;
    final MessageType type = MessageType.of(object);
    switch(type) {
        case USER_MESSAGE:
            data = userMessageSerde.toBytes(object);
            break;
        case WATERMARK:
            data = watermarkSerde.toBytes((WatermarkMessage) object);
            break;
        case END_OF_STREAM:
            data = eosSerde.toBytes((EndOfStreamMessage) object);
            break;
        default:
            throw new SamzaException("Unknown message type: " + type.name());
    }
    final byte[] bytes = new byte[data.length + 1];
    bytes[0] = (byte) type.ordinal();
    System.arraycopy(data, 0, bytes, 1, data.length);
    return bytes;
}

Example 4 with WatermarkMessage

use of org.apache.samza.system.WatermarkMessage in project samza by apache.

the class TestWatermarkStates method testUpdate.

@Test
public void testUpdate() {
    SystemStream input = new SystemStream("system", "input");
    SystemStream intermediate = new SystemStream("system", "intermediate");
    Set<SystemStreamPartition> ssps = new HashSet<>();
    SystemStreamPartition inputPartition0 = new SystemStreamPartition(input, new Partition(0));
    SystemStreamPartition intPartition0 = new SystemStreamPartition(intermediate, new Partition(0));
    SystemStreamPartition intPartition1 = new SystemStreamPartition(intermediate, new Partition(1));
    ssps.add(inputPartition0);
    ssps.add(intPartition0);
    ssps.add(intPartition1);
    Map<SystemStream, Integer> producerCounts = new HashMap<>();
    producerCounts.put(intermediate, 2);
    // advance watermark on input to 5
    WatermarkStates watermarkStates = new WatermarkStates(ssps, producerCounts, new MetricsRegistryMap());
    IncomingMessageEnvelope envelope = IncomingMessageEnvelope.buildWatermarkEnvelope(inputPartition0, 5L);
    watermarkStates.update((WatermarkMessage) envelope.getMessage(), envelope.getSystemStreamPartition());
    assertEquals(watermarkStates.getWatermark(input), 5L);
    assertEquals(watermarkStates.getWatermark(intermediate), WATERMARK_NOT_EXIST);
    // watermark from task 0 on int p0 to 6
    WatermarkMessage watermarkMessage = new WatermarkMessage(6L, "task 0");
    watermarkStates.update(watermarkMessage, intPartition0);
    assertEquals(watermarkStates.getWatermarkPerSSP(intPartition0), WATERMARK_NOT_EXIST);
    assertEquals(watermarkStates.getWatermark(intermediate), WATERMARK_NOT_EXIST);
    // watermark from task 1 on int p0 to 3
    watermarkMessage = new WatermarkMessage(3L, "task 1");
    watermarkStates.update(watermarkMessage, intPartition0);
    assertEquals(watermarkStates.getWatermarkPerSSP(intPartition0), 3L);
    assertEquals(watermarkStates.getWatermark(intermediate), WATERMARK_NOT_EXIST);
    // watermark from task 0 on int p1 to 10
    watermarkMessage = new WatermarkMessage(10L, "task 0");
    watermarkStates.update(watermarkMessage, intPartition1);
    assertEquals(watermarkStates.getWatermarkPerSSP(intPartition1), WATERMARK_NOT_EXIST);
    assertEquals(watermarkStates.getWatermark(intermediate), WATERMARK_NOT_EXIST);
    // watermark from task 1 on int p1 to 4
    watermarkMessage = new WatermarkMessage(4L, "task 1");
    watermarkStates.update(watermarkMessage, intPartition1);
    assertEquals(watermarkStates.getWatermarkPerSSP(intPartition1), 4L);
    // verify we got a watermark 3 (min) for int stream
    assertEquals(watermarkStates.getWatermark(intermediate), 3L);
    // advance watermark from task 1 on int p0 to 8
    watermarkMessage = new WatermarkMessage(8L, "task 1");
    watermarkStates.update(watermarkMessage, intPartition0);
    assertEquals(watermarkStates.getWatermarkPerSSP(intPartition0), 6L);
    // verify we got a watermark 4 (min) for int stream
    assertEquals(watermarkStates.getWatermark(intermediate), 4L);
    // advance watermark from task 1 on int p1 to 7
    watermarkMessage = new WatermarkMessage(7L, "task 1");
    watermarkStates.update(watermarkMessage, intPartition1);
    assertEquals(watermarkStates.getWatermarkPerSSP(intPartition1), 7L);
    // verify we got a watermark 6 (min) for int stream
    assertEquals(watermarkStates.getWatermark(intermediate), 6L);
}

Example 5 with WatermarkMessage

use of org.apache.samza.system.WatermarkMessage in project beam by apache.

the class TranslationContext method createDummyStreamDescriptor.

/**
 * The dummy stream created will only be used in Beam tests.
 */
private static InputDescriptor<OpMessage<String>, ?> createDummyStreamDescriptor(String id) {
    final GenericSystemDescriptor dummySystem = new GenericSystemDescriptor(id, InMemorySystemFactory.class.getName());
    final GenericInputDescriptor<OpMessage<String>> dummyInput = dummySystem.getInputDescriptor(id, new NoOpSerde<>());
    dummyInput.withOffsetDefault(SystemStreamMetadata.OffsetType.OLDEST);
    final Config config = new MapConfig(dummyInput.toConfig(), dummySystem.toConfig());
    final SystemFactory factory = new InMemorySystemFactory();
    final StreamSpec dummyStreamSpec = new StreamSpec(id, id, id, 1);
    factory.getAdmin(id, config).createStream(dummyStreamSpec);
    final SystemProducer producer = factory.getProducer(id, config, null);
    final SystemStream sysStream = new SystemStream(id, id);
    final Consumer<Object> sendFn = (msg) -> {
        producer.send(id, new OutgoingMessageEnvelope(sysStream, 0, null, msg));
    };
    final WindowedValue<String> windowedValue = WindowedValue.timestampedValueInGlobalWindow("dummy", new Instant());
    sendFn.accept(OpMessage.ofElement(windowedValue));
    sendFn.accept(new WatermarkMessage(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()));
    sendFn.accept(new EndOfStreamMessage(null));
    return dummyInput;
}