Examples with WatermarkFunction org.apache.samza.operators.functions.WatermarkFunction used on opensource projects

Example 1 with WatermarkFunction

use of org.apache.samza.operators.functions.WatermarkFunction in project samza by apache.

the class OperatorImpl method onMessageAsync.

public final CompletionStage<Void> onMessageAsync(M message, MessageCollector collector, TaskCoordinator coordinator) {
    this.numMessage.inc();
    long startNs = this.highResClock.nanoTime();
    CompletionStage<Collection<RM>> completableResultsFuture;
    try {
        completableResultsFuture = handleMessageAsync(message, collector, coordinator);
    } catch (ClassCastException e) {
        String actualType = e.getMessage().replaceFirst(" cannot be cast to .*", "");
        String expectedType = e.getMessage().replaceFirst(".* cannot be cast to ", "");
        throw new SamzaException(String.format("Error applying operator %s (created at %s) to its input message. " + "Expected input message to be of type %s, but found it to be of type %s. " + "Are Serdes for the inputs to this operator configured correctly?", getOpImplId(), getOperatorSpec().getSourceLocation(), expectedType, actualType), e);
    }
    CompletionStage<Void> result = completableResultsFuture.thenCompose(results -> {
        long endNs = this.highResClock.nanoTime();
        this.handleMessageNs.update(endNs - startNs);
        return CompletableFuture.allOf(results.stream().flatMap(r -> this.registeredOperators.stream().map(op -> op.onMessageAsync(r, collector, coordinator))).toArray(CompletableFuture[]::new));
    });
    WatermarkFunction watermarkFn = getOperatorSpec().getWatermarkFn();
    if (watermarkFn != null) {
        // check whether there is new watermark emitted from the user function
        Long outputWm = watermarkFn.getOutputWatermark();
        return result.thenCompose(ignored -> propagateWatermark(outputWm, collector, coordinator));
    }
    return result;
}

Example 2 with WatermarkFunction

use of org.apache.samza.operators.functions.WatermarkFunction in project samza by apache.

the class OperatorImpl method onWatermark.

/**
 * A watermark comes from an upstream operator. This function decides whether we should update the
 * input watermark based on the watermark time of all the previous operators, and then call handleWatermark()
 * to let the inherited operator to act on it.
 * @param watermark incoming watermark from an upstream operator
 * @param collector message collector
 * @param coordinator task coordinator
 */
private CompletionStage<Void> onWatermark(long watermark, MessageCollector collector, TaskCoordinator coordinator) {
    final long inputWatermarkMin;
    if (prevOperators.isEmpty()) {
        // for input operator, use the watermark time coming from the source input
        inputWatermarkMin = watermark;
    } else {
        // InputWatermark(op) = min { OutputWatermark(op') | op' is upstream of op}
        inputWatermarkMin = prevOperators.stream().map(op -> op.getOutputWatermark()).min(Long::compare).get();
    }
    CompletionStage<Void> watermarkFuture = CompletableFuture.completedFuture(null);
    if (currentWatermark < inputWatermarkMin) {
        // advance the watermark time of this operator
        currentWatermark = inputWatermarkMin;
        LOG.trace("Advance input watermark to {} in operator {}", currentWatermark, getOpImplId());
        final Long outputWm;
        final Collection<RM> output;
        final WatermarkFunction watermarkFn = getOperatorSpec().getWatermarkFn();
        if (watermarkFn != null) {
            // user-overrided watermark handling here
            output = (Collection<RM>) watermarkFn.processWatermark(currentWatermark);
            outputWm = watermarkFn.getOutputWatermark();
        } else {
            // use samza-provided watermark handling
            // default is to propagate the input watermark
            output = handleWatermark(currentWatermark, collector, coordinator);
            outputWm = currentWatermark;
        }
        if (!output.isEmpty()) {
            watermarkFuture = CompletableFuture.allOf(output.stream().flatMap(rm -> this.registeredOperators.stream().map(op -> op.onMessageAsync(rm, collector, coordinator))).toArray(CompletableFuture[]::new));
        }
        watermarkFuture = watermarkFuture.thenCompose(res -> propagateWatermark(outputWm, collector, coordinator));
    }
    return watermarkFuture;
}