Examples with MessageType org.apache.samza.system.MessageType used on opensource projects

Example 1 with MessageType

use of org.apache.samza.system.MessageType in project samza by apache.

the class StreamOperatorTask method processAsync.

/**
 * Passes the incoming message envelopes along to the {@link InputOperatorImpl} node
 * for the input {@link SystemStream}. It is non-blocking and dispatches the message to the container thread
 * pool. The thread pool size is configured through job.container.thread.pool.size. In the absence of the config,
 * the task executes the DAG on the run loop thread.
 * <p>
 * From then on, each {@link org.apache.samza.operators.impl.OperatorImpl} propagates its transformed output to
 * its chained {@link org.apache.samza.operators.impl.OperatorImpl}s itself.
 *
 * @param ime incoming message envelope to process
 * @param collector the collector to send messages with
 * @param coordinator the coordinator to request commits or shutdown
 * @param callback the task callback handle
 */
@Override
public final void processAsync(IncomingMessageEnvelope ime, MessageCollector collector, TaskCoordinator coordinator, TaskCallback callback) {
    Runnable processRunnable = () -> {
        try {
            SystemStream systemStream = ime.getSystemStreamPartition().getSystemStream();
            InputOperatorImpl inputOpImpl = operatorImplGraph.getInputOperator(systemStream);
            if (inputOpImpl != null) {
                CompletionStage<Void> processFuture;
                MessageType messageType = MessageType.of(ime.getMessage());
                switch(messageType) {
                    case USER_MESSAGE:
                        processFuture = inputOpImpl.onMessageAsync(ime, collector, coordinator);
                        break;
                    case END_OF_STREAM:
                        EndOfStreamMessage eosMessage = (EndOfStreamMessage) ime.getMessage();
                        processFuture = inputOpImpl.aggregateEndOfStream(eosMessage, ime.getSystemStreamPartition(), collector, coordinator);
                        break;
                    case WATERMARK:
                        WatermarkMessage watermarkMessage = (WatermarkMessage) ime.getMessage();
                        processFuture = inputOpImpl.aggregateWatermark(watermarkMessage, ime.getSystemStreamPartition(), collector, coordinator);
                        break;
                    default:
                        processFuture = failedFuture(new SamzaException("Unknown message type " + messageType + " encountered."));
                        break;
                }
                processFuture.whenComplete((val, ex) -> {
                    if (ex != null) {
                        callback.failure(ex);
                    } else {
                        callback.complete();
                    }
                });
            } else {
                // If InputOperator is not found in the operator graph for a given SystemStream, throw an exception else the
                // job will timeout due to async task callback timeout (TaskCallbackTimeoutException)
                final String errMessage = String.format("InputOperator not found in OperatorGraph for %s. The available input" + " operators are: %s. Please check SystemStream configuration for the `SystemConsumer` and/or task.inputs" + " task configuration.", systemStream, operatorImplGraph.getAllInputOperators());
                LOG.error(errMessage);
                callback.failure(new SamzaException(errMessage));
            }
        } catch (Exception e) {
            LOG.error("Failed to process the incoming message due to ", e);
            callback.failure(e);
        }
    };
    if (taskThreadPool != null) {
        LOG.debug("Processing message using thread pool.");
        taskThreadPool.submit(processRunnable);
    } else {
        LOG.debug("Processing message on the run loop thread.");
        processRunnable.run();
    }
}

Example 2 with MessageType

use of org.apache.samza.system.MessageType in project samza by apache.

the class RunLoop method getWorkersForEnvelope.

/**
 * when elasticity is not enabled, fetch the workers from sspToTaskWorkerMapping using envelope.getSSP()
 * when elasticity is enabled,
 *       sspToTaskWorkerMapping has workers for a SSP which has keyBucket
 *       hence need to use envelop.getSSP(elasticityFactor)
 *       Additionally, when envelope is EnofStream or Watermark, it needs to be sent to all works for the ssp irrespective of keyBucket
 * @param envelope
 * @return list of workers for the envelope
 */
private List<AsyncTaskWorker> getWorkersForEnvelope(IncomingMessageEnvelope envelope) {
    if (elasticityFactor <= 1) {
        return sspToTaskWorkerMapping.get(envelope.getSystemStreamPartition());
    }
    final SystemStreamPartition sspOfEnvelope = envelope.getSystemStreamPartition(elasticityFactor);
    List<AsyncTaskWorker> listOfWorkersForEnvelope = null;
    // if envelope is end of stream or watermark, it needs to be routed to all tasks consuming the ssp irresp of keybucket
    MessageType messageType = MessageType.of(envelope.getMessage());
    if (envelope.isEndOfStream() || MessageType.END_OF_STREAM == messageType || MessageType.WATERMARK == messageType) {
        // sspToTaskWorkerMapping has ssps with keybucket so extract and check only system, stream and partition and ignore the keybucket
        listOfWorkersForEnvelope = sspToTaskWorkerMapping.entrySet().stream().filter(sspToTask -> sspToTask.getKey().getSystemStream().equals(sspOfEnvelope.getSystemStream()) && sspToTask.getKey().getPartition().equals(sspOfEnvelope.getPartition())).map(sspToWorker -> sspToWorker.getValue()).flatMap(Collection::stream).collect(Collectors.toList());
    } else {
        listOfWorkersForEnvelope = sspToTaskWorkerMapping.get(sspOfEnvelope);
    }
    return listOfWorkersForEnvelope;
}

Example 3 with MessageType

use of org.apache.samza.system.MessageType in project samza by apache.

the class IntermediateMessageSerde method toBytes.

@Override
public byte[] toBytes(Object object) {
    final byte[] data;
    final MessageType type = MessageType.of(object);
    switch(type) {
        case USER_MESSAGE:
            data = userMessageSerde.toBytes(object);
            break;
        case WATERMARK:
            data = watermarkSerde.toBytes((WatermarkMessage) object);
            break;
        case END_OF_STREAM:
            data = eosSerde.toBytes((EndOfStreamMessage) object);
            break;
        default:
            throw new SamzaException("Unknown message type: " + type.name());
    }
    final byte[] bytes = new byte[data.length + 1];
    bytes[0] = (byte) type.ordinal();
    System.arraycopy(data, 0, bytes, 1, data.length);
    return bytes;
}

Example 4 with MessageType

use of org.apache.samza.system.MessageType in project samza by apache.

the class IntermediateMessageSerde method fromBytes.

@Override
public Object fromBytes(byte[] bytes) {
    try {
        final Object object;
        final MessageType type;
        try {
            type = MessageType.values()[bytes[0]];
        } catch (ArrayIndexOutOfBoundsException e) {
            // has reached retention time.
            throw new SamzaException("Error reading the message type from intermediate message. This may happen if you " + "have recently upgraded from samza version older than 0.13.1 or there are still old messages in the " + "intermediate stream.", e);
        }
        final byte[] data = Arrays.copyOfRange(bytes, 1, bytes.length);
        switch(type) {
            case USER_MESSAGE:
                object = userMessageSerde.fromBytes(data);
                break;
            case WATERMARK:
                object = watermarkSerde.fromBytes(data);
                break;
            case END_OF_STREAM:
                object = eosSerde.fromBytes(data);
                break;
            default:
                throw new UnsupportedOperationException(String.format("Message type %s is not supported", type.name()));
        }
        return object;
    } catch (UnsupportedOperationException ue) {
        throw new SamzaException(ue);
    } catch (Exception e) {
        throw e;
    }
}