Examples with SystemStream org.apache.samza.system.SystemStream used on opensource projects

Example 16 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class StreamPartitionCountMonitor method updatePartitionCountMetric.

/**
   * Fetches the current partition count for each system stream from the cache, compares the current count to the
   * original count and updates the metric for that system stream with the delta.
   */
void updatePartitionCountMetric() {
    try {
        Map<SystemStream, SystemStreamMetadata> currentMetadata = getMetadata(streamsToMonitor, metadataCache);
        for (Map.Entry<SystemStream, SystemStreamMetadata> metadataEntry : initialMetadata.entrySet()) {
            SystemStream systemStream = metadataEntry.getKey();
            SystemStreamMetadata metadata = metadataEntry.getValue();
            int currentPartitionCount = currentMetadata.get(systemStream).getSystemStreamPartitionMetadata().keySet().size();
            int prevPartitionCount = metadata.getSystemStreamPartitionMetadata().keySet().size();
            Gauge gauge = gauges.get(systemStream);
            gauge.set(currentPartitionCount - prevPartitionCount);
        }
    } catch (Exception e) {
        log.error("Exception while updating partition count metric.", e);
    }
}

Example 17 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class ExecutionPlanner method updateExistingPartitions.

/**
   * Fetch the partitions of source/sink streams and update the StreamEdges.
   * @param jobGraph {@link JobGraph}
   * @param streamManager the {@StreamManager} to interface with the streams.
   */
/* package private */
static void updateExistingPartitions(JobGraph jobGraph, StreamManager streamManager) {
    Set<StreamEdge> existingStreams = new HashSet<>();
    existingStreams.addAll(jobGraph.getSources());
    existingStreams.addAll(jobGraph.getSinks());
    Multimap<String, StreamEdge> systemToStreamEdges = HashMultimap.create();
    // group the StreamEdge(s) based on the system name
    existingStreams.forEach(streamEdge -> {
        SystemStream systemStream = streamEdge.getSystemStream();
        systemToStreamEdges.put(systemStream.getSystem(), streamEdge);
    });
    for (Map.Entry<String, Collection<StreamEdge>> entry : systemToStreamEdges.asMap().entrySet()) {
        String systemName = entry.getKey();
        Collection<StreamEdge> streamEdges = entry.getValue();
        Map<String, StreamEdge> streamToStreamEdge = new HashMap<>();
        // create the stream name to StreamEdge mapping for this system
        streamEdges.forEach(streamEdge -> streamToStreamEdge.put(streamEdge.getSystemStream().getStream(), streamEdge));
        // retrieve the partition counts for the streams in this system
        Map<String, Integer> streamToPartitionCount = streamManager.getStreamPartitionCounts(systemName, streamToStreamEdge.keySet());
        // set the partitions of a stream to its StreamEdge
        streamToPartitionCount.forEach((stream, partitionCount) -> {
            streamToStreamEdge.get(stream).setPartitionCount(partitionCount);
            log.debug("Partition count is {} for stream {}", partitionCount, stream);
        });
    }
}

Example 18 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class StreamOperatorTask method process.

/**
   * Passes the incoming message envelopes along to the {@link org.apache.samza.operators.impl.RootOperatorImpl} node
   * for the input {@link SystemStream}.
   * <p>
   * From then on, each {@link org.apache.samza.operators.impl.OperatorImpl} propagates its transformed output to
   * its chained {@link org.apache.samza.operators.impl.OperatorImpl}s itself.
   *
   * @param ime incoming message envelope to process
   * @param collector the collector to send messages with
   * @param coordinator the coordinator to request commits or shutdown
   */
@Override
public final void process(IncomingMessageEnvelope ime, MessageCollector collector, TaskCoordinator coordinator) {
    SystemStream systemStream = ime.getSystemStreamPartition().getSystemStream();
    InputStreamInternal inputStream = inputSystemStreamToInputStream.get(systemStream);
    RootOperatorImpl rootOperatorImpl = operatorImplGraph.getRootOperator(systemStream);
    if (rootOperatorImpl != null) {
        // TODO: SAMZA-1148 - Cast to appropriate input (key, msg) types based on the serde
        // before applying the msgBuilder.
        Object message = inputStream.getMsgBuilder().apply(ime.getKey(), ime.getMessage());
        rootOperatorImpl.onMessage(message, collector, coordinator);
    }
}

Example 19 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class OperatorImplGraph method init.

/**
   * Initialize the DAG of {@link OperatorImpl}s for the input {@link MessageStreamImpl} in the provided
   * {@link StreamGraphImpl}.
   *
   * @param streamGraph  the logical {@link StreamGraphImpl}
   * @param config  the {@link Config} required to instantiate operators
   * @param context  the {@link TaskContext} required to instantiate operators
   */
public void init(StreamGraphImpl streamGraph, Config config, TaskContext context) {
    streamGraph.getInputStreams().forEach((streamSpec, inputStream) -> {
        SystemStream systemStream = new SystemStream(streamSpec.getSystemName(), streamSpec.getPhysicalName());
        this.rootOperators.put(systemStream, this.createOperatorImpls((MessageStreamImpl) inputStream, config, context));
    });
}

Example 20 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class TaskConfigJava method getBroadcastSystemStreams.

/**
   * Get the SystemStreams for the configured broadcast streams.
   *
   * @return the set of SystemStreams for which there are broadcast stream SSPs configured.
   */
public Set<SystemStream> getBroadcastSystemStreams() {
    Set<SystemStream> broadcastSS = new HashSet<>();
    Set<SystemStreamPartition> broadcastSSPs = getBroadcastSystemStreamPartitions();
    for (SystemStreamPartition bssp : broadcastSSPs) {
        broadcastSS.add(bssp.getSystemStream());
    }
    return Collections.unmodifiableSet(broadcastSS);
}