use of org.apache.samza.operators.spec.InputOperatorSpec in project samza by apache.
the class TestInputOperatorImpl method testWithKeyedInput.
@Test
public void testWithKeyedInput() {
InputOperatorImpl inputOperator = new InputOperatorImpl(new InputOperatorSpec("stream-id", null, null, null, true, "input-op-id"));
IncomingMessageEnvelope ime = new IncomingMessageEnvelope(mock(SystemStreamPartition.class), "123", "key", "msg");
Collection<Object> results = inputOperator.handleMessage(ime, mock(MessageCollector.class), mock(TaskCoordinator.class));
Object result = results.iterator().next();
assertEquals("key", ((KV) result).getKey());
assertEquals("msg", ((KV) result).getValue());
}
use of org.apache.samza.operators.spec.InputOperatorSpec in project samza by apache.
the class TestOperatorSpecGraph method setUp.
@Before
public void setUp() {
this.mockAppDesc = mock(StreamApplicationDescriptorImpl.class);
/**
* Setup two linear transformation pipelines:
* 1) input1 --> filter --> sendTo
* 2) input2 --> map --> sink
*/
String inputStreamId1 = "test-input-1";
String outputStreamId = "test-output-1";
InputOperatorSpec testInput = new InputOperatorSpec(inputStreamId1, new NoOpSerde(), new NoOpSerde(), null, true, inputStreamId1);
StreamOperatorSpec filterOp = OperatorSpecs.createFilterOperatorSpec(m -> true, "test-filter-2");
OutputStreamImpl outputStream1 = new OutputStreamImpl(outputStreamId, null, null, true);
OutputOperatorSpec outputSpec = OperatorSpecs.createSendToOperatorSpec(outputStream1, "test-output-3");
testInput.registerNextOperatorSpec(filterOp);
filterOp.registerNextOperatorSpec(outputSpec);
String streamId2 = "test-input-2";
InputOperatorSpec testInput2 = new InputOperatorSpec(streamId2, new NoOpSerde(), new NoOpSerde(), null, true, "test-input-4");
StreamOperatorSpec testMap = OperatorSpecs.createMapOperatorSpec(m -> m, "test-map-5");
SinkOperatorSpec testSink = OperatorSpecs.createSinkOperatorSpec((m, mc, tc) -> {
}, "test-sink-6");
testInput2.registerNextOperatorSpec(testMap);
testMap.registerNextOperatorSpec(testSink);
this.inputOpSpecMap = new LinkedHashMap<>();
inputOpSpecMap.put(inputStreamId1, testInput);
inputOpSpecMap.put(streamId2, testInput2);
this.outputStrmMap = new LinkedHashMap<>();
outputStrmMap.put(outputStreamId, outputStream1);
when(mockAppDesc.getInputOperators()).thenReturn(Collections.unmodifiableMap(inputOpSpecMap));
when(mockAppDesc.getOutputStreams()).thenReturn(Collections.unmodifiableMap(outputStrmMap));
this.allOpSpecs = new HashSet<OperatorSpec>() {
{
this.add(testInput);
this.add(filterOp);
this.add(outputSpec);
this.add(testInput2);
this.add(testMap);
this.add(testSink);
}
};
}
use of org.apache.samza.operators.spec.InputOperatorSpec in project samza by apache.
the class TestStreamApplicationDescriptorImpl method testGetInputStreamWithExpandingSystem.
@Test
public void testGetInputStreamWithExpandingSystem() {
String streamId = "test-stream-1";
String expandedStreamId = "expanded-stream";
AtomicInteger expandCallCount = new AtomicInteger();
StreamExpander expander = (sg, isd) -> {
expandCallCount.incrementAndGet();
InputDescriptor expandedISD = new GenericSystemDescriptor("expanded-system", "mockFactoryClass").getInputDescriptor(expandedStreamId, new IntegerSerde());
return sg.getInputStream(expandedISD);
};
MockExpandingSystemDescriptor sd = new MockExpandingSystemDescriptor("mock-system", expander);
MockInputDescriptor isd = sd.getInputDescriptor(streamId, new IntegerSerde());
StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
appDesc.getInputStream(isd);
}, getConfig());
InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(expandedStreamId);
assertEquals(OpCode.INPUT, inputOpSpec.getOpCode());
assertEquals(1, expandCallCount.get());
assertFalse(streamAppDesc.getInputOperators().containsKey(streamId));
assertFalse(streamAppDesc.getInputDescriptors().containsKey(streamId));
assertTrue(streamAppDesc.getInputDescriptors().containsKey(expandedStreamId));
assertEquals(expandedStreamId, inputOpSpec.getStreamId());
}
use of org.apache.samza.operators.spec.InputOperatorSpec in project samza by apache.
the class StreamApplicationDescriptorImpl method getInputStream.
@Override
public <M> MessageStream<M> getInputStream(InputDescriptor<M, ?> inputDescriptor) {
SystemDescriptor systemDescriptor = inputDescriptor.getSystemDescriptor();
Optional<StreamExpander> expander = systemDescriptor.getExpander();
if (expander.isPresent()) {
return expander.get().apply(this, inputDescriptor);
}
// TODO: SAMZA-1841: need to add to the broadcast streams if inputDescriptor is for a broadcast stream
addInputDescriptor(inputDescriptor);
String streamId = inputDescriptor.getStreamId();
Serde serde = inputDescriptor.getSerde();
KV<Serde, Serde> kvSerdes = getOrCreateStreamSerdes(streamId, serde);
boolean isKeyed = serde instanceof KVSerde;
InputTransformer transformer = inputDescriptor.getTransformer().orElse(null);
InputOperatorSpec inputOperatorSpec = OperatorSpecs.createInputOperatorSpec(streamId, kvSerdes.getKey(), kvSerdes.getValue(), transformer, isKeyed, this.getNextOpId(OpCode.INPUT, null));
inputOperators.put(streamId, inputOperatorSpec);
return new MessageStreamImpl(this, inputOperators.get(streamId));
}
use of org.apache.samza.operators.spec.InputOperatorSpec in project samza by apache.
the class StreamApplicationDescriptorImpl method getIntermediateStream.
/**
* Internal helper for {@link MessageStreamImpl} to add an intermediate {@link MessageStream} to the graph.
* An intermediate {@link MessageStream} is both an output and an input stream.
*
* @param streamId the id of the stream to be created.
* @param serde the {@link Serde} to use for the message in the intermediate stream. If null, the default serde
* is used.
* @param isBroadcast whether the stream is a broadcast stream.
* @param <M> the type of messages in the intermediate {@link MessageStream}
* @return the intermediate {@link MessageStreamImpl}
*/
@VisibleForTesting
public <M> IntermediateMessageStreamImpl<M> getIntermediateStream(String streamId, Serde<M> serde, boolean isBroadcast) {
Preconditions.checkNotNull(serde, "serde must not be null for intermediate stream: " + streamId);
Preconditions.checkState(!inputOperators.containsKey(streamId) && !outputStreams.containsKey(streamId), "getIntermediateStream must not be called multiple times with the same streamId: " + streamId);
if (isBroadcast) {
intermediateBroadcastStreamIds.add(streamId);
}
boolean isKeyed = serde instanceof KVSerde;
KV<Serde, Serde> kvSerdes = getOrCreateStreamSerdes(streamId, serde);
InputTransformer transformer = (InputTransformer) getDefaultSystemDescriptor().flatMap(SystemDescriptor::getTransformer).orElse(null);
InputOperatorSpec inputOperatorSpec = OperatorSpecs.createInputOperatorSpec(streamId, kvSerdes.getKey(), kvSerdes.getValue(), transformer, isKeyed, this.getNextOpId(OpCode.INPUT, null));
inputOperators.put(streamId, inputOperatorSpec);
outputStreams.put(streamId, new OutputStreamImpl(streamId, kvSerdes.getKey(), kvSerdes.getValue(), isKeyed));
return new IntermediateMessageStreamImpl<>(this, inputOperators.get(streamId), outputStreams.get(streamId));
}
Aggregations