Search in sources :

Example 16 with InputOperatorSpec

use of org.apache.samza.operators.spec.InputOperatorSpec in project samza by apache.

the class TestInputOperatorImpl method testWithKeyedInput.

@Test
public void testWithKeyedInput() {
    InputOperatorImpl inputOperator = new InputOperatorImpl(new InputOperatorSpec("stream-id", null, null, null, true, "input-op-id"));
    IncomingMessageEnvelope ime = new IncomingMessageEnvelope(mock(SystemStreamPartition.class), "123", "key", "msg");
    Collection<Object> results = inputOperator.handleMessage(ime, mock(MessageCollector.class), mock(TaskCoordinator.class));
    Object result = results.iterator().next();
    assertEquals("key", ((KV) result).getKey());
    assertEquals("msg", ((KV) result).getValue());
}
Also used : InputOperatorSpec(org.apache.samza.operators.spec.InputOperatorSpec) IncomingMessageEnvelope(org.apache.samza.system.IncomingMessageEnvelope) MessageCollector(org.apache.samza.task.MessageCollector) TaskCoordinator(org.apache.samza.task.TaskCoordinator) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) Test(org.junit.Test)

Example 17 with InputOperatorSpec

use of org.apache.samza.operators.spec.InputOperatorSpec in project samza by apache.

the class TestOperatorSpecGraph method setUp.

@Before
public void setUp() {
    this.mockAppDesc = mock(StreamApplicationDescriptorImpl.class);
    /**
     * Setup two linear transformation pipelines:
     * 1) input1 --> filter --> sendTo
     * 2) input2 --> map --> sink
     */
    String inputStreamId1 = "test-input-1";
    String outputStreamId = "test-output-1";
    InputOperatorSpec testInput = new InputOperatorSpec(inputStreamId1, new NoOpSerde(), new NoOpSerde(), null, true, inputStreamId1);
    StreamOperatorSpec filterOp = OperatorSpecs.createFilterOperatorSpec(m -> true, "test-filter-2");
    OutputStreamImpl outputStream1 = new OutputStreamImpl(outputStreamId, null, null, true);
    OutputOperatorSpec outputSpec = OperatorSpecs.createSendToOperatorSpec(outputStream1, "test-output-3");
    testInput.registerNextOperatorSpec(filterOp);
    filterOp.registerNextOperatorSpec(outputSpec);
    String streamId2 = "test-input-2";
    InputOperatorSpec testInput2 = new InputOperatorSpec(streamId2, new NoOpSerde(), new NoOpSerde(), null, true, "test-input-4");
    StreamOperatorSpec testMap = OperatorSpecs.createMapOperatorSpec(m -> m, "test-map-5");
    SinkOperatorSpec testSink = OperatorSpecs.createSinkOperatorSpec((m, mc, tc) -> {
    }, "test-sink-6");
    testInput2.registerNextOperatorSpec(testMap);
    testMap.registerNextOperatorSpec(testSink);
    this.inputOpSpecMap = new LinkedHashMap<>();
    inputOpSpecMap.put(inputStreamId1, testInput);
    inputOpSpecMap.put(streamId2, testInput2);
    this.outputStrmMap = new LinkedHashMap<>();
    outputStrmMap.put(outputStreamId, outputStream1);
    when(mockAppDesc.getInputOperators()).thenReturn(Collections.unmodifiableMap(inputOpSpecMap));
    when(mockAppDesc.getOutputStreams()).thenReturn(Collections.unmodifiableMap(outputStrmMap));
    this.allOpSpecs = new HashSet<OperatorSpec>() {

        {
            this.add(testInput);
            this.add(filterOp);
            this.add(outputSpec);
            this.add(testInput2);
            this.add(testMap);
            this.add(testSink);
        }
    };
}
Also used : StreamOperatorSpec(org.apache.samza.operators.spec.StreamOperatorSpec) OperatorSpec(org.apache.samza.operators.spec.OperatorSpec) SinkOperatorSpec(org.apache.samza.operators.spec.SinkOperatorSpec) OutputOperatorSpec(org.apache.samza.operators.spec.OutputOperatorSpec) InputOperatorSpec(org.apache.samza.operators.spec.InputOperatorSpec) InputOperatorSpec(org.apache.samza.operators.spec.InputOperatorSpec) StreamOperatorSpec(org.apache.samza.operators.spec.StreamOperatorSpec) OutputStreamImpl(org.apache.samza.operators.spec.OutputStreamImpl) StreamApplicationDescriptorImpl(org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl) NoOpSerde(org.apache.samza.serializers.NoOpSerde) OutputOperatorSpec(org.apache.samza.operators.spec.OutputOperatorSpec) SinkOperatorSpec(org.apache.samza.operators.spec.SinkOperatorSpec) Before(org.junit.Before)

Example 18 with InputOperatorSpec

use of org.apache.samza.operators.spec.InputOperatorSpec in project samza by apache.

the class TestStreamApplicationDescriptorImpl method testGetInputStreamWithExpandingSystem.

@Test
public void testGetInputStreamWithExpandingSystem() {
    String streamId = "test-stream-1";
    String expandedStreamId = "expanded-stream";
    AtomicInteger expandCallCount = new AtomicInteger();
    StreamExpander expander = (sg, isd) -> {
        expandCallCount.incrementAndGet();
        InputDescriptor expandedISD = new GenericSystemDescriptor("expanded-system", "mockFactoryClass").getInputDescriptor(expandedStreamId, new IntegerSerde());
        return sg.getInputStream(expandedISD);
    };
    MockExpandingSystemDescriptor sd = new MockExpandingSystemDescriptor("mock-system", expander);
    MockInputDescriptor isd = sd.getInputDescriptor(streamId, new IntegerSerde());
    StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
        appDesc.getInputStream(isd);
    }, getConfig());
    InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(expandedStreamId);
    assertEquals(OpCode.INPUT, inputOpSpec.getOpCode());
    assertEquals(1, expandCallCount.get());
    assertFalse(streamAppDesc.getInputOperators().containsKey(streamId));
    assertFalse(streamAppDesc.getInputDescriptors().containsKey(streamId));
    assertTrue(streamAppDesc.getInputDescriptors().containsKey(expandedStreamId));
    assertEquals(expandedStreamId, inputOpSpec.getStreamId());
}
Also used : SystemDescriptor(org.apache.samza.system.descriptors.SystemDescriptor) GenericSystemDescriptor(org.apache.samza.system.descriptors.GenericSystemDescriptor) IntermediateMessageStreamImpl(org.apache.samza.operators.stream.IntermediateMessageStreamImpl) HashMap(java.util.HashMap) Serde(org.apache.samza.serializers.Serde) GenericInputDescriptor(org.apache.samza.system.descriptors.GenericInputDescriptor) AtomicReference(java.util.concurrent.atomic.AtomicReference) TableImpl(org.apache.samza.operators.TableImpl) ArrayList(java.util.ArrayList) OutputStreamImpl(org.apache.samza.operators.spec.OutputStreamImpl) OperatorSpec(org.apache.samza.operators.spec.OperatorSpec) BaseTableDescriptor(org.apache.samza.table.descriptors.BaseTableDescriptor) ImmutableList(com.google.common.collect.ImmutableList) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ApplicationConfig(org.apache.samza.config.ApplicationConfig) InputTransformer(org.apache.samza.system.descriptors.InputTransformer) ProcessorLifecycleListenerFactory(org.apache.samza.runtime.ProcessorLifecycleListenerFactory) Assert.fail(org.junit.Assert.fail) MapConfig(org.apache.samza.config.MapConfig) IntegerSerde(org.apache.samza.serializers.IntegerSerde) NoOpSerde(org.apache.samza.serializers.NoOpSerde) Mockito.doReturn(org.mockito.Mockito.doReturn) TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) OpCode(org.apache.samza.operators.spec.OperatorSpec.OpCode) InputDescriptor(org.apache.samza.system.descriptors.InputDescriptor) ApplicationContainerContextFactory(org.apache.samza.context.ApplicationContainerContextFactory) GenericOutputDescriptor(org.apache.samza.system.descriptors.GenericOutputDescriptor) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) TransformingInputDescriptorProvider(org.apache.samza.system.descriptors.TransformingInputDescriptorProvider) SamzaException(org.apache.samza.SamzaException) ApplicationTaskContextFactory(org.apache.samza.context.ApplicationTaskContextFactory) ExpandingInputDescriptorProvider(org.apache.samza.system.descriptors.ExpandingInputDescriptorProvider) StreamExpander(org.apache.samza.system.descriptors.StreamExpander) Mockito.verify(org.mockito.Mockito.verify) List(java.util.List) Assert.assertFalse(org.junit.Assert.assertFalse) Optional(java.util.Optional) Config(org.apache.samza.config.Config) KVSerde(org.apache.samza.serializers.KVSerde) StreamApplication(org.apache.samza.application.StreamApplication) Assert.assertEquals(org.junit.Assert.assertEquals) InputOperatorSpec(org.apache.samza.operators.spec.InputOperatorSpec) Mockito.mock(org.mockito.Mockito.mock) GenericInputDescriptor(org.apache.samza.system.descriptors.GenericInputDescriptor) InputDescriptor(org.apache.samza.system.descriptors.InputDescriptor) InputOperatorSpec(org.apache.samza.operators.spec.InputOperatorSpec) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) StreamExpander(org.apache.samza.system.descriptors.StreamExpander) GenericSystemDescriptor(org.apache.samza.system.descriptors.GenericSystemDescriptor) IntegerSerde(org.apache.samza.serializers.IntegerSerde) Test(org.junit.Test)

Example 19 with InputOperatorSpec

use of org.apache.samza.operators.spec.InputOperatorSpec in project samza by apache.

the class StreamApplicationDescriptorImpl method getInputStream.

@Override
public <M> MessageStream<M> getInputStream(InputDescriptor<M, ?> inputDescriptor) {
    SystemDescriptor systemDescriptor = inputDescriptor.getSystemDescriptor();
    Optional<StreamExpander> expander = systemDescriptor.getExpander();
    if (expander.isPresent()) {
        return expander.get().apply(this, inputDescriptor);
    }
    // TODO: SAMZA-1841: need to add to the broadcast streams if inputDescriptor is for a broadcast stream
    addInputDescriptor(inputDescriptor);
    String streamId = inputDescriptor.getStreamId();
    Serde serde = inputDescriptor.getSerde();
    KV<Serde, Serde> kvSerdes = getOrCreateStreamSerdes(streamId, serde);
    boolean isKeyed = serde instanceof KVSerde;
    InputTransformer transformer = inputDescriptor.getTransformer().orElse(null);
    InputOperatorSpec inputOperatorSpec = OperatorSpecs.createInputOperatorSpec(streamId, kvSerdes.getKey(), kvSerdes.getValue(), transformer, isKeyed, this.getNextOpId(OpCode.INPUT, null));
    inputOperators.put(streamId, inputOperatorSpec);
    return new MessageStreamImpl(this, inputOperators.get(streamId));
}
Also used : Serde(org.apache.samza.serializers.Serde) KVSerde(org.apache.samza.serializers.KVSerde) InputOperatorSpec(org.apache.samza.operators.spec.InputOperatorSpec) IntermediateMessageStreamImpl(org.apache.samza.operators.stream.IntermediateMessageStreamImpl) MessageStreamImpl(org.apache.samza.operators.MessageStreamImpl) SystemDescriptor(org.apache.samza.system.descriptors.SystemDescriptor) KVSerde(org.apache.samza.serializers.KVSerde) InputTransformer(org.apache.samza.system.descriptors.InputTransformer) StreamExpander(org.apache.samza.system.descriptors.StreamExpander)

Example 20 with InputOperatorSpec

use of org.apache.samza.operators.spec.InputOperatorSpec in project samza by apache.

the class StreamApplicationDescriptorImpl method getIntermediateStream.

/**
 * Internal helper for {@link MessageStreamImpl} to add an intermediate {@link MessageStream} to the graph.
 * An intermediate {@link MessageStream} is both an output and an input stream.
 *
 * @param streamId the id of the stream to be created.
 * @param serde the {@link Serde} to use for the message in the intermediate stream. If null, the default serde
 *              is used.
 * @param isBroadcast whether the stream is a broadcast stream.
 * @param <M> the type of messages in the intermediate {@link MessageStream}
 * @return  the intermediate {@link MessageStreamImpl}
 */
@VisibleForTesting
public <M> IntermediateMessageStreamImpl<M> getIntermediateStream(String streamId, Serde<M> serde, boolean isBroadcast) {
    Preconditions.checkNotNull(serde, "serde must not be null for intermediate stream: " + streamId);
    Preconditions.checkState(!inputOperators.containsKey(streamId) && !outputStreams.containsKey(streamId), "getIntermediateStream must not be called multiple times with the same streamId: " + streamId);
    if (isBroadcast) {
        intermediateBroadcastStreamIds.add(streamId);
    }
    boolean isKeyed = serde instanceof KVSerde;
    KV<Serde, Serde> kvSerdes = getOrCreateStreamSerdes(streamId, serde);
    InputTransformer transformer = (InputTransformer) getDefaultSystemDescriptor().flatMap(SystemDescriptor::getTransformer).orElse(null);
    InputOperatorSpec inputOperatorSpec = OperatorSpecs.createInputOperatorSpec(streamId, kvSerdes.getKey(), kvSerdes.getValue(), transformer, isKeyed, this.getNextOpId(OpCode.INPUT, null));
    inputOperators.put(streamId, inputOperatorSpec);
    outputStreams.put(streamId, new OutputStreamImpl(streamId, kvSerdes.getKey(), kvSerdes.getValue(), isKeyed));
    return new IntermediateMessageStreamImpl<>(this, inputOperators.get(streamId), outputStreams.get(streamId));
}
Also used : Serde(org.apache.samza.serializers.Serde) KVSerde(org.apache.samza.serializers.KVSerde) InputOperatorSpec(org.apache.samza.operators.spec.InputOperatorSpec) KVSerde(org.apache.samza.serializers.KVSerde) SystemDescriptor(org.apache.samza.system.descriptors.SystemDescriptor) OutputStreamImpl(org.apache.samza.operators.spec.OutputStreamImpl) IntermediateMessageStreamImpl(org.apache.samza.operators.stream.IntermediateMessageStreamImpl) InputTransformer(org.apache.samza.system.descriptors.InputTransformer) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Aggregations

InputOperatorSpec (org.apache.samza.operators.spec.InputOperatorSpec)23 Test (org.junit.Test)13 Serde (org.apache.samza.serializers.Serde)11 KVSerde (org.apache.samza.serializers.KVSerde)10 OperatorSpec (org.apache.samza.operators.spec.OperatorSpec)9 NoOpSerde (org.apache.samza.serializers.NoOpSerde)9 IntegerSerde (org.apache.samza.serializers.IntegerSerde)8 GenericSystemDescriptor (org.apache.samza.system.descriptors.GenericSystemDescriptor)7 GenericInputDescriptor (org.apache.samza.system.descriptors.GenericInputDescriptor)6 ArrayList (java.util.ArrayList)5 OutputStreamImpl (org.apache.samza.operators.spec.OutputStreamImpl)5 StreamTableJoinOperatorSpec (org.apache.samza.operators.spec.StreamTableJoinOperatorSpec)5 IntermediateMessageStreamImpl (org.apache.samza.operators.stream.IntermediateMessageStreamImpl)5 Config (org.apache.samza.config.Config)4 HashSet (java.util.HashSet)3 ApplicationConfig (org.apache.samza.config.ApplicationConfig)3 MapConfig (org.apache.samza.config.MapConfig)3 TestMessageEnvelope (org.apache.samza.operators.data.TestMessageEnvelope)3 IncomingMessageEnvelope (org.apache.samza.system.IncomingMessageEnvelope)3 SystemStreamPartition (org.apache.samza.system.SystemStreamPartition)3