Examples with InputOperatorSpec org.apache.samza.operators.spec.InputOperatorSpec used on opensource projects

Example 6 with InputOperatorSpec

use of org.apache.samza.operators.spec.InputOperatorSpec in project samza by apache.

the class TestStreamApplicationDescriptorImpl method testGetInputStreamWithTransformFunction.

@Test
public void testGetInputStreamWithTransformFunction() {
    String streamId = "test-stream-1";
    Serde mockValueSerde = mock(Serde.class);
    InputTransformer transformer = ime -> ime;
    MockTransformingSystemDescriptor sd = new MockTransformingSystemDescriptor("mockSystem", transformer);
    MockInputDescriptor isd = sd.getInputDescriptor(streamId, mockValueSerde);
    StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
        appDesc.getInputStream(isd);
    }, getConfig());
    InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(streamId);
    assertEquals(OpCode.INPUT, inputOpSpec.getOpCode());
    assertEquals(streamId, inputOpSpec.getStreamId());
    assertEquals(isd, streamAppDesc.getInputDescriptors().get(streamId));
    assertEquals(transformer, inputOpSpec.getTransformer());
}

Example 7 with InputOperatorSpec

use of org.apache.samza.operators.spec.InputOperatorSpec in project samza by apache.

the class TestStreamApplicationDescriptorImpl method testGetIntermediateStreamWithKeyValueSerde.

@Test
public void testGetIntermediateStreamWithKeyValueSerde() {
    String streamId = "streamId";
    StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
    }, getConfig());
    KVSerde mockKVSerde = mock(KVSerde.class);
    Serde mockKeySerde = mock(Serde.class);
    Serde mockValueSerde = mock(Serde.class);
    doReturn(mockKeySerde).when(mockKVSerde).getKeySerde();
    doReturn(mockValueSerde).when(mockKVSerde).getValueSerde();
    IntermediateMessageStreamImpl<TestMessageEnvelope> intermediateStreamImpl = streamAppDesc.getIntermediateStream(streamId, mockKVSerde, false);
    assertEquals(streamAppDesc.getInputOperators().get(streamId), intermediateStreamImpl.getOperatorSpec());
    assertEquals(streamAppDesc.getOutputStreams().get(streamId), intermediateStreamImpl.getOutputStream());
    assertEquals(streamId, intermediateStreamImpl.getStreamId());
    assertEquals(mockKeySerde, intermediateStreamImpl.getOutputStream().getKeySerde());
    assertEquals(mockValueSerde, intermediateStreamImpl.getOutputStream().getValueSerde());
    assertEquals(mockKeySerde, ((InputOperatorSpec) (OperatorSpec) intermediateStreamImpl.getOperatorSpec()).getKeySerde());
    assertEquals(mockValueSerde, ((InputOperatorSpec) (OperatorSpec) intermediateStreamImpl.getOperatorSpec()).getValueSerde());
}

Example 8 with InputOperatorSpec

use of org.apache.samza.operators.spec.InputOperatorSpec in project samza by apache.

the class OperatorSpecGraph method findAllOperatorSpecs.

private HashSet<OperatorSpec> findAllOperatorSpecs() {
    Collection<InputOperatorSpec> inputOperatorSpecs = this.inputOperators.values();
    HashSet<OperatorSpec> operatorSpecs = new HashSet<>();
    for (InputOperatorSpec inputOperatorSpec : inputOperatorSpecs) {
        operatorSpecs.add(inputOperatorSpec);
        doGetOperatorSpecs(inputOperatorSpec, operatorSpecs);
    }
    return operatorSpecs;
}

Example 9 with InputOperatorSpec

use of org.apache.samza.operators.spec.InputOperatorSpec in project samza by apache.

the class ExecutionPlanner method getStreamSet.

/**
 * Creates a {@link StreamSet} whose Id is {@code setId}, and {@link StreamEdge}s
 * correspond to the provided {@code inputOpSpecs}.
 */
private static StreamSet getStreamSet(String setId, Iterable<InputOperatorSpec> inputOpSpecs, JobGraph jobGraph) {
    Set<StreamEdge> streamEdges = new HashSet<>();
    for (InputOperatorSpec inputOpSpec : inputOpSpecs) {
        StreamEdge streamEdge = jobGraph.getStreamEdge(inputOpSpec.getStreamId());
        streamEdges.add(streamEdge);
    }
    return new StreamSet(setId, streamEdges);
}

Example 10 with InputOperatorSpec

use of org.apache.samza.operators.spec.InputOperatorSpec in project samza by apache.

the class OperatorSpecGraphAnalyzer method getJoinToInputOperatorSpecs.

/**
 * Returns a grouping of {@link InputOperatorSpec}s by the joins, i.e. {@link JoinOperatorSpec}s and
 * {@link StreamTableJoinOperatorSpec}s, they participate in.
 *
 * The key of the returned Multimap is of type {@link OperatorSpec} due to the lack of a stricter
 * base type for {@link JoinOperatorSpec} and {@link StreamTableJoinOperatorSpec}. However, key
 * objects are guaranteed to be of either type only.
 */
public static Multimap<OperatorSpec, InputOperatorSpec> getJoinToInputOperatorSpecs(Collection<InputOperatorSpec> inputOpSpecs) {
    Multimap<OperatorSpec, InputOperatorSpec> joinToInputOpSpecs = HashMultimap.create();
    // Create a getNextOpSpecs() function that emulates connections between every SendToTableOperatorSpec
    // — which are terminal OperatorSpecs — and all StreamTableJoinOperatorSpecs referencing the same table.
    // 
    // This is necessary to support Stream-Table Join scenarios because it allows us to associate streams behind
    // SendToTableOperatorSpecs with streams participating in Stream-Table Joins, a connection that would not be
    // easy to make otherwise since SendToTableOperatorSpecs are terminal operator specs.
    Function<OperatorSpec, Iterable<OperatorSpec>> getNextOpSpecs = getCustomGetNextOpSpecs(inputOpSpecs);
    // and join-related operator specs.
    for (InputOperatorSpec inputOpSpec : inputOpSpecs) {
        // Observe all join-related operator specs reachable from this input operator spec.
        JoinVisitor joinVisitor = new JoinVisitor();
        traverse(inputOpSpec, joinVisitor, getNextOpSpecs);
        // Associate every encountered join-related operator spec with this input operator spec.
        for (OperatorSpec joinOpSpec : joinVisitor.getJoins()) {
            joinToInputOpSpecs.put(joinOpSpec, inputOpSpec);
        }
    }
    return joinToInputOpSpecs;
}