Search in sources :

Example 1 with SinkOperatorSpec

use of org.apache.samza.operators.spec.SinkOperatorSpec in project samza by apache.

the class OperatorJsonUtils method operatorToMap.

/**
   * Format the operator properties into a map
   * @param spec a {@link OperatorSpec} instance
   * @return map of the operator properties
   */
public static Map<String, Object> operatorToMap(OperatorSpec spec) {
    Map<String, Object> map = new HashMap<>();
    map.put(OP_CODE, spec.getOpCode().name());
    map.put(OP_ID, spec.getOpId());
    map.put(SOURCE_LOCATION, spec.getSourceLocation());
    if (spec.getNextStream() != null) {
        Collection<OperatorSpec> nextOperators = spec.getNextStream().getRegisteredOperatorSpecs();
        map.put(NEXT_OPERATOR_IDS, nextOperators.stream().map(OperatorSpec::getOpId).collect(Collectors.toSet()));
    } else {
        map.put(NEXT_OPERATOR_IDS, Collections.emptySet());
    }
    if (spec instanceof SinkOperatorSpec) {
        OutputStreamInternal outputStream = ((SinkOperatorSpec) spec).getOutputStream();
        if (outputStream != null) {
            map.put(OUTPUT_STREAM_ID, outputStream.getStreamSpec().getId());
        }
    }
    if (spec instanceof PartialJoinOperatorSpec) {
        map.put(TTL_MS, ((PartialJoinOperatorSpec) spec).getTtlMs());
    }
    return map;
}
Also used : OperatorSpec(org.apache.samza.operators.spec.OperatorSpec) PartialJoinOperatorSpec(org.apache.samza.operators.spec.PartialJoinOperatorSpec) SinkOperatorSpec(org.apache.samza.operators.spec.SinkOperatorSpec) HashMap(java.util.HashMap) PartialJoinOperatorSpec(org.apache.samza.operators.spec.PartialJoinOperatorSpec) SinkOperatorSpec(org.apache.samza.operators.spec.SinkOperatorSpec) OutputStreamInternal(org.apache.samza.operators.stream.OutputStreamInternal)

Example 2 with SinkOperatorSpec

use of org.apache.samza.operators.spec.SinkOperatorSpec in project samza by apache.

the class TestOperatorImpls method testCreateOperator.

@Test
public void testCreateOperator() throws NoSuchFieldException, IllegalAccessException, InvocationTargetException {
    // get window operator
    WindowOperatorSpec mockWnd = mock(WindowOperatorSpec.class);
    WindowInternal<TestMessageEnvelope, String, Integer> windowInternal = new WindowInternal<>(null, null, null, null, null, WindowType.TUMBLING);
    when(mockWnd.getWindow()).thenReturn(windowInternal);
    Config mockConfig = mock(Config.class);
    TaskContext mockContext = mock(TaskContext.class);
    OperatorImplGraph opGraph = new OperatorImplGraph();
    OperatorImpl<TestMessageEnvelope, ?> opImpl = (OperatorImpl<TestMessageEnvelope, ?>) createOpMethod.invoke(opGraph, mockWnd, mockConfig, mockContext);
    assertTrue(opImpl instanceof WindowOperatorImpl);
    Field wndInternalField = WindowOperatorImpl.class.getDeclaredField("window");
    wndInternalField.setAccessible(true);
    WindowInternal wndInternal = (WindowInternal) wndInternalField.get(opImpl);
    assertEquals(wndInternal, windowInternal);
    // get simple operator
    StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope> mockSimpleOp = mock(StreamOperatorSpec.class);
    FlatMapFunction<TestMessageEnvelope, TestOutputMessageEnvelope> mockTxfmFn = mock(FlatMapFunction.class);
    when(mockSimpleOp.getTransformFn()).thenReturn(mockTxfmFn);
    opImpl = (OperatorImpl<TestMessageEnvelope, ?>) createOpMethod.invoke(opGraph, mockSimpleOp, mockConfig, mockContext);
    assertTrue(opImpl instanceof StreamOperatorImpl);
    Field txfmFnField = StreamOperatorImpl.class.getDeclaredField("transformFn");
    txfmFnField.setAccessible(true);
    assertEquals(mockTxfmFn, txfmFnField.get(opImpl));
    // get sink operator
    SinkFunction<TestMessageEnvelope> sinkFn = (m, mc, tc) -> {
    };
    SinkOperatorSpec<TestMessageEnvelope> sinkOp = mock(SinkOperatorSpec.class);
    when(sinkOp.getSinkFn()).thenReturn(sinkFn);
    opImpl = (OperatorImpl<TestMessageEnvelope, ?>) createOpMethod.invoke(opGraph, sinkOp, mockConfig, mockContext);
    assertTrue(opImpl instanceof SinkOperatorImpl);
    Field sinkFnField = SinkOperatorImpl.class.getDeclaredField("sinkFn");
    sinkFnField.setAccessible(true);
    assertEquals(sinkFn, sinkFnField.get(opImpl));
    // get join operator
    PartialJoinOperatorSpec<String, TestMessageEnvelope, TestMessageEnvelope, TestOutputMessageEnvelope> joinOp = mock(PartialJoinOperatorSpec.class);
    opImpl = (OperatorImpl<TestMessageEnvelope, ?>) createOpMethod.invoke(opGraph, joinOp, mockConfig, mockContext);
    assertTrue(opImpl instanceof PartialJoinOperatorImpl);
}
Also used : StreamOperatorSpec(org.apache.samza.operators.spec.StreamOperatorSpec) Assert.assertNotSame(org.junit.Assert.assertNotSame) ArrayList(java.util.ArrayList) OperatorSpec(org.apache.samza.operators.spec.OperatorSpec) PartialJoinOperatorSpec(org.apache.samza.operators.spec.PartialJoinOperatorSpec) SinkOperatorSpec(org.apache.samza.operators.spec.SinkOperatorSpec) MessageStreamImpl(org.apache.samza.operators.MessageStreamImpl) Duration(java.time.Duration) TestOutputMessageEnvelope(org.apache.samza.operators.data.TestOutputMessageEnvelope) Method(java.lang.reflect.Method) Before(org.junit.Before) TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) WindowType(org.apache.samza.operators.windows.internal.WindowType) TaskContext(org.apache.samza.task.TaskContext) Windows(org.apache.samza.operators.windows.Windows) Iterator(java.util.Iterator) WindowOperatorSpec(org.apache.samza.operators.spec.WindowOperatorSpec) Set(java.util.Set) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) JoinFunction(org.apache.samza.operators.functions.JoinFunction) Field(java.lang.reflect.Field) FlatMapFunction(org.apache.samza.operators.functions.FlatMapFunction) StreamGraphImpl(org.apache.samza.operators.StreamGraphImpl) InvocationTargetException(java.lang.reflect.InvocationTargetException) SinkFunction(org.apache.samza.operators.functions.SinkFunction) Config(org.apache.samza.config.Config) TestMessageStreamImplUtil(org.apache.samza.operators.TestMessageStreamImplUtil) WindowInternal(org.apache.samza.operators.windows.internal.WindowInternal) Assert.assertEquals(org.junit.Assert.assertEquals) MetricsRegistryMap(org.apache.samza.metrics.MetricsRegistryMap) Mockito.mock(org.mockito.Mockito.mock) TaskContext(org.apache.samza.task.TaskContext) WindowInternal(org.apache.samza.operators.windows.internal.WindowInternal) Config(org.apache.samza.config.Config) WindowOperatorSpec(org.apache.samza.operators.spec.WindowOperatorSpec) Field(java.lang.reflect.Field) TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) TestOutputMessageEnvelope(org.apache.samza.operators.data.TestOutputMessageEnvelope) Test(org.junit.Test)

Example 3 with SinkOperatorSpec

use of org.apache.samza.operators.spec.SinkOperatorSpec in project samza by apache.

the class TestMessageStreamImpl method testSink.

@Test
public void testSink() {
    StreamApplicationDescriptorImpl mockGraph = mock(StreamApplicationDescriptorImpl.class);
    OperatorSpec mockOpSpec = mock(OperatorSpec.class);
    MessageStreamImpl<TestMessageEnvelope> inputStream = new MessageStreamImpl<>(mockGraph, mockOpSpec);
    inputStream.sink(mock(SinkFunction.class));
    ArgumentCaptor<OperatorSpec> registeredOpCaptor = ArgumentCaptor.forClass(OperatorSpec.class);
    verify(mockOpSpec).registerNextOperatorSpec(registeredOpCaptor.capture());
    OperatorSpec<?, TestMessageEnvelope> registeredOpSpec = registeredOpCaptor.getValue();
    assertTrue(registeredOpSpec instanceof SinkOperatorSpec);
    assertNotNull(((SinkOperatorSpec) registeredOpSpec).getSinkFn());
    assertEquals(OpCode.SINK, registeredOpSpec.getOpCode());
}
Also used : StreamOperatorSpec(org.apache.samza.operators.spec.StreamOperatorSpec) PartitionByOperatorSpec(org.apache.samza.operators.spec.PartitionByOperatorSpec) JoinOperatorSpec(org.apache.samza.operators.spec.JoinOperatorSpec) SendToTableOperatorSpec(org.apache.samza.operators.spec.SendToTableOperatorSpec) OperatorSpec(org.apache.samza.operators.spec.OperatorSpec) SinkOperatorSpec(org.apache.samza.operators.spec.SinkOperatorSpec) OutputOperatorSpec(org.apache.samza.operators.spec.OutputOperatorSpec) WindowOperatorSpec(org.apache.samza.operators.spec.WindowOperatorSpec) StreamTableJoinOperatorSpec(org.apache.samza.operators.spec.StreamTableJoinOperatorSpec) IntermediateMessageStreamImpl(org.apache.samza.operators.stream.IntermediateMessageStreamImpl) TestMessageEnvelope(org.apache.samza.operators.data.TestMessageEnvelope) SinkFunction(org.apache.samza.operators.functions.SinkFunction) StreamApplicationDescriptorImpl(org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl) SinkOperatorSpec(org.apache.samza.operators.spec.SinkOperatorSpec) Test(org.junit.Test)

Example 4 with SinkOperatorSpec

use of org.apache.samza.operators.spec.SinkOperatorSpec in project samza by apache.

the class TestOperatorSpecGraph method setUp.

@Before
public void setUp() {
    this.mockAppDesc = mock(StreamApplicationDescriptorImpl.class);
    /**
     * Setup two linear transformation pipelines:
     * 1) input1 --> filter --> sendTo
     * 2) input2 --> map --> sink
     */
    String inputStreamId1 = "test-input-1";
    String outputStreamId = "test-output-1";
    InputOperatorSpec testInput = new InputOperatorSpec(inputStreamId1, new NoOpSerde(), new NoOpSerde(), null, true, inputStreamId1);
    StreamOperatorSpec filterOp = OperatorSpecs.createFilterOperatorSpec(m -> true, "test-filter-2");
    OutputStreamImpl outputStream1 = new OutputStreamImpl(outputStreamId, null, null, true);
    OutputOperatorSpec outputSpec = OperatorSpecs.createSendToOperatorSpec(outputStream1, "test-output-3");
    testInput.registerNextOperatorSpec(filterOp);
    filterOp.registerNextOperatorSpec(outputSpec);
    String streamId2 = "test-input-2";
    InputOperatorSpec testInput2 = new InputOperatorSpec(streamId2, new NoOpSerde(), new NoOpSerde(), null, true, "test-input-4");
    StreamOperatorSpec testMap = OperatorSpecs.createMapOperatorSpec(m -> m, "test-map-5");
    SinkOperatorSpec testSink = OperatorSpecs.createSinkOperatorSpec((m, mc, tc) -> {
    }, "test-sink-6");
    testInput2.registerNextOperatorSpec(testMap);
    testMap.registerNextOperatorSpec(testSink);
    this.inputOpSpecMap = new LinkedHashMap<>();
    inputOpSpecMap.put(inputStreamId1, testInput);
    inputOpSpecMap.put(streamId2, testInput2);
    this.outputStrmMap = new LinkedHashMap<>();
    outputStrmMap.put(outputStreamId, outputStream1);
    when(mockAppDesc.getInputOperators()).thenReturn(Collections.unmodifiableMap(inputOpSpecMap));
    when(mockAppDesc.getOutputStreams()).thenReturn(Collections.unmodifiableMap(outputStrmMap));
    this.allOpSpecs = new HashSet<OperatorSpec>() {

        {
            this.add(testInput);
            this.add(filterOp);
            this.add(outputSpec);
            this.add(testInput2);
            this.add(testMap);
            this.add(testSink);
        }
    };
}
Also used : StreamOperatorSpec(org.apache.samza.operators.spec.StreamOperatorSpec) OperatorSpec(org.apache.samza.operators.spec.OperatorSpec) SinkOperatorSpec(org.apache.samza.operators.spec.SinkOperatorSpec) OutputOperatorSpec(org.apache.samza.operators.spec.OutputOperatorSpec) InputOperatorSpec(org.apache.samza.operators.spec.InputOperatorSpec) InputOperatorSpec(org.apache.samza.operators.spec.InputOperatorSpec) StreamOperatorSpec(org.apache.samza.operators.spec.StreamOperatorSpec) OutputStreamImpl(org.apache.samza.operators.spec.OutputStreamImpl) StreamApplicationDescriptorImpl(org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl) NoOpSerde(org.apache.samza.serializers.NoOpSerde) OutputOperatorSpec(org.apache.samza.operators.spec.OutputOperatorSpec) SinkOperatorSpec(org.apache.samza.operators.spec.SinkOperatorSpec) Before(org.junit.Before)

Example 5 with SinkOperatorSpec

use of org.apache.samza.operators.spec.SinkOperatorSpec in project samza by apache.

the class OperatorImplGraph method createOperatorImpl.

/**
 * Creates a new {@link OperatorImpl} instance for the provided {@link OperatorSpec}.
 *
 * @param prevOperatorSpec the original {@link OperatorSpec} that produces output for {@code operatorSpec} from {@link OperatorSpecGraph}
 * @param operatorSpec  the original {@link OperatorSpec} from {@link OperatorSpecGraph}
 * @param context  the {@link Context} required to instantiate operators
 * @return  the {@link OperatorImpl} implementation instance
 */
OperatorImpl createOperatorImpl(OperatorSpec prevOperatorSpec, OperatorSpec operatorSpec, Context context) {
    Config config = context.getJobContext().getConfig();
    StreamConfig streamConfig = new StreamConfig(config);
    if (operatorSpec instanceof InputOperatorSpec) {
        return new InputOperatorImpl((InputOperatorSpec) operatorSpec);
    } else if (operatorSpec instanceof StreamOperatorSpec) {
        return new FlatmapOperatorImpl((StreamOperatorSpec) operatorSpec);
    } else if (operatorSpec instanceof SinkOperatorSpec) {
        return new SinkOperatorImpl((SinkOperatorSpec) operatorSpec);
    } else if (operatorSpec instanceof OutputOperatorSpec) {
        String streamId = ((OutputOperatorSpec) operatorSpec).getOutputStream().getStreamId();
        SystemStream systemStream = streamConfig.streamIdToSystemStream(streamId);
        return new OutputOperatorImpl((OutputOperatorSpec) operatorSpec, systemStream);
    } else if (operatorSpec instanceof PartitionByOperatorSpec) {
        String streamId = ((PartitionByOperatorSpec) operatorSpec).getOutputStream().getStreamId();
        SystemStream systemStream = streamConfig.streamIdToSystemStream(streamId);
        return new PartitionByOperatorImpl((PartitionByOperatorSpec) operatorSpec, systemStream, internalTaskContext);
    } else if (operatorSpec instanceof WindowOperatorSpec) {
        return new WindowOperatorImpl((WindowOperatorSpec) operatorSpec, clock);
    } else if (operatorSpec instanceof JoinOperatorSpec) {
        return getOrCreatePartialJoinOpImpls((JoinOperatorSpec) operatorSpec, prevOperatorSpec.equals(((JoinOperatorSpec) operatorSpec).getLeftInputOpSpec()), clock);
    } else if (operatorSpec instanceof StreamTableJoinOperatorSpec) {
        return new StreamTableJoinOperatorImpl((StreamTableJoinOperatorSpec) operatorSpec, context);
    } else if (operatorSpec instanceof SendToTableOperatorSpec) {
        return new SendToTableOperatorImpl((SendToTableOperatorSpec) operatorSpec, context);
    } else if (operatorSpec instanceof SendToTableWithUpdateOperatorSpec) {
        return new SendToTableWithUpdateOperatorImpl((SendToTableWithUpdateOperatorSpec) operatorSpec, context);
    } else if (operatorSpec instanceof BroadcastOperatorSpec) {
        String streamId = ((BroadcastOperatorSpec) operatorSpec).getOutputStream().getStreamId();
        SystemStream systemStream = streamConfig.streamIdToSystemStream(streamId);
        return new BroadcastOperatorImpl((BroadcastOperatorSpec) operatorSpec, systemStream, context);
    } else if (operatorSpec instanceof AsyncFlatMapOperatorSpec) {
        return new AsyncFlatmapOperatorImpl((AsyncFlatMapOperatorSpec) operatorSpec);
    }
    throw new IllegalArgumentException(String.format("Unsupported OperatorSpec: %s", operatorSpec.getClass().getName()));
}
Also used : StreamConfig(org.apache.samza.config.StreamConfig) Config(org.apache.samza.config.Config) JoinOperatorSpec(org.apache.samza.operators.spec.JoinOperatorSpec) StreamTableJoinOperatorSpec(org.apache.samza.operators.spec.StreamTableJoinOperatorSpec) OutputOperatorSpec(org.apache.samza.operators.spec.OutputOperatorSpec) StreamOperatorSpec(org.apache.samza.operators.spec.StreamOperatorSpec) PartitionByOperatorSpec(org.apache.samza.operators.spec.PartitionByOperatorSpec) BroadcastOperatorSpec(org.apache.samza.operators.spec.BroadcastOperatorSpec) InputOperatorSpec(org.apache.samza.operators.spec.InputOperatorSpec) SystemStream(org.apache.samza.system.SystemStream) StreamConfig(org.apache.samza.config.StreamConfig) WindowOperatorSpec(org.apache.samza.operators.spec.WindowOperatorSpec) SendToTableWithUpdateOperatorSpec(org.apache.samza.operators.spec.SendToTableWithUpdateOperatorSpec) SendToTableOperatorSpec(org.apache.samza.operators.spec.SendToTableOperatorSpec) AsyncFlatMapOperatorSpec(org.apache.samza.operators.spec.AsyncFlatMapOperatorSpec) SinkOperatorSpec(org.apache.samza.operators.spec.SinkOperatorSpec) StreamTableJoinOperatorSpec(org.apache.samza.operators.spec.StreamTableJoinOperatorSpec)

Aggregations

SinkOperatorSpec (org.apache.samza.operators.spec.SinkOperatorSpec)5 OperatorSpec (org.apache.samza.operators.spec.OperatorSpec)4 StreamOperatorSpec (org.apache.samza.operators.spec.StreamOperatorSpec)4 OutputOperatorSpec (org.apache.samza.operators.spec.OutputOperatorSpec)3 WindowOperatorSpec (org.apache.samza.operators.spec.WindowOperatorSpec)3 StreamApplicationDescriptorImpl (org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl)2 Config (org.apache.samza.config.Config)2 TestMessageEnvelope (org.apache.samza.operators.data.TestMessageEnvelope)2 SinkFunction (org.apache.samza.operators.functions.SinkFunction)2 InputOperatorSpec (org.apache.samza.operators.spec.InputOperatorSpec)2 JoinOperatorSpec (org.apache.samza.operators.spec.JoinOperatorSpec)2 PartialJoinOperatorSpec (org.apache.samza.operators.spec.PartialJoinOperatorSpec)2 PartitionByOperatorSpec (org.apache.samza.operators.spec.PartitionByOperatorSpec)2 SendToTableOperatorSpec (org.apache.samza.operators.spec.SendToTableOperatorSpec)2 StreamTableJoinOperatorSpec (org.apache.samza.operators.spec.StreamTableJoinOperatorSpec)2 Before (org.junit.Before)2 Test (org.junit.Test)2 Field (java.lang.reflect.Field)1 InvocationTargetException (java.lang.reflect.InvocationTargetException)1 Method (java.lang.reflect.Method)1