use of org.apache.samza.operators.spec.SinkOperatorSpec in project samza by apache.
the class OperatorJsonUtils method operatorToMap.
/**
* Format the operator properties into a map
* @param spec a {@link OperatorSpec} instance
* @return map of the operator properties
*/
public static Map<String, Object> operatorToMap(OperatorSpec spec) {
Map<String, Object> map = new HashMap<>();
map.put(OP_CODE, spec.getOpCode().name());
map.put(OP_ID, spec.getOpId());
map.put(SOURCE_LOCATION, spec.getSourceLocation());
if (spec.getNextStream() != null) {
Collection<OperatorSpec> nextOperators = spec.getNextStream().getRegisteredOperatorSpecs();
map.put(NEXT_OPERATOR_IDS, nextOperators.stream().map(OperatorSpec::getOpId).collect(Collectors.toSet()));
} else {
map.put(NEXT_OPERATOR_IDS, Collections.emptySet());
}
if (spec instanceof SinkOperatorSpec) {
OutputStreamInternal outputStream = ((SinkOperatorSpec) spec).getOutputStream();
if (outputStream != null) {
map.put(OUTPUT_STREAM_ID, outputStream.getStreamSpec().getId());
}
}
if (spec instanceof PartialJoinOperatorSpec) {
map.put(TTL_MS, ((PartialJoinOperatorSpec) spec).getTtlMs());
}
return map;
}
use of org.apache.samza.operators.spec.SinkOperatorSpec in project samza by apache.
the class TestOperatorImpls method testCreateOperator.
@Test
public void testCreateOperator() throws NoSuchFieldException, IllegalAccessException, InvocationTargetException {
// get window operator
WindowOperatorSpec mockWnd = mock(WindowOperatorSpec.class);
WindowInternal<TestMessageEnvelope, String, Integer> windowInternal = new WindowInternal<>(null, null, null, null, null, WindowType.TUMBLING);
when(mockWnd.getWindow()).thenReturn(windowInternal);
Config mockConfig = mock(Config.class);
TaskContext mockContext = mock(TaskContext.class);
OperatorImplGraph opGraph = new OperatorImplGraph();
OperatorImpl<TestMessageEnvelope, ?> opImpl = (OperatorImpl<TestMessageEnvelope, ?>) createOpMethod.invoke(opGraph, mockWnd, mockConfig, mockContext);
assertTrue(opImpl instanceof WindowOperatorImpl);
Field wndInternalField = WindowOperatorImpl.class.getDeclaredField("window");
wndInternalField.setAccessible(true);
WindowInternal wndInternal = (WindowInternal) wndInternalField.get(opImpl);
assertEquals(wndInternal, windowInternal);
// get simple operator
StreamOperatorSpec<TestMessageEnvelope, TestOutputMessageEnvelope> mockSimpleOp = mock(StreamOperatorSpec.class);
FlatMapFunction<TestMessageEnvelope, TestOutputMessageEnvelope> mockTxfmFn = mock(FlatMapFunction.class);
when(mockSimpleOp.getTransformFn()).thenReturn(mockTxfmFn);
opImpl = (OperatorImpl<TestMessageEnvelope, ?>) createOpMethod.invoke(opGraph, mockSimpleOp, mockConfig, mockContext);
assertTrue(opImpl instanceof StreamOperatorImpl);
Field txfmFnField = StreamOperatorImpl.class.getDeclaredField("transformFn");
txfmFnField.setAccessible(true);
assertEquals(mockTxfmFn, txfmFnField.get(opImpl));
// get sink operator
SinkFunction<TestMessageEnvelope> sinkFn = (m, mc, tc) -> {
};
SinkOperatorSpec<TestMessageEnvelope> sinkOp = mock(SinkOperatorSpec.class);
when(sinkOp.getSinkFn()).thenReturn(sinkFn);
opImpl = (OperatorImpl<TestMessageEnvelope, ?>) createOpMethod.invoke(opGraph, sinkOp, mockConfig, mockContext);
assertTrue(opImpl instanceof SinkOperatorImpl);
Field sinkFnField = SinkOperatorImpl.class.getDeclaredField("sinkFn");
sinkFnField.setAccessible(true);
assertEquals(sinkFn, sinkFnField.get(opImpl));
// get join operator
PartialJoinOperatorSpec<String, TestMessageEnvelope, TestMessageEnvelope, TestOutputMessageEnvelope> joinOp = mock(PartialJoinOperatorSpec.class);
opImpl = (OperatorImpl<TestMessageEnvelope, ?>) createOpMethod.invoke(opGraph, joinOp, mockConfig, mockContext);
assertTrue(opImpl instanceof PartialJoinOperatorImpl);
}
use of org.apache.samza.operators.spec.SinkOperatorSpec in project samza by apache.
the class TestMessageStreamImpl method testSink.
@Test
public void testSink() {
StreamApplicationDescriptorImpl mockGraph = mock(StreamApplicationDescriptorImpl.class);
OperatorSpec mockOpSpec = mock(OperatorSpec.class);
MessageStreamImpl<TestMessageEnvelope> inputStream = new MessageStreamImpl<>(mockGraph, mockOpSpec);
inputStream.sink(mock(SinkFunction.class));
ArgumentCaptor<OperatorSpec> registeredOpCaptor = ArgumentCaptor.forClass(OperatorSpec.class);
verify(mockOpSpec).registerNextOperatorSpec(registeredOpCaptor.capture());
OperatorSpec<?, TestMessageEnvelope> registeredOpSpec = registeredOpCaptor.getValue();
assertTrue(registeredOpSpec instanceof SinkOperatorSpec);
assertNotNull(((SinkOperatorSpec) registeredOpSpec).getSinkFn());
assertEquals(OpCode.SINK, registeredOpSpec.getOpCode());
}
use of org.apache.samza.operators.spec.SinkOperatorSpec in project samza by apache.
the class TestOperatorSpecGraph method setUp.
@Before
public void setUp() {
this.mockAppDesc = mock(StreamApplicationDescriptorImpl.class);
/**
* Setup two linear transformation pipelines:
* 1) input1 --> filter --> sendTo
* 2) input2 --> map --> sink
*/
String inputStreamId1 = "test-input-1";
String outputStreamId = "test-output-1";
InputOperatorSpec testInput = new InputOperatorSpec(inputStreamId1, new NoOpSerde(), new NoOpSerde(), null, true, inputStreamId1);
StreamOperatorSpec filterOp = OperatorSpecs.createFilterOperatorSpec(m -> true, "test-filter-2");
OutputStreamImpl outputStream1 = new OutputStreamImpl(outputStreamId, null, null, true);
OutputOperatorSpec outputSpec = OperatorSpecs.createSendToOperatorSpec(outputStream1, "test-output-3");
testInput.registerNextOperatorSpec(filterOp);
filterOp.registerNextOperatorSpec(outputSpec);
String streamId2 = "test-input-2";
InputOperatorSpec testInput2 = new InputOperatorSpec(streamId2, new NoOpSerde(), new NoOpSerde(), null, true, "test-input-4");
StreamOperatorSpec testMap = OperatorSpecs.createMapOperatorSpec(m -> m, "test-map-5");
SinkOperatorSpec testSink = OperatorSpecs.createSinkOperatorSpec((m, mc, tc) -> {
}, "test-sink-6");
testInput2.registerNextOperatorSpec(testMap);
testMap.registerNextOperatorSpec(testSink);
this.inputOpSpecMap = new LinkedHashMap<>();
inputOpSpecMap.put(inputStreamId1, testInput);
inputOpSpecMap.put(streamId2, testInput2);
this.outputStrmMap = new LinkedHashMap<>();
outputStrmMap.put(outputStreamId, outputStream1);
when(mockAppDesc.getInputOperators()).thenReturn(Collections.unmodifiableMap(inputOpSpecMap));
when(mockAppDesc.getOutputStreams()).thenReturn(Collections.unmodifiableMap(outputStrmMap));
this.allOpSpecs = new HashSet<OperatorSpec>() {
{
this.add(testInput);
this.add(filterOp);
this.add(outputSpec);
this.add(testInput2);
this.add(testMap);
this.add(testSink);
}
};
}
use of org.apache.samza.operators.spec.SinkOperatorSpec in project samza by apache.
the class OperatorImplGraph method createOperatorImpl.
/**
* Creates a new {@link OperatorImpl} instance for the provided {@link OperatorSpec}.
*
* @param prevOperatorSpec the original {@link OperatorSpec} that produces output for {@code operatorSpec} from {@link OperatorSpecGraph}
* @param operatorSpec the original {@link OperatorSpec} from {@link OperatorSpecGraph}
* @param context the {@link Context} required to instantiate operators
* @return the {@link OperatorImpl} implementation instance
*/
OperatorImpl createOperatorImpl(OperatorSpec prevOperatorSpec, OperatorSpec operatorSpec, Context context) {
Config config = context.getJobContext().getConfig();
StreamConfig streamConfig = new StreamConfig(config);
if (operatorSpec instanceof InputOperatorSpec) {
return new InputOperatorImpl((InputOperatorSpec) operatorSpec);
} else if (operatorSpec instanceof StreamOperatorSpec) {
return new FlatmapOperatorImpl((StreamOperatorSpec) operatorSpec);
} else if (operatorSpec instanceof SinkOperatorSpec) {
return new SinkOperatorImpl((SinkOperatorSpec) operatorSpec);
} else if (operatorSpec instanceof OutputOperatorSpec) {
String streamId = ((OutputOperatorSpec) operatorSpec).getOutputStream().getStreamId();
SystemStream systemStream = streamConfig.streamIdToSystemStream(streamId);
return new OutputOperatorImpl((OutputOperatorSpec) operatorSpec, systemStream);
} else if (operatorSpec instanceof PartitionByOperatorSpec) {
String streamId = ((PartitionByOperatorSpec) operatorSpec).getOutputStream().getStreamId();
SystemStream systemStream = streamConfig.streamIdToSystemStream(streamId);
return new PartitionByOperatorImpl((PartitionByOperatorSpec) operatorSpec, systemStream, internalTaskContext);
} else if (operatorSpec instanceof WindowOperatorSpec) {
return new WindowOperatorImpl((WindowOperatorSpec) operatorSpec, clock);
} else if (operatorSpec instanceof JoinOperatorSpec) {
return getOrCreatePartialJoinOpImpls((JoinOperatorSpec) operatorSpec, prevOperatorSpec.equals(((JoinOperatorSpec) operatorSpec).getLeftInputOpSpec()), clock);
} else if (operatorSpec instanceof StreamTableJoinOperatorSpec) {
return new StreamTableJoinOperatorImpl((StreamTableJoinOperatorSpec) operatorSpec, context);
} else if (operatorSpec instanceof SendToTableOperatorSpec) {
return new SendToTableOperatorImpl((SendToTableOperatorSpec) operatorSpec, context);
} else if (operatorSpec instanceof SendToTableWithUpdateOperatorSpec) {
return new SendToTableWithUpdateOperatorImpl((SendToTableWithUpdateOperatorSpec) operatorSpec, context);
} else if (operatorSpec instanceof BroadcastOperatorSpec) {
String streamId = ((BroadcastOperatorSpec) operatorSpec).getOutputStream().getStreamId();
SystemStream systemStream = streamConfig.streamIdToSystemStream(streamId);
return new BroadcastOperatorImpl((BroadcastOperatorSpec) operatorSpec, systemStream, context);
} else if (operatorSpec instanceof AsyncFlatMapOperatorSpec) {
return new AsyncFlatmapOperatorImpl((AsyncFlatMapOperatorSpec) operatorSpec);
}
throw new IllegalArgumentException(String.format("Unsupported OperatorSpec: %s", operatorSpec.getClass().getName()));
}
Aggregations