Examples with OperatorImplGraph org.apache.samza.operators.impl.OperatorImplGraph used on opensource projects

Example 1 with OperatorImplGraph

use of org.apache.samza.operators.impl.OperatorImplGraph in project samza by apache.

the class StreamOperatorTask method init.

/**
   * Initializes this task during startup.
   * <p>
   * Implementation: Initializes the user-implemented {@link StreamApplication}. The {@link StreamApplication} sets
   * the input and output streams and the task-wide context manager using the {@link StreamGraphImpl} APIs,
   * and the logical transforms using the {@link org.apache.samza.operators.MessageStream} APIs.
   *<p>
   * It then uses the {@link StreamGraphImpl} to create the {@link OperatorImplGraph} corresponding to the logical
   * DAG. It also saves the mapping between input {@link SystemStream}s and their corresponding
   * {@link InputStreamInternal}s for delivering incoming messages to the appropriate sub-DAG.
   *
   * @param config allows accessing of fields in the configuration files that this StreamTask is specified in
   * @param context allows initializing and accessing contextual data of this StreamTask
   * @throws Exception in case of initialization errors
   */
@Override
public final void init(Config config, TaskContext context) throws Exception {
    StreamGraphImpl streamGraph = new StreamGraphImpl(runner, config);
    // initialize the user-implemented stream application.
    this.streamApplication.init(streamGraph, config);
    // get the user-implemented context manager and initialize it
    this.contextManager = streamGraph.getContextManager();
    if (this.contextManager != null) {
        this.contextManager.init(config, context);
    }
    // create the operator impl DAG corresponding to the logical operator spec DAG
    OperatorImplGraph operatorImplGraph = new OperatorImplGraph(clock);
    operatorImplGraph.init(streamGraph, config, context);
    this.operatorImplGraph = operatorImplGraph;
    // TODO: SAMZA-1118 - Remove mapping after SystemConsumer starts returning logical streamId with incoming messages
    inputSystemStreamToInputStream = new HashMap<>();
    streamGraph.getInputStreams().forEach((streamSpec, inputStream) -> {
        SystemStream systemStream = new SystemStream(streamSpec.getSystemName(), streamSpec.getPhysicalName());
        inputSystemStreamToInputStream.put(systemStream, inputStream);
    });
}

Example 2 with OperatorImplGraph

use of org.apache.samza.operators.impl.OperatorImplGraph in project samza by apache.

the class WatermarkIntegrationTest method testWatermark.

@Test
public void testWatermark() throws Exception {
    Map<String, String> configs = new HashMap<>();
    configs.put(ApplicationConfig.APP_RUNNER_CLASS, MockLocalApplicationRunner.class.getName());
    configs.put("systems.test.samza.factory", TestSystemFactory.class.getName());
    configs.put("streams.PageView.samza.system", "test");
    configs.put("streams.PageView.partitionCount", String.valueOf(PARTITION_COUNT));
    configs.put(JobConfig.JOB_NAME, "test-watermark-job");
    configs.put(JobConfig.PROCESSOR_ID, "1");
    configs.put(JobCoordinatorConfig.JOB_COORDINATOR_FACTORY, PassthroughJobCoordinatorFactory.class.getName());
    configs.put(TaskConfig.GROUPER_FACTORY, SingleContainerGrouperFactory.class.getName());
    configs.put("systems.kafka.samza.factory", "org.apache.samza.system.kafka.KafkaSystemFactory");
    configs.put("systems.kafka.producer.bootstrap.servers", bootstrapUrl());
    configs.put("systems.kafka.consumer.zookeeper.connect", zkConnect());
    configs.put("systems.kafka.samza.key.serde", "int");
    configs.put("systems.kafka.samza.msg.serde", "json");
    configs.put("systems.kafka.default.stream.replication.factor", "1");
    configs.put("job.default.system", "kafka");
    configs.put("serializers.registry.int.class", IntegerSerdeFactory.class.getName());
    configs.put("serializers.registry.string.class", StringSerdeFactory.class.getName());
    configs.put("serializers.registry.json.class", PageViewJsonSerdeFactory.class.getName());
    List<PageView> received = new ArrayList<>();
    class TestStreamApp implements StreamApplication {

        @Override
        public void describe(StreamApplicationDescriptor appDescriptor) {
            DelegatingSystemDescriptor sd = new DelegatingSystemDescriptor("test");
            GenericInputDescriptor<KV<String, PageView>> isd = sd.getInputDescriptor("PageView", KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()));
            appDescriptor.getInputStream(isd).map(KV::getValue).partitionBy(pv -> pv.getMemberId(), pv -> pv, KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()), "p1").sink((m, collector, coordinator) -> {
                received.add(m.getValue());
            });
        }
    }
    Config config = new MapConfig(configs);
    final ApplicationRunner runner = ApplicationRunners.getApplicationRunner(new TestStreamApp(), config);
    executeRun(runner, config);
    // processors are only available when the app is running
    Map<String, StreamOperatorTask> tasks = getTaskOperationGraphs((MockLocalApplicationRunner) runner);
    runner.waitForFinish();
    // wait for the completion to ensure that all tasks are actually initialized and the OperatorImplGraph is initialized
    StreamOperatorTask task0 = tasks.get("Partition 0");
    OperatorImplGraph graph = TestStreamOperatorTask.getOperatorImplGraph(task0);
    OperatorImpl pb = getOperator(graph, OperatorSpec.OpCode.PARTITION_BY);
    assertEquals(TestOperatorImpl.getInputWatermark(pb), 4);
    assertEquals(TestOperatorImpl.getOutputWatermark(pb), 4);
    OperatorImpl sink = getOperator(graph, OperatorSpec.OpCode.SINK);
    assertEquals(TestOperatorImpl.getInputWatermark(sink), 3);
    assertEquals(TestOperatorImpl.getOutputWatermark(sink), 3);
    StreamOperatorTask task1 = tasks.get("Partition 1");
    graph = TestStreamOperatorTask.getOperatorImplGraph(task1);
    pb = getOperator(graph, OperatorSpec.OpCode.PARTITION_BY);
    assertEquals(TestOperatorImpl.getInputWatermark(pb), 3);
    assertEquals(TestOperatorImpl.getOutputWatermark(pb), 3);
    sink = getOperator(graph, OperatorSpec.OpCode.SINK);
    assertEquals(TestOperatorImpl.getInputWatermark(sink), 3);
    assertEquals(TestOperatorImpl.getOutputWatermark(sink), 3);
}

Example 3 with OperatorImplGraph

use of org.apache.samza.operators.impl.OperatorImplGraph in project samza by apache.

the class TestStreamOperatorTask method testExceptionIfInputOperatorMissing.

/**
 * Tests if the appropriate SamzaException is propagated to the TaskCallback if there is no InputOperator for a given
 * SystemStream in the OperatorGraph.
 */
@Test
public void testExceptionIfInputOperatorMissing() throws NoSuchFieldException, IllegalAccessException {
    IncomingMessageEnvelope mockIme = mock(IncomingMessageEnvelope.class, RETURNS_DEEP_STUBS);
    SystemStream testSystemStream = new SystemStream("foo", "bar");
    when(mockIme.getSystemStreamPartition().getSystemStream()).thenReturn(testSystemStream);
    OperatorImplGraph mockOperatorImplGraph = mock(OperatorImplGraph.class);
    when(mockOperatorImplGraph.getInputOperator(anyObject())).thenReturn(null);
    StreamOperatorTask operatorTask = new StreamOperatorTask(mock(OperatorSpecGraph.class));
    operatorTask.setOperatorImplGraph(mockOperatorImplGraph);
    TaskCallback mockTaskCallback = mock(TaskCallback.class);
    operatorTask.processAsync(mockIme, mock(MessageCollector.class), mock(TaskCoordinator.class), mockTaskCallback);
    ArgumentCaptor<Throwable> throwableCaptor = ArgumentCaptor.forClass(Throwable.class);
    verify(mockTaskCallback, only()).failure(throwableCaptor.capture());
    assertEquals(throwableCaptor.getValue().getClass(), SamzaException.class);
    String expectedErrMessage = String.format("InputOperator not found in OperatorGraph for %s. The available input" + " operators are: %s. Please check SystemStream configuration for the `SystemConsumer` and/or task.inputs" + " task configuration.", testSystemStream, mockOperatorImplGraph.getAllInputOperators());
    assertEquals(throwableCaptor.getValue().getMessage(), expectedErrMessage);
}