Examples with SystemProducer org.apache.samza.system.SystemProducer used on opensource projects

Example 11 with SystemProducer

use of org.apache.samza.system.SystemProducer in project samza by apache.

the class MetricsSnapshotReporterFactory method getProducer.

protected SystemProducer getProducer(String reporterName, Config config, MetricsRegistryMap registry) {
    SystemConfig systemConfig = new SystemConfig(config);
    String systemName = getSystemStream(reporterName, config).getSystem();
    String systemFactoryClassName = systemConfig.getSystemFactory(systemName).orElseThrow(() -> new SamzaException(String.format("Trying to fetch system factory for system %s, which isn't defined in config.", systemName)));
    SystemFactory systemFactory = ReflectionUtil.getObj(systemFactoryClassName, SystemFactory.class);
    LOG.info("Got system factory {}.", systemFactory);
    SystemProducer producer = systemFactory.getProducer(systemName, config, registry);
    LOG.info("Got producer {}.", producer);
    return producer;
}

Example 12 with SystemProducer

use of org.apache.samza.system.SystemProducer in project samza by apache.

the class TestKafkaCheckpointManager method testWriteCheckpointShouldRecreateSystemProducerOnFailure.

@Test
public void testWriteCheckpointShouldRecreateSystemProducerOnFailure() {
    setupSystemFactory(config());
    SystemProducer secondKafkaProducer = mock(SystemProducer.class);
    // override default mock behavior to return a second producer on the second call to create a producer
    when(this.systemFactory.getProducer(CHECKPOINT_SYSTEM, config(), this.metricsRegistry, KafkaCheckpointManager.class.getSimpleName())).thenReturn(this.systemProducer, secondKafkaProducer);
    // first producer throws an exception on flush
    doThrow(new RuntimeException("flush failed")).when(this.systemProducer).flush(TASK0.getTaskName());
    KafkaCheckpointManager kafkaCheckpointManager = buildKafkaCheckpointManager(true, config());
    kafkaCheckpointManager.register(TASK0);
    CheckpointV1 checkpointV1 = buildCheckpointV1(INPUT_SSP0, "0");
    kafkaCheckpointManager.writeCheckpoint(TASK0, checkpointV1);
    // first producer should be stopped
    verify(this.systemProducer).stop();
    // register and start the second producer
    verify(secondKafkaProducer).register(TASK0.getTaskName());
    verify(secondKafkaProducer).start();
    // check that the second producer was given the message to send out
    ArgumentCaptor<OutgoingMessageEnvelope> outgoingMessageEnvelopeArgumentCaptor = ArgumentCaptor.forClass(OutgoingMessageEnvelope.class);
    verify(secondKafkaProducer).send(eq(TASK0.getTaskName()), outgoingMessageEnvelopeArgumentCaptor.capture());
    assertEquals(CHECKPOINT_SSP, outgoingMessageEnvelopeArgumentCaptor.getValue().getSystemStream());
    assertEquals(new KafkaCheckpointLogKey(KafkaCheckpointLogKey.CHECKPOINT_V1_KEY_TYPE, TASK0, GROUPER_FACTORY_CLASS), KAFKA_CHECKPOINT_LOG_KEY_SERDE.fromBytes((byte[]) outgoingMessageEnvelopeArgumentCaptor.getValue().getKey()));
    assertEquals(checkpointV1, CHECKPOINT_V1_SERDE.fromBytes((byte[]) outgoingMessageEnvelopeArgumentCaptor.getValue().getMessage()));
    verify(secondKafkaProducer).flush(TASK0.getTaskName());
}

Example 13 with SystemProducer

use of org.apache.samza.system.SystemProducer in project samza by apache.

the class TestDiagnosticsManager method testDiagnosticsManagerStart.

@Test
public void testDiagnosticsManagerStart() {
    SystemProducer mockSystemProducer = Mockito.mock(SystemProducer.class);
    DiagnosticsManager diagnosticsManager = new DiagnosticsManager(JOB_NAME, JOB_ID, containerModels, CONTAINER_MB, CONTAINER_NUM_CORES, NUM_PERSISTENT_STORES, MAX_HEAP_SIZE, CONTAINER_THREAD_POOL_SIZE, "0", EXECUTION_ENV_CONTAINER_ID, SAMZA_EPOCH_ID, TASK_CLASS_VERSION, SAMZA_VERSION, HOSTNAME, diagnosticsSystemStream, mockSystemProducer, Duration.ofSeconds(1), mockExecutorService, AUTOSIZING_ENABLED, config, this.clock);
    diagnosticsManager.start();
    Mockito.verify(mockSystemProducer, Mockito.times(1)).start();
    Mockito.verify(mockExecutorService, Mockito.times(1)).scheduleWithFixedDelay(Mockito.any(Runnable.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.any(TimeUnit.class));
}

Example 14 with SystemProducer

use of org.apache.samza.system.SystemProducer in project samza by apache.

the class TestDiagnosticsManager method testDiagnosticsManagerForceStop.

@Test
public void testDiagnosticsManagerForceStop() throws InterruptedException {
    SystemProducer mockSystemProducer = Mockito.mock(SystemProducer.class);
    Mockito.when(mockExecutorService.isTerminated()).thenReturn(false);
    Duration terminationDuration = Duration.ofSeconds(1);
    DiagnosticsManager diagnosticsManager = new DiagnosticsManager(JOB_NAME, JOB_ID, containerModels, CONTAINER_MB, CONTAINER_NUM_CORES, NUM_PERSISTENT_STORES, MAX_HEAP_SIZE, CONTAINER_THREAD_POOL_SIZE, "0", EXECUTION_ENV_CONTAINER_ID, SAMZA_EPOCH_ID, TASK_CLASS_VERSION, SAMZA_VERSION, HOSTNAME, diagnosticsSystemStream, mockSystemProducer, terminationDuration, mockExecutorService, AUTOSIZING_ENABLED, config, this.clock);
    diagnosticsManager.stop();
    Mockito.verify(mockExecutorService, Mockito.times(1)).shutdown();
    Mockito.verify(mockExecutorService, Mockito.times(1)).awaitTermination(terminationDuration.toMillis(), TimeUnit.MILLISECONDS);
    Mockito.verify(mockExecutorService, Mockito.times(1)).shutdownNow();
    Mockito.verify(mockSystemProducer, Mockito.times(1)).stop();
}