Examples with StartpointManager org.apache.samza.startpoint.StartpointManager used on opensource projects

Example 6 with StartpointManager

use of org.apache.samza.startpoint.StartpointManager in project samza by apache.

the class StreamProcessor method createSamzaContainer.

@VisibleForTesting
SamzaContainer createSamzaContainer(String processorId, JobModel jobModel) {
    // Creating diagnostics manager and wiring it respectively
    String jobName = new JobConfig(config).getName().get();
    String jobId = new JobConfig(config).getJobId();
    Optional<DiagnosticsManager> diagnosticsManager = DiagnosticsUtil.buildDiagnosticsManager(jobName, jobId, jobModel, processorId, Optional.empty(), Optional.empty(), config);
    // Metadata store lifecycle managed outside of the SamzaContainer.
    // All manager lifecycles are managed in the SamzaContainer including startpointManager
    StartpointManager startpointManager = null;
    if (metadataStore != null && new JobConfig(config).getStartpointEnabled()) {
        startpointManager = new StartpointManager(metadataStore);
    } else if (!new JobConfig(config).getStartpointEnabled()) {
        LOGGER.warn("StartpointManager not instantiated because startpoints is not enabled");
    } else {
        LOGGER.warn("StartpointManager cannot be instantiated because no metadata store defined for this stream processor");
    }
    /*
     * StreamProcessor has a metricsRegistry instance variable, but StreamProcessor registers its metrics on its own
     * with the reporters. Therefore, don't reuse the StreamProcessor.metricsRegistry, because SamzaContainer also
     * registers the registry, and that will result in unnecessary duplicate metrics.
     */
    MetricsRegistryMap metricsRegistryMap = new MetricsRegistryMap();
    return SamzaContainer.apply(processorId, jobModel, ScalaJavaUtil.toScalaMap(this.customMetricsReporter), metricsRegistryMap, this.taskFactory, JobContextImpl.fromConfigWithDefaults(this.config, jobModel), Option.apply(this.applicationDefinedContainerContextFactoryOptional.orElse(null)), Option.apply(this.applicationDefinedTaskContextFactoryOptional.orElse(null)), Option.apply(this.externalContextOptional.orElse(null)), null, startpointManager, Option.apply(diagnosticsManager.orElse(null)));
}

Example 7 with StartpointManager

use of org.apache.samza.startpoint.StartpointManager in project samza by apache.

the class TestZkJobCoordinator method testDoOnProcessorChange.

@Test
public void testDoOnProcessorChange() {
    when(zkUtils.getJobModel(TEST_JOB_MODEL_VERSION)).thenReturn(jobModel);
    StartpointManager mockStartpointManager = Mockito.mock(StartpointManager.class);
    ZkJobCoordinator zkJobCoordinator = Mockito.spy(new ZkJobCoordinator(PROCESSOR_ID, config, new NoOpMetricsRegistry(), zkUtils, zkMetadataStore, coordinatorStreamStore));
    doReturn(mockStartpointManager).when(zkJobCoordinator).createStartpointManager();
    doReturn(jobModel).when(zkJobCoordinator).generateNewJobModel(any());
    doNothing().when(zkJobCoordinator).loadMetadataResources(jobModel);
    zkJobCoordinator.doOnProcessorChange();
    verify(zkUtils).publishJobModelVersion(anyString(), anyString());
    verify(zkJobCoordinator).loadMetadataResources(eq(jobModel));
}

Example 8 with StartpointManager

use of org.apache.samza.startpoint.StartpointManager in project samza by apache.

the class TestZkJobCoordinator method testLoadMetadataResources.

@Test
public void testLoadMetadataResources() throws IOException {
    when(zkUtils.getJobModel(TEST_JOB_MODEL_VERSION)).thenReturn(jobModel);
    StartpointManager mockStartpointManager = Mockito.mock(StartpointManager.class);
    ZkJobCoordinator zkJobCoordinator = Mockito.spy(new ZkJobCoordinator(PROCESSOR_ID, config, new NoOpMetricsRegistry(), zkUtils, zkMetadataStore, coordinatorStreamStore));
    doReturn(mockStartpointManager).when(zkJobCoordinator).createStartpointManager();
    MetadataResourceUtil mockMetadataResourceUtil = mock(MetadataResourceUtil.class);
    doReturn(mockMetadataResourceUtil).when(zkJobCoordinator).createMetadataResourceUtil(any(), any(Config.class));
    verifyZeroInteractions(mockStartpointManager);
    zkJobCoordinator.loadMetadataResources(jobModel);
    verify(mockMetadataResourceUtil).createResources();
    verify(mockStartpointManager).start();
    verify(mockStartpointManager).fanOut(any());
    verify(mockStartpointManager).stop();
}

Example 9 with StartpointManager

use of org.apache.samza.startpoint.StartpointManager in project samza by apache.

the class TestStartpoint method testStartpointOldest.

@Test
public void testStartpointOldest() throws InterruptedException {
    publishKafkaEventsWithDelayPerEvent(inputKafkaTopic3, 0, NUM_KAFKA_EVENTS, PROCESSOR_IDS[2], Duration.ofMillis(2));
    ConcurrentHashMap<String, IncomingMessageEnvelope> recvEventsInputStartpointOldest = new ConcurrentHashMap<>();
    CoordinatorStreamStore coordinatorStreamStore = createCoordinatorStreamStore(applicationConfig1);
    coordinatorStreamStore.init();
    StartpointManager startpointManager = new StartpointManager(coordinatorStreamStore);
    startpointManager.start();
    StartpointOldest startpointOldest = new StartpointOldest();
    writeStartpoints(startpointManager, inputKafkaTopic3, ZK_TEST_PARTITION_COUNT, startpointOldest);
    startpointManager.stop();
    coordinatorStreamStore.close();
    TestTaskApplication.TaskApplicationProcessCallback processedCallback = (IncomingMessageEnvelope ime, TaskCallback callback) -> {
        try {
            String streamName = ime.getSystemStreamPartition().getStream();
            TestKafkaEvent testKafkaEvent = TestKafkaEvent.fromString((String) ime.getMessage());
            String eventIndex = testKafkaEvent.getEventData();
            if (inputKafkaTopic3.equals(streamName)) {
                recvEventsInputStartpointOldest.put(eventIndex, ime);
            } else {
                throw new RuntimeException("Unexpected input stream: " + streamName);
            }
            callback.complete();
        } catch (Exception ex) {
            callback.failure(ex);
        }
    };
    // Fetch all since consuming from oldest
    CountDownLatch processedMessagesLatchStartpointOldest = new CountDownLatch(NUM_KAFKA_EVENTS);
    CountDownLatch shutdownLatchStartpointOldest = new CountDownLatch(1);
    TestTaskApplication testTaskApplicationStartpointOldest = new TestTaskApplication(TEST_SYSTEM, inputKafkaTopic3, outputKafkaTopic, processedMessagesLatchStartpointOldest, shutdownLatchStartpointOldest, Optional.of(processedCallback));
    ApplicationRunner appRunner = ApplicationRunners.getApplicationRunner(testTaskApplicationStartpointOldest, applicationConfig3);
    executeRun(appRunner, applicationConfig3);
    assertTrue(processedMessagesLatchStartpointOldest.await(1, TimeUnit.MINUTES));
    appRunner.kill();
    appRunner.waitForFinish();
    assertTrue(shutdownLatchStartpointOldest.await(1, TimeUnit.MINUTES));
    assertEquals("Expecting to have processed all the events", NUM_KAFKA_EVENTS, recvEventsInputStartpointOldest.size());
}

Example 10 with StartpointManager

use of org.apache.samza.startpoint.StartpointManager in project samza by apache.

the class TestStartpoint method testStartpointUpcoming.

@Test
public void testStartpointUpcoming() throws InterruptedException {
    publishKafkaEventsWithDelayPerEvent(inputKafkaTopic4, 0, NUM_KAFKA_EVENTS, PROCESSOR_IDS[3], Duration.ofMillis(2));
    ConcurrentHashMap<String, IncomingMessageEnvelope> recvEventsInputStartpointUpcoming = new ConcurrentHashMap<>();
    CoordinatorStreamStore coordinatorStreamStore = createCoordinatorStreamStore(applicationConfig1);
    coordinatorStreamStore.init();
    StartpointManager startpointManager = new StartpointManager(coordinatorStreamStore);
    startpointManager.start();
    StartpointUpcoming startpointUpcoming = new StartpointUpcoming();
    writeStartpoints(startpointManager, inputKafkaTopic4, ZK_TEST_PARTITION_COUNT, startpointUpcoming);
    startpointManager.stop();
    coordinatorStreamStore.close();
    TestTaskApplication.TaskApplicationProcessCallback processedCallback = (IncomingMessageEnvelope ime, TaskCallback callback) -> {
        try {
            String streamName = ime.getSystemStreamPartition().getStream();
            TestKafkaEvent testKafkaEvent = TestKafkaEvent.fromString((String) ime.getMessage());
            String eventIndex = testKafkaEvent.getEventData();
            if (inputKafkaTopic4.equals(streamName)) {
                recvEventsInputStartpointUpcoming.put(eventIndex, ime);
            } else {
                throw new RuntimeException("Unexpected input stream: " + streamName);
            }
            callback.complete();
        } catch (Exception ex) {
            callback.failure(ex);
        }
    };
    // Expecting none, so just attempt a small number of fetches.
    CountDownLatch processedMessagesLatchStartpointUpcoming = new CountDownLatch(5);
    CountDownLatch shutdownLatchStartpointUpcoming = new CountDownLatch(1);
    TestTaskApplication testTaskApplicationStartpointUpcoming = new TestTaskApplication(TEST_SYSTEM, inputKafkaTopic4, outputKafkaTopic, processedMessagesLatchStartpointUpcoming, shutdownLatchStartpointUpcoming, Optional.of(processedCallback));
    // Startpoint upcoming
    ApplicationRunner appRunner = ApplicationRunners.getApplicationRunner(testTaskApplicationStartpointUpcoming, applicationConfig4);
    executeRun(appRunner, applicationConfig4);
    assertFalse("Expecting to timeout and not process any old messages.", processedMessagesLatchStartpointUpcoming.await(15, TimeUnit.SECONDS));
    assertEquals("Expecting not to process any old messages.", 0, recvEventsInputStartpointUpcoming.size());
    appRunner.kill();
    appRunner.waitForFinish();
    assertTrue(shutdownLatchStartpointUpcoming.await(1, TimeUnit.MINUTES));
}