Examples with StartpointManager org.apache.samza.startpoint.StartpointManager used on opensource projects

Example 11 with StartpointManager

use of org.apache.samza.startpoint.StartpointManager in project samza by apache.

the class TestStartpoint method testStartpointSpecific.

@Test
public void testStartpointSpecific() throws InterruptedException {
    Map<Integer, RecordMetadata> sentEvents1 = publishKafkaEventsWithDelayPerEvent(inputKafkaTopic1, 0, NUM_KAFKA_EVENTS, PROCESSOR_IDS[0], Duration.ofMillis(2));
    ConcurrentHashMap<String, IncomingMessageEnvelope> recvEventsInputStartpointSpecific = new ConcurrentHashMap<>();
    CoordinatorStreamStore coordinatorStreamStore = createCoordinatorStreamStore(applicationConfig1);
    coordinatorStreamStore.init();
    StartpointManager startpointManager = new StartpointManager(coordinatorStreamStore);
    startpointManager.start();
    StartpointSpecific startpointSpecific = new StartpointSpecific(String.valueOf(sentEvents1.get(100).offset()));
    writeStartpoints(startpointManager, inputKafkaTopic1, ZK_TEST_PARTITION_COUNT, startpointSpecific);
    startpointManager.stop();
    coordinatorStreamStore.close();
    TestTaskApplication.TaskApplicationProcessCallback processedCallback = (IncomingMessageEnvelope ime, TaskCallback callback) -> {
        try {
            String streamName = ime.getSystemStreamPartition().getStream();
            TestKafkaEvent testKafkaEvent = TestKafkaEvent.fromString((String) ime.getMessage());
            String eventIndex = testKafkaEvent.getEventData();
            if (inputKafkaTopic1.equals(streamName)) {
                recvEventsInputStartpointSpecific.put(eventIndex, ime);
            } else {
                throw new RuntimeException("Unexpected input stream: " + streamName);
            }
            callback.complete();
        } catch (Exception ex) {
            callback.failure(ex);
        }
    };
    // Just fetch a few messages
    CountDownLatch processedMessagesLatchStartpointSpecific = new CountDownLatch(100);
    CountDownLatch shutdownLatchStartpointSpecific = new CountDownLatch(1);
    TestTaskApplication testTaskApplicationStartpointSpecific = new TestTaskApplication(TEST_SYSTEM, inputKafkaTopic1, outputKafkaTopic, processedMessagesLatchStartpointSpecific, shutdownLatchStartpointSpecific, Optional.of(processedCallback));
    ApplicationRunner appRunner = ApplicationRunners.getApplicationRunner(testTaskApplicationStartpointSpecific, applicationConfig1);
    executeRun(appRunner, applicationConfig1);
    assertTrue(processedMessagesLatchStartpointSpecific.await(1, TimeUnit.MINUTES));
    appRunner.kill();
    appRunner.waitForFinish();
    assertTrue(shutdownLatchStartpointSpecific.await(1, TimeUnit.MINUTES));
    Integer startpointSpecificOffset = Integer.valueOf(startpointSpecific.getSpecificOffset());
    for (IncomingMessageEnvelope ime : recvEventsInputStartpointSpecific.values()) {
        Integer eventOffset = Integer.valueOf(ime.getOffset());
        String assertMsg = String.format("Expecting message offset: %d >= Startpoint specific offset: %d", eventOffset, startpointSpecificOffset);
        assertTrue(assertMsg, eventOffset >= startpointSpecificOffset);
    }
}

Example 12 with StartpointManager

use of org.apache.samza.startpoint.StartpointManager in project samza by apache.

the class TestStartpoint method testStartpointTimestamp.

@Test
public void testStartpointTimestamp() throws InterruptedException {
    Map<Integer, RecordMetadata> sentEvents2 = publishKafkaEventsWithDelayPerEvent(inputKafkaTopic2, 0, NUM_KAFKA_EVENTS, PROCESSOR_IDS[1], Duration.ofMillis(2));
    ConcurrentHashMap<String, IncomingMessageEnvelope> recvEventsInputStartpointTimestamp = new ConcurrentHashMap<>();
    CoordinatorStreamStore coordinatorStreamStore = createCoordinatorStreamStore(applicationConfig1);
    coordinatorStreamStore.init();
    StartpointManager startpointManager = new StartpointManager(coordinatorStreamStore);
    startpointManager.start();
    StartpointTimestamp startpointTimestamp = new StartpointTimestamp(sentEvents2.get(150).timestamp());
    writeStartpoints(startpointManager, inputKafkaTopic2, ZK_TEST_PARTITION_COUNT, startpointTimestamp);
    startpointManager.stop();
    coordinatorStreamStore.close();
    TestTaskApplication.TaskApplicationProcessCallback processedCallback = (IncomingMessageEnvelope ime, TaskCallback callback) -> {
        try {
            String streamName = ime.getSystemStreamPartition().getStream();
            TestKafkaEvent testKafkaEvent = TestKafkaEvent.fromString((String) ime.getMessage());
            String eventIndex = testKafkaEvent.getEventData();
            if (inputKafkaTopic2.equals(streamName)) {
                recvEventsInputStartpointTimestamp.put(eventIndex, ime);
            } else {
                throw new RuntimeException("Unexpected input stream: " + streamName);
            }
            callback.complete();
        } catch (Exception ex) {
            callback.failure(ex);
        }
    };
    // Just fetch a few messages
    CountDownLatch processedMessagesLatchStartpointTimestamp = new CountDownLatch(100);
    CountDownLatch shutdownLatchStartpointTimestamp = new CountDownLatch(1);
    TestTaskApplication testTaskApplicationStartpointTimestamp = new TestTaskApplication(TEST_SYSTEM, inputKafkaTopic2, outputKafkaTopic, processedMessagesLatchStartpointTimestamp, shutdownLatchStartpointTimestamp, Optional.of(processedCallback));
    ApplicationRunner appRunner = ApplicationRunners.getApplicationRunner(testTaskApplicationStartpointTimestamp, applicationConfig2);
    executeRun(appRunner, applicationConfig2);
    assertTrue(processedMessagesLatchStartpointTimestamp.await(1, TimeUnit.MINUTES));
    appRunner.kill();
    appRunner.waitForFinish();
    assertTrue(shutdownLatchStartpointTimestamp.await(1, TimeUnit.MINUTES));
    for (IncomingMessageEnvelope ime : recvEventsInputStartpointTimestamp.values()) {
        Integer eventOffset = Integer.valueOf(ime.getOffset());
        // sanity check
        assertNotEquals(0, ime.getEventTime());
        String assertMsg = String.format("Expecting message timestamp: %d >= Startpoint timestamp: %d", ime.getEventTime(), startpointTimestamp.getTimestampOffset());
        assertTrue(assertMsg, ime.getEventTime() >= startpointTimestamp.getTimestampOffset());
    }
}

Example 13 with StartpointManager

use of org.apache.samza.startpoint.StartpointManager in project samza by apache.

the class StaticResourceJobCoordinatorFactory method getJobCoordinator.

@Override
public JobCoordinator getJobCoordinator(String processorId, Config config, MetricsRegistry metricsRegistry, MetadataStore metadataStore) {
    JobInfoServingContext jobModelServingContext = new JobInfoServingContext();
    JobConfig jobConfig = new JobConfig(config);
    CoordinatorCommunicationContext context = new CoordinatorCommunicationContext(jobModelServingContext, config, metricsRegistry);
    CoordinatorCommunication coordinatorCommunication = new HttpCoordinatorToWorkerCommunicationFactory().coordinatorCommunication(context);
    JobCoordinatorMetadataManager jobCoordinatorMetadataManager = new JobCoordinatorMetadataManager(new NamespaceAwareCoordinatorStreamStore(metadataStore, SetJobCoordinatorMetadataMessage.TYPE), JobCoordinatorMetadataManager.ClusterType.NON_YARN, metricsRegistry);
    ChangelogStreamManager changelogStreamManager = new ChangelogStreamManager(new NamespaceAwareCoordinatorStreamStore(metadataStore, SetChangelogMapping.TYPE));
    JobRestartSignal jobRestartSignal = ReflectionUtil.getObj(new JobCoordinatorConfig(config).getJobRestartSignalFactory(), JobRestartSignalFactory.class).build(new JobRestartSignalFactoryContext(config));
    Optional<StartpointManager> startpointManager = jobConfig.getStartpointEnabled() ? Optional.of(new StartpointManager(metadataStore)) : Optional.empty();
    SystemAdmins systemAdmins = new SystemAdmins(config, StaticResourceJobCoordinator.class.getSimpleName());
    StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance());
    JobModelHelper jobModelHelper = buildJobModelHelper(metadataStore, streamMetadataCache);
    StreamPartitionCountMonitorFactory streamPartitionCountMonitorFactory = new StreamPartitionCountMonitorFactory(streamMetadataCache, metricsRegistry);
    StreamRegexMonitorFactory streamRegexMonitorFactory = new StreamRegexMonitorFactory(streamMetadataCache, metricsRegistry);
    Optional<String> executionEnvContainerId = Optional.ofNullable(System.getenv(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID));
    Optional<String> samzaEpochId = Optional.ofNullable(System.getenv(EnvironmentVariables.SAMZA_EPOCH_ID));
    return new StaticResourceJobCoordinator(processorId, jobModelHelper, jobModelServingContext, coordinatorCommunication, jobCoordinatorMetadataManager, streamPartitionCountMonitorFactory, streamRegexMonitorFactory, startpointManager, changelogStreamManager, jobRestartSignal, metricsRegistry, systemAdmins, executionEnvContainerId, samzaEpochId, config);
}