Examples with CoordinatorStreamStore org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore used on opensource projects

Example 16 with CoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore in project samza by apache.

the class TestStartpoint method testStartpointSpecific.

@Test
public void testStartpointSpecific() throws InterruptedException {
    Map<Integer, RecordMetadata> sentEvents1 = publishKafkaEventsWithDelayPerEvent(inputKafkaTopic1, 0, NUM_KAFKA_EVENTS, PROCESSOR_IDS[0], Duration.ofMillis(2));
    ConcurrentHashMap<String, IncomingMessageEnvelope> recvEventsInputStartpointSpecific = new ConcurrentHashMap<>();
    CoordinatorStreamStore coordinatorStreamStore = createCoordinatorStreamStore(applicationConfig1);
    coordinatorStreamStore.init();
    StartpointManager startpointManager = new StartpointManager(coordinatorStreamStore);
    startpointManager.start();
    StartpointSpecific startpointSpecific = new StartpointSpecific(String.valueOf(sentEvents1.get(100).offset()));
    writeStartpoints(startpointManager, inputKafkaTopic1, ZK_TEST_PARTITION_COUNT, startpointSpecific);
    startpointManager.stop();
    coordinatorStreamStore.close();
    TestTaskApplication.TaskApplicationProcessCallback processedCallback = (IncomingMessageEnvelope ime, TaskCallback callback) -> {
        try {
            String streamName = ime.getSystemStreamPartition().getStream();
            TestKafkaEvent testKafkaEvent = TestKafkaEvent.fromString((String) ime.getMessage());
            String eventIndex = testKafkaEvent.getEventData();
            if (inputKafkaTopic1.equals(streamName)) {
                recvEventsInputStartpointSpecific.put(eventIndex, ime);
            } else {
                throw new RuntimeException("Unexpected input stream: " + streamName);
            }
            callback.complete();
        } catch (Exception ex) {
            callback.failure(ex);
        }
    };
    // Just fetch a few messages
    CountDownLatch processedMessagesLatchStartpointSpecific = new CountDownLatch(100);
    CountDownLatch shutdownLatchStartpointSpecific = new CountDownLatch(1);
    TestTaskApplication testTaskApplicationStartpointSpecific = new TestTaskApplication(TEST_SYSTEM, inputKafkaTopic1, outputKafkaTopic, processedMessagesLatchStartpointSpecific, shutdownLatchStartpointSpecific, Optional.of(processedCallback));
    ApplicationRunner appRunner = ApplicationRunners.getApplicationRunner(testTaskApplicationStartpointSpecific, applicationConfig1);
    executeRun(appRunner, applicationConfig1);
    assertTrue(processedMessagesLatchStartpointSpecific.await(1, TimeUnit.MINUTES));
    appRunner.kill();
    appRunner.waitForFinish();
    assertTrue(shutdownLatchStartpointSpecific.await(1, TimeUnit.MINUTES));
    Integer startpointSpecificOffset = Integer.valueOf(startpointSpecific.getSpecificOffset());
    for (IncomingMessageEnvelope ime : recvEventsInputStartpointSpecific.values()) {
        Integer eventOffset = Integer.valueOf(ime.getOffset());
        String assertMsg = String.format("Expecting message offset: %d >= Startpoint specific offset: %d", eventOffset, startpointSpecificOffset);
        assertTrue(assertMsg, eventOffset >= startpointSpecificOffset);
    }
}

Example 17 with CoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore in project samza by apache.

the class YarnJobValidationTool method validateJmxMetrics.

public void validateJmxMetrics() throws Exception {
    MetricsRegistry metricsRegistry = new MetricsRegistryMap();
    CoordinatorStreamStore coordinatorStreamStore = new CoordinatorStreamStore(config, metricsRegistry);
    coordinatorStreamStore.init();
    try {
        LocalityManager localityManager = new LocalityManager(new NamespaceAwareCoordinatorStreamStore(coordinatorStreamStore, SetConfig.TYPE));
        validator.init(config);
        LocalityModel localityModel = localityManager.readLocality();
        for (ProcessorLocality processorLocality : localityModel.getProcessorLocalities().values()) {
            String containerId = processorLocality.id();
            String jmxUrl = processorLocality.jmxTunnelingUrl();
            if (StringUtils.isNotBlank(jmxUrl)) {
                log.info("validate container " + containerId + " metrics with JMX: " + jmxUrl);
                JmxMetricsAccessor jmxMetrics = new JmxMetricsAccessor(jmxUrl);
                jmxMetrics.connect();
                validator.validate(jmxMetrics);
                jmxMetrics.close();
                log.info("validate container " + containerId + " successfully");
            }
        }
        validator.complete();
    } finally {
        coordinatorStreamStore.close();
    }
}

Example 18 with CoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore in project samza by apache.

the class TestStartpoint method testStartpointTimestamp.

@Test
public void testStartpointTimestamp() throws InterruptedException {
    Map<Integer, RecordMetadata> sentEvents2 = publishKafkaEventsWithDelayPerEvent(inputKafkaTopic2, 0, NUM_KAFKA_EVENTS, PROCESSOR_IDS[1], Duration.ofMillis(2));
    ConcurrentHashMap<String, IncomingMessageEnvelope> recvEventsInputStartpointTimestamp = new ConcurrentHashMap<>();
    CoordinatorStreamStore coordinatorStreamStore = createCoordinatorStreamStore(applicationConfig1);
    coordinatorStreamStore.init();
    StartpointManager startpointManager = new StartpointManager(coordinatorStreamStore);
    startpointManager.start();
    StartpointTimestamp startpointTimestamp = new StartpointTimestamp(sentEvents2.get(150).timestamp());
    writeStartpoints(startpointManager, inputKafkaTopic2, ZK_TEST_PARTITION_COUNT, startpointTimestamp);
    startpointManager.stop();
    coordinatorStreamStore.close();
    TestTaskApplication.TaskApplicationProcessCallback processedCallback = (IncomingMessageEnvelope ime, TaskCallback callback) -> {
        try {
            String streamName = ime.getSystemStreamPartition().getStream();
            TestKafkaEvent testKafkaEvent = TestKafkaEvent.fromString((String) ime.getMessage());
            String eventIndex = testKafkaEvent.getEventData();
            if (inputKafkaTopic2.equals(streamName)) {
                recvEventsInputStartpointTimestamp.put(eventIndex, ime);
            } else {
                throw new RuntimeException("Unexpected input stream: " + streamName);
            }
            callback.complete();
        } catch (Exception ex) {
            callback.failure(ex);
        }
    };
    // Just fetch a few messages
    CountDownLatch processedMessagesLatchStartpointTimestamp = new CountDownLatch(100);
    CountDownLatch shutdownLatchStartpointTimestamp = new CountDownLatch(1);
    TestTaskApplication testTaskApplicationStartpointTimestamp = new TestTaskApplication(TEST_SYSTEM, inputKafkaTopic2, outputKafkaTopic, processedMessagesLatchStartpointTimestamp, shutdownLatchStartpointTimestamp, Optional.of(processedCallback));
    ApplicationRunner appRunner = ApplicationRunners.getApplicationRunner(testTaskApplicationStartpointTimestamp, applicationConfig2);
    executeRun(appRunner, applicationConfig2);
    assertTrue(processedMessagesLatchStartpointTimestamp.await(1, TimeUnit.MINUTES));
    appRunner.kill();
    appRunner.waitForFinish();
    assertTrue(shutdownLatchStartpointTimestamp.await(1, TimeUnit.MINUTES));
    for (IncomingMessageEnvelope ime : recvEventsInputStartpointTimestamp.values()) {
        Integer eventOffset = Integer.valueOf(ime.getOffset());
        // sanity check
        assertNotEquals(0, ime.getEventTime());
        String assertMsg = String.format("Expecting message timestamp: %d >= Startpoint timestamp: %d", ime.getEventTime(), startpointTimestamp.getTimestampOffset());
        assertTrue(assertMsg, ime.getEventTime() >= startpointTimestamp.getTimestampOffset());
    }
}

Example 19 with CoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore in project samza by apache.

the class TestLocalApplicationRunner method testCreateCoordinatorStreamWithCoordinatorFactory.

/**
 * Underlying coordinator stream should be created if using CoordinatorStreamMetadataStoreFactory
 */
@Test
public void testCreateCoordinatorStreamWithCoordinatorFactory() throws Exception {
    CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class);
    CoordinatorStreamMetadataStoreFactory coordinatorStreamMetadataStoreFactory = mock(CoordinatorStreamMetadataStoreFactory.class);
    doReturn(coordinatorStreamStore).when(coordinatorStreamMetadataStoreFactory).getMetadataStore(anyString(), any(Config.class), any(MetricsRegistry.class));
    SystemAdmins systemAdmins = mock(SystemAdmins.class);
    PowerMockito.whenNew(SystemAdmins.class).withAnyArguments().thenReturn(systemAdmins);
    LocalApplicationRunner localApplicationRunner = spy(new LocalApplicationRunner(mockApp, config, coordinatorStreamMetadataStoreFactory));
    // create store only if successful in creating the underlying coordinator stream
    doReturn(true).when(localApplicationRunner).createUnderlyingCoordinatorStream(eq(config));
    assertEquals(coordinatorStreamStore, localApplicationRunner.createCoordinatorStreamStore(config));
    verify(localApplicationRunner).createUnderlyingCoordinatorStream(eq(config));
    // do not create store if creating the underlying coordinator stream fails
    doReturn(false).when(localApplicationRunner).createUnderlyingCoordinatorStream(eq(config));
    assertNull(localApplicationRunner.createCoordinatorStreamStore(config));
}

Example 20 with CoordinatorStreamStore

use of org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore in project samza by apache.

the class TestLocalApplicationRunner method testRunComplete.

@Test
public void testRunComplete() throws Exception {
    Map<String, String> cfgs = new HashMap<>();
    cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
    config = new MapConfig(cfgs);
    ProcessorLifecycleListenerFactory mockFactory = (pContext, cfg) -> mock(ProcessorLifecycleListener.class);
    mockApp = (StreamApplication) appDesc -> appDesc.withProcessorLifecycleListenerFactory(mockFactory);
    prepareTest();
    // return the jobConfigs from the planner
    doReturn(Collections.singletonList(new JobConfig(new MapConfig(config)))).when(localPlanner).prepareJobs();
    StreamProcessor sp = mock(StreamProcessor.class);
    CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class);
    ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);
    doAnswer(i -> {
        ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
        listener.afterStart();
        listener.afterStop();
        return null;
    }).when(sp).start();
    ExternalContext externalContext = mock(ExternalContext.class);
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), any(CoordinatorStreamStore.class));
    doReturn(coordinatorStreamStore).when(runner).createCoordinatorStreamStore(any(Config.class));
    runner.run(externalContext);
    runner.waitForFinish();
    verify(coordinatorStreamStore).init();
    verify(coordinatorStreamStore).close();
    assertEquals(runner.status(), ApplicationStatus.SuccessfulFinish);
}