Examples with ZkMetadataStore org.apache.samza.zk.ZkMetadataStore used on opensource projects

Example 1 with ZkMetadataStore

use of org.apache.samza.zk.ZkMetadataStore in project samza by apache.

the class TestLocalJobPlanner method createLocalJobPlanner.

private LocalJobPlanner createLocalJobPlanner(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) throws Exception {
    CoordinationUtils coordinationUtils = mock(CoordinationUtils.class);
    DistributedLock distributedLock = mock(DistributedLock.class);
    when(distributedLock.lock(anyObject())).thenReturn(true);
    when(coordinationUtils.getLock(anyString())).thenReturn(distributedLock);
    ZkMetadataStore zkMetadataStore = mock(ZkMetadataStore.class);
    when(zkMetadataStore.get(any())).thenReturn(null);
    PowerMockito.whenNew(ZkMetadataStore.class).withAnyArguments().thenReturn(zkMetadataStore);
    return spy(new LocalJobPlanner(appDesc, coordinationUtils, "FAKE_UID", "FAKE_RUNID"));
}

Example 2 with ZkMetadataStore

use of org.apache.samza.zk.ZkMetadataStore in project samza by apache.

the class TestZkLocalApplicationRunner method setUp.

@Override
public void setUp() {
    super.setUp();
    String uniqueTestId = UUID.randomUUID().toString();
    testStreamAppName = String.format("test-app-name-%s", uniqueTestId);
    testStreamAppId = String.format("test-app-id-%s", uniqueTestId);
    inputKafkaTopic = String.format("test-input-topic-%s", uniqueTestId);
    outputKafkaTopic = String.format("test-output-topic-%s", uniqueTestId);
    inputSinglePartitionKafkaTopic = String.format("test-input-single-partition-topic-%s", uniqueTestId);
    outputSinglePartitionKafkaTopic = String.format("test-output-single-partition-topic-%s", uniqueTestId);
    // Set up stream application config map with the given testStreamAppName, testStreamAppId and test kafka system
    // TODO: processorId should typically come up from a processorID generator as processor.id will be deprecated in 0.14.0+
    Map<String, String> configMap = buildStreamApplicationConfigMap(testStreamAppName, testStreamAppId, false, Optional.empty());
    configMap.put(JobConfig.PROCESSOR_ID, PROCESSOR_IDS[0]);
    applicationConfig1 = new ApplicationConfig(new MapConfig(configMap));
    configMap.put(JobConfig.PROCESSOR_ID, PROCESSOR_IDS[1]);
    applicationConfig2 = new ApplicationConfig(new MapConfig(configMap));
    configMap.put(JobConfig.PROCESSOR_ID, PROCESSOR_IDS[2]);
    applicationConfig3 = new ApplicationConfig(new MapConfig(configMap));
    ZkClient zkClient = new ZkClient(zkConnect(), ZK_CONNECTION_TIMEOUT_MS, ZK_CONNECTION_TIMEOUT_MS, new ZkStringSerializer());
    ZkKeyBuilder zkKeyBuilder = new ZkKeyBuilder(ZkJobCoordinatorFactory.getJobCoordinationZkPath(applicationConfig1));
    zkUtils = new ZkUtils(zkKeyBuilder, zkClient, ZK_CONNECTION_TIMEOUT_MS, ZK_SESSION_TIMEOUT_MS, new NoOpMetricsRegistry());
    zkUtils.connect();
    topicToPartitionCount = ImmutableMap.of(inputSinglePartitionKafkaTopic, 1, outputSinglePartitionKafkaTopic, 1, inputKafkaTopic, NUM_PARTITION, outputKafkaTopic, NUM_PARTITION);
    List<NewTopic> newTopics = ImmutableList.of(inputKafkaTopic, outputKafkaTopic, inputSinglePartitionKafkaTopic, outputSinglePartitionKafkaTopic).stream().map(topic -> new NewTopic(topic, topicToPartitionCount.get(topic), (short) 1)).collect(Collectors.toList());
    assertTrue("Encountered errors during test setup. Failed to create topics.", createTopics(newTopics));
    zkMetadataStore = new ZkMetadataStore(zkUtils.getKeyBuilder().getRootPath(), new MapConfig(configMap), new NoOpMetricsRegistry());
}

Example 3 with ZkMetadataStore

use of org.apache.samza.zk.ZkMetadataStore in project samza by apache.

the class TestZkLocalApplicationRunner method shouldStopNewProcessorsJoiningGroupWhenNumContainersIsGreaterThanNumTasks.

/**
 * sspGrouper is set to GroupBySystemStreamPartitionFactory.
 * Run a stream application(appRunner1) consuming messages from input topic(effectively one container).
 *
 * In the callback triggered by appRunner1 after processing a message, bring up an another stream application(appRunner2).
 *
 * Assertions:
 *           A) JobModel generated before and after the addition of appRunner2 should be equal.
 *           B) Second stream application(appRunner2) should not join the group and process any message.
 */
@Test
public void shouldStopNewProcessorsJoiningGroupWhenNumContainersIsGreaterThanNumTasks() throws InterruptedException {
    // Set up kafka topics.
    publishKafkaEvents(inputSinglePartitionKafkaTopic, 0, NUM_KAFKA_EVENTS * 2, PROCESSOR_IDS[0]);
    // Configuration, verification variables
    MapConfig testConfig = new MapConfig(ImmutableMap.of(JobConfig.SSP_GROUPER_FACTORY, "org.apache.samza.container.grouper.stream.GroupBySystemStreamPartitionFactory", JobConfig.JOB_DEBOUNCE_TIME_MS, "10", ClusterManagerConfig.JOB_HOST_AFFINITY_ENABLED, "true"));
    // Declared as final array to update it from streamApplication callback(Variable should be declared final to access in lambda block).
    final JobModel[] previousJobModel = new JobModel[1];
    final String[] previousJobModelVersion = new String[1];
    AtomicBoolean hasSecondProcessorJoined = new AtomicBoolean(false);
    final CountDownLatch secondProcessorRegistered = new CountDownLatch(1);
    zkUtils.subscribeToProcessorChange((parentPath, currentChilds) -> {
        // When appRunner2 with id: PROCESSOR_IDS[1] is registered, run processing message in appRunner1.
        if (currentChilds.contains(PROCESSOR_IDS[1])) {
            secondProcessorRegistered.countDown();
        }
    });
    // Set up stream app appRunner2.
    CountDownLatch processedMessagesLatch = new CountDownLatch(NUM_KAFKA_EVENTS);
    Config localTestConfig2 = new MapConfig(applicationConfig2, testConfig);
    ApplicationRunner appRunner2 = ApplicationRunners.getApplicationRunner(TestStreamApplication.getInstance(TEST_SYSTEM, ImmutableList.of(inputSinglePartitionKafkaTopic), outputSinglePartitionKafkaTopic, processedMessagesLatch, null, null, localTestConfig2), localTestConfig2);
    // Callback handler for appRunner1.
    TestStreamApplication.StreamApplicationCallback callback = m -> {
        if (hasSecondProcessorJoined.compareAndSet(false, true)) {
            previousJobModelVersion[0] = zkUtils.getJobModelVersion();
            previousJobModel[0] = JobModelUtil.readJobModel(previousJobModelVersion[0], zkMetadataStore);
            executeRun(appRunner2, localTestConfig2);
            try {
                // Wait for appRunner2 to register with zookeeper.
                secondProcessorRegistered.await();
            } catch (InterruptedException e) {
            }
        }
    };
    CountDownLatch kafkaEventsConsumedLatch = new CountDownLatch(NUM_KAFKA_EVENTS * 2);
    // Set up stream app appRunner1.
    Config localTestConfig1 = new MapConfig(applicationConfig1, testConfig);
    ApplicationRunner appRunner1 = ApplicationRunners.getApplicationRunner(TestStreamApplication.getInstance(TEST_SYSTEM, ImmutableList.of(inputSinglePartitionKafkaTopic), outputSinglePartitionKafkaTopic, null, callback, kafkaEventsConsumedLatch, localTestConfig1), localTestConfig1);
    executeRun(appRunner1, localTestConfig1);
    kafkaEventsConsumedLatch.await();
    String currentJobModelVersion = zkUtils.getJobModelVersion();
    JobModel updatedJobModel = JobModelUtil.readJobModel(currentJobModelVersion, zkMetadataStore);
    // Job model before and after the addition of second stream processor should be the same.
    assertEquals(previousJobModel[0], updatedJobModel);
    assertEquals(new MapConfig(), updatedJobModel.getConfig());
    assertEquals(NUM_KAFKA_EVENTS, processedMessagesLatch.getCount());
    appRunner2.kill();
    appRunner2.waitForFinish();
    assertEquals(appRunner2.status(), ApplicationStatus.UnsuccessfulFinish);
    appRunner1.kill();
    appRunner1.waitForFinish();
    assertEquals(appRunner1.status(), ApplicationStatus.SuccessfulFinish);
}

Example 4 with ZkMetadataStore

use of org.apache.samza.zk.ZkMetadataStore in project samza by apache.

the class TestZkLocalApplicationRunner method shouldUpdateJobModelWhenNewProcessorJoiningGroupUsingAllSspToSingleTaskGrouperFactory.

/**
 * sspGrouper is set to AllSspToSingleTaskGrouperFactory (All ssps from input kafka topic are mapped to a single task per container).
 * AllSspToSingleTaskGrouperFactory should be used only with high-level consumers which do the partition management
 * by themselves. Using the factory with the consumers that do not do the partition management will result in
 * each processor/task consuming all the messages from all the partitions.
 * Run a stream application(streamApp1) consuming messages from input topic(effectively one container).
 *
 * In the callback triggered by streamApp1 after processing a message, bring up an another stream application(streamApp2).
 *
 * Assertions:
 *           A) JobModel generated before and after the addition of streamApp2 should not be equal.
 *           B) Second stream application(streamApp2) should join the group and process all the messages.
 */
@Test
public void shouldUpdateJobModelWhenNewProcessorJoiningGroupUsingAllSspToSingleTaskGrouperFactory() throws InterruptedException {
    // Set up kafka topics.
    publishKafkaEvents(inputKafkaTopic, 0, NUM_KAFKA_EVENTS * 2, PROCESSOR_IDS[0]);
    // Configuration, verification variables
    MapConfig testConfig = new MapConfig(ImmutableMap.of(JobConfig.SSP_GROUPER_FACTORY, "org.apache.samza.container.grouper.stream.AllSspToSingleTaskGrouperFactory", JobConfig.JOB_DEBOUNCE_TIME_MS, "10", ClusterManagerConfig.HOST_AFFINITY_ENABLED, "false"));
    // Declared as final array to update it from streamApplication callback(Variable should be declared final to access in lambda block).
    final JobModel[] previousJobModel = new JobModel[1];
    final String[] previousJobModelVersion = new String[1];
    AtomicBoolean hasSecondProcessorJoined = new AtomicBoolean(false);
    final CountDownLatch secondProcessorRegistered = new CountDownLatch(1);
    zkUtils.subscribeToProcessorChange((parentPath, currentChilds) -> {
        // When appRunner2 with id: PROCESSOR_IDS[1] is registered, start processing message in appRunner1.
        if (currentChilds.contains(PROCESSOR_IDS[1])) {
            secondProcessorRegistered.countDown();
        }
    });
    // Set up appRunner2.
    CountDownLatch processedMessagesLatch = new CountDownLatch(NUM_KAFKA_EVENTS * 2);
    Config testAppConfig2 = new MapConfig(applicationConfig2, testConfig);
    ApplicationRunner appRunner2 = ApplicationRunners.getApplicationRunner(TestStreamApplication.getInstance(TEST_SYSTEM, ImmutableList.of(inputKafkaTopic), outputKafkaTopic, processedMessagesLatch, null, null, testAppConfig2), testAppConfig2);
    // Callback handler for appRunner1.
    TestStreamApplication.StreamApplicationCallback streamApplicationCallback = message -> {
        if (hasSecondProcessorJoined.compareAndSet(false, true)) {
            previousJobModelVersion[0] = zkUtils.getJobModelVersion();
            previousJobModel[0] = JobModelUtil.readJobModel(previousJobModelVersion[0], zkMetadataStore);
            executeRun(appRunner2, testAppConfig2);
            try {
                // Wait for appRunner2 to register with zookeeper.
                secondProcessorRegistered.await();
            } catch (InterruptedException e) {
            }
        }
    };
    // This is the latch for the messages received by appRunner1. Since appRunner1 is run first, it gets one event
    // redelivered due to re-balancing done by Zk after the appRunner2 joins (See the callback above).
    CountDownLatch kafkaEventsConsumedLatch = new CountDownLatch(NUM_KAFKA_EVENTS * 2 + 1);
    // Set up stream app appRunner1.
    Config testAppConfig1 = new MapConfig(applicationConfig1, testConfig);
    ApplicationRunner appRunner1 = ApplicationRunners.getApplicationRunner(TestStreamApplication.getInstance(TEST_SYSTEM, ImmutableList.of(inputKafkaTopic), outputKafkaTopic, null, streamApplicationCallback, kafkaEventsConsumedLatch, testAppConfig1), testAppConfig1);
    executeRun(appRunner1, testAppConfig1);
    kafkaEventsConsumedLatch.await();
    String currentJobModelVersion = zkUtils.getJobModelVersion();
    JobModel updatedJobModel = JobModelUtil.readJobModel(currentJobModelVersion, zkMetadataStore);
    // JobModelVersion check to verify that leader publishes new jobModel.
    assertTrue(Integer.parseInt(previousJobModelVersion[0]) < Integer.parseInt(currentJobModelVersion));
    // Job model before and after the addition of second stream processor should not be the same.
    assertTrue(!previousJobModel[0].equals(updatedJobModel));
    // Task names in the job model should be different but the set of partitions should be the same and each task name
    // should be assigned to a different container.
    assertEquals(new MapConfig(), previousJobModel[0].getConfig());
    assertEquals(previousJobModel[0].getContainers().get(PROCESSOR_IDS[0]).getTasks().size(), 1);
    assertEquals(new MapConfig(), updatedJobModel.getConfig());
    assertEquals(updatedJobModel.getContainers().get(PROCESSOR_IDS[0]).getTasks().size(), 1);
    assertEquals(updatedJobModel.getContainers().get(PROCESSOR_IDS[1]).getTasks().size(), 1);
    Map<TaskName, TaskModel> updatedTaskModelMap1 = updatedJobModel.getContainers().get(PROCESSOR_IDS[0]).getTasks();
    Map<TaskName, TaskModel> updatedTaskModelMap2 = updatedJobModel.getContainers().get(PROCESSOR_IDS[1]).getTasks();
    assertEquals(updatedTaskModelMap1.size(), 1);
    assertEquals(updatedTaskModelMap2.size(), 1);
    TaskModel taskModel1 = updatedTaskModelMap1.values().stream().findFirst().get();
    TaskModel taskModel2 = updatedTaskModelMap2.values().stream().findFirst().get();
    assertEquals(taskModel1.getSystemStreamPartitions(), taskModel2.getSystemStreamPartitions());
    assertFalse(taskModel1.getTaskName().getTaskName().equals(taskModel2.getTaskName().getTaskName()));
    processedMessagesLatch.await();
    assertEquals(ApplicationStatus.Running, appRunner2.status());
    appRunner1.kill();
    appRunner1.waitForFinish();
    appRunner2.kill();
    appRunner2.waitForFinish();
    assertEquals(ApplicationStatus.SuccessfulFinish, appRunner1.status());
}

Example 5 with ZkMetadataStore

use of org.apache.samza.zk.ZkMetadataStore in project samza by apache.

the class TestLocalApplicationRunner method prepareTest.

private void prepareTest() throws Exception {
    CoordinationUtils coordinationUtils = mock(CoordinationUtils.class);
    DistributedLock distributedLock = mock(DistributedLock.class);
    when(distributedLock.lock(anyObject())).thenReturn(true);
    when(coordinationUtils.getLock(anyString())).thenReturn(distributedLock);
    ZkMetadataStore zkMetadataStore = mock(ZkMetadataStore.class);
    when(zkMetadataStore.get(any())).thenReturn(null);
    PowerMockito.whenNew(ZkMetadataStore.class).withAnyArguments().thenReturn(zkMetadataStore);
    ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc = ApplicationDescriptorUtil.getAppDescriptor(mockApp, config);
    localPlanner = spy(new LocalJobPlanner(appDesc, coordinationUtils, "FAKE_UID", "FAKE_RUNID"));
    runner = spy(new LocalApplicationRunner(mockApp, config, coordinationUtils));
    doReturn(localPlanner).when(runner).getPlanner();
}