Search in sources :

Example 1 with ZkKeyBuilder

use of org.apache.samza.zk.ZkKeyBuilder in project samza by apache.

the class TestZkLocalApplicationRunner method setUp.

@Override
public void setUp() {
    super.setUp();
    String uniqueTestId = UUID.randomUUID().toString();
    testStreamAppName = String.format("test-app-name-%s", uniqueTestId);
    testStreamAppId = String.format("test-app-id-%s", uniqueTestId);
    inputKafkaTopic = String.format("test-input-topic-%s", uniqueTestId);
    outputKafkaTopic = String.format("test-output-topic-%s", uniqueTestId);
    inputSinglePartitionKafkaTopic = String.format("test-input-single-partition-topic-%s", uniqueTestId);
    outputSinglePartitionKafkaTopic = String.format("test-output-single-partition-topic-%s", uniqueTestId);
    // Set up stream application config map with the given testStreamAppName, testStreamAppId and test kafka system
    // TODO: processorId should typically come up from a processorID generator as processor.id will be deprecated in 0.14.0+
    Map<String, String> configMap = buildStreamApplicationConfigMap(testStreamAppName, testStreamAppId, false, Optional.empty());
    configMap.put(JobConfig.PROCESSOR_ID, PROCESSOR_IDS[0]);
    applicationConfig1 = new ApplicationConfig(new MapConfig(configMap));
    configMap.put(JobConfig.PROCESSOR_ID, PROCESSOR_IDS[1]);
    applicationConfig2 = new ApplicationConfig(new MapConfig(configMap));
    configMap.put(JobConfig.PROCESSOR_ID, PROCESSOR_IDS[2]);
    applicationConfig3 = new ApplicationConfig(new MapConfig(configMap));
    ZkClient zkClient = new ZkClient(zkConnect(), ZK_CONNECTION_TIMEOUT_MS, ZK_CONNECTION_TIMEOUT_MS, new ZkStringSerializer());
    ZkKeyBuilder zkKeyBuilder = new ZkKeyBuilder(ZkJobCoordinatorFactory.getJobCoordinationZkPath(applicationConfig1));
    zkUtils = new ZkUtils(zkKeyBuilder, zkClient, ZK_CONNECTION_TIMEOUT_MS, ZK_SESSION_TIMEOUT_MS, new NoOpMetricsRegistry());
    zkUtils.connect();
    topicToPartitionCount = ImmutableMap.of(inputSinglePartitionKafkaTopic, 1, outputSinglePartitionKafkaTopic, 1, inputKafkaTopic, NUM_PARTITION, outputKafkaTopic, NUM_PARTITION);
    List<NewTopic> newTopics = ImmutableList.of(inputKafkaTopic, outputKafkaTopic, inputSinglePartitionKafkaTopic, outputSinglePartitionKafkaTopic).stream().map(topic -> new NewTopic(topic, topicToPartitionCount.get(topic), (short) 1)).collect(Collectors.toList());
    assertTrue("Encountered errors during test setup. Failed to create topics.", createTopics(newTopics));
    zkMetadataStore = new ZkMetadataStore(zkUtils.getKeyBuilder().getRootPath(), new MapConfig(configMap), new NoOpMetricsRegistry());
}
Also used : ZkClient(org.I0Itec.zkclient.ZkClient) Arrays(java.util.Arrays) LoggerFactory(org.slf4j.LoggerFactory) TaskModel(org.apache.samza.job.model.TaskModel) StoreProperties(org.apache.samza.storage.StoreProperties) Map(java.util.Map) StorageEngine(org.apache.samza.storage.StorageEngine) StandaloneTestUtils(org.apache.samza.test.StandaloneTestUtils) MapConfig(org.apache.samza.config.MapConfig) ApplicationRunner(org.apache.samza.runtime.ApplicationRunner) StorageConfig(org.apache.samza.config.StorageConfig) ImmutableSet(com.google.common.collect.ImmutableSet) TaskName(org.apache.samza.container.TaskName) ZkStringSerializer(org.apache.samza.zk.ZkStringSerializer) ImmutableMap(com.google.common.collect.ImmutableMap) TaskApplication(org.apache.samza.application.TaskApplication) ZkClient(org.I0Itec.zkclient.ZkClient) Set(java.util.Set) UUID(java.util.UUID) MetricsRegistry(org.apache.samza.metrics.MetricsRegistry) NoOpMetricsRegistry(org.apache.samza.util.NoOpMetricsRegistry) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) Objects(java.util.Objects) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) MetadataStoreFactory(org.apache.samza.metadatastore.MetadataStoreFactory) Assert.assertFalse(org.junit.Assert.assertFalse) IntegrationTestHarness(org.apache.samza.test.harness.IntegrationTestHarness) Optional(java.util.Optional) JobCoordinatorConfig(org.apache.samza.config.JobCoordinatorConfig) Config(org.apache.samza.config.Config) ApplicationStatus(org.apache.samza.job.ApplicationStatus) MetricsRegistryMap(org.apache.samza.metrics.MetricsRegistryMap) CoordinatorStreamStore(org.apache.samza.coordinator.metadatastore.CoordinatorStreamStore) ProducerRecord(org.apache.kafka.clients.producer.ProducerRecord) JobModelUtil(org.apache.samza.job.model.JobModelUtil) ZkKeyBuilder(org.apache.samza.zk.ZkKeyBuilder) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) JobConfig(org.apache.samza.config.JobConfig) HashMap(java.util.HashMap) ClusterManagerConfig(org.apache.samza.config.ClusterManagerConfig) Serde(org.apache.samza.serializers.Serde) SystemStreamPartition(org.apache.samza.system.SystemStreamPartition) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) MetadataStore(org.apache.samza.metadatastore.MetadataStore) ImmutableList(com.google.common.collect.ImmutableList) MessageCollector(org.apache.samza.task.MessageCollector) ZkUtils(org.apache.samza.zk.ZkUtils) Timeout(org.junit.rules.Timeout) ApplicationConfig(org.apache.samza.config.ApplicationConfig) ApplicationRunners(org.apache.samza.runtime.ApplicationRunners) ExpectedException(org.junit.rules.ExpectedException) JobModel(org.apache.samza.job.model.JobModel) LocalApplicationRunner(org.apache.samza.runtime.LocalApplicationRunner) ZkJobCoordinatorFactory(org.apache.samza.zk.ZkJobCoordinatorFactory) Logger(org.slf4j.Logger) TaskConfig(org.apache.samza.config.TaskConfig) JobContext(org.apache.samza.context.JobContext) ZkMetadataStore(org.apache.samza.zk.ZkMetadataStore) NewTopic(org.apache.kafka.clients.admin.NewTopic) Partition(org.apache.samza.Partition) ContainerContext(org.apache.samza.context.ContainerContext) Assert.assertTrue(org.junit.Assert.assertTrue) CoordinatorStreamValueSerde(org.apache.samza.coordinator.stream.CoordinatorStreamValueSerde) Test(org.junit.Test) ZkConfig(org.apache.samza.config.ZkConfig) Maps(com.google.common.collect.Maps) File(java.io.File) SamzaException(org.apache.samza.SamzaException) TestKafkaEvent(org.apache.samza.test.util.TestKafkaEvent) Assert.assertNotEquals(org.junit.Assert.assertNotEquals) TaskMode(org.apache.samza.job.model.TaskMode) MockStorageEngine(org.apache.samza.storage.MockStorageEngine) Rule(org.junit.Rule) ReflectionUtil(org.apache.samza.util.ReflectionUtil) ContainerModel(org.apache.samza.job.model.ContainerModel) StorageEngineFactory(org.apache.samza.storage.StorageEngineFactory) Assert(org.junit.Assert) Assert.assertEquals(org.junit.Assert.assertEquals) ZkUtils(org.apache.samza.zk.ZkUtils) ZkKeyBuilder(org.apache.samza.zk.ZkKeyBuilder) NoOpMetricsRegistry(org.apache.samza.util.NoOpMetricsRegistry) ApplicationConfig(org.apache.samza.config.ApplicationConfig) ZkMetadataStore(org.apache.samza.zk.ZkMetadataStore) NewTopic(org.apache.kafka.clients.admin.NewTopic) MapConfig(org.apache.samza.config.MapConfig) ZkStringSerializer(org.apache.samza.zk.ZkStringSerializer)

Aggregations

ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 ImmutableSet (com.google.common.collect.ImmutableSet)1 Maps (com.google.common.collect.Maps)1 Sets (com.google.common.collect.Sets)1 File (java.io.File)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Map (java.util.Map)1 Objects (java.util.Objects)1 Optional (java.util.Optional)1 Set (java.util.Set)1 UUID (java.util.UUID)1 CountDownLatch (java.util.concurrent.CountDownLatch)1 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)1 Collectors (java.util.stream.Collectors)1 ZkClient (org.I0Itec.zkclient.ZkClient)1