Examples with DistributedLogConfiguration com.twitter.distributedlog.DistributedLogConfiguration used on opensource projects

Example 46 with DistributedLogConfiguration

use of com.twitter.distributedlog.DistributedLogConfiguration in project distributedlog by twitter.

the class TestConfigurationSubscription method testReloadConfiguration.

@Test(timeout = 60000)
public void testReloadConfiguration() throws Exception {
    PropertiesWriter writer = new PropertiesWriter();
    FileConfigurationBuilder builder = new PropertiesConfigurationBuilder(writer.getFile().toURI().toURL());
    ConcurrentConstConfiguration conf = new ConcurrentConstConfiguration(new DistributedLogConfiguration());
    DeterministicScheduler executorService = new DeterministicScheduler();
    List<FileConfigurationBuilder> fileConfigBuilders = Lists.newArrayList(builder);
    ConfigurationSubscription confSub = new ConfigurationSubscription(conf, fileConfigBuilders, executorService, 100, TimeUnit.MILLISECONDS);
    final AtomicReference<ConcurrentBaseConfiguration> confHolder = new AtomicReference<>();
    confSub.registerListener(new com.twitter.distributedlog.config.ConfigurationListener() {

        @Override
        public void onReload(ConcurrentBaseConfiguration conf) {
            confHolder.set(conf);
        }
    });
    assertEquals(null, conf.getProperty("prop1"));
    // add
    writer.setProperty("prop1", "1");
    writer.save();
    // ensure the file change reloading event can be triggered
    ensureConfigReloaded();
    // reload the config
    confSub.reload();
    assertNotNull(confHolder.get());
    assertTrue(conf == confHolder.get());
    assertEquals("1", conf.getProperty("prop1"));
}

Example 47 with DistributedLogConfiguration

use of com.twitter.distributedlog.DistributedLogConfiguration in project distributedlog by twitter.

the class TestDynamicConfigurationFeatureProvider method testReloadFeaturesFromOverlay.

@Test(timeout = 60000)
public void testReloadFeaturesFromOverlay() throws Exception {
    PropertiesWriter writer = new PropertiesWriter();
    writer.setProperty("feature_1", "10000");
    writer.setProperty("feature_2", "5000");
    writer.save();
    PropertiesWriter overlayWriter = new PropertiesWriter();
    overlayWriter.setProperty("feature_2", "6000");
    overlayWriter.setProperty("feature_4", "6000");
    overlayWriter.save();
    DistributedLogConfiguration conf = new DistributedLogConfiguration().setDynamicConfigReloadIntervalSec(Integer.MAX_VALUE).setFileFeatureProviderBaseConfigPath(writer.getFile().toURI().toURL().getPath()).setFileFeatureProviderOverlayConfigPath(overlayWriter.getFile().toURI().toURL().getPath());
    DynamicConfigurationFeatureProvider provider = new DynamicConfigurationFeatureProvider("", conf, NullStatsLogger.INSTANCE);
    provider.start();
    ensureConfigReloaded();
    Feature feature1 = provider.getFeature("feature_1");
    assertTrue(feature1.isAvailable());
    assertEquals(10000, feature1.availability());
    Feature feature2 = provider.getFeature("feature_2");
    assertTrue(feature2.isAvailable());
    assertEquals(6000, feature2.availability());
    Feature feature3 = provider.getFeature("feature_3");
    assertFalse(feature3.isAvailable());
    assertEquals(0, feature3.availability());
    Feature feature4 = provider.getFeature("feature_4");
    assertTrue(feature4.isAvailable());
    assertEquals(6000, feature4.availability());
    Feature feature5 = provider.getFeature("unknown_feature");
    assertFalse(feature5.isAvailable());
    assertEquals(0, feature5.availability());
    // dynamic load config
    provider.getFeatureConf().setProperty("feature_1", 3000);
    provider.getFeatureConf().setProperty("feature_2", 7000);
    provider.getFeatureConf().setProperty("feature_3", 8000);
    provider.getFeatureConf().setProperty("feature_4", 9000);
    provider.onReload(provider.getFeatureConf());
    feature1 = provider.getFeature("feature_1");
    assertTrue(feature1.isAvailable());
    assertEquals(3000, feature1.availability());
    feature2 = provider.getFeature("feature_2");
    assertTrue(feature2.isAvailable());
    assertEquals(7000, feature2.availability());
    feature3 = provider.getFeature("feature_3");
    assertTrue(feature3.isAvailable());
    assertEquals(8000, feature3.availability());
    feature4 = provider.getFeature("feature_4");
    assertTrue(feature4.isAvailable());
    assertEquals(9000, feature4.availability());
    provider.stop();
}

Example 48 with DistributedLogConfiguration

use of com.twitter.distributedlog.DistributedLogConfiguration in project distributedlog by twitter.

the class TestZKNamespaceWatcher method testNamespaceListener.

@Test(timeout = 60000)
public void testNamespaceListener() throws Exception {
    URI uri = createDLMURI("/" + runtime.getMethodName());
    zkc.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    conf.addConfiguration(baseConf);
    ZKNamespaceWatcher watcher = new ZKNamespaceWatcher(conf, uri, zkc, scheduler);
    final CountDownLatch[] latches = new CountDownLatch[10];
    for (int i = 0; i < 10; i++) {
        latches[i] = new CountDownLatch(1);
    }
    final AtomicInteger numUpdates = new AtomicInteger(0);
    final AtomicReference<Set<String>> receivedLogs = new AtomicReference<Set<String>>(null);
    watcher.registerListener(new NamespaceListener() {

        @Override
        public void onStreamsChanged(Iterator<String> streams) {
            Set<String> streamSet = Sets.newHashSet(streams);
            int updates = numUpdates.incrementAndGet();
            receivedLogs.set(streamSet);
            latches[updates - 1].countDown();
        }
    });
    // first update
    final Set<String> expectedLogs = Sets.newHashSet();
    latches[0].await();
    validateReceivedLogs(expectedLogs, receivedLogs.get());
    // create test1
    expectedLogs.add("test1");
    createLogInNamespace(uri, "test1");
    latches[1].await();
    validateReceivedLogs(expectedLogs, receivedLogs.get());
    // create invalid log
    createLogInNamespace(uri, ".test1");
    latches[2].await();
    validateReceivedLogs(expectedLogs, receivedLogs.get());
    // create test2
    expectedLogs.add("test2");
    createLogInNamespace(uri, "test2");
    latches[3].await();
    validateReceivedLogs(expectedLogs, receivedLogs.get());
    // delete test1
    expectedLogs.remove("test1");
    deleteLogInNamespace(uri, "test1");
    latches[4].await();
    validateReceivedLogs(expectedLogs, receivedLogs.get());
}

Example 49 with DistributedLogConfiguration

use of com.twitter.distributedlog.DistributedLogConfiguration in project distributedlog by twitter.

the class TestFederatedZKLogMetadataStore method testDuplicatedLogs.

@Test(timeout = 60000)
public void testDuplicatedLogs() throws Exception {
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    conf.addConfiguration(baseConf);
    String logName = "test-log";
    FutureUtils.result(metadataStore.createLog(logName));
    URI subNs1 = FutureUtils.result(metadataStore.createSubNamespace());
    URI subNs2 = FutureUtils.result(metadataStore.createSubNamespace());
    String duplicatedLogName = "test-duplicated-logs";
    // Create same log in different sub namespaces
    metadataStore.createLogInNamespaceSync(subNs1, duplicatedLogName);
    metadataStore.createLogInNamespaceSync(subNs2, duplicatedLogName);
    try {
        FutureUtils.result(metadataStore.createLog("non-existent-log"));
        fail("should throw exception when duplicated log found");
    } catch (UnexpectedException ue) {
        // should throw unexpected exception
        assertTrue(metadataStore.duplicatedLogFound.get());
    }
    try {
        FutureUtils.result(metadataStore.getLogLocation(logName));
        fail("should throw exception when duplicated log found");
    } catch (UnexpectedException ue) {
        // should throw unexpected exception
        assertTrue(metadataStore.duplicatedLogFound.get());
    }
    try {
        FutureUtils.result(metadataStore.getLogLocation("non-existent-log"));
        fail("should throw exception when duplicated log found");
    } catch (UnexpectedException ue) {
        // should throw unexpected exception
        assertTrue(metadataStore.duplicatedLogFound.get());
    }
    try {
        FutureUtils.result(metadataStore.getLogLocation(duplicatedLogName));
        fail("should throw exception when duplicated log found");
    } catch (UnexpectedException ue) {
        // should throw unexpected exception
        assertTrue(metadataStore.duplicatedLogFound.get());
    }
    try {
        FutureUtils.result(metadataStore.getLogs());
        fail("should throw exception when duplicated log found");
    } catch (UnexpectedException ue) {
        // should throw unexpected exception
        assertTrue(metadataStore.duplicatedLogFound.get());
    }
}

Example 50 with DistributedLogConfiguration

use of com.twitter.distributedlog.DistributedLogConfiguration in project distributedlog by twitter.

the class TestFederatedZKLogMetadataStore method testZooKeeperSessionExpired.

@Test(timeout = 60000)
public void testZooKeeperSessionExpired() throws Exception {
    Set<String> allLogs = createLogs(2 * maxLogsPerSubnamespace, "test-zookeeper-session-expired-");
    TestNamespaceListenerWithExpectedSize listener = new TestNamespaceListenerWithExpectedSize(2 * maxLogsPerSubnamespace + 1);
    metadataStore.registerNamespaceListener(listener);
    ZooKeeperClientUtils.expireSession(zkc, DLUtils.getZKServersFromDLUri(uri), zkSessionTimeoutMs);
    String testLogName = "test-log-name";
    allLogs.add(testLogName);
    DistributedLogConfiguration anotherConf = new DistributedLogConfiguration();
    anotherConf.addConfiguration(baseConf);
    ZooKeeperClient anotherZkc = TestZooKeeperClientBuilder.newBuilder().uri(uri).sessionTimeoutMs(zkSessionTimeoutMs).build();
    FederatedZKLogMetadataStore anotherMetadataStore = new FederatedZKLogMetadataStore(anotherConf, uri, anotherZkc, scheduler);
    FutureUtils.result(anotherMetadataStore.createLog(testLogName));
    listener.waitForDone();
    Set<String> receivedLogs = listener.getResult();
    assertEquals(2 * maxLogsPerSubnamespace + 1, receivedLogs.size());
    assertEquals(allLogs, receivedLogs);
}