Examples with DistributedLogConfiguration org.apache.distributedlog.DistributedLogConfiguration used on opensource projects

Example 11 with DistributedLogConfiguration

use of org.apache.distributedlog.DistributedLogConfiguration in project bookkeeper by apache.

the class TestFederatedZKLogMetadataStore method testDuplicatedLogs.

@Test(timeout = 60000)
public void testDuplicatedLogs() throws Exception {
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    conf.addConfiguration(baseConf);
    String logName = "test-log";
    Utils.ioResult(metadataStore.createLog(logName));
    URI subNs1 = Utils.ioResult(metadataStore.createSubNamespace());
    URI subNs2 = Utils.ioResult(metadataStore.createSubNamespace());
    String duplicatedLogName = "test-duplicated-logs";
    // Create same log in different sub namespaces
    metadataStore.createLogInNamespaceSync(subNs1, duplicatedLogName);
    metadataStore.createLogInNamespaceSync(subNs2, duplicatedLogName);
    try {
        Utils.ioResult(metadataStore.createLog("non-existent-log"));
        fail("should throw exception when duplicated log found");
    } catch (UnexpectedException ue) {
        // should throw unexpected exception
        assertTrue(metadataStore.duplicatedLogFound.get());
    }
    try {
        Utils.ioResult(metadataStore.getLogLocation(logName));
        fail("should throw exception when duplicated log found");
    } catch (UnexpectedException ue) {
        // should throw unexpected exception
        assertTrue(metadataStore.duplicatedLogFound.get());
    }
    try {
        Utils.ioResult(metadataStore.getLogLocation("non-existent-log"));
        fail("should throw exception when duplicated log found");
    } catch (UnexpectedException ue) {
        // should throw unexpected exception
        assertTrue(metadataStore.duplicatedLogFound.get());
    }
    try {
        Utils.ioResult(metadataStore.getLogLocation(duplicatedLogName));
        fail("should throw exception when duplicated log found");
    } catch (UnexpectedException ue) {
        // should throw unexpected exception
        assertTrue(metadataStore.duplicatedLogFound.get());
    }
    try {
        Utils.ioResult(metadataStore.getLogs(""));
        fail("should throw exception when duplicated log found");
    } catch (UnexpectedException ue) {
        // should throw unexpected exception
        assertTrue(metadataStore.duplicatedLogFound.get());
    }
}

Example 12 with DistributedLogConfiguration

use of org.apache.distributedlog.DistributedLogConfiguration in project bookkeeper by apache.

the class TestDLCK method testCheckAndRepairDLNamespace.

@Test(timeout = 60000)
@SuppressWarnings("deprecation")
public void testCheckAndRepairDLNamespace() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setOutputBufferSize(0);
    confLocal.setLogSegmentSequenceNumberValidationEnabled(false);
    confLocal.setLogSegmentCacheEnabled(false);
    URI uri = createDLMURI("/check-and-repair-dl-namespace");
    zkc.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(confLocal).uri(uri).build();
    OrderedScheduler scheduler = OrderedScheduler.newSchedulerBuilder().name("dlck-tool").numThreads(1).build();
    ExecutorService executorService = Executors.newCachedThreadPool();
    String streamName = "check-and-repair-dl-namespace";
    // Create completed log segments
    DistributedLogManager dlm = namespace.openLog(streamName);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 1L, 1L, 10, false);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 2L, 11L, 10, true);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 3L, 21L, 10, false);
    DLMTestUtil.injectLogSegmentWithLastDLSN(dlm, confLocal, 4L, 31L, 10, true);
    // dryrun
    DistributedLogAdmin.checkAndRepairDLNamespace(uri, namespace, new DryrunLogSegmentMetadataStoreUpdater(confLocal, getLogSegmentMetadataStore(namespace)), scheduler, false, false);
    Map<Long, LogSegmentMetadata> segments = getLogSegments(dlm);
    LOG.info("segments after drynrun {}", segments);
    verifyLogSegment(segments, new DLSN(1L, 18L, 0L), 1L, 10, 10L);
    verifyLogSegment(segments, new DLSN(2L, 16L, 0L), 2L, 9, 19L);
    verifyLogSegment(segments, new DLSN(3L, 18L, 0L), 3L, 10, 30L);
    verifyLogSegment(segments, new DLSN(4L, 16L, 0L), 4L, 9, 39L);
    // check and repair
    DistributedLogAdmin.checkAndRepairDLNamespace(uri, namespace, LogSegmentMetadataStoreUpdater.createMetadataUpdater(confLocal, getLogSegmentMetadataStore(namespace)), scheduler, false, false);
    segments = getLogSegments(dlm);
    LOG.info("segments after repair {}", segments);
    verifyLogSegment(segments, new DLSN(1L, 18L, 0L), 1L, 10, 10L);
    verifyLogSegment(segments, new DLSN(2L, 18L, 0L), 2L, 10, 20L);
    verifyLogSegment(segments, new DLSN(3L, 18L, 0L), 3L, 10, 30L);
    verifyLogSegment(segments, new DLSN(4L, 18L, 0L), 4L, 10, 40L);
    dlm.close();
    SchedulerUtils.shutdownScheduler(executorService, 5, TimeUnit.MINUTES);
    namespace.close();
}

Example 13 with DistributedLogConfiguration

use of org.apache.distributedlog.DistributedLogConfiguration in project bookkeeper by apache.

the class TestDynamicConfigurationFeatureProvider method testReloadFeaturesFromOverlay.

@Test(timeout = 60000)
public void testReloadFeaturesFromOverlay() throws Exception {
    PropertiesWriter writer = new PropertiesWriter();
    writer.setProperty("feature_1", "10000");
    writer.setProperty("feature_2", "5000");
    writer.save();
    PropertiesWriter overlayWriter = new PropertiesWriter();
    overlayWriter.setProperty("feature_2", "6000");
    overlayWriter.setProperty("feature_4", "6000");
    overlayWriter.save();
    DistributedLogConfiguration conf = new DistributedLogConfiguration().setDynamicConfigReloadIntervalSec(Integer.MAX_VALUE).setFileFeatureProviderBaseConfigPath(writer.getFile().toURI().toURL().getPath()).setFileFeatureProviderOverlayConfigPath(overlayWriter.getFile().toURI().toURL().getPath());
    DynamicConfigurationFeatureProvider provider = new DynamicConfigurationFeatureProvider("", conf, NullStatsLogger.INSTANCE);
    provider.start();
    ensureConfigReloaded();
    Feature feature1 = provider.getFeature("feature_1");
    assertTrue(feature1.isAvailable());
    assertEquals(10000, feature1.availability());
    Feature feature2 = provider.getFeature("feature_2");
    assertTrue(feature2.isAvailable());
    assertEquals(6000, feature2.availability());
    Feature feature3 = provider.getFeature("feature_3");
    assertFalse(feature3.isAvailable());
    assertEquals(0, feature3.availability());
    Feature feature4 = provider.getFeature("feature_4");
    assertTrue(feature4.isAvailable());
    assertEquals(6000, feature4.availability());
    Feature feature5 = provider.getFeature("unknown_feature");
    assertFalse(feature5.isAvailable());
    assertEquals(0, feature5.availability());
    // dynamic load config
    provider.getFeatureConf().setProperty("feature_1", 3000);
    provider.getFeatureConf().setProperty("feature_2", 7000);
    provider.getFeatureConf().setProperty("feature_3", 8000);
    provider.getFeatureConf().setProperty("feature_4", 9000);
    provider.onReload(provider.getFeatureConf());
    feature1 = provider.getFeature("feature_1");
    assertTrue(feature1.isAvailable());
    assertEquals(3000, feature1.availability());
    feature2 = provider.getFeature("feature_2");
    assertTrue(feature2.isAvailable());
    assertEquals(7000, feature2.availability());
    feature3 = provider.getFeature("feature_3");
    assertTrue(feature3.isAvailable());
    assertEquals(8000, feature3.availability());
    feature4 = provider.getFeature("feature_4");
    assertTrue(feature4.isAvailable());
    assertEquals(9000, feature4.availability());
    provider.stop();
}

Example 14 with DistributedLogConfiguration

use of org.apache.distributedlog.DistributedLogConfiguration in project bookkeeper by apache.

the class TestZKNamespaceWatcher method testSessionExpired.

@Test(timeout = 60000)
public void testSessionExpired() throws Exception {
    URI uri = createDLMURI("/" + runtime.getMethodName());
    zkc.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    conf.addConfiguration(baseConf);
    ZKNamespaceWatcher watcher = new ZKNamespaceWatcher(conf, uri, zkc, scheduler);
    final CountDownLatch[] latches = new CountDownLatch[10];
    for (int i = 0; i < 10; i++) {
        latches[i] = new CountDownLatch(1);
    }
    final AtomicInteger numUpdates = new AtomicInteger(0);
    final AtomicReference<Set<String>> receivedLogs = new AtomicReference<Set<String>>(null);
    watcher.registerListener(new NamespaceListener() {

        @Override
        public void onStreamsChanged(Iterator<String> streams) {
            Set<String> streamSet = Sets.newHashSet(streams);
            int updates = numUpdates.incrementAndGet();
            receivedLogs.set(streamSet);
            latches[updates - 1].countDown();
        }
    });
    latches[0].await();
    createLogInNamespace(uri, "test1");
    latches[1].await();
    createLogInNamespace(uri, "test2");
    latches[2].await();
    assertEquals(2, receivedLogs.get().size());
    ZooKeeperClientUtils.expireSession(zkc, BKNamespaceDriver.getZKServersFromDLUri(uri), zkSessionTimeoutMs);
    latches[3].await();
    assertEquals(2, receivedLogs.get().size());
    createLogInNamespace(uri, "test3");
    latches[4].await();
    assertEquals(3, receivedLogs.get().size());
}

Example 15 with DistributedLogConfiguration

use of org.apache.distributedlog.DistributedLogConfiguration in project bookkeeper by apache.

the class TestZKAccessControlManager method setup.

@Before
public void setup() throws Exception {
    executorService = Executors.newSingleThreadScheduledExecutor();
    zkc = TestZooKeeperClientBuilder.newBuilder().uri(createURI("/")).build();
    conf = new DistributedLogConfiguration();
}