Examples with Namespace org.apache.distributedlog.api.namespace.Namespace used on opensource projects

Example 1 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project bookkeeper by apache.

the class TestAsyncReaderLock method testReaderLockMultiReadersScenario.

@Test(timeout = 60000)
public void testReaderLockMultiReadersScenario() throws Exception {
    final String name = runtime.getMethodName();
    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    DistributedLogConfiguration localConf = new DistributedLogConfiguration();
    localConf.addConfiguration(conf);
    localConf.setImmediateFlushEnabled(false);
    localConf.setPeriodicFlushFrequencyMilliSeconds(60 * 1000);
    localConf.setOutputBufferSize(0);
    // Otherwise, we won't be able to run scheduled threads for readahead when we're in a callback.
    localConf.setNumWorkerThreads(2);
    localConf.setLockTimeout(Long.MAX_VALUE);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(localConf).uri(uri).clientId("main").build();
    DistributedLogManager dlm0 = namespace.openLog(name);
    DLMTestUtil.generateCompletedLogSegments(dlm0, localConf, 9, 100);
    dlm0.close();
    int recordCount = 0;
    AtomicReference<DLSN> currentDLSN = new AtomicReference<DLSN>(DLSN.InitialDLSN);
    String clientId1 = "reader1";
    Namespace namespace1 = NamespaceBuilder.newBuilder().conf(localConf).uri(uri).clientId(clientId1).build();
    DistributedLogManager dlm1 = namespace1.openLog(name);
    String clientId2 = "reader2";
    Namespace namespace2 = NamespaceBuilder.newBuilder().conf(localConf).uri(uri).clientId(clientId2).build();
    DistributedLogManager dlm2 = namespace2.openLog(name);
    String clientId3 = "reader3";
    Namespace namespace3 = NamespaceBuilder.newBuilder().conf(localConf).uri(uri).clientId(clientId3).build();
    DistributedLogManager dlm3 = namespace3.openLog(name);
    LOG.info("{} is opening reader on stream {}", clientId1, name);
    CompletableFuture<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    AsyncLogReader reader1 = Utils.ioResult(futureReader1);
    LOG.info("{} opened reader on stream {}", clientId1, name);
    LOG.info("{} is opening reader on stream {}", clientId2, name);
    CompletableFuture<AsyncLogReader> futureReader2 = dlm2.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    LOG.info("{} is opening reader on stream {}", clientId3, name);
    CompletableFuture<AsyncLogReader> futureReader3 = dlm3.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    ExecutorService executorService = Executors.newCachedThreadPool();
    ReadRecordsListener listener2 = new ReadRecordsListener(currentDLSN, clientId2, executorService);
    ReadRecordsListener listener3 = new ReadRecordsListener(currentDLSN, clientId3, executorService);
    futureReader2.whenComplete(listener2);
    futureReader3.whenComplete(listener3);
    // Get reader1 and start reading.
    for (; recordCount < 200; recordCount++) {
        LogRecordWithDLSN record = Utils.ioResult(reader1.readNext());
        currentDLSN.set(record.getDlsn());
    }
    // Take a break, reader2 decides to stop waiting and cancels.
    Thread.sleep(1000);
    assertFalse(listener2.done());
    futureReader2.cancel(true);
    listener2.getLatch().await();
    assertTrue(listener2.done());
    assertTrue(listener2.failed());
    // Reader1 starts reading again.
    for (; recordCount < 300; recordCount++) {
        LogRecordWithDLSN record = Utils.ioResult(reader1.readNext());
        currentDLSN.set(record.getDlsn());
    }
    // Reader1 is done, someone else can take over. Since reader2 was
    // aborted, reader3 should take its place.
    assertFalse(listener3.done());
    Utils.close(reader1);
    listener3.getLatch().await();
    assertTrue(listener3.done());
    assertFalse(listener3.failed());
    assertEquals(new DLSN(3, 99, 0), currentDLSN.get());
    try {
        Utils.ioResult(futureReader2);
    } catch (Exception ex) {
    // Can't get this one to close it--the dlm will take care of it.
    }
    Utils.close(Utils.ioResult(futureReader3));
    dlm1.close();
    namespace1.close();
    dlm2.close();
    namespace2.close();
    dlm3.close();
    namespace3.close();
    executorService.shutdown();
}

Example 2 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project bookkeeper by apache.

the class TestAsyncReaderWriter method testCreateLogStreamWithDifferentReplicationFactor.

@Test(timeout = 60000)
public void testCreateLogStreamWithDifferentReplicationFactor() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    ConcurrentBaseConfiguration baseConf = new ConcurrentConstConfiguration(confLocal);
    DynamicDistributedLogConfiguration dynConf = new DynamicDistributedLogConfiguration(baseConf);
    dynConf.setProperty(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE, DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT - 1);
    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(confLocal).uri(uri).build();
    // use the pool
    DistributedLogManager dlm = namespace.openLog(name + "-pool");
    AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(1, segments.size());
    long ledgerId = segments.get(0).getLogSegmentId();
    LedgerHandle lh = ((BKNamespaceDriver) namespace.getNamespaceDriver()).getReaderBKC().get().openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, confLocal.getBKDigestPW().getBytes(UTF_8));
    LedgerMetadata metadata = BookKeeperAccessor.getLedgerMetadata(lh);
    assertEquals(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT, metadata.getEnsembleSize());
    lh.close();
    Utils.close(writer);
    dlm.close();
    // use customized configuration
    dlm = namespace.openLog(name + "-custom", java.util.Optional.empty(), java.util.Optional.of(dynConf), java.util.Optional.empty());
    writer = dlm.startAsyncLogSegmentNonPartitioned();
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
    segments = dlm.getLogSegments();
    assertEquals(1, segments.size());
    ledgerId = segments.get(0).getLogSegmentId();
    lh = ((BKNamespaceDriver) namespace.getNamespaceDriver()).getReaderBKC().get().openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, confLocal.getBKDigestPW().getBytes(UTF_8));
    metadata = BookKeeperAccessor.getLedgerMetadata(lh);
    assertEquals(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT - 1, metadata.getEnsembleSize());
    lh.close();
    Utils.close(writer);
    dlm.close();
    namespace.close();
}

Example 3 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project bookkeeper by apache.

the class TestBKDistributedLogNamespace method testUseNamespaceAfterCloseShouldFailFast.

@Test(timeout = 60000)
public void testUseNamespaceAfterCloseShouldFailFast() throws Exception {
    URI uri = createDLMURI("/" + runtime.getMethodName());
    Namespace namespace = NamespaceBuilder.newBuilder().conf(conf).uri(uri).build();
    // before closing the namespace, no exception should be thrown
    String logName = "test-stream";
    // create a log
    namespace.createLog(logName);
    // log exists
    Assert.assertTrue(namespace.logExists(logName));
    // create a dlm
    DistributedLogManager dlm = namespace.openLog(logName);
    // do some writes
    BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
    for (long i = 0; i < 3; i++) {
        LogRecord record = DLMTestUtil.getLargeLogRecordInstance(i);
        writer.write(record);
    }
    writer.closeAndComplete();
    // do some reads
    LogReader reader = dlm.getInputStream(0);
    for (long i = 0; i < 3; i++) {
        Assert.assertEquals(reader.readNext(false).getTransactionId(), i);
    }
    namespace.deleteLog(logName);
    Assert.assertFalse(namespace.logExists(logName));
    // now try to close the namespace
    namespace.close();
    try {
        namespace.createLog(logName);
        fail("Should throw exception after namespace is closed");
    } catch (AlreadyClosedException e) {
    // No-ops
    }
    try {
        namespace.openLog(logName);
        fail("Should throw exception after namespace is closed");
    } catch (AlreadyClosedException e) {
    // No-ops
    }
    try {
        namespace.logExists(logName);
        fail("Should throw exception after namespace is closed");
    } catch (AlreadyClosedException e) {
    // No-ops
    }
    try {
        namespace.getLogs();
        fail("Should throw exception after namespace is closed");
    } catch (AlreadyClosedException e) {
    // No-ops
    }
    try {
        namespace.deleteLog(logName);
        fail("Should throw exception after namespace is closed");
    } catch (AlreadyClosedException e) {
    // No-ops
    }
    try {
        namespace.createAccessControlManager();
        fail("Should throw exception after namespace is closed");
    } catch (AlreadyClosedException e) {
    // No-ops
    }
}

Example 4 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project bookkeeper by apache.

the class TestBKDistributedLogNamespace method testInvalidStreamName.

@Test(timeout = 60000)
public void testInvalidStreamName() throws Exception {
    assertFalse(DLUtils.isReservedStreamName("test"));
    assertTrue(DLUtils.isReservedStreamName(".test"));
    URI uri = createDLMURI("/" + runtime.getMethodName());
    Namespace namespace = NamespaceBuilder.newBuilder().conf(conf).uri(uri).build();
    try {
        namespace.openLog(".test1");
        fail("Should fail to create invalid stream .test");
    } catch (InvalidStreamNameException isne) {
    // expected
    }
    DistributedLogManager dlm = namespace.openLog("test1");
    LogWriter writer = dlm.startLogSegmentNonPartitioned();
    writer.write(DLMTestUtil.getLogRecordInstance(1));
    writer.close();
    dlm.close();
    try {
        namespace.openLog(".test2");
        fail("Should fail to create invalid stream .test2");
    } catch (InvalidStreamNameException isne) {
    // expected
    }
    try {
        namespace.openLog("/ test2");
        fail("should fail to create invalid stream / test2");
    } catch (InvalidStreamNameException isne) {
    // expected
    }
    try {
        char[] chars = new char[6];
        for (int i = 0; i < chars.length; i++) {
            chars[i] = 'a';
        }
        chars[0] = 0;
        String streamName = new String(chars);
        namespace.openLog(streamName);
        fail("should fail to create invalid stream " + streamName);
    } catch (InvalidStreamNameException isne) {
    // expected
    }
    try {
        char[] chars = new char[6];
        for (int i = 0; i < chars.length; i++) {
            chars[i] = 'a';
        }
        chars[3] = '\u0010';
        String streamName = new String(chars);
        namespace.openLog(streamName);
        fail("should fail to create invalid stream " + streamName);
    } catch (InvalidStreamNameException isne) {
    // expected
    }
    DistributedLogManager newDLM = namespace.openLog("test_2-3");
    LogWriter newWriter = newDLM.startLogSegmentNonPartitioned();
    newWriter.write(DLMTestUtil.getLogRecordInstance(1));
    newWriter.close();
    newDLM.close();
    Iterator<String> streamIter = namespace.getLogs();
    Set<String> streamSet = Sets.newHashSet(streamIter);
    assertEquals(2, streamSet.size());
    assertTrue(streamSet.contains("test1"));
    assertTrue(streamSet.contains("test_2-3"));
    namespace.close();
}

Example 5 with Namespace

use of org.apache.distributedlog.api.namespace.Namespace in project bookkeeper by apache.

the class TestBKDistributedLogNamespace method initDlogMeta.

private void initDlogMeta(String dlNamespace, String un, String streamName) throws Exception {
    URI uri = createDLMURI(dlNamespace);
    DistributedLogConfiguration newConf = new DistributedLogConfiguration();
    newConf.addConfiguration(conf);
    newConf.setCreateStreamIfNotExists(true);
    newConf.setZkAclId(un);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(newConf).uri(uri).build();
    DistributedLogManager dlm = namespace.openLog(streamName);
    LogWriter writer = dlm.startLogSegmentNonPartitioned();
    for (int i = 0; i < 10; i++) {
        writer.write(DLMTestUtil.getLogRecordInstance(1L));
    }
    writer.close();
    dlm.close();
    namespace.close();
}