Examples with DistributedLogManager org.apache.distributedlog.api.DistributedLogManager used on opensource projects

Example 51 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderWriter method testBulkReadNotWaitingMoreRecords.

@Test(timeout = 60000)
public void testBulkReadNotWaitingMoreRecords() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned();
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
    LogRecord controlRecord = DLMTestUtil.getLogRecordInstance(1L);
    controlRecord.setControl();
    Utils.ioResult(writer.write(controlRecord));
    BKAsyncLogReader reader = (BKAsyncLogReader) dlm.getAsyncLogReader(DLSN.InitialDLSN);
    CompletableFuture<List<LogRecordWithDLSN>> bulkReadFuture = reader.readBulk(2, 0, TimeUnit.MILLISECONDS);
    CompletableFuture<LogRecordWithDLSN> readFuture = reader.readNext();
    List<LogRecordWithDLSN> bulkReadRecords = Utils.ioResult(bulkReadFuture);
    assertEquals(1, bulkReadRecords.size());
    assertEquals(1L, bulkReadRecords.get(0).getTransactionId());
    for (LogRecordWithDLSN record : bulkReadRecords) {
        DLMTestUtil.verifyLogRecord(record);
    }
    // write another records
    for (int i = 0; i < 5; i++) {
        long txid = 2L + i;
        Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txid)));
        controlRecord = DLMTestUtil.getLogRecordInstance(txid);
        controlRecord.setControl();
        Utils.ioResult(writer.write(controlRecord));
    }
    LogRecordWithDLSN record = Utils.ioResult(readFuture);
    assertEquals(2L, record.getTransactionId());
    DLMTestUtil.verifyLogRecord(record);
    Utils.close(reader);
    writer.close();
    dlm.close();
}

Example 52 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderWriter method testReadBrokenEntries.

@Test(timeout = 60000)
public void testReadBrokenEntries() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(1);
    confLocal.setPositionGapDetectionEnabled(false);
    confLocal.setReadAheadSkipBrokenEntries(true);
    confLocal.setEIInjectReadAheadBrokenEntries(true);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    int numLogSegments = 3;
    int numRecordsPerLogSegment = 10;
    long txid = 1L;
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false);
    AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);
    // record in each ledger is discarded, for 30 - 3 = 27 records.
    for (int i = 0; i < 27; i++) {
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        assertFalse(record.getDlsn().getEntryId() % 10 == 0);
    }
    Utils.close(reader);
    dlm.close();
}

Example 53 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestBKDistributedLogNamespace method testUseNamespaceAfterCloseShouldFailFast.

@Test(timeout = 60000)
public void testUseNamespaceAfterCloseShouldFailFast() throws Exception {
    URI uri = createDLMURI("/" + runtime.getMethodName());
    Namespace namespace = NamespaceBuilder.newBuilder().conf(conf).uri(uri).build();
    // before closing the namespace, no exception should be thrown
    String logName = "test-stream";
    // create a log
    namespace.createLog(logName);
    // log exists
    Assert.assertTrue(namespace.logExists(logName));
    // create a dlm
    DistributedLogManager dlm = namespace.openLog(logName);
    // do some writes
    BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());
    for (long i = 0; i < 3; i++) {
        LogRecord record = DLMTestUtil.getLargeLogRecordInstance(i);
        writer.write(record);
    }
    writer.closeAndComplete();
    // do some reads
    LogReader reader = dlm.getInputStream(0);
    for (long i = 0; i < 3; i++) {
        Assert.assertEquals(reader.readNext(false).getTransactionId(), i);
    }
    namespace.deleteLog(logName);
    Assert.assertFalse(namespace.logExists(logName));
    // now try to close the namespace
    namespace.close();
    try {
        namespace.createLog(logName);
        fail("Should throw exception after namespace is closed");
    } catch (AlreadyClosedException e) {
    // No-ops
    }
    try {
        namespace.openLog(logName);
        fail("Should throw exception after namespace is closed");
    } catch (AlreadyClosedException e) {
    // No-ops
    }
    try {
        namespace.logExists(logName);
        fail("Should throw exception after namespace is closed");
    } catch (AlreadyClosedException e) {
    // No-ops
    }
    try {
        namespace.getLogs();
        fail("Should throw exception after namespace is closed");
    } catch (AlreadyClosedException e) {
    // No-ops
    }
    try {
        namespace.deleteLog(logName);
        fail("Should throw exception after namespace is closed");
    } catch (AlreadyClosedException e) {
    // No-ops
    }
    try {
        namespace.createAccessControlManager();
        fail("Should throw exception after namespace is closed");
    } catch (AlreadyClosedException e) {
    // No-ops
    }
}

Example 54 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestBKDistributedLogNamespace method testInvalidStreamName.

@Test(timeout = 60000)
public void testInvalidStreamName() throws Exception {
    assertFalse(DLUtils.isReservedStreamName("test"));
    assertTrue(DLUtils.isReservedStreamName(".test"));
    URI uri = createDLMURI("/" + runtime.getMethodName());
    Namespace namespace = NamespaceBuilder.newBuilder().conf(conf).uri(uri).build();
    try {
        namespace.openLog(".test1");
        fail("Should fail to create invalid stream .test");
    } catch (InvalidStreamNameException isne) {
    // expected
    }
    DistributedLogManager dlm = namespace.openLog("test1");
    LogWriter writer = dlm.startLogSegmentNonPartitioned();
    writer.write(DLMTestUtil.getLogRecordInstance(1));
    writer.close();
    dlm.close();
    try {
        namespace.openLog(".test2");
        fail("Should fail to create invalid stream .test2");
    } catch (InvalidStreamNameException isne) {
    // expected
    }
    try {
        namespace.openLog("/ test2");
        fail("should fail to create invalid stream / test2");
    } catch (InvalidStreamNameException isne) {
    // expected
    }
    try {
        char[] chars = new char[6];
        for (int i = 0; i < chars.length; i++) {
            chars[i] = 'a';
        }
        chars[0] = 0;
        String streamName = new String(chars);
        namespace.openLog(streamName);
        fail("should fail to create invalid stream " + streamName);
    } catch (InvalidStreamNameException isne) {
    // expected
    }
    try {
        char[] chars = new char[6];
        for (int i = 0; i < chars.length; i++) {
            chars[i] = 'a';
        }
        chars[3] = '\u0010';
        String streamName = new String(chars);
        namespace.openLog(streamName);
        fail("should fail to create invalid stream " + streamName);
    } catch (InvalidStreamNameException isne) {
    // expected
    }
    DistributedLogManager newDLM = namespace.openLog("test_2-3");
    LogWriter newWriter = newDLM.startLogSegmentNonPartitioned();
    newWriter.write(DLMTestUtil.getLogRecordInstance(1));
    newWriter.close();
    newDLM.close();
    Iterator<String> streamIter = namespace.getLogs();
    Set<String> streamSet = Sets.newHashSet(streamIter);
    assertEquals(2, streamSet.size());
    assertTrue(streamSet.contains("test1"));
    assertTrue(streamSet.contains("test_2-3"));
    namespace.close();
}

Example 55 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestBKDistributedLogNamespace method initDlogMeta.

private void initDlogMeta(String dlNamespace, String un, String streamName) throws Exception {
    URI uri = createDLMURI(dlNamespace);
    DistributedLogConfiguration newConf = new DistributedLogConfiguration();
    newConf.addConfiguration(conf);
    newConf.setCreateStreamIfNotExists(true);
    newConf.setZkAclId(un);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(newConf).uri(uri).build();
    DistributedLogManager dlm = namespace.openLog(streamName);
    LogWriter writer = dlm.startLogSegmentNonPartitioned();
    for (int i = 0; i < 10; i++) {
        writer.write(DLMTestUtil.getLogRecordInstance(1L));
    }
    writer.close();
    dlm.close();
    namespace.close();
}