Examples with AsyncLogWriter org.apache.distributedlog.api.AsyncLogWriter used on opensource projects

Example 6 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestTruncate method testPurgeLogs.

@Test(timeout = 60000)
public void testPurgeLogs() throws Exception {
    String name = "distrlog-purge-logs";
    URI uri = createDLMURI("/" + name);
    populateData(new HashMap<Long, DLSN>(), conf, name, 10, 10, false);
    DistributedLogManager distributedLogManager = createNewDLM(conf, name);
    List<LogSegmentMetadata> segments = distributedLogManager.getLogSegments();
    LOG.info("Segments before modifying completion time : {}", segments);
    ZooKeeperClient zkc = TestZooKeeperClientBuilder.newBuilder(conf).uri(uri).build();
    // Update completion time of first 5 segments
    long newTimeMs = System.currentTimeMillis() - 60 * 60 * 1000 * 2;
    for (int i = 0; i < 5; i++) {
        LogSegmentMetadata segment = segments.get(i);
        updateCompletionTime(zkc, segment, newTimeMs + i);
    }
    zkc.close();
    segments = distributedLogManager.getLogSegments();
    LOG.info("Segments after modifying completion time : {}", segments);
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setRetentionPeriodHours(1);
    confLocal.setExplicitTruncationByApplication(false);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
    long txid = 1 + 10 * 10;
    for (int j = 1; j <= 10; j++) {
        Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txid++)));
    }
    // wait until truncation task to be completed.
    BKAsyncLogWriter bkLogWriter = (BKAsyncLogWriter) writer;
    CompletableFuture<List<LogSegmentMetadata>> truncationAttempt = bkLogWriter.getLastTruncationAttempt();
    while (truncationAttempt == null || !truncationAttempt.isDone()) {
        TimeUnit.MILLISECONDS.sleep(20);
        truncationAttempt = bkLogWriter.getLastTruncationAttempt();
    }
    assertEquals(6, distributedLogManager.getLogSegments().size());
    Utils.close(writer);
    dlm.close();
    distributedLogManager.close();
}

Example 7 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestTruncate method populateData.

private Pair<DistributedLogManager, AsyncLogWriter> populateData(Map<Long, DLSN> txid2DLSN, DistributedLogConfiguration confLocal, String name, int numLogSegments, int numEntriesPerLogSegment, boolean createInprogressLogSegment) throws Exception {
    long txid = 1;
    for (long i = 1; i <= numLogSegments; i++) {
        LOG.info("Writing Log Segment {}.", i);
        DistributedLogManager dlm = createNewDLM(confLocal, name);
        AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
        for (int j = 1; j <= numEntriesPerLogSegment; j++) {
            long curTxId = txid++;
            DLSN dlsn = Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(curTxId)));
            txid2DLSN.put(curTxId, dlsn);
        }
        Utils.close(writer);
        dlm.close();
    }
    if (createInprogressLogSegment) {
        DistributedLogManager dlm = createNewDLM(confLocal, name);
        AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
        for (int j = 1; j <= 10; j++) {
            long curTxId = txid++;
            DLSN dlsn = Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(curTxId)));
            txid2DLSN.put(curTxId, dlsn);
        }
        return new ImmutablePair<DistributedLogManager, AsyncLogWriter>(dlm, writer);
    } else {
        return null;
    }
}

Example 8 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestAsyncReaderWriter method testMaxReadAheadRecords.

@Test(timeout = 60000)
public void testMaxReadAheadRecords() throws Exception {
    int maxRecords = 1;
    int batchSize = 8;
    int maxAllowedCachedRecords = maxRecords + batchSize - 1;
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(Integer.MAX_VALUE);
    confLocal.setReadAheadMaxRecords(maxRecords);
    confLocal.setReadAheadBatchSize(batchSize);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
    int numRecords = 40;
    for (int i = 1; i <= numRecords; i++) {
        Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(i)));
        assertEquals("last tx id should become " + i, i, writer.getLastTxId());
    }
    LogRecord record = DLMTestUtil.getLogRecordInstance(numRecords);
    record.setControl();
    Utils.ioResult(writer.write(record));
    BKAsyncLogReader reader = (BKAsyncLogReader) dlm.getAsyncLogReader(DLSN.InitialDLSN);
    record = Utils.ioResult(reader.readNext());
    LOG.info("Read record {}", record);
    assertEquals(1L, record.getTransactionId());
    assertNotNull(reader.getReadAheadReader());
    assertTrue(reader.getReadAheadReader().getNumCachedEntries() <= maxAllowedCachedRecords);
    for (int i = 2; i <= numRecords; i++) {
        record = Utils.ioResult(reader.readNext());
        LOG.info("Read record {}", record);
        assertEquals((long) i, record.getTransactionId());
        TimeUnit.MILLISECONDS.sleep(20);
        int numCachedEntries = reader.getReadAheadReader().getNumCachedEntries();
        assertTrue("Should cache less than " + batchSize + " records but already found " + numCachedEntries + " records when reading " + i + "th record", numCachedEntries <= maxAllowedCachedRecords);
    }
    Utils.close(reader);
}

Example 9 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestAsyncReaderWriter method testCreateLogStreamWithDifferentReplicationFactor.

@Test(timeout = 60000)
public void testCreateLogStreamWithDifferentReplicationFactor() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    ConcurrentBaseConfiguration baseConf = new ConcurrentConstConfiguration(confLocal);
    DynamicDistributedLogConfiguration dynConf = new DynamicDistributedLogConfiguration(baseConf);
    dynConf.setProperty(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE, DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT - 1);
    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(confLocal).uri(uri).build();
    // use the pool
    DistributedLogManager dlm = namespace.openLog(name + "-pool");
    AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(1, segments.size());
    long ledgerId = segments.get(0).getLogSegmentId();
    LedgerHandle lh = ((BKNamespaceDriver) namespace.getNamespaceDriver()).getReaderBKC().get().openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, confLocal.getBKDigestPW().getBytes(UTF_8));
    LedgerMetadata metadata = BookKeeperAccessor.getLedgerMetadata(lh);
    assertEquals(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT, metadata.getEnsembleSize());
    lh.close();
    Utils.close(writer);
    dlm.close();
    // use customized configuration
    dlm = namespace.openLog(name + "-custom", java.util.Optional.empty(), java.util.Optional.of(dynConf), java.util.Optional.empty());
    writer = dlm.startAsyncLogSegmentNonPartitioned();
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
    segments = dlm.getLogSegments();
    assertEquals(1, segments.size());
    ledgerId = segments.get(0).getLogSegmentId();
    lh = ((BKNamespaceDriver) namespace.getNamespaceDriver()).getReaderBKC().get().openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, confLocal.getBKDigestPW().getBytes(UTF_8));
    metadata = BookKeeperAccessor.getLedgerMetadata(lh);
    assertEquals(DistributedLogConfiguration.BKDL_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT - 1, metadata.getEnsembleSize());
    lh.close();
    Utils.close(writer);
    dlm.close();
    namespace.close();
}

Example 10 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestAsyncReaderWriter method testGetLastTxId.

@Test(timeout = 60000)
public void testGetLastTxId() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(true);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
    int numRecords = 10;
    for (int i = 0; i < numRecords; i++) {
        Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(i)));
        assertEquals("last tx id should become " + i, i, writer.getLastTxId());
    }
    // open a writer to recover the inprogress log segment
    AsyncLogWriter recoverWriter = dlm.startAsyncLogSegmentNonPartitioned();
    assertEquals("recovered last tx id should be " + (numRecords - 1), numRecords - 1, recoverWriter.getLastTxId());
}