Examples with AsyncLogWriter org.apache.distributedlog.api.AsyncLogWriter used on opensource projects

Example 1 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestBKLogReadHandler method testGetLogRecordCountWithSingleInProgressLedger.

@Test(timeout = 60000)
public void testGetLogRecordCountWithSingleInProgressLedger() throws Exception {
    String streamName = runtime.getMethodName();
    BKDistributedLogManager bkdlm = (BKDistributedLogManager) createNewDLM(conf, streamName);
    AsyncLogWriter out = bkdlm.startAsyncLogSegmentNonPartitioned();
    int txid = 1;
    Utils.ioResult(out.write(DLMTestUtil.getLargeLogRecordInstance(txid++, false)));
    Utils.ioResult(out.write(DLMTestUtil.getLargeLogRecordInstance(txid++, false)));
    Utils.ioResult(out.write(DLMTestUtil.getLargeLogRecordInstance(txid++, false)));
    BKLogReadHandler readHandler = bkdlm.createReadHandler();
    List<LogSegmentMetadata> ledgerList = Utils.ioResult(readHandler.readLogSegmentsFromStore(LogSegmentMetadata.COMPARATOR, LogSegmentFilter.DEFAULT_FILTER, null)).getValue();
    assertEquals(1, ledgerList.size());
    assertTrue(ledgerList.get(0).isInProgress());
    CompletableFuture<Long> count = null;
    count = readHandler.asyncGetLogRecordCount(new DLSN(1, 0, 0));
    assertEquals(2, Utils.ioResult(count).longValue());
    Utils.close(out);
}

Example 2 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestBKLogReadHandler method testGetLogRecordCountWithCompletedAndInprogressLedgers.

@Test(timeout = 60000)
public void testGetLogRecordCountWithCompletedAndInprogressLedgers() throws Exception {
    String streamName = runtime.getMethodName();
    BKDistributedLogManager bkdlm = (BKDistributedLogManager) createNewDLM(conf, streamName);
    long txid = 1;
    txid += DLMTestUtil.generateLogSegmentNonPartitioned(bkdlm, 0, 5, txid);
    AsyncLogWriter out = bkdlm.startAsyncLogSegmentNonPartitioned();
    Utils.ioResult(out.write(DLMTestUtil.getLargeLogRecordInstance(txid++, false)));
    Utils.ioResult(out.write(DLMTestUtil.getLargeLogRecordInstance(txid++, false)));
    Utils.ioResult(out.write(DLMTestUtil.getLargeLogRecordInstance(txid++, false)));
    BKLogReadHandler readHandler = bkdlm.createReadHandler();
    List<LogSegmentMetadata> ledgerList = Utils.ioResult(readHandler.readLogSegmentsFromStore(LogSegmentMetadata.COMPARATOR, LogSegmentFilter.DEFAULT_FILTER, null)).getValue();
    assertEquals(2, ledgerList.size());
    assertFalse(ledgerList.get(0).isInProgress());
    assertTrue(ledgerList.get(1).isInProgress());
    CompletableFuture<Long> count = null;
    count = readHandler.asyncGetLogRecordCount(new DLSN(1, 0, 0));
    assertEquals(7, Utils.ioResult(count).longValue());
    Utils.close(out);
}

Example 3 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestBKLogReadHandler method testGetFirstDLSNAfterCleanTruncation.

@Test(timeout = 60000)
public void testGetFirstDLSNAfterCleanTruncation() throws Exception {
    String dlName = runtime.getMethodName();
    prepareLogSegmentsNonPartitioned(dlName, 3, 10);
    DistributedLogManager dlm = createNewDLM(conf, dlName);
    BKLogReadHandler readHandler = ((BKDistributedLogManager) dlm).createReadHandler();
    AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
    CompletableFuture<Boolean> futureSuccess = writer.truncate(new DLSN(2, 0, 0));
    Boolean success = Utils.ioResult(futureSuccess);
    assertTrue(success);
    CompletableFuture<LogRecordWithDLSN> futureRecord = readHandler.asyncGetFirstLogRecord();
    LogRecordWithDLSN record = Utils.ioResult(futureRecord);
    assertEquals(new DLSN(2, 0, 0), record.getDlsn());
}

Example 4 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestBKLogReadHandler method testGetFirstDLSNWithOpenLedger.

@Test(timeout = 60000)
public void testGetFirstDLSNWithOpenLedger() throws Exception {
    String dlName = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(conf);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setOutputBufferSize(0);
    int numEntriesPerSegment = 10;
    DistributedLogManager dlm1 = createNewDLM(confLocal, dlName);
    long txid = 1;
    ArrayList<CompletableFuture<DLSN>> futures = new ArrayList<CompletableFuture<DLSN>>(numEntriesPerSegment);
    AsyncLogWriter out = dlm1.startAsyncLogSegmentNonPartitioned();
    for (int eid = 0; eid < numEntriesPerSegment; ++eid) {
        futures.add(out.write(DLMTestUtil.getLogRecordInstance(txid)));
        ++txid;
    }
    Utils.ioResult(FutureUtils.collect(futures));
    // commit
    LogRecord controlRecord = new LogRecord(txid, DistributedLogConstants.CONTROL_RECORD_CONTENT);
    controlRecord.setControl();
    Utils.ioResult(out.write(controlRecord));
    DLSN last = dlm1.getLastDLSN();
    assertEquals(new DLSN(1, 9, 0), last);
    DLSN first = Utils.ioResult(dlm1.getFirstDLSNAsync());
    assertEquals(new DLSN(1, 0, 0), first);
    Utils.close(out);
}

Example 5 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestTruncate method testOnlyPurgeSegmentsBeforeNoneFullyTruncatedSegment.

@Test(timeout = 60000)
public void testOnlyPurgeSegmentsBeforeNoneFullyTruncatedSegment() throws Exception {
    String name = "distrlog-only-purge-segments-before-none-fully-truncated-segment";
    URI uri = createDLMURI("/" + name);
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setExplicitTruncationByApplication(true);
    // populate data
    populateData(new HashMap<Long, DLSN>(), confLocal, name, 4, 10, false);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    LOG.info("Segments before modifying segment status : {}", segments);
    ZooKeeperClient zkc = TestZooKeeperClientBuilder.newBuilder(conf).uri(uri).build();
    setTruncationStatus(zkc, segments.get(0), TruncationStatus.PARTIALLY_TRUNCATED);
    for (int i = 1; i < 4; i++) {
        LogSegmentMetadata segment = segments.get(i);
        setTruncationStatus(zkc, segment, TruncationStatus.TRUNCATED);
    }
    List<LogSegmentMetadata> segmentsAfterTruncated = dlm.getLogSegments();
    dlm.purgeLogsOlderThan(999999);
    List<LogSegmentMetadata> newSegments = dlm.getLogSegments();
    LOG.info("Segments after purge segments older than 999999 : {}", newSegments);
    assertArrayEquals(segmentsAfterTruncated.toArray(new LogSegmentMetadata[segmentsAfterTruncated.size()]), newSegments.toArray(new LogSegmentMetadata[newSegments.size()]));
    dlm.close();
    // Update completion time of all 4 segments
    long newTimeMs = System.currentTimeMillis() - 60 * 60 * 1000 * 10;
    for (int i = 0; i < 4; i++) {
        LogSegmentMetadata segment = newSegments.get(i);
        updateCompletionTime(zkc, segment, newTimeMs + i);
    }
    DistributedLogConfiguration newConf = new DistributedLogConfiguration();
    newConf.addConfiguration(confLocal);
    newConf.setRetentionPeriodHours(1);
    DistributedLogManager newDLM = createNewDLM(newConf, name);
    AsyncLogWriter newWriter = newDLM.startAsyncLogSegmentNonPartitioned();
    long txid = 1 + 4 * 10;
    for (int j = 1; j <= 10; j++) {
        Utils.ioResult(newWriter.write(DLMTestUtil.getLogRecordInstance(txid++)));
    }
    // to make sure the truncation task is executed
    DLSN lastDLSN = Utils.ioResult(newDLM.getLastDLSNAsync());
    LOG.info("Get last dlsn of stream {} : {}", name, lastDLSN);
    assertEquals(5, newDLM.getLogSegments().size());
    Utils.close(newWriter);
    newDLM.close();
    zkc.close();
}