Examples with AsyncLogWriter org.apache.distributedlog.api.AsyncLogWriter used on opensource projects

Example 26 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestNonBlockingReadsMultiReader method testMultiReaders.

@Test(timeout = 60000)
public void testMultiReaders() throws Exception {
    String name = "distrlog-multireaders";
    final RateLimiter limiter = RateLimiter.create(1000);
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(true);
    DistributedLogManager dlmwrite = createNewDLM(confLocal, name);
    final AsyncLogWriter writer = dlmwrite.startAsyncLogSegmentNonPartitioned();
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(0)));
    Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(1)));
    final AtomicInteger writeCount = new AtomicInteger(2);
    DistributedLogManager dlmread = createNewDLM(conf, name);
    BKSyncLogReader reader0 = (BKSyncLogReader) dlmread.getInputStream(0);
    try {
        ReaderThread[] readerThreads = new ReaderThread[1];
        readerThreads[0] = new ReaderThread("reader0-non-blocking", reader0, false);
        // readerThreads[1] = new ReaderThread("reader1-non-blocking", reader0, false);
        final AtomicBoolean running = new AtomicBoolean(true);
        Thread writerThread = new Thread("WriteThread") {

            @Override
            public void run() {
                try {
                    long txid = 2;
                    DLSN dlsn = DLSN.InvalidDLSN;
                    while (running.get()) {
                        limiter.acquire();
                        long curTxId = txid++;
                        dlsn = Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(curTxId)));
                        writeCount.incrementAndGet();
                        if (curTxId % 1000 == 0) {
                            LOG.info("writer write {}", curTxId);
                        }
                    }
                    LOG.info("Completed writing record at {}", dlsn);
                    Utils.close(writer);
                } catch (DLInterruptedException die) {
                    Thread.currentThread().interrupt();
                } catch (Exception e) {
                }
            }
        };
        for (ReaderThread rt : readerThreads) {
            rt.start();
        }
        writerThread.start();
        TimeUnit.SECONDS.sleep(5);
        LOG.info("Stopping writer");
        running.set(false);
        writerThread.join();
        LOG.info("Writer stopped after writing {} records, waiting for reader to complete", writeCount.get());
        while (writeCount.get() > (readerThreads[0].getReadCount())) {
            LOG.info("Write Count = {}, Read Count = {}", new Object[] { writeCount.get(), readerThreads[0].getReadCount() });
            TimeUnit.MILLISECONDS.sleep(100);
        }
        assertEquals(writeCount.get(), (readerThreads[0].getReadCount()));
        for (ReaderThread readerThread : readerThreads) {
            readerThread.stopReading();
        }
    } finally {
        dlmwrite.close();
        reader0.close();
        dlmread.close();
    }
}

Example 27 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestReadUtils method testGetLogRecordGreaterThanTxIdOnSameTxId.

@Test(timeout = 60000)
public void testGetLogRecordGreaterThanTxIdOnSameTxId() throws Exception {
    String streamName = runtime.getMethodName();
    BKDistributedLogManager bkdlm = createNewDLM(conf, streamName);
    AsyncLogWriter out = bkdlm.startAsyncLogSegmentNonPartitioned();
    long txid = 1L;
    for (int i = 0; i < 10; ++i) {
        LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid);
        Utils.ioResult(out.write(record));
        txid += 1;
    }
    long txidToSearch = txid;
    for (int i = 0; i < 10; ++i) {
        LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txidToSearch);
        Utils.ioResult(out.write(record));
    }
    for (int i = 0; i < 10; ++i) {
        LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid);
        Utils.ioResult(out.write(record));
        txid += 1;
    }
    Utils.close(out);
    Optional<LogRecordWithDLSN> result = Utils.ioResult(getLogRecordNotLessThanTxId(bkdlm, 0, txidToSearch));
    assertTrue(result.isPresent());
    assertEquals(10L, result.get().getDlsn().getEntryId());
}

Example 28 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestReadUtils method testGetLastRecordControlRecord.

@Test(timeout = 60000)
public void testGetLastRecordControlRecord() throws Exception {
    String streamName = runtime.getMethodName();
    BKDistributedLogManager bkdlm = (BKDistributedLogManager) createNewDLM(conf, streamName);
    AsyncLogWriter out = bkdlm.startAsyncLogSegmentNonPartitioned();
    int txid = 1;
    Utils.ioResult(out.write(DLMTestUtil.getLargeLogRecordInstance(txid++, false)));
    Utils.ioResult(out.write(DLMTestUtil.getLargeLogRecordInstance(txid++, false)));
    Utils.ioResult(out.write(DLMTestUtil.getLargeLogRecordInstance(txid++, false)));
    Utils.ioResult(out.write(DLMTestUtil.getLargeLogRecordInstance(txid++, true)));
    Utils.ioResult(out.write(DLMTestUtil.getLargeLogRecordInstance(txid++, true)));
    Utils.close(out);
    CompletableFuture<LogRecordWithDLSN> futureLogrec = getLastUserRecord(bkdlm, 0);
    LogRecordWithDLSN logrec = Utils.ioResult(futureLogrec);
    assertEquals(new DLSN(1, 2, 0), logrec.getDlsn());
    bkdlm.close();
}

Example 29 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestBKLogReadHandler method testGetFirstDLSNAfterPartialTruncation.

@Test(timeout = 60000)
public void testGetFirstDLSNAfterPartialTruncation() throws Exception {
    String dlName = runtime.getMethodName();
    prepareLogSegmentsNonPartitioned(dlName, 3, 10);
    DistributedLogManager dlm = createNewDLM(conf, dlName);
    BKLogReadHandler readHandler = ((BKDistributedLogManager) dlm).createReadHandler();
    AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
    // Only truncates at ledger boundary.
    CompletableFuture<Boolean> futureSuccess = writer.truncate(new DLSN(2, 5, 0));
    Boolean success = Utils.ioResult(futureSuccess);
    assertTrue(success);
    CompletableFuture<LogRecordWithDLSN> futureRecord = readHandler.asyncGetFirstLogRecord();
    LogRecordWithDLSN record = Utils.ioResult(futureRecord);
    assertEquals(new DLSN(2, 0, 0), record.getDlsn());
}

Example 30 with AsyncLogWriter

use of org.apache.distributedlog.api.AsyncLogWriter in project bookkeeper by apache.

the class TestBKLogWriteHandler method testAbortTransactionOnStartLogSegment.

/**
 * Testcase: when write handler encounters exceptions on starting log segment
 * it should abort the transaction and return the ledger to the pool.
 */
@Test(timeout = 60000)
public void testAbortTransactionOnStartLogSegment() throws Exception {
    URI uri = createDLMURI("/" + runtime.getMethodName());
    ensureURICreated(zkc, uri);
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setEnableLedgerAllocatorPool(true);
    confLocal.setLedgerAllocatorPoolCoreSize(1);
    confLocal.setLedgerAllocatorPoolName("test-allocator-pool");
    BKDistributedLogNamespace namespace = (BKDistributedLogNamespace) NamespaceBuilder.newBuilder().conf(confLocal).uri(uri).build();
    DistributedLogManager dlm = namespace.openLog("test-stream");
    FailpointUtils.setFailpoint(FailpointUtils.FailPointName.FP_StartLogSegmentOnAssignLogSegmentSequenceNumber, FailpointUtils.FailPointActions.FailPointAction_Throw);
    try {
        AsyncLogWriter writer = Utils.ioResult(dlm.openAsyncLogWriter());
        Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
        fail("Should fail opening the writer");
    } catch (IOException ioe) {
    // expected
    } finally {
        FailpointUtils.removeFailpoint(FailpointUtils.FailPointName.FP_StartLogSegmentOnAssignLogSegmentSequenceNumber);
    }
    LedgerAllocator allocator = ((BKNamespaceDriver) namespace.getNamespaceDriver()).getLedgerAllocator();
    assertTrue(allocator instanceof LedgerAllocatorPool);
    LedgerAllocatorPool allocatorPool = (LedgerAllocatorPool) allocator;
    assertEquals(0, allocatorPool.obtainMapSize());
    AsyncLogWriter writer = Utils.ioResult(dlm.openAsyncLogWriter());
    writer.write(DLMTestUtil.getLogRecordInstance(1L));
    Utils.close(writer);
}