Examples with DistributedLogManager org.apache.distributedlog.api.DistributedLogManager used on opensource projects

Example 6 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderLock method testReaderLockFutureCancelledWhileWaiting.

@Test(timeout = 60000)
public void testReaderLockFutureCancelledWhileWaiting() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogManager dlm0 = createNewDLM(conf, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm0.startAsyncLogSegmentNonPartitioned());
    writer.write(DLMTestUtil.getLogRecordInstance(1L));
    writer.write(DLMTestUtil.getLogRecordInstance(2L));
    writer.closeAndComplete();
    DistributedLogManager dlm1 = createNewDLM(conf, name);
    CompletableFuture<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    AsyncLogReader reader1 = Utils.ioResult(futureReader1);
    DistributedLogManager dlm2 = createNewDLM(conf, name);
    CompletableFuture<AsyncLogReader> futureReader2 = dlm2.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    try {
        futureReader2.cancel(true);
        Utils.ioResult(futureReader2);
        fail("Should fail getting log reader as it is cancelled");
    } catch (CancellationException ce) {
    } catch (LockClosedException ex) {
    } catch (LockCancelledException ex) {
    } catch (OwnershipAcquireFailedException oafe) {
    }
    futureReader2 = dlm2.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    Utils.close(reader1);
    Utils.ioResult(futureReader2);
    dlm0.close();
    dlm1.close();
    dlm2.close();
}

Example 7 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderLock method testReaderLockFutureCancelledWhileLocked.

@Test(timeout = 60000)
public void testReaderLockFutureCancelledWhileLocked() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogManager dlm0 = createNewDLM(conf, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm0.startAsyncLogSegmentNonPartitioned());
    writer.write(DLMTestUtil.getLogRecordInstance(1L));
    writer.write(DLMTestUtil.getLogRecordInstance(2L));
    writer.closeAndComplete();
    DistributedLogManager dlm1 = createNewDLM(conf, name);
    CompletableFuture<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    // Must not throw or cancel or do anything bad, future already completed.
    Utils.ioResult(futureReader1);
    futureReader1.cancel(true);
    AsyncLogReader reader1 = Utils.ioResult(futureReader1);
    Utils.ioResult(reader1.readNext());
    dlm0.close();
    dlm1.close();
}

Example 8 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderLock method testReaderLockMultiReadersScenario.

@Test(timeout = 60000)
public void testReaderLockMultiReadersScenario() throws Exception {
    final String name = runtime.getMethodName();
    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);
    DistributedLogConfiguration localConf = new DistributedLogConfiguration();
    localConf.addConfiguration(conf);
    localConf.setImmediateFlushEnabled(false);
    localConf.setPeriodicFlushFrequencyMilliSeconds(60 * 1000);
    localConf.setOutputBufferSize(0);
    // Otherwise, we won't be able to run scheduled threads for readahead when we're in a callback.
    localConf.setNumWorkerThreads(2);
    localConf.setLockTimeout(Long.MAX_VALUE);
    Namespace namespace = NamespaceBuilder.newBuilder().conf(localConf).uri(uri).clientId("main").build();
    DistributedLogManager dlm0 = namespace.openLog(name);
    DLMTestUtil.generateCompletedLogSegments(dlm0, localConf, 9, 100);
    dlm0.close();
    int recordCount = 0;
    AtomicReference<DLSN> currentDLSN = new AtomicReference<DLSN>(DLSN.InitialDLSN);
    String clientId1 = "reader1";
    Namespace namespace1 = NamespaceBuilder.newBuilder().conf(localConf).uri(uri).clientId(clientId1).build();
    DistributedLogManager dlm1 = namespace1.openLog(name);
    String clientId2 = "reader2";
    Namespace namespace2 = NamespaceBuilder.newBuilder().conf(localConf).uri(uri).clientId(clientId2).build();
    DistributedLogManager dlm2 = namespace2.openLog(name);
    String clientId3 = "reader3";
    Namespace namespace3 = NamespaceBuilder.newBuilder().conf(localConf).uri(uri).clientId(clientId3).build();
    DistributedLogManager dlm3 = namespace3.openLog(name);
    LOG.info("{} is opening reader on stream {}", clientId1, name);
    CompletableFuture<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    AsyncLogReader reader1 = Utils.ioResult(futureReader1);
    LOG.info("{} opened reader on stream {}", clientId1, name);
    LOG.info("{} is opening reader on stream {}", clientId2, name);
    CompletableFuture<AsyncLogReader> futureReader2 = dlm2.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    LOG.info("{} is opening reader on stream {}", clientId3, name);
    CompletableFuture<AsyncLogReader> futureReader3 = dlm3.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    ExecutorService executorService = Executors.newCachedThreadPool();
    ReadRecordsListener listener2 = new ReadRecordsListener(currentDLSN, clientId2, executorService);
    ReadRecordsListener listener3 = new ReadRecordsListener(currentDLSN, clientId3, executorService);
    futureReader2.whenComplete(listener2);
    futureReader3.whenComplete(listener3);
    // Get reader1 and start reading.
    for (; recordCount < 200; recordCount++) {
        LogRecordWithDLSN record = Utils.ioResult(reader1.readNext());
        currentDLSN.set(record.getDlsn());
    }
    // Take a break, reader2 decides to stop waiting and cancels.
    Thread.sleep(1000);
    assertFalse(listener2.done());
    futureReader2.cancel(true);
    listener2.getLatch().await();
    assertTrue(listener2.done());
    assertTrue(listener2.failed());
    // Reader1 starts reading again.
    for (; recordCount < 300; recordCount++) {
        LogRecordWithDLSN record = Utils.ioResult(reader1.readNext());
        currentDLSN.set(record.getDlsn());
    }
    // Reader1 is done, someone else can take over. Since reader2 was
    // aborted, reader3 should take its place.
    assertFalse(listener3.done());
    Utils.close(reader1);
    listener3.getLatch().await();
    assertTrue(listener3.done());
    assertFalse(listener3.failed());
    assertEquals(new DLSN(3, 99, 0), currentDLSN.get());
    try {
        Utils.ioResult(futureReader2);
    } catch (Exception ex) {
    // Can't get this one to close it--the dlm will take care of it.
    }
    Utils.close(Utils.ioResult(futureReader3));
    dlm1.close();
    namespace1.close();
    dlm2.close();
    namespace2.close();
    dlm3.close();
    namespace3.close();
    executorService.shutdown();
}

Example 9 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestAsyncReaderLock method testReaderLockSharedDlmDoesNotConflict.

@Test(timeout = 60000)
public void testReaderLockSharedDlmDoesNotConflict() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogManager dlm0 = createNewDLM(conf, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm0.startAsyncLogSegmentNonPartitioned());
    writer.write(DLMTestUtil.getLogRecordInstance(1L));
    writer.write(DLMTestUtil.getLogRecordInstance(2L));
    writer.closeAndComplete();
    DistributedLogManager dlm1 = createNewDLM(conf, name);
    CompletableFuture<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    CompletableFuture<AsyncLogReader> futureReader2 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    // Both use the same client id, so there's no lock conflict. Not necessarily ideal, but how the
    // system currently works.
    Utils.ioResult(futureReader1);
    Utils.ioResult(futureReader2);
    dlm0.close();
    dlm1.close();
}

Example 10 with DistributedLogManager

use of org.apache.distributedlog.api.DistributedLogManager in project bookkeeper by apache.

the class TestBKDistributedLogManager method testNumberOfTransactionsWithInprogressAtEnd.

@Test(timeout = 60000)
public void testNumberOfTransactionsWithInprogressAtEnd() throws Exception {
    String name = "distrlog-inprogressAtEnd";
    DistributedLogManager dlm = createNewDLM(conf, name);
    long txid = 1;
    for (long i = 0; i < 3; i++) {
        long start = txid;
        BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) {
            LogRecord op = DLMTestUtil.getLogRecordInstance(txid++);
            out.write(op);
        }
        BKLogSegmentWriter perStreamLogWriter = out.getCachedLogWriter();
        out.closeAndComplete();
        BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true);
        assertNotNull(zkc.exists(blplm.completedLedgerZNode(start, txid - 1, perStreamLogWriter.getLogSegmentSequenceNumber()), false));
        Utils.ioResult(blplm.asyncClose());
    }
    BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    for (long j = 1; j <= DEFAULT_SEGMENT_SIZE / 2; j++) {
        LogRecord op = DLMTestUtil.getLogRecordInstance(txid++);
        out.write(op);
    }
    out.flush();
    out.commit();
    out.close();
    long numTrans = DLMTestUtil.getNumberofLogRecords(createNewDLM(conf, name), 1);
    assertEquals((txid - 1), numTrans);
}